Diff of the two buildlogs: -- --- b1/build.log 2025-03-20 03:48:13.252422460 +0000 +++ b2/build.log 2025-03-20 03:51:39.071158054 +0000 @@ -1,6 +1,6 @@ I: pbuilder: network access will be disabled during build -I: Current time: Tue Apr 21 22:08:13 -12 2026 -I: pbuilder-time-stamp: 1776852493 +I: Current time: Thu Mar 20 17:48:15 +14 2025 +I: pbuilder-time-stamp: 1742442495 I: Building the build Environment I: extracting base tarball [/var/cache/pbuilder/unstable-reproducible-base.tgz] I: copying local configuration @@ -25,54 +25,86 @@ dpkg-source: info: applying 0001-Skip-unreliable-tests.patch I: Not using root during the build. I: Installing the build-deps -I: user script /srv/workspace/pbuilder/53424/tmp/hooks/D02_print_environment starting +I: user script /srv/workspace/pbuilder/44275/tmp/hooks/D01_modify_environment starting +debug: Running on ionos2-i386. +I: Changing host+domainname to test build reproducibility +I: Adding a custom variable just for the fun of it... +I: Changing /bin/sh to bash +'/bin/sh' -> '/bin/bash' +lrwxrwxrwx 1 root root 9 Mar 20 03:48 /bin/sh -> /bin/bash +I: Setting pbuilder2's login shell to /bin/bash +I: Setting pbuilder2's GECOS to second user,second room,second work-phone,second home-phone,second other +I: user script /srv/workspace/pbuilder/44275/tmp/hooks/D01_modify_environment finished +I: user script /srv/workspace/pbuilder/44275/tmp/hooks/D02_print_environment starting I: set - BUILDDIR='/build/reproducible-path' - BUILDUSERGECOS='first user,first room,first work-phone,first home-phone,first other' - BUILDUSERNAME='pbuilder1' - BUILD_ARCH='i386' - DEBIAN_FRONTEND='noninteractive' - DEB_BUILD_OPTIONS='buildinfo=+all reproducible=+all parallel=22 ' - DISTRIBUTION='unstable' - HOME='/root' - HOST_ARCH='i386' + BASH=/bin/sh + BASHOPTS=checkwinsize:cmdhist:complete_fullquote:extquote:force_fignore:globasciiranges:globskipdots:hostcomplete:interactive_comments:patsub_replacement:progcomp:promptvars:sourcepath + BASH_ALIASES=() + BASH_ARGC=() + BASH_ARGV=() + BASH_CMDS=() + BASH_LINENO=([0]="12" [1]="0") + BASH_LOADABLES_PATH=/usr/local/lib/bash:/usr/lib/bash:/opt/local/lib/bash:/usr/pkg/lib/bash:/opt/pkg/lib/bash:. + BASH_SOURCE=([0]="/tmp/hooks/D02_print_environment" [1]="/tmp/hooks/D02_print_environment") + BASH_VERSINFO=([0]="5" [1]="2" [2]="37" [3]="1" [4]="release" [5]="i686-pc-linux-gnu") + BASH_VERSION='5.2.37(1)-release' + BUILDDIR=/build/reproducible-path + BUILDUSERGECOS='second user,second room,second work-phone,second home-phone,second other' + BUILDUSERNAME=pbuilder2 + BUILD_ARCH=i386 + DEBIAN_FRONTEND=noninteractive + DEB_BUILD_OPTIONS='buildinfo=+all reproducible=+all parallel=10 ' + DIRSTACK=() + DISTRIBUTION=unstable + EUID=0 + FUNCNAME=([0]="Echo" [1]="main") + GROUPS=() + HOME=/root + HOSTNAME=i-capture-the-hostname + HOSTTYPE=i686 + HOST_ARCH=i386 IFS=' ' - INVOCATION_ID='1954aad46e9043bfa5d277658b5ab597' - LANG='C' - LANGUAGE='en_US:en' - LC_ALL='C' - LD_LIBRARY_PATH='/usr/lib/libeatmydata' - LD_PRELOAD='libeatmydata.so' - MAIL='/var/mail/root' - OPTIND='1' - PATH='/usr/sbin:/usr/bin:/sbin:/bin:/usr/games' - PBCURRENTCOMMANDLINEOPERATION='build' - PBUILDER_OPERATION='build' - PBUILDER_PKGDATADIR='/usr/share/pbuilder' - PBUILDER_PKGLIBDIR='/usr/lib/pbuilder' - PBUILDER_SYSCONFDIR='/etc' - PPID='53424' - PS1='# ' - PS2='> ' + INVOCATION_ID=f06f448ab29a448d94aa0b14d9477066 + LANG=C + LANGUAGE=de_CH:de + LC_ALL=C + LD_LIBRARY_PATH=/usr/lib/libeatmydata + LD_PRELOAD=libeatmydata.so + MACHTYPE=i686-pc-linux-gnu + MAIL=/var/mail/root + OPTERR=1 + OPTIND=1 + OSTYPE=linux-gnu + PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path + PBCURRENTCOMMANDLINEOPERATION=build + PBUILDER_OPERATION=build + PBUILDER_PKGDATADIR=/usr/share/pbuilder + PBUILDER_PKGLIBDIR=/usr/lib/pbuilder + PBUILDER_SYSCONFDIR=/etc + PIPESTATUS=([0]="0") + POSIXLY_CORRECT=y + PPID=44275 PS4='+ ' - PWD='/' - SHELL='/bin/bash' - SHLVL='2' - SUDO_COMMAND='/usr/bin/timeout -k 18.1h 18h /usr/bin/ionice -c 3 /usr/bin/nice /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/r-b-build.H0QjU75I/pbuilderrc_8hoc --distribution unstable --hookdir /etc/pbuilder/first-build-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/unstable-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/r-b-build.H0QjU75I/b1 --logfile b1/build.log golang-github-hashicorp-raft_1.5.0-1.dsc' - SUDO_GID='112' - SUDO_UID='107' - SUDO_USER='jenkins' - TERM='unknown' - TZ='/usr/share/zoneinfo/Etc/GMT+12' - USER='root' - _='/usr/bin/systemd-run' - http_proxy='http://213.165.73.152:3128' + PWD=/ + SHELL=/bin/bash + SHELLOPTS=braceexpand:errexit:hashall:interactive-comments:posix + SHLVL=3 + SUDO_COMMAND='/usr/bin/timeout -k 24.1h 24h /usr/bin/ionice -c 3 /usr/bin/nice -n 11 /usr/bin/unshare --uts -- /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/r-b-build.H0QjU75I/pbuilderrc_dNBY --distribution unstable --hookdir /etc/pbuilder/rebuild-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/unstable-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/r-b-build.H0QjU75I/b2 --logfile b2/build.log golang-github-hashicorp-raft_1.5.0-1.dsc' + SUDO_GID=112 + SUDO_UID=107 + SUDO_USER=jenkins + TERM=unknown + TZ=/usr/share/zoneinfo/Etc/GMT-14 + UID=0 + USER=root + _='I: set' + http_proxy=http://46.16.76.132:3128 I: uname -a - Linux ionos16-i386 6.1.0-32-amd64 #1 SMP PREEMPT_DYNAMIC Debian 6.1.129-1 (2025-03-06) x86_64 GNU/Linux + Linux i-capture-the-hostname 6.1.0-32-amd64 #1 SMP PREEMPT_DYNAMIC Debian 6.1.129-1 (2025-03-06) x86_64 GNU/Linux I: ls -l /bin - lrwxrwxrwx 1 root root 7 Mar 4 2025 /bin -> usr/bin -I: user script /srv/workspace/pbuilder/53424/tmp/hooks/D02_print_environment finished + lrwxrwxrwx 1 root root 7 Mar 4 11:20 /bin -> usr/bin +I: user script /srv/workspace/pbuilder/44275/tmp/hooks/D02_print_environment finished -> Attempting to satisfy build-dependencies -> Creating pbuilder-satisfydepends-dummy package Package: pbuilder-satisfydepends-dummy @@ -227,7 +259,7 @@ Get: 105 http://deb.debian.org/debian unstable/main i386 golang-gopkg-check.v1-dev all 0.0+git20200902.038fdea-2 [31.2 kB] Get: 106 http://deb.debian.org/debian unstable/main i386 golang-gopkg-mgo.v2-dev all 2016.08.01-7 [316 kB] Get: 107 http://deb.debian.org/debian unstable/main i386 golang-github-hashicorp-go-msgpack-dev all 0.5.5-1 [43.3 kB] -Fetched 130 MB in 1s (101 MB/s) +Fetched 130 MB in 1s (92.7 MB/s) Preconfiguring packages ... Selecting previously unselected package sensible-utils. (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 19770 files and directories currently installed.) @@ -670,7 +702,11 @@ Building tag database... -> Finished parsing the build-deps I: Building the package -I: Running cd /build/reproducible-path/golang-github-hashicorp-raft-1.5.0/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" HOME="/nonexistent/first-build" dpkg-buildpackage -us -uc -b && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" HOME="/nonexistent/first-build" dpkg-genchanges -S > ../golang-github-hashicorp-raft_1.5.0-1_source.changes +I: user script /srv/workspace/pbuilder/44275/tmp/hooks/A99_set_merged_usr starting +Not re-configuring usrmerge for unstable +I: user script /srv/workspace/pbuilder/44275/tmp/hooks/A99_set_merged_usr finished +hostname: Name or service not known +I: Running cd /build/reproducible-path/golang-github-hashicorp-raft-1.5.0/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" HOME="/nonexistent/second-build" dpkg-buildpackage -us -uc -b && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" HOME="/nonexistent/second-build" dpkg-genchanges -S > ../golang-github-hashicorp-raft_1.5.0-1_source.changes dpkg-buildpackage: info: source package golang-github-hashicorp-raft dpkg-buildpackage: info: source version 1.5.0-1 dpkg-buildpackage: info: source distribution unstable @@ -688,193 +724,193 @@ dh_autoreconf -O--buildsystem=golang dh_auto_configure -O--buildsystem=golang dh_auto_build -O--buildsystem=golang - cd obj-i686-linux-gnu && go install -trimpath -v -p 22 github.com/hashicorp/raft github.com/hashicorp/raft/bench -internal/asan -internal/goarch -container/list -internal/coverage/rtcov -internal/msan + cd obj-i686-linux-gnu && go install -trimpath -v -p 10 github.com/hashicorp/raft github.com/hashicorp/raft/bench internal/profilerecord internal/goos -unicode/utf8 -internal/byteorder -crypto/internal/fips140/alias -math/bits -internal/cpu -internal/itoa +internal/coverage/rtcov internal/godebugs -sync/atomic -internal/runtime/atomic -internal/goexperiment +internal/goarch +internal/cpu internal/unsafeheader -crypto/internal/boring/sig -internal/runtime/syscall -cmp -unicode -encoding -unicode/utf16 -log/internal -internal/nettrace -vendor/golang.org/x/crypto/cryptobyte/asn1 -vendor/golang.org/x/crypto/internal/alias +internal/byteorder +internal/goexperiment +internal/runtime/atomic +internal/asan internal/runtime/math +internal/msan internal/runtime/sys internal/abi +internal/runtime/syscall +sync/atomic +math/bits internal/chacha8rand -crypto/internal/fips140deps/byteorder -math -crypto/internal/fips140deps/cpu +unicode/utf8 +unicode +container/list +internal/itoa internal/bytealg +math +crypto/internal/fips140/alias internal/runtime/exithook +crypto/internal/fips140deps/byteorder +crypto/internal/fips140deps/cpu +cmp +crypto/internal/boring/sig +encoding +log/internal +unicode/utf16 +internal/nettrace +vendor/golang.org/x/crypto/cryptobyte/asn1 internal/stringslite +vendor/golang.org/x/crypto/internal/alias internal/race internal/sync internal/runtime/maps runtime -crypto/internal/fips140/subtle iter internal/reflectlite -weak sync +weak +crypto/internal/fips140/subtle maps slices crypto/subtle errors sort internal/testlog -internal/singleflight internal/bisect +internal/singleflight unique +runtime/cgo internal/oserror -path io -github.com/hashicorp/golang-lru/simplelru -math/rand/v2 -vendor/golang.org/x/net/dns/dnsmessage +path strconv -runtime/cgo +math/rand/v2 +github.com/hashicorp/golang-lru/simplelru syscall internal/godebug -hash -internal/saferio +vendor/golang.org/x/net/dns/dnsmessage bytes strings -crypto/internal/fips140deps/godebug -hash/crc64 +internal/saferio +hash +crypto/internal/randutil hash/crc32 +hash/crc64 +crypto/internal/fips140deps/godebug math/rand -crypto/internal/randutil -crypto/internal/fips140 +crypto +reflect net/netip +vendor/golang.org/x/text/transform bufio -crypto -net/http/internal/ascii +crypto/internal/fips140 github.com/hashicorp/go-immutable-radix crypto/internal/impl -vendor/golang.org/x/text/transform -regexp/syntax -html -reflect +net/http/internal/ascii +crypto/internal/fips140/sha256 crypto/internal/fips140/sha3 crypto/internal/fips140/sha512 -crypto/internal/fips140/sha256 crypto/tls/internal/fips140tls -crypto/sha3 +regexp/syntax +html crypto/internal/fips140/hmac +crypto/sha3 +internal/syscall/unix +time +internal/syscall/execenv crypto/internal/fips140/check crypto/internal/fips140hash -crypto/internal/fips140/bigmod crypto/internal/fips140/edwards25519/field -crypto/internal/fips140/hkdf -crypto/internal/fips140/tls12 crypto/internal/fips140/aes +crypto/internal/fips140/bigmod +crypto/internal/fips140/hkdf crypto/internal/fips140/nistec/fiat +crypto/internal/fips140/tls12 crypto/internal/fips140/tls13 crypto/internal/fips140/edwards25519 -internal/syscall/execenv -internal/syscall/unix -time regexp context io/fs internal/poll +internal/filepathlite +os internal/fmtsort encoding/binary -internal/filepathlite crypto/internal/fips140/nistec -os encoding/base64 vendor/golang.org/x/crypto/internal/poly1305 golang.org/x/sys/unix encoding/pem crypto/internal/sysrand fmt -path/filepath +internal/sysinfo io/ioutil +path/filepath os/signal -internal/sysinfo net crypto/internal/entropy crypto/internal/fips140/drbg -crypto/internal/fips140/aes/gcm crypto/internal/fips140only -crypto/internal/fips140/ecdh crypto/internal/fips140/ed25519 -crypto/internal/fips140/ecdsa -crypto/internal/fips140/mlkem +crypto/internal/fips140/aes/gcm crypto/internal/fips140/rsa +crypto/internal/fips140/mlkem crypto/md5 crypto/rc4 crypto/cipher -encoding/json -encoding/hex +crypto/internal/boring +vendor/golang.org/x/crypto/chacha20 +crypto/des log -net/url +encoding/json compress/flate -vendor/golang.org/x/net/http2/hpack math/big +crypto/aes +crypto/sha512 +crypto/hmac +vendor/golang.org/x/crypto/chacha20poly1305 +crypto/internal/fips140/ecdh +crypto/internal/fips140/ecdsa +crypto/sha1 +crypto/ecdh +crypto/sha256 +encoding/hex +net/url +vendor/golang.org/x/text/unicode/bidi vendor/golang.org/x/text/unicode/norm -go/token +compress/gzip +vendor/golang.org/x/net/http2/hpack mime mime/quotedprintable net/http/internal encoding/gob +go/token text/template/parse +vendor/golang.org/x/text/secure/bidirule flag -runtime/trace runtime/debug -crypto/des -crypto/internal/boring -vendor/golang.org/x/crypto/chacha20 -vendor/golang.org/x/text/unicode/bidi -crypto/aes -crypto/ecdh -crypto/hmac -crypto/sha512 -vendor/golang.org/x/crypto/chacha20poly1305 -crypto/sha1 -crypto/sha256 -compress/gzip -testing -vendor/golang.org/x/text/secure/bidirule -github.com/mattn/go-isatty -text/template -github.com/mattn/go-colorable +runtime/trace vendor/golang.org/x/net/idna -github.com/fatih/color -github.com/hashicorp/go-hclog crypto/rand crypto/elliptic crypto/internal/boring/bbig crypto/dsa encoding/asn1 crypto/ed25519 +github.com/mattn/go-isatty crypto/internal/hpke crypto/rsa -html/template -vendor/golang.org/x/crypto/cryptobyte +testing +github.com/mattn/go-colorable +text/template +github.com/fatih/color crypto/x509/pkix +vendor/golang.org/x/crypto/cryptobyte +github.com/hashicorp/go-hclog crypto/ecdsa +html/template vendor/golang.org/x/net/http/httpproxy net/textproto crypto/x509 @@ -883,15 +919,15 @@ crypto/tls net/http/httptrace net/http -github.com/armon/go-metrics net/rpc +github.com/armon/go-metrics github.com/hashicorp/go-msgpack/codec github.com/hashicorp/raft github.com/hashicorp/raft/bench debian/rules override_dh_auto_test make[1]: Entering directory '/build/reproducible-path/golang-github-hashicorp-raft-1.5.0' dh_auto_test - cd obj-i686-linux-gnu && go test -vet=off -v -p 22 github.com/hashicorp/raft github.com/hashicorp/raft/bench + cd obj-i686-linux-gnu && go test -vet=off -v -p 10 github.com/hashicorp/raft github.com/hashicorp/raft/bench === RUN TestCommitment_setVoters --- PASS: TestCommitment_setVoters (0.00s) === RUN TestCommitment_match_max @@ -933,33 +969,33 @@ === RUN TestFileSnapshotSinkImpl --- PASS: TestFileSnapshotSinkImpl (0.00s) === RUN TestFileSS_CreateSnapshotMissingParentDir -2026-04-22T10:09:13.978Z [INFO] creating new snapshot: path=/tmp/raft1320065332/raft496559154/snapshots/3-10-1776852553978.tmp +2025-03-20T03:49:28.329Z [INFO] creating new snapshot: path=/tmp/raft3612408171/raft1423176524/snapshots/3-10-1742442568329.tmp --- PASS: TestFileSS_CreateSnapshotMissingParentDir (0.01s) === RUN TestFileSS_CreateSnapshot -2026-04-22T10:09:13.984Z [INFO] creating new snapshot: path=/tmp/raft960437121/snapshots/3-10-1776852553984.tmp -2026-04-22T10:09:13.990Z [WARN] found temporary snapshot: name=3-10-1776852553984.tmp +2025-03-20T03:49:28.336Z [INFO] creating new snapshot: path=/tmp/raft1534132571/snapshots/3-10-1742442568336.tmp +2025-03-20T03:49:28.338Z [WARN] found temporary snapshot: name=3-10-1742442568336.tmp --- PASS: TestFileSS_CreateSnapshot (0.01s) === RUN TestFileSS_CancelSnapshot -2026-04-22T10:09:13.997Z [INFO] creating new snapshot: path=/tmp/raft1765692347/snapshots/3-10-1776852553997.tmp +2025-03-20T03:49:28.342Z [INFO] creating new snapshot: path=/tmp/raft4054342303/snapshots/3-10-1742442568342.tmp --- PASS: TestFileSS_CancelSnapshot (0.00s) === RUN TestFileSS_Retention -2026-04-22T10:09:14.002Z [INFO] creating new snapshot: path=/tmp/raft298372192/snapshots/3-10-1776852554002.tmp -2026-04-22T10:09:14.010Z [INFO] creating new snapshot: path=/tmp/raft298372192/snapshots/3-11-1776852554010.tmp -2026-04-22T10:09:14.019Z [INFO] creating new snapshot: path=/tmp/raft298372192/snapshots/3-12-1776852554019.tmp -2026-04-22T10:09:14.029Z [INFO] reaping snapshot: path=/tmp/raft298372192/snapshots/3-10-1776852554002 -2026-04-22T10:09:14.029Z [INFO] creating new snapshot: path=/tmp/raft298372192/snapshots/3-13-1776852554029.tmp -2026-04-22T10:09:14.038Z [INFO] reaping snapshot: path=/tmp/raft298372192/snapshots/3-11-1776852554010 -2026-04-22T10:09:14.038Z [INFO] creating new snapshot: path=/tmp/raft298372192/snapshots/3-14-1776852554038.tmp -2026-04-22T10:09:14.047Z [INFO] reaping snapshot: path=/tmp/raft298372192/snapshots/3-12-1776852554019 ---- PASS: TestFileSS_Retention (0.05s) +2025-03-20T03:49:28.345Z [INFO] creating new snapshot: path=/tmp/raft3585632230/snapshots/3-10-1742442568345.tmp +2025-03-20T03:49:28.349Z [INFO] creating new snapshot: path=/tmp/raft3585632230/snapshots/3-11-1742442568349.tmp +2025-03-20T03:49:28.353Z [INFO] creating new snapshot: path=/tmp/raft3585632230/snapshots/3-12-1742442568353.tmp +2025-03-20T03:49:28.358Z [INFO] reaping snapshot: path=/tmp/raft3585632230/snapshots/3-10-1742442568345 +2025-03-20T03:49:28.358Z [INFO] creating new snapshot: path=/tmp/raft3585632230/snapshots/3-13-1742442568358.tmp +2025-03-20T03:49:28.362Z [INFO] reaping snapshot: path=/tmp/raft3585632230/snapshots/3-11-1742442568349 +2025-03-20T03:49:28.362Z [INFO] creating new snapshot: path=/tmp/raft3585632230/snapshots/3-14-1742442568362.tmp +2025-03-20T03:49:28.369Z [INFO] reaping snapshot: path=/tmp/raft3585632230/snapshots/3-12-1742442568353 +--- PASS: TestFileSS_Retention (0.02s) === RUN TestFileSS_BadPerm --- PASS: TestFileSS_BadPerm (0.00s) === RUN TestFileSS_MissingParentDir --- PASS: TestFileSS_MissingParentDir (0.00s) === RUN TestFileSS_Ordering -2026-04-22T10:09:14.049Z [INFO] creating new snapshot: path=/tmp/raft174869081/snapshots/5-130350-1776852554049.tmp -2026-04-22T10:09:14.057Z [INFO] creating new snapshot: path=/tmp/raft174869081/snapshots/36-204917-1776852554057.tmp ---- PASS: TestFileSS_Ordering (0.02s) +2025-03-20T03:49:28.370Z [INFO] creating new snapshot: path=/tmp/raft1794335650/snapshots/5-130350-1742442568370.tmp +2025-03-20T03:49:28.375Z [INFO] creating new snapshot: path=/tmp/raft1794335650/snapshots/36-204917-1742442568375.tmp +--- PASS: TestFileSS_Ordering (0.01s) === RUN TestDeferFutureSuccess --- PASS: TestDeferFutureSuccess (0.00s) === RUN TestDeferFutureError @@ -995,7 +1031,7 @@ === RUN TestEmitsLogStoreMetrics --- PASS: TestEmitsLogStoreMetrics (0.01s) === RUN TestNetworkTransport_CloseStreams ---- PASS: TestNetworkTransport_CloseStreams (0.00s) +--- PASS: TestNetworkTransport_CloseStreams (0.01s) === RUN TestNetworkTransport_StartStop --- PASS: TestNetworkTransport_StartStop (0.00s) === RUN TestNetworkTransport_Heartbeat_FastPath @@ -1008,12 +1044,12 @@ --- PASS: TestNetworkTransport_AppendEntriesPipeline_CloseStreams (0.02s) === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=0 -2026-04-22T10:09:14.164Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" +2025-03-20T03:49:28.492Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=1 === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=2 +2025-03-20T03:49:28.545Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=10 -2026-04-22T10:09:14.216Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" -2026-04-22T10:09:14.268Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" +2025-03-20T03:49:28.598Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight (0.16s) --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=0 (0.05s) --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=1 (0.00s) @@ -1030,27 +1066,27 @@ === RUN TestNetworkTransport_PooledConn --- PASS: TestNetworkTransport_PooledConn (0.00s) === RUN TestNetworkTransport_ListenBackoff - net_transport_test.go:839: countingWriter: 2026-04-22T10:09:14.274Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-03-20T03:49:28.610Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2026-04-22T10:09:14.279Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-03-20T03:49:28.615Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2026-04-22T10:09:14.290Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-03-20T03:49:28.626Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2026-04-22T10:09:14.310Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-03-20T03:49:28.646Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2026-04-22T10:09:14.351Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-03-20T03:49:28.687Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2026-04-22T10:09:14.431Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-03-20T03:49:28.768Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2026-04-22T10:09:14.592Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-03-20T03:49:28.929Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2026-04-22T10:09:14.912Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-03-20T03:49:29.249Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2026-04-22T10:09:15.553Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-03-20T03:49:29.891Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2026-04-22T10:09:16.555Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-03-20T03:49:30.892Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2026-04-22T10:09:17.557Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-03-20T03:49:31.894Z [ERROR] test: failed to accept connection: error="intentional error in test" --- PASS: TestNetworkTransport_ListenBackoff (4.00s) === RUN TestPeersJSON_BadConfiguration @@ -1060,2534 +1096,2619 @@ === RUN TestPeersJSON_ReadConfigJSON --- PASS: TestPeersJSON_ReadConfigJSON (0.00s) === RUN TestRaft_StartStop -2026-04-22T10:09:18.276Z [INFO] server-b0dd2471-59a7-8e19-9e66-0767b24b0baa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b0dd2471-59a7-8e19-9e66-0767b24b0baa Address:b0dd2471-59a7-8e19-9e66-0767b24b0baa}]" +2025-03-20T03:49:32.612Z [INFO] server-bdaf1992-5610-4569-5076-40acf341ccc0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bdaf1992-5610-4569-5076-40acf341ccc0 Address:bdaf1992-5610-4569-5076-40acf341ccc0}]" --- PASS: TestRaft_StartStop (0.00s) === RUN TestRaft_AfterShutdown -2026-04-22T10:09:18.277Z [INFO] server-9a33264c-a7df-5903-3d05-7a8e92cc4023: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9a33264c-a7df-5903-3d05-7a8e92cc4023 Address:9a33264c-a7df-5903-3d05-7a8e92cc4023}]" +2025-03-20T03:49:32.613Z [INFO] server-4b4c9891-99bc-7d85-6919-49aa00289b6e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4b4c9891-99bc-7d85-6919-49aa00289b6e Address:4b4c9891-99bc-7d85-6919-49aa00289b6e}]" --- PASS: TestRaft_AfterShutdown (0.00s) === RUN TestRaft_LiveBootstrap -2026-04-22T10:09:18.278Z [INFO] server-52d0754e-b7a3-3283-75d8-98a76ed0eb27: initial configuration: index=0 servers=[] -2026-04-22T10:09:18.278Z [INFO] server-a9238553-d59b-9369-b4a2-be728263ba87: initial configuration: index=0 servers=[] -2026-04-22T10:09:18.278Z [INFO] server-52d0754e-b7a3-3283-75d8-98a76ed0eb27: entering follower state: follower="Node at 52d0754e-b7a3-3283-75d8-98a76ed0eb27 [Follower]" leader-address= leader-id= -2026-04-22T10:09:18.278Z [INFO] server-a9238553-d59b-9369-b4a2-be728263ba87: entering follower state: follower="Node at a9238553-d59b-9369-b4a2-be728263ba87 [Follower]" leader-address= leader-id= -2026-04-22T10:09:18.278Z [INFO] server-05ec8efb-eeb7-eca7-3c4d-c94ed8900f8e: initial configuration: index=0 servers=[] -2026-04-22T10:09:18.278Z [INFO] server-05ec8efb-eeb7-eca7-3c4d-c94ed8900f8e: entering follower state: follower="Node at 05ec8efb-eeb7-eca7-3c4d-c94ed8900f8e [Follower]" leader-address= leader-id= -2026-04-22T10:09:18.278Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:09:18.332Z [WARN] server-a9238553-d59b-9369-b4a2-be728263ba87: no known peers, aborting election -2026-04-22T10:09:18.341Z [WARN] server-05ec8efb-eeb7-eca7-3c4d-c94ed8900f8e: no known peers, aborting election -2026-04-22T10:09:18.356Z [WARN] server-52d0754e-b7a3-3283-75d8-98a76ed0eb27: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:18.356Z [INFO] server-52d0754e-b7a3-3283-75d8-98a76ed0eb27: entering candidate state: node="Node at 52d0754e-b7a3-3283-75d8-98a76ed0eb27 [Candidate]" term=2 -2026-04-22T10:09:18.356Z [INFO] server-52d0754e-b7a3-3283-75d8-98a76ed0eb27: election won: term=2 tally=2 -2026-04-22T10:09:18.356Z [INFO] server-52d0754e-b7a3-3283-75d8-98a76ed0eb27: entering leader state: leader="Node at 52d0754e-b7a3-3283-75d8-98a76ed0eb27 [Leader]" -2026-04-22T10:09:18.356Z [INFO] server-52d0754e-b7a3-3283-75d8-98a76ed0eb27: added peer, starting replication: peer=server-a9238553-d59b-9369-b4a2-be728263ba87 -2026-04-22T10:09:18.356Z [INFO] server-52d0754e-b7a3-3283-75d8-98a76ed0eb27: added peer, starting replication: peer=server-05ec8efb-eeb7-eca7-3c4d-c94ed8900f8e -2026-04-22T10:09:18.356Z [WARN] server-a9238553-d59b-9369-b4a2-be728263ba87: failed to get previous log: previous-index=1 last-index=0 error="log not found" -2026-04-22T10:09:18.357Z [WARN] server-52d0754e-b7a3-3283-75d8-98a76ed0eb27: appendEntries rejected, sending older logs: peer="{Voter server-a9238553-d59b-9369-b4a2-be728263ba87 a9238553-d59b-9369-b4a2-be728263ba87}" next=1 -2026-04-22T10:09:18.357Z [WARN] server-05ec8efb-eeb7-eca7-3c4d-c94ed8900f8e: failed to get previous log: previous-index=1 last-index=0 error="log not found" -2026-04-22T10:09:18.357Z [WARN] server-52d0754e-b7a3-3283-75d8-98a76ed0eb27: appendEntries rejected, sending older logs: peer="{Voter server-05ec8efb-eeb7-eca7-3c4d-c94ed8900f8e 05ec8efb-eeb7-eca7-3c4d-c94ed8900f8e}" next=1 -2026-04-22T10:09:18.357Z [INFO] server-52d0754e-b7a3-3283-75d8-98a76ed0eb27: pipelining replication: peer="{Voter server-a9238553-d59b-9369-b4a2-be728263ba87 a9238553-d59b-9369-b4a2-be728263ba87}" -2026-04-22T10:09:18.357Z [INFO] server-52d0754e-b7a3-3283-75d8-98a76ed0eb27: pipelining replication: peer="{Voter server-05ec8efb-eeb7-eca7-3c4d-c94ed8900f8e 05ec8efb-eeb7-eca7-3c4d-c94ed8900f8e}" -2026-04-22T10:09:18.463Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:09:18.357595267 +0000 UTC m=+4.381391413 (2 nodes), 79.212234ms from start of poll, 79.48729ms from cluster start. Timeout at 2026-04-22 10:09:18.462661232 +0000 UTC m=+4.486457324, 105.065911ms after stability -2026-04-22T10:09:18.463Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:18.569Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:18.463510217 +0000 UTC m=+4.487306305 (1 nodes), 699ns from start of poll, 185.402182ms from cluster start. Timeout at 2026-04-22 10:09:18.568513782 +0000 UTC m=+4.592309868, 105.003563ms after stability -2026-04-22T10:09:18.580Z [INFO] server-52d0754e-b7a3-3283-75d8-98a76ed0eb27: aborting pipeline replication: peer="{Voter server-05ec8efb-eeb7-eca7-3c4d-c94ed8900f8e 05ec8efb-eeb7-eca7-3c4d-c94ed8900f8e}" -2026-04-22T10:09:18.580Z [INFO] server-52d0754e-b7a3-3283-75d8-98a76ed0eb27: aborting pipeline replication: peer="{Voter server-a9238553-d59b-9369-b4a2-be728263ba87 a9238553-d59b-9369-b4a2-be728263ba87}" ---- PASS: TestRaft_LiveBootstrap (0.30s) +2025-03-20T03:49:32.614Z [INFO] server-851d465b-80ed-10c7-720f-d854e5465d63: initial configuration: index=0 servers=[] +2025-03-20T03:49:32.614Z [INFO] server-3b28d9d2-3741-0ee6-1c9b-747247396ced: initial configuration: index=0 servers=[] +2025-03-20T03:49:32.614Z [INFO] server-e9ec5c55-8221-9127-815b-145c53dfbc13: initial configuration: index=0 servers=[] +2025-03-20T03:49:32.614Z [INFO] server-3b28d9d2-3741-0ee6-1c9b-747247396ced: entering follower state: follower="Node at 3b28d9d2-3741-0ee6-1c9b-747247396ced [Follower]" leader-address= leader-id= +2025-03-20T03:49:32.614Z [INFO] server-851d465b-80ed-10c7-720f-d854e5465d63: entering follower state: follower="Node at 851d465b-80ed-10c7-720f-d854e5465d63 [Follower]" leader-address= leader-id= +2025-03-20T03:49:32.614Z [INFO] server-e9ec5c55-8221-9127-815b-145c53dfbc13: entering follower state: follower="Node at e9ec5c55-8221-9127-815b-145c53dfbc13 [Follower]" leader-address= leader-id= +2025-03-20T03:49:32.614Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:49:32.666Z [WARN] server-e9ec5c55-8221-9127-815b-145c53dfbc13: no known peers, aborting election +2025-03-20T03:49:32.674Z [WARN] server-851d465b-80ed-10c7-720f-d854e5465d63: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:32.674Z [INFO] server-851d465b-80ed-10c7-720f-d854e5465d63: entering candidate state: node="Node at 851d465b-80ed-10c7-720f-d854e5465d63 [Candidate]" term=2 +2025-03-20T03:49:32.674Z [INFO] server-851d465b-80ed-10c7-720f-d854e5465d63: election won: term=2 tally=2 +2025-03-20T03:49:32.674Z [INFO] server-851d465b-80ed-10c7-720f-d854e5465d63: entering leader state: leader="Node at 851d465b-80ed-10c7-720f-d854e5465d63 [Leader]" +2025-03-20T03:49:32.674Z [INFO] server-851d465b-80ed-10c7-720f-d854e5465d63: added peer, starting replication: peer=server-3b28d9d2-3741-0ee6-1c9b-747247396ced +2025-03-20T03:49:32.674Z [INFO] server-851d465b-80ed-10c7-720f-d854e5465d63: added peer, starting replication: peer=server-e9ec5c55-8221-9127-815b-145c53dfbc13 +2025-03-20T03:49:32.674Z [WARN] server-3b28d9d2-3741-0ee6-1c9b-747247396ced: failed to get previous log: previous-index=1 last-index=0 error="log not found" +2025-03-20T03:49:32.674Z [WARN] server-e9ec5c55-8221-9127-815b-145c53dfbc13: failed to get previous log: previous-index=1 last-index=0 error="log not found" +2025-03-20T03:49:32.674Z [WARN] server-851d465b-80ed-10c7-720f-d854e5465d63: appendEntries rejected, sending older logs: peer="{Voter server-3b28d9d2-3741-0ee6-1c9b-747247396ced 3b28d9d2-3741-0ee6-1c9b-747247396ced}" next=1 +2025-03-20T03:49:32.675Z [WARN] server-851d465b-80ed-10c7-720f-d854e5465d63: appendEntries rejected, sending older logs: peer="{Voter server-e9ec5c55-8221-9127-815b-145c53dfbc13 e9ec5c55-8221-9127-815b-145c53dfbc13}" next=1 +2025-03-20T03:49:32.675Z [INFO] server-851d465b-80ed-10c7-720f-d854e5465d63: pipelining replication: peer="{Voter server-3b28d9d2-3741-0ee6-1c9b-747247396ced 3b28d9d2-3741-0ee6-1c9b-747247396ced}" +2025-03-20T03:49:32.675Z [INFO] server-851d465b-80ed-10c7-720f-d854e5465d63: pipelining replication: peer="{Voter server-e9ec5c55-8221-9127-815b-145c53dfbc13 e9ec5c55-8221-9127-815b-145c53dfbc13}" +2025-03-20T03:49:32.780Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:49:32.67473218 +0000 UTC m=+4.349186401 (2 nodes), 59.912987ms from start of poll, 60.593775ms from cluster start. Timeout at 2025-03-20 03:49:32.779736308 +0000 UTC m=+4.454189114, 105.002713ms after stability +2025-03-20T03:49:32.780Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:32.885Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:32.780583505 +0000 UTC m=+4.455036492 (1 nodes), 2.22µs from start of poll, 166.443866ms from cluster start. Timeout at 2025-03-20 03:49:32.885597023 +0000 UTC m=+4.560049785, 105.013293ms after stability +2025-03-20T03:49:32.897Z [INFO] server-851d465b-80ed-10c7-720f-d854e5465d63: aborting pipeline replication: peer="{Voter server-e9ec5c55-8221-9127-815b-145c53dfbc13 e9ec5c55-8221-9127-815b-145c53dfbc13}" +2025-03-20T03:49:32.897Z [INFO] server-851d465b-80ed-10c7-720f-d854e5465d63: aborting pipeline replication: peer="{Voter server-3b28d9d2-3741-0ee6-1c9b-747247396ced 3b28d9d2-3741-0ee6-1c9b-747247396ced}" +--- PASS: TestRaft_LiveBootstrap (0.28s) === RUN TestRaft_LiveBootstrap_From_NonVoter -2026-04-22T10:09:18.581Z [INFO] server-227a4b70-2d39-e02e-9ac7-70f56c7eeb64: initial configuration: index=0 servers=[] -2026-04-22T10:09:18.581Z [INFO] server-227a4b70-2d39-e02e-9ac7-70f56c7eeb64: entering follower state: follower="Node at 227a4b70-2d39-e02e-9ac7-70f56c7eeb64 [Follower]" leader-address= leader-id= -2026-04-22T10:09:18.581Z [INFO] server-5e72e0d4-8d44-6b0b-f9f1-e9783a6812cb: initial configuration: index=0 servers=[] -2026-04-22T10:09:18.581Z [INFO] server-5e72e0d4-8d44-6b0b-f9f1-e9783a6812cb: entering follower state: follower="Node at 5e72e0d4-8d44-6b0b-f9f1-e9783a6812cb [Follower]" leader-address= leader-id= +2025-03-20T03:49:32.898Z [INFO] server-3d2805d2-24cf-1e84-82b6-ed346b2a6a55: initial configuration: index=0 servers=[] +2025-03-20T03:49:32.898Z [INFO] server-b3bac581-111a-f771-0505-503cc069b3fc: initial configuration: index=0 servers=[] +2025-03-20T03:49:32.898Z [INFO] server-3d2805d2-24cf-1e84-82b6-ed346b2a6a55: entering follower state: follower="Node at 3d2805d2-24cf-1e84-82b6-ed346b2a6a55 [Follower]" leader-address= leader-id= --- PASS: TestRaft_LiveBootstrap_From_NonVoter (0.00s) === RUN TestRaft_RecoverCluster_NoState -2026-04-22T10:09:18.581Z [INFO] server-2d07da6e-b1c4-7aa6-2b0d-7ae5965707fc: initial configuration: index=0 servers=[] -2026-04-22T10:09:18.581Z [INFO] server-2d07da6e-b1c4-7aa6-2b0d-7ae5965707fc: entering follower state: follower="Node at 2d07da6e-b1c4-7aa6-2b0d-7ae5965707fc [Follower]" leader-address= leader-id= +2025-03-20T03:49:32.899Z [INFO] server-2027fdfa-294e-b3f8-3fc5-c411f1e6022d: initial configuration: index=0 servers=[] --- PASS: TestRaft_RecoverCluster_NoState (0.00s) === RUN TestRaft_RecoverCluster === RUN TestRaft_RecoverCluster/no_snapshot,_no_trailing_logs -2026-04-22T10:09:18.582Z [INFO] server-3244534b-f4e9-d8a2-ad82-af518b84b638: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3244534b-f4e9-d8a2-ad82-af518b84b638 Address:3244534b-f4e9-d8a2-ad82-af518b84b638} {Suffrage:Voter ID:server-5b4c6018-176a-09c3-0a51-b3aa89ef2ae8 Address:5b4c6018-176a-09c3-0a51-b3aa89ef2ae8} {Suffrage:Voter ID:server-6b596672-1647-4512-b963-f14a6fa59b74 Address:6b596672-1647-4512-b963-f14a6fa59b74}]" -2026-04-22T10:09:18.582Z [INFO] server-5b4c6018-176a-09c3-0a51-b3aa89ef2ae8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3244534b-f4e9-d8a2-ad82-af518b84b638 Address:3244534b-f4e9-d8a2-ad82-af518b84b638} {Suffrage:Voter ID:server-5b4c6018-176a-09c3-0a51-b3aa89ef2ae8 Address:5b4c6018-176a-09c3-0a51-b3aa89ef2ae8} {Suffrage:Voter ID:server-6b596672-1647-4512-b963-f14a6fa59b74 Address:6b596672-1647-4512-b963-f14a6fa59b74}]" -2026-04-22T10:09:18.582Z [INFO] server-6b596672-1647-4512-b963-f14a6fa59b74: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3244534b-f4e9-d8a2-ad82-af518b84b638 Address:3244534b-f4e9-d8a2-ad82-af518b84b638} {Suffrage:Voter ID:server-5b4c6018-176a-09c3-0a51-b3aa89ef2ae8 Address:5b4c6018-176a-09c3-0a51-b3aa89ef2ae8} {Suffrage:Voter ID:server-6b596672-1647-4512-b963-f14a6fa59b74 Address:6b596672-1647-4512-b963-f14a6fa59b74}]" -2026-04-22T10:09:18.582Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:18.582Z [INFO] server-3244534b-f4e9-d8a2-ad82-af518b84b638: entering follower state: follower="Node at 3244534b-f4e9-d8a2-ad82-af518b84b638 [Follower]" leader-address= leader-id= -2026-04-22T10:09:18.583Z [INFO] server-5b4c6018-176a-09c3-0a51-b3aa89ef2ae8: entering follower state: follower="Node at 5b4c6018-176a-09c3-0a51-b3aa89ef2ae8 [Follower]" leader-address= leader-id= -2026-04-22T10:09:18.583Z [INFO] server-6b596672-1647-4512-b963-f14a6fa59b74: entering follower state: follower="Node at 6b596672-1647-4512-b963-f14a6fa59b74 [Follower]" leader-address= leader-id= -2026-04-22T10:09:18.635Z [WARN] server-6b596672-1647-4512-b963-f14a6fa59b74: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:18.635Z [INFO] server-6b596672-1647-4512-b963-f14a6fa59b74: entering candidate state: node="Node at 6b596672-1647-4512-b963-f14a6fa59b74 [Candidate]" term=2 -2026-04-22T10:09:18.636Z [INFO] server-6b596672-1647-4512-b963-f14a6fa59b74: election won: term=2 tally=2 -2026-04-22T10:09:18.636Z [INFO] server-6b596672-1647-4512-b963-f14a6fa59b74: entering leader state: leader="Node at 6b596672-1647-4512-b963-f14a6fa59b74 [Leader]" -2026-04-22T10:09:18.636Z [INFO] server-6b596672-1647-4512-b963-f14a6fa59b74: added peer, starting replication: peer=server-3244534b-f4e9-d8a2-ad82-af518b84b638 -2026-04-22T10:09:18.636Z [INFO] server-6b596672-1647-4512-b963-f14a6fa59b74: added peer, starting replication: peer=server-5b4c6018-176a-09c3-0a51-b3aa89ef2ae8 -2026-04-22T10:09:18.636Z [INFO] server-6b596672-1647-4512-b963-f14a6fa59b74: pipelining replication: peer="{Voter server-5b4c6018-176a-09c3-0a51-b3aa89ef2ae8 5b4c6018-176a-09c3-0a51-b3aa89ef2ae8}" -2026-04-22T10:09:18.636Z [INFO] server-6b596672-1647-4512-b963-f14a6fa59b74: pipelining replication: peer="{Voter server-3244534b-f4e9-d8a2-ad82-af518b84b638 3244534b-f4e9-d8a2-ad82-af518b84b638}" -2026-04-22T10:09:18.741Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:18.636050015 +0000 UTC m=+4.659846114 (1 nodes), 53.06919ms from start of poll, 53.390068ms from cluster start. Timeout at 2026-04-22 10:09:18.741053327 +0000 UTC m=+4.764849410, 105.003296ms after stability -2026-04-22T10:09:18.741Z [INFO] server-6b596672-1647-4512-b963-f14a6fa59b74: aborting pipeline replication: peer="{Voter server-5b4c6018-176a-09c3-0a51-b3aa89ef2ae8 5b4c6018-176a-09c3-0a51-b3aa89ef2ae8}" -2026-04-22T10:09:18.741Z [INFO] server-6b596672-1647-4512-b963-f14a6fa59b74: aborting pipeline replication: peer="{Voter server-3244534b-f4e9-d8a2-ad82-af518b84b638 3244534b-f4e9-d8a2-ad82-af518b84b638}" -2026-04-22T10:09:18.741Z [INFO] creating new snapshot: path=/tmp/raft3187329625/snapshots/2-2-1776852558741.tmp -2026-04-22T10:09:18.742Z [INFO] server-3244534b-f4e9-d8a2-ad82-af518b84b638: starting restore from snapshot: id=2-2-1776852558741 last-index=2 last-term=2 size-in-bytes=1 -2026-04-22T10:09:18.742Z [INFO] server-3244534b-f4e9-d8a2-ad82-af518b84b638: snapshot restore progress: id=2-2-1776852558741 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% -2026-04-22T10:09:18.742Z [INFO] server-3244534b-f4e9-d8a2-ad82-af518b84b638: restored from snapshot: id=2-2-1776852558741 last-index=2 last-term=2 size-in-bytes=1 -2026-04-22T10:09:18.742Z [INFO] server-3244534b-f4e9-d8a2-ad82-af518b84b638: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3244534b-f4e9-d8a2-ad82-af518b84b638 Address:3244534b-f4e9-d8a2-ad82-af518b84b638} {Suffrage:Voter ID:server-5b4c6018-176a-09c3-0a51-b3aa89ef2ae8 Address:5b4c6018-176a-09c3-0a51-b3aa89ef2ae8} {Suffrage:Voter ID:server-6b596672-1647-4512-b963-f14a6fa59b74 Address:6b596672-1647-4512-b963-f14a6fa59b74}]" -2026-04-22T10:09:18.742Z [INFO] server-3244534b-f4e9-d8a2-ad82-af518b84b638: entering follower state: follower="Node at 3244534b-f4e9-d8a2-ad82-af518b84b638 [Follower]" leader-address= leader-id= -2026-04-22T10:09:18.742Z [INFO] creating new snapshot: path=/tmp/raft201431586/snapshots/2-2-1776852558742.tmp -2026-04-22T10:09:18.742Z [INFO] server-5b4c6018-176a-09c3-0a51-b3aa89ef2ae8: starting restore from snapshot: id=2-2-1776852558742 last-index=2 last-term=2 size-in-bytes=1 -2026-04-22T10:09:18.742Z [INFO] server-5b4c6018-176a-09c3-0a51-b3aa89ef2ae8: snapshot restore progress: id=2-2-1776852558742 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% -2026-04-22T10:09:18.742Z [INFO] server-5b4c6018-176a-09c3-0a51-b3aa89ef2ae8: restored from snapshot: id=2-2-1776852558742 last-index=2 last-term=2 size-in-bytes=1 -2026-04-22T10:09:18.742Z [INFO] server-5b4c6018-176a-09c3-0a51-b3aa89ef2ae8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3244534b-f4e9-d8a2-ad82-af518b84b638 Address:3244534b-f4e9-d8a2-ad82-af518b84b638} {Suffrage:Voter ID:server-5b4c6018-176a-09c3-0a51-b3aa89ef2ae8 Address:5b4c6018-176a-09c3-0a51-b3aa89ef2ae8} {Suffrage:Voter ID:server-6b596672-1647-4512-b963-f14a6fa59b74 Address:6b596672-1647-4512-b963-f14a6fa59b74}]" -2026-04-22T10:09:18.743Z [INFO] server-5b4c6018-176a-09c3-0a51-b3aa89ef2ae8: entering follower state: follower="Node at 5b4c6018-176a-09c3-0a51-b3aa89ef2ae8 [Follower]" leader-address= leader-id= -2026-04-22T10:09:18.743Z [INFO] creating new snapshot: path=/tmp/raft3849371788/snapshots/2-2-1776852558743.tmp -2026-04-22T10:09:18.744Z [INFO] server-6b596672-1647-4512-b963-f14a6fa59b74: starting restore from snapshot: id=2-2-1776852558743 last-index=2 last-term=2 size-in-bytes=1 -2026-04-22T10:09:18.744Z [INFO] server-6b596672-1647-4512-b963-f14a6fa59b74: snapshot restore progress: id=2-2-1776852558743 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% -2026-04-22T10:09:18.744Z [INFO] server-6b596672-1647-4512-b963-f14a6fa59b74: restored from snapshot: id=2-2-1776852558743 last-index=2 last-term=2 size-in-bytes=1 -2026-04-22T10:09:18.744Z [INFO] server-6b596672-1647-4512-b963-f14a6fa59b74: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3244534b-f4e9-d8a2-ad82-af518b84b638 Address:3244534b-f4e9-d8a2-ad82-af518b84b638} {Suffrage:Voter ID:server-5b4c6018-176a-09c3-0a51-b3aa89ef2ae8 Address:5b4c6018-176a-09c3-0a51-b3aa89ef2ae8} {Suffrage:Voter ID:server-6b596672-1647-4512-b963-f14a6fa59b74 Address:6b596672-1647-4512-b963-f14a6fa59b74}]" -2026-04-22T10:09:18.744Z [INFO] server-6b596672-1647-4512-b963-f14a6fa59b74: entering follower state: follower="Node at 6b596672-1647-4512-b963-f14a6fa59b74 [Follower]" leader-address= leader-id= -2026-04-22T10:09:18.809Z [WARN] server-6b596672-1647-4512-b963-f14a6fa59b74: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:18.809Z [INFO] server-6b596672-1647-4512-b963-f14a6fa59b74: entering candidate state: node="Node at 6b596672-1647-4512-b963-f14a6fa59b74 [Candidate]" term=3 -2026-04-22T10:09:18.809Z [INFO] server-6b596672-1647-4512-b963-f14a6fa59b74: election won: term=3 tally=2 -2026-04-22T10:09:18.809Z [INFO] server-6b596672-1647-4512-b963-f14a6fa59b74: entering leader state: leader="Node at 6b596672-1647-4512-b963-f14a6fa59b74 [Leader]" -2026-04-22T10:09:18.809Z [INFO] server-6b596672-1647-4512-b963-f14a6fa59b74: added peer, starting replication: peer=server-3244534b-f4e9-d8a2-ad82-af518b84b638 -2026-04-22T10:09:18.809Z [INFO] server-6b596672-1647-4512-b963-f14a6fa59b74: added peer, starting replication: peer=server-5b4c6018-176a-09c3-0a51-b3aa89ef2ae8 -2026-04-22T10:09:18.810Z [INFO] server-6b596672-1647-4512-b963-f14a6fa59b74: pipelining replication: peer="{Voter server-5b4c6018-176a-09c3-0a51-b3aa89ef2ae8 5b4c6018-176a-09c3-0a51-b3aa89ef2ae8}" -2026-04-22T10:09:18.810Z [INFO] server-6b596672-1647-4512-b963-f14a6fa59b74: pipelining replication: peer="{Voter server-3244534b-f4e9-d8a2-ad82-af518b84b638 3244534b-f4e9-d8a2-ad82-af518b84b638}" -2026-04-22T10:09:19.060Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:19.166Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:19.060161906 +0000 UTC m=+5.083957992 (1 nodes), 23.028µs from start of poll, 477.501946ms from cluster start. Timeout at 2026-04-22 10:09:19.165167022 +0000 UTC m=+5.188963130, 105.005138ms after stability -2026-04-22T10:09:19.166Z [INFO] server-6b596672-1647-4512-b963-f14a6fa59b74: aborting pipeline replication: peer="{Voter server-5b4c6018-176a-09c3-0a51-b3aa89ef2ae8 5b4c6018-176a-09c3-0a51-b3aa89ef2ae8}" -2026-04-22T10:09:19.166Z [INFO] server-6b596672-1647-4512-b963-f14a6fa59b74: aborting pipeline replication: peer="{Voter server-3244534b-f4e9-d8a2-ad82-af518b84b638 3244534b-f4e9-d8a2-ad82-af518b84b638}" +2025-03-20T03:49:32.900Z [INFO] server-03c9f7d9-e8b8-e512-3089-ac07095d6e95: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-03c9f7d9-e8b8-e512-3089-ac07095d6e95 Address:03c9f7d9-e8b8-e512-3089-ac07095d6e95} {Suffrage:Voter ID:server-a0bcd02c-6d28-e499-62f3-498dde8d52a5 Address:a0bcd02c-6d28-e499-62f3-498dde8d52a5} {Suffrage:Voter ID:server-dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a Address:dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a}]" +2025-03-20T03:49:32.900Z [INFO] server-a0bcd02c-6d28-e499-62f3-498dde8d52a5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-03c9f7d9-e8b8-e512-3089-ac07095d6e95 Address:03c9f7d9-e8b8-e512-3089-ac07095d6e95} {Suffrage:Voter ID:server-a0bcd02c-6d28-e499-62f3-498dde8d52a5 Address:a0bcd02c-6d28-e499-62f3-498dde8d52a5} {Suffrage:Voter ID:server-dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a Address:dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a}]" +2025-03-20T03:49:32.900Z [INFO] server-dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-03c9f7d9-e8b8-e512-3089-ac07095d6e95 Address:03c9f7d9-e8b8-e512-3089-ac07095d6e95} {Suffrage:Voter ID:server-a0bcd02c-6d28-e499-62f3-498dde8d52a5 Address:a0bcd02c-6d28-e499-62f3-498dde8d52a5} {Suffrage:Voter ID:server-dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a Address:dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a}]" +2025-03-20T03:49:32.900Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:32.900Z [INFO] server-03c9f7d9-e8b8-e512-3089-ac07095d6e95: entering follower state: follower="Node at 03c9f7d9-e8b8-e512-3089-ac07095d6e95 [Follower]" leader-address= leader-id= +2025-03-20T03:49:32.900Z [INFO] server-dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a: entering follower state: follower="Node at dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a [Follower]" leader-address= leader-id= +2025-03-20T03:49:32.900Z [INFO] server-a0bcd02c-6d28-e499-62f3-498dde8d52a5: entering follower state: follower="Node at a0bcd02c-6d28-e499-62f3-498dde8d52a5 [Follower]" leader-address= leader-id= +2025-03-20T03:49:32.952Z [WARN] server-dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:32.952Z [INFO] server-dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a: entering candidate state: node="Node at dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a [Candidate]" term=2 +2025-03-20T03:49:32.953Z [INFO] server-dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a: election won: term=2 tally=2 +2025-03-20T03:49:32.953Z [INFO] server-dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a: entering leader state: leader="Node at dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a [Leader]" +2025-03-20T03:49:32.953Z [INFO] server-dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a: added peer, starting replication: peer=server-03c9f7d9-e8b8-e512-3089-ac07095d6e95 +2025-03-20T03:49:32.953Z [INFO] server-dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a: added peer, starting replication: peer=server-a0bcd02c-6d28-e499-62f3-498dde8d52a5 +2025-03-20T03:49:32.953Z [INFO] server-dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a: pipelining replication: peer="{Voter server-a0bcd02c-6d28-e499-62f3-498dde8d52a5 a0bcd02c-6d28-e499-62f3-498dde8d52a5}" +2025-03-20T03:49:32.953Z [INFO] server-dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a: pipelining replication: peer="{Voter server-03c9f7d9-e8b8-e512-3089-ac07095d6e95 03c9f7d9-e8b8-e512-3089-ac07095d6e95}" +2025-03-20T03:49:33.059Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:32.953098018 +0000 UTC m=+4.627550823 (1 nodes), 52.475691ms from start of poll, 52.832475ms from cluster start. Timeout at 2025-03-20 03:49:33.058107387 +0000 UTC m=+4.732560217, 105.009394ms after stability +2025-03-20T03:49:33.059Z [INFO] server-dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a: aborting pipeline replication: peer="{Voter server-a0bcd02c-6d28-e499-62f3-498dde8d52a5 a0bcd02c-6d28-e499-62f3-498dde8d52a5}" +2025-03-20T03:49:33.059Z [INFO] server-dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a: aborting pipeline replication: peer="{Voter server-03c9f7d9-e8b8-e512-3089-ac07095d6e95 03c9f7d9-e8b8-e512-3089-ac07095d6e95}" +2025-03-20T03:49:33.059Z [INFO] creating new snapshot: path=/tmp/raft4173403060/snapshots/2-2-1742442573059.tmp +2025-03-20T03:49:33.060Z [INFO] server-03c9f7d9-e8b8-e512-3089-ac07095d6e95: starting restore from snapshot: id=2-2-1742442573059 last-index=2 last-term=2 size-in-bytes=1 +2025-03-20T03:49:33.060Z [INFO] server-03c9f7d9-e8b8-e512-3089-ac07095d6e95: snapshot restore progress: id=2-2-1742442573059 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% +2025-03-20T03:49:33.060Z [INFO] server-03c9f7d9-e8b8-e512-3089-ac07095d6e95: restored from snapshot: id=2-2-1742442573059 last-index=2 last-term=2 size-in-bytes=1 +2025-03-20T03:49:33.060Z [INFO] server-03c9f7d9-e8b8-e512-3089-ac07095d6e95: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-03c9f7d9-e8b8-e512-3089-ac07095d6e95 Address:03c9f7d9-e8b8-e512-3089-ac07095d6e95} {Suffrage:Voter ID:server-a0bcd02c-6d28-e499-62f3-498dde8d52a5 Address:a0bcd02c-6d28-e499-62f3-498dde8d52a5} {Suffrage:Voter ID:server-dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a Address:dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a}]" +2025-03-20T03:49:33.060Z [INFO] creating new snapshot: path=/tmp/raft216540559/snapshots/2-2-1742442573060.tmp +2025-03-20T03:49:33.061Z [INFO] server-a0bcd02c-6d28-e499-62f3-498dde8d52a5: starting restore from snapshot: id=2-2-1742442573060 last-index=2 last-term=2 size-in-bytes=1 +2025-03-20T03:49:33.061Z [INFO] server-a0bcd02c-6d28-e499-62f3-498dde8d52a5: snapshot restore progress: id=2-2-1742442573060 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% +2025-03-20T03:49:33.061Z [INFO] server-a0bcd02c-6d28-e499-62f3-498dde8d52a5: restored from snapshot: id=2-2-1742442573060 last-index=2 last-term=2 size-in-bytes=1 +2025-03-20T03:49:33.061Z [INFO] server-a0bcd02c-6d28-e499-62f3-498dde8d52a5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-03c9f7d9-e8b8-e512-3089-ac07095d6e95 Address:03c9f7d9-e8b8-e512-3089-ac07095d6e95} {Suffrage:Voter ID:server-a0bcd02c-6d28-e499-62f3-498dde8d52a5 Address:a0bcd02c-6d28-e499-62f3-498dde8d52a5} {Suffrage:Voter ID:server-dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a Address:dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a}]" +2025-03-20T03:49:33.061Z [INFO] creating new snapshot: path=/tmp/raft2091110312/snapshots/2-2-1742442573061.tmp +2025-03-20T03:49:33.062Z [INFO] server-dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a: starting restore from snapshot: id=2-2-1742442573061 last-index=2 last-term=2 size-in-bytes=1 +2025-03-20T03:49:33.062Z [INFO] server-dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a: snapshot restore progress: id=2-2-1742442573061 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% +2025-03-20T03:49:33.062Z [INFO] server-dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a: restored from snapshot: id=2-2-1742442573061 last-index=2 last-term=2 size-in-bytes=1 +2025-03-20T03:49:33.062Z [INFO] server-dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-03c9f7d9-e8b8-e512-3089-ac07095d6e95 Address:03c9f7d9-e8b8-e512-3089-ac07095d6e95} {Suffrage:Voter ID:server-a0bcd02c-6d28-e499-62f3-498dde8d52a5 Address:a0bcd02c-6d28-e499-62f3-498dde8d52a5} {Suffrage:Voter ID:server-dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a Address:dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a}]" +2025-03-20T03:49:33.062Z [INFO] server-03c9f7d9-e8b8-e512-3089-ac07095d6e95: entering follower state: follower="Node at 03c9f7d9-e8b8-e512-3089-ac07095d6e95 [Follower]" leader-address= leader-id= +2025-03-20T03:49:33.062Z [INFO] server-a0bcd02c-6d28-e499-62f3-498dde8d52a5: entering follower state: follower="Node at a0bcd02c-6d28-e499-62f3-498dde8d52a5 [Follower]" leader-address= leader-id= +2025-03-20T03:49:33.063Z [INFO] server-dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a: entering follower state: follower="Node at dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a [Follower]" leader-address= leader-id= +2025-03-20T03:49:33.133Z [WARN] server-dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:33.133Z [INFO] server-dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a: entering candidate state: node="Node at dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a [Candidate]" term=3 +2025-03-20T03:49:33.133Z [INFO] server-dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a: election won: term=3 tally=2 +2025-03-20T03:49:33.133Z [INFO] server-dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a: entering leader state: leader="Node at dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a [Leader]" +2025-03-20T03:49:33.133Z [INFO] server-dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a: added peer, starting replication: peer=server-03c9f7d9-e8b8-e512-3089-ac07095d6e95 +2025-03-20T03:49:33.133Z [INFO] server-dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a: added peer, starting replication: peer=server-a0bcd02c-6d28-e499-62f3-498dde8d52a5 +2025-03-20T03:49:33.133Z [INFO] server-dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a: pipelining replication: peer="{Voter server-a0bcd02c-6d28-e499-62f3-498dde8d52a5 a0bcd02c-6d28-e499-62f3-498dde8d52a5}" +2025-03-20T03:49:33.134Z [INFO] server-dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a: pipelining replication: peer="{Voter server-03c9f7d9-e8b8-e512-3089-ac07095d6e95 03c9f7d9-e8b8-e512-3089-ac07095d6e95}" +2025-03-20T03:49:33.378Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:33.485Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:33.37903827 +0000 UTC m=+5.053491052 (1 nodes), 55.013µs from start of poll, 478.772704ms from cluster start. Timeout at 2025-03-20 03:49:33.484064592 +0000 UTC m=+5.158517587, 105.026535ms after stability +2025-03-20T03:49:33.485Z [INFO] server-dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a: aborting pipeline replication: peer="{Voter server-a0bcd02c-6d28-e499-62f3-498dde8d52a5 a0bcd02c-6d28-e499-62f3-498dde8d52a5}" +2025-03-20T03:49:33.485Z [INFO] server-dcc2f855-0a46-e7ba-8c3c-78f06f16bf2a: aborting pipeline replication: peer="{Voter server-03c9f7d9-e8b8-e512-3089-ac07095d6e95 03c9f7d9-e8b8-e512-3089-ac07095d6e95}" === RUN TestRaft_RecoverCluster/no_snapshot,_some_trailing_logs -2026-04-22T10:09:19.342Z [INFO] server-77176985-871c-76cf-c0e7-d004dc150816: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-77176985-871c-76cf-c0e7-d004dc150816 Address:77176985-871c-76cf-c0e7-d004dc150816} {Suffrage:Voter ID:server-8ffb20ef-6433-8bfc-fade-d5a74bffbe3f Address:8ffb20ef-6433-8bfc-fade-d5a74bffbe3f} {Suffrage:Voter ID:server-f5f56c4a-e57b-3330-1976-085698632c76 Address:f5f56c4a-e57b-3330-1976-085698632c76}]" -2026-04-22T10:09:19.342Z [INFO] server-77176985-871c-76cf-c0e7-d004dc150816: entering follower state: follower="Node at 77176985-871c-76cf-c0e7-d004dc150816 [Follower]" leader-address= leader-id= -2026-04-22T10:09:19.342Z [INFO] server-8ffb20ef-6433-8bfc-fade-d5a74bffbe3f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-77176985-871c-76cf-c0e7-d004dc150816 Address:77176985-871c-76cf-c0e7-d004dc150816} {Suffrage:Voter ID:server-8ffb20ef-6433-8bfc-fade-d5a74bffbe3f Address:8ffb20ef-6433-8bfc-fade-d5a74bffbe3f} {Suffrage:Voter ID:server-f5f56c4a-e57b-3330-1976-085698632c76 Address:f5f56c4a-e57b-3330-1976-085698632c76}]" -2026-04-22T10:09:19.342Z [INFO] server-f5f56c4a-e57b-3330-1976-085698632c76: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-77176985-871c-76cf-c0e7-d004dc150816 Address:77176985-871c-76cf-c0e7-d004dc150816} {Suffrage:Voter ID:server-8ffb20ef-6433-8bfc-fade-d5a74bffbe3f Address:8ffb20ef-6433-8bfc-fade-d5a74bffbe3f} {Suffrage:Voter ID:server-f5f56c4a-e57b-3330-1976-085698632c76 Address:f5f56c4a-e57b-3330-1976-085698632c76}]" -2026-04-22T10:09:19.342Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:19.342Z [INFO] server-8ffb20ef-6433-8bfc-fade-d5a74bffbe3f: entering follower state: follower="Node at 8ffb20ef-6433-8bfc-fade-d5a74bffbe3f [Follower]" leader-address= leader-id= -2026-04-22T10:09:19.342Z [INFO] server-f5f56c4a-e57b-3330-1976-085698632c76: entering follower state: follower="Node at f5f56c4a-e57b-3330-1976-085698632c76 [Follower]" leader-address= leader-id= -2026-04-22T10:09:19.398Z [WARN] server-f5f56c4a-e57b-3330-1976-085698632c76: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:19.398Z [INFO] server-f5f56c4a-e57b-3330-1976-085698632c76: entering candidate state: node="Node at f5f56c4a-e57b-3330-1976-085698632c76 [Candidate]" term=2 -2026-04-22T10:09:19.398Z [INFO] server-f5f56c4a-e57b-3330-1976-085698632c76: election won: term=2 tally=2 -2026-04-22T10:09:19.398Z [INFO] server-f5f56c4a-e57b-3330-1976-085698632c76: entering leader state: leader="Node at f5f56c4a-e57b-3330-1976-085698632c76 [Leader]" -2026-04-22T10:09:19.398Z [INFO] server-f5f56c4a-e57b-3330-1976-085698632c76: added peer, starting replication: peer=server-77176985-871c-76cf-c0e7-d004dc150816 -2026-04-22T10:09:19.398Z [INFO] server-f5f56c4a-e57b-3330-1976-085698632c76: added peer, starting replication: peer=server-8ffb20ef-6433-8bfc-fade-d5a74bffbe3f -2026-04-22T10:09:19.398Z [INFO] server-f5f56c4a-e57b-3330-1976-085698632c76: pipelining replication: peer="{Voter server-8ffb20ef-6433-8bfc-fade-d5a74bffbe3f 8ffb20ef-6433-8bfc-fade-d5a74bffbe3f}" -2026-04-22T10:09:19.399Z [INFO] server-f5f56c4a-e57b-3330-1976-085698632c76: pipelining replication: peer="{Voter server-77176985-871c-76cf-c0e7-d004dc150816 77176985-871c-76cf-c0e7-d004dc150816}" -2026-04-22T10:09:19.504Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:19.399083805 +0000 UTC m=+5.422879920 (1 nodes), 56.483022ms from start of poll, 57.005586ms from cluster start. Timeout at 2026-04-22 10:09:19.504084536 +0000 UTC m=+5.527880618, 105.000698ms after stability -2026-04-22T10:09:19.505Z [INFO] server-f5f56c4a-e57b-3330-1976-085698632c76: aborting pipeline replication: peer="{Voter server-8ffb20ef-6433-8bfc-fade-d5a74bffbe3f 8ffb20ef-6433-8bfc-fade-d5a74bffbe3f}" -2026-04-22T10:09:19.505Z [INFO] server-f5f56c4a-e57b-3330-1976-085698632c76: aborting pipeline replication: peer="{Voter server-77176985-871c-76cf-c0e7-d004dc150816 77176985-871c-76cf-c0e7-d004dc150816}" -2026-04-22T10:09:19.505Z [INFO] creating new snapshot: path=/tmp/raft3727105579/snapshots/2-6-1776852559505.tmp -2026-04-22T10:09:19.549Z [INFO] server-77176985-871c-76cf-c0e7-d004dc150816: starting restore from snapshot: id=2-6-1776852559505 last-index=6 last-term=2 size-in-bytes=25 -2026-04-22T10:09:19.549Z [INFO] server-77176985-871c-76cf-c0e7-d004dc150816: snapshot restore progress: id=2-6-1776852559505 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% -2026-04-22T10:09:19.549Z [INFO] server-77176985-871c-76cf-c0e7-d004dc150816: restored from snapshot: id=2-6-1776852559505 last-index=6 last-term=2 size-in-bytes=25 -2026-04-22T10:09:19.550Z [INFO] server-77176985-871c-76cf-c0e7-d004dc150816: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-77176985-871c-76cf-c0e7-d004dc150816 Address:77176985-871c-76cf-c0e7-d004dc150816} {Suffrage:Voter ID:server-8ffb20ef-6433-8bfc-fade-d5a74bffbe3f Address:8ffb20ef-6433-8bfc-fade-d5a74bffbe3f} {Suffrage:Voter ID:server-f5f56c4a-e57b-3330-1976-085698632c76 Address:f5f56c4a-e57b-3330-1976-085698632c76}]" -2026-04-22T10:09:19.550Z [INFO] creating new snapshot: path=/tmp/raft3326683447/snapshots/2-6-1776852559550.tmp -2026-04-22T10:09:19.550Z [INFO] server-77176985-871c-76cf-c0e7-d004dc150816: entering follower state: follower="Node at 77176985-871c-76cf-c0e7-d004dc150816 [Follower]" leader-address= leader-id= -2026-04-22T10:09:19.550Z [INFO] server-8ffb20ef-6433-8bfc-fade-d5a74bffbe3f: starting restore from snapshot: id=2-6-1776852559550 last-index=6 last-term=2 size-in-bytes=25 -2026-04-22T10:09:19.550Z [INFO] server-8ffb20ef-6433-8bfc-fade-d5a74bffbe3f: snapshot restore progress: id=2-6-1776852559550 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% -2026-04-22T10:09:19.550Z [INFO] server-8ffb20ef-6433-8bfc-fade-d5a74bffbe3f: restored from snapshot: id=2-6-1776852559550 last-index=6 last-term=2 size-in-bytes=25 -2026-04-22T10:09:19.550Z [INFO] server-8ffb20ef-6433-8bfc-fade-d5a74bffbe3f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-77176985-871c-76cf-c0e7-d004dc150816 Address:77176985-871c-76cf-c0e7-d004dc150816} {Suffrage:Voter ID:server-8ffb20ef-6433-8bfc-fade-d5a74bffbe3f Address:8ffb20ef-6433-8bfc-fade-d5a74bffbe3f} {Suffrage:Voter ID:server-f5f56c4a-e57b-3330-1976-085698632c76 Address:f5f56c4a-e57b-3330-1976-085698632c76}]" -2026-04-22T10:09:19.550Z [INFO] creating new snapshot: path=/tmp/raft1487374809/snapshots/2-6-1776852559550.tmp -2026-04-22T10:09:19.550Z [INFO] server-8ffb20ef-6433-8bfc-fade-d5a74bffbe3f: entering follower state: follower="Node at 8ffb20ef-6433-8bfc-fade-d5a74bffbe3f [Follower]" leader-address= leader-id= -2026-04-22T10:09:19.606Z [WARN] server-8ffb20ef-6433-8bfc-fade-d5a74bffbe3f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:19.606Z [INFO] server-8ffb20ef-6433-8bfc-fade-d5a74bffbe3f: entering candidate state: node="Node at 8ffb20ef-6433-8bfc-fade-d5a74bffbe3f [Candidate]" term=3 -2026-04-22T10:09:19.606Z [ERROR] server-8ffb20ef-6433-8bfc-fade-d5a74bffbe3f: failed to make requestVote RPC: target="{Voter server-f5f56c4a-e57b-3330-1976-085698632c76 f5f56c4a-e57b-3330-1976-085698632c76}" error="failed to connect to peer: f5f56c4a-e57b-3330-1976-085698632c76" term=3 -2026-04-22T10:09:19.606Z [ERROR] server-8ffb20ef-6433-8bfc-fade-d5a74bffbe3f: failed to make requestVote RPC: target="{Voter server-77176985-871c-76cf-c0e7-d004dc150816 77176985-871c-76cf-c0e7-d004dc150816}" error="failed to connect to peer: 77176985-871c-76cf-c0e7-d004dc150816" term=3 -2026-04-22T10:09:19.636Z [WARN] server-77176985-871c-76cf-c0e7-d004dc150816: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:19.636Z [INFO] server-77176985-871c-76cf-c0e7-d004dc150816: entering candidate state: node="Node at 77176985-871c-76cf-c0e7-d004dc150816 [Candidate]" term=3 -2026-04-22T10:09:19.636Z [ERROR] server-77176985-871c-76cf-c0e7-d004dc150816: failed to make requestVote RPC: target="{Voter server-f5f56c4a-e57b-3330-1976-085698632c76 f5f56c4a-e57b-3330-1976-085698632c76}" error="failed to connect to peer: f5f56c4a-e57b-3330-1976-085698632c76" term=3 -2026-04-22T10:09:19.636Z [ERROR] server-77176985-871c-76cf-c0e7-d004dc150816: failed to make requestVote RPC: target="{Voter server-8ffb20ef-6433-8bfc-fade-d5a74bffbe3f 8ffb20ef-6433-8bfc-fade-d5a74bffbe3f}" error="failed to connect to peer: 8ffb20ef-6433-8bfc-fade-d5a74bffbe3f" term=3 -2026-04-22T10:09:19.672Z [WARN] server-8ffb20ef-6433-8bfc-fade-d5a74bffbe3f: Election timeout reached, restarting election -2026-04-22T10:09:19.672Z [INFO] server-8ffb20ef-6433-8bfc-fade-d5a74bffbe3f: entering candidate state: node="Node at 8ffb20ef-6433-8bfc-fade-d5a74bffbe3f [Candidate]" term=4 -2026-04-22T10:09:19.672Z [ERROR] server-8ffb20ef-6433-8bfc-fade-d5a74bffbe3f: failed to make requestVote RPC: target="{Voter server-f5f56c4a-e57b-3330-1976-085698632c76 f5f56c4a-e57b-3330-1976-085698632c76}" error="failed to connect to peer: f5f56c4a-e57b-3330-1976-085698632c76" term=4 -2026-04-22T10:09:19.672Z [ERROR] server-8ffb20ef-6433-8bfc-fade-d5a74bffbe3f: failed to make requestVote RPC: target="{Voter server-77176985-871c-76cf-c0e7-d004dc150816 77176985-871c-76cf-c0e7-d004dc150816}" error="failed to connect to peer: 77176985-871c-76cf-c0e7-d004dc150816" term=4 -2026-04-22T10:09:19.689Z [INFO] server-f5f56c4a-e57b-3330-1976-085698632c76: starting restore from snapshot: id=2-6-1776852559550 last-index=6 last-term=2 size-in-bytes=25 -2026-04-22T10:09:19.689Z [INFO] server-f5f56c4a-e57b-3330-1976-085698632c76: snapshot restore progress: id=2-6-1776852559550 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% -2026-04-22T10:09:19.689Z [INFO] server-f5f56c4a-e57b-3330-1976-085698632c76: restored from snapshot: id=2-6-1776852559550 last-index=6 last-term=2 size-in-bytes=25 -2026-04-22T10:09:19.689Z [INFO] server-f5f56c4a-e57b-3330-1976-085698632c76: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-77176985-871c-76cf-c0e7-d004dc150816 Address:77176985-871c-76cf-c0e7-d004dc150816} {Suffrage:Voter ID:server-8ffb20ef-6433-8bfc-fade-d5a74bffbe3f Address:8ffb20ef-6433-8bfc-fade-d5a74bffbe3f} {Suffrage:Voter ID:server-f5f56c4a-e57b-3330-1976-085698632c76 Address:f5f56c4a-e57b-3330-1976-085698632c76}]" -2026-04-22T10:09:19.689Z [INFO] server-f5f56c4a-e57b-3330-1976-085698632c76: entering follower state: follower="Node at f5f56c4a-e57b-3330-1976-085698632c76 [Follower]" leader-address= leader-id= -2026-04-22T10:09:19.716Z [WARN] server-77176985-871c-76cf-c0e7-d004dc150816: Election timeout reached, restarting election -2026-04-22T10:09:19.716Z [INFO] server-77176985-871c-76cf-c0e7-d004dc150816: entering candidate state: node="Node at 77176985-871c-76cf-c0e7-d004dc150816 [Candidate]" term=4 -2026-04-22T10:09:19.716Z [INFO] server-8ffb20ef-6433-8bfc-fade-d5a74bffbe3f: duplicate requestVote for same term: term=4 -2026-04-22T10:09:19.716Z [INFO] server-77176985-871c-76cf-c0e7-d004dc150816: election won: term=4 tally=2 -2026-04-22T10:09:19.716Z [INFO] server-77176985-871c-76cf-c0e7-d004dc150816: entering leader state: leader="Node at 77176985-871c-76cf-c0e7-d004dc150816 [Leader]" -2026-04-22T10:09:19.716Z [INFO] server-77176985-871c-76cf-c0e7-d004dc150816: added peer, starting replication: peer=server-8ffb20ef-6433-8bfc-fade-d5a74bffbe3f -2026-04-22T10:09:19.716Z [INFO] server-77176985-871c-76cf-c0e7-d004dc150816: added peer, starting replication: peer=server-f5f56c4a-e57b-3330-1976-085698632c76 -2026-04-22T10:09:19.716Z [INFO] server-77176985-871c-76cf-c0e7-d004dc150816: pipelining replication: peer="{Voter server-f5f56c4a-e57b-3330-1976-085698632c76 f5f56c4a-e57b-3330-1976-085698632c76}" -2026-04-22T10:09:19.716Z [INFO] server-8ffb20ef-6433-8bfc-fade-d5a74bffbe3f: entering follower state: follower="Node at 8ffb20ef-6433-8bfc-fade-d5a74bffbe3f [Follower]" leader-address=77176985-871c-76cf-c0e7-d004dc150816 leader-id=server-77176985-871c-76cf-c0e7-d004dc150816 -2026-04-22T10:09:19.716Z [INFO] server-77176985-871c-76cf-c0e7-d004dc150816: pipelining replication: peer="{Voter server-8ffb20ef-6433-8bfc-fade-d5a74bffbe3f 8ffb20ef-6433-8bfc-fade-d5a74bffbe3f}" -2026-04-22T10:09:20.005Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:20.110Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:20.005129374 +0000 UTC m=+6.028925477 (1 nodes), 26.098µs from start of poll, 663.051143ms from cluster start. Timeout at 2026-04-22 10:09:20.11013378 +0000 UTC m=+6.133929878, 105.004401ms after stability -2026-04-22T10:09:20.110Z [INFO] server-77176985-871c-76cf-c0e7-d004dc150816: aborting pipeline replication: peer="{Voter server-f5f56c4a-e57b-3330-1976-085698632c76 f5f56c4a-e57b-3330-1976-085698632c76}" -2026-04-22T10:09:20.110Z [INFO] server-77176985-871c-76cf-c0e7-d004dc150816: aborting pipeline replication: peer="{Voter server-8ffb20ef-6433-8bfc-fade-d5a74bffbe3f 8ffb20ef-6433-8bfc-fade-d5a74bffbe3f}" +2025-03-20T03:49:33.487Z [INFO] server-c79ddb86-0969-5f5c-0634-08037c935cf4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c79ddb86-0969-5f5c-0634-08037c935cf4 Address:c79ddb86-0969-5f5c-0634-08037c935cf4} {Suffrage:Voter ID:server-5ec9104e-6ff7-3c06-4fdd-7454e331a106 Address:5ec9104e-6ff7-3c06-4fdd-7454e331a106} {Suffrage:Voter ID:server-ca26d9c1-5cf4-3b79-d6be-c03d054aa462 Address:ca26d9c1-5cf4-3b79-d6be-c03d054aa462}]" +2025-03-20T03:49:33.487Z [INFO] server-5ec9104e-6ff7-3c06-4fdd-7454e331a106: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c79ddb86-0969-5f5c-0634-08037c935cf4 Address:c79ddb86-0969-5f5c-0634-08037c935cf4} {Suffrage:Voter ID:server-5ec9104e-6ff7-3c06-4fdd-7454e331a106 Address:5ec9104e-6ff7-3c06-4fdd-7454e331a106} {Suffrage:Voter ID:server-ca26d9c1-5cf4-3b79-d6be-c03d054aa462 Address:ca26d9c1-5cf4-3b79-d6be-c03d054aa462}]" +2025-03-20T03:49:33.487Z [INFO] server-ca26d9c1-5cf4-3b79-d6be-c03d054aa462: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c79ddb86-0969-5f5c-0634-08037c935cf4 Address:c79ddb86-0969-5f5c-0634-08037c935cf4} {Suffrage:Voter ID:server-5ec9104e-6ff7-3c06-4fdd-7454e331a106 Address:5ec9104e-6ff7-3c06-4fdd-7454e331a106} {Suffrage:Voter ID:server-ca26d9c1-5cf4-3b79-d6be-c03d054aa462 Address:ca26d9c1-5cf4-3b79-d6be-c03d054aa462}]" +2025-03-20T03:49:33.487Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:33.487Z [INFO] server-c79ddb86-0969-5f5c-0634-08037c935cf4: entering follower state: follower="Node at c79ddb86-0969-5f5c-0634-08037c935cf4 [Follower]" leader-address= leader-id= +2025-03-20T03:49:33.487Z [INFO] server-5ec9104e-6ff7-3c06-4fdd-7454e331a106: entering follower state: follower="Node at 5ec9104e-6ff7-3c06-4fdd-7454e331a106 [Follower]" leader-address= leader-id= +2025-03-20T03:49:33.487Z [INFO] server-ca26d9c1-5cf4-3b79-d6be-c03d054aa462: entering follower state: follower="Node at ca26d9c1-5cf4-3b79-d6be-c03d054aa462 [Follower]" leader-address= leader-id= +2025-03-20T03:49:33.565Z [WARN] server-ca26d9c1-5cf4-3b79-d6be-c03d054aa462: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:33.565Z [INFO] server-ca26d9c1-5cf4-3b79-d6be-c03d054aa462: entering candidate state: node="Node at ca26d9c1-5cf4-3b79-d6be-c03d054aa462 [Candidate]" term=2 +2025-03-20T03:49:33.565Z [INFO] server-ca26d9c1-5cf4-3b79-d6be-c03d054aa462: election won: term=2 tally=2 +2025-03-20T03:49:33.565Z [INFO] server-ca26d9c1-5cf4-3b79-d6be-c03d054aa462: entering leader state: leader="Node at ca26d9c1-5cf4-3b79-d6be-c03d054aa462 [Leader]" +2025-03-20T03:49:33.565Z [INFO] server-ca26d9c1-5cf4-3b79-d6be-c03d054aa462: added peer, starting replication: peer=server-c79ddb86-0969-5f5c-0634-08037c935cf4 +2025-03-20T03:49:33.565Z [INFO] server-ca26d9c1-5cf4-3b79-d6be-c03d054aa462: added peer, starting replication: peer=server-5ec9104e-6ff7-3c06-4fdd-7454e331a106 +2025-03-20T03:49:33.565Z [INFO] server-ca26d9c1-5cf4-3b79-d6be-c03d054aa462: pipelining replication: peer="{Voter server-5ec9104e-6ff7-3c06-4fdd-7454e331a106 5ec9104e-6ff7-3c06-4fdd-7454e331a106}" +2025-03-20T03:49:33.565Z [INFO] server-ca26d9c1-5cf4-3b79-d6be-c03d054aa462: pipelining replication: peer="{Voter server-c79ddb86-0969-5f5c-0634-08037c935cf4 c79ddb86-0969-5f5c-0634-08037c935cf4}" +2025-03-20T03:49:33.671Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:33.566062682 +0000 UTC m=+5.240515735 (1 nodes), 78.402121ms from start of poll, 78.81514ms from cluster start. Timeout at 2025-03-20 03:49:33.671065109 +0000 UTC m=+5.345517887, 105.002152ms after stability +2025-03-20T03:49:33.672Z [INFO] server-ca26d9c1-5cf4-3b79-d6be-c03d054aa462: aborting pipeline replication: peer="{Voter server-c79ddb86-0969-5f5c-0634-08037c935cf4 c79ddb86-0969-5f5c-0634-08037c935cf4}" +2025-03-20T03:49:33.672Z [INFO] server-ca26d9c1-5cf4-3b79-d6be-c03d054aa462: aborting pipeline replication: peer="{Voter server-5ec9104e-6ff7-3c06-4fdd-7454e331a106 5ec9104e-6ff7-3c06-4fdd-7454e331a106}" +2025-03-20T03:49:33.672Z [INFO] creating new snapshot: path=/tmp/raft647169395/snapshots/2-6-1742442573672.tmp +2025-03-20T03:49:33.673Z [INFO] server-c79ddb86-0969-5f5c-0634-08037c935cf4: starting restore from snapshot: id=2-6-1742442573672 last-index=6 last-term=2 size-in-bytes=25 +2025-03-20T03:49:33.673Z [INFO] server-c79ddb86-0969-5f5c-0634-08037c935cf4: snapshot restore progress: id=2-6-1742442573672 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% +2025-03-20T03:49:33.673Z [INFO] server-c79ddb86-0969-5f5c-0634-08037c935cf4: restored from snapshot: id=2-6-1742442573672 last-index=6 last-term=2 size-in-bytes=25 +2025-03-20T03:49:33.673Z [INFO] server-c79ddb86-0969-5f5c-0634-08037c935cf4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c79ddb86-0969-5f5c-0634-08037c935cf4 Address:c79ddb86-0969-5f5c-0634-08037c935cf4} {Suffrage:Voter ID:server-5ec9104e-6ff7-3c06-4fdd-7454e331a106 Address:5ec9104e-6ff7-3c06-4fdd-7454e331a106} {Suffrage:Voter ID:server-ca26d9c1-5cf4-3b79-d6be-c03d054aa462 Address:ca26d9c1-5cf4-3b79-d6be-c03d054aa462}]" +2025-03-20T03:49:33.673Z [INFO] creating new snapshot: path=/tmp/raft1470385293/snapshots/2-6-1742442573673.tmp +2025-03-20T03:49:33.674Z [INFO] server-5ec9104e-6ff7-3c06-4fdd-7454e331a106: starting restore from snapshot: id=2-6-1742442573673 last-index=6 last-term=2 size-in-bytes=25 +2025-03-20T03:49:33.674Z [INFO] server-5ec9104e-6ff7-3c06-4fdd-7454e331a106: snapshot restore progress: id=2-6-1742442573673 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% +2025-03-20T03:49:33.674Z [INFO] server-5ec9104e-6ff7-3c06-4fdd-7454e331a106: restored from snapshot: id=2-6-1742442573673 last-index=6 last-term=2 size-in-bytes=25 +2025-03-20T03:49:33.674Z [INFO] server-5ec9104e-6ff7-3c06-4fdd-7454e331a106: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c79ddb86-0969-5f5c-0634-08037c935cf4 Address:c79ddb86-0969-5f5c-0634-08037c935cf4} {Suffrage:Voter ID:server-5ec9104e-6ff7-3c06-4fdd-7454e331a106 Address:5ec9104e-6ff7-3c06-4fdd-7454e331a106} {Suffrage:Voter ID:server-ca26d9c1-5cf4-3b79-d6be-c03d054aa462 Address:ca26d9c1-5cf4-3b79-d6be-c03d054aa462}]" +2025-03-20T03:49:33.675Z [INFO] creating new snapshot: path=/tmp/raft1722311865/snapshots/2-6-1742442573674.tmp +2025-03-20T03:49:33.675Z [INFO] server-ca26d9c1-5cf4-3b79-d6be-c03d054aa462: starting restore from snapshot: id=2-6-1742442573674 last-index=6 last-term=2 size-in-bytes=25 +2025-03-20T03:49:33.675Z [INFO] server-ca26d9c1-5cf4-3b79-d6be-c03d054aa462: snapshot restore progress: id=2-6-1742442573674 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% +2025-03-20T03:49:33.675Z [INFO] server-ca26d9c1-5cf4-3b79-d6be-c03d054aa462: restored from snapshot: id=2-6-1742442573674 last-index=6 last-term=2 size-in-bytes=25 +2025-03-20T03:49:33.675Z [INFO] server-ca26d9c1-5cf4-3b79-d6be-c03d054aa462: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c79ddb86-0969-5f5c-0634-08037c935cf4 Address:c79ddb86-0969-5f5c-0634-08037c935cf4} {Suffrage:Voter ID:server-5ec9104e-6ff7-3c06-4fdd-7454e331a106 Address:5ec9104e-6ff7-3c06-4fdd-7454e331a106} {Suffrage:Voter ID:server-ca26d9c1-5cf4-3b79-d6be-c03d054aa462 Address:ca26d9c1-5cf4-3b79-d6be-c03d054aa462}]" +2025-03-20T03:49:33.675Z [INFO] server-c79ddb86-0969-5f5c-0634-08037c935cf4: entering follower state: follower="Node at c79ddb86-0969-5f5c-0634-08037c935cf4 [Follower]" leader-address= leader-id= +2025-03-20T03:49:33.675Z [INFO] server-5ec9104e-6ff7-3c06-4fdd-7454e331a106: entering follower state: follower="Node at 5ec9104e-6ff7-3c06-4fdd-7454e331a106 [Follower]" leader-address= leader-id= +2025-03-20T03:49:33.675Z [INFO] server-ca26d9c1-5cf4-3b79-d6be-c03d054aa462: entering follower state: follower="Node at ca26d9c1-5cf4-3b79-d6be-c03d054aa462 [Follower]" leader-address= leader-id= +2025-03-20T03:49:33.726Z [WARN] server-c79ddb86-0969-5f5c-0634-08037c935cf4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:33.726Z [INFO] server-c79ddb86-0969-5f5c-0634-08037c935cf4: entering candidate state: node="Node at c79ddb86-0969-5f5c-0634-08037c935cf4 [Candidate]" term=3 +2025-03-20T03:49:33.727Z [INFO] server-c79ddb86-0969-5f5c-0634-08037c935cf4: election won: term=3 tally=2 +2025-03-20T03:49:33.727Z [INFO] server-c79ddb86-0969-5f5c-0634-08037c935cf4: entering leader state: leader="Node at c79ddb86-0969-5f5c-0634-08037c935cf4 [Leader]" +2025-03-20T03:49:33.727Z [INFO] server-c79ddb86-0969-5f5c-0634-08037c935cf4: added peer, starting replication: peer=server-5ec9104e-6ff7-3c06-4fdd-7454e331a106 +2025-03-20T03:49:33.727Z [INFO] server-c79ddb86-0969-5f5c-0634-08037c935cf4: added peer, starting replication: peer=server-ca26d9c1-5cf4-3b79-d6be-c03d054aa462 +2025-03-20T03:49:33.727Z [INFO] server-c79ddb86-0969-5f5c-0634-08037c935cf4: pipelining replication: peer="{Voter server-ca26d9c1-5cf4-3b79-d6be-c03d054aa462 ca26d9c1-5cf4-3b79-d6be-c03d054aa462}" +2025-03-20T03:49:33.727Z [INFO] server-c79ddb86-0969-5f5c-0634-08037c935cf4: pipelining replication: peer="{Voter server-5ec9104e-6ff7-3c06-4fdd-7454e331a106 5ec9104e-6ff7-3c06-4fdd-7454e331a106}" +2025-03-20T03:49:33.991Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:34.098Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:33.991857896 +0000 UTC m=+5.666310679 (1 nodes), 70.783µs from start of poll, 504.610084ms from cluster start. Timeout at 2025-03-20 03:49:34.096860693 +0000 UTC m=+5.771313698, 105.003019ms after stability +2025-03-20T03:49:34.098Z [INFO] server-c79ddb86-0969-5f5c-0634-08037c935cf4: aborting pipeline replication: peer="{Voter server-ca26d9c1-5cf4-3b79-d6be-c03d054aa462 ca26d9c1-5cf4-3b79-d6be-c03d054aa462}" +2025-03-20T03:49:34.098Z [INFO] server-c79ddb86-0969-5f5c-0634-08037c935cf4: aborting pipeline replication: peer="{Voter server-5ec9104e-6ff7-3c06-4fdd-7454e331a106 5ec9104e-6ff7-3c06-4fdd-7454e331a106}" +2025-03-20T03:49:34.598Z [ERROR] server-c79ddb86-0969-5f5c-0634-08037c935cf4: failed to heartbeat to: peer=ca26d9c1-5cf4-3b79-d6be-c03d054aa462 backoff time=10ms error="command timed out" === RUN TestRaft_RecoverCluster/snapshot,_with_trailing_logs -2026-04-22T10:09:20.111Z [INFO] server-e59ba41c-10e0-f573-f2d8-5b04f955dd0c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e59ba41c-10e0-f573-f2d8-5b04f955dd0c Address:e59ba41c-10e0-f573-f2d8-5b04f955dd0c} {Suffrage:Voter ID:server-5949481e-b89e-a129-4e2d-f55f10cdcf93 Address:5949481e-b89e-a129-4e2d-f55f10cdcf93} {Suffrage:Voter ID:server-c1426241-9fa7-6427-cf86-9b86bc9a8872 Address:c1426241-9fa7-6427-cf86-9b86bc9a8872}]" -2026-04-22T10:09:20.111Z [INFO] server-e59ba41c-10e0-f573-f2d8-5b04f955dd0c: entering follower state: follower="Node at e59ba41c-10e0-f573-f2d8-5b04f955dd0c [Follower]" leader-address= leader-id= -2026-04-22T10:09:20.111Z [INFO] server-5949481e-b89e-a129-4e2d-f55f10cdcf93: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e59ba41c-10e0-f573-f2d8-5b04f955dd0c Address:e59ba41c-10e0-f573-f2d8-5b04f955dd0c} {Suffrage:Voter ID:server-5949481e-b89e-a129-4e2d-f55f10cdcf93 Address:5949481e-b89e-a129-4e2d-f55f10cdcf93} {Suffrage:Voter ID:server-c1426241-9fa7-6427-cf86-9b86bc9a8872 Address:c1426241-9fa7-6427-cf86-9b86bc9a8872}]" -2026-04-22T10:09:20.111Z [INFO] server-5949481e-b89e-a129-4e2d-f55f10cdcf93: entering follower state: follower="Node at 5949481e-b89e-a129-4e2d-f55f10cdcf93 [Follower]" leader-address= leader-id= -2026-04-22T10:09:20.112Z [INFO] server-c1426241-9fa7-6427-cf86-9b86bc9a8872: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e59ba41c-10e0-f573-f2d8-5b04f955dd0c Address:e59ba41c-10e0-f573-f2d8-5b04f955dd0c} {Suffrage:Voter ID:server-5949481e-b89e-a129-4e2d-f55f10cdcf93 Address:5949481e-b89e-a129-4e2d-f55f10cdcf93} {Suffrage:Voter ID:server-c1426241-9fa7-6427-cf86-9b86bc9a8872 Address:c1426241-9fa7-6427-cf86-9b86bc9a8872}]" -2026-04-22T10:09:20.112Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:20.112Z [INFO] server-c1426241-9fa7-6427-cf86-9b86bc9a8872: entering follower state: follower="Node at c1426241-9fa7-6427-cf86-9b86bc9a8872 [Follower]" leader-address= leader-id= -2026-04-22T10:09:20.168Z [WARN] server-5949481e-b89e-a129-4e2d-f55f10cdcf93: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:20.168Z [INFO] server-5949481e-b89e-a129-4e2d-f55f10cdcf93: entering candidate state: node="Node at 5949481e-b89e-a129-4e2d-f55f10cdcf93 [Candidate]" term=2 -2026-04-22T10:09:20.168Z [INFO] server-5949481e-b89e-a129-4e2d-f55f10cdcf93: election won: term=2 tally=2 -2026-04-22T10:09:20.168Z [INFO] server-5949481e-b89e-a129-4e2d-f55f10cdcf93: entering leader state: leader="Node at 5949481e-b89e-a129-4e2d-f55f10cdcf93 [Leader]" -2026-04-22T10:09:20.168Z [INFO] server-5949481e-b89e-a129-4e2d-f55f10cdcf93: added peer, starting replication: peer=server-e59ba41c-10e0-f573-f2d8-5b04f955dd0c -2026-04-22T10:09:20.168Z [INFO] server-5949481e-b89e-a129-4e2d-f55f10cdcf93: added peer, starting replication: peer=server-c1426241-9fa7-6427-cf86-9b86bc9a8872 -2026-04-22T10:09:20.168Z [INFO] server-5949481e-b89e-a129-4e2d-f55f10cdcf93: pipelining replication: peer="{Voter server-c1426241-9fa7-6427-cf86-9b86bc9a8872 c1426241-9fa7-6427-cf86-9b86bc9a8872}" -2026-04-22T10:09:20.168Z [INFO] server-5949481e-b89e-a129-4e2d-f55f10cdcf93: pipelining replication: peer="{Voter server-e59ba41c-10e0-f573-f2d8-5b04f955dd0c e59ba41c-10e0-f573-f2d8-5b04f955dd0c}" -2026-04-22T10:09:20.273Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:20.168310757 +0000 UTC m=+6.192106844 (1 nodes), 56.230964ms from start of poll, 56.638213ms from cluster start. Timeout at 2026-04-22 10:09:20.273312006 +0000 UTC m=+6.297108091, 105.001247ms after stability -2026-04-22T10:09:20.276Z [INFO] server-5949481e-b89e-a129-4e2d-f55f10cdcf93: aborting pipeline replication: peer="{Voter server-e59ba41c-10e0-f573-f2d8-5b04f955dd0c e59ba41c-10e0-f573-f2d8-5b04f955dd0c}" -2026-04-22T10:09:20.276Z [INFO] server-5949481e-b89e-a129-4e2d-f55f10cdcf93: aborting pipeline replication: peer="{Voter server-c1426241-9fa7-6427-cf86-9b86bc9a8872 c1426241-9fa7-6427-cf86-9b86bc9a8872}" -2026-04-22T10:09:20.276Z [INFO] creating new snapshot: path=/tmp/raft3522790038/snapshots/2-27-1776852560276.tmp -2026-04-22T10:09:20.279Z [INFO] server-e59ba41c-10e0-f573-f2d8-5b04f955dd0c: starting restore from snapshot: id=2-27-1776852560276 last-index=27 last-term=2 size-in-bytes=168 -2026-04-22T10:09:20.279Z [INFO] server-e59ba41c-10e0-f573-f2d8-5b04f955dd0c: snapshot restore progress: id=2-27-1776852560276 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% -2026-04-22T10:09:20.279Z [INFO] server-e59ba41c-10e0-f573-f2d8-5b04f955dd0c: restored from snapshot: id=2-27-1776852560276 last-index=27 last-term=2 size-in-bytes=168 -2026-04-22T10:09:20.279Z [INFO] server-e59ba41c-10e0-f573-f2d8-5b04f955dd0c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e59ba41c-10e0-f573-f2d8-5b04f955dd0c Address:e59ba41c-10e0-f573-f2d8-5b04f955dd0c} {Suffrage:Voter ID:server-5949481e-b89e-a129-4e2d-f55f10cdcf93 Address:5949481e-b89e-a129-4e2d-f55f10cdcf93} {Suffrage:Voter ID:server-c1426241-9fa7-6427-cf86-9b86bc9a8872 Address:c1426241-9fa7-6427-cf86-9b86bc9a8872}]" -2026-04-22T10:09:20.279Z [INFO] server-e59ba41c-10e0-f573-f2d8-5b04f955dd0c: entering follower state: follower="Node at e59ba41c-10e0-f573-f2d8-5b04f955dd0c [Follower]" leader-address= leader-id= -2026-04-22T10:09:20.279Z [INFO] creating new snapshot: path=/tmp/raft1465422886/snapshots/2-27-1776852560279.tmp -2026-04-22T10:09:20.279Z [INFO] server-5949481e-b89e-a129-4e2d-f55f10cdcf93: starting restore from snapshot: id=2-27-1776852560279 last-index=27 last-term=2 size-in-bytes=168 -2026-04-22T10:09:20.279Z [INFO] server-5949481e-b89e-a129-4e2d-f55f10cdcf93: snapshot restore progress: id=2-27-1776852560279 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% -2026-04-22T10:09:20.280Z [INFO] server-5949481e-b89e-a129-4e2d-f55f10cdcf93: restored from snapshot: id=2-27-1776852560279 last-index=27 last-term=2 size-in-bytes=168 -2026-04-22T10:09:20.280Z [INFO] server-5949481e-b89e-a129-4e2d-f55f10cdcf93: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e59ba41c-10e0-f573-f2d8-5b04f955dd0c Address:e59ba41c-10e0-f573-f2d8-5b04f955dd0c} {Suffrage:Voter ID:server-5949481e-b89e-a129-4e2d-f55f10cdcf93 Address:5949481e-b89e-a129-4e2d-f55f10cdcf93} {Suffrage:Voter ID:server-c1426241-9fa7-6427-cf86-9b86bc9a8872 Address:c1426241-9fa7-6427-cf86-9b86bc9a8872}]" -2026-04-22T10:09:20.280Z [INFO] server-5949481e-b89e-a129-4e2d-f55f10cdcf93: entering follower state: follower="Node at 5949481e-b89e-a129-4e2d-f55f10cdcf93 [Follower]" leader-address= leader-id= -2026-04-22T10:09:20.280Z [INFO] creating new snapshot: path=/tmp/raft2712672720/snapshots/2-27-1776852560280.tmp -2026-04-22T10:09:20.280Z [INFO] server-c1426241-9fa7-6427-cf86-9b86bc9a8872: starting restore from snapshot: id=2-27-1776852560280 last-index=27 last-term=2 size-in-bytes=168 -2026-04-22T10:09:20.280Z [INFO] server-c1426241-9fa7-6427-cf86-9b86bc9a8872: snapshot restore progress: id=2-27-1776852560280 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% -2026-04-22T10:09:20.280Z [INFO] server-c1426241-9fa7-6427-cf86-9b86bc9a8872: restored from snapshot: id=2-27-1776852560280 last-index=27 last-term=2 size-in-bytes=168 -2026-04-22T10:09:20.280Z [INFO] server-c1426241-9fa7-6427-cf86-9b86bc9a8872: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e59ba41c-10e0-f573-f2d8-5b04f955dd0c Address:e59ba41c-10e0-f573-f2d8-5b04f955dd0c} {Suffrage:Voter ID:server-5949481e-b89e-a129-4e2d-f55f10cdcf93 Address:5949481e-b89e-a129-4e2d-f55f10cdcf93} {Suffrage:Voter ID:server-c1426241-9fa7-6427-cf86-9b86bc9a8872 Address:c1426241-9fa7-6427-cf86-9b86bc9a8872}]" -2026-04-22T10:09:20.280Z [INFO] server-c1426241-9fa7-6427-cf86-9b86bc9a8872: entering follower state: follower="Node at c1426241-9fa7-6427-cf86-9b86bc9a8872 [Follower]" leader-address= leader-id= -2026-04-22T10:09:20.346Z [WARN] server-c1426241-9fa7-6427-cf86-9b86bc9a8872: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:20.346Z [INFO] server-c1426241-9fa7-6427-cf86-9b86bc9a8872: entering candidate state: node="Node at c1426241-9fa7-6427-cf86-9b86bc9a8872 [Candidate]" term=3 -2026-04-22T10:09:20.346Z [INFO] server-c1426241-9fa7-6427-cf86-9b86bc9a8872: election won: term=3 tally=2 -2026-04-22T10:09:20.346Z [INFO] server-c1426241-9fa7-6427-cf86-9b86bc9a8872: entering leader state: leader="Node at c1426241-9fa7-6427-cf86-9b86bc9a8872 [Leader]" -2026-04-22T10:09:20.346Z [INFO] server-c1426241-9fa7-6427-cf86-9b86bc9a8872: added peer, starting replication: peer=server-e59ba41c-10e0-f573-f2d8-5b04f955dd0c -2026-04-22T10:09:20.346Z [INFO] server-c1426241-9fa7-6427-cf86-9b86bc9a8872: added peer, starting replication: peer=server-5949481e-b89e-a129-4e2d-f55f10cdcf93 -2026-04-22T10:09:20.346Z [INFO] server-c1426241-9fa7-6427-cf86-9b86bc9a8872: pipelining replication: peer="{Voter server-5949481e-b89e-a129-4e2d-f55f10cdcf93 5949481e-b89e-a129-4e2d-f55f10cdcf93}" -2026-04-22T10:09:20.347Z [INFO] server-c1426241-9fa7-6427-cf86-9b86bc9a8872: pipelining replication: peer="{Voter server-e59ba41c-10e0-f573-f2d8-5b04f955dd0c e59ba41c-10e0-f573-f2d8-5b04f955dd0c}" -2026-04-22T10:09:20.596Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:20.702Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:20.596346642 +0000 UTC m=+6.620142740 (1 nodes), 32.946µs from start of poll, 484.674109ms from cluster start. Timeout at 2026-04-22 10:09:20.70135993 +0000 UTC m=+6.725156056, 105.013316ms after stability -2026-04-22T10:09:20.702Z [INFO] server-c1426241-9fa7-6427-cf86-9b86bc9a8872: aborting pipeline replication: peer="{Voter server-5949481e-b89e-a129-4e2d-f55f10cdcf93 5949481e-b89e-a129-4e2d-f55f10cdcf93}" -2026-04-22T10:09:20.702Z [INFO] server-c1426241-9fa7-6427-cf86-9b86bc9a8872: aborting pipeline replication: peer="{Voter server-e59ba41c-10e0-f573-f2d8-5b04f955dd0c e59ba41c-10e0-f573-f2d8-5b04f955dd0c}" ---- PASS: TestRaft_RecoverCluster (2.12s) - --- PASS: TestRaft_RecoverCluster/no_snapshot,_no_trailing_logs (0.76s) - --- PASS: TestRaft_RecoverCluster/no_snapshot,_some_trailing_logs (0.77s) - --- PASS: TestRaft_RecoverCluster/snapshot,_with_trailing_logs (0.59s) +2025-03-20T03:49:34.601Z [INFO] server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5 Address:7a2ee38c-ce94-d7d1-eba2-70c14b7985b5} {Suffrage:Voter ID:server-4a209e07-e311-7e4f-74b8-86c18bba4348 Address:4a209e07-e311-7e4f-74b8-86c18bba4348} {Suffrage:Voter ID:server-d62f5133-9c0e-b630-587c-76ac6dbb8151 Address:d62f5133-9c0e-b630-587c-76ac6dbb8151}]" +2025-03-20T03:49:34.601Z [INFO] server-4a209e07-e311-7e4f-74b8-86c18bba4348: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5 Address:7a2ee38c-ce94-d7d1-eba2-70c14b7985b5} {Suffrage:Voter ID:server-4a209e07-e311-7e4f-74b8-86c18bba4348 Address:4a209e07-e311-7e4f-74b8-86c18bba4348} {Suffrage:Voter ID:server-d62f5133-9c0e-b630-587c-76ac6dbb8151 Address:d62f5133-9c0e-b630-587c-76ac6dbb8151}]" +2025-03-20T03:49:34.601Z [INFO] server-d62f5133-9c0e-b630-587c-76ac6dbb8151: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5 Address:7a2ee38c-ce94-d7d1-eba2-70c14b7985b5} {Suffrage:Voter ID:server-4a209e07-e311-7e4f-74b8-86c18bba4348 Address:4a209e07-e311-7e4f-74b8-86c18bba4348} {Suffrage:Voter ID:server-d62f5133-9c0e-b630-587c-76ac6dbb8151 Address:d62f5133-9c0e-b630-587c-76ac6dbb8151}]" +2025-03-20T03:49:34.601Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:34.601Z [INFO] server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5: entering follower state: follower="Node at 7a2ee38c-ce94-d7d1-eba2-70c14b7985b5 [Follower]" leader-address= leader-id= +2025-03-20T03:49:34.601Z [INFO] server-4a209e07-e311-7e4f-74b8-86c18bba4348: entering follower state: follower="Node at 4a209e07-e311-7e4f-74b8-86c18bba4348 [Follower]" leader-address= leader-id= +2025-03-20T03:49:34.601Z [INFO] server-d62f5133-9c0e-b630-587c-76ac6dbb8151: entering follower state: follower="Node at d62f5133-9c0e-b630-587c-76ac6dbb8151 [Follower]" leader-address= leader-id= +2025-03-20T03:49:34.655Z [WARN] server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:34.655Z [INFO] server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5: entering candidate state: node="Node at 7a2ee38c-ce94-d7d1-eba2-70c14b7985b5 [Candidate]" term=2 +2025-03-20T03:49:34.656Z [INFO] server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5: election won: term=2 tally=2 +2025-03-20T03:49:34.656Z [INFO] server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5: entering leader state: leader="Node at 7a2ee38c-ce94-d7d1-eba2-70c14b7985b5 [Leader]" +2025-03-20T03:49:34.656Z [INFO] server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5: added peer, starting replication: peer=server-4a209e07-e311-7e4f-74b8-86c18bba4348 +2025-03-20T03:49:34.656Z [INFO] server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5: added peer, starting replication: peer=server-d62f5133-9c0e-b630-587c-76ac6dbb8151 +2025-03-20T03:49:34.656Z [INFO] server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5: pipelining replication: peer="{Voter server-d62f5133-9c0e-b630-587c-76ac6dbb8151 d62f5133-9c0e-b630-587c-76ac6dbb8151}" +2025-03-20T03:49:34.656Z [INFO] server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5: pipelining replication: peer="{Voter server-4a209e07-e311-7e4f-74b8-86c18bba4348 4a209e07-e311-7e4f-74b8-86c18bba4348}" +2025-03-20T03:49:34.762Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:34.657088204 +0000 UTC m=+6.331541033 (1 nodes), 55.384741ms from start of poll, 55.89364ms from cluster start. Timeout at 2025-03-20 03:49:34.762091293 +0000 UTC m=+6.436544365, 105.003332ms after stability +2025-03-20T03:49:34.775Z [INFO] server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5: aborting pipeline replication: peer="{Voter server-4a209e07-e311-7e4f-74b8-86c18bba4348 4a209e07-e311-7e4f-74b8-86c18bba4348}" +2025-03-20T03:49:34.775Z [INFO] server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5: aborting pipeline replication: peer="{Voter server-d62f5133-9c0e-b630-587c-76ac6dbb8151 d62f5133-9c0e-b630-587c-76ac6dbb8151}" +2025-03-20T03:49:34.775Z [INFO] creating new snapshot: path=/tmp/raft2676022995/snapshots/2-27-1742442574775.tmp +2025-03-20T03:49:34.778Z [INFO] server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5: starting restore from snapshot: id=2-27-1742442574775 last-index=27 last-term=2 size-in-bytes=168 +2025-03-20T03:49:34.779Z [INFO] server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5: snapshot restore progress: id=2-27-1742442574775 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% +2025-03-20T03:49:34.779Z [INFO] server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5: restored from snapshot: id=2-27-1742442574775 last-index=27 last-term=2 size-in-bytes=168 +2025-03-20T03:49:34.779Z [INFO] server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5 Address:7a2ee38c-ce94-d7d1-eba2-70c14b7985b5} {Suffrage:Voter ID:server-4a209e07-e311-7e4f-74b8-86c18bba4348 Address:4a209e07-e311-7e4f-74b8-86c18bba4348} {Suffrage:Voter ID:server-d62f5133-9c0e-b630-587c-76ac6dbb8151 Address:d62f5133-9c0e-b630-587c-76ac6dbb8151}]" +2025-03-20T03:49:34.779Z [INFO] creating new snapshot: path=/tmp/raft3379389949/snapshots/2-27-1742442574779.tmp +2025-03-20T03:49:34.780Z [INFO] server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5: entering follower state: follower="Node at 7a2ee38c-ce94-d7d1-eba2-70c14b7985b5 [Follower]" leader-address= leader-id= +2025-03-20T03:49:34.781Z [INFO] server-4a209e07-e311-7e4f-74b8-86c18bba4348: starting restore from snapshot: id=2-27-1742442574779 last-index=27 last-term=2 size-in-bytes=168 +2025-03-20T03:49:34.781Z [INFO] server-4a209e07-e311-7e4f-74b8-86c18bba4348: snapshot restore progress: id=2-27-1742442574779 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% +2025-03-20T03:49:34.781Z [INFO] server-4a209e07-e311-7e4f-74b8-86c18bba4348: restored from snapshot: id=2-27-1742442574779 last-index=27 last-term=2 size-in-bytes=168 +2025-03-20T03:49:34.781Z [INFO] server-4a209e07-e311-7e4f-74b8-86c18bba4348: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5 Address:7a2ee38c-ce94-d7d1-eba2-70c14b7985b5} {Suffrage:Voter ID:server-4a209e07-e311-7e4f-74b8-86c18bba4348 Address:4a209e07-e311-7e4f-74b8-86c18bba4348} {Suffrage:Voter ID:server-d62f5133-9c0e-b630-587c-76ac6dbb8151 Address:d62f5133-9c0e-b630-587c-76ac6dbb8151}]" +2025-03-20T03:49:34.781Z [INFO] creating new snapshot: path=/tmp/raft2094700744/snapshots/2-19-1742442574781.tmp +2025-03-20T03:49:34.782Z [INFO] server-4a209e07-e311-7e4f-74b8-86c18bba4348: entering follower state: follower="Node at 4a209e07-e311-7e4f-74b8-86c18bba4348 [Follower]" leader-address= leader-id= +2025-03-20T03:49:34.783Z [INFO] server-d62f5133-9c0e-b630-587c-76ac6dbb8151: starting restore from snapshot: id=2-19-1742442574781 last-index=19 last-term=2 size-in-bytes=112 +2025-03-20T03:49:34.783Z [INFO] server-d62f5133-9c0e-b630-587c-76ac6dbb8151: snapshot restore progress: id=2-19-1742442574781 last-index=19 last-term=2 size-in-bytes=112 read-bytes=112 percent-complete=100.00% +2025-03-20T03:49:34.783Z [INFO] server-d62f5133-9c0e-b630-587c-76ac6dbb8151: restored from snapshot: id=2-19-1742442574781 last-index=19 last-term=2 size-in-bytes=112 +2025-03-20T03:49:34.783Z [INFO] server-d62f5133-9c0e-b630-587c-76ac6dbb8151: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5 Address:7a2ee38c-ce94-d7d1-eba2-70c14b7985b5} {Suffrage:Voter ID:server-4a209e07-e311-7e4f-74b8-86c18bba4348 Address:4a209e07-e311-7e4f-74b8-86c18bba4348} {Suffrage:Voter ID:server-d62f5133-9c0e-b630-587c-76ac6dbb8151 Address:d62f5133-9c0e-b630-587c-76ac6dbb8151}]" +2025-03-20T03:49:34.783Z [INFO] server-d62f5133-9c0e-b630-587c-76ac6dbb8151: entering follower state: follower="Node at d62f5133-9c0e-b630-587c-76ac6dbb8151 [Follower]" leader-address= leader-id= +2025-03-20T03:49:34.832Z [WARN] server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:34.832Z [INFO] server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5: entering candidate state: node="Node at 7a2ee38c-ce94-d7d1-eba2-70c14b7985b5 [Candidate]" term=3 +2025-03-20T03:49:34.832Z [INFO] server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5: election won: term=3 tally=2 +2025-03-20T03:49:34.832Z [INFO] server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5: entering leader state: leader="Node at 7a2ee38c-ce94-d7d1-eba2-70c14b7985b5 [Leader]" +2025-03-20T03:49:34.832Z [INFO] server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5: added peer, starting replication: peer=server-4a209e07-e311-7e4f-74b8-86c18bba4348 +2025-03-20T03:49:34.833Z [INFO] server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5: added peer, starting replication: peer=server-d62f5133-9c0e-b630-587c-76ac6dbb8151 +2025-03-20T03:49:34.833Z [WARN] server-d62f5133-9c0e-b630-587c-76ac6dbb8151: failed to get previous log: previous-index=27 last-index=19 error="log not found" +2025-03-20T03:49:34.833Z [WARN] server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5: appendEntries rejected, sending older logs: peer="{Voter server-d62f5133-9c0e-b630-587c-76ac6dbb8151 d62f5133-9c0e-b630-587c-76ac6dbb8151}" next=20 +2025-03-20T03:49:34.833Z [ERROR] server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5: failed to get log: index=19 error="log not found" +2025-03-20T03:49:34.833Z [INFO] creating new snapshot: path=/tmp/raft2094700744/snapshots/2-27-1742442574833.tmp +2025-03-20T03:49:34.834Z [INFO] server-d62f5133-9c0e-b630-587c-76ac6dbb8151: snapshot network transfer progress: read-bytes=168 percent-complete=100.00% +2025-03-20T03:49:34.834Z [INFO] server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5: pipelining replication: peer="{Voter server-4a209e07-e311-7e4f-74b8-86c18bba4348 4a209e07-e311-7e4f-74b8-86c18bba4348}" +2025-03-20T03:49:34.870Z [INFO] server-d62f5133-9c0e-b630-587c-76ac6dbb8151: copied to local snapshot: bytes=168 +2025-03-20T03:49:34.871Z [INFO] server-d62f5133-9c0e-b630-587c-76ac6dbb8151: snapshot restore progress: id=2-27-1742442574833 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% +2025-03-20T03:49:34.871Z [INFO] server-d62f5133-9c0e-b630-587c-76ac6dbb8151: Installed remote snapshot +2025-03-20T03:49:34.871Z [INFO] server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5: pipelining replication: peer="{Voter server-d62f5133-9c0e-b630-587c-76ac6dbb8151 d62f5133-9c0e-b630-587c-76ac6dbb8151}" +2025-03-20T03:49:35.099Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:35.204Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:35.099804787 +0000 UTC m=+6.774257755 (1 nodes), 77.847µs from start of poll, 498.610362ms from cluster start. Timeout at 2025-03-20 03:49:35.204822742 +0000 UTC m=+6.879275726, 105.017971ms after stability +2025-03-20T03:49:35.205Z [INFO] server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5: aborting pipeline replication: peer="{Voter server-d62f5133-9c0e-b630-587c-76ac6dbb8151 d62f5133-9c0e-b630-587c-76ac6dbb8151}" +2025-03-20T03:49:35.205Z [INFO] server-7a2ee38c-ce94-d7d1-eba2-70c14b7985b5: aborting pipeline replication: peer="{Voter server-4a209e07-e311-7e4f-74b8-86c18bba4348 4a209e07-e311-7e4f-74b8-86c18bba4348}" +--- PASS: TestRaft_RecoverCluster (2.31s) + --- PASS: TestRaft_RecoverCluster/no_snapshot,_no_trailing_logs (0.59s) + --- PASS: TestRaft_RecoverCluster/no_snapshot,_some_trailing_logs (1.11s) + --- PASS: TestRaft_RecoverCluster/snapshot,_with_trailing_logs (0.61s) === RUN TestRaft_HasExistingState raft_test.go:258: Skipping unreliable test --- SKIP: TestRaft_HasExistingState (0.00s) === RUN TestRaft_SingleNode -2026-04-22T10:09:20.703Z [INFO] server-281618b9-9da6-85b8-d452-e93bbad90f32: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-281618b9-9da6-85b8-d452-e93bbad90f32 Address:281618b9-9da6-85b8-d452-e93bbad90f32}]" -2026-04-22T10:09:20.703Z [INFO] server-281618b9-9da6-85b8-d452-e93bbad90f32: entering follower state: follower="Node at 281618b9-9da6-85b8-d452-e93bbad90f32 [Follower]" leader-address= leader-id= -2026-04-22T10:09:20.800Z [WARN] server-281618b9-9da6-85b8-d452-e93bbad90f32: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:20.800Z [INFO] server-281618b9-9da6-85b8-d452-e93bbad90f32: entering candidate state: node="Node at 281618b9-9da6-85b8-d452-e93bbad90f32 [Candidate]" term=2 -2026-04-22T10:09:20.800Z [INFO] server-281618b9-9da6-85b8-d452-e93bbad90f32: election won: term=2 tally=1 -2026-04-22T10:09:20.800Z [INFO] server-281618b9-9da6-85b8-d452-e93bbad90f32: entering leader state: leader="Node at 281618b9-9da6-85b8-d452-e93bbad90f32 [Leader]" ---- PASS: TestRaft_SingleNode (0.10s) +2025-03-20T03:49:35.207Z [INFO] server-a6fc59ab-d8f0-7b05-b7fd-b7d9c13a0f1e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a6fc59ab-d8f0-7b05-b7fd-b7d9c13a0f1e Address:a6fc59ab-d8f0-7b05-b7fd-b7d9c13a0f1e}]" +2025-03-20T03:49:35.207Z [INFO] server-a6fc59ab-d8f0-7b05-b7fd-b7d9c13a0f1e: entering follower state: follower="Node at a6fc59ab-d8f0-7b05-b7fd-b7d9c13a0f1e [Follower]" leader-address= leader-id= +2025-03-20T03:49:35.291Z [WARN] server-a6fc59ab-d8f0-7b05-b7fd-b7d9c13a0f1e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:35.291Z [INFO] server-a6fc59ab-d8f0-7b05-b7fd-b7d9c13a0f1e: entering candidate state: node="Node at a6fc59ab-d8f0-7b05-b7fd-b7d9c13a0f1e [Candidate]" term=2 +2025-03-20T03:49:35.291Z [INFO] server-a6fc59ab-d8f0-7b05-b7fd-b7d9c13a0f1e: election won: term=2 tally=1 +2025-03-20T03:49:35.291Z [INFO] server-a6fc59ab-d8f0-7b05-b7fd-b7d9c13a0f1e: entering leader state: leader="Node at a6fc59ab-d8f0-7b05-b7fd-b7d9c13a0f1e [Leader]" +--- PASS: TestRaft_SingleNode (0.08s) === RUN TestRaft_TripleNode -2026-04-22T10:09:20.801Z [INFO] server-f87fb15b-f73c-1cd3-a8df-6696bb292d87: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f87fb15b-f73c-1cd3-a8df-6696bb292d87 Address:f87fb15b-f73c-1cd3-a8df-6696bb292d87} {Suffrage:Voter ID:server-560b2ce4-268a-bb85-555f-a2efcf570087 Address:560b2ce4-268a-bb85-555f-a2efcf570087} {Suffrage:Voter ID:server-4da96110-e121-eeb4-83eb-7c0e9fa7d6be Address:4da96110-e121-eeb4-83eb-7c0e9fa7d6be}]" -2026-04-22T10:09:20.801Z [INFO] server-f87fb15b-f73c-1cd3-a8df-6696bb292d87: entering follower state: follower="Node at f87fb15b-f73c-1cd3-a8df-6696bb292d87 [Follower]" leader-address= leader-id= -2026-04-22T10:09:20.801Z [INFO] server-560b2ce4-268a-bb85-555f-a2efcf570087: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f87fb15b-f73c-1cd3-a8df-6696bb292d87 Address:f87fb15b-f73c-1cd3-a8df-6696bb292d87} {Suffrage:Voter ID:server-560b2ce4-268a-bb85-555f-a2efcf570087 Address:560b2ce4-268a-bb85-555f-a2efcf570087} {Suffrage:Voter ID:server-4da96110-e121-eeb4-83eb-7c0e9fa7d6be Address:4da96110-e121-eeb4-83eb-7c0e9fa7d6be}]" -2026-04-22T10:09:20.801Z [INFO] server-560b2ce4-268a-bb85-555f-a2efcf570087: entering follower state: follower="Node at 560b2ce4-268a-bb85-555f-a2efcf570087 [Follower]" leader-address= leader-id= -2026-04-22T10:09:20.801Z [INFO] server-4da96110-e121-eeb4-83eb-7c0e9fa7d6be: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f87fb15b-f73c-1cd3-a8df-6696bb292d87 Address:f87fb15b-f73c-1cd3-a8df-6696bb292d87} {Suffrage:Voter ID:server-560b2ce4-268a-bb85-555f-a2efcf570087 Address:560b2ce4-268a-bb85-555f-a2efcf570087} {Suffrage:Voter ID:server-4da96110-e121-eeb4-83eb-7c0e9fa7d6be Address:4da96110-e121-eeb4-83eb-7c0e9fa7d6be}]" -2026-04-22T10:09:20.801Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:09:20.801Z [INFO] server-4da96110-e121-eeb4-83eb-7c0e9fa7d6be: entering follower state: follower="Node at 4da96110-e121-eeb4-83eb-7c0e9fa7d6be [Follower]" leader-address= leader-id= -2026-04-22T10:09:20.859Z [WARN] server-4da96110-e121-eeb4-83eb-7c0e9fa7d6be: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:20.859Z [INFO] server-4da96110-e121-eeb4-83eb-7c0e9fa7d6be: entering candidate state: node="Node at 4da96110-e121-eeb4-83eb-7c0e9fa7d6be [Candidate]" term=2 -2026-04-22T10:09:20.859Z [INFO] server-4da96110-e121-eeb4-83eb-7c0e9fa7d6be: election won: term=2 tally=2 -2026-04-22T10:09:20.859Z [INFO] server-4da96110-e121-eeb4-83eb-7c0e9fa7d6be: entering leader state: leader="Node at 4da96110-e121-eeb4-83eb-7c0e9fa7d6be [Leader]" -2026-04-22T10:09:20.859Z [INFO] server-4da96110-e121-eeb4-83eb-7c0e9fa7d6be: added peer, starting replication: peer=server-f87fb15b-f73c-1cd3-a8df-6696bb292d87 -2026-04-22T10:09:20.859Z [INFO] server-4da96110-e121-eeb4-83eb-7c0e9fa7d6be: added peer, starting replication: peer=server-560b2ce4-268a-bb85-555f-a2efcf570087 -2026-04-22T10:09:20.859Z [INFO] server-4da96110-e121-eeb4-83eb-7c0e9fa7d6be: pipelining replication: peer="{Voter server-560b2ce4-268a-bb85-555f-a2efcf570087 560b2ce4-268a-bb85-555f-a2efcf570087}" -2026-04-22T10:09:20.859Z [INFO] server-4da96110-e121-eeb4-83eb-7c0e9fa7d6be: pipelining replication: peer="{Voter server-f87fb15b-f73c-1cd3-a8df-6696bb292d87 f87fb15b-f73c-1cd3-a8df-6696bb292d87}" -2026-04-22T10:09:20.965Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:09:20.859496678 +0000 UTC m=+6.883292790 (2 nodes), 58.160182ms from start of poll, 58.520757ms from cluster start. Timeout at 2026-04-22 10:09:20.964499852 +0000 UTC m=+6.988295935, 105.003145ms after stability -2026-04-22T10:09:20.965Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:21.070Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:20.965231826 +0000 UTC m=+6.989027920 (1 nodes), 786ns from start of poll, 164.255887ms from cluster start. Timeout at 2026-04-22 10:09:21.070236887 +0000 UTC m=+7.094032969, 105.005049ms after stability -2026-04-22T10:09:21.081Z [INFO] server-4da96110-e121-eeb4-83eb-7c0e9fa7d6be: aborting pipeline replication: peer="{Voter server-560b2ce4-268a-bb85-555f-a2efcf570087 560b2ce4-268a-bb85-555f-a2efcf570087}" -2026-04-22T10:09:21.081Z [INFO] server-4da96110-e121-eeb4-83eb-7c0e9fa7d6be: aborting pipeline replication: peer="{Voter server-f87fb15b-f73c-1cd3-a8df-6696bb292d87 f87fb15b-f73c-1cd3-a8df-6696bb292d87}" ---- PASS: TestRaft_TripleNode (0.28s) +2025-03-20T03:49:35.293Z [INFO] server-d2eb01fd-3f83-8fb2-8f3b-e5a34c7120a2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d2eb01fd-3f83-8fb2-8f3b-e5a34c7120a2 Address:d2eb01fd-3f83-8fb2-8f3b-e5a34c7120a2} {Suffrage:Voter ID:server-32044c4d-1063-d863-c973-f8ea27565855 Address:32044c4d-1063-d863-c973-f8ea27565855} {Suffrage:Voter ID:server-c777deb1-af0a-f06b-05c5-93047f915b5d Address:c777deb1-af0a-f06b-05c5-93047f915b5d}]" +2025-03-20T03:49:35.293Z [INFO] server-32044c4d-1063-d863-c973-f8ea27565855: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d2eb01fd-3f83-8fb2-8f3b-e5a34c7120a2 Address:d2eb01fd-3f83-8fb2-8f3b-e5a34c7120a2} {Suffrage:Voter ID:server-32044c4d-1063-d863-c973-f8ea27565855 Address:32044c4d-1063-d863-c973-f8ea27565855} {Suffrage:Voter ID:server-c777deb1-af0a-f06b-05c5-93047f915b5d Address:c777deb1-af0a-f06b-05c5-93047f915b5d}]" +2025-03-20T03:49:35.293Z [INFO] server-c777deb1-af0a-f06b-05c5-93047f915b5d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d2eb01fd-3f83-8fb2-8f3b-e5a34c7120a2 Address:d2eb01fd-3f83-8fb2-8f3b-e5a34c7120a2} {Suffrage:Voter ID:server-32044c4d-1063-d863-c973-f8ea27565855 Address:32044c4d-1063-d863-c973-f8ea27565855} {Suffrage:Voter ID:server-c777deb1-af0a-f06b-05c5-93047f915b5d Address:c777deb1-af0a-f06b-05c5-93047f915b5d}]" +2025-03-20T03:49:35.293Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:49:35.293Z [INFO] server-d2eb01fd-3f83-8fb2-8f3b-e5a34c7120a2: entering follower state: follower="Node at d2eb01fd-3f83-8fb2-8f3b-e5a34c7120a2 [Follower]" leader-address= leader-id= +2025-03-20T03:49:35.293Z [INFO] server-32044c4d-1063-d863-c973-f8ea27565855: entering follower state: follower="Node at 32044c4d-1063-d863-c973-f8ea27565855 [Follower]" leader-address= leader-id= +2025-03-20T03:49:35.293Z [INFO] server-c777deb1-af0a-f06b-05c5-93047f915b5d: entering follower state: follower="Node at c777deb1-af0a-f06b-05c5-93047f915b5d [Follower]" leader-address= leader-id= +2025-03-20T03:49:35.362Z [WARN] server-c777deb1-af0a-f06b-05c5-93047f915b5d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:35.363Z [INFO] server-c777deb1-af0a-f06b-05c5-93047f915b5d: entering candidate state: node="Node at c777deb1-af0a-f06b-05c5-93047f915b5d [Candidate]" term=2 +2025-03-20T03:49:35.363Z [INFO] server-c777deb1-af0a-f06b-05c5-93047f915b5d: election won: term=2 tally=2 +2025-03-20T03:49:35.363Z [INFO] server-c777deb1-af0a-f06b-05c5-93047f915b5d: entering leader state: leader="Node at c777deb1-af0a-f06b-05c5-93047f915b5d [Leader]" +2025-03-20T03:49:35.363Z [INFO] server-c777deb1-af0a-f06b-05c5-93047f915b5d: added peer, starting replication: peer=server-d2eb01fd-3f83-8fb2-8f3b-e5a34c7120a2 +2025-03-20T03:49:35.363Z [INFO] server-c777deb1-af0a-f06b-05c5-93047f915b5d: added peer, starting replication: peer=server-32044c4d-1063-d863-c973-f8ea27565855 +2025-03-20T03:49:35.363Z [INFO] server-c777deb1-af0a-f06b-05c5-93047f915b5d: pipelining replication: peer="{Voter server-32044c4d-1063-d863-c973-f8ea27565855 32044c4d-1063-d863-c973-f8ea27565855}" +2025-03-20T03:49:35.363Z [INFO] server-c777deb1-af0a-f06b-05c5-93047f915b5d: pipelining replication: peer="{Voter server-d2eb01fd-3f83-8fb2-8f3b-e5a34c7120a2 d2eb01fd-3f83-8fb2-8f3b-e5a34c7120a2}" +2025-03-20T03:49:35.470Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:49:35.363864189 +0000 UTC m=+7.038317070 (2 nodes), 70.495979ms from start of poll, 70.923592ms from cluster start. Timeout at 2025-03-20 03:49:35.468871929 +0000 UTC m=+7.143324827, 105.007757ms after stability +2025-03-20T03:49:35.470Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:35.582Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:35.470218133 +0000 UTC m=+7.144670987 (1 nodes), 3.912µs from start of poll, 177.277509ms from cluster start. Timeout at 2025-03-20 03:49:35.575246466 +0000 UTC m=+7.249699682, 105.028695ms after stability +2025-03-20T03:49:35.594Z [INFO] server-c777deb1-af0a-f06b-05c5-93047f915b5d: aborting pipeline replication: peer="{Voter server-32044c4d-1063-d863-c973-f8ea27565855 32044c4d-1063-d863-c973-f8ea27565855}" +2025-03-20T03:49:35.594Z [INFO] server-c777deb1-af0a-f06b-05c5-93047f915b5d: aborting pipeline replication: peer="{Voter server-d2eb01fd-3f83-8fb2-8f3b-e5a34c7120a2 d2eb01fd-3f83-8fb2-8f3b-e5a34c7120a2}" +--- PASS: TestRaft_TripleNode (0.30s) === RUN TestRaft_LeaderFail -2026-04-22T10:09:21.082Z [INFO] server-35baf66f-8e1d-2cae-d0c0-68d47521f922: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-35baf66f-8e1d-2cae-d0c0-68d47521f922 Address:35baf66f-8e1d-2cae-d0c0-68d47521f922} {Suffrage:Voter ID:server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a Address:f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a} {Suffrage:Voter ID:server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b Address:fb6044d1-2b30-69f3-0ac3-bf2f201eb89b}]" -2026-04-22T10:09:21.082Z [INFO] server-35baf66f-8e1d-2cae-d0c0-68d47521f922: entering follower state: follower="Node at 35baf66f-8e1d-2cae-d0c0-68d47521f922 [Follower]" leader-address= leader-id= -2026-04-22T10:09:21.082Z [INFO] server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-35baf66f-8e1d-2cae-d0c0-68d47521f922 Address:35baf66f-8e1d-2cae-d0c0-68d47521f922} {Suffrage:Voter ID:server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a Address:f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a} {Suffrage:Voter ID:server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b Address:fb6044d1-2b30-69f3-0ac3-bf2f201eb89b}]" -2026-04-22T10:09:21.082Z [INFO] server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a: entering follower state: follower="Node at f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a [Follower]" leader-address= leader-id= -2026-04-22T10:09:21.082Z [INFO] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-35baf66f-8e1d-2cae-d0c0-68d47521f922 Address:35baf66f-8e1d-2cae-d0c0-68d47521f922} {Suffrage:Voter ID:server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a Address:f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a} {Suffrage:Voter ID:server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b Address:fb6044d1-2b30-69f3-0ac3-bf2f201eb89b}]" -2026-04-22T10:09:21.082Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:09:21.082Z [INFO] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: entering follower state: follower="Node at fb6044d1-2b30-69f3-0ac3-bf2f201eb89b [Follower]" leader-address= leader-id= -2026-04-22T10:09:21.134Z [WARN] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:21.135Z [INFO] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: entering candidate state: node="Node at fb6044d1-2b30-69f3-0ac3-bf2f201eb89b [Candidate]" term=2 -2026-04-22T10:09:21.135Z [INFO] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: election won: term=2 tally=2 -2026-04-22T10:09:21.135Z [INFO] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: entering leader state: leader="Node at fb6044d1-2b30-69f3-0ac3-bf2f201eb89b [Leader]" -2026-04-22T10:09:21.135Z [INFO] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: added peer, starting replication: peer=server-35baf66f-8e1d-2cae-d0c0-68d47521f922 -2026-04-22T10:09:21.135Z [INFO] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: added peer, starting replication: peer=server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a -2026-04-22T10:09:21.135Z [INFO] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: pipelining replication: peer="{Voter server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a}" -2026-04-22T10:09:21.135Z [INFO] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: pipelining replication: peer="{Voter server-35baf66f-8e1d-2cae-d0c0-68d47521f922 35baf66f-8e1d-2cae-d0c0-68d47521f922}" -2026-04-22T10:09:21.241Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:09:21.135506093 +0000 UTC m=+7.159302218 (2 nodes), 52.837321ms from start of poll, 53.182234ms from cluster start. Timeout at 2026-04-22 10:09:21.240511907 +0000 UTC m=+7.264308022, 105.005804ms after stability -2026-04-22T10:09:21.241Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:21.347Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:21.241320092 +0000 UTC m=+7.265116200 (1 nodes), 939ns from start of poll, 158.996216ms from cluster start. Timeout at 2026-04-22 10:09:21.346329597 +0000 UTC m=+7.370125703, 105.009503ms after stability - raft_test.go:378: [INFO] Disconnecting Node at fb6044d1-2b30-69f3-0ac3-bf2f201eb89b [Leader] -2026-04-22T10:09:21.364Z [ERROR] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: failed to pipeline appendEntries: peer="{Voter server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a}" error="append pipeline closed" -2026-04-22T10:09:21.364Z [INFO] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: aborting pipeline replication: peer="{Voter server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a}" -2026-04-22T10:09:21.365Z [ERROR] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: failed to heartbeat to: peer=f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a backoff time=10ms error="failed to connect to peer: f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a" -2026-04-22T10:09:21.365Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:21.366Z [ERROR] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: failed to heartbeat to: peer=35baf66f-8e1d-2cae-d0c0-68d47521f922 backoff time=10ms error="failed to connect to peer: 35baf66f-8e1d-2cae-d0c0-68d47521f922" -2026-04-22T10:09:21.370Z [ERROR] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: failed to pipeline appendEntries: peer="{Voter server-35baf66f-8e1d-2cae-d0c0-68d47521f922 35baf66f-8e1d-2cae-d0c0-68d47521f922}" error="append pipeline closed" -2026-04-22T10:09:21.370Z [INFO] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: aborting pipeline replication: peer="{Voter server-35baf66f-8e1d-2cae-d0c0-68d47521f922 35baf66f-8e1d-2cae-d0c0-68d47521f922}" -2026-04-22T10:09:21.373Z [ERROR] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: failed to appendEntries to: peer="{Voter server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a}" error="failed to connect to peer: f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a" -2026-04-22T10:09:21.377Z [ERROR] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: failed to appendEntries to: peer="{Voter server-35baf66f-8e1d-2cae-d0c0-68d47521f922 35baf66f-8e1d-2cae-d0c0-68d47521f922}" error="failed to connect to peer: 35baf66f-8e1d-2cae-d0c0-68d47521f922" -2026-04-22T10:09:21.384Z [ERROR] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: failed to heartbeat to: peer=f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a backoff time=10ms error="failed to connect to peer: f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a" -2026-04-22T10:09:21.386Z [ERROR] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: failed to heartbeat to: peer=35baf66f-8e1d-2cae-d0c0-68d47521f922 backoff time=10ms error="failed to connect to peer: 35baf66f-8e1d-2cae-d0c0-68d47521f922" -2026-04-22T10:09:21.394Z [ERROR] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: failed to appendEntries to: peer="{Voter server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a}" error="failed to connect to peer: f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a" -2026-04-22T10:09:21.397Z [ERROR] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: failed to appendEntries to: peer="{Voter server-35baf66f-8e1d-2cae-d0c0-68d47521f922 35baf66f-8e1d-2cae-d0c0-68d47521f922}" error="failed to connect to peer: 35baf66f-8e1d-2cae-d0c0-68d47521f922" -2026-04-22T10:09:21.402Z [ERROR] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: failed to heartbeat to: peer=35baf66f-8e1d-2cae-d0c0-68d47521f922 backoff time=10ms error="failed to connect to peer: 35baf66f-8e1d-2cae-d0c0-68d47521f922" -2026-04-22T10:09:21.405Z [ERROR] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: failed to heartbeat to: peer=f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a backoff time=10ms error="failed to connect to peer: f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a" -2026-04-22T10:09:21.408Z [WARN] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: failed to contact: server-id=server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a time=50.356083ms -2026-04-22T10:09:21.415Z [ERROR] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: failed to appendEntries to: peer="{Voter server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a}" error="failed to connect to peer: f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a" -2026-04-22T10:09:21.418Z [ERROR] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: failed to appendEntries to: peer="{Voter server-35baf66f-8e1d-2cae-d0c0-68d47521f922 35baf66f-8e1d-2cae-d0c0-68d47521f922}" error="failed to connect to peer: 35baf66f-8e1d-2cae-d0c0-68d47521f922" -2026-04-22T10:09:21.419Z [WARN] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: failed to contact: server-id=server-35baf66f-8e1d-2cae-d0c0-68d47521f922 time=58.175808ms -2026-04-22T10:09:21.419Z [WARN] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: failed to contact: server-id=server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a time=61.406952ms -2026-04-22T10:09:21.419Z [WARN] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: failed to contact quorum of nodes, stepping down -2026-04-22T10:09:21.419Z [INFO] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: entering follower state: follower="Node at fb6044d1-2b30-69f3-0ac3-bf2f201eb89b [Follower]" leader-address= leader-id= -2026-04-22T10:09:21.449Z [WARN] server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a: heartbeat timeout reached, starting election: last-leader-addr=fb6044d1-2b30-69f3-0ac3-bf2f201eb89b last-leader-id=server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b -2026-04-22T10:09:21.449Z [INFO] server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a: entering candidate state: node="Node at f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a [Candidate]" term=3 -2026-04-22T10:09:21.449Z [ERROR] server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a: failed to make requestVote RPC: target="{Voter server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b fb6044d1-2b30-69f3-0ac3-bf2f201eb89b}" error="failed to connect to peer: fb6044d1-2b30-69f3-0ac3-bf2f201eb89b" term=3 -2026-04-22T10:09:21.449Z [WARN] server-35baf66f-8e1d-2cae-d0c0-68d47521f922: rejecting vote request since we have a leader: from=f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a leader=fb6044d1-2b30-69f3-0ac3-bf2f201eb89b leader-id=server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b -2026-04-22T10:09:21.469Z [WARN] server-35baf66f-8e1d-2cae-d0c0-68d47521f922: heartbeat timeout reached, starting election: last-leader-addr=fb6044d1-2b30-69f3-0ac3-bf2f201eb89b last-leader-id=server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b -2026-04-22T10:09:21.469Z [INFO] server-35baf66f-8e1d-2cae-d0c0-68d47521f922: entering candidate state: node="Node at 35baf66f-8e1d-2cae-d0c0-68d47521f922 [Candidate]" term=3 -2026-04-22T10:09:21.469Z [ERROR] server-35baf66f-8e1d-2cae-d0c0-68d47521f922: failed to make requestVote RPC: target="{Voter server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b fb6044d1-2b30-69f3-0ac3-bf2f201eb89b}" error="failed to connect to peer: fb6044d1-2b30-69f3-0ac3-bf2f201eb89b" term=3 -2026-04-22T10:09:21.469Z [INFO] server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a: duplicate requestVote for same term: term=3 -2026-04-22T10:09:21.511Z [WARN] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:21.511Z [INFO] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: entering candidate state: node="Node at fb6044d1-2b30-69f3-0ac3-bf2f201eb89b [Candidate]" term=3 -2026-04-22T10:09:21.511Z [ERROR] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: failed to make requestVote RPC: target="{Voter server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a}" error="failed to connect to peer: f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a" term=3 -2026-04-22T10:09:21.511Z [ERROR] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: failed to make requestVote RPC: target="{Voter server-35baf66f-8e1d-2cae-d0c0-68d47521f922 35baf66f-8e1d-2cae-d0c0-68d47521f922}" error="failed to connect to peer: 35baf66f-8e1d-2cae-d0c0-68d47521f922" term=3 -2026-04-22T10:09:21.516Z [WARN] server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a: Election timeout reached, restarting election -2026-04-22T10:09:21.516Z [INFO] server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a: entering candidate state: node="Node at f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a [Candidate]" term=4 -2026-04-22T10:09:21.516Z [ERROR] server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a: failed to make requestVote RPC: target="{Voter server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b fb6044d1-2b30-69f3-0ac3-bf2f201eb89b}" error="failed to connect to peer: fb6044d1-2b30-69f3-0ac3-bf2f201eb89b" term=4 -2026-04-22T10:09:21.516Z [INFO] server-35baf66f-8e1d-2cae-d0c0-68d47521f922: entering follower state: follower="Node at 35baf66f-8e1d-2cae-d0c0-68d47521f922 [Follower]" leader-address= leader-id= -2026-04-22T10:09:21.516Z [INFO] server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a: election won: term=4 tally=2 -2026-04-22T10:09:21.516Z [INFO] server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a: entering leader state: leader="Node at f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a [Leader]" -2026-04-22T10:09:21.516Z [INFO] server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a: added peer, starting replication: peer=server-35baf66f-8e1d-2cae-d0c0-68d47521f922 -2026-04-22T10:09:21.516Z [INFO] server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a: added peer, starting replication: peer=server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b -2026-04-22T10:09:21.516Z [ERROR] server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a: failed to appendEntries to: peer="{Voter server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b fb6044d1-2b30-69f3-0ac3-bf2f201eb89b}" error="failed to connect to peer: fb6044d1-2b30-69f3-0ac3-bf2f201eb89b" -2026-04-22T10:09:21.516Z [INFO] server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a: pipelining replication: peer="{Voter server-35baf66f-8e1d-2cae-d0c0-68d47521f922 35baf66f-8e1d-2cae-d0c0-68d47521f922}" -2026-04-22T10:09:21.526Z [ERROR] server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a: failed to heartbeat to: peer=fb6044d1-2b30-69f3-0ac3-bf2f201eb89b backoff time=10ms error="failed to connect to peer: fb6044d1-2b30-69f3-0ac3-bf2f201eb89b" -2026-04-22T10:09:21.534Z [ERROR] server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a: failed to appendEntries to: peer="{Voter server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b fb6044d1-2b30-69f3-0ac3-bf2f201eb89b}" error="failed to connect to peer: fb6044d1-2b30-69f3-0ac3-bf2f201eb89b" -2026-04-22T10:09:21.547Z [ERROR] server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a: failed to heartbeat to: peer=fb6044d1-2b30-69f3-0ac3-bf2f201eb89b backoff time=10ms error="failed to connect to peer: fb6044d1-2b30-69f3-0ac3-bf2f201eb89b" -2026-04-22T10:09:21.553Z [ERROR] server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a: failed to appendEntries to: peer="{Voter server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b fb6044d1-2b30-69f3-0ac3-bf2f201eb89b}" error="failed to connect to peer: fb6044d1-2b30-69f3-0ac3-bf2f201eb89b" -2026-04-22T10:09:21.566Z [WARN] server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a: failed to contact: server-id=server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b time=50.117302ms -2026-04-22T10:09:21.569Z [ERROR] server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a: failed to heartbeat to: peer=fb6044d1-2b30-69f3-0ac3-bf2f201eb89b backoff time=10ms error="failed to connect to peer: fb6044d1-2b30-69f3-0ac3-bf2f201eb89b" -2026-04-22T10:09:21.579Z [ERROR] server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a: failed to appendEntries to: peer="{Voter server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b fb6044d1-2b30-69f3-0ac3-bf2f201eb89b}" error="failed to connect to peer: fb6044d1-2b30-69f3-0ac3-bf2f201eb89b" -2026-04-22T10:09:21.583Z [WARN] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: Election timeout reached, restarting election -2026-04-22T10:09:21.583Z [INFO] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: entering candidate state: node="Node at fb6044d1-2b30-69f3-0ac3-bf2f201eb89b [Candidate]" term=4 -2026-04-22T10:09:21.583Z [ERROR] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: failed to make requestVote RPC: target="{Voter server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a}" error="failed to connect to peer: f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a" term=4 -2026-04-22T10:09:21.583Z [ERROR] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: failed to make requestVote RPC: target="{Voter server-35baf66f-8e1d-2cae-d0c0-68d47521f922 35baf66f-8e1d-2cae-d0c0-68d47521f922}" error="failed to connect to peer: 35baf66f-8e1d-2cae-d0c0-68d47521f922" term=4 -2026-04-22T10:09:21.589Z [ERROR] server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a: failed to heartbeat to: peer=fb6044d1-2b30-69f3-0ac3-bf2f201eb89b backoff time=20ms error="failed to connect to peer: fb6044d1-2b30-69f3-0ac3-bf2f201eb89b" -2026-04-22T10:09:21.616Z [WARN] server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a: failed to contact: server-id=server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b time=100.115458ms -2026-04-22T10:09:21.618Z [ERROR] server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a: failed to heartbeat to: peer=fb6044d1-2b30-69f3-0ac3-bf2f201eb89b backoff time=25ms error="failed to connect to peer: fb6044d1-2b30-69f3-0ac3-bf2f201eb89b" -2026-04-22T10:09:21.622Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:21.516692355 +0000 UTC m=+7.540488465 (1 nodes), 150.954735ms from start of poll, 434.368481ms from cluster start. Timeout at 2026-04-22 10:09:21.621696043 +0000 UTC m=+7.645492125, 105.00366ms after stability - raft_test.go:413: [INFO] Reconnecting Node at fb6044d1-2b30-69f3-0ac3-bf2f201eb89b [Candidate] -2026-04-22T10:09:21.627Z [INFO] server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b: entering follower state: follower="Node at fb6044d1-2b30-69f3-0ac3-bf2f201eb89b [Follower]" leader-address=f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a leader-id=server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a -2026-04-22T10:09:21.627Z [INFO] server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a: pipelining replication: peer="{Voter server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b fb6044d1-2b30-69f3-0ac3-bf2f201eb89b}" -2026-04-22T10:09:21.633Z [INFO] server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a: aborting pipeline replication: peer="{Voter server-fb6044d1-2b30-69f3-0ac3-bf2f201eb89b fb6044d1-2b30-69f3-0ac3-bf2f201eb89b}" -2026-04-22T10:09:21.633Z [INFO] server-f8102dde-10fd-e9ec-ecee-8a2f8b5eb12a: aborting pipeline replication: peer="{Voter server-35baf66f-8e1d-2cae-d0c0-68d47521f922 35baf66f-8e1d-2cae-d0c0-68d47521f922}" ---- PASS: TestRaft_LeaderFail (0.55s) +2025-03-20T03:49:35.596Z [INFO] server-fb98b940-75fc-8a9e-1591-543f92661749: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fb98b940-75fc-8a9e-1591-543f92661749 Address:fb98b940-75fc-8a9e-1591-543f92661749} {Suffrage:Voter ID:server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf Address:c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf} {Suffrage:Voter ID:server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc Address:c88fec5b-6161-ea7b-6c73-d274ea66a7cc}]" +2025-03-20T03:49:35.596Z [INFO] server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fb98b940-75fc-8a9e-1591-543f92661749 Address:fb98b940-75fc-8a9e-1591-543f92661749} {Suffrage:Voter ID:server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf Address:c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf} {Suffrage:Voter ID:server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc Address:c88fec5b-6161-ea7b-6c73-d274ea66a7cc}]" +2025-03-20T03:49:35.596Z [INFO] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fb98b940-75fc-8a9e-1591-543f92661749 Address:fb98b940-75fc-8a9e-1591-543f92661749} {Suffrage:Voter ID:server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf Address:c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf} {Suffrage:Voter ID:server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc Address:c88fec5b-6161-ea7b-6c73-d274ea66a7cc}]" +2025-03-20T03:49:35.596Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:49:35.596Z [INFO] server-fb98b940-75fc-8a9e-1591-543f92661749: entering follower state: follower="Node at fb98b940-75fc-8a9e-1591-543f92661749 [Follower]" leader-address= leader-id= +2025-03-20T03:49:35.596Z [INFO] server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf: entering follower state: follower="Node at c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf [Follower]" leader-address= leader-id= +2025-03-20T03:49:35.596Z [INFO] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: entering follower state: follower="Node at c88fec5b-6161-ea7b-6c73-d274ea66a7cc [Follower]" leader-address= leader-id= +2025-03-20T03:49:35.658Z [WARN] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:35.658Z [INFO] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: entering candidate state: node="Node at c88fec5b-6161-ea7b-6c73-d274ea66a7cc [Candidate]" term=2 +2025-03-20T03:49:35.658Z [INFO] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: election won: term=2 tally=2 +2025-03-20T03:49:35.658Z [INFO] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: entering leader state: leader="Node at c88fec5b-6161-ea7b-6c73-d274ea66a7cc [Leader]" +2025-03-20T03:49:35.658Z [INFO] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: added peer, starting replication: peer=server-fb98b940-75fc-8a9e-1591-543f92661749 +2025-03-20T03:49:35.658Z [INFO] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: added peer, starting replication: peer=server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf +2025-03-20T03:49:35.658Z [INFO] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: pipelining replication: peer="{Voter server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf}" +2025-03-20T03:49:35.659Z [INFO] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: pipelining replication: peer="{Voter server-fb98b940-75fc-8a9e-1591-543f92661749 fb98b940-75fc-8a9e-1591-543f92661749}" +2025-03-20T03:49:35.766Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:49:35.659092389 +0000 UTC m=+7.333545346 (2 nodes), 62.369409ms from start of poll, 62.916473ms from cluster start. Timeout at 2025-03-20 03:49:35.764097524 +0000 UTC m=+7.438550501, 105.005155ms after stability +2025-03-20T03:49:35.766Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:35.878Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:35.766335478 +0000 UTC m=+7.440788443 (1 nodes), 2.474µs from start of poll, 170.15957ms from cluster start. Timeout at 2025-03-20 03:49:35.87142565 +0000 UTC m=+7.545878723, 105.09028ms after stability + raft_test.go:378: [INFO] Disconnecting Node at c88fec5b-6161-ea7b-6c73-d274ea66a7cc [Leader] +2025-03-20T03:49:35.890Z [ERROR] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: failed to pipeline appendEntries: peer="{Voter server-fb98b940-75fc-8a9e-1591-543f92661749 fb98b940-75fc-8a9e-1591-543f92661749}" error="append pipeline closed" +2025-03-20T03:49:35.891Z [INFO] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: aborting pipeline replication: peer="{Voter server-fb98b940-75fc-8a9e-1591-543f92661749 fb98b940-75fc-8a9e-1591-543f92661749}" +2025-03-20T03:49:35.895Z [ERROR] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: failed to heartbeat to: peer=c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf backoff time=10ms error="failed to connect to peer: c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf" +2025-03-20T03:49:35.895Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:35.897Z [ERROR] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: failed to appendEntries to: peer="{Voter server-fb98b940-75fc-8a9e-1591-543f92661749 fb98b940-75fc-8a9e-1591-543f92661749}" error="failed to connect to peer: fb98b940-75fc-8a9e-1591-543f92661749" +2025-03-20T03:49:35.898Z [ERROR] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: failed to heartbeat to: peer=fb98b940-75fc-8a9e-1591-543f92661749 backoff time=10ms error="failed to connect to peer: fb98b940-75fc-8a9e-1591-543f92661749" +2025-03-20T03:49:35.900Z [ERROR] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: failed to pipeline appendEntries: peer="{Voter server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf}" error="append pipeline closed" +2025-03-20T03:49:35.900Z [INFO] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: aborting pipeline replication: peer="{Voter server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf}" +2025-03-20T03:49:35.909Z [ERROR] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: failed to appendEntries to: peer="{Voter server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf}" error="failed to connect to peer: c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf" +2025-03-20T03:49:35.915Z [ERROR] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: failed to appendEntries to: peer="{Voter server-fb98b940-75fc-8a9e-1591-543f92661749 fb98b940-75fc-8a9e-1591-543f92661749}" error="failed to connect to peer: fb98b940-75fc-8a9e-1591-543f92661749" +2025-03-20T03:49:35.916Z [ERROR] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: failed to heartbeat to: peer=c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf backoff time=10ms error="failed to connect to peer: c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf" +2025-03-20T03:49:35.919Z [ERROR] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: failed to heartbeat to: peer=fb98b940-75fc-8a9e-1591-543f92661749 backoff time=10ms error="failed to connect to peer: fb98b940-75fc-8a9e-1591-543f92661749" +2025-03-20T03:49:35.933Z [ERROR] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: failed to appendEntries to: peer="{Voter server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf}" error="failed to connect to peer: c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf" +2025-03-20T03:49:35.936Z [ERROR] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: failed to appendEntries to: peer="{Voter server-fb98b940-75fc-8a9e-1591-543f92661749 fb98b940-75fc-8a9e-1591-543f92661749}" error="failed to connect to peer: fb98b940-75fc-8a9e-1591-543f92661749" +2025-03-20T03:49:35.939Z [WARN] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: failed to contact: server-id=server-fb98b940-75fc-8a9e-1591-543f92661749 time=50.345754ms +2025-03-20T03:49:35.939Z [WARN] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: failed to contact: server-id=server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf time=51.557645ms +2025-03-20T03:49:35.939Z [WARN] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: failed to contact quorum of nodes, stepping down +2025-03-20T03:49:35.939Z [INFO] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: entering follower state: follower="Node at c88fec5b-6161-ea7b-6c73-d274ea66a7cc [Follower]" leader-address= leader-id= +2025-03-20T03:49:35.956Z [WARN] server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf: heartbeat timeout reached, starting election: last-leader-addr=c88fec5b-6161-ea7b-6c73-d274ea66a7cc last-leader-id=server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc +2025-03-20T03:49:35.956Z [INFO] server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf: entering candidate state: node="Node at c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf [Candidate]" term=3 +2025-03-20T03:49:35.956Z [ERROR] server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf: failed to make requestVote RPC: target="{Voter server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc c88fec5b-6161-ea7b-6c73-d274ea66a7cc}" error="failed to connect to peer: c88fec5b-6161-ea7b-6c73-d274ea66a7cc" term=3 +2025-03-20T03:49:35.956Z [WARN] server-fb98b940-75fc-8a9e-1591-543f92661749: rejecting vote request since we have a leader: from=c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf leader=c88fec5b-6161-ea7b-6c73-d274ea66a7cc leader-id=server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc +2025-03-20T03:49:35.986Z [WARN] server-fb98b940-75fc-8a9e-1591-543f92661749: heartbeat timeout reached, starting election: last-leader-addr=c88fec5b-6161-ea7b-6c73-d274ea66a7cc last-leader-id=server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc +2025-03-20T03:49:35.986Z [INFO] server-fb98b940-75fc-8a9e-1591-543f92661749: entering candidate state: node="Node at fb98b940-75fc-8a9e-1591-543f92661749 [Candidate]" term=3 +2025-03-20T03:49:35.987Z [ERROR] server-fb98b940-75fc-8a9e-1591-543f92661749: failed to make requestVote RPC: target="{Voter server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc c88fec5b-6161-ea7b-6c73-d274ea66a7cc}" error="failed to connect to peer: c88fec5b-6161-ea7b-6c73-d274ea66a7cc" term=3 +2025-03-20T03:49:35.987Z [INFO] server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf: duplicate requestVote for same term: term=3 +2025-03-20T03:49:36.021Z [WARN] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:36.021Z [INFO] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: entering candidate state: node="Node at c88fec5b-6161-ea7b-6c73-d274ea66a7cc [Candidate]" term=3 +2025-03-20T03:49:36.021Z [ERROR] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: failed to make requestVote RPC: target="{Voter server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf}" error="failed to connect to peer: c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf" term=3 +2025-03-20T03:49:36.021Z [ERROR] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: failed to make requestVote RPC: target="{Voter server-fb98b940-75fc-8a9e-1591-543f92661749 fb98b940-75fc-8a9e-1591-543f92661749}" error="failed to connect to peer: fb98b940-75fc-8a9e-1591-543f92661749" term=3 +2025-03-20T03:49:36.050Z [WARN] server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf: Election timeout reached, restarting election +2025-03-20T03:49:36.050Z [INFO] server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf: entering candidate state: node="Node at c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf [Candidate]" term=4 +2025-03-20T03:49:36.050Z [ERROR] server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf: failed to make requestVote RPC: target="{Voter server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc c88fec5b-6161-ea7b-6c73-d274ea66a7cc}" error="failed to connect to peer: c88fec5b-6161-ea7b-6c73-d274ea66a7cc" term=4 +2025-03-20T03:49:36.051Z [INFO] server-fb98b940-75fc-8a9e-1591-543f92661749: entering follower state: follower="Node at fb98b940-75fc-8a9e-1591-543f92661749 [Follower]" leader-address= leader-id= +2025-03-20T03:49:36.051Z [INFO] server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf: election won: term=4 tally=2 +2025-03-20T03:49:36.051Z [INFO] server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf: entering leader state: leader="Node at c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf [Leader]" +2025-03-20T03:49:36.051Z [INFO] server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf: added peer, starting replication: peer=server-fb98b940-75fc-8a9e-1591-543f92661749 +2025-03-20T03:49:36.051Z [INFO] server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf: added peer, starting replication: peer=server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc +2025-03-20T03:49:36.051Z [ERROR] server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf: failed to appendEntries to: peer="{Voter server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc c88fec5b-6161-ea7b-6c73-d274ea66a7cc}" error="failed to connect to peer: c88fec5b-6161-ea7b-6c73-d274ea66a7cc" +2025-03-20T03:49:36.051Z [INFO] server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf: pipelining replication: peer="{Voter server-fb98b940-75fc-8a9e-1591-543f92661749 fb98b940-75fc-8a9e-1591-543f92661749}" +2025-03-20T03:49:36.057Z [ERROR] server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf: failed to heartbeat to: peer=c88fec5b-6161-ea7b-6c73-d274ea66a7cc backoff time=10ms error="failed to connect to peer: c88fec5b-6161-ea7b-6c73-d274ea66a7cc" +2025-03-20T03:49:36.070Z [ERROR] server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf: failed to appendEntries to: peer="{Voter server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc c88fec5b-6161-ea7b-6c73-d274ea66a7cc}" error="failed to connect to peer: c88fec5b-6161-ea7b-6c73-d274ea66a7cc" +2025-03-20T03:49:36.078Z [ERROR] server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf: failed to heartbeat to: peer=c88fec5b-6161-ea7b-6c73-d274ea66a7cc backoff time=10ms error="failed to connect to peer: c88fec5b-6161-ea7b-6c73-d274ea66a7cc" +2025-03-20T03:49:36.089Z [ERROR] server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf: failed to appendEntries to: peer="{Voter server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc c88fec5b-6161-ea7b-6c73-d274ea66a7cc}" error="failed to connect to peer: c88fec5b-6161-ea7b-6c73-d274ea66a7cc" +2025-03-20T03:49:36.099Z [ERROR] server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf: failed to heartbeat to: peer=c88fec5b-6161-ea7b-6c73-d274ea66a7cc backoff time=10ms error="failed to connect to peer: c88fec5b-6161-ea7b-6c73-d274ea66a7cc" +2025-03-20T03:49:36.102Z [WARN] server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf: failed to contact: server-id=server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc time=50.778873ms +2025-03-20T03:49:36.115Z [WARN] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: Election timeout reached, restarting election +2025-03-20T03:49:36.115Z [INFO] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: entering candidate state: node="Node at c88fec5b-6161-ea7b-6c73-d274ea66a7cc [Candidate]" term=4 +2025-03-20T03:49:36.115Z [ERROR] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: failed to make requestVote RPC: target="{Voter server-fb98b940-75fc-8a9e-1591-543f92661749 fb98b940-75fc-8a9e-1591-543f92661749}" error="failed to connect to peer: fb98b940-75fc-8a9e-1591-543f92661749" term=4 +2025-03-20T03:49:36.115Z [ERROR] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: failed to make requestVote RPC: target="{Voter server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf}" error="failed to connect to peer: c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf" term=4 +2025-03-20T03:49:36.122Z [ERROR] server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf: failed to heartbeat to: peer=c88fec5b-6161-ea7b-6c73-d274ea66a7cc backoff time=20ms error="failed to connect to peer: c88fec5b-6161-ea7b-6c73-d274ea66a7cc" +2025-03-20T03:49:36.124Z [ERROR] server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf: failed to appendEntries to: peer="{Voter server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc c88fec5b-6161-ea7b-6c73-d274ea66a7cc}" error="failed to connect to peer: c88fec5b-6161-ea7b-6c73-d274ea66a7cc" +2025-03-20T03:49:36.150Z [ERROR] server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf: failed to heartbeat to: peer=c88fec5b-6161-ea7b-6c73-d274ea66a7cc backoff time=25ms error="failed to connect to peer: c88fec5b-6161-ea7b-6c73-d274ea66a7cc" +2025-03-20T03:49:36.151Z [WARN] server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf: failed to contact: server-id=server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc time=100.035947ms +2025-03-20T03:49:36.157Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:36.051402854 +0000 UTC m=+7.725855683 (1 nodes), 156.218157ms from start of poll, 455.22681ms from cluster start. Timeout at 2025-03-20 03:49:36.156405475 +0000 UTC m=+7.830858340, 105.002657ms after stability + raft_test.go:413: [INFO] Reconnecting Node at c88fec5b-6161-ea7b-6c73-d274ea66a7cc [Candidate] +2025-03-20T03:49:36.171Z [INFO] server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc: entering follower state: follower="Node at c88fec5b-6161-ea7b-6c73-d274ea66a7cc [Follower]" leader-address=c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf leader-id=server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf +2025-03-20T03:49:36.171Z [INFO] server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf: pipelining replication: peer="{Voter server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc c88fec5b-6161-ea7b-6c73-d274ea66a7cc}" +2025-03-20T03:49:36.177Z [INFO] server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf: aborting pipeline replication: peer="{Voter server-c88fec5b-6161-ea7b-6c73-d274ea66a7cc c88fec5b-6161-ea7b-6c73-d274ea66a7cc}" +2025-03-20T03:49:36.177Z [INFO] server-c6b4b47b-6638-a8d3-b62d-c1d9bb9d16bf: aborting pipeline replication: peer="{Voter server-fb98b940-75fc-8a9e-1591-543f92661749 fb98b940-75fc-8a9e-1591-543f92661749}" +--- PASS: TestRaft_LeaderFail (0.58s) === RUN TestRaft_BehindFollower -2026-04-22T10:09:21.634Z [INFO] server-428683ea-9b24-eef8-bc76-95e6f02e00a0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-428683ea-9b24-eef8-bc76-95e6f02e00a0 Address:428683ea-9b24-eef8-bc76-95e6f02e00a0} {Suffrage:Voter ID:server-f75509ed-8427-f556-871d-d8f064505597 Address:f75509ed-8427-f556-871d-d8f064505597} {Suffrage:Voter ID:server-8c7247e0-bda8-4c4e-fa6d-cfc9dad03a43 Address:8c7247e0-bda8-4c4e-fa6d-cfc9dad03a43}]" -2026-04-22T10:09:21.634Z [INFO] server-f75509ed-8427-f556-871d-d8f064505597: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-428683ea-9b24-eef8-bc76-95e6f02e00a0 Address:428683ea-9b24-eef8-bc76-95e6f02e00a0} {Suffrage:Voter ID:server-f75509ed-8427-f556-871d-d8f064505597 Address:f75509ed-8427-f556-871d-d8f064505597} {Suffrage:Voter ID:server-8c7247e0-bda8-4c4e-fa6d-cfc9dad03a43 Address:8c7247e0-bda8-4c4e-fa6d-cfc9dad03a43}]" -2026-04-22T10:09:21.634Z [INFO] server-8c7247e0-bda8-4c4e-fa6d-cfc9dad03a43: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-428683ea-9b24-eef8-bc76-95e6f02e00a0 Address:428683ea-9b24-eef8-bc76-95e6f02e00a0} {Suffrage:Voter ID:server-f75509ed-8427-f556-871d-d8f064505597 Address:f75509ed-8427-f556-871d-d8f064505597} {Suffrage:Voter ID:server-8c7247e0-bda8-4c4e-fa6d-cfc9dad03a43 Address:8c7247e0-bda8-4c4e-fa6d-cfc9dad03a43}]" -2026-04-22T10:09:21.634Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:21.634Z [INFO] server-428683ea-9b24-eef8-bc76-95e6f02e00a0: entering follower state: follower="Node at 428683ea-9b24-eef8-bc76-95e6f02e00a0 [Follower]" leader-address= leader-id= -2026-04-22T10:09:21.634Z [INFO] server-f75509ed-8427-f556-871d-d8f064505597: entering follower state: follower="Node at f75509ed-8427-f556-871d-d8f064505597 [Follower]" leader-address= leader-id= -2026-04-22T10:09:21.634Z [INFO] server-8c7247e0-bda8-4c4e-fa6d-cfc9dad03a43: entering follower state: follower="Node at 8c7247e0-bda8-4c4e-fa6d-cfc9dad03a43 [Follower]" leader-address= leader-id= -2026-04-22T10:09:21.697Z [WARN] server-428683ea-9b24-eef8-bc76-95e6f02e00a0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:21.697Z [INFO] server-428683ea-9b24-eef8-bc76-95e6f02e00a0: entering candidate state: node="Node at 428683ea-9b24-eef8-bc76-95e6f02e00a0 [Candidate]" term=2 -2026-04-22T10:09:21.698Z [INFO] server-428683ea-9b24-eef8-bc76-95e6f02e00a0: election won: term=2 tally=2 -2026-04-22T10:09:21.698Z [INFO] server-428683ea-9b24-eef8-bc76-95e6f02e00a0: entering leader state: leader="Node at 428683ea-9b24-eef8-bc76-95e6f02e00a0 [Leader]" -2026-04-22T10:09:21.698Z [INFO] server-428683ea-9b24-eef8-bc76-95e6f02e00a0: added peer, starting replication: peer=server-f75509ed-8427-f556-871d-d8f064505597 -2026-04-22T10:09:21.698Z [INFO] server-428683ea-9b24-eef8-bc76-95e6f02e00a0: added peer, starting replication: peer=server-8c7247e0-bda8-4c4e-fa6d-cfc9dad03a43 -2026-04-22T10:09:21.698Z [INFO] server-428683ea-9b24-eef8-bc76-95e6f02e00a0: pipelining replication: peer="{Voter server-8c7247e0-bda8-4c4e-fa6d-cfc9dad03a43 8c7247e0-bda8-4c4e-fa6d-cfc9dad03a43}" -2026-04-22T10:09:21.698Z [INFO] server-428683ea-9b24-eef8-bc76-95e6f02e00a0: pipelining replication: peer="{Voter server-f75509ed-8427-f556-871d-d8f064505597 f75509ed-8427-f556-871d-d8f064505597}" -2026-04-22T10:09:21.804Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:21.698464198 +0000 UTC m=+7.722260309 (1 nodes), 63.825396ms from start of poll, 64.18175ms from cluster start. Timeout at 2026-04-22 10:09:21.803467847 +0000 UTC m=+7.827263930, 105.003621ms after stability -2026-04-22T10:09:21.804Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:09:21.909Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:09:21.804421533 +0000 UTC m=+7.828217650 (2 nodes), 2.384µs from start of poll, 170.139091ms from cluster start. Timeout at 2026-04-22 10:09:21.909428341 +0000 UTC m=+7.933224424, 105.006774ms after stability -2026-04-22T10:09:21.910Z [ERROR] server-428683ea-9b24-eef8-bc76-95e6f02e00a0: failed to pipeline appendEntries: peer="{Voter server-f75509ed-8427-f556-871d-d8f064505597 f75509ed-8427-f556-871d-d8f064505597}" error="append pipeline closed" -2026-04-22T10:09:21.910Z [INFO] server-428683ea-9b24-eef8-bc76-95e6f02e00a0: aborting pipeline replication: peer="{Voter server-f75509ed-8427-f556-871d-d8f064505597 f75509ed-8427-f556-871d-d8f064505597}" -2026-04-22T10:09:21.910Z [ERROR] server-428683ea-9b24-eef8-bc76-95e6f02e00a0: failed to appendEntries to: peer="{Voter server-f75509ed-8427-f556-871d-d8f064505597 f75509ed-8427-f556-871d-d8f064505597}" error="failed to connect to peer: f75509ed-8427-f556-871d-d8f064505597" +2025-03-20T03:49:36.179Z [INFO] server-3098dbf6-cc20-12fa-1ff5-5bb41d46057a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3098dbf6-cc20-12fa-1ff5-5bb41d46057a Address:3098dbf6-cc20-12fa-1ff5-5bb41d46057a} {Suffrage:Voter ID:server-4fc10606-dddb-ff68-ee32-dcd60af00fa5 Address:4fc10606-dddb-ff68-ee32-dcd60af00fa5} {Suffrage:Voter ID:server-7c91779d-66ac-0b9f-f0d6-f75ab4ea21b2 Address:7c91779d-66ac-0b9f-f0d6-f75ab4ea21b2}]" +2025-03-20T03:49:36.179Z [INFO] server-4fc10606-dddb-ff68-ee32-dcd60af00fa5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3098dbf6-cc20-12fa-1ff5-5bb41d46057a Address:3098dbf6-cc20-12fa-1ff5-5bb41d46057a} {Suffrage:Voter ID:server-4fc10606-dddb-ff68-ee32-dcd60af00fa5 Address:4fc10606-dddb-ff68-ee32-dcd60af00fa5} {Suffrage:Voter ID:server-7c91779d-66ac-0b9f-f0d6-f75ab4ea21b2 Address:7c91779d-66ac-0b9f-f0d6-f75ab4ea21b2}]" +2025-03-20T03:49:36.179Z [INFO] server-7c91779d-66ac-0b9f-f0d6-f75ab4ea21b2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3098dbf6-cc20-12fa-1ff5-5bb41d46057a Address:3098dbf6-cc20-12fa-1ff5-5bb41d46057a} {Suffrage:Voter ID:server-4fc10606-dddb-ff68-ee32-dcd60af00fa5 Address:4fc10606-dddb-ff68-ee32-dcd60af00fa5} {Suffrage:Voter ID:server-7c91779d-66ac-0b9f-f0d6-f75ab4ea21b2 Address:7c91779d-66ac-0b9f-f0d6-f75ab4ea21b2}]" +2025-03-20T03:49:36.179Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:36.179Z [INFO] server-3098dbf6-cc20-12fa-1ff5-5bb41d46057a: entering follower state: follower="Node at 3098dbf6-cc20-12fa-1ff5-5bb41d46057a [Follower]" leader-address= leader-id= +2025-03-20T03:49:36.179Z [INFO] server-4fc10606-dddb-ff68-ee32-dcd60af00fa5: entering follower state: follower="Node at 4fc10606-dddb-ff68-ee32-dcd60af00fa5 [Follower]" leader-address= leader-id= +2025-03-20T03:49:36.180Z [INFO] server-7c91779d-66ac-0b9f-f0d6-f75ab4ea21b2: entering follower state: follower="Node at 7c91779d-66ac-0b9f-f0d6-f75ab4ea21b2 [Follower]" leader-address= leader-id= +2025-03-20T03:49:36.254Z [WARN] server-4fc10606-dddb-ff68-ee32-dcd60af00fa5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:36.254Z [INFO] server-4fc10606-dddb-ff68-ee32-dcd60af00fa5: entering candidate state: node="Node at 4fc10606-dddb-ff68-ee32-dcd60af00fa5 [Candidate]" term=2 +2025-03-20T03:49:36.254Z [INFO] server-4fc10606-dddb-ff68-ee32-dcd60af00fa5: election won: term=2 tally=2 +2025-03-20T03:49:36.254Z [INFO] server-4fc10606-dddb-ff68-ee32-dcd60af00fa5: entering leader state: leader="Node at 4fc10606-dddb-ff68-ee32-dcd60af00fa5 [Leader]" +2025-03-20T03:49:36.254Z [INFO] server-4fc10606-dddb-ff68-ee32-dcd60af00fa5: added peer, starting replication: peer=server-3098dbf6-cc20-12fa-1ff5-5bb41d46057a +2025-03-20T03:49:36.254Z [INFO] server-4fc10606-dddb-ff68-ee32-dcd60af00fa5: added peer, starting replication: peer=server-7c91779d-66ac-0b9f-f0d6-f75ab4ea21b2 +2025-03-20T03:49:36.254Z [INFO] server-4fc10606-dddb-ff68-ee32-dcd60af00fa5: pipelining replication: peer="{Voter server-7c91779d-66ac-0b9f-f0d6-f75ab4ea21b2 7c91779d-66ac-0b9f-f0d6-f75ab4ea21b2}" +2025-03-20T03:49:36.255Z [INFO] server-4fc10606-dddb-ff68-ee32-dcd60af00fa5: pipelining replication: peer="{Voter server-3098dbf6-cc20-12fa-1ff5-5bb41d46057a 3098dbf6-cc20-12fa-1ff5-5bb41d46057a}" +2025-03-20T03:49:36.361Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:36.255420127 +0000 UTC m=+7.929873568 (1 nodes), 75.536252ms from start of poll, 75.959595ms from cluster start. Timeout at 2025-03-20 03:49:36.360424122 +0000 UTC m=+8.034876947, 105.003379ms after stability +2025-03-20T03:49:36.361Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:49:36.468Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:49:36.361819735 +0000 UTC m=+8.036272541 (2 nodes), 2.519µs from start of poll, 182.358568ms from cluster start. Timeout at 2025-03-20 03:49:36.466834393 +0000 UTC m=+8.141287249, 105.014708ms after stability +2025-03-20T03:49:36.468Z [ERROR] server-4fc10606-dddb-ff68-ee32-dcd60af00fa5: failed to pipeline appendEntries: peer="{Voter server-3098dbf6-cc20-12fa-1ff5-5bb41d46057a 3098dbf6-cc20-12fa-1ff5-5bb41d46057a}" error="append pipeline closed" +2025-03-20T03:49:36.468Z [INFO] server-4fc10606-dddb-ff68-ee32-dcd60af00fa5: aborting pipeline replication: peer="{Voter server-3098dbf6-cc20-12fa-1ff5-5bb41d46057a 3098dbf6-cc20-12fa-1ff5-5bb41d46057a}" +2025-03-20T03:49:36.468Z [ERROR] server-4fc10606-dddb-ff68-ee32-dcd60af00fa5: failed to appendEntries to: peer="{Voter server-3098dbf6-cc20-12fa-1ff5-5bb41d46057a 3098dbf6-cc20-12fa-1ff5-5bb41d46057a}" error="failed to connect to peer: 3098dbf6-cc20-12fa-1ff5-5bb41d46057a" +2025-03-20T03:49:36.477Z [ERROR] server-4fc10606-dddb-ff68-ee32-dcd60af00fa5: failed to heartbeat to: peer=3098dbf6-cc20-12fa-1ff5-5bb41d46057a backoff time=10ms error="failed to connect to peer: 3098dbf6-cc20-12fa-1ff5-5bb41d46057a" +2025-03-20T03:49:36.479Z [ERROR] server-4fc10606-dddb-ff68-ee32-dcd60af00fa5: failed to appendEntries to: peer="{Voter server-3098dbf6-cc20-12fa-1ff5-5bb41d46057a 3098dbf6-cc20-12fa-1ff5-5bb41d46057a}" error="failed to connect to peer: 3098dbf6-cc20-12fa-1ff5-5bb41d46057a" raft_test.go:459: [INFO] Finished apply without behind follower -2026-04-22T10:09:21.921Z [INFO] server-428683ea-9b24-eef8-bc76-95e6f02e00a0: pipelining replication: peer="{Voter server-f75509ed-8427-f556-871d-d8f064505597 f75509ed-8427-f556-871d-d8f064505597}" -2026-04-22T10:09:21.937Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:22.043Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:21.937660194 +0000 UTC m=+7.961456302 (1 nodes), 1.166µs from start of poll, 303.377743ms from cluster start. Timeout at 2026-04-22 10:09:22.042668559 +0000 UTC m=+8.066464652, 105.00835ms after stability -2026-04-22T10:09:22.043Z [INFO] server-428683ea-9b24-eef8-bc76-95e6f02e00a0: aborting pipeline replication: peer="{Voter server-8c7247e0-bda8-4c4e-fa6d-cfc9dad03a43 8c7247e0-bda8-4c4e-fa6d-cfc9dad03a43}" -2026-04-22T10:09:22.043Z [INFO] server-428683ea-9b24-eef8-bc76-95e6f02e00a0: aborting pipeline replication: peer="{Voter server-f75509ed-8427-f556-871d-d8f064505597 f75509ed-8427-f556-871d-d8f064505597}" ---- PASS: TestRaft_BehindFollower (0.41s) +2025-03-20T03:49:36.491Z [INFO] server-4fc10606-dddb-ff68-ee32-dcd60af00fa5: pipelining replication: peer="{Voter server-3098dbf6-cc20-12fa-1ff5-5bb41d46057a 3098dbf6-cc20-12fa-1ff5-5bb41d46057a}" +2025-03-20T03:49:36.498Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:36.603Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:36.498170295 +0000 UTC m=+8.172623335 (1 nodes), 3.654µs from start of poll, 318.709362ms from cluster start. Timeout at 2025-03-20 03:49:36.603190992 +0000 UTC m=+8.277643900, 105.020565ms after stability +2025-03-20T03:49:36.604Z [INFO] server-4fc10606-dddb-ff68-ee32-dcd60af00fa5: aborting pipeline replication: peer="{Voter server-7c91779d-66ac-0b9f-f0d6-f75ab4ea21b2 7c91779d-66ac-0b9f-f0d6-f75ab4ea21b2}" +2025-03-20T03:49:36.604Z [INFO] server-4fc10606-dddb-ff68-ee32-dcd60af00fa5: aborting pipeline replication: peer="{Voter server-3098dbf6-cc20-12fa-1ff5-5bb41d46057a 3098dbf6-cc20-12fa-1ff5-5bb41d46057a}" +--- PASS: TestRaft_BehindFollower (0.43s) === RUN TestRaft_ApplyNonLeader -2026-04-22T10:09:22.044Z [INFO] server-051a4dd4-a43e-7302-f030-384fc4d2b135: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-051a4dd4-a43e-7302-f030-384fc4d2b135 Address:051a4dd4-a43e-7302-f030-384fc4d2b135} {Suffrage:Voter ID:server-15ae070a-55f5-558f-2ffb-6c61b7e231f8 Address:15ae070a-55f5-558f-2ffb-6c61b7e231f8} {Suffrage:Voter ID:server-b5df207f-0c84-606c-28c1-6ffe8ae7ae60 Address:b5df207f-0c84-606c-28c1-6ffe8ae7ae60}]" -2026-04-22T10:09:22.044Z [INFO] server-051a4dd4-a43e-7302-f030-384fc4d2b135: entering follower state: follower="Node at 051a4dd4-a43e-7302-f030-384fc4d2b135 [Follower]" leader-address= leader-id= -2026-04-22T10:09:22.044Z [INFO] server-15ae070a-55f5-558f-2ffb-6c61b7e231f8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-051a4dd4-a43e-7302-f030-384fc4d2b135 Address:051a4dd4-a43e-7302-f030-384fc4d2b135} {Suffrage:Voter ID:server-15ae070a-55f5-558f-2ffb-6c61b7e231f8 Address:15ae070a-55f5-558f-2ffb-6c61b7e231f8} {Suffrage:Voter ID:server-b5df207f-0c84-606c-28c1-6ffe8ae7ae60 Address:b5df207f-0c84-606c-28c1-6ffe8ae7ae60}]" -2026-04-22T10:09:22.044Z [INFO] server-15ae070a-55f5-558f-2ffb-6c61b7e231f8: entering follower state: follower="Node at 15ae070a-55f5-558f-2ffb-6c61b7e231f8 [Follower]" leader-address= leader-id= -2026-04-22T10:09:22.044Z [INFO] server-b5df207f-0c84-606c-28c1-6ffe8ae7ae60: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-051a4dd4-a43e-7302-f030-384fc4d2b135 Address:051a4dd4-a43e-7302-f030-384fc4d2b135} {Suffrage:Voter ID:server-15ae070a-55f5-558f-2ffb-6c61b7e231f8 Address:15ae070a-55f5-558f-2ffb-6c61b7e231f8} {Suffrage:Voter ID:server-b5df207f-0c84-606c-28c1-6ffe8ae7ae60 Address:b5df207f-0c84-606c-28c1-6ffe8ae7ae60}]" -2026-04-22T10:09:22.044Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:22.044Z [INFO] server-b5df207f-0c84-606c-28c1-6ffe8ae7ae60: entering follower state: follower="Node at b5df207f-0c84-606c-28c1-6ffe8ae7ae60 [Follower]" leader-address= leader-id= -2026-04-22T10:09:22.106Z [WARN] server-b5df207f-0c84-606c-28c1-6ffe8ae7ae60: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:22.106Z [INFO] server-b5df207f-0c84-606c-28c1-6ffe8ae7ae60: entering candidate state: node="Node at b5df207f-0c84-606c-28c1-6ffe8ae7ae60 [Candidate]" term=2 -2026-04-22T10:09:22.107Z [INFO] server-b5df207f-0c84-606c-28c1-6ffe8ae7ae60: election won: term=2 tally=2 -2026-04-22T10:09:22.107Z [INFO] server-b5df207f-0c84-606c-28c1-6ffe8ae7ae60: entering leader state: leader="Node at b5df207f-0c84-606c-28c1-6ffe8ae7ae60 [Leader]" -2026-04-22T10:09:22.107Z [INFO] server-b5df207f-0c84-606c-28c1-6ffe8ae7ae60: added peer, starting replication: peer=server-051a4dd4-a43e-7302-f030-384fc4d2b135 -2026-04-22T10:09:22.107Z [INFO] server-b5df207f-0c84-606c-28c1-6ffe8ae7ae60: added peer, starting replication: peer=server-15ae070a-55f5-558f-2ffb-6c61b7e231f8 -2026-04-22T10:09:22.107Z [INFO] server-b5df207f-0c84-606c-28c1-6ffe8ae7ae60: pipelining replication: peer="{Voter server-051a4dd4-a43e-7302-f030-384fc4d2b135 051a4dd4-a43e-7302-f030-384fc4d2b135}" -2026-04-22T10:09:22.107Z [INFO] server-b5df207f-0c84-606c-28c1-6ffe8ae7ae60: pipelining replication: peer="{Voter server-15ae070a-55f5-558f-2ffb-6c61b7e231f8 15ae070a-55f5-558f-2ffb-6c61b7e231f8}" -2026-04-22T10:09:22.213Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:22.107215266 +0000 UTC m=+8.131011395 (1 nodes), 62.750275ms from start of poll, 63.083755ms from cluster start. Timeout at 2026-04-22 10:09:22.212216379 +0000 UTC m=+8.236012462, 105.001067ms after stability -2026-04-22T10:09:22.213Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:09:22.318Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:09:22.213126202 +0000 UTC m=+8.236922310 (2 nodes), 1.22µs from start of poll, 168.99467ms from cluster start. Timeout at 2026-04-22 10:09:22.318132571 +0000 UTC m=+8.341928651, 105.006341ms after stability -2026-04-22T10:09:22.318Z [INFO] server-b5df207f-0c84-606c-28c1-6ffe8ae7ae60: aborting pipeline replication: peer="{Voter server-15ae070a-55f5-558f-2ffb-6c61b7e231f8 15ae070a-55f5-558f-2ffb-6c61b7e231f8}" -2026-04-22T10:09:22.318Z [INFO] server-b5df207f-0c84-606c-28c1-6ffe8ae7ae60: aborting pipeline replication: peer="{Voter server-051a4dd4-a43e-7302-f030-384fc4d2b135 051a4dd4-a43e-7302-f030-384fc4d2b135}" ---- PASS: TestRaft_ApplyNonLeader (0.28s) +2025-03-20T03:49:36.606Z [INFO] server-fcde8d29-9819-b368-c724-f40cbe7d86fd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fcde8d29-9819-b368-c724-f40cbe7d86fd Address:fcde8d29-9819-b368-c724-f40cbe7d86fd} {Suffrage:Voter ID:server-76d87ec3-0032-bfdf-0569-41b9e7b8e95a Address:76d87ec3-0032-bfdf-0569-41b9e7b8e95a} {Suffrage:Voter ID:server-d727c9ab-83d2-777b-8e87-45a51b6f1f7f Address:d727c9ab-83d2-777b-8e87-45a51b6f1f7f}]" +2025-03-20T03:49:36.606Z [INFO] server-76d87ec3-0032-bfdf-0569-41b9e7b8e95a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fcde8d29-9819-b368-c724-f40cbe7d86fd Address:fcde8d29-9819-b368-c724-f40cbe7d86fd} {Suffrage:Voter ID:server-76d87ec3-0032-bfdf-0569-41b9e7b8e95a Address:76d87ec3-0032-bfdf-0569-41b9e7b8e95a} {Suffrage:Voter ID:server-d727c9ab-83d2-777b-8e87-45a51b6f1f7f Address:d727c9ab-83d2-777b-8e87-45a51b6f1f7f}]" +2025-03-20T03:49:36.606Z [INFO] server-d727c9ab-83d2-777b-8e87-45a51b6f1f7f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fcde8d29-9819-b368-c724-f40cbe7d86fd Address:fcde8d29-9819-b368-c724-f40cbe7d86fd} {Suffrage:Voter ID:server-76d87ec3-0032-bfdf-0569-41b9e7b8e95a Address:76d87ec3-0032-bfdf-0569-41b9e7b8e95a} {Suffrage:Voter ID:server-d727c9ab-83d2-777b-8e87-45a51b6f1f7f Address:d727c9ab-83d2-777b-8e87-45a51b6f1f7f}]" +2025-03-20T03:49:36.606Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:36.606Z [INFO] server-fcde8d29-9819-b368-c724-f40cbe7d86fd: entering follower state: follower="Node at fcde8d29-9819-b368-c724-f40cbe7d86fd [Follower]" leader-address= leader-id= +2025-03-20T03:49:36.606Z [INFO] server-76d87ec3-0032-bfdf-0569-41b9e7b8e95a: entering follower state: follower="Node at 76d87ec3-0032-bfdf-0569-41b9e7b8e95a [Follower]" leader-address= leader-id= +2025-03-20T03:49:36.606Z [INFO] server-d727c9ab-83d2-777b-8e87-45a51b6f1f7f: entering follower state: follower="Node at d727c9ab-83d2-777b-8e87-45a51b6f1f7f [Follower]" leader-address= leader-id= +2025-03-20T03:49:36.665Z [WARN] server-76d87ec3-0032-bfdf-0569-41b9e7b8e95a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:36.665Z [INFO] server-76d87ec3-0032-bfdf-0569-41b9e7b8e95a: entering candidate state: node="Node at 76d87ec3-0032-bfdf-0569-41b9e7b8e95a [Candidate]" term=2 +2025-03-20T03:49:36.665Z [INFO] server-76d87ec3-0032-bfdf-0569-41b9e7b8e95a: election won: term=2 tally=2 +2025-03-20T03:49:36.665Z [INFO] server-76d87ec3-0032-bfdf-0569-41b9e7b8e95a: entering leader state: leader="Node at 76d87ec3-0032-bfdf-0569-41b9e7b8e95a [Leader]" +2025-03-20T03:49:36.665Z [INFO] server-76d87ec3-0032-bfdf-0569-41b9e7b8e95a: added peer, starting replication: peer=server-fcde8d29-9819-b368-c724-f40cbe7d86fd +2025-03-20T03:49:36.665Z [INFO] server-76d87ec3-0032-bfdf-0569-41b9e7b8e95a: added peer, starting replication: peer=server-d727c9ab-83d2-777b-8e87-45a51b6f1f7f +2025-03-20T03:49:36.665Z [INFO] server-76d87ec3-0032-bfdf-0569-41b9e7b8e95a: pipelining replication: peer="{Voter server-d727c9ab-83d2-777b-8e87-45a51b6f1f7f d727c9ab-83d2-777b-8e87-45a51b6f1f7f}" +2025-03-20T03:49:36.665Z [INFO] server-76d87ec3-0032-bfdf-0569-41b9e7b8e95a: pipelining replication: peer="{Voter server-fcde8d29-9819-b368-c724-f40cbe7d86fd fcde8d29-9819-b368-c724-f40cbe7d86fd}" +2025-03-20T03:49:36.771Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:36.666078143 +0000 UTC m=+8.340531013 (1 nodes), 59.627968ms from start of poll, 60.172507ms from cluster start. Timeout at 2025-03-20 03:49:36.771095588 +0000 UTC m=+8.445548512, 105.017499ms after stability +2025-03-20T03:49:36.771Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:49:36.877Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:49:36.7716662 +0000 UTC m=+8.446119045 (2 nodes), 2.42µs from start of poll, 165.760539ms from cluster start. Timeout at 2025-03-20 03:49:36.876683595 +0000 UTC m=+8.551136505, 105.01746ms after stability +2025-03-20T03:49:36.877Z [INFO] server-76d87ec3-0032-bfdf-0569-41b9e7b8e95a: aborting pipeline replication: peer="{Voter server-d727c9ab-83d2-777b-8e87-45a51b6f1f7f d727c9ab-83d2-777b-8e87-45a51b6f1f7f}" +2025-03-20T03:49:36.877Z [INFO] server-76d87ec3-0032-bfdf-0569-41b9e7b8e95a: aborting pipeline replication: peer="{Voter server-fcde8d29-9819-b368-c724-f40cbe7d86fd fcde8d29-9819-b368-c724-f40cbe7d86fd}" +--- PASS: TestRaft_ApplyNonLeader (0.27s) === RUN TestRaft_ApplyConcurrent -2026-04-22T10:09:22.319Z [INFO] server-7326745e-254a-bfb4-3be1-36e97c978490: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7326745e-254a-bfb4-3be1-36e97c978490 Address:7326745e-254a-bfb4-3be1-36e97c978490} {Suffrage:Voter ID:server-cd403e3c-7dfe-5200-41ee-a75f96447917 Address:cd403e3c-7dfe-5200-41ee-a75f96447917} {Suffrage:Voter ID:server-9a71e383-1c53-d2d4-c5bb-6a24696e0a8f Address:9a71e383-1c53-d2d4-c5bb-6a24696e0a8f}]" -2026-04-22T10:09:22.319Z [INFO] server-7326745e-254a-bfb4-3be1-36e97c978490: entering follower state: follower="Node at 7326745e-254a-bfb4-3be1-36e97c978490 [Follower]" leader-address= leader-id= -2026-04-22T10:09:22.319Z [INFO] server-cd403e3c-7dfe-5200-41ee-a75f96447917: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7326745e-254a-bfb4-3be1-36e97c978490 Address:7326745e-254a-bfb4-3be1-36e97c978490} {Suffrage:Voter ID:server-cd403e3c-7dfe-5200-41ee-a75f96447917 Address:cd403e3c-7dfe-5200-41ee-a75f96447917} {Suffrage:Voter ID:server-9a71e383-1c53-d2d4-c5bb-6a24696e0a8f Address:9a71e383-1c53-d2d4-c5bb-6a24696e0a8f}]" -2026-04-22T10:09:22.319Z [INFO] server-cd403e3c-7dfe-5200-41ee-a75f96447917: entering follower state: follower="Node at cd403e3c-7dfe-5200-41ee-a75f96447917 [Follower]" leader-address= leader-id= -2026-04-22T10:09:22.319Z [INFO] server-9a71e383-1c53-d2d4-c5bb-6a24696e0a8f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7326745e-254a-bfb4-3be1-36e97c978490 Address:7326745e-254a-bfb4-3be1-36e97c978490} {Suffrage:Voter ID:server-cd403e3c-7dfe-5200-41ee-a75f96447917 Address:cd403e3c-7dfe-5200-41ee-a75f96447917} {Suffrage:Voter ID:server-9a71e383-1c53-d2d4-c5bb-6a24696e0a8f Address:9a71e383-1c53-d2d4-c5bb-6a24696e0a8f}]" -2026-04-22T10:09:22.319Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:22.319Z [INFO] server-9a71e383-1c53-d2d4-c5bb-6a24696e0a8f: entering follower state: follower="Node at 9a71e383-1c53-d2d4-c5bb-6a24696e0a8f [Follower]" leader-address= leader-id= -2026-04-22T10:09:22.445Z [WARN] server-9a71e383-1c53-d2d4-c5bb-6a24696e0a8f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:22.445Z [INFO] server-9a71e383-1c53-d2d4-c5bb-6a24696e0a8f: entering candidate state: node="Node at 9a71e383-1c53-d2d4-c5bb-6a24696e0a8f [Candidate]" term=2 -2026-04-22T10:09:22.446Z [INFO] server-9a71e383-1c53-d2d4-c5bb-6a24696e0a8f: election won: term=2 tally=2 -2026-04-22T10:09:22.446Z [INFO] server-9a71e383-1c53-d2d4-c5bb-6a24696e0a8f: entering leader state: leader="Node at 9a71e383-1c53-d2d4-c5bb-6a24696e0a8f [Leader]" -2026-04-22T10:09:22.446Z [INFO] server-9a71e383-1c53-d2d4-c5bb-6a24696e0a8f: added peer, starting replication: peer=server-7326745e-254a-bfb4-3be1-36e97c978490 -2026-04-22T10:09:22.446Z [INFO] server-9a71e383-1c53-d2d4-c5bb-6a24696e0a8f: added peer, starting replication: peer=server-cd403e3c-7dfe-5200-41ee-a75f96447917 -2026-04-22T10:09:22.446Z [INFO] server-9a71e383-1c53-d2d4-c5bb-6a24696e0a8f: pipelining replication: peer="{Voter server-7326745e-254a-bfb4-3be1-36e97c978490 7326745e-254a-bfb4-3be1-36e97c978490}" -2026-04-22T10:09:22.446Z [INFO] server-9a71e383-1c53-d2d4-c5bb-6a24696e0a8f: pipelining replication: peer="{Voter server-cd403e3c-7dfe-5200-41ee-a75f96447917 cd403e3c-7dfe-5200-41ee-a75f96447917}" -2026-04-22T10:09:22.653Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:22.446108213 +0000 UTC m=+8.469904340 (1 nodes), 126.263698ms from start of poll, 126.596936ms from cluster start. Timeout at 2026-04-22 10:09:22.651114606 +0000 UTC m=+8.674910731, 205.006391ms after stability -2026-04-22T10:09:22.671Z [INFO] server-9a71e383-1c53-d2d4-c5bb-6a24696e0a8f: aborting pipeline replication: peer="{Voter server-cd403e3c-7dfe-5200-41ee-a75f96447917 cd403e3c-7dfe-5200-41ee-a75f96447917}" -2026-04-22T10:09:22.671Z [INFO] server-9a71e383-1c53-d2d4-c5bb-6a24696e0a8f: aborting pipeline replication: peer="{Voter server-7326745e-254a-bfb4-3be1-36e97c978490 7326745e-254a-bfb4-3be1-36e97c978490}" ---- PASS: TestRaft_ApplyConcurrent (0.35s) +2025-03-20T03:49:36.879Z [INFO] server-579d6afb-3e8a-4a2d-8354-2d18628bd6af: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-579d6afb-3e8a-4a2d-8354-2d18628bd6af Address:579d6afb-3e8a-4a2d-8354-2d18628bd6af} {Suffrage:Voter ID:server-e898ab90-f8f4-b141-ebee-4b0d1524204b Address:e898ab90-f8f4-b141-ebee-4b0d1524204b} {Suffrage:Voter ID:server-6c7f2476-fe60-c530-0773-4171b5aed643 Address:6c7f2476-fe60-c530-0773-4171b5aed643}]" +2025-03-20T03:49:36.879Z [INFO] server-e898ab90-f8f4-b141-ebee-4b0d1524204b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-579d6afb-3e8a-4a2d-8354-2d18628bd6af Address:579d6afb-3e8a-4a2d-8354-2d18628bd6af} {Suffrage:Voter ID:server-e898ab90-f8f4-b141-ebee-4b0d1524204b Address:e898ab90-f8f4-b141-ebee-4b0d1524204b} {Suffrage:Voter ID:server-6c7f2476-fe60-c530-0773-4171b5aed643 Address:6c7f2476-fe60-c530-0773-4171b5aed643}]" +2025-03-20T03:49:36.879Z [INFO] server-6c7f2476-fe60-c530-0773-4171b5aed643: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-579d6afb-3e8a-4a2d-8354-2d18628bd6af Address:579d6afb-3e8a-4a2d-8354-2d18628bd6af} {Suffrage:Voter ID:server-e898ab90-f8f4-b141-ebee-4b0d1524204b Address:e898ab90-f8f4-b141-ebee-4b0d1524204b} {Suffrage:Voter ID:server-6c7f2476-fe60-c530-0773-4171b5aed643 Address:6c7f2476-fe60-c530-0773-4171b5aed643}]" +2025-03-20T03:49:36.879Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:36.879Z [INFO] server-579d6afb-3e8a-4a2d-8354-2d18628bd6af: entering follower state: follower="Node at 579d6afb-3e8a-4a2d-8354-2d18628bd6af [Follower]" leader-address= leader-id= +2025-03-20T03:49:36.879Z [INFO] server-e898ab90-f8f4-b141-ebee-4b0d1524204b: entering follower state: follower="Node at e898ab90-f8f4-b141-ebee-4b0d1524204b [Follower]" leader-address= leader-id= +2025-03-20T03:49:36.879Z [INFO] server-6c7f2476-fe60-c530-0773-4171b5aed643: entering follower state: follower="Node at 6c7f2476-fe60-c530-0773-4171b5aed643 [Follower]" leader-address= leader-id= +2025-03-20T03:49:37.000Z [WARN] server-6c7f2476-fe60-c530-0773-4171b5aed643: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:37.000Z [INFO] server-6c7f2476-fe60-c530-0773-4171b5aed643: entering candidate state: node="Node at 6c7f2476-fe60-c530-0773-4171b5aed643 [Candidate]" term=2 +2025-03-20T03:49:37.000Z [INFO] server-6c7f2476-fe60-c530-0773-4171b5aed643: election won: term=2 tally=2 +2025-03-20T03:49:37.000Z [INFO] server-6c7f2476-fe60-c530-0773-4171b5aed643: entering leader state: leader="Node at 6c7f2476-fe60-c530-0773-4171b5aed643 [Leader]" +2025-03-20T03:49:37.000Z [INFO] server-6c7f2476-fe60-c530-0773-4171b5aed643: added peer, starting replication: peer=server-579d6afb-3e8a-4a2d-8354-2d18628bd6af +2025-03-20T03:49:37.001Z [INFO] server-6c7f2476-fe60-c530-0773-4171b5aed643: added peer, starting replication: peer=server-e898ab90-f8f4-b141-ebee-4b0d1524204b +2025-03-20T03:49:37.001Z [INFO] server-6c7f2476-fe60-c530-0773-4171b5aed643: pipelining replication: peer="{Voter server-e898ab90-f8f4-b141-ebee-4b0d1524204b e898ab90-f8f4-b141-ebee-4b0d1524204b}" +2025-03-20T03:49:37.001Z [INFO] server-6c7f2476-fe60-c530-0773-4171b5aed643: pipelining replication: peer="{Voter server-579d6afb-3e8a-4a2d-8354-2d18628bd6af 579d6afb-3e8a-4a2d-8354-2d18628bd6af}" +2025-03-20T03:49:37.207Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:37.00181499 +0000 UTC m=+8.676270027 (1 nodes), 122.079822ms from start of poll, 122.54075ms from cluster start. Timeout at 2025-03-20 03:49:37.206821385 +0000 UTC m=+8.881274428, 205.004401ms after stability +2025-03-20T03:49:37.236Z [INFO] server-6c7f2476-fe60-c530-0773-4171b5aed643: aborting pipeline replication: peer="{Voter server-e898ab90-f8f4-b141-ebee-4b0d1524204b e898ab90-f8f4-b141-ebee-4b0d1524204b}" +2025-03-20T03:49:37.236Z [INFO] server-6c7f2476-fe60-c530-0773-4171b5aed643: aborting pipeline replication: peer="{Voter server-579d6afb-3e8a-4a2d-8354-2d18628bd6af 579d6afb-3e8a-4a2d-8354-2d18628bd6af}" +--- PASS: TestRaft_ApplyConcurrent (0.36s) === RUN TestRaft_ApplyConcurrent_Timeout -2026-04-22T10:09:22.672Z [INFO] server-c5ee96ec-5dfc-b691-ee1d-f4f9ee355e8b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c5ee96ec-5dfc-b691-ee1d-f4f9ee355e8b Address:c5ee96ec-5dfc-b691-ee1d-f4f9ee355e8b}]" -2026-04-22T10:09:22.672Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:22.672Z [INFO] server-c5ee96ec-5dfc-b691-ee1d-f4f9ee355e8b: entering follower state: follower="Node at c5ee96ec-5dfc-b691-ee1d-f4f9ee355e8b [Follower]" leader-address= leader-id= -2026-04-22T10:09:22.841Z [WARN] server-c5ee96ec-5dfc-b691-ee1d-f4f9ee355e8b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:22.841Z [INFO] server-c5ee96ec-5dfc-b691-ee1d-f4f9ee355e8b: entering candidate state: node="Node at c5ee96ec-5dfc-b691-ee1d-f4f9ee355e8b [Candidate]" term=2 -2026-04-22T10:09:22.841Z [INFO] server-c5ee96ec-5dfc-b691-ee1d-f4f9ee355e8b: election won: term=2 tally=1 -2026-04-22T10:09:22.841Z [INFO] server-c5ee96ec-5dfc-b691-ee1d-f4f9ee355e8b: entering leader state: leader="Node at c5ee96ec-5dfc-b691-ee1d-f4f9ee355e8b [Leader]" -2026-04-22T10:09:23.045Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:22.841836554 +0000 UTC m=+8.865632668 (1 nodes), 169.126822ms from start of poll, 169.273773ms from cluster start. Timeout at 2026-04-22 10:09:23.042847528 +0000 UTC m=+9.066643640, 201.010972ms after stability ---- PASS: TestRaft_ApplyConcurrent_Timeout (0.37s) +2025-03-20T03:49:37.237Z [INFO] server-b424b2c8-ce92-c450-99aa-36cdffa3e18c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b424b2c8-ce92-c450-99aa-36cdffa3e18c Address:b424b2c8-ce92-c450-99aa-36cdffa3e18c}]" +2025-03-20T03:49:37.237Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:37.237Z [INFO] server-b424b2c8-ce92-c450-99aa-36cdffa3e18c: entering follower state: follower="Node at b424b2c8-ce92-c450-99aa-36cdffa3e18c [Follower]" leader-address= leader-id= +2025-03-20T03:49:37.360Z [WARN] server-b424b2c8-ce92-c450-99aa-36cdffa3e18c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:37.360Z [INFO] server-b424b2c8-ce92-c450-99aa-36cdffa3e18c: entering candidate state: node="Node at b424b2c8-ce92-c450-99aa-36cdffa3e18c [Candidate]" term=2 +2025-03-20T03:49:37.360Z [INFO] server-b424b2c8-ce92-c450-99aa-36cdffa3e18c: election won: term=2 tally=1 +2025-03-20T03:49:37.360Z [INFO] server-b424b2c8-ce92-c450-99aa-36cdffa3e18c: entering leader state: leader="Node at b424b2c8-ce92-c450-99aa-36cdffa3e18c [Leader]" +2025-03-20T03:49:37.562Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:37.3603252 +0000 UTC m=+9.034778109 (1 nodes), 122.386992ms from start of poll, 122.543238ms from cluster start. Timeout at 2025-03-20 03:49:37.561330473 +0000 UTC m=+9.235783380, 201.005271ms after stability +--- PASS: TestRaft_ApplyConcurrent_Timeout (0.33s) === RUN TestRaft_JoinNode -2026-04-22T10:09:23.046Z [INFO] server-96e251b9-7953-1866-857b-85a162ac79e0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-96e251b9-7953-1866-857b-85a162ac79e0 Address:96e251b9-7953-1866-857b-85a162ac79e0} {Suffrage:Voter ID:server-567ddd18-ef01-cc99-f387-34ac0aa95ed2 Address:567ddd18-ef01-cc99-f387-34ac0aa95ed2}]" -2026-04-22T10:09:23.046Z [INFO] server-567ddd18-ef01-cc99-f387-34ac0aa95ed2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-96e251b9-7953-1866-857b-85a162ac79e0 Address:96e251b9-7953-1866-857b-85a162ac79e0} {Suffrage:Voter ID:server-567ddd18-ef01-cc99-f387-34ac0aa95ed2 Address:567ddd18-ef01-cc99-f387-34ac0aa95ed2}]" -2026-04-22T10:09:23.047Z [INFO] server-96e251b9-7953-1866-857b-85a162ac79e0: entering follower state: follower="Node at 96e251b9-7953-1866-857b-85a162ac79e0 [Follower]" leader-address= leader-id= -2026-04-22T10:09:23.047Z [INFO] server-567ddd18-ef01-cc99-f387-34ac0aa95ed2: entering follower state: follower="Node at 567ddd18-ef01-cc99-f387-34ac0aa95ed2 [Follower]" leader-address= leader-id= -2026-04-22T10:09:23.047Z [INFO] server-ccbf10e9-7aef-6a84-6972-1f6c437570f9: initial configuration: index=0 servers=[] -2026-04-22T10:09:23.047Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:23.047Z [INFO] server-ccbf10e9-7aef-6a84-6972-1f6c437570f9: entering follower state: follower="Node at ccbf10e9-7aef-6a84-6972-1f6c437570f9 [Follower]" leader-address= leader-id= -2026-04-22T10:09:23.104Z [WARN] server-ccbf10e9-7aef-6a84-6972-1f6c437570f9: no known peers, aborting election -2026-04-22T10:09:23.143Z [WARN] server-96e251b9-7953-1866-857b-85a162ac79e0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:23.143Z [INFO] server-96e251b9-7953-1866-857b-85a162ac79e0: entering candidate state: node="Node at 96e251b9-7953-1866-857b-85a162ac79e0 [Candidate]" term=2 -2026-04-22T10:09:23.143Z [INFO] server-96e251b9-7953-1866-857b-85a162ac79e0: election won: term=2 tally=2 -2026-04-22T10:09:23.143Z [INFO] server-96e251b9-7953-1866-857b-85a162ac79e0: entering leader state: leader="Node at 96e251b9-7953-1866-857b-85a162ac79e0 [Leader]" -2026-04-22T10:09:23.143Z [INFO] server-96e251b9-7953-1866-857b-85a162ac79e0: added peer, starting replication: peer=server-567ddd18-ef01-cc99-f387-34ac0aa95ed2 -2026-04-22T10:09:23.143Z [INFO] server-96e251b9-7953-1866-857b-85a162ac79e0: pipelining replication: peer="{Voter server-567ddd18-ef01-cc99-f387-34ac0aa95ed2 567ddd18-ef01-cc99-f387-34ac0aa95ed2}" -2026-04-22T10:09:23.248Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:23.143645313 +0000 UTC m=+9.167441426 (1 nodes), 96.416015ms from start of poll, 96.927241ms from cluster start. Timeout at 2026-04-22 10:09:23.248651561 +0000 UTC m=+9.272447737, 105.006311ms after stability -2026-04-22T10:09:23.249Z [INFO] server-96e251b9-7953-1866-857b-85a162ac79e0: updating configuration: command=AddVoter server-id=server-ccbf10e9-7aef-6a84-6972-1f6c437570f9 server-addr=ccbf10e9-7aef-6a84-6972-1f6c437570f9 servers="[{Suffrage:Voter ID:server-96e251b9-7953-1866-857b-85a162ac79e0 Address:96e251b9-7953-1866-857b-85a162ac79e0} {Suffrage:Voter ID:server-567ddd18-ef01-cc99-f387-34ac0aa95ed2 Address:567ddd18-ef01-cc99-f387-34ac0aa95ed2} {Suffrage:Voter ID:server-ccbf10e9-7aef-6a84-6972-1f6c437570f9 Address:ccbf10e9-7aef-6a84-6972-1f6c437570f9}]" -2026-04-22T10:09:23.249Z [INFO] server-96e251b9-7953-1866-857b-85a162ac79e0: added peer, starting replication: peer=server-ccbf10e9-7aef-6a84-6972-1f6c437570f9 -2026-04-22T10:09:23.249Z [WARN] server-ccbf10e9-7aef-6a84-6972-1f6c437570f9: failed to get previous log: previous-index=3 last-index=0 error="log not found" -2026-04-22T10:09:23.249Z [WARN] server-96e251b9-7953-1866-857b-85a162ac79e0: appendEntries rejected, sending older logs: peer="{Voter server-ccbf10e9-7aef-6a84-6972-1f6c437570f9 ccbf10e9-7aef-6a84-6972-1f6c437570f9}" next=1 -2026-04-22T10:09:23.249Z [INFO] server-96e251b9-7953-1866-857b-85a162ac79e0: pipelining replication: peer="{Voter server-ccbf10e9-7aef-6a84-6972-1f6c437570f9 ccbf10e9-7aef-6a84-6972-1f6c437570f9}" -2026-04-22T10:09:23.249Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:23.355Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:23.24947961 +0000 UTC m=+9.273275721 (1 nodes), 937ns from start of poll, 202.761536ms from cluster start. Timeout at 2026-04-22 10:09:23.354484172 +0000 UTC m=+9.378280291, 105.00457ms after stability -2026-04-22T10:09:23.355Z [INFO] server-96e251b9-7953-1866-857b-85a162ac79e0: aborting pipeline replication: peer="{Voter server-567ddd18-ef01-cc99-f387-34ac0aa95ed2 567ddd18-ef01-cc99-f387-34ac0aa95ed2}" -2026-04-22T10:09:23.355Z [INFO] server-96e251b9-7953-1866-857b-85a162ac79e0: aborting pipeline replication: peer="{Voter server-ccbf10e9-7aef-6a84-6972-1f6c437570f9 ccbf10e9-7aef-6a84-6972-1f6c437570f9}" ---- PASS: TestRaft_JoinNode (0.31s) +2025-03-20T03:49:37.566Z [INFO] server-1afc7cd0-c796-3885-dffa-3ca37caecb89: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1afc7cd0-c796-3885-dffa-3ca37caecb89 Address:1afc7cd0-c796-3885-dffa-3ca37caecb89} {Suffrage:Voter ID:server-f2d0a129-bfa7-24f3-af9f-a1ae2ae9c0b3 Address:f2d0a129-bfa7-24f3-af9f-a1ae2ae9c0b3}]" +2025-03-20T03:49:37.566Z [INFO] server-f2d0a129-bfa7-24f3-af9f-a1ae2ae9c0b3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1afc7cd0-c796-3885-dffa-3ca37caecb89 Address:1afc7cd0-c796-3885-dffa-3ca37caecb89} {Suffrage:Voter ID:server-f2d0a129-bfa7-24f3-af9f-a1ae2ae9c0b3 Address:f2d0a129-bfa7-24f3-af9f-a1ae2ae9c0b3}]" +2025-03-20T03:49:37.566Z [INFO] server-64c0b822-b390-3d5d-4db8-469f8912a176: initial configuration: index=0 servers=[] +2025-03-20T03:49:37.566Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:37.566Z [INFO] server-1afc7cd0-c796-3885-dffa-3ca37caecb89: entering follower state: follower="Node at 1afc7cd0-c796-3885-dffa-3ca37caecb89 [Follower]" leader-address= leader-id= +2025-03-20T03:49:37.567Z [INFO] server-f2d0a129-bfa7-24f3-af9f-a1ae2ae9c0b3: entering follower state: follower="Node at f2d0a129-bfa7-24f3-af9f-a1ae2ae9c0b3 [Follower]" leader-address= leader-id= +2025-03-20T03:49:37.567Z [INFO] server-64c0b822-b390-3d5d-4db8-469f8912a176: entering follower state: follower="Node at 64c0b822-b390-3d5d-4db8-469f8912a176 [Follower]" leader-address= leader-id= +2025-03-20T03:49:37.624Z [WARN] server-1afc7cd0-c796-3885-dffa-3ca37caecb89: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:37.624Z [INFO] server-1afc7cd0-c796-3885-dffa-3ca37caecb89: entering candidate state: node="Node at 1afc7cd0-c796-3885-dffa-3ca37caecb89 [Candidate]" term=2 +2025-03-20T03:49:37.624Z [INFO] server-1afc7cd0-c796-3885-dffa-3ca37caecb89: election won: term=2 tally=2 +2025-03-20T03:49:37.624Z [INFO] server-1afc7cd0-c796-3885-dffa-3ca37caecb89: entering leader state: leader="Node at 1afc7cd0-c796-3885-dffa-3ca37caecb89 [Leader]" +2025-03-20T03:49:37.624Z [INFO] server-1afc7cd0-c796-3885-dffa-3ca37caecb89: added peer, starting replication: peer=server-f2d0a129-bfa7-24f3-af9f-a1ae2ae9c0b3 +2025-03-20T03:49:37.624Z [INFO] server-1afc7cd0-c796-3885-dffa-3ca37caecb89: pipelining replication: peer="{Voter server-f2d0a129-bfa7-24f3-af9f-a1ae2ae9c0b3 f2d0a129-bfa7-24f3-af9f-a1ae2ae9c0b3}" +2025-03-20T03:49:37.650Z [WARN] server-64c0b822-b390-3d5d-4db8-469f8912a176: no known peers, aborting election +2025-03-20T03:49:37.730Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:37.62479812 +0000 UTC m=+9.299251209 (1 nodes), 57.858918ms from start of poll, 58.510227ms from cluster start. Timeout at 2025-03-20 03:49:37.72980606 +0000 UTC m=+9.404258891, 105.007682ms after stability +2025-03-20T03:49:37.730Z [INFO] server-1afc7cd0-c796-3885-dffa-3ca37caecb89: updating configuration: command=AddVoter server-id=server-64c0b822-b390-3d5d-4db8-469f8912a176 server-addr=64c0b822-b390-3d5d-4db8-469f8912a176 servers="[{Suffrage:Voter ID:server-1afc7cd0-c796-3885-dffa-3ca37caecb89 Address:1afc7cd0-c796-3885-dffa-3ca37caecb89} {Suffrage:Voter ID:server-f2d0a129-bfa7-24f3-af9f-a1ae2ae9c0b3 Address:f2d0a129-bfa7-24f3-af9f-a1ae2ae9c0b3} {Suffrage:Voter ID:server-64c0b822-b390-3d5d-4db8-469f8912a176 Address:64c0b822-b390-3d5d-4db8-469f8912a176}]" +2025-03-20T03:49:37.730Z [INFO] server-1afc7cd0-c796-3885-dffa-3ca37caecb89: added peer, starting replication: peer=server-64c0b822-b390-3d5d-4db8-469f8912a176 +2025-03-20T03:49:37.730Z [WARN] server-64c0b822-b390-3d5d-4db8-469f8912a176: failed to get previous log: previous-index=3 last-index=0 error="log not found" +2025-03-20T03:49:37.730Z [WARN] server-1afc7cd0-c796-3885-dffa-3ca37caecb89: appendEntries rejected, sending older logs: peer="{Voter server-64c0b822-b390-3d5d-4db8-469f8912a176 64c0b822-b390-3d5d-4db8-469f8912a176}" next=1 +2025-03-20T03:49:37.730Z [INFO] server-1afc7cd0-c796-3885-dffa-3ca37caecb89: pipelining replication: peer="{Voter server-64c0b822-b390-3d5d-4db8-469f8912a176 64c0b822-b390-3d5d-4db8-469f8912a176}" +2025-03-20T03:49:37.730Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:37.843Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:37.73075811 +0000 UTC m=+9.405211156 (1 nodes), 2.408µs from start of poll, 164.470174ms from cluster start. Timeout at 2025-03-20 03:49:37.835777918 +0000 UTC m=+9.510231074, 105.019918ms after stability +2025-03-20T03:49:37.843Z [INFO] server-1afc7cd0-c796-3885-dffa-3ca37caecb89: aborting pipeline replication: peer="{Voter server-64c0b822-b390-3d5d-4db8-469f8912a176 64c0b822-b390-3d5d-4db8-469f8912a176}" +2025-03-20T03:49:37.843Z [INFO] server-1afc7cd0-c796-3885-dffa-3ca37caecb89: aborting pipeline replication: peer="{Voter server-f2d0a129-bfa7-24f3-af9f-a1ae2ae9c0b3 f2d0a129-bfa7-24f3-af9f-a1ae2ae9c0b3}" +--- PASS: TestRaft_JoinNode (0.29s) === RUN TestRaft_JoinNode_ConfigStore -2026-04-22T10:09:23.356Z [INFO] server-a36bc265-32f0-5640-d74f-621e816ac017: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a36bc265-32f0-5640-d74f-621e816ac017 Address:a36bc265-32f0-5640-d74f-621e816ac017}]" -2026-04-22T10:09:23.356Z [INFO] server-a36bc265-32f0-5640-d74f-621e816ac017: entering follower state: follower="Node at a36bc265-32f0-5640-d74f-621e816ac017 [Follower]" leader-address= leader-id= -2026-04-22T10:09:23.356Z [INFO] server-9ba5cedf-8650-8379-8f7b-29b22e9fed8e: initial configuration: index=0 servers=[] -2026-04-22T10:09:23.357Z [INFO] server-f68e78de-7705-7c56-6e40-d5fbcf734a77: initial configuration: index=0 servers=[] -2026-04-22T10:09:23.357Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:23.357Z [INFO] server-9ba5cedf-8650-8379-8f7b-29b22e9fed8e: entering follower state: follower="Node at 9ba5cedf-8650-8379-8f7b-29b22e9fed8e [Follower]" leader-address= leader-id= -2026-04-22T10:09:23.357Z [INFO] server-f68e78de-7705-7c56-6e40-d5fbcf734a77: entering follower state: follower="Node at f68e78de-7705-7c56-6e40-d5fbcf734a77 [Follower]" leader-address= leader-id= -2026-04-22T10:09:23.417Z [WARN] server-a36bc265-32f0-5640-d74f-621e816ac017: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:23.417Z [INFO] server-a36bc265-32f0-5640-d74f-621e816ac017: entering candidate state: node="Node at a36bc265-32f0-5640-d74f-621e816ac017 [Candidate]" term=2 -2026-04-22T10:09:23.417Z [INFO] server-a36bc265-32f0-5640-d74f-621e816ac017: election won: term=2 tally=1 -2026-04-22T10:09:23.417Z [INFO] server-a36bc265-32f0-5640-d74f-621e816ac017: entering leader state: leader="Node at a36bc265-32f0-5640-d74f-621e816ac017 [Leader]" -2026-04-22T10:09:23.428Z [WARN] server-f68e78de-7705-7c56-6e40-d5fbcf734a77: no known peers, aborting election -2026-04-22T10:09:23.450Z [WARN] server-9ba5cedf-8650-8379-8f7b-29b22e9fed8e: no known peers, aborting election -2026-04-22T10:09:23.525Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:23.417901622 +0000 UTC m=+9.441697713 (1 nodes), 60.750933ms from start of poll, 61.480608ms from cluster start. Timeout at 2026-04-22 10:09:23.522904335 +0000 UTC m=+9.546700500, 105.002787ms after stability -2026-04-22T10:09:23.525Z [INFO] server-a36bc265-32f0-5640-d74f-621e816ac017: updating configuration: command=AddVoter server-id=server-9ba5cedf-8650-8379-8f7b-29b22e9fed8e server-addr=9ba5cedf-8650-8379-8f7b-29b22e9fed8e servers="[{Suffrage:Voter ID:server-a36bc265-32f0-5640-d74f-621e816ac017 Address:a36bc265-32f0-5640-d74f-621e816ac017} {Suffrage:Voter ID:server-9ba5cedf-8650-8379-8f7b-29b22e9fed8e Address:9ba5cedf-8650-8379-8f7b-29b22e9fed8e}]" -2026-04-22T10:09:23.525Z [INFO] server-a36bc265-32f0-5640-d74f-621e816ac017: added peer, starting replication: peer=server-9ba5cedf-8650-8379-8f7b-29b22e9fed8e -2026-04-22T10:09:23.526Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:23.526Z [WARN] server-9ba5cedf-8650-8379-8f7b-29b22e9fed8e: failed to get previous log: previous-index=3 last-index=0 error="log not found" -2026-04-22T10:09:23.526Z [WARN] server-a36bc265-32f0-5640-d74f-621e816ac017: appendEntries rejected, sending older logs: peer="{Voter server-9ba5cedf-8650-8379-8f7b-29b22e9fed8e 9ba5cedf-8650-8379-8f7b-29b22e9fed8e}" next=1 -2026-04-22T10:09:23.526Z [INFO] server-a36bc265-32f0-5640-d74f-621e816ac017: pipelining replication: peer="{Voter server-9ba5cedf-8650-8379-8f7b-29b22e9fed8e 9ba5cedf-8650-8379-8f7b-29b22e9fed8e}" -2026-04-22T10:09:23.631Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:23.526050361 +0000 UTC m=+9.549846472 (1 nodes), 917ns from start of poll, 169.629367ms from cluster start. Timeout at 2026-04-22 10:09:23.631054926 +0000 UTC m=+9.654851017, 105.004545ms after stability -2026-04-22T10:09:23.631Z [INFO] server-a36bc265-32f0-5640-d74f-621e816ac017: updating configuration: command=AddVoter server-id=server-f68e78de-7705-7c56-6e40-d5fbcf734a77 server-addr=f68e78de-7705-7c56-6e40-d5fbcf734a77 servers="[{Suffrage:Voter ID:server-a36bc265-32f0-5640-d74f-621e816ac017 Address:a36bc265-32f0-5640-d74f-621e816ac017} {Suffrage:Voter ID:server-9ba5cedf-8650-8379-8f7b-29b22e9fed8e Address:9ba5cedf-8650-8379-8f7b-29b22e9fed8e} {Suffrage:Voter ID:server-f68e78de-7705-7c56-6e40-d5fbcf734a77 Address:f68e78de-7705-7c56-6e40-d5fbcf734a77}]" -2026-04-22T10:09:23.631Z [INFO] server-a36bc265-32f0-5640-d74f-621e816ac017: added peer, starting replication: peer=server-f68e78de-7705-7c56-6e40-d5fbcf734a77 -2026-04-22T10:09:23.631Z [WARN] server-f68e78de-7705-7c56-6e40-d5fbcf734a77: failed to get previous log: previous-index=4 last-index=0 error="log not found" -2026-04-22T10:09:23.631Z [WARN] server-a36bc265-32f0-5640-d74f-621e816ac017: appendEntries rejected, sending older logs: peer="{Voter server-f68e78de-7705-7c56-6e40-d5fbcf734a77 f68e78de-7705-7c56-6e40-d5fbcf734a77}" next=1 -2026-04-22T10:09:23.631Z [INFO] server-a36bc265-32f0-5640-d74f-621e816ac017: pipelining replication: peer="{Voter server-f68e78de-7705-7c56-6e40-d5fbcf734a77 f68e78de-7705-7c56-6e40-d5fbcf734a77}" -2026-04-22T10:09:23.631Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:23.737Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:23.631843657 +0000 UTC m=+9.655639751 (1 nodes), 900ns from start of poll, 275.422646ms from cluster start. Timeout at 2026-04-22 10:09:23.736849225 +0000 UTC m=+9.760645323, 105.005572ms after stability -2026-04-22T10:09:23.737Z [INFO] server-a36bc265-32f0-5640-d74f-621e816ac017: aborting pipeline replication: peer="{Voter server-9ba5cedf-8650-8379-8f7b-29b22e9fed8e 9ba5cedf-8650-8379-8f7b-29b22e9fed8e}" -2026-04-22T10:09:23.737Z [INFO] server-a36bc265-32f0-5640-d74f-621e816ac017: aborting pipeline replication: peer="{Voter server-f68e78de-7705-7c56-6e40-d5fbcf734a77 f68e78de-7705-7c56-6e40-d5fbcf734a77}" ---- PASS: TestRaft_JoinNode_ConfigStore (0.38s) +2025-03-20T03:49:37.853Z [INFO] server-57a81e1c-144b-bdad-f9b4-925e244cdbd5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-57a81e1c-144b-bdad-f9b4-925e244cdbd5 Address:57a81e1c-144b-bdad-f9b4-925e244cdbd5}]" +2025-03-20T03:49:37.853Z [INFO] server-57a81e1c-144b-bdad-f9b4-925e244cdbd5: entering follower state: follower="Node at 57a81e1c-144b-bdad-f9b4-925e244cdbd5 [Follower]" leader-address= leader-id= +2025-03-20T03:49:37.853Z [INFO] server-56e2389f-2322-7c96-f862-848e825377a0: initial configuration: index=0 servers=[] +2025-03-20T03:49:37.853Z [INFO] server-56e2389f-2322-7c96-f862-848e825377a0: entering follower state: follower="Node at 56e2389f-2322-7c96-f862-848e825377a0 [Follower]" leader-address= leader-id= +2025-03-20T03:49:37.854Z [INFO] server-62209364-f0f9-3248-3462-d6d28f14f6f3: initial configuration: index=0 servers=[] +2025-03-20T03:49:37.854Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:37.854Z [INFO] server-62209364-f0f9-3248-3462-d6d28f14f6f3: entering follower state: follower="Node at 62209364-f0f9-3248-3462-d6d28f14f6f3 [Follower]" leader-address= leader-id= +2025-03-20T03:49:37.917Z [WARN] server-62209364-f0f9-3248-3462-d6d28f14f6f3: no known peers, aborting election +2025-03-20T03:49:37.932Z [WARN] server-56e2389f-2322-7c96-f862-848e825377a0: no known peers, aborting election +2025-03-20T03:49:37.935Z [WARN] server-57a81e1c-144b-bdad-f9b4-925e244cdbd5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:37.935Z [INFO] server-57a81e1c-144b-bdad-f9b4-925e244cdbd5: entering candidate state: node="Node at 57a81e1c-144b-bdad-f9b4-925e244cdbd5 [Candidate]" term=2 +2025-03-20T03:49:37.935Z [INFO] server-57a81e1c-144b-bdad-f9b4-925e244cdbd5: election won: term=2 tally=1 +2025-03-20T03:49:37.935Z [INFO] server-57a81e1c-144b-bdad-f9b4-925e244cdbd5: entering leader state: leader="Node at 57a81e1c-144b-bdad-f9b4-925e244cdbd5 [Leader]" +2025-03-20T03:49:38.040Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:37.935648918 +0000 UTC m=+9.610101901 (1 nodes), 81.442598ms from start of poll, 82.382272ms from cluster start. Timeout at 2025-03-20 03:49:38.040664456 +0000 UTC m=+9.715118094, 105.016193ms after stability +2025-03-20T03:49:38.041Z [INFO] server-57a81e1c-144b-bdad-f9b4-925e244cdbd5: updating configuration: command=AddVoter server-id=server-56e2389f-2322-7c96-f862-848e825377a0 server-addr=56e2389f-2322-7c96-f862-848e825377a0 servers="[{Suffrage:Voter ID:server-57a81e1c-144b-bdad-f9b4-925e244cdbd5 Address:57a81e1c-144b-bdad-f9b4-925e244cdbd5} {Suffrage:Voter ID:server-56e2389f-2322-7c96-f862-848e825377a0 Address:56e2389f-2322-7c96-f862-848e825377a0}]" +2025-03-20T03:49:38.041Z [INFO] server-57a81e1c-144b-bdad-f9b4-925e244cdbd5: added peer, starting replication: peer=server-56e2389f-2322-7c96-f862-848e825377a0 +2025-03-20T03:49:38.041Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:38.044Z [WARN] server-56e2389f-2322-7c96-f862-848e825377a0: failed to get previous log: previous-index=3 last-index=0 error="log not found" +2025-03-20T03:49:38.044Z [WARN] server-57a81e1c-144b-bdad-f9b4-925e244cdbd5: appendEntries rejected, sending older logs: peer="{Voter server-56e2389f-2322-7c96-f862-848e825377a0 56e2389f-2322-7c96-f862-848e825377a0}" next=1 +2025-03-20T03:49:38.044Z [INFO] server-57a81e1c-144b-bdad-f9b4-925e244cdbd5: pipelining replication: peer="{Voter server-56e2389f-2322-7c96-f862-848e825377a0 56e2389f-2322-7c96-f862-848e825377a0}" +2025-03-20T03:49:38.150Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:38.041599473 +0000 UTC m=+9.716052787 (1 nodes), 3.4µs from start of poll, 188.333158ms from cluster start. Timeout at 2025-03-20 03:49:38.14661359 +0000 UTC m=+9.821066474, 105.013687ms after stability +2025-03-20T03:49:38.150Z [INFO] server-57a81e1c-144b-bdad-f9b4-925e244cdbd5: updating configuration: command=AddVoter server-id=server-62209364-f0f9-3248-3462-d6d28f14f6f3 server-addr=62209364-f0f9-3248-3462-d6d28f14f6f3 servers="[{Suffrage:Voter ID:server-57a81e1c-144b-bdad-f9b4-925e244cdbd5 Address:57a81e1c-144b-bdad-f9b4-925e244cdbd5} {Suffrage:Voter ID:server-56e2389f-2322-7c96-f862-848e825377a0 Address:56e2389f-2322-7c96-f862-848e825377a0} {Suffrage:Voter ID:server-62209364-f0f9-3248-3462-d6d28f14f6f3 Address:62209364-f0f9-3248-3462-d6d28f14f6f3}]" +2025-03-20T03:49:38.150Z [INFO] server-57a81e1c-144b-bdad-f9b4-925e244cdbd5: added peer, starting replication: peer=server-62209364-f0f9-3248-3462-d6d28f14f6f3 +2025-03-20T03:49:38.150Z [WARN] server-62209364-f0f9-3248-3462-d6d28f14f6f3: failed to get previous log: previous-index=4 last-index=0 error="log not found" +2025-03-20T03:49:38.150Z [WARN] server-57a81e1c-144b-bdad-f9b4-925e244cdbd5: appendEntries rejected, sending older logs: peer="{Voter server-62209364-f0f9-3248-3462-d6d28f14f6f3 62209364-f0f9-3248-3462-d6d28f14f6f3}" next=1 +2025-03-20T03:49:38.150Z [INFO] server-57a81e1c-144b-bdad-f9b4-925e244cdbd5: pipelining replication: peer="{Voter server-62209364-f0f9-3248-3462-d6d28f14f6f3 62209364-f0f9-3248-3462-d6d28f14f6f3}" +2025-03-20T03:49:38.150Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:38.256Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:38.150909025 +0000 UTC m=+9.825361819 (1 nodes), 2.06µs from start of poll, 297.64219ms from cluster start. Timeout at 2025-03-20 03:49:38.255924444 +0000 UTC m=+9.930377429, 105.01561ms after stability +2025-03-20T03:49:38.256Z [INFO] server-57a81e1c-144b-bdad-f9b4-925e244cdbd5: aborting pipeline replication: peer="{Voter server-62209364-f0f9-3248-3462-d6d28f14f6f3 62209364-f0f9-3248-3462-d6d28f14f6f3}" +2025-03-20T03:49:38.256Z [INFO] server-57a81e1c-144b-bdad-f9b4-925e244cdbd5: aborting pipeline replication: peer="{Voter server-56e2389f-2322-7c96-f862-848e825377a0 56e2389f-2322-7c96-f862-848e825377a0}" +--- PASS: TestRaft_JoinNode_ConfigStore (0.40s) === RUN TestRaft_RemoveFollower -2026-04-22T10:09:23.738Z [INFO] server-9de23053-a6c2-12dd-2043-fdc4836b95a7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9de23053-a6c2-12dd-2043-fdc4836b95a7 Address:9de23053-a6c2-12dd-2043-fdc4836b95a7} {Suffrage:Voter ID:server-7834b15f-f109-44af-0c6d-d5d6decc54f2 Address:7834b15f-f109-44af-0c6d-d5d6decc54f2} {Suffrage:Voter ID:server-5fc94849-62b8-4edf-f429-a44e6c269562 Address:5fc94849-62b8-4edf-f429-a44e6c269562}]" -2026-04-22T10:09:23.738Z [INFO] server-9de23053-a6c2-12dd-2043-fdc4836b95a7: entering follower state: follower="Node at 9de23053-a6c2-12dd-2043-fdc4836b95a7 [Follower]" leader-address= leader-id= -2026-04-22T10:09:23.738Z [INFO] server-7834b15f-f109-44af-0c6d-d5d6decc54f2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9de23053-a6c2-12dd-2043-fdc4836b95a7 Address:9de23053-a6c2-12dd-2043-fdc4836b95a7} {Suffrage:Voter ID:server-7834b15f-f109-44af-0c6d-d5d6decc54f2 Address:7834b15f-f109-44af-0c6d-d5d6decc54f2} {Suffrage:Voter ID:server-5fc94849-62b8-4edf-f429-a44e6c269562 Address:5fc94849-62b8-4edf-f429-a44e6c269562}]" -2026-04-22T10:09:23.738Z [INFO] server-7834b15f-f109-44af-0c6d-d5d6decc54f2: entering follower state: follower="Node at 7834b15f-f109-44af-0c6d-d5d6decc54f2 [Follower]" leader-address= leader-id= -2026-04-22T10:09:23.738Z [INFO] server-5fc94849-62b8-4edf-f429-a44e6c269562: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9de23053-a6c2-12dd-2043-fdc4836b95a7 Address:9de23053-a6c2-12dd-2043-fdc4836b95a7} {Suffrage:Voter ID:server-7834b15f-f109-44af-0c6d-d5d6decc54f2 Address:7834b15f-f109-44af-0c6d-d5d6decc54f2} {Suffrage:Voter ID:server-5fc94849-62b8-4edf-f429-a44e6c269562 Address:5fc94849-62b8-4edf-f429-a44e6c269562}]" -2026-04-22T10:09:23.738Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:23.738Z [INFO] server-5fc94849-62b8-4edf-f429-a44e6c269562: entering follower state: follower="Node at 5fc94849-62b8-4edf-f429-a44e6c269562 [Follower]" leader-address= leader-id= -2026-04-22T10:09:23.790Z [WARN] server-5fc94849-62b8-4edf-f429-a44e6c269562: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:23.790Z [INFO] server-5fc94849-62b8-4edf-f429-a44e6c269562: entering candidate state: node="Node at 5fc94849-62b8-4edf-f429-a44e6c269562 [Candidate]" term=2 -2026-04-22T10:09:23.790Z [INFO] server-5fc94849-62b8-4edf-f429-a44e6c269562: election won: term=2 tally=2 -2026-04-22T10:09:23.790Z [INFO] server-5fc94849-62b8-4edf-f429-a44e6c269562: entering leader state: leader="Node at 5fc94849-62b8-4edf-f429-a44e6c269562 [Leader]" -2026-04-22T10:09:23.790Z [INFO] server-5fc94849-62b8-4edf-f429-a44e6c269562: added peer, starting replication: peer=server-9de23053-a6c2-12dd-2043-fdc4836b95a7 -2026-04-22T10:09:23.790Z [INFO] server-5fc94849-62b8-4edf-f429-a44e6c269562: added peer, starting replication: peer=server-7834b15f-f109-44af-0c6d-d5d6decc54f2 -2026-04-22T10:09:23.791Z [INFO] server-5fc94849-62b8-4edf-f429-a44e6c269562: pipelining replication: peer="{Voter server-7834b15f-f109-44af-0c6d-d5d6decc54f2 7834b15f-f109-44af-0c6d-d5d6decc54f2}" -2026-04-22T10:09:23.791Z [INFO] server-5fc94849-62b8-4edf-f429-a44e6c269562: pipelining replication: peer="{Voter server-9de23053-a6c2-12dd-2043-fdc4836b95a7 9de23053-a6c2-12dd-2043-fdc4836b95a7}" -2026-04-22T10:09:23.896Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:23.791202092 +0000 UTC m=+9.814998202 (1 nodes), 52.288578ms from start of poll, 52.555654ms from cluster start. Timeout at 2026-04-22 10:09:23.896203253 +0000 UTC m=+9.919999340, 105.001138ms after stability -2026-04-22T10:09:23.901Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:09:24.008Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:09:23.901923745 +0000 UTC m=+9.925719838 (2 nodes), 1.353µs from start of poll, 163.27729ms from cluster start. Timeout at 2026-04-22 10:09:24.006932065 +0000 UTC m=+10.030728171, 105.008333ms after stability -2026-04-22T10:09:24.008Z [INFO] server-5fc94849-62b8-4edf-f429-a44e6c269562: updating configuration: command=RemoveServer server-id=server-9de23053-a6c2-12dd-2043-fdc4836b95a7 server-addr= servers="[{Suffrage:Voter ID:server-7834b15f-f109-44af-0c6d-d5d6decc54f2 Address:7834b15f-f109-44af-0c6d-d5d6decc54f2} {Suffrage:Voter ID:server-5fc94849-62b8-4edf-f429-a44e6c269562 Address:5fc94849-62b8-4edf-f429-a44e6c269562}]" -2026-04-22T10:09:24.008Z [INFO] server-5fc94849-62b8-4edf-f429-a44e6c269562: removed peer, stopping replication: peer=server-9de23053-a6c2-12dd-2043-fdc4836b95a7 last-index=3 -2026-04-22T10:09:24.008Z [INFO] server-5fc94849-62b8-4edf-f429-a44e6c269562: aborting pipeline replication: peer="{Voter server-9de23053-a6c2-12dd-2043-fdc4836b95a7 9de23053-a6c2-12dd-2043-fdc4836b95a7}" -2026-04-22T10:09:24.084Z [WARN] server-9de23053-a6c2-12dd-2043-fdc4836b95a7: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election -2026-04-22T10:09:24.114Z [INFO] server-5fc94849-62b8-4edf-f429-a44e6c269562: aborting pipeline replication: peer="{Voter server-7834b15f-f109-44af-0c6d-d5d6decc54f2 7834b15f-f109-44af-0c6d-d5d6decc54f2}" ---- PASS: TestRaft_RemoveFollower (0.38s) +2025-03-20T03:49:38.276Z [INFO] server-dab63d02-b84a-af34-8953-9e04165edcce: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dab63d02-b84a-af34-8953-9e04165edcce Address:dab63d02-b84a-af34-8953-9e04165edcce} {Suffrage:Voter ID:server-2d0b610d-aac7-24eb-c779-dac315ae1cda Address:2d0b610d-aac7-24eb-c779-dac315ae1cda} {Suffrage:Voter ID:server-c754b242-79da-4b2b-f269-5db8d3769e6b Address:c754b242-79da-4b2b-f269-5db8d3769e6b}]" +2025-03-20T03:49:38.277Z [INFO] server-2d0b610d-aac7-24eb-c779-dac315ae1cda: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dab63d02-b84a-af34-8953-9e04165edcce Address:dab63d02-b84a-af34-8953-9e04165edcce} {Suffrage:Voter ID:server-2d0b610d-aac7-24eb-c779-dac315ae1cda Address:2d0b610d-aac7-24eb-c779-dac315ae1cda} {Suffrage:Voter ID:server-c754b242-79da-4b2b-f269-5db8d3769e6b Address:c754b242-79da-4b2b-f269-5db8d3769e6b}]" +2025-03-20T03:49:38.277Z [INFO] server-c754b242-79da-4b2b-f269-5db8d3769e6b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dab63d02-b84a-af34-8953-9e04165edcce Address:dab63d02-b84a-af34-8953-9e04165edcce} {Suffrage:Voter ID:server-2d0b610d-aac7-24eb-c779-dac315ae1cda Address:2d0b610d-aac7-24eb-c779-dac315ae1cda} {Suffrage:Voter ID:server-c754b242-79da-4b2b-f269-5db8d3769e6b Address:c754b242-79da-4b2b-f269-5db8d3769e6b}]" +2025-03-20T03:49:38.277Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:38.277Z [INFO] server-dab63d02-b84a-af34-8953-9e04165edcce: entering follower state: follower="Node at dab63d02-b84a-af34-8953-9e04165edcce [Follower]" leader-address= leader-id= +2025-03-20T03:49:38.277Z [INFO] server-2d0b610d-aac7-24eb-c779-dac315ae1cda: entering follower state: follower="Node at 2d0b610d-aac7-24eb-c779-dac315ae1cda [Follower]" leader-address= leader-id= +2025-03-20T03:49:38.277Z [INFO] server-c754b242-79da-4b2b-f269-5db8d3769e6b: entering follower state: follower="Node at c754b242-79da-4b2b-f269-5db8d3769e6b [Follower]" leader-address= leader-id= +2025-03-20T03:49:38.347Z [WARN] server-dab63d02-b84a-af34-8953-9e04165edcce: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:38.347Z [INFO] server-dab63d02-b84a-af34-8953-9e04165edcce: entering candidate state: node="Node at dab63d02-b84a-af34-8953-9e04165edcce [Candidate]" term=2 +2025-03-20T03:49:38.347Z [INFO] server-dab63d02-b84a-af34-8953-9e04165edcce: election won: term=2 tally=2 +2025-03-20T03:49:38.347Z [INFO] server-dab63d02-b84a-af34-8953-9e04165edcce: entering leader state: leader="Node at dab63d02-b84a-af34-8953-9e04165edcce [Leader]" +2025-03-20T03:49:38.347Z [INFO] server-dab63d02-b84a-af34-8953-9e04165edcce: added peer, starting replication: peer=server-2d0b610d-aac7-24eb-c779-dac315ae1cda +2025-03-20T03:49:38.347Z [INFO] server-dab63d02-b84a-af34-8953-9e04165edcce: added peer, starting replication: peer=server-c754b242-79da-4b2b-f269-5db8d3769e6b +2025-03-20T03:49:38.348Z [INFO] server-dab63d02-b84a-af34-8953-9e04165edcce: pipelining replication: peer="{Voter server-c754b242-79da-4b2b-f269-5db8d3769e6b c754b242-79da-4b2b-f269-5db8d3769e6b}" +2025-03-20T03:49:38.348Z [INFO] server-dab63d02-b84a-af34-8953-9e04165edcce: pipelining replication: peer="{Voter server-2d0b610d-aac7-24eb-c779-dac315ae1cda 2d0b610d-aac7-24eb-c779-dac315ae1cda}" +2025-03-20T03:49:38.454Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:38.348407164 +0000 UTC m=+10.022859953 (1 nodes), 71.056256ms from start of poll, 71.670629ms from cluster start. Timeout at 2025-03-20 03:49:38.453411022 +0000 UTC m=+10.127864054, 105.004101ms after stability +2025-03-20T03:49:38.460Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:49:38.565Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:49:38.460109492 +0000 UTC m=+10.134562424 (2 nodes), 2.788µs from start of poll, 183.3731ms from cluster start. Timeout at 2025-03-20 03:49:38.565119913 +0000 UTC m=+10.239572803, 105.010379ms after stability +2025-03-20T03:49:38.565Z [INFO] server-dab63d02-b84a-af34-8953-9e04165edcce: updating configuration: command=RemoveServer server-id=server-2d0b610d-aac7-24eb-c779-dac315ae1cda server-addr= servers="[{Suffrage:Voter ID:server-dab63d02-b84a-af34-8953-9e04165edcce Address:dab63d02-b84a-af34-8953-9e04165edcce} {Suffrage:Voter ID:server-c754b242-79da-4b2b-f269-5db8d3769e6b Address:c754b242-79da-4b2b-f269-5db8d3769e6b}]" +2025-03-20T03:49:38.565Z [INFO] server-dab63d02-b84a-af34-8953-9e04165edcce: removed peer, stopping replication: peer=server-2d0b610d-aac7-24eb-c779-dac315ae1cda last-index=3 +2025-03-20T03:49:38.565Z [INFO] server-dab63d02-b84a-af34-8953-9e04165edcce: aborting pipeline replication: peer="{Voter server-2d0b610d-aac7-24eb-c779-dac315ae1cda 2d0b610d-aac7-24eb-c779-dac315ae1cda}" +2025-03-20T03:49:38.646Z [WARN] server-2d0b610d-aac7-24eb-c779-dac315ae1cda: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +2025-03-20T03:49:38.674Z [INFO] server-dab63d02-b84a-af34-8953-9e04165edcce: aborting pipeline replication: peer="{Voter server-c754b242-79da-4b2b-f269-5db8d3769e6b c754b242-79da-4b2b-f269-5db8d3769e6b}" +--- PASS: TestRaft_RemoveFollower (0.42s) === RUN TestRaft_RemoveLeader -2026-04-22T10:09:24.115Z [INFO] server-6e2a8005-70f4-abd1-fde9-5711dc1e7806: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6e2a8005-70f4-abd1-fde9-5711dc1e7806 Address:6e2a8005-70f4-abd1-fde9-5711dc1e7806} {Suffrage:Voter ID:server-e95d6517-f700-823c-e155-07c8d319f1bf Address:e95d6517-f700-823c-e155-07c8d319f1bf} {Suffrage:Voter ID:server-dd64a51e-eb28-25e2-7c64-f19ade66bb87 Address:dd64a51e-eb28-25e2-7c64-f19ade66bb87}]" -2026-04-22T10:09:24.115Z [INFO] server-6e2a8005-70f4-abd1-fde9-5711dc1e7806: entering follower state: follower="Node at 6e2a8005-70f4-abd1-fde9-5711dc1e7806 [Follower]" leader-address= leader-id= -2026-04-22T10:09:24.115Z [INFO] server-e95d6517-f700-823c-e155-07c8d319f1bf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6e2a8005-70f4-abd1-fde9-5711dc1e7806 Address:6e2a8005-70f4-abd1-fde9-5711dc1e7806} {Suffrage:Voter ID:server-e95d6517-f700-823c-e155-07c8d319f1bf Address:e95d6517-f700-823c-e155-07c8d319f1bf} {Suffrage:Voter ID:server-dd64a51e-eb28-25e2-7c64-f19ade66bb87 Address:dd64a51e-eb28-25e2-7c64-f19ade66bb87}]" -2026-04-22T10:09:24.115Z [INFO] server-e95d6517-f700-823c-e155-07c8d319f1bf: entering follower state: follower="Node at e95d6517-f700-823c-e155-07c8d319f1bf [Follower]" leader-address= leader-id= -2026-04-22T10:09:24.115Z [INFO] server-dd64a51e-eb28-25e2-7c64-f19ade66bb87: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6e2a8005-70f4-abd1-fde9-5711dc1e7806 Address:6e2a8005-70f4-abd1-fde9-5711dc1e7806} {Suffrage:Voter ID:server-e95d6517-f700-823c-e155-07c8d319f1bf Address:e95d6517-f700-823c-e155-07c8d319f1bf} {Suffrage:Voter ID:server-dd64a51e-eb28-25e2-7c64-f19ade66bb87 Address:dd64a51e-eb28-25e2-7c64-f19ade66bb87}]" -2026-04-22T10:09:24.115Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:24.116Z [INFO] server-dd64a51e-eb28-25e2-7c64-f19ade66bb87: entering follower state: follower="Node at dd64a51e-eb28-25e2-7c64-f19ade66bb87 [Follower]" leader-address= leader-id= -2026-04-22T10:09:24.171Z [WARN] server-e95d6517-f700-823c-e155-07c8d319f1bf: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:24.171Z [INFO] server-e95d6517-f700-823c-e155-07c8d319f1bf: entering candidate state: node="Node at e95d6517-f700-823c-e155-07c8d319f1bf [Candidate]" term=2 -2026-04-22T10:09:24.171Z [INFO] server-e95d6517-f700-823c-e155-07c8d319f1bf: election won: term=2 tally=2 -2026-04-22T10:09:24.171Z [INFO] server-e95d6517-f700-823c-e155-07c8d319f1bf: entering leader state: leader="Node at e95d6517-f700-823c-e155-07c8d319f1bf [Leader]" -2026-04-22T10:09:24.171Z [INFO] server-e95d6517-f700-823c-e155-07c8d319f1bf: added peer, starting replication: peer=server-6e2a8005-70f4-abd1-fde9-5711dc1e7806 -2026-04-22T10:09:24.171Z [INFO] server-e95d6517-f700-823c-e155-07c8d319f1bf: added peer, starting replication: peer=server-dd64a51e-eb28-25e2-7c64-f19ade66bb87 -2026-04-22T10:09:24.171Z [INFO] server-e95d6517-f700-823c-e155-07c8d319f1bf: pipelining replication: peer="{Voter server-dd64a51e-eb28-25e2-7c64-f19ade66bb87 dd64a51e-eb28-25e2-7c64-f19ade66bb87}" -2026-04-22T10:09:24.171Z [INFO] server-e95d6517-f700-823c-e155-07c8d319f1bf: pipelining replication: peer="{Voter server-6e2a8005-70f4-abd1-fde9-5711dc1e7806 6e2a8005-70f4-abd1-fde9-5711dc1e7806}" -2026-04-22T10:09:24.277Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:24.171511382 +0000 UTC m=+10.195307528 (1 nodes), 55.503459ms from start of poll, 55.899183ms from cluster start. Timeout at 2026-04-22 10:09:24.276515385 +0000 UTC m=+10.300311468, 105.00394ms after stability -2026-04-22T10:09:24.282Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:09:24.388Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:09:24.282487251 +0000 UTC m=+10.306283368 (2 nodes), 1.234µs from start of poll, 166.875023ms from cluster start. Timeout at 2026-04-22 10:09:24.387505891 +0000 UTC m=+10.411302022, 105.018654ms after stability -2026-04-22T10:09:24.388Z [INFO] server-e95d6517-f700-823c-e155-07c8d319f1bf: updating configuration: command=RemoveServer server-id=server-e95d6517-f700-823c-e155-07c8d319f1bf server-addr= servers="[{Suffrage:Voter ID:server-6e2a8005-70f4-abd1-fde9-5711dc1e7806 Address:6e2a8005-70f4-abd1-fde9-5711dc1e7806} {Suffrage:Voter ID:server-dd64a51e-eb28-25e2-7c64-f19ade66bb87 Address:dd64a51e-eb28-25e2-7c64-f19ade66bb87}]" -2026-04-22T10:09:24.388Z [INFO] server-e95d6517-f700-823c-e155-07c8d319f1bf: removed ourself, shutting down -2026-04-22T10:09:24.388Z [INFO] server-e95d6517-f700-823c-e155-07c8d319f1bf: aborting pipeline replication: peer="{Voter server-dd64a51e-eb28-25e2-7c64-f19ade66bb87 dd64a51e-eb28-25e2-7c64-f19ade66bb87}" -2026-04-22T10:09:24.389Z [INFO] server-e95d6517-f700-823c-e155-07c8d319f1bf: aborting pipeline replication: peer="{Voter server-6e2a8005-70f4-abd1-fde9-5711dc1e7806 6e2a8005-70f4-abd1-fde9-5711dc1e7806}" -2026-04-22T10:09:24.477Z [WARN] server-6e2a8005-70f4-abd1-fde9-5711dc1e7806: heartbeat timeout reached, starting election: last-leader-addr=e95d6517-f700-823c-e155-07c8d319f1bf last-leader-id=server-e95d6517-f700-823c-e155-07c8d319f1bf -2026-04-22T10:09:24.477Z [INFO] server-6e2a8005-70f4-abd1-fde9-5711dc1e7806: entering candidate state: node="Node at 6e2a8005-70f4-abd1-fde9-5711dc1e7806 [Candidate]" term=3 -2026-04-22T10:09:24.477Z [WARN] server-dd64a51e-eb28-25e2-7c64-f19ade66bb87: rejecting vote request since we have a leader: from=6e2a8005-70f4-abd1-fde9-5711dc1e7806 leader=e95d6517-f700-823c-e155-07c8d319f1bf leader-id=server-e95d6517-f700-823c-e155-07c8d319f1bf -2026-04-22T10:09:24.491Z [WARN] server-dd64a51e-eb28-25e2-7c64-f19ade66bb87: heartbeat timeout reached, starting election: last-leader-addr=e95d6517-f700-823c-e155-07c8d319f1bf last-leader-id=server-e95d6517-f700-823c-e155-07c8d319f1bf -2026-04-22T10:09:24.491Z [INFO] server-dd64a51e-eb28-25e2-7c64-f19ade66bb87: entering candidate state: node="Node at dd64a51e-eb28-25e2-7c64-f19ade66bb87 [Candidate]" term=3 -2026-04-22T10:09:24.491Z [INFO] server-6e2a8005-70f4-abd1-fde9-5711dc1e7806: duplicate requestVote for same term: term=3 -2026-04-22T10:09:24.544Z [WARN] server-6e2a8005-70f4-abd1-fde9-5711dc1e7806: Election timeout reached, restarting election -2026-04-22T10:09:24.544Z [INFO] server-6e2a8005-70f4-abd1-fde9-5711dc1e7806: entering candidate state: node="Node at 6e2a8005-70f4-abd1-fde9-5711dc1e7806 [Candidate]" term=4 -2026-04-22T10:09:24.544Z [INFO] server-dd64a51e-eb28-25e2-7c64-f19ade66bb87: entering follower state: follower="Node at dd64a51e-eb28-25e2-7c64-f19ade66bb87 [Follower]" leader-address= leader-id= -2026-04-22T10:09:24.544Z [INFO] server-6e2a8005-70f4-abd1-fde9-5711dc1e7806: election won: term=4 tally=2 -2026-04-22T10:09:24.544Z [INFO] server-6e2a8005-70f4-abd1-fde9-5711dc1e7806: entering leader state: leader="Node at 6e2a8005-70f4-abd1-fde9-5711dc1e7806 [Leader]" -2026-04-22T10:09:24.544Z [INFO] server-6e2a8005-70f4-abd1-fde9-5711dc1e7806: added peer, starting replication: peer=server-dd64a51e-eb28-25e2-7c64-f19ade66bb87 -2026-04-22T10:09:24.545Z [INFO] server-6e2a8005-70f4-abd1-fde9-5711dc1e7806: pipelining replication: peer="{Voter server-dd64a51e-eb28-25e2-7c64-f19ade66bb87 dd64a51e-eb28-25e2-7c64-f19ade66bb87}" -2026-04-22T10:09:24.602Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:24.708Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:24.602897401 +0000 UTC m=+10.626693504 (1 nodes), 51.211µs from start of poll, 487.285159ms from cluster start. Timeout at 2026-04-22 10:09:24.707926939 +0000 UTC m=+10.731723062, 105.029558ms after stability -2026-04-22T10:09:24.708Z [INFO] server-6e2a8005-70f4-abd1-fde9-5711dc1e7806: aborting pipeline replication: peer="{Voter server-dd64a51e-eb28-25e2-7c64-f19ade66bb87 dd64a51e-eb28-25e2-7c64-f19ade66bb87}" ---- PASS: TestRaft_RemoveLeader (0.60s) +2025-03-20T03:49:38.676Z [INFO] server-74ba5650-439e-11d6-8fef-f6cd707ca26b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-74ba5650-439e-11d6-8fef-f6cd707ca26b Address:74ba5650-439e-11d6-8fef-f6cd707ca26b} {Suffrage:Voter ID:server-d963fe94-bef1-318c-a918-b9dc45c9fb2f Address:d963fe94-bef1-318c-a918-b9dc45c9fb2f} {Suffrage:Voter ID:server-f7766b6e-71b1-166c-1067-c52bdfbda41e Address:f7766b6e-71b1-166c-1067-c52bdfbda41e}]" +2025-03-20T03:49:38.676Z [INFO] server-d963fe94-bef1-318c-a918-b9dc45c9fb2f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-74ba5650-439e-11d6-8fef-f6cd707ca26b Address:74ba5650-439e-11d6-8fef-f6cd707ca26b} {Suffrage:Voter ID:server-d963fe94-bef1-318c-a918-b9dc45c9fb2f Address:d963fe94-bef1-318c-a918-b9dc45c9fb2f} {Suffrage:Voter ID:server-f7766b6e-71b1-166c-1067-c52bdfbda41e Address:f7766b6e-71b1-166c-1067-c52bdfbda41e}]" +2025-03-20T03:49:38.676Z [INFO] server-f7766b6e-71b1-166c-1067-c52bdfbda41e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-74ba5650-439e-11d6-8fef-f6cd707ca26b Address:74ba5650-439e-11d6-8fef-f6cd707ca26b} {Suffrage:Voter ID:server-d963fe94-bef1-318c-a918-b9dc45c9fb2f Address:d963fe94-bef1-318c-a918-b9dc45c9fb2f} {Suffrage:Voter ID:server-f7766b6e-71b1-166c-1067-c52bdfbda41e Address:f7766b6e-71b1-166c-1067-c52bdfbda41e}]" +2025-03-20T03:49:38.676Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:38.676Z [INFO] server-74ba5650-439e-11d6-8fef-f6cd707ca26b: entering follower state: follower="Node at 74ba5650-439e-11d6-8fef-f6cd707ca26b [Follower]" leader-address= leader-id= +2025-03-20T03:49:38.676Z [INFO] server-d963fe94-bef1-318c-a918-b9dc45c9fb2f: entering follower state: follower="Node at d963fe94-bef1-318c-a918-b9dc45c9fb2f [Follower]" leader-address= leader-id= +2025-03-20T03:49:38.676Z [INFO] server-f7766b6e-71b1-166c-1067-c52bdfbda41e: entering follower state: follower="Node at f7766b6e-71b1-166c-1067-c52bdfbda41e [Follower]" leader-address= leader-id= +2025-03-20T03:49:38.748Z [WARN] server-74ba5650-439e-11d6-8fef-f6cd707ca26b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:38.748Z [INFO] server-74ba5650-439e-11d6-8fef-f6cd707ca26b: entering candidate state: node="Node at 74ba5650-439e-11d6-8fef-f6cd707ca26b [Candidate]" term=2 +2025-03-20T03:49:38.748Z [INFO] server-74ba5650-439e-11d6-8fef-f6cd707ca26b: election won: term=2 tally=2 +2025-03-20T03:49:38.748Z [INFO] server-74ba5650-439e-11d6-8fef-f6cd707ca26b: entering leader state: leader="Node at 74ba5650-439e-11d6-8fef-f6cd707ca26b [Leader]" +2025-03-20T03:49:38.748Z [INFO] server-74ba5650-439e-11d6-8fef-f6cd707ca26b: added peer, starting replication: peer=server-d963fe94-bef1-318c-a918-b9dc45c9fb2f +2025-03-20T03:49:38.748Z [INFO] server-74ba5650-439e-11d6-8fef-f6cd707ca26b: added peer, starting replication: peer=server-f7766b6e-71b1-166c-1067-c52bdfbda41e +2025-03-20T03:49:38.748Z [INFO] server-74ba5650-439e-11d6-8fef-f6cd707ca26b: pipelining replication: peer="{Voter server-f7766b6e-71b1-166c-1067-c52bdfbda41e f7766b6e-71b1-166c-1067-c52bdfbda41e}" +2025-03-20T03:49:38.749Z [INFO] server-74ba5650-439e-11d6-8fef-f6cd707ca26b: pipelining replication: peer="{Voter server-d963fe94-bef1-318c-a918-b9dc45c9fb2f d963fe94-bef1-318c-a918-b9dc45c9fb2f}" +2025-03-20T03:49:38.854Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:38.749447362 +0000 UTC m=+10.423900227 (1 nodes), 73.13932ms from start of poll, 73.501906ms from cluster start. Timeout at 2025-03-20 03:49:38.854465925 +0000 UTC m=+10.528918843, 105.018616ms after stability +2025-03-20T03:49:38.860Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:49:38.966Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:49:38.860665755 +0000 UTC m=+10.535119159 (2 nodes), 3.491µs from start of poll, 184.720838ms from cluster start. Timeout at 2025-03-20 03:49:38.965688276 +0000 UTC m=+10.640141346, 105.022187ms after stability +2025-03-20T03:49:38.966Z [INFO] server-74ba5650-439e-11d6-8fef-f6cd707ca26b: updating configuration: command=RemoveServer server-id=server-74ba5650-439e-11d6-8fef-f6cd707ca26b server-addr= servers="[{Suffrage:Voter ID:server-d963fe94-bef1-318c-a918-b9dc45c9fb2f Address:d963fe94-bef1-318c-a918-b9dc45c9fb2f} {Suffrage:Voter ID:server-f7766b6e-71b1-166c-1067-c52bdfbda41e Address:f7766b6e-71b1-166c-1067-c52bdfbda41e}]" +2025-03-20T03:49:38.967Z [INFO] server-74ba5650-439e-11d6-8fef-f6cd707ca26b: removed ourself, shutting down +2025-03-20T03:49:38.967Z [INFO] server-74ba5650-439e-11d6-8fef-f6cd707ca26b: aborting pipeline replication: peer="{Voter server-f7766b6e-71b1-166c-1067-c52bdfbda41e f7766b6e-71b1-166c-1067-c52bdfbda41e}" +2025-03-20T03:49:38.967Z [INFO] server-74ba5650-439e-11d6-8fef-f6cd707ca26b: aborting pipeline replication: peer="{Voter server-d963fe94-bef1-318c-a918-b9dc45c9fb2f d963fe94-bef1-318c-a918-b9dc45c9fb2f}" +2025-03-20T03:49:39.043Z [WARN] server-f7766b6e-71b1-166c-1067-c52bdfbda41e: heartbeat timeout reached, starting election: last-leader-addr=74ba5650-439e-11d6-8fef-f6cd707ca26b last-leader-id=server-74ba5650-439e-11d6-8fef-f6cd707ca26b +2025-03-20T03:49:39.043Z [INFO] server-f7766b6e-71b1-166c-1067-c52bdfbda41e: entering candidate state: node="Node at f7766b6e-71b1-166c-1067-c52bdfbda41e [Candidate]" term=3 +2025-03-20T03:49:39.043Z [WARN] server-d963fe94-bef1-318c-a918-b9dc45c9fb2f: rejecting vote request since we have a leader: from=f7766b6e-71b1-166c-1067-c52bdfbda41e leader=74ba5650-439e-11d6-8fef-f6cd707ca26b leader-id=server-74ba5650-439e-11d6-8fef-f6cd707ca26b +2025-03-20T03:49:39.072Z [WARN] server-d963fe94-bef1-318c-a918-b9dc45c9fb2f: heartbeat timeout reached, starting election: last-leader-addr=74ba5650-439e-11d6-8fef-f6cd707ca26b last-leader-id=server-74ba5650-439e-11d6-8fef-f6cd707ca26b +2025-03-20T03:49:39.072Z [INFO] server-d963fe94-bef1-318c-a918-b9dc45c9fb2f: entering candidate state: node="Node at d963fe94-bef1-318c-a918-b9dc45c9fb2f [Candidate]" term=3 +2025-03-20T03:49:39.072Z [INFO] server-f7766b6e-71b1-166c-1067-c52bdfbda41e: duplicate requestVote for same term: term=3 +2025-03-20T03:49:39.142Z [WARN] server-f7766b6e-71b1-166c-1067-c52bdfbda41e: Election timeout reached, restarting election +2025-03-20T03:49:39.142Z [INFO] server-f7766b6e-71b1-166c-1067-c52bdfbda41e: entering candidate state: node="Node at f7766b6e-71b1-166c-1067-c52bdfbda41e [Candidate]" term=4 +2025-03-20T03:49:39.142Z [INFO] server-d963fe94-bef1-318c-a918-b9dc45c9fb2f: entering follower state: follower="Node at d963fe94-bef1-318c-a918-b9dc45c9fb2f [Follower]" leader-address= leader-id= +2025-03-20T03:49:39.142Z [INFO] server-f7766b6e-71b1-166c-1067-c52bdfbda41e: election won: term=4 tally=2 +2025-03-20T03:49:39.142Z [INFO] server-f7766b6e-71b1-166c-1067-c52bdfbda41e: entering leader state: leader="Node at f7766b6e-71b1-166c-1067-c52bdfbda41e [Leader]" +2025-03-20T03:49:39.143Z [INFO] server-f7766b6e-71b1-166c-1067-c52bdfbda41e: added peer, starting replication: peer=server-d963fe94-bef1-318c-a918-b9dc45c9fb2f +2025-03-20T03:49:39.143Z [INFO] server-f7766b6e-71b1-166c-1067-c52bdfbda41e: pipelining replication: peer="{Voter server-d963fe94-bef1-318c-a918-b9dc45c9fb2f d963fe94-bef1-318c-a918-b9dc45c9fb2f}" +2025-03-20T03:49:39.178Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:39.295Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:39.179097408 +0000 UTC m=+10.853550256 (1 nodes), 89.263µs from start of poll, 503.151935ms from cluster start. Timeout at 2025-03-20 03:49:39.284109874 +0000 UTC m=+10.958562881, 105.012625ms after stability +2025-03-20T03:49:39.295Z [INFO] server-f7766b6e-71b1-166c-1067-c52bdfbda41e: aborting pipeline replication: peer="{Voter server-d963fe94-bef1-318c-a918-b9dc45c9fb2f d963fe94-bef1-318c-a918-b9dc45c9fb2f}" +2025-03-20T03:49:39.795Z [ERROR] server-f7766b6e-71b1-166c-1067-c52bdfbda41e: failed to heartbeat to: peer=d963fe94-bef1-318c-a918-b9dc45c9fb2f backoff time=10ms error="command timed out" +--- PASS: TestRaft_RemoveLeader (1.12s) === RUN TestRaft_RemoveLeader_NoShutdown -2026-04-22T10:09:24.711Z [INFO] server-e59d6a67-e1dd-ff37-7dbe-3f8777f65d33: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e59d6a67-e1dd-ff37-7dbe-3f8777f65d33 Address:e59d6a67-e1dd-ff37-7dbe-3f8777f65d33} {Suffrage:Voter ID:server-e43f3ac2-a129-5bf7-3b56-66e800a4743a Address:e43f3ac2-a129-5bf7-3b56-66e800a4743a} {Suffrage:Voter ID:server-92e2ca1b-e1a6-0667-0632-68d88a66d4fd Address:92e2ca1b-e1a6-0667-0632-68d88a66d4fd}]" -2026-04-22T10:09:24.711Z [INFO] server-e43f3ac2-a129-5bf7-3b56-66e800a4743a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e59d6a67-e1dd-ff37-7dbe-3f8777f65d33 Address:e59d6a67-e1dd-ff37-7dbe-3f8777f65d33} {Suffrage:Voter ID:server-e43f3ac2-a129-5bf7-3b56-66e800a4743a Address:e43f3ac2-a129-5bf7-3b56-66e800a4743a} {Suffrage:Voter ID:server-92e2ca1b-e1a6-0667-0632-68d88a66d4fd Address:92e2ca1b-e1a6-0667-0632-68d88a66d4fd}]" -2026-04-22T10:09:24.711Z [INFO] server-92e2ca1b-e1a6-0667-0632-68d88a66d4fd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e59d6a67-e1dd-ff37-7dbe-3f8777f65d33 Address:e59d6a67-e1dd-ff37-7dbe-3f8777f65d33} {Suffrage:Voter ID:server-e43f3ac2-a129-5bf7-3b56-66e800a4743a Address:e43f3ac2-a129-5bf7-3b56-66e800a4743a} {Suffrage:Voter ID:server-92e2ca1b-e1a6-0667-0632-68d88a66d4fd Address:92e2ca1b-e1a6-0667-0632-68d88a66d4fd}]" -2026-04-22T10:09:24.711Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:09:24.712Z [INFO] server-e59d6a67-e1dd-ff37-7dbe-3f8777f65d33: entering follower state: follower="Node at e59d6a67-e1dd-ff37-7dbe-3f8777f65d33 [Follower]" leader-address= leader-id= -2026-04-22T10:09:24.712Z [INFO] server-e43f3ac2-a129-5bf7-3b56-66e800a4743a: entering follower state: follower="Node at e43f3ac2-a129-5bf7-3b56-66e800a4743a [Follower]" leader-address= leader-id= -2026-04-22T10:09:24.712Z [INFO] server-92e2ca1b-e1a6-0667-0632-68d88a66d4fd: entering follower state: follower="Node at 92e2ca1b-e1a6-0667-0632-68d88a66d4fd [Follower]" leader-address= leader-id= -2026-04-22T10:09:24.768Z [WARN] server-92e2ca1b-e1a6-0667-0632-68d88a66d4fd: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:24.768Z [INFO] server-92e2ca1b-e1a6-0667-0632-68d88a66d4fd: entering candidate state: node="Node at 92e2ca1b-e1a6-0667-0632-68d88a66d4fd [Candidate]" term=2 -2026-04-22T10:09:24.768Z [INFO] server-92e2ca1b-e1a6-0667-0632-68d88a66d4fd: election won: term=2 tally=2 -2026-04-22T10:09:24.768Z [INFO] server-92e2ca1b-e1a6-0667-0632-68d88a66d4fd: entering leader state: leader="Node at 92e2ca1b-e1a6-0667-0632-68d88a66d4fd [Leader]" -2026-04-22T10:09:24.768Z [INFO] server-92e2ca1b-e1a6-0667-0632-68d88a66d4fd: added peer, starting replication: peer=server-e59d6a67-e1dd-ff37-7dbe-3f8777f65d33 -2026-04-22T10:09:24.768Z [INFO] server-92e2ca1b-e1a6-0667-0632-68d88a66d4fd: added peer, starting replication: peer=server-e43f3ac2-a129-5bf7-3b56-66e800a4743a -2026-04-22T10:09:24.768Z [INFO] server-92e2ca1b-e1a6-0667-0632-68d88a66d4fd: pipelining replication: peer="{Voter server-e43f3ac2-a129-5bf7-3b56-66e800a4743a e43f3ac2-a129-5bf7-3b56-66e800a4743a}" -2026-04-22T10:09:24.768Z [INFO] server-92e2ca1b-e1a6-0667-0632-68d88a66d4fd: pipelining replication: peer="{Voter server-e59d6a67-e1dd-ff37-7dbe-3f8777f65d33 e59d6a67-e1dd-ff37-7dbe-3f8777f65d33}" -2026-04-22T10:09:24.874Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:09:24.768979347 +0000 UTC m=+10.792775460 (2 nodes), 56.973137ms from start of poll, 57.336611ms from cluster start. Timeout at 2026-04-22 10:09:24.873986189 +0000 UTC m=+10.897782312, 105.006852ms after stability -2026-04-22T10:09:24.874Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:24.981Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:24.874504689 +0000 UTC m=+10.898300796 (1 nodes), 1.589µs from start of poll, 162.861947ms from cluster start. Timeout at 2026-04-22 10:09:24.97951502 +0000 UTC m=+11.003311148, 105.010352ms after stability -2026-04-22T10:09:24.991Z [INFO] server-92e2ca1b-e1a6-0667-0632-68d88a66d4fd: updating configuration: command=RemoveServer server-id=server-92e2ca1b-e1a6-0667-0632-68d88a66d4fd server-addr= servers="[{Suffrage:Voter ID:server-e59d6a67-e1dd-ff37-7dbe-3f8777f65d33 Address:e59d6a67-e1dd-ff37-7dbe-3f8777f65d33} {Suffrage:Voter ID:server-e43f3ac2-a129-5bf7-3b56-66e800a4743a Address:e43f3ac2-a129-5bf7-3b56-66e800a4743a}]" -2026-04-22T10:09:24.992Z [INFO] server-92e2ca1b-e1a6-0667-0632-68d88a66d4fd: removed ourself, transitioning to follower -2026-04-22T10:09:24.992Z [INFO] server-92e2ca1b-e1a6-0667-0632-68d88a66d4fd: entering follower state: follower="Node at 92e2ca1b-e1a6-0667-0632-68d88a66d4fd [Follower]" leader-address= leader-id= -2026-04-22T10:09:24.992Z [INFO] server-92e2ca1b-e1a6-0667-0632-68d88a66d4fd: aborting pipeline replication: peer="{Voter server-e43f3ac2-a129-5bf7-3b56-66e800a4743a e43f3ac2-a129-5bf7-3b56-66e800a4743a}" -2026-04-22T10:09:24.992Z [INFO] server-92e2ca1b-e1a6-0667-0632-68d88a66d4fd: aborting pipeline replication: peer="{Voter server-e59d6a67-e1dd-ff37-7dbe-3f8777f65d33 e59d6a67-e1dd-ff37-7dbe-3f8777f65d33}" -2026-04-22T10:09:25.047Z [WARN] server-e59d6a67-e1dd-ff37-7dbe-3f8777f65d33: heartbeat timeout reached, starting election: last-leader-addr=92e2ca1b-e1a6-0667-0632-68d88a66d4fd last-leader-id=server-92e2ca1b-e1a6-0667-0632-68d88a66d4fd -2026-04-22T10:09:25.047Z [INFO] server-e59d6a67-e1dd-ff37-7dbe-3f8777f65d33: entering candidate state: node="Node at e59d6a67-e1dd-ff37-7dbe-3f8777f65d33 [Candidate]" term=3 -2026-04-22T10:09:25.047Z [WARN] server-e43f3ac2-a129-5bf7-3b56-66e800a4743a: rejecting vote request since we have a leader: from=e59d6a67-e1dd-ff37-7dbe-3f8777f65d33 leader=92e2ca1b-e1a6-0667-0632-68d88a66d4fd leader-id=server-92e2ca1b-e1a6-0667-0632-68d88a66d4fd -2026-04-22T10:09:25.073Z [WARN] server-92e2ca1b-e1a6-0667-0632-68d88a66d4fd: not part of stable configuration, aborting election -2026-04-22T10:09:25.097Z [WARN] server-e43f3ac2-a129-5bf7-3b56-66e800a4743a: heartbeat timeout reached, starting election: last-leader-addr=92e2ca1b-e1a6-0667-0632-68d88a66d4fd last-leader-id=server-92e2ca1b-e1a6-0667-0632-68d88a66d4fd -2026-04-22T10:09:25.097Z [INFO] server-e43f3ac2-a129-5bf7-3b56-66e800a4743a: entering candidate state: node="Node at e43f3ac2-a129-5bf7-3b56-66e800a4743a [Candidate]" term=3 -2026-04-22T10:09:25.097Z [INFO] server-e59d6a67-e1dd-ff37-7dbe-3f8777f65d33: duplicate requestVote for same term: term=3 -2026-04-22T10:09:25.097Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:25.105Z [WARN] server-e59d6a67-e1dd-ff37-7dbe-3f8777f65d33: Election timeout reached, restarting election -2026-04-22T10:09:25.105Z [INFO] server-e59d6a67-e1dd-ff37-7dbe-3f8777f65d33: entering candidate state: node="Node at e59d6a67-e1dd-ff37-7dbe-3f8777f65d33 [Candidate]" term=4 -2026-04-22T10:09:25.105Z [INFO] server-e43f3ac2-a129-5bf7-3b56-66e800a4743a: entering follower state: follower="Node at e43f3ac2-a129-5bf7-3b56-66e800a4743a [Follower]" leader-address= leader-id= -2026-04-22T10:09:25.105Z [INFO] server-e59d6a67-e1dd-ff37-7dbe-3f8777f65d33: election won: term=4 tally=2 -2026-04-22T10:09:25.105Z [INFO] server-e59d6a67-e1dd-ff37-7dbe-3f8777f65d33: entering leader state: leader="Node at e59d6a67-e1dd-ff37-7dbe-3f8777f65d33 [Leader]" -2026-04-22T10:09:25.105Z [INFO] server-e59d6a67-e1dd-ff37-7dbe-3f8777f65d33: added peer, starting replication: peer=server-e43f3ac2-a129-5bf7-3b56-66e800a4743a -2026-04-22T10:09:25.105Z [INFO] server-e59d6a67-e1dd-ff37-7dbe-3f8777f65d33: pipelining replication: peer="{Voter server-e43f3ac2-a129-5bf7-3b56-66e800a4743a e43f3ac2-a129-5bf7-3b56-66e800a4743a}" -2026-04-22T10:09:25.649Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:25.10599145 +0000 UTC m=+11.129787579 (1 nodes), 8.259393ms from start of poll, 394.34873ms from cluster start. Timeout at 2026-04-22 10:09:25.211005561 +0000 UTC m=+11.234801674, 105.014095ms after stability -2026-04-22T10:09:25.757Z [INFO] server-e59d6a67-e1dd-ff37-7dbe-3f8777f65d33: aborting pipeline replication: peer="{Voter server-e43f3ac2-a129-5bf7-3b56-66e800a4743a e43f3ac2-a129-5bf7-3b56-66e800a4743a}" ---- PASS: TestRaft_RemoveLeader_NoShutdown (1.05s) +2025-03-20T03:49:39.797Z [INFO] server-dfd43007-b724-dc78-8c6f-62c0ed45a40c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dfd43007-b724-dc78-8c6f-62c0ed45a40c Address:dfd43007-b724-dc78-8c6f-62c0ed45a40c} {Suffrage:Voter ID:server-567ca2c1-25a7-276d-e885-54903e2a493b Address:567ca2c1-25a7-276d-e885-54903e2a493b} {Suffrage:Voter ID:server-f9b4023d-be01-31df-34e0-afaf59c5e70e Address:f9b4023d-be01-31df-34e0-afaf59c5e70e}]" +2025-03-20T03:49:39.797Z [INFO] server-567ca2c1-25a7-276d-e885-54903e2a493b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dfd43007-b724-dc78-8c6f-62c0ed45a40c Address:dfd43007-b724-dc78-8c6f-62c0ed45a40c} {Suffrage:Voter ID:server-567ca2c1-25a7-276d-e885-54903e2a493b Address:567ca2c1-25a7-276d-e885-54903e2a493b} {Suffrage:Voter ID:server-f9b4023d-be01-31df-34e0-afaf59c5e70e Address:f9b4023d-be01-31df-34e0-afaf59c5e70e}]" +2025-03-20T03:49:39.797Z [INFO] server-f9b4023d-be01-31df-34e0-afaf59c5e70e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dfd43007-b724-dc78-8c6f-62c0ed45a40c Address:dfd43007-b724-dc78-8c6f-62c0ed45a40c} {Suffrage:Voter ID:server-567ca2c1-25a7-276d-e885-54903e2a493b Address:567ca2c1-25a7-276d-e885-54903e2a493b} {Suffrage:Voter ID:server-f9b4023d-be01-31df-34e0-afaf59c5e70e Address:f9b4023d-be01-31df-34e0-afaf59c5e70e}]" +2025-03-20T03:49:39.797Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:49:39.797Z [INFO] server-dfd43007-b724-dc78-8c6f-62c0ed45a40c: entering follower state: follower="Node at dfd43007-b724-dc78-8c6f-62c0ed45a40c [Follower]" leader-address= leader-id= +2025-03-20T03:49:39.797Z [INFO] server-567ca2c1-25a7-276d-e885-54903e2a493b: entering follower state: follower="Node at 567ca2c1-25a7-276d-e885-54903e2a493b [Follower]" leader-address= leader-id= +2025-03-20T03:49:39.797Z [INFO] server-f9b4023d-be01-31df-34e0-afaf59c5e70e: entering follower state: follower="Node at f9b4023d-be01-31df-34e0-afaf59c5e70e [Follower]" leader-address= leader-id= +2025-03-20T03:49:39.850Z [WARN] server-dfd43007-b724-dc78-8c6f-62c0ed45a40c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:39.850Z [INFO] server-dfd43007-b724-dc78-8c6f-62c0ed45a40c: entering candidate state: node="Node at dfd43007-b724-dc78-8c6f-62c0ed45a40c [Candidate]" term=2 +2025-03-20T03:49:39.850Z [INFO] server-dfd43007-b724-dc78-8c6f-62c0ed45a40c: election won: term=2 tally=2 +2025-03-20T03:49:39.850Z [INFO] server-dfd43007-b724-dc78-8c6f-62c0ed45a40c: entering leader state: leader="Node at dfd43007-b724-dc78-8c6f-62c0ed45a40c [Leader]" +2025-03-20T03:49:39.850Z [INFO] server-dfd43007-b724-dc78-8c6f-62c0ed45a40c: added peer, starting replication: peer=server-567ca2c1-25a7-276d-e885-54903e2a493b +2025-03-20T03:49:39.850Z [INFO] server-dfd43007-b724-dc78-8c6f-62c0ed45a40c: added peer, starting replication: peer=server-f9b4023d-be01-31df-34e0-afaf59c5e70e +2025-03-20T03:49:39.851Z [INFO] server-dfd43007-b724-dc78-8c6f-62c0ed45a40c: pipelining replication: peer="{Voter server-f9b4023d-be01-31df-34e0-afaf59c5e70e f9b4023d-be01-31df-34e0-afaf59c5e70e}" +2025-03-20T03:49:39.851Z [INFO] server-dfd43007-b724-dc78-8c6f-62c0ed45a40c: pipelining replication: peer="{Voter server-567ca2c1-25a7-276d-e885-54903e2a493b 567ca2c1-25a7-276d-e885-54903e2a493b}" +2025-03-20T03:49:39.956Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:49:39.851455104 +0000 UTC m=+11.525907987 (2 nodes), 53.708899ms from start of poll, 54.117448ms from cluster start. Timeout at 2025-03-20 03:49:39.956459891 +0000 UTC m=+11.630912870, 105.004883ms after stability +2025-03-20T03:49:39.956Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:40.062Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:39.956805444 +0000 UTC m=+11.631258290 (1 nodes), 1.827µs from start of poll, 159.467751ms from cluster start. Timeout at 2025-03-20 03:49:40.061813034 +0000 UTC m=+11.736266058, 105.007768ms after stability +2025-03-20T03:49:40.076Z [INFO] server-dfd43007-b724-dc78-8c6f-62c0ed45a40c: updating configuration: command=RemoveServer server-id=server-dfd43007-b724-dc78-8c6f-62c0ed45a40c server-addr= servers="[{Suffrage:Voter ID:server-567ca2c1-25a7-276d-e885-54903e2a493b Address:567ca2c1-25a7-276d-e885-54903e2a493b} {Suffrage:Voter ID:server-f9b4023d-be01-31df-34e0-afaf59c5e70e Address:f9b4023d-be01-31df-34e0-afaf59c5e70e}]" +2025-03-20T03:49:40.078Z [INFO] server-dfd43007-b724-dc78-8c6f-62c0ed45a40c: removed ourself, transitioning to follower +2025-03-20T03:49:40.078Z [INFO] server-dfd43007-b724-dc78-8c6f-62c0ed45a40c: entering follower state: follower="Node at dfd43007-b724-dc78-8c6f-62c0ed45a40c [Follower]" leader-address= leader-id= +2025-03-20T03:49:40.078Z [INFO] server-dfd43007-b724-dc78-8c6f-62c0ed45a40c: aborting pipeline replication: peer="{Voter server-f9b4023d-be01-31df-34e0-afaf59c5e70e f9b4023d-be01-31df-34e0-afaf59c5e70e}" +2025-03-20T03:49:40.078Z [INFO] server-dfd43007-b724-dc78-8c6f-62c0ed45a40c: aborting pipeline replication: peer="{Voter server-567ca2c1-25a7-276d-e885-54903e2a493b 567ca2c1-25a7-276d-e885-54903e2a493b}" +2025-03-20T03:49:40.137Z [WARN] server-f9b4023d-be01-31df-34e0-afaf59c5e70e: heartbeat timeout reached, starting election: last-leader-addr=dfd43007-b724-dc78-8c6f-62c0ed45a40c last-leader-id=server-dfd43007-b724-dc78-8c6f-62c0ed45a40c +2025-03-20T03:49:40.137Z [INFO] server-f9b4023d-be01-31df-34e0-afaf59c5e70e: entering candidate state: node="Node at f9b4023d-be01-31df-34e0-afaf59c5e70e [Candidate]" term=3 +2025-03-20T03:49:40.137Z [WARN] server-567ca2c1-25a7-276d-e885-54903e2a493b: rejecting vote request since we have a leader: from=f9b4023d-be01-31df-34e0-afaf59c5e70e leader=dfd43007-b724-dc78-8c6f-62c0ed45a40c leader-id=server-dfd43007-b724-dc78-8c6f-62c0ed45a40c +2025-03-20T03:49:40.156Z [WARN] server-dfd43007-b724-dc78-8c6f-62c0ed45a40c: not part of stable configuration, aborting election +2025-03-20T03:49:40.178Z [WARN] server-567ca2c1-25a7-276d-e885-54903e2a493b: heartbeat timeout reached, starting election: last-leader-addr=dfd43007-b724-dc78-8c6f-62c0ed45a40c last-leader-id=server-dfd43007-b724-dc78-8c6f-62c0ed45a40c +2025-03-20T03:49:40.178Z [INFO] server-567ca2c1-25a7-276d-e885-54903e2a493b: entering candidate state: node="Node at 567ca2c1-25a7-276d-e885-54903e2a493b [Candidate]" term=3 +2025-03-20T03:49:40.178Z [INFO] server-f9b4023d-be01-31df-34e0-afaf59c5e70e: duplicate requestVote for same term: term=3 +2025-03-20T03:49:40.184Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:40.218Z [WARN] server-f9b4023d-be01-31df-34e0-afaf59c5e70e: Election timeout reached, restarting election +2025-03-20T03:49:40.218Z [INFO] server-f9b4023d-be01-31df-34e0-afaf59c5e70e: entering candidate state: node="Node at f9b4023d-be01-31df-34e0-afaf59c5e70e [Candidate]" term=4 +2025-03-20T03:49:40.219Z [INFO] server-567ca2c1-25a7-276d-e885-54903e2a493b: entering follower state: follower="Node at 567ca2c1-25a7-276d-e885-54903e2a493b [Follower]" leader-address= leader-id= +2025-03-20T03:49:40.219Z [INFO] server-f9b4023d-be01-31df-34e0-afaf59c5e70e: election won: term=4 tally=2 +2025-03-20T03:49:40.219Z [INFO] server-f9b4023d-be01-31df-34e0-afaf59c5e70e: entering leader state: leader="Node at f9b4023d-be01-31df-34e0-afaf59c5e70e [Leader]" +2025-03-20T03:49:40.219Z [INFO] server-f9b4023d-be01-31df-34e0-afaf59c5e70e: added peer, starting replication: peer=server-567ca2c1-25a7-276d-e885-54903e2a493b +2025-03-20T03:49:40.219Z [INFO] server-f9b4023d-be01-31df-34e0-afaf59c5e70e: pipelining replication: peer="{Voter server-567ca2c1-25a7-276d-e885-54903e2a493b 567ca2c1-25a7-276d-e885-54903e2a493b}" +2025-03-20T03:49:40.325Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:40.219389198 +0000 UTC m=+11.893842070 (1 nodes), 34.987978ms from start of poll, 422.051531ms from cluster start. Timeout at 2025-03-20 03:49:40.324395218 +0000 UTC m=+11.998848110, 105.00604ms after stability +2025-03-20T03:49:40.430Z [INFO] server-f9b4023d-be01-31df-34e0-afaf59c5e70e: aborting pipeline replication: peer="{Voter server-567ca2c1-25a7-276d-e885-54903e2a493b 567ca2c1-25a7-276d-e885-54903e2a493b}" +--- PASS: TestRaft_RemoveLeader_NoShutdown (0.64s) === RUN TestRaft_RemoveFollower_SplitCluster -2026-04-22T10:09:25.759Z [INFO] server-022c65c4-5b03-4100-9d02-329c05d6d9e8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-022c65c4-5b03-4100-9d02-329c05d6d9e8 Address:022c65c4-5b03-4100-9d02-329c05d6d9e8} {Suffrage:Voter ID:server-adffe494-0224-4fde-2822-ff5110673309 Address:adffe494-0224-4fde-2822-ff5110673309} {Suffrage:Voter ID:server-6e2809f4-b3fe-466e-5eb3-6f6394030c30 Address:6e2809f4-b3fe-466e-5eb3-6f6394030c30} {Suffrage:Voter ID:server-a6e2f418-d993-1a67-ab11-d1d8599c19bc Address:a6e2f418-d993-1a67-ab11-d1d8599c19bc}]" -2026-04-22T10:09:25.759Z [INFO] server-adffe494-0224-4fde-2822-ff5110673309: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-022c65c4-5b03-4100-9d02-329c05d6d9e8 Address:022c65c4-5b03-4100-9d02-329c05d6d9e8} {Suffrage:Voter ID:server-adffe494-0224-4fde-2822-ff5110673309 Address:adffe494-0224-4fde-2822-ff5110673309} {Suffrage:Voter ID:server-6e2809f4-b3fe-466e-5eb3-6f6394030c30 Address:6e2809f4-b3fe-466e-5eb3-6f6394030c30} {Suffrage:Voter ID:server-a6e2f418-d993-1a67-ab11-d1d8599c19bc Address:a6e2f418-d993-1a67-ab11-d1d8599c19bc}]" -2026-04-22T10:09:25.759Z [INFO] server-6e2809f4-b3fe-466e-5eb3-6f6394030c30: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-022c65c4-5b03-4100-9d02-329c05d6d9e8 Address:022c65c4-5b03-4100-9d02-329c05d6d9e8} {Suffrage:Voter ID:server-adffe494-0224-4fde-2822-ff5110673309 Address:adffe494-0224-4fde-2822-ff5110673309} {Suffrage:Voter ID:server-6e2809f4-b3fe-466e-5eb3-6f6394030c30 Address:6e2809f4-b3fe-466e-5eb3-6f6394030c30} {Suffrage:Voter ID:server-a6e2f418-d993-1a67-ab11-d1d8599c19bc Address:a6e2f418-d993-1a67-ab11-d1d8599c19bc}]" -2026-04-22T10:09:25.759Z [INFO] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-022c65c4-5b03-4100-9d02-329c05d6d9e8 Address:022c65c4-5b03-4100-9d02-329c05d6d9e8} {Suffrage:Voter ID:server-adffe494-0224-4fde-2822-ff5110673309 Address:adffe494-0224-4fde-2822-ff5110673309} {Suffrage:Voter ID:server-6e2809f4-b3fe-466e-5eb3-6f6394030c30 Address:6e2809f4-b3fe-466e-5eb3-6f6394030c30} {Suffrage:Voter ID:server-a6e2f418-d993-1a67-ab11-d1d8599c19bc Address:a6e2f418-d993-1a67-ab11-d1d8599c19bc}]" -2026-04-22T10:09:25.759Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:25.759Z [INFO] server-022c65c4-5b03-4100-9d02-329c05d6d9e8: entering follower state: follower="Node at 022c65c4-5b03-4100-9d02-329c05d6d9e8 [Follower]" leader-address= leader-id= -2026-04-22T10:09:25.759Z [INFO] server-adffe494-0224-4fde-2822-ff5110673309: entering follower state: follower="Node at adffe494-0224-4fde-2822-ff5110673309 [Follower]" leader-address= leader-id= -2026-04-22T10:09:25.759Z [INFO] server-6e2809f4-b3fe-466e-5eb3-6f6394030c30: entering follower state: follower="Node at 6e2809f4-b3fe-466e-5eb3-6f6394030c30 [Follower]" leader-address= leader-id= -2026-04-22T10:09:25.759Z [INFO] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: entering follower state: follower="Node at a6e2f418-d993-1a67-ab11-d1d8599c19bc [Follower]" leader-address= leader-id= -2026-04-22T10:09:25.829Z [WARN] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:25.829Z [INFO] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: entering candidate state: node="Node at a6e2f418-d993-1a67-ab11-d1d8599c19bc [Candidate]" term=2 -2026-04-22T10:09:25.829Z [WARN] server-6e2809f4-b3fe-466e-5eb3-6f6394030c30: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:25.829Z [INFO] server-6e2809f4-b3fe-466e-5eb3-6f6394030c30: entering candidate state: node="Node at 6e2809f4-b3fe-466e-5eb3-6f6394030c30 [Candidate]" term=2 -2026-04-22T10:09:25.829Z [INFO] server-6e2809f4-b3fe-466e-5eb3-6f6394030c30: duplicate requestVote for same term: term=2 -2026-04-22T10:09:25.829Z [INFO] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: election won: term=2 tally=3 -2026-04-22T10:09:25.829Z [INFO] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: entering leader state: leader="Node at a6e2f418-d993-1a67-ab11-d1d8599c19bc [Leader]" -2026-04-22T10:09:25.830Z [INFO] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: added peer, starting replication: peer=server-022c65c4-5b03-4100-9d02-329c05d6d9e8 -2026-04-22T10:09:25.830Z [INFO] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: added peer, starting replication: peer=server-adffe494-0224-4fde-2822-ff5110673309 -2026-04-22T10:09:25.830Z [INFO] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: added peer, starting replication: peer=server-6e2809f4-b3fe-466e-5eb3-6f6394030c30 -2026-04-22T10:09:25.830Z [INFO] server-6e2809f4-b3fe-466e-5eb3-6f6394030c30: entering follower state: follower="Node at 6e2809f4-b3fe-466e-5eb3-6f6394030c30 [Follower]" leader-address=a6e2f418-d993-1a67-ab11-d1d8599c19bc leader-id=server-a6e2f418-d993-1a67-ab11-d1d8599c19bc -2026-04-22T10:09:25.830Z [INFO] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: pipelining replication: peer="{Voter server-6e2809f4-b3fe-466e-5eb3-6f6394030c30 6e2809f4-b3fe-466e-5eb3-6f6394030c30}" -2026-04-22T10:09:25.830Z [INFO] server-022c65c4-5b03-4100-9d02-329c05d6d9e8: duplicate requestVote for same term: term=2 -2026-04-22T10:09:25.830Z [INFO] server-adffe494-0224-4fde-2822-ff5110673309: duplicate requestVote for same term: term=2 -2026-04-22T10:09:25.830Z [WARN] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: rejecting vote request since we have a leader: from=6e2809f4-b3fe-466e-5eb3-6f6394030c30 leader=a6e2f418-d993-1a67-ab11-d1d8599c19bc leader-id=server-a6e2f418-d993-1a67-ab11-d1d8599c19bc -2026-04-22T10:09:25.830Z [INFO] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: pipelining replication: peer="{Voter server-022c65c4-5b03-4100-9d02-329c05d6d9e8 022c65c4-5b03-4100-9d02-329c05d6d9e8}" -2026-04-22T10:09:25.830Z [INFO] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: pipelining replication: peer="{Voter server-adffe494-0224-4fde-2822-ff5110673309 adffe494-0224-4fde-2822-ff5110673309}" -2026-04-22T10:09:25.935Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:25.830343347 +0000 UTC m=+11.854139451 (1 nodes), 70.860199ms from start of poll, 71.332163ms from cluster start. Timeout at 2026-04-22 10:09:25.935344576 +0000 UTC m=+11.959140701, 105.00125ms after stability -2026-04-22T10:09:26.043Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:09:26.151Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:09:26.043717356 +0000 UTC m=+12.067513472 (3 nodes), 1.597µs from start of poll, 284.706184ms from cluster start. Timeout at 2026-04-22 10:09:26.148728849 +0000 UTC m=+12.172524959, 105.011487ms after stability -2026-04-22T10:09:26.151Z [INFO] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: updating configuration: command=RemoveServer server-id=server-6e2809f4-b3fe-466e-5eb3-6f6394030c30 server-addr= servers="[{Suffrage:Voter ID:server-022c65c4-5b03-4100-9d02-329c05d6d9e8 Address:022c65c4-5b03-4100-9d02-329c05d6d9e8} {Suffrage:Voter ID:server-adffe494-0224-4fde-2822-ff5110673309 Address:adffe494-0224-4fde-2822-ff5110673309} {Suffrage:Voter ID:server-a6e2f418-d993-1a67-ab11-d1d8599c19bc Address:a6e2f418-d993-1a67-ab11-d1d8599c19bc}]" -2026-04-22T10:09:26.152Z [INFO] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: removed peer, stopping replication: peer=server-6e2809f4-b3fe-466e-5eb3-6f6394030c30 last-index=3 -2026-04-22T10:09:26.152Z [INFO] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: aborting pipeline replication: peer="{Voter server-6e2809f4-b3fe-466e-5eb3-6f6394030c30 6e2809f4-b3fe-466e-5eb3-6f6394030c30}" -2026-04-22T10:09:26.152Z [ERROR] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: failed to pipeline appendEntries: peer="{Voter server-022c65c4-5b03-4100-9d02-329c05d6d9e8 022c65c4-5b03-4100-9d02-329c05d6d9e8}" error="append pipeline closed" -2026-04-22T10:09:26.152Z [INFO] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: aborting pipeline replication: peer="{Voter server-022c65c4-5b03-4100-9d02-329c05d6d9e8 022c65c4-5b03-4100-9d02-329c05d6d9e8}" -2026-04-22T10:09:26.152Z [ERROR] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: failed to pipeline appendEntries: peer="{Voter server-adffe494-0224-4fde-2822-ff5110673309 adffe494-0224-4fde-2822-ff5110673309}" error="append pipeline closed" -2026-04-22T10:09:26.152Z [INFO] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: aborting pipeline replication: peer="{Voter server-adffe494-0224-4fde-2822-ff5110673309 adffe494-0224-4fde-2822-ff5110673309}" -2026-04-22T10:09:26.157Z [ERROR] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: failed to heartbeat to: peer=adffe494-0224-4fde-2822-ff5110673309 backoff time=10ms error="failed to connect to peer: adffe494-0224-4fde-2822-ff5110673309" -2026-04-22T10:09:26.157Z [ERROR] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: failed to heartbeat to: peer=022c65c4-5b03-4100-9d02-329c05d6d9e8 backoff time=10ms error="failed to connect to peer: 022c65c4-5b03-4100-9d02-329c05d6d9e8" -2026-04-22T10:09:26.161Z [ERROR] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: failed to appendEntries to: peer="{Voter server-022c65c4-5b03-4100-9d02-329c05d6d9e8 022c65c4-5b03-4100-9d02-329c05d6d9e8}" error="failed to connect to peer: 022c65c4-5b03-4100-9d02-329c05d6d9e8" -2026-04-22T10:09:26.161Z [ERROR] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: failed to appendEntries to: peer="{Voter server-adffe494-0224-4fde-2822-ff5110673309 adffe494-0224-4fde-2822-ff5110673309}" error="failed to connect to peer: adffe494-0224-4fde-2822-ff5110673309" -2026-04-22T10:09:26.176Z [ERROR] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: failed to heartbeat to: peer=022c65c4-5b03-4100-9d02-329c05d6d9e8 backoff time=10ms error="failed to connect to peer: 022c65c4-5b03-4100-9d02-329c05d6d9e8" -2026-04-22T10:09:26.177Z [ERROR] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: failed to heartbeat to: peer=adffe494-0224-4fde-2822-ff5110673309 backoff time=10ms error="failed to connect to peer: adffe494-0224-4fde-2822-ff5110673309" -2026-04-22T10:09:26.184Z [ERROR] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: failed to appendEntries to: peer="{Voter server-022c65c4-5b03-4100-9d02-329c05d6d9e8 022c65c4-5b03-4100-9d02-329c05d6d9e8}" error="failed to connect to peer: 022c65c4-5b03-4100-9d02-329c05d6d9e8" -2026-04-22T10:09:26.184Z [ERROR] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: failed to appendEntries to: peer="{Voter server-adffe494-0224-4fde-2822-ff5110673309 adffe494-0224-4fde-2822-ff5110673309}" error="failed to connect to peer: adffe494-0224-4fde-2822-ff5110673309" -2026-04-22T10:09:26.194Z [ERROR] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: failed to heartbeat to: peer=022c65c4-5b03-4100-9d02-329c05d6d9e8 backoff time=10ms error="failed to connect to peer: 022c65c4-5b03-4100-9d02-329c05d6d9e8" -2026-04-22T10:09:26.195Z [WARN] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: failed to contact: server-id=server-022c65c4-5b03-4100-9d02-329c05d6d9e8 time=50.448013ms -2026-04-22T10:09:26.199Z [ERROR] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: failed to heartbeat to: peer=adffe494-0224-4fde-2822-ff5110673309 backoff time=10ms error="failed to connect to peer: adffe494-0224-4fde-2822-ff5110673309" -2026-04-22T10:09:26.201Z [ERROR] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: failed to appendEntries to: peer="{Voter server-022c65c4-5b03-4100-9d02-329c05d6d9e8 022c65c4-5b03-4100-9d02-329c05d6d9e8}" error="failed to connect to peer: 022c65c4-5b03-4100-9d02-329c05d6d9e8" -2026-04-22T10:09:26.205Z [ERROR] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: failed to appendEntries to: peer="{Voter server-adffe494-0224-4fde-2822-ff5110673309 adffe494-0224-4fde-2822-ff5110673309}" error="failed to connect to peer: adffe494-0224-4fde-2822-ff5110673309" -2026-04-22T10:09:26.206Z [WARN] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: failed to contact: server-id=server-022c65c4-5b03-4100-9d02-329c05d6d9e8 time=61.440102ms -2026-04-22T10:09:26.206Z [WARN] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: failed to contact: server-id=server-adffe494-0224-4fde-2822-ff5110673309 time=55.284054ms -2026-04-22T10:09:26.206Z [WARN] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: failed to contact quorum of nodes, stepping down -2026-04-22T10:09:26.207Z [INFO] server-a6e2f418-d993-1a67-ab11-d1d8599c19bc: entering follower state: follower="Node at a6e2f418-d993-1a67-ab11-d1d8599c19bc [Follower]" leader-address= leader-id= ---- PASS: TestRaft_RemoveFollower_SplitCluster (0.45s) +2025-03-20T03:49:40.433Z [INFO] server-12177400-25cf-9a5b-8f97-1ff7b561f22e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-12177400-25cf-9a5b-8f97-1ff7b561f22e Address:12177400-25cf-9a5b-8f97-1ff7b561f22e} {Suffrage:Voter ID:server-80cf8ffd-13ea-88fe-837f-b897870ad3ad Address:80cf8ffd-13ea-88fe-837f-b897870ad3ad} {Suffrage:Voter ID:server-187b5c57-b8e1-adb3-e2d5-949979850587 Address:187b5c57-b8e1-adb3-e2d5-949979850587} {Suffrage:Voter ID:server-e5d48d39-64c0-089d-263c-5979154a8743 Address:e5d48d39-64c0-089d-263c-5979154a8743}]" +2025-03-20T03:49:40.433Z [INFO] server-12177400-25cf-9a5b-8f97-1ff7b561f22e: entering follower state: follower="Node at 12177400-25cf-9a5b-8f97-1ff7b561f22e [Follower]" leader-address= leader-id= +2025-03-20T03:49:40.433Z [INFO] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-12177400-25cf-9a5b-8f97-1ff7b561f22e Address:12177400-25cf-9a5b-8f97-1ff7b561f22e} {Suffrage:Voter ID:server-80cf8ffd-13ea-88fe-837f-b897870ad3ad Address:80cf8ffd-13ea-88fe-837f-b897870ad3ad} {Suffrage:Voter ID:server-187b5c57-b8e1-adb3-e2d5-949979850587 Address:187b5c57-b8e1-adb3-e2d5-949979850587} {Suffrage:Voter ID:server-e5d48d39-64c0-089d-263c-5979154a8743 Address:e5d48d39-64c0-089d-263c-5979154a8743}]" +2025-03-20T03:49:40.433Z [INFO] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: entering follower state: follower="Node at 80cf8ffd-13ea-88fe-837f-b897870ad3ad [Follower]" leader-address= leader-id= +2025-03-20T03:49:40.433Z [INFO] server-187b5c57-b8e1-adb3-e2d5-949979850587: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-12177400-25cf-9a5b-8f97-1ff7b561f22e Address:12177400-25cf-9a5b-8f97-1ff7b561f22e} {Suffrage:Voter ID:server-80cf8ffd-13ea-88fe-837f-b897870ad3ad Address:80cf8ffd-13ea-88fe-837f-b897870ad3ad} {Suffrage:Voter ID:server-187b5c57-b8e1-adb3-e2d5-949979850587 Address:187b5c57-b8e1-adb3-e2d5-949979850587} {Suffrage:Voter ID:server-e5d48d39-64c0-089d-263c-5979154a8743 Address:e5d48d39-64c0-089d-263c-5979154a8743}]" +2025-03-20T03:49:40.433Z [INFO] server-187b5c57-b8e1-adb3-e2d5-949979850587: entering follower state: follower="Node at 187b5c57-b8e1-adb3-e2d5-949979850587 [Follower]" leader-address= leader-id= +2025-03-20T03:49:40.433Z [INFO] server-e5d48d39-64c0-089d-263c-5979154a8743: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-12177400-25cf-9a5b-8f97-1ff7b561f22e Address:12177400-25cf-9a5b-8f97-1ff7b561f22e} {Suffrage:Voter ID:server-80cf8ffd-13ea-88fe-837f-b897870ad3ad Address:80cf8ffd-13ea-88fe-837f-b897870ad3ad} {Suffrage:Voter ID:server-187b5c57-b8e1-adb3-e2d5-949979850587 Address:187b5c57-b8e1-adb3-e2d5-949979850587} {Suffrage:Voter ID:server-e5d48d39-64c0-089d-263c-5979154a8743 Address:e5d48d39-64c0-089d-263c-5979154a8743}]" +2025-03-20T03:49:40.433Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:40.433Z [INFO] server-e5d48d39-64c0-089d-263c-5979154a8743: entering follower state: follower="Node at e5d48d39-64c0-089d-263c-5979154a8743 [Follower]" leader-address= leader-id= +2025-03-20T03:49:40.493Z [WARN] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:40.493Z [INFO] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: entering candidate state: node="Node at 80cf8ffd-13ea-88fe-837f-b897870ad3ad [Candidate]" term=2 +2025-03-20T03:49:40.493Z [INFO] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: election won: term=2 tally=3 +2025-03-20T03:49:40.493Z [INFO] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: entering leader state: leader="Node at 80cf8ffd-13ea-88fe-837f-b897870ad3ad [Leader]" +2025-03-20T03:49:40.493Z [INFO] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: added peer, starting replication: peer=server-12177400-25cf-9a5b-8f97-1ff7b561f22e +2025-03-20T03:49:40.493Z [INFO] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: added peer, starting replication: peer=server-187b5c57-b8e1-adb3-e2d5-949979850587 +2025-03-20T03:49:40.493Z [INFO] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: added peer, starting replication: peer=server-e5d48d39-64c0-089d-263c-5979154a8743 +2025-03-20T03:49:40.494Z [INFO] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: pipelining replication: peer="{Voter server-e5d48d39-64c0-089d-263c-5979154a8743 e5d48d39-64c0-089d-263c-5979154a8743}" +2025-03-20T03:49:40.494Z [INFO] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: pipelining replication: peer="{Voter server-12177400-25cf-9a5b-8f97-1ff7b561f22e 12177400-25cf-9a5b-8f97-1ff7b561f22e}" +2025-03-20T03:49:40.494Z [INFO] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: pipelining replication: peer="{Voter server-187b5c57-b8e1-adb3-e2d5-949979850587 187b5c57-b8e1-adb3-e2d5-949979850587}" +2025-03-20T03:49:40.599Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:40.494192342 +0000 UTC m=+12.168645182 (1 nodes), 60.377189ms from start of poll, 61.12878ms from cluster start. Timeout at 2025-03-20 03:49:40.599197945 +0000 UTC m=+12.273650774, 105.005592ms after stability +2025-03-20T03:49:40.705Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:49:40.810Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:49:40.705402345 +0000 UTC m=+12.379855222 (3 nodes), 2.156µs from start of poll, 272.33882ms from cluster start. Timeout at 2025-03-20 03:49:40.81041576 +0000 UTC m=+12.484868734, 105.013512ms after stability +2025-03-20T03:49:40.810Z [INFO] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: updating configuration: command=RemoveServer server-id=server-e5d48d39-64c0-089d-263c-5979154a8743 server-addr= servers="[{Suffrage:Voter ID:server-12177400-25cf-9a5b-8f97-1ff7b561f22e Address:12177400-25cf-9a5b-8f97-1ff7b561f22e} {Suffrage:Voter ID:server-80cf8ffd-13ea-88fe-837f-b897870ad3ad Address:80cf8ffd-13ea-88fe-837f-b897870ad3ad} {Suffrage:Voter ID:server-187b5c57-b8e1-adb3-e2d5-949979850587 Address:187b5c57-b8e1-adb3-e2d5-949979850587}]" +2025-03-20T03:49:40.811Z [INFO] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: removed peer, stopping replication: peer=server-e5d48d39-64c0-089d-263c-5979154a8743 last-index=3 +2025-03-20T03:49:40.811Z [INFO] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: aborting pipeline replication: peer="{Voter server-e5d48d39-64c0-089d-263c-5979154a8743 e5d48d39-64c0-089d-263c-5979154a8743}" +2025-03-20T03:49:40.811Z [ERROR] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: failed to pipeline appendEntries: peer="{Voter server-12177400-25cf-9a5b-8f97-1ff7b561f22e 12177400-25cf-9a5b-8f97-1ff7b561f22e}" error="append pipeline closed" +2025-03-20T03:49:40.811Z [INFO] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: aborting pipeline replication: peer="{Voter server-12177400-25cf-9a5b-8f97-1ff7b561f22e 12177400-25cf-9a5b-8f97-1ff7b561f22e}" +2025-03-20T03:49:40.811Z [ERROR] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: failed to pipeline appendEntries: peer="{Voter server-187b5c57-b8e1-adb3-e2d5-949979850587 187b5c57-b8e1-adb3-e2d5-949979850587}" error="append pipeline closed" +2025-03-20T03:49:40.811Z [INFO] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: aborting pipeline replication: peer="{Voter server-187b5c57-b8e1-adb3-e2d5-949979850587 187b5c57-b8e1-adb3-e2d5-949979850587}" +2025-03-20T03:49:40.814Z [ERROR] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: failed to heartbeat to: peer=12177400-25cf-9a5b-8f97-1ff7b561f22e backoff time=10ms error="failed to connect to peer: 12177400-25cf-9a5b-8f97-1ff7b561f22e" +2025-03-20T03:49:40.816Z [ERROR] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: failed to heartbeat to: peer=187b5c57-b8e1-adb3-e2d5-949979850587 backoff time=10ms error="failed to connect to peer: 187b5c57-b8e1-adb3-e2d5-949979850587" +2025-03-20T03:49:40.818Z [ERROR] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: failed to appendEntries to: peer="{Voter server-187b5c57-b8e1-adb3-e2d5-949979850587 187b5c57-b8e1-adb3-e2d5-949979850587}" error="failed to connect to peer: 187b5c57-b8e1-adb3-e2d5-949979850587" +2025-03-20T03:49:40.819Z [ERROR] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: failed to appendEntries to: peer="{Voter server-12177400-25cf-9a5b-8f97-1ff7b561f22e 12177400-25cf-9a5b-8f97-1ff7b561f22e}" error="failed to connect to peer: 12177400-25cf-9a5b-8f97-1ff7b561f22e" +2025-03-20T03:49:40.833Z [ERROR] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: failed to heartbeat to: peer=187b5c57-b8e1-adb3-e2d5-949979850587 backoff time=10ms error="failed to connect to peer: 187b5c57-b8e1-adb3-e2d5-949979850587" +2025-03-20T03:49:40.834Z [ERROR] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: failed to heartbeat to: peer=12177400-25cf-9a5b-8f97-1ff7b561f22e backoff time=10ms error="failed to connect to peer: 12177400-25cf-9a5b-8f97-1ff7b561f22e" +2025-03-20T03:49:40.835Z [ERROR] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: failed to appendEntries to: peer="{Voter server-187b5c57-b8e1-adb3-e2d5-949979850587 187b5c57-b8e1-adb3-e2d5-949979850587}" error="failed to connect to peer: 187b5c57-b8e1-adb3-e2d5-949979850587" +2025-03-20T03:49:40.836Z [ERROR] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: failed to appendEntries to: peer="{Voter server-12177400-25cf-9a5b-8f97-1ff7b561f22e 12177400-25cf-9a5b-8f97-1ff7b561f22e}" error="failed to connect to peer: 12177400-25cf-9a5b-8f97-1ff7b561f22e" +2025-03-20T03:49:40.852Z [ERROR] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: failed to heartbeat to: peer=187b5c57-b8e1-adb3-e2d5-949979850587 backoff time=10ms error="failed to connect to peer: 187b5c57-b8e1-adb3-e2d5-949979850587" +2025-03-20T03:49:40.854Z [ERROR] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: failed to heartbeat to: peer=12177400-25cf-9a5b-8f97-1ff7b561f22e backoff time=10ms error="failed to connect to peer: 12177400-25cf-9a5b-8f97-1ff7b561f22e" +2025-03-20T03:49:40.854Z [ERROR] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: failed to appendEntries to: peer="{Voter server-187b5c57-b8e1-adb3-e2d5-949979850587 187b5c57-b8e1-adb3-e2d5-949979850587}" error="failed to connect to peer: 187b5c57-b8e1-adb3-e2d5-949979850587" +2025-03-20T03:49:40.854Z [ERROR] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: failed to appendEntries to: peer="{Voter server-12177400-25cf-9a5b-8f97-1ff7b561f22e 12177400-25cf-9a5b-8f97-1ff7b561f22e}" error="failed to connect to peer: 12177400-25cf-9a5b-8f97-1ff7b561f22e" +2025-03-20T03:49:40.859Z [WARN] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: failed to contact: server-id=server-12177400-25cf-9a5b-8f97-1ff7b561f22e time=51.134278ms +2025-03-20T03:49:40.868Z [ERROR] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: failed to heartbeat to: peer=187b5c57-b8e1-adb3-e2d5-949979850587 backoff time=20ms error="failed to connect to peer: 187b5c57-b8e1-adb3-e2d5-949979850587" +2025-03-20T03:49:40.869Z [WARN] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: failed to contact: server-id=server-12177400-25cf-9a5b-8f97-1ff7b561f22e time=61.442688ms +2025-03-20T03:49:40.869Z [WARN] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: failed to contact: server-id=server-187b5c57-b8e1-adb3-e2d5-949979850587 time=60.130035ms +2025-03-20T03:49:40.869Z [WARN] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: failed to contact quorum of nodes, stepping down +2025-03-20T03:49:40.869Z [INFO] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: entering follower state: follower="Node at 80cf8ffd-13ea-88fe-837f-b897870ad3ad [Follower]" leader-address= leader-id= +2025-03-20T03:49:40.870Z [ERROR] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: failed to appendEntries to: peer="{Voter server-12177400-25cf-9a5b-8f97-1ff7b561f22e 12177400-25cf-9a5b-8f97-1ff7b561f22e}" error="failed to connect to peer: 12177400-25cf-9a5b-8f97-1ff7b561f22e" +2025-03-20T03:49:40.870Z [ERROR] server-80cf8ffd-13ea-88fe-837f-b897870ad3ad: failed to appendEntries to: peer="{Voter server-187b5c57-b8e1-adb3-e2d5-949979850587 187b5c57-b8e1-adb3-e2d5-949979850587}" error="failed to connect to peer: 187b5c57-b8e1-adb3-e2d5-949979850587" +--- PASS: TestRaft_RemoveFollower_SplitCluster (0.44s) === RUN TestRaft_AddKnownPeer -2026-04-22T10:09:26.208Z [INFO] server-21405755-d305-b391-a77a-024ad6d06275: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-21405755-d305-b391-a77a-024ad6d06275 Address:21405755-d305-b391-a77a-024ad6d06275} {Suffrage:Voter ID:server-ae32f513-0d09-fbbe-fe92-87e2d4c2710e Address:ae32f513-0d09-fbbe-fe92-87e2d4c2710e} {Suffrage:Voter ID:server-a2243f9e-153c-770f-4ee7-62b06de9eba6 Address:a2243f9e-153c-770f-4ee7-62b06de9eba6}]" -2026-04-22T10:09:26.208Z [INFO] server-ae32f513-0d09-fbbe-fe92-87e2d4c2710e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-21405755-d305-b391-a77a-024ad6d06275 Address:21405755-d305-b391-a77a-024ad6d06275} {Suffrage:Voter ID:server-ae32f513-0d09-fbbe-fe92-87e2d4c2710e Address:ae32f513-0d09-fbbe-fe92-87e2d4c2710e} {Suffrage:Voter ID:server-a2243f9e-153c-770f-4ee7-62b06de9eba6 Address:a2243f9e-153c-770f-4ee7-62b06de9eba6}]" -2026-04-22T10:09:26.208Z [INFO] server-a2243f9e-153c-770f-4ee7-62b06de9eba6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-21405755-d305-b391-a77a-024ad6d06275 Address:21405755-d305-b391-a77a-024ad6d06275} {Suffrage:Voter ID:server-ae32f513-0d09-fbbe-fe92-87e2d4c2710e Address:ae32f513-0d09-fbbe-fe92-87e2d4c2710e} {Suffrage:Voter ID:server-a2243f9e-153c-770f-4ee7-62b06de9eba6 Address:a2243f9e-153c-770f-4ee7-62b06de9eba6}]" -2026-04-22T10:09:26.208Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:26.208Z [INFO] server-21405755-d305-b391-a77a-024ad6d06275: entering follower state: follower="Node at 21405755-d305-b391-a77a-024ad6d06275 [Follower]" leader-address= leader-id= -2026-04-22T10:09:26.208Z [INFO] server-ae32f513-0d09-fbbe-fe92-87e2d4c2710e: entering follower state: follower="Node at ae32f513-0d09-fbbe-fe92-87e2d4c2710e [Follower]" leader-address= leader-id= -2026-04-22T10:09:26.208Z [INFO] server-a2243f9e-153c-770f-4ee7-62b06de9eba6: entering follower state: follower="Node at a2243f9e-153c-770f-4ee7-62b06de9eba6 [Follower]" leader-address= leader-id= -2026-04-22T10:09:26.269Z [WARN] server-21405755-d305-b391-a77a-024ad6d06275: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:26.269Z [INFO] server-21405755-d305-b391-a77a-024ad6d06275: entering candidate state: node="Node at 21405755-d305-b391-a77a-024ad6d06275 [Candidate]" term=2 -2026-04-22T10:09:26.269Z [INFO] server-21405755-d305-b391-a77a-024ad6d06275: election won: term=2 tally=2 -2026-04-22T10:09:26.269Z [INFO] server-21405755-d305-b391-a77a-024ad6d06275: entering leader state: leader="Node at 21405755-d305-b391-a77a-024ad6d06275 [Leader]" -2026-04-22T10:09:26.269Z [INFO] server-21405755-d305-b391-a77a-024ad6d06275: added peer, starting replication: peer=server-ae32f513-0d09-fbbe-fe92-87e2d4c2710e -2026-04-22T10:09:26.269Z [INFO] server-21405755-d305-b391-a77a-024ad6d06275: added peer, starting replication: peer=server-a2243f9e-153c-770f-4ee7-62b06de9eba6 -2026-04-22T10:09:26.270Z [INFO] server-21405755-d305-b391-a77a-024ad6d06275: pipelining replication: peer="{Voter server-a2243f9e-153c-770f-4ee7-62b06de9eba6 a2243f9e-153c-770f-4ee7-62b06de9eba6}" -2026-04-22T10:09:26.270Z [INFO] server-21405755-d305-b391-a77a-024ad6d06275: pipelining replication: peer="{Voter server-ae32f513-0d09-fbbe-fe92-87e2d4c2710e ae32f513-0d09-fbbe-fe92-87e2d4c2710e}" -2026-04-22T10:09:26.377Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:26.270625208 +0000 UTC m=+12.294421326 (1 nodes), 61.936392ms from start of poll, 62.28398ms from cluster start. Timeout at 2026-04-22 10:09:26.375630407 +0000 UTC m=+12.399426497, 105.005171ms after stability -2026-04-22T10:09:26.377Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:09:26.483Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:09:26.377704189 +0000 UTC m=+12.401500304 (2 nodes), 1.244µs from start of poll, 169.362958ms from cluster start. Timeout at 2026-04-22 10:09:26.482718966 +0000 UTC m=+12.506515107, 105.014803ms after stability -2026-04-22T10:09:26.483Z [INFO] server-21405755-d305-b391-a77a-024ad6d06275: updating configuration: command=AddVoter server-id=server-ae32f513-0d09-fbbe-fe92-87e2d4c2710e server-addr=ae32f513-0d09-fbbe-fe92-87e2d4c2710e servers="[{Suffrage:Voter ID:server-21405755-d305-b391-a77a-024ad6d06275 Address:21405755-d305-b391-a77a-024ad6d06275} {Suffrage:Voter ID:server-ae32f513-0d09-fbbe-fe92-87e2d4c2710e Address:ae32f513-0d09-fbbe-fe92-87e2d4c2710e} {Suffrage:Voter ID:server-a2243f9e-153c-770f-4ee7-62b06de9eba6 Address:a2243f9e-153c-770f-4ee7-62b06de9eba6}]" -2026-04-22T10:09:26.483Z [INFO] server-21405755-d305-b391-a77a-024ad6d06275: aborting pipeline replication: peer="{Voter server-a2243f9e-153c-770f-4ee7-62b06de9eba6 a2243f9e-153c-770f-4ee7-62b06de9eba6}" -2026-04-22T10:09:26.483Z [INFO] server-21405755-d305-b391-a77a-024ad6d06275: aborting pipeline replication: peer="{Voter server-ae32f513-0d09-fbbe-fe92-87e2d4c2710e ae32f513-0d09-fbbe-fe92-87e2d4c2710e}" +2025-03-20T03:49:40.872Z [INFO] server-64a3357e-a8cc-64e1-ed7d-00fb53bdd48c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-64a3357e-a8cc-64e1-ed7d-00fb53bdd48c Address:64a3357e-a8cc-64e1-ed7d-00fb53bdd48c} {Suffrage:Voter ID:server-7f764b4c-d2dc-15fa-96f0-311cdcaeb572 Address:7f764b4c-d2dc-15fa-96f0-311cdcaeb572} {Suffrage:Voter ID:server-f749f321-c5d2-ca21-dfba-998352edcd47 Address:f749f321-c5d2-ca21-dfba-998352edcd47}]" +2025-03-20T03:49:40.872Z [INFO] server-7f764b4c-d2dc-15fa-96f0-311cdcaeb572: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-64a3357e-a8cc-64e1-ed7d-00fb53bdd48c Address:64a3357e-a8cc-64e1-ed7d-00fb53bdd48c} {Suffrage:Voter ID:server-7f764b4c-d2dc-15fa-96f0-311cdcaeb572 Address:7f764b4c-d2dc-15fa-96f0-311cdcaeb572} {Suffrage:Voter ID:server-f749f321-c5d2-ca21-dfba-998352edcd47 Address:f749f321-c5d2-ca21-dfba-998352edcd47}]" +2025-03-20T03:49:40.872Z [INFO] server-f749f321-c5d2-ca21-dfba-998352edcd47: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-64a3357e-a8cc-64e1-ed7d-00fb53bdd48c Address:64a3357e-a8cc-64e1-ed7d-00fb53bdd48c} {Suffrage:Voter ID:server-7f764b4c-d2dc-15fa-96f0-311cdcaeb572 Address:7f764b4c-d2dc-15fa-96f0-311cdcaeb572} {Suffrage:Voter ID:server-f749f321-c5d2-ca21-dfba-998352edcd47 Address:f749f321-c5d2-ca21-dfba-998352edcd47}]" +2025-03-20T03:49:40.872Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:40.872Z [INFO] server-64a3357e-a8cc-64e1-ed7d-00fb53bdd48c: entering follower state: follower="Node at 64a3357e-a8cc-64e1-ed7d-00fb53bdd48c [Follower]" leader-address= leader-id= +2025-03-20T03:49:40.872Z [INFO] server-7f764b4c-d2dc-15fa-96f0-311cdcaeb572: entering follower state: follower="Node at 7f764b4c-d2dc-15fa-96f0-311cdcaeb572 [Follower]" leader-address= leader-id= +2025-03-20T03:49:40.872Z [INFO] server-f749f321-c5d2-ca21-dfba-998352edcd47: entering follower state: follower="Node at f749f321-c5d2-ca21-dfba-998352edcd47 [Follower]" leader-address= leader-id= +2025-03-20T03:49:40.935Z [WARN] server-7f764b4c-d2dc-15fa-96f0-311cdcaeb572: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:40.935Z [INFO] server-7f764b4c-d2dc-15fa-96f0-311cdcaeb572: entering candidate state: node="Node at 7f764b4c-d2dc-15fa-96f0-311cdcaeb572 [Candidate]" term=2 +2025-03-20T03:49:40.936Z [INFO] server-7f764b4c-d2dc-15fa-96f0-311cdcaeb572: election won: term=2 tally=2 +2025-03-20T03:49:40.936Z [INFO] server-7f764b4c-d2dc-15fa-96f0-311cdcaeb572: entering leader state: leader="Node at 7f764b4c-d2dc-15fa-96f0-311cdcaeb572 [Leader]" +2025-03-20T03:49:40.936Z [INFO] server-7f764b4c-d2dc-15fa-96f0-311cdcaeb572: added peer, starting replication: peer=server-64a3357e-a8cc-64e1-ed7d-00fb53bdd48c +2025-03-20T03:49:40.936Z [INFO] server-7f764b4c-d2dc-15fa-96f0-311cdcaeb572: added peer, starting replication: peer=server-f749f321-c5d2-ca21-dfba-998352edcd47 +2025-03-20T03:49:40.936Z [INFO] server-7f764b4c-d2dc-15fa-96f0-311cdcaeb572: pipelining replication: peer="{Voter server-f749f321-c5d2-ca21-dfba-998352edcd47 f749f321-c5d2-ca21-dfba-998352edcd47}" +2025-03-20T03:49:40.936Z [INFO] server-7f764b4c-d2dc-15fa-96f0-311cdcaeb572: pipelining replication: peer="{Voter server-64a3357e-a8cc-64e1-ed7d-00fb53bdd48c 64a3357e-a8cc-64e1-ed7d-00fb53bdd48c}" +2025-03-20T03:49:41.041Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:40.936709146 +0000 UTC m=+12.611162174 (1 nodes), 64.374757ms from start of poll, 64.775589ms from cluster start. Timeout at 2025-03-20 03:49:41.041714048 +0000 UTC m=+12.716166941, 105.004767ms after stability +2025-03-20T03:49:41.041Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:49:41.147Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:49:41.042020603 +0000 UTC m=+12.716476522 (2 nodes), 30.019µs from start of poll, 170.089937ms from cluster start. Timeout at 2025-03-20 03:49:41.147037771 +0000 UTC m=+12.821490633, 105.014111ms after stability +2025-03-20T03:49:41.147Z [INFO] server-7f764b4c-d2dc-15fa-96f0-311cdcaeb572: updating configuration: command=AddVoter server-id=server-64a3357e-a8cc-64e1-ed7d-00fb53bdd48c server-addr=64a3357e-a8cc-64e1-ed7d-00fb53bdd48c servers="[{Suffrage:Voter ID:server-64a3357e-a8cc-64e1-ed7d-00fb53bdd48c Address:64a3357e-a8cc-64e1-ed7d-00fb53bdd48c} {Suffrage:Voter ID:server-7f764b4c-d2dc-15fa-96f0-311cdcaeb572 Address:7f764b4c-d2dc-15fa-96f0-311cdcaeb572} {Suffrage:Voter ID:server-f749f321-c5d2-ca21-dfba-998352edcd47 Address:f749f321-c5d2-ca21-dfba-998352edcd47}]" +2025-03-20T03:49:41.148Z [INFO] server-7f764b4c-d2dc-15fa-96f0-311cdcaeb572: aborting pipeline replication: peer="{Voter server-f749f321-c5d2-ca21-dfba-998352edcd47 f749f321-c5d2-ca21-dfba-998352edcd47}" +2025-03-20T03:49:41.148Z [INFO] server-7f764b4c-d2dc-15fa-96f0-311cdcaeb572: aborting pipeline replication: peer="{Voter server-64a3357e-a8cc-64e1-ed7d-00fb53bdd48c 64a3357e-a8cc-64e1-ed7d-00fb53bdd48c}" --- PASS: TestRaft_AddKnownPeer (0.28s) === RUN TestRaft_RemoveUnknownPeer -2026-04-22T10:09:26.484Z [INFO] server-42c99b24-97a6-1ae1-98e9-c8bb331808ac: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-42c99b24-97a6-1ae1-98e9-c8bb331808ac Address:42c99b24-97a6-1ae1-98e9-c8bb331808ac} {Suffrage:Voter ID:server-164c7a2a-4375-4a74-e697-5275ab269583 Address:164c7a2a-4375-4a74-e697-5275ab269583} {Suffrage:Voter ID:server-443073a1-fe74-5b49-7655-fd141b84bef6 Address:443073a1-fe74-5b49-7655-fd141b84bef6}]" -2026-04-22T10:09:26.484Z [INFO] server-164c7a2a-4375-4a74-e697-5275ab269583: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-42c99b24-97a6-1ae1-98e9-c8bb331808ac Address:42c99b24-97a6-1ae1-98e9-c8bb331808ac} {Suffrage:Voter ID:server-164c7a2a-4375-4a74-e697-5275ab269583 Address:164c7a2a-4375-4a74-e697-5275ab269583} {Suffrage:Voter ID:server-443073a1-fe74-5b49-7655-fd141b84bef6 Address:443073a1-fe74-5b49-7655-fd141b84bef6}]" -2026-04-22T10:09:26.484Z [INFO] server-42c99b24-97a6-1ae1-98e9-c8bb331808ac: entering follower state: follower="Node at 42c99b24-97a6-1ae1-98e9-c8bb331808ac [Follower]" leader-address= leader-id= -2026-04-22T10:09:26.484Z [INFO] server-443073a1-fe74-5b49-7655-fd141b84bef6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-42c99b24-97a6-1ae1-98e9-c8bb331808ac Address:42c99b24-97a6-1ae1-98e9-c8bb331808ac} {Suffrage:Voter ID:server-164c7a2a-4375-4a74-e697-5275ab269583 Address:164c7a2a-4375-4a74-e697-5275ab269583} {Suffrage:Voter ID:server-443073a1-fe74-5b49-7655-fd141b84bef6 Address:443073a1-fe74-5b49-7655-fd141b84bef6}]" -2026-04-22T10:09:26.484Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:26.484Z [INFO] server-164c7a2a-4375-4a74-e697-5275ab269583: entering follower state: follower="Node at 164c7a2a-4375-4a74-e697-5275ab269583 [Follower]" leader-address= leader-id= -2026-04-22T10:09:26.484Z [INFO] server-443073a1-fe74-5b49-7655-fd141b84bef6: entering follower state: follower="Node at 443073a1-fe74-5b49-7655-fd141b84bef6 [Follower]" leader-address= leader-id= -2026-04-22T10:09:26.537Z [WARN] server-164c7a2a-4375-4a74-e697-5275ab269583: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:26.537Z [INFO] server-164c7a2a-4375-4a74-e697-5275ab269583: entering candidate state: node="Node at 164c7a2a-4375-4a74-e697-5275ab269583 [Candidate]" term=2 -2026-04-22T10:09:26.537Z [INFO] server-164c7a2a-4375-4a74-e697-5275ab269583: election won: term=2 tally=2 -2026-04-22T10:09:26.537Z [INFO] server-164c7a2a-4375-4a74-e697-5275ab269583: entering leader state: leader="Node at 164c7a2a-4375-4a74-e697-5275ab269583 [Leader]" -2026-04-22T10:09:26.537Z [INFO] server-164c7a2a-4375-4a74-e697-5275ab269583: added peer, starting replication: peer=server-42c99b24-97a6-1ae1-98e9-c8bb331808ac -2026-04-22T10:09:26.537Z [INFO] server-164c7a2a-4375-4a74-e697-5275ab269583: added peer, starting replication: peer=server-443073a1-fe74-5b49-7655-fd141b84bef6 -2026-04-22T10:09:26.537Z [INFO] server-164c7a2a-4375-4a74-e697-5275ab269583: pipelining replication: peer="{Voter server-443073a1-fe74-5b49-7655-fd141b84bef6 443073a1-fe74-5b49-7655-fd141b84bef6}" -2026-04-22T10:09:26.537Z [WARN] server-42c99b24-97a6-1ae1-98e9-c8bb331808ac: rejecting vote request since our last term is greater: candidate=164c7a2a-4375-4a74-e697-5275ab269583 last-term=2 last-candidate-term=1 -2026-04-22T10:09:26.537Z [INFO] server-164c7a2a-4375-4a74-e697-5275ab269583: pipelining replication: peer="{Voter server-42c99b24-97a6-1ae1-98e9-c8bb331808ac 42c99b24-97a6-1ae1-98e9-c8bb331808ac}" -2026-04-22T10:09:26.643Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:26.537931801 +0000 UTC m=+12.561727899 (1 nodes), 53.00164ms from start of poll, 53.240564ms from cluster start. Timeout at 2026-04-22 10:09:26.642936395 +0000 UTC m=+12.666732509, 105.00461ms after stability -2026-04-22T10:09:26.643Z [INFO] server-164c7a2a-4375-4a74-e697-5275ab269583: updating configuration: command=RemoveServer server-id=d52cedc8-f911-b7a6-1db4-db7f31b29de1 server-addr= servers="[{Suffrage:Voter ID:server-42c99b24-97a6-1ae1-98e9-c8bb331808ac Address:42c99b24-97a6-1ae1-98e9-c8bb331808ac} {Suffrage:Voter ID:server-164c7a2a-4375-4a74-e697-5275ab269583 Address:164c7a2a-4375-4a74-e697-5275ab269583} {Suffrage:Voter ID:server-443073a1-fe74-5b49-7655-fd141b84bef6 Address:443073a1-fe74-5b49-7655-fd141b84bef6}]" -2026-04-22T10:09:26.643Z [INFO] server-164c7a2a-4375-4a74-e697-5275ab269583: aborting pipeline replication: peer="{Voter server-443073a1-fe74-5b49-7655-fd141b84bef6 443073a1-fe74-5b49-7655-fd141b84bef6}" -2026-04-22T10:09:26.643Z [INFO] server-164c7a2a-4375-4a74-e697-5275ab269583: aborting pipeline replication: peer="{Voter server-42c99b24-97a6-1ae1-98e9-c8bb331808ac 42c99b24-97a6-1ae1-98e9-c8bb331808ac}" ---- PASS: TestRaft_RemoveUnknownPeer (0.16s) +2025-03-20T03:49:41.150Z [INFO] server-4fd2f9eb-a732-22f3-69b9-f89ce376da9f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4fd2f9eb-a732-22f3-69b9-f89ce376da9f Address:4fd2f9eb-a732-22f3-69b9-f89ce376da9f} {Suffrage:Voter ID:server-f2a99e3a-5553-64d1-47d7-e854c847526e Address:f2a99e3a-5553-64d1-47d7-e854c847526e} {Suffrage:Voter ID:server-af3dcc3c-8328-1d6a-a720-b70f50f34f2a Address:af3dcc3c-8328-1d6a-a720-b70f50f34f2a}]" +2025-03-20T03:49:41.150Z [INFO] server-f2a99e3a-5553-64d1-47d7-e854c847526e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4fd2f9eb-a732-22f3-69b9-f89ce376da9f Address:4fd2f9eb-a732-22f3-69b9-f89ce376da9f} {Suffrage:Voter ID:server-f2a99e3a-5553-64d1-47d7-e854c847526e Address:f2a99e3a-5553-64d1-47d7-e854c847526e} {Suffrage:Voter ID:server-af3dcc3c-8328-1d6a-a720-b70f50f34f2a Address:af3dcc3c-8328-1d6a-a720-b70f50f34f2a}]" +2025-03-20T03:49:41.150Z [INFO] server-af3dcc3c-8328-1d6a-a720-b70f50f34f2a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4fd2f9eb-a732-22f3-69b9-f89ce376da9f Address:4fd2f9eb-a732-22f3-69b9-f89ce376da9f} {Suffrage:Voter ID:server-f2a99e3a-5553-64d1-47d7-e854c847526e Address:f2a99e3a-5553-64d1-47d7-e854c847526e} {Suffrage:Voter ID:server-af3dcc3c-8328-1d6a-a720-b70f50f34f2a Address:af3dcc3c-8328-1d6a-a720-b70f50f34f2a}]" +2025-03-20T03:49:41.150Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:41.150Z [INFO] server-4fd2f9eb-a732-22f3-69b9-f89ce376da9f: entering follower state: follower="Node at 4fd2f9eb-a732-22f3-69b9-f89ce376da9f [Follower]" leader-address= leader-id= +2025-03-20T03:49:41.150Z [INFO] server-f2a99e3a-5553-64d1-47d7-e854c847526e: entering follower state: follower="Node at f2a99e3a-5553-64d1-47d7-e854c847526e [Follower]" leader-address= leader-id= +2025-03-20T03:49:41.150Z [INFO] server-af3dcc3c-8328-1d6a-a720-b70f50f34f2a: entering follower state: follower="Node at af3dcc3c-8328-1d6a-a720-b70f50f34f2a [Follower]" leader-address= leader-id= +2025-03-20T03:49:41.224Z [WARN] server-f2a99e3a-5553-64d1-47d7-e854c847526e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:41.224Z [INFO] server-f2a99e3a-5553-64d1-47d7-e854c847526e: entering candidate state: node="Node at f2a99e3a-5553-64d1-47d7-e854c847526e [Candidate]" term=2 +2025-03-20T03:49:41.224Z [INFO] server-f2a99e3a-5553-64d1-47d7-e854c847526e: election won: term=2 tally=2 +2025-03-20T03:49:41.224Z [INFO] server-f2a99e3a-5553-64d1-47d7-e854c847526e: entering leader state: leader="Node at f2a99e3a-5553-64d1-47d7-e854c847526e [Leader]" +2025-03-20T03:49:41.224Z [INFO] server-f2a99e3a-5553-64d1-47d7-e854c847526e: added peer, starting replication: peer=server-4fd2f9eb-a732-22f3-69b9-f89ce376da9f +2025-03-20T03:49:41.225Z [INFO] server-f2a99e3a-5553-64d1-47d7-e854c847526e: added peer, starting replication: peer=server-af3dcc3c-8328-1d6a-a720-b70f50f34f2a +2025-03-20T03:49:41.225Z [INFO] server-f2a99e3a-5553-64d1-47d7-e854c847526e: pipelining replication: peer="{Voter server-af3dcc3c-8328-1d6a-a720-b70f50f34f2a af3dcc3c-8328-1d6a-a720-b70f50f34f2a}" +2025-03-20T03:49:41.225Z [INFO] server-f2a99e3a-5553-64d1-47d7-e854c847526e: pipelining replication: peer="{Voter server-4fd2f9eb-a732-22f3-69b9-f89ce376da9f 4fd2f9eb-a732-22f3-69b9-f89ce376da9f}" +2025-03-20T03:49:41.331Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:41.225564305 +0000 UTC m=+12.900017171 (1 nodes), 75.234459ms from start of poll, 75.665965ms from cluster start. Timeout at 2025-03-20 03:49:41.330567568 +0000 UTC m=+13.005020466, 105.003295ms after stability +2025-03-20T03:49:41.331Z [INFO] server-f2a99e3a-5553-64d1-47d7-e854c847526e: updating configuration: command=RemoveServer server-id=872eda3d-7ad3-ace0-7bee-2df72edd4d63 server-addr= servers="[{Suffrage:Voter ID:server-4fd2f9eb-a732-22f3-69b9-f89ce376da9f Address:4fd2f9eb-a732-22f3-69b9-f89ce376da9f} {Suffrage:Voter ID:server-f2a99e3a-5553-64d1-47d7-e854c847526e Address:f2a99e3a-5553-64d1-47d7-e854c847526e} {Suffrage:Voter ID:server-af3dcc3c-8328-1d6a-a720-b70f50f34f2a Address:af3dcc3c-8328-1d6a-a720-b70f50f34f2a}]" +2025-03-20T03:49:41.332Z [INFO] server-f2a99e3a-5553-64d1-47d7-e854c847526e: aborting pipeline replication: peer="{Voter server-af3dcc3c-8328-1d6a-a720-b70f50f34f2a af3dcc3c-8328-1d6a-a720-b70f50f34f2a}" +2025-03-20T03:49:41.332Z [INFO] server-f2a99e3a-5553-64d1-47d7-e854c847526e: aborting pipeline replication: peer="{Voter server-4fd2f9eb-a732-22f3-69b9-f89ce376da9f 4fd2f9eb-a732-22f3-69b9-f89ce376da9f}" +2025-03-20T03:49:41.833Z [ERROR] server-f2a99e3a-5553-64d1-47d7-e854c847526e: failed to heartbeat to: peer=af3dcc3c-8328-1d6a-a720-b70f50f34f2a backoff time=10ms error="command timed out" +--- PASS: TestRaft_RemoveUnknownPeer (0.69s) === RUN TestRaft_SnapshotRestore -2026-04-22T10:09:26.644Z [INFO] server-e7a732fd-cf69-e0ea-e6c8-3ebca551546c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e7a732fd-cf69-e0ea-e6c8-3ebca551546c Address:e7a732fd-cf69-e0ea-e6c8-3ebca551546c}]" -2026-04-22T10:09:26.644Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:26.644Z [INFO] server-e7a732fd-cf69-e0ea-e6c8-3ebca551546c: entering follower state: follower="Node at e7a732fd-cf69-e0ea-e6c8-3ebca551546c [Follower]" leader-address= leader-id= -2026-04-22T10:09:26.716Z [WARN] server-e7a732fd-cf69-e0ea-e6c8-3ebca551546c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:26.716Z [INFO] server-e7a732fd-cf69-e0ea-e6c8-3ebca551546c: entering candidate state: node="Node at e7a732fd-cf69-e0ea-e6c8-3ebca551546c [Candidate]" term=2 -2026-04-22T10:09:26.716Z [INFO] server-e7a732fd-cf69-e0ea-e6c8-3ebca551546c: election won: term=2 tally=1 -2026-04-22T10:09:26.716Z [INFO] server-e7a732fd-cf69-e0ea-e6c8-3ebca551546c: entering leader state: leader="Node at e7a732fd-cf69-e0ea-e6c8-3ebca551546c [Leader]" -2026-04-22T10:09:26.821Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:26.716399561 +0000 UTC m=+12.740195673 (1 nodes), 72.111323ms from start of poll, 72.216484ms from cluster start. Timeout at 2026-04-22 10:09:26.821401802 +0000 UTC m=+12.845197904, 105.002231ms after stability -2026-04-22T10:09:26.825Z [INFO] server-e7a732fd-cf69-e0ea-e6c8-3ebca551546c: starting snapshot up to: index=102 -2026-04-22T10:09:26.825Z [INFO] creating new snapshot: path=/tmp/raft2944875025/snapshots/2-102-1776852566825.tmp -2026-04-22T10:09:26.831Z [INFO] server-e7a732fd-cf69-e0ea-e6c8-3ebca551546c: compacting logs: from=1 to=92 -2026-04-22T10:09:26.831Z [INFO] server-e7a732fd-cf69-e0ea-e6c8-3ebca551546c: snapshot complete up to: index=102 -2026-04-22T10:09:26.831Z [INFO] server-e7a732fd-cf69-e0ea-e6c8-3ebca551546c: starting restore from snapshot: id=2-102-1776852566825 last-index=102 last-term=2 size-in-bytes=693 -2026-04-22T10:09:26.832Z [INFO] server-e7a732fd-cf69-e0ea-e6c8-3ebca551546c: snapshot restore progress: id=2-102-1776852566825 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% -2026-04-22T10:09:26.832Z [INFO] server-e7a732fd-cf69-e0ea-e6c8-3ebca551546c: restored from snapshot: id=2-102-1776852566825 last-index=102 last-term=2 size-in-bytes=693 -2026-04-22T10:09:26.832Z [INFO] server-e7a732fd-cf69-e0ea-e6c8-3ebca551546c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e7a732fd-cf69-e0ea-e6c8-3ebca551546c Address:e7a732fd-cf69-e0ea-e6c8-3ebca551546c}]" ---- PASS: TestRaft_SnapshotRestore (0.20s) +2025-03-20T03:49:41.836Z [INFO] server-33264a54-f5b0-ec8a-c48d-b94389dad8e5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-33264a54-f5b0-ec8a-c48d-b94389dad8e5 Address:33264a54-f5b0-ec8a-c48d-b94389dad8e5}]" +2025-03-20T03:49:41.836Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:41.837Z [INFO] server-33264a54-f5b0-ec8a-c48d-b94389dad8e5: entering follower state: follower="Node at 33264a54-f5b0-ec8a-c48d-b94389dad8e5 [Follower]" leader-address= leader-id= +2025-03-20T03:49:41.917Z [WARN] server-33264a54-f5b0-ec8a-c48d-b94389dad8e5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:41.917Z [INFO] server-33264a54-f5b0-ec8a-c48d-b94389dad8e5: entering candidate state: node="Node at 33264a54-f5b0-ec8a-c48d-b94389dad8e5 [Candidate]" term=2 +2025-03-20T03:49:41.917Z [INFO] server-33264a54-f5b0-ec8a-c48d-b94389dad8e5: election won: term=2 tally=1 +2025-03-20T03:49:41.917Z [INFO] server-33264a54-f5b0-ec8a-c48d-b94389dad8e5: entering leader state: leader="Node at 33264a54-f5b0-ec8a-c48d-b94389dad8e5 [Leader]" +2025-03-20T03:49:42.023Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:41.917895874 +0000 UTC m=+13.592348756 (1 nodes), 80.960339ms from start of poll, 81.173683ms from cluster start. Timeout at 2025-03-20 03:49:42.022903863 +0000 UTC m=+13.697356785, 105.008029ms after stability +2025-03-20T03:49:42.031Z [INFO] server-33264a54-f5b0-ec8a-c48d-b94389dad8e5: starting snapshot up to: index=102 +2025-03-20T03:49:42.034Z [INFO] creating new snapshot: path=/tmp/raft4090261838/snapshots/2-102-1742442582034.tmp +2025-03-20T03:49:42.035Z [INFO] server-33264a54-f5b0-ec8a-c48d-b94389dad8e5: compacting logs: from=1 to=92 +2025-03-20T03:49:42.035Z [INFO] server-33264a54-f5b0-ec8a-c48d-b94389dad8e5: snapshot complete up to: index=102 +2025-03-20T03:49:42.036Z [INFO] server-33264a54-f5b0-ec8a-c48d-b94389dad8e5: starting restore from snapshot: id=2-102-1742442582034 last-index=102 last-term=2 size-in-bytes=693 +2025-03-20T03:49:42.036Z [INFO] server-33264a54-f5b0-ec8a-c48d-b94389dad8e5: snapshot restore progress: id=2-102-1742442582034 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% +2025-03-20T03:49:42.036Z [INFO] server-33264a54-f5b0-ec8a-c48d-b94389dad8e5: restored from snapshot: id=2-102-1742442582034 last-index=102 last-term=2 size-in-bytes=693 +2025-03-20T03:49:42.036Z [INFO] server-33264a54-f5b0-ec8a-c48d-b94389dad8e5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-33264a54-f5b0-ec8a-c48d-b94389dad8e5 Address:33264a54-f5b0-ec8a-c48d-b94389dad8e5}]" +--- PASS: TestRaft_SnapshotRestore (0.21s) === RUN TestRaft_RestoreSnapshotOnStartup_Monotonic -2026-04-22T10:09:26.840Z [INFO] server-ed20eced-f87a-e739-d489-1ea36fdd710d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ed20eced-f87a-e739-d489-1ea36fdd710d Address:ed20eced-f87a-e739-d489-1ea36fdd710d}]" -2026-04-22T10:09:26.840Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:26.840Z [INFO] server-ed20eced-f87a-e739-d489-1ea36fdd710d: entering follower state: follower="Node at ed20eced-f87a-e739-d489-1ea36fdd710d [Follower]" leader-address= leader-id= -2026-04-22T10:09:26.893Z [WARN] server-ed20eced-f87a-e739-d489-1ea36fdd710d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:26.893Z [INFO] server-ed20eced-f87a-e739-d489-1ea36fdd710d: entering candidate state: node="Node at ed20eced-f87a-e739-d489-1ea36fdd710d [Candidate]" term=2 -2026-04-22T10:09:26.893Z [INFO] server-ed20eced-f87a-e739-d489-1ea36fdd710d: election won: term=2 tally=1 -2026-04-22T10:09:26.893Z [INFO] server-ed20eced-f87a-e739-d489-1ea36fdd710d: entering leader state: leader="Node at ed20eced-f87a-e739-d489-1ea36fdd710d [Leader]" -2026-04-22T10:09:26.999Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:26.893345208 +0000 UTC m=+12.917141296 (1 nodes), 52.965194ms from start of poll, 53.155994ms from cluster start. Timeout at 2026-04-22 10:09:26.998350838 +0000 UTC m=+13.022146964, 105.005668ms after stability -2026-04-22T10:09:27.002Z [INFO] server-ed20eced-f87a-e739-d489-1ea36fdd710d: starting snapshot up to: index=102 -2026-04-22T10:09:27.002Z [INFO] creating new snapshot: path=/tmp/raft3802017344/snapshots/2-102-1776852567002.tmp -2026-04-22T10:09:27.031Z [INFO] server-ed20eced-f87a-e739-d489-1ea36fdd710d: compacting logs: from=1 to=92 -2026-04-22T10:09:27.031Z [INFO] server-ed20eced-f87a-e739-d489-1ea36fdd710d: snapshot complete up to: index=102 -2026-04-22T10:09:27.032Z [INFO] server-ed20eced-f87a-e739-d489-1ea36fdd710d: starting restore from snapshot: id=2-102-1776852567002 last-index=102 last-term=2 size-in-bytes=693 -2026-04-22T10:09:27.032Z [INFO] server-ed20eced-f87a-e739-d489-1ea36fdd710d: snapshot restore progress: id=2-102-1776852567002 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% -2026-04-22T10:09:27.032Z [INFO] server-ed20eced-f87a-e739-d489-1ea36fdd710d: restored from snapshot: id=2-102-1776852567002 last-index=102 last-term=2 size-in-bytes=693 -2026-04-22T10:09:27.032Z [INFO] server-ed20eced-f87a-e739-d489-1ea36fdd710d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ed20eced-f87a-e739-d489-1ea36fdd710d Address:ed20eced-f87a-e739-d489-1ea36fdd710d}]" -2026-04-22T10:09:27.032Z [INFO] server-ed20eced-f87a-e739-d489-1ea36fdd710d: entering follower state: follower="Node at ed20eced-f87a-e739-d489-1ea36fdd710d [Shutdown]" leader-address= leader-id= ---- PASS: TestRaft_RestoreSnapshotOnStartup_Monotonic (0.19s) +2025-03-20T03:49:42.043Z [INFO] server-fcb26cde-cdb6-3dc5-c857-08ff2b97cb94: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fcb26cde-cdb6-3dc5-c857-08ff2b97cb94 Address:fcb26cde-cdb6-3dc5-c857-08ff2b97cb94}]" +2025-03-20T03:49:42.043Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:42.043Z [INFO] server-fcb26cde-cdb6-3dc5-c857-08ff2b97cb94: entering follower state: follower="Node at fcb26cde-cdb6-3dc5-c857-08ff2b97cb94 [Follower]" leader-address= leader-id= +2025-03-20T03:49:42.138Z [WARN] server-fcb26cde-cdb6-3dc5-c857-08ff2b97cb94: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:42.138Z [INFO] server-fcb26cde-cdb6-3dc5-c857-08ff2b97cb94: entering candidate state: node="Node at fcb26cde-cdb6-3dc5-c857-08ff2b97cb94 [Candidate]" term=2 +2025-03-20T03:49:42.138Z [INFO] server-fcb26cde-cdb6-3dc5-c857-08ff2b97cb94: election won: term=2 tally=1 +2025-03-20T03:49:42.138Z [INFO] server-fcb26cde-cdb6-3dc5-c857-08ff2b97cb94: entering leader state: leader="Node at fcb26cde-cdb6-3dc5-c857-08ff2b97cb94 [Leader]" +2025-03-20T03:49:42.244Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:42.138566469 +0000 UTC m=+13.813019326 (1 nodes), 94.659766ms from start of poll, 94.929383ms from cluster start. Timeout at 2025-03-20 03:49:42.243570154 +0000 UTC m=+13.918023004, 105.003678ms after stability +2025-03-20T03:49:42.250Z [INFO] server-fcb26cde-cdb6-3dc5-c857-08ff2b97cb94: starting snapshot up to: index=102 +2025-03-20T03:49:42.250Z [INFO] creating new snapshot: path=/tmp/raft1014009879/snapshots/2-102-1742442582250.tmp +2025-03-20T03:49:42.251Z [INFO] server-fcb26cde-cdb6-3dc5-c857-08ff2b97cb94: compacting logs: from=1 to=92 +2025-03-20T03:49:42.251Z [INFO] server-fcb26cde-cdb6-3dc5-c857-08ff2b97cb94: snapshot complete up to: index=102 +2025-03-20T03:49:42.252Z [INFO] server-fcb26cde-cdb6-3dc5-c857-08ff2b97cb94: starting restore from snapshot: id=2-102-1742442582250 last-index=102 last-term=2 size-in-bytes=693 +2025-03-20T03:49:42.252Z [INFO] server-fcb26cde-cdb6-3dc5-c857-08ff2b97cb94: snapshot restore progress: id=2-102-1742442582250 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% +2025-03-20T03:49:42.252Z [INFO] server-fcb26cde-cdb6-3dc5-c857-08ff2b97cb94: restored from snapshot: id=2-102-1742442582250 last-index=102 last-term=2 size-in-bytes=693 +2025-03-20T03:49:42.252Z [INFO] server-fcb26cde-cdb6-3dc5-c857-08ff2b97cb94: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fcb26cde-cdb6-3dc5-c857-08ff2b97cb94 Address:fcb26cde-cdb6-3dc5-c857-08ff2b97cb94}]" +--- PASS: TestRaft_RestoreSnapshotOnStartup_Monotonic (0.21s) === RUN TestRaft_SnapshotRestore_Progress -2026-04-22T10:09:27.033Z [INFO] server-6501ca49-e2ca-4ff1-a198-23ba0ae26ef0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6501ca49-e2ca-4ff1-a198-23ba0ae26ef0 Address:6501ca49-e2ca-4ff1-a198-23ba0ae26ef0}]" -2026-04-22T10:09:27.033Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:27.033Z [INFO] server-6501ca49-e2ca-4ff1-a198-23ba0ae26ef0: entering follower state: follower="Node at 6501ca49-e2ca-4ff1-a198-23ba0ae26ef0 [Follower]" leader-address= leader-id= -2026-04-22T10:09:27.108Z [WARN] server-6501ca49-e2ca-4ff1-a198-23ba0ae26ef0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:27.108Z [INFO] server-6501ca49-e2ca-4ff1-a198-23ba0ae26ef0: entering candidate state: node="Node at 6501ca49-e2ca-4ff1-a198-23ba0ae26ef0 [Candidate]" term=2 -2026-04-22T10:09:27.108Z [INFO] server-6501ca49-e2ca-4ff1-a198-23ba0ae26ef0: election won: term=2 tally=1 -2026-04-22T10:09:27.108Z [INFO] server-6501ca49-e2ca-4ff1-a198-23ba0ae26ef0: entering leader state: leader="Node at 6501ca49-e2ca-4ff1-a198-23ba0ae26ef0 [Leader]" -2026-04-22T10:09:27.214Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:27.108895621 +0000 UTC m=+13.132691707 (1 nodes), 75.453211ms from start of poll, 75.604436ms from cluster start. Timeout at 2026-04-22 10:09:27.213898726 +0000 UTC m=+13.237694825, 105.003118ms after stability -2026-04-22T10:09:27.217Z [INFO] server-6501ca49-e2ca-4ff1-a198-23ba0ae26ef0: starting snapshot up to: index=102 -2026-04-22T10:09:27.217Z [INFO] creating new snapshot: path=/tmp/raft962602678/snapshots/2-102-1776852567217.tmp -2026-04-22T10:09:27.255Z [INFO] server-6501ca49-e2ca-4ff1-a198-23ba0ae26ef0: compacting logs: from=1 to=92 -2026-04-22T10:09:27.255Z [INFO] server-6501ca49-e2ca-4ff1-a198-23ba0ae26ef0: snapshot complete up to: index=102 ---- PASS: TestRaft_SnapshotRestore_Progress (0.22s) +2025-03-20T03:49:42.253Z [INFO] server-656df152-3d1c-d1cc-b9e6-545b77d5e4e0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-656df152-3d1c-d1cc-b9e6-545b77d5e4e0 Address:656df152-3d1c-d1cc-b9e6-545b77d5e4e0}]" +2025-03-20T03:49:42.253Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:42.253Z [INFO] server-656df152-3d1c-d1cc-b9e6-545b77d5e4e0: entering follower state: follower="Node at 656df152-3d1c-d1cc-b9e6-545b77d5e4e0 [Follower]" leader-address= leader-id= +2025-03-20T03:49:42.315Z [WARN] server-656df152-3d1c-d1cc-b9e6-545b77d5e4e0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:42.315Z [INFO] server-656df152-3d1c-d1cc-b9e6-545b77d5e4e0: entering candidate state: node="Node at 656df152-3d1c-d1cc-b9e6-545b77d5e4e0 [Candidate]" term=2 +2025-03-20T03:49:42.315Z [INFO] server-656df152-3d1c-d1cc-b9e6-545b77d5e4e0: election won: term=2 tally=1 +2025-03-20T03:49:42.315Z [INFO] server-656df152-3d1c-d1cc-b9e6-545b77d5e4e0: entering leader state: leader="Node at 656df152-3d1c-d1cc-b9e6-545b77d5e4e0 [Leader]" +2025-03-20T03:49:42.421Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:42.315335123 +0000 UTC m=+13.989787991 (1 nodes), 61.78462ms from start of poll, 61.941003ms from cluster start. Timeout at 2025-03-20 03:49:42.42034366 +0000 UTC m=+14.094796580, 105.008589ms after stability +2025-03-20T03:49:42.427Z [INFO] server-656df152-3d1c-d1cc-b9e6-545b77d5e4e0: starting snapshot up to: index=102 +2025-03-20T03:49:42.427Z [INFO] creating new snapshot: path=/tmp/raft604890512/snapshots/2-102-1742442582427.tmp +2025-03-20T03:49:42.428Z [INFO] server-656df152-3d1c-d1cc-b9e6-545b77d5e4e0: compacting logs: from=1 to=92 +2025-03-20T03:49:42.428Z [INFO] server-656df152-3d1c-d1cc-b9e6-545b77d5e4e0: snapshot complete up to: index=102 +--- PASS: TestRaft_SnapshotRestore_Progress (0.18s) === RUN TestRaft_NoRestoreOnStart -2026-04-22T10:09:27.257Z [INFO] server-4123a9f7-c209-7b35-e81d-46eb7e57aa0a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4123a9f7-c209-7b35-e81d-46eb7e57aa0a Address:4123a9f7-c209-7b35-e81d-46eb7e57aa0a}]" -2026-04-22T10:09:27.257Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:27.257Z [INFO] server-4123a9f7-c209-7b35-e81d-46eb7e57aa0a: entering follower state: follower="Node at 4123a9f7-c209-7b35-e81d-46eb7e57aa0a [Follower]" leader-address= leader-id= -2026-04-22T10:09:27.347Z [WARN] server-4123a9f7-c209-7b35-e81d-46eb7e57aa0a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:27.347Z [INFO] server-4123a9f7-c209-7b35-e81d-46eb7e57aa0a: entering candidate state: node="Node at 4123a9f7-c209-7b35-e81d-46eb7e57aa0a [Candidate]" term=2 -2026-04-22T10:09:27.347Z [INFO] server-4123a9f7-c209-7b35-e81d-46eb7e57aa0a: election won: term=2 tally=1 -2026-04-22T10:09:27.347Z [INFO] server-4123a9f7-c209-7b35-e81d-46eb7e57aa0a: entering leader state: leader="Node at 4123a9f7-c209-7b35-e81d-46eb7e57aa0a [Leader]" -2026-04-22T10:09:27.452Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:27.347754545 +0000 UTC m=+13.371550662 (1 nodes), 89.79247ms from start of poll, 89.947435ms from cluster start. Timeout at 2026-04-22 10:09:27.452755545 +0000 UTC m=+13.476551639, 105.000977ms after stability -2026-04-22T10:09:27.455Z [INFO] server-4123a9f7-c209-7b35-e81d-46eb7e57aa0a: starting snapshot up to: index=102 -2026-04-22T10:09:27.455Z [INFO] creating new snapshot: path=/tmp/raft3042018787/snapshots/2-102-1776852567455.tmp -2026-04-22T10:09:27.480Z [INFO] server-4123a9f7-c209-7b35-e81d-46eb7e57aa0a: compacting logs: from=1 to=92 -2026-04-22T10:09:27.480Z [INFO] server-4123a9f7-c209-7b35-e81d-46eb7e57aa0a: snapshot complete up to: index=102 -2026-04-22T10:09:27.481Z [INFO] server-4123a9f7-c209-7b35-e81d-46eb7e57aa0a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4123a9f7-c209-7b35-e81d-46eb7e57aa0a Address:4123a9f7-c209-7b35-e81d-46eb7e57aa0a}]" ---- PASS: TestRaft_NoRestoreOnStart (0.22s) +2025-03-20T03:49:42.430Z [INFO] server-dc13e336-5962-4766-8055-a70f687c4f2f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dc13e336-5962-4766-8055-a70f687c4f2f Address:dc13e336-5962-4766-8055-a70f687c4f2f}]" +2025-03-20T03:49:42.430Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:42.431Z [INFO] server-dc13e336-5962-4766-8055-a70f687c4f2f: entering follower state: follower="Node at dc13e336-5962-4766-8055-a70f687c4f2f [Follower]" leader-address= leader-id= +2025-03-20T03:49:42.492Z [WARN] server-dc13e336-5962-4766-8055-a70f687c4f2f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:42.492Z [INFO] server-dc13e336-5962-4766-8055-a70f687c4f2f: entering candidate state: node="Node at dc13e336-5962-4766-8055-a70f687c4f2f [Candidate]" term=2 +2025-03-20T03:49:42.492Z [INFO] server-dc13e336-5962-4766-8055-a70f687c4f2f: election won: term=2 tally=1 +2025-03-20T03:49:42.492Z [INFO] server-dc13e336-5962-4766-8055-a70f687c4f2f: entering leader state: leader="Node at dc13e336-5962-4766-8055-a70f687c4f2f [Leader]" +2025-03-20T03:49:42.597Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:42.492550824 +0000 UTC m=+14.167003627 (1 nodes), 61.550295ms from start of poll, 61.76535ms from cluster start. Timeout at 2025-03-20 03:49:42.597556586 +0000 UTC m=+14.272009414, 105.005787ms after stability +2025-03-20T03:49:42.603Z [INFO] server-dc13e336-5962-4766-8055-a70f687c4f2f: starting snapshot up to: index=102 +2025-03-20T03:49:42.604Z [INFO] creating new snapshot: path=/tmp/raft3907964936/snapshots/2-102-1742442582604.tmp +2025-03-20T03:49:42.605Z [INFO] server-dc13e336-5962-4766-8055-a70f687c4f2f: compacting logs: from=1 to=92 +2025-03-20T03:49:42.605Z [INFO] server-dc13e336-5962-4766-8055-a70f687c4f2f: snapshot complete up to: index=102 +2025-03-20T03:49:42.605Z [INFO] server-dc13e336-5962-4766-8055-a70f687c4f2f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dc13e336-5962-4766-8055-a70f687c4f2f Address:dc13e336-5962-4766-8055-a70f687c4f2f}]" +--- PASS: TestRaft_NoRestoreOnStart (0.18s) === RUN TestRaft_SnapshotRestore_PeerChange -2026-04-22T10:09:27.481Z [INFO] server-4123a9f7-c209-7b35-e81d-46eb7e57aa0a: entering follower state: follower="Node at 4123a9f7-c209-7b35-e81d-46eb7e57aa0a [Follower]" leader-address= leader-id= raft_test.go:1266: Skipping unreliable test --- SKIP: TestRaft_SnapshotRestore_PeerChange (0.00s) === RUN TestRaft_AutoSnapshot -2026-04-22T10:09:27.481Z [INFO] server-17ec5c83-a0dc-f67a-81ec-926d5d82326c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-17ec5c83-a0dc-f67a-81ec-926d5d82326c Address:17ec5c83-a0dc-f67a-81ec-926d5d82326c}]" -2026-04-22T10:09:27.481Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:27.481Z [INFO] server-17ec5c83-a0dc-f67a-81ec-926d5d82326c: entering follower state: follower="Node at 17ec5c83-a0dc-f67a-81ec-926d5d82326c [Follower]" leader-address= leader-id= -2026-04-22T10:09:27.541Z [WARN] server-4123a9f7-c209-7b35-e81d-46eb7e57aa0a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:27.541Z [INFO] server-4123a9f7-c209-7b35-e81d-46eb7e57aa0a: entering candidate state: node="Node at 4123a9f7-c209-7b35-e81d-46eb7e57aa0a [Candidate]" term=3 -2026-04-22T10:09:27.541Z [INFO] server-4123a9f7-c209-7b35-e81d-46eb7e57aa0a: election won: term=3 tally=1 -2026-04-22T10:09:27.541Z [INFO] server-4123a9f7-c209-7b35-e81d-46eb7e57aa0a: entering leader state: leader="Node at 4123a9f7-c209-7b35-e81d-46eb7e57aa0a [Leader]" -2026-04-22T10:09:27.557Z [WARN] server-17ec5c83-a0dc-f67a-81ec-926d5d82326c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:27.557Z [INFO] server-17ec5c83-a0dc-f67a-81ec-926d5d82326c: entering candidate state: node="Node at 17ec5c83-a0dc-f67a-81ec-926d5d82326c [Candidate]" term=2 -2026-04-22T10:09:27.557Z [INFO] server-17ec5c83-a0dc-f67a-81ec-926d5d82326c: election won: term=2 tally=1 -2026-04-22T10:09:27.557Z [INFO] server-17ec5c83-a0dc-f67a-81ec-926d5d82326c: entering leader state: leader="Node at 17ec5c83-a0dc-f67a-81ec-926d5d82326c [Leader]" -2026-04-22T10:09:27.663Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:27.557275027 +0000 UTC m=+13.581071119 (1 nodes), 75.727762ms from start of poll, 75.847369ms from cluster start. Timeout at 2026-04-22 10:09:27.6622777 +0000 UTC m=+13.686073781, 105.002662ms after stability -2026-04-22T10:09:27.667Z [INFO] server-17ec5c83-a0dc-f67a-81ec-926d5d82326c: starting snapshot up to: index=102 -2026-04-22T10:09:27.667Z [INFO] creating new snapshot: path=/tmp/raft3916032151/snapshots/2-102-1776852567667.tmp -2026-04-22T10:09:27.694Z [INFO] server-17ec5c83-a0dc-f67a-81ec-926d5d82326c: compacting logs: from=1 to=92 -2026-04-22T10:09:27.694Z [INFO] server-17ec5c83-a0dc-f67a-81ec-926d5d82326c: snapshot complete up to: index=102 +2025-03-20T03:49:42.606Z [INFO] server-dc13e336-5962-4766-8055-a70f687c4f2f: entering follower state: follower="Node at dc13e336-5962-4766-8055-a70f687c4f2f [Follower]" leader-address= leader-id= +2025-03-20T03:49:42.606Z [INFO] server-c8c26b74-e128-4fc9-7251-53f70e92a05c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c8c26b74-e128-4fc9-7251-53f70e92a05c Address:c8c26b74-e128-4fc9-7251-53f70e92a05c}]" +2025-03-20T03:49:42.606Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:42.606Z [INFO] server-c8c26b74-e128-4fc9-7251-53f70e92a05c: entering follower state: follower="Node at c8c26b74-e128-4fc9-7251-53f70e92a05c [Follower]" leader-address= leader-id= +2025-03-20T03:49:42.678Z [WARN] server-c8c26b74-e128-4fc9-7251-53f70e92a05c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:42.678Z [INFO] server-c8c26b74-e128-4fc9-7251-53f70e92a05c: entering candidate state: node="Node at c8c26b74-e128-4fc9-7251-53f70e92a05c [Candidate]" term=2 +2025-03-20T03:49:42.678Z [INFO] server-c8c26b74-e128-4fc9-7251-53f70e92a05c: election won: term=2 tally=1 +2025-03-20T03:49:42.678Z [INFO] server-c8c26b74-e128-4fc9-7251-53f70e92a05c: entering leader state: leader="Node at c8c26b74-e128-4fc9-7251-53f70e92a05c [Leader]" +2025-03-20T03:49:42.689Z [WARN] server-dc13e336-5962-4766-8055-a70f687c4f2f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:42.689Z [INFO] server-dc13e336-5962-4766-8055-a70f687c4f2f: entering candidate state: node="Node at dc13e336-5962-4766-8055-a70f687c4f2f [Candidate]" term=3 +2025-03-20T03:49:42.689Z [INFO] server-dc13e336-5962-4766-8055-a70f687c4f2f: election won: term=3 tally=1 +2025-03-20T03:49:42.689Z [INFO] server-dc13e336-5962-4766-8055-a70f687c4f2f: entering leader state: leader="Node at dc13e336-5962-4766-8055-a70f687c4f2f [Leader]" +2025-03-20T03:49:42.784Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:42.678449458 +0000 UTC m=+14.352902249 (1 nodes), 71.8057ms from start of poll, 71.979099ms from cluster start. Timeout at 2025-03-20 03:49:42.783453406 +0000 UTC m=+14.457906212, 105.003963ms after stability +2025-03-20T03:49:42.794Z [INFO] server-c8c26b74-e128-4fc9-7251-53f70e92a05c: starting snapshot up to: index=102 +2025-03-20T03:49:42.794Z [INFO] creating new snapshot: path=/tmp/raft1073057005/snapshots/2-102-1742442582794.tmp +2025-03-20T03:49:42.796Z [INFO] server-c8c26b74-e128-4fc9-7251-53f70e92a05c: compacting logs: from=1 to=92 +2025-03-20T03:49:42.796Z [INFO] server-c8c26b74-e128-4fc9-7251-53f70e92a05c: snapshot complete up to: index=102 --- PASS: TestRaft_AutoSnapshot (0.29s) === RUN TestRaft_UserSnapshot -2026-04-22T10:09:27.771Z [INFO] server-124bdfe4-ae45-ab06-5a49-4f9ba36094df: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-124bdfe4-ae45-ab06-5a49-4f9ba36094df Address:124bdfe4-ae45-ab06-5a49-4f9ba36094df}]" -2026-04-22T10:09:27.771Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:27.771Z [INFO] server-124bdfe4-ae45-ab06-5a49-4f9ba36094df: entering follower state: follower="Node at 124bdfe4-ae45-ab06-5a49-4f9ba36094df [Follower]" leader-address= leader-id= -2026-04-22T10:09:27.844Z [WARN] server-124bdfe4-ae45-ab06-5a49-4f9ba36094df: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:27.844Z [INFO] server-124bdfe4-ae45-ab06-5a49-4f9ba36094df: entering candidate state: node="Node at 124bdfe4-ae45-ab06-5a49-4f9ba36094df [Candidate]" term=2 -2026-04-22T10:09:27.844Z [INFO] server-124bdfe4-ae45-ab06-5a49-4f9ba36094df: election won: term=2 tally=1 -2026-04-22T10:09:27.844Z [INFO] server-124bdfe4-ae45-ab06-5a49-4f9ba36094df: entering leader state: leader="Node at 124bdfe4-ae45-ab06-5a49-4f9ba36094df [Leader]" -2026-04-22T10:09:27.949Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:27.844316986 +0000 UTC m=+13.868113083 (1 nodes), 72.334202ms from start of poll, 72.457086ms from cluster start. Timeout at 2026-04-22 10:09:27.949320395 +0000 UTC m=+13.973116488, 105.003405ms after stability -2026-04-22T10:09:27.949Z [ERROR] server-124bdfe4-ae45-ab06-5a49-4f9ba36094df: failed to take snapshot: error="nothing new to snapshot" -2026-04-22T10:09:27.950Z [INFO] server-124bdfe4-ae45-ab06-5a49-4f9ba36094df: starting snapshot up to: index=12 -2026-04-22T10:09:27.950Z [INFO] creating new snapshot: path=/tmp/raft3781532292/snapshots/2-12-1776852567950.tmp -2026-04-22T10:09:27.950Z [INFO] server-124bdfe4-ae45-ab06-5a49-4f9ba36094df: compacting logs: from=1 to=2 -2026-04-22T10:09:27.950Z [INFO] server-124bdfe4-ae45-ab06-5a49-4f9ba36094df: snapshot complete up to: index=12 ---- PASS: TestRaft_UserSnapshot (0.18s) +2025-03-20T03:49:42.900Z [INFO] server-50617c79-adc6-1441-fb16-c9d523ca3df8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-50617c79-adc6-1441-fb16-c9d523ca3df8 Address:50617c79-adc6-1441-fb16-c9d523ca3df8}]" +2025-03-20T03:49:42.900Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:42.900Z [INFO] server-50617c79-adc6-1441-fb16-c9d523ca3df8: entering follower state: follower="Node at 50617c79-adc6-1441-fb16-c9d523ca3df8 [Follower]" leader-address= leader-id= +2025-03-20T03:49:42.953Z [WARN] server-50617c79-adc6-1441-fb16-c9d523ca3df8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:42.953Z [INFO] server-50617c79-adc6-1441-fb16-c9d523ca3df8: entering candidate state: node="Node at 50617c79-adc6-1441-fb16-c9d523ca3df8 [Candidate]" term=2 +2025-03-20T03:49:42.953Z [INFO] server-50617c79-adc6-1441-fb16-c9d523ca3df8: election won: term=2 tally=1 +2025-03-20T03:49:42.953Z [INFO] server-50617c79-adc6-1441-fb16-c9d523ca3df8: entering leader state: leader="Node at 50617c79-adc6-1441-fb16-c9d523ca3df8 [Leader]" +2025-03-20T03:49:43.059Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:42.954187961 +0000 UTC m=+14.628640835 (1 nodes), 53.702856ms from start of poll, 53.921399ms from cluster start. Timeout at 2025-03-20 03:49:43.059194134 +0000 UTC m=+14.733647010, 105.006175ms after stability +2025-03-20T03:49:43.059Z [ERROR] server-50617c79-adc6-1441-fb16-c9d523ca3df8: failed to take snapshot: error="nothing new to snapshot" +2025-03-20T03:49:43.060Z [INFO] server-50617c79-adc6-1441-fb16-c9d523ca3df8: starting snapshot up to: index=12 +2025-03-20T03:49:43.060Z [INFO] creating new snapshot: path=/tmp/raft2876347836/snapshots/2-12-1742442583060.tmp +2025-03-20T03:49:43.061Z [INFO] server-50617c79-adc6-1441-fb16-c9d523ca3df8: compacting logs: from=1 to=2 +2025-03-20T03:49:43.061Z [INFO] server-50617c79-adc6-1441-fb16-c9d523ca3df8: snapshot complete up to: index=12 +--- PASS: TestRaft_UserSnapshot (0.16s) === RUN TestRaft_UserRestore === RUN TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_false -2026-04-22T10:09:27.951Z [INFO] server-31bdcbab-7903-43cf-434e-63d1c8b786c7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31bdcbab-7903-43cf-434e-63d1c8b786c7 Address:31bdcbab-7903-43cf-434e-63d1c8b786c7} {Suffrage:Voter ID:server-004abba6-2704-f02c-4a1d-2f74074e297c Address:004abba6-2704-f02c-4a1d-2f74074e297c} {Suffrage:Voter ID:server-9280583d-a2ed-9ffa-4c97-536f654f8445 Address:9280583d-a2ed-9ffa-4c97-536f654f8445}]" -2026-04-22T10:09:27.951Z [INFO] server-31bdcbab-7903-43cf-434e-63d1c8b786c7: entering follower state: follower="Node at 31bdcbab-7903-43cf-434e-63d1c8b786c7 [Follower]" leader-address= leader-id= -2026-04-22T10:09:27.951Z [INFO] server-004abba6-2704-f02c-4a1d-2f74074e297c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31bdcbab-7903-43cf-434e-63d1c8b786c7 Address:31bdcbab-7903-43cf-434e-63d1c8b786c7} {Suffrage:Voter ID:server-004abba6-2704-f02c-4a1d-2f74074e297c Address:004abba6-2704-f02c-4a1d-2f74074e297c} {Suffrage:Voter ID:server-9280583d-a2ed-9ffa-4c97-536f654f8445 Address:9280583d-a2ed-9ffa-4c97-536f654f8445}]" -2026-04-22T10:09:27.951Z [INFO] server-004abba6-2704-f02c-4a1d-2f74074e297c: entering follower state: follower="Node at 004abba6-2704-f02c-4a1d-2f74074e297c [Follower]" leader-address= leader-id= -2026-04-22T10:09:27.952Z [INFO] server-9280583d-a2ed-9ffa-4c97-536f654f8445: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31bdcbab-7903-43cf-434e-63d1c8b786c7 Address:31bdcbab-7903-43cf-434e-63d1c8b786c7} {Suffrage:Voter ID:server-004abba6-2704-f02c-4a1d-2f74074e297c Address:004abba6-2704-f02c-4a1d-2f74074e297c} {Suffrage:Voter ID:server-9280583d-a2ed-9ffa-4c97-536f654f8445 Address:9280583d-a2ed-9ffa-4c97-536f654f8445}]" -2026-04-22T10:09:27.952Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:27.952Z [INFO] server-9280583d-a2ed-9ffa-4c97-536f654f8445: entering follower state: follower="Node at 9280583d-a2ed-9ffa-4c97-536f654f8445 [Follower]" leader-address= leader-id= -2026-04-22T10:09:28.599Z [WARN] server-31bdcbab-7903-43cf-434e-63d1c8b786c7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:28.599Z [INFO] server-31bdcbab-7903-43cf-434e-63d1c8b786c7: entering candidate state: node="Node at 31bdcbab-7903-43cf-434e-63d1c8b786c7 [Candidate]" term=2 -2026-04-22T10:09:28.599Z [INFO] server-31bdcbab-7903-43cf-434e-63d1c8b786c7: election won: term=2 tally=2 -2026-04-22T10:09:28.599Z [INFO] server-31bdcbab-7903-43cf-434e-63d1c8b786c7: entering leader state: leader="Node at 31bdcbab-7903-43cf-434e-63d1c8b786c7 [Leader]" -2026-04-22T10:09:28.599Z [INFO] server-31bdcbab-7903-43cf-434e-63d1c8b786c7: added peer, starting replication: peer=server-004abba6-2704-f02c-4a1d-2f74074e297c -2026-04-22T10:09:28.599Z [INFO] server-31bdcbab-7903-43cf-434e-63d1c8b786c7: added peer, starting replication: peer=server-9280583d-a2ed-9ffa-4c97-536f654f8445 -2026-04-22T10:09:28.599Z [INFO] server-31bdcbab-7903-43cf-434e-63d1c8b786c7: pipelining replication: peer="{Voter server-9280583d-a2ed-9ffa-4c97-536f654f8445 9280583d-a2ed-9ffa-4c97-536f654f8445}" -2026-04-22T10:09:28.599Z [INFO] server-31bdcbab-7903-43cf-434e-63d1c8b786c7: pipelining replication: peer="{Voter server-004abba6-2704-f02c-4a1d-2f74074e297c 004abba6-2704-f02c-4a1d-2f74074e297c}" -2026-04-22T10:09:29.604Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:28.599516741 +0000 UTC m=+14.623312839 (1 nodes), 647.432638ms from start of poll, 647.797295ms from cluster start. Timeout at 2026-04-22 10:09:29.604522824 +0000 UTC m=+15.628318948, 1.005006109s after stability -2026-04-22T10:09:29.605Z [INFO] server-31bdcbab-7903-43cf-434e-63d1c8b786c7: starting snapshot up to: index=12 -2026-04-22T10:09:29.605Z [INFO] creating new snapshot: path=/tmp/raft1998147525/snapshots/2-12-1776852569605.tmp -2026-04-22T10:09:29.606Z [INFO] server-31bdcbab-7903-43cf-434e-63d1c8b786c7: snapshot complete up to: index=12 -2026-04-22T10:09:29.606Z [INFO] creating new snapshot: path=/tmp/raft1998147525/snapshots/2-23-1776852569606.tmp -2026-04-22T10:09:29.607Z [INFO] server-31bdcbab-7903-43cf-434e-63d1c8b786c7: copied to local snapshot: bytes=71 -2026-04-22T10:09:29.607Z [INFO] server-31bdcbab-7903-43cf-434e-63d1c8b786c7: snapshot restore progress: id=2-23-1776852569606 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:29.607Z [INFO] server-31bdcbab-7903-43cf-434e-63d1c8b786c7: restored user snapshot: index=23 -2026-04-22T10:09:29.608Z [ERROR] server-31bdcbab-7903-43cf-434e-63d1c8b786c7: failed to get log: index=23 error="log not found" -2026-04-22T10:09:29.608Z [INFO] server-31bdcbab-7903-43cf-434e-63d1c8b786c7: aborting pipeline replication: peer="{Voter server-9280583d-a2ed-9ffa-4c97-536f654f8445 9280583d-a2ed-9ffa-4c97-536f654f8445}" -2026-04-22T10:09:29.608Z [ERROR] server-31bdcbab-7903-43cf-434e-63d1c8b786c7: failed to get log: index=23 error="log not found" -2026-04-22T10:09:29.608Z [INFO] server-31bdcbab-7903-43cf-434e-63d1c8b786c7: aborting pipeline replication: peer="{Voter server-004abba6-2704-f02c-4a1d-2f74074e297c 004abba6-2704-f02c-4a1d-2f74074e297c}" -2026-04-22T10:09:29.617Z [ERROR] server-31bdcbab-7903-43cf-434e-63d1c8b786c7: failed to get log: index=23 error="log not found" -2026-04-22T10:09:29.617Z [ERROR] server-31bdcbab-7903-43cf-434e-63d1c8b786c7: failed to get log: index=23 error="log not found" -2026-04-22T10:09:29.618Z [INFO] creating new snapshot: path=/tmp/raft3391203387/snapshots/2-23-1776852569617.tmp -2026-04-22T10:09:29.618Z [INFO] creating new snapshot: path=/tmp/raft1203374000/snapshots/2-23-1776852569618.tmp -2026-04-22T10:09:29.618Z [INFO] server-9280583d-a2ed-9ffa-4c97-536f654f8445: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:29.618Z [INFO] server-9280583d-a2ed-9ffa-4c97-536f654f8445: copied to local snapshot: bytes=71 -2026-04-22T10:09:29.618Z [INFO] server-004abba6-2704-f02c-4a1d-2f74074e297c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:29.618Z [INFO] server-9280583d-a2ed-9ffa-4c97-536f654f8445: snapshot restore progress: id=2-23-1776852569617 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:29.618Z [INFO] server-9280583d-a2ed-9ffa-4c97-536f654f8445: Installed remote snapshot -2026-04-22T10:09:29.619Z [WARN] server-9280583d-a2ed-9ffa-4c97-536f654f8445: skipping application of old log: index=22 -2026-04-22T10:09:29.619Z [INFO] server-31bdcbab-7903-43cf-434e-63d1c8b786c7: pipelining replication: peer="{Voter server-9280583d-a2ed-9ffa-4c97-536f654f8445 9280583d-a2ed-9ffa-4c97-536f654f8445}" -2026-04-22T10:09:29.619Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:09:29.619Z [INFO] server-004abba6-2704-f02c-4a1d-2f74074e297c: copied to local snapshot: bytes=71 -2026-04-22T10:09:29.619Z [INFO] server-004abba6-2704-f02c-4a1d-2f74074e297c: snapshot restore progress: id=2-23-1776852569618 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:29.619Z [INFO] server-004abba6-2704-f02c-4a1d-2f74074e297c: Installed remote snapshot -2026-04-22T10:09:29.619Z [INFO] server-31bdcbab-7903-43cf-434e-63d1c8b786c7: pipelining replication: peer="{Voter server-004abba6-2704-f02c-4a1d-2f74074e297c 004abba6-2704-f02c-4a1d-2f74074e297c}" -2026-04-22T10:09:30.624Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:09:29.619188034 +0000 UTC m=+15.642984122 (2 nodes), 673ns from start of poll, 1.667468578s from cluster start. Timeout at 2026-04-22 10:09:30.624193244 +0000 UTC m=+16.647989326, 1.005005204s after stability -2026-04-22T10:09:30.636Z [INFO] server-31bdcbab-7903-43cf-434e-63d1c8b786c7: aborting pipeline replication: peer="{Voter server-9280583d-a2ed-9ffa-4c97-536f654f8445 9280583d-a2ed-9ffa-4c97-536f654f8445}" -2026-04-22T10:09:30.636Z [INFO] server-31bdcbab-7903-43cf-434e-63d1c8b786c7: aborting pipeline replication: peer="{Voter server-004abba6-2704-f02c-4a1d-2f74074e297c 004abba6-2704-f02c-4a1d-2f74074e297c}" +2025-03-20T03:49:43.063Z [INFO] server-82ad609c-ed5a-ef90-8bed-02eb5008a551: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-82ad609c-ed5a-ef90-8bed-02eb5008a551 Address:82ad609c-ed5a-ef90-8bed-02eb5008a551} {Suffrage:Voter ID:server-3c701e80-74fe-b83d-a563-6265ba66f080 Address:3c701e80-74fe-b83d-a563-6265ba66f080} {Suffrage:Voter ID:server-c4d92b9b-d5cf-918a-b63e-0f8c8e77cde1 Address:c4d92b9b-d5cf-918a-b63e-0f8c8e77cde1}]" +2025-03-20T03:49:43.063Z [INFO] server-3c701e80-74fe-b83d-a563-6265ba66f080: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-82ad609c-ed5a-ef90-8bed-02eb5008a551 Address:82ad609c-ed5a-ef90-8bed-02eb5008a551} {Suffrage:Voter ID:server-3c701e80-74fe-b83d-a563-6265ba66f080 Address:3c701e80-74fe-b83d-a563-6265ba66f080} {Suffrage:Voter ID:server-c4d92b9b-d5cf-918a-b63e-0f8c8e77cde1 Address:c4d92b9b-d5cf-918a-b63e-0f8c8e77cde1}]" +2025-03-20T03:49:43.063Z [INFO] server-c4d92b9b-d5cf-918a-b63e-0f8c8e77cde1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-82ad609c-ed5a-ef90-8bed-02eb5008a551 Address:82ad609c-ed5a-ef90-8bed-02eb5008a551} {Suffrage:Voter ID:server-3c701e80-74fe-b83d-a563-6265ba66f080 Address:3c701e80-74fe-b83d-a563-6265ba66f080} {Suffrage:Voter ID:server-c4d92b9b-d5cf-918a-b63e-0f8c8e77cde1 Address:c4d92b9b-d5cf-918a-b63e-0f8c8e77cde1}]" +2025-03-20T03:49:43.063Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:43.063Z [INFO] server-82ad609c-ed5a-ef90-8bed-02eb5008a551: entering follower state: follower="Node at 82ad609c-ed5a-ef90-8bed-02eb5008a551 [Follower]" leader-address= leader-id= +2025-03-20T03:49:43.063Z [INFO] server-3c701e80-74fe-b83d-a563-6265ba66f080: entering follower state: follower="Node at 3c701e80-74fe-b83d-a563-6265ba66f080 [Follower]" leader-address= leader-id= +2025-03-20T03:49:43.063Z [INFO] server-c4d92b9b-d5cf-918a-b63e-0f8c8e77cde1: entering follower state: follower="Node at c4d92b9b-d5cf-918a-b63e-0f8c8e77cde1 [Follower]" leader-address= leader-id= +2025-03-20T03:49:43.717Z [WARN] server-82ad609c-ed5a-ef90-8bed-02eb5008a551: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:43.717Z [INFO] server-82ad609c-ed5a-ef90-8bed-02eb5008a551: entering candidate state: node="Node at 82ad609c-ed5a-ef90-8bed-02eb5008a551 [Candidate]" term=2 +2025-03-20T03:49:43.718Z [INFO] server-82ad609c-ed5a-ef90-8bed-02eb5008a551: election won: term=2 tally=2 +2025-03-20T03:49:43.718Z [INFO] server-82ad609c-ed5a-ef90-8bed-02eb5008a551: entering leader state: leader="Node at 82ad609c-ed5a-ef90-8bed-02eb5008a551 [Leader]" +2025-03-20T03:49:43.718Z [INFO] server-82ad609c-ed5a-ef90-8bed-02eb5008a551: added peer, starting replication: peer=server-3c701e80-74fe-b83d-a563-6265ba66f080 +2025-03-20T03:49:43.718Z [INFO] server-82ad609c-ed5a-ef90-8bed-02eb5008a551: added peer, starting replication: peer=server-c4d92b9b-d5cf-918a-b63e-0f8c8e77cde1 +2025-03-20T03:49:43.718Z [INFO] server-82ad609c-ed5a-ef90-8bed-02eb5008a551: pipelining replication: peer="{Voter server-c4d92b9b-d5cf-918a-b63e-0f8c8e77cde1 c4d92b9b-d5cf-918a-b63e-0f8c8e77cde1}" +2025-03-20T03:49:43.718Z [INFO] server-82ad609c-ed5a-ef90-8bed-02eb5008a551: pipelining replication: peer="{Voter server-3c701e80-74fe-b83d-a563-6265ba66f080 3c701e80-74fe-b83d-a563-6265ba66f080}" +2025-03-20T03:49:44.737Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:43.719084216 +0000 UTC m=+15.393537067 (1 nodes), 655.771647ms from start of poll, 656.193818ms from cluster start. Timeout at 2025-03-20 03:49:44.72409144 +0000 UTC m=+16.398544295, 1.005007228s after stability +2025-03-20T03:49:44.740Z [INFO] server-82ad609c-ed5a-ef90-8bed-02eb5008a551: starting snapshot up to: index=12 +2025-03-20T03:49:44.740Z [INFO] creating new snapshot: path=/tmp/raft2472651297/snapshots/2-12-1742442584740.tmp +2025-03-20T03:49:44.741Z [INFO] server-82ad609c-ed5a-ef90-8bed-02eb5008a551: snapshot complete up to: index=12 +2025-03-20T03:49:44.743Z [INFO] creating new snapshot: path=/tmp/raft2472651297/snapshots/2-23-1742442584743.tmp +2025-03-20T03:49:44.744Z [INFO] server-82ad609c-ed5a-ef90-8bed-02eb5008a551: copied to local snapshot: bytes=71 +2025-03-20T03:49:44.745Z [INFO] server-82ad609c-ed5a-ef90-8bed-02eb5008a551: snapshot restore progress: id=2-23-1742442584743 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:49:44.745Z [INFO] server-82ad609c-ed5a-ef90-8bed-02eb5008a551: restored user snapshot: index=23 +2025-03-20T03:49:44.745Z [ERROR] server-82ad609c-ed5a-ef90-8bed-02eb5008a551: failed to get log: index=23 error="log not found" +2025-03-20T03:49:44.745Z [INFO] server-82ad609c-ed5a-ef90-8bed-02eb5008a551: aborting pipeline replication: peer="{Voter server-c4d92b9b-d5cf-918a-b63e-0f8c8e77cde1 c4d92b9b-d5cf-918a-b63e-0f8c8e77cde1}" +2025-03-20T03:49:44.745Z [ERROR] server-82ad609c-ed5a-ef90-8bed-02eb5008a551: failed to get log: index=23 error="log not found" +2025-03-20T03:49:44.745Z [INFO] server-82ad609c-ed5a-ef90-8bed-02eb5008a551: aborting pipeline replication: peer="{Voter server-3c701e80-74fe-b83d-a563-6265ba66f080 3c701e80-74fe-b83d-a563-6265ba66f080}" +2025-03-20T03:49:44.752Z [ERROR] server-82ad609c-ed5a-ef90-8bed-02eb5008a551: failed to get log: index=23 error="log not found" +2025-03-20T03:49:44.753Z [ERROR] server-82ad609c-ed5a-ef90-8bed-02eb5008a551: failed to get log: index=23 error="log not found" +2025-03-20T03:49:44.753Z [INFO] creating new snapshot: path=/tmp/raft1064077599/snapshots/2-23-1742442584753.tmp +2025-03-20T03:49:44.754Z [INFO] server-3c701e80-74fe-b83d-a563-6265ba66f080: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:49:44.754Z [INFO] server-3c701e80-74fe-b83d-a563-6265ba66f080: copied to local snapshot: bytes=71 +2025-03-20T03:49:44.754Z [INFO] creating new snapshot: path=/tmp/raft4060961207/snapshots/2-23-1742442584754.tmp +2025-03-20T03:49:44.755Z [INFO] server-c4d92b9b-d5cf-918a-b63e-0f8c8e77cde1: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:49:44.755Z [INFO] server-c4d92b9b-d5cf-918a-b63e-0f8c8e77cde1: copied to local snapshot: bytes=71 +2025-03-20T03:49:44.755Z [INFO] server-c4d92b9b-d5cf-918a-b63e-0f8c8e77cde1: snapshot restore progress: id=2-23-1742442584754 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:49:44.755Z [INFO] server-c4d92b9b-d5cf-918a-b63e-0f8c8e77cde1: Installed remote snapshot +2025-03-20T03:49:44.756Z [WARN] server-c4d92b9b-d5cf-918a-b63e-0f8c8e77cde1: skipping application of old log: index=22 +2025-03-20T03:49:44.756Z [INFO] server-82ad609c-ed5a-ef90-8bed-02eb5008a551: pipelining replication: peer="{Voter server-c4d92b9b-d5cf-918a-b63e-0f8c8e77cde1 c4d92b9b-d5cf-918a-b63e-0f8c8e77cde1}" +2025-03-20T03:49:44.756Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:49:44.756Z [INFO] server-3c701e80-74fe-b83d-a563-6265ba66f080: snapshot restore progress: id=2-23-1742442584753 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:49:44.756Z [INFO] server-3c701e80-74fe-b83d-a563-6265ba66f080: Installed remote snapshot +2025-03-20T03:49:44.756Z [INFO] server-82ad609c-ed5a-ef90-8bed-02eb5008a551: pipelining replication: peer="{Voter server-3c701e80-74fe-b83d-a563-6265ba66f080 3c701e80-74fe-b83d-a563-6265ba66f080}" +2025-03-20T03:49:45.762Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:49:44.756276493 +0000 UTC m=+16.430729302 (2 nodes), 1.906µs from start of poll, 1.693386053s from cluster start. Timeout at 2025-03-20 03:49:45.761292217 +0000 UTC m=+17.435745005, 1.005015703s after stability +2025-03-20T03:49:45.775Z [INFO] server-82ad609c-ed5a-ef90-8bed-02eb5008a551: aborting pipeline replication: peer="{Voter server-3c701e80-74fe-b83d-a563-6265ba66f080 3c701e80-74fe-b83d-a563-6265ba66f080}" +2025-03-20T03:49:45.776Z [INFO] server-82ad609c-ed5a-ef90-8bed-02eb5008a551: aborting pipeline replication: peer="{Voter server-c4d92b9b-d5cf-918a-b63e-0f8c8e77cde1 c4d92b9b-d5cf-918a-b63e-0f8c8e77cde1}" === RUN TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_false -2026-04-22T10:09:30.637Z [INFO] server-4e947b87-ab30-61d6-1a41-2880ba821696: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4e947b87-ab30-61d6-1a41-2880ba821696 Address:4e947b87-ab30-61d6-1a41-2880ba821696} {Suffrage:Voter ID:server-c45dcf87-f7ea-0bad-e946-91b7beca8a51 Address:c45dcf87-f7ea-0bad-e946-91b7beca8a51} {Suffrage:Voter ID:server-b60e906f-adda-1903-b2a4-386a694bb5db Address:b60e906f-adda-1903-b2a4-386a694bb5db}]" -2026-04-22T10:09:30.637Z [INFO] server-4e947b87-ab30-61d6-1a41-2880ba821696: entering follower state: follower="Node at 4e947b87-ab30-61d6-1a41-2880ba821696 [Follower]" leader-address= leader-id= -2026-04-22T10:09:30.637Z [INFO] server-c45dcf87-f7ea-0bad-e946-91b7beca8a51: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4e947b87-ab30-61d6-1a41-2880ba821696 Address:4e947b87-ab30-61d6-1a41-2880ba821696} {Suffrage:Voter ID:server-c45dcf87-f7ea-0bad-e946-91b7beca8a51 Address:c45dcf87-f7ea-0bad-e946-91b7beca8a51} {Suffrage:Voter ID:server-b60e906f-adda-1903-b2a4-386a694bb5db Address:b60e906f-adda-1903-b2a4-386a694bb5db}]" -2026-04-22T10:09:30.638Z [INFO] server-b60e906f-adda-1903-b2a4-386a694bb5db: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4e947b87-ab30-61d6-1a41-2880ba821696 Address:4e947b87-ab30-61d6-1a41-2880ba821696} {Suffrage:Voter ID:server-c45dcf87-f7ea-0bad-e946-91b7beca8a51 Address:c45dcf87-f7ea-0bad-e946-91b7beca8a51} {Suffrage:Voter ID:server-b60e906f-adda-1903-b2a4-386a694bb5db Address:b60e906f-adda-1903-b2a4-386a694bb5db}]" -2026-04-22T10:09:30.638Z [INFO] server-c45dcf87-f7ea-0bad-e946-91b7beca8a51: entering follower state: follower="Node at c45dcf87-f7ea-0bad-e946-91b7beca8a51 [Follower]" leader-address= leader-id= -2026-04-22T10:09:30.638Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:30.638Z [INFO] server-b60e906f-adda-1903-b2a4-386a694bb5db: entering follower state: follower="Node at b60e906f-adda-1903-b2a4-386a694bb5db [Follower]" leader-address= leader-id= -2026-04-22T10:09:31.147Z [WARN] server-b60e906f-adda-1903-b2a4-386a694bb5db: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:31.147Z [INFO] server-b60e906f-adda-1903-b2a4-386a694bb5db: entering candidate state: node="Node at b60e906f-adda-1903-b2a4-386a694bb5db [Candidate]" term=2 -2026-04-22T10:09:31.147Z [INFO] server-b60e906f-adda-1903-b2a4-386a694bb5db: election won: term=2 tally=2 -2026-04-22T10:09:31.147Z [INFO] server-b60e906f-adda-1903-b2a4-386a694bb5db: entering leader state: leader="Node at b60e906f-adda-1903-b2a4-386a694bb5db [Leader]" -2026-04-22T10:09:31.147Z [INFO] server-b60e906f-adda-1903-b2a4-386a694bb5db: added peer, starting replication: peer=server-4e947b87-ab30-61d6-1a41-2880ba821696 -2026-04-22T10:09:31.147Z [INFO] server-b60e906f-adda-1903-b2a4-386a694bb5db: added peer, starting replication: peer=server-c45dcf87-f7ea-0bad-e946-91b7beca8a51 -2026-04-22T10:09:31.147Z [INFO] server-b60e906f-adda-1903-b2a4-386a694bb5db: pipelining replication: peer="{Voter server-c45dcf87-f7ea-0bad-e946-91b7beca8a51 c45dcf87-f7ea-0bad-e946-91b7beca8a51}" -2026-04-22T10:09:31.147Z [INFO] server-b60e906f-adda-1903-b2a4-386a694bb5db: pipelining replication: peer="{Voter server-4e947b87-ab30-61d6-1a41-2880ba821696 4e947b87-ab30-61d6-1a41-2880ba821696}" -2026-04-22T10:09:32.153Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:31.147546263 +0000 UTC m=+17.171342351 (1 nodes), 509.490051ms from start of poll, 509.781962ms from cluster start. Timeout at 2026-04-22 10:09:32.152558631 +0000 UTC m=+18.176354751, 1.0050124s after stability -2026-04-22T10:09:32.155Z [INFO] server-b60e906f-adda-1903-b2a4-386a694bb5db: starting snapshot up to: index=12 -2026-04-22T10:09:32.155Z [INFO] creating new snapshot: path=/tmp/raft422226262/snapshots/2-12-1776852572155.tmp -2026-04-22T10:09:32.157Z [INFO] server-b60e906f-adda-1903-b2a4-386a694bb5db: snapshot complete up to: index=12 -2026-04-22T10:09:32.709Z [WARN] server-b60e906f-adda-1903-b2a4-386a694bb5db: failed to contact: server-id=server-c45dcf87-f7ea-0bad-e946-91b7beca8a51 time=554.26795ms -2026-04-22T10:09:32.710Z [INFO] creating new snapshot: path=/tmp/raft422226262/snapshots/2-23-1776852572710.tmp -2026-04-22T10:09:32.710Z [INFO] server-b60e906f-adda-1903-b2a4-386a694bb5db: copied to local snapshot: bytes=71 -2026-04-22T10:09:32.714Z [INFO] server-b60e906f-adda-1903-b2a4-386a694bb5db: snapshot restore progress: id=2-23-1776852572710 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:32.714Z [INFO] server-b60e906f-adda-1903-b2a4-386a694bb5db: removing all old logs from log store -2026-04-22T10:09:32.714Z [INFO] server-b60e906f-adda-1903-b2a4-386a694bb5db: compacting logs: from=1 to=22 -2026-04-22T10:09:32.714Z [INFO] server-b60e906f-adda-1903-b2a4-386a694bb5db: restored user snapshot: index=23 -2026-04-22T10:09:32.714Z [ERROR] server-b60e906f-adda-1903-b2a4-386a694bb5db: failed to get log: index=22 error="log not found" -2026-04-22T10:09:32.714Z [INFO] server-b60e906f-adda-1903-b2a4-386a694bb5db: aborting pipeline replication: peer="{Voter server-c45dcf87-f7ea-0bad-e946-91b7beca8a51 c45dcf87-f7ea-0bad-e946-91b7beca8a51}" -2026-04-22T10:09:32.714Z [ERROR] server-b60e906f-adda-1903-b2a4-386a694bb5db: failed to get log: index=22 error="log not found" -2026-04-22T10:09:32.714Z [INFO] server-b60e906f-adda-1903-b2a4-386a694bb5db: aborting pipeline replication: peer="{Voter server-4e947b87-ab30-61d6-1a41-2880ba821696 4e947b87-ab30-61d6-1a41-2880ba821696}" -2026-04-22T10:09:32.721Z [ERROR] server-b60e906f-adda-1903-b2a4-386a694bb5db: failed to get log: index=22 error="log not found" -2026-04-22T10:09:32.721Z [INFO] creating new snapshot: path=/tmp/raft3832349352/snapshots/2-23-1776852572721.tmp -2026-04-22T10:09:32.722Z [INFO] server-4e947b87-ab30-61d6-1a41-2880ba821696: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:32.724Z [ERROR] server-b60e906f-adda-1903-b2a4-386a694bb5db: failed to get log: index=22 error="log not found" -2026-04-22T10:09:32.725Z [INFO] creating new snapshot: path=/tmp/raft3371125227/snapshots/2-23-1776852572725.tmp -2026-04-22T10:09:32.726Z [INFO] server-4e947b87-ab30-61d6-1a41-2880ba821696: copied to local snapshot: bytes=71 -2026-04-22T10:09:32.726Z [INFO] server-c45dcf87-f7ea-0bad-e946-91b7beca8a51: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:32.726Z [INFO] server-4e947b87-ab30-61d6-1a41-2880ba821696: snapshot restore progress: id=2-23-1776852572721 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:32.726Z [INFO] server-4e947b87-ab30-61d6-1a41-2880ba821696: removing all old logs from log store -2026-04-22T10:09:32.726Z [INFO] server-4e947b87-ab30-61d6-1a41-2880ba821696: compacting logs: from=1 to=22 -2026-04-22T10:09:32.726Z [INFO] server-4e947b87-ab30-61d6-1a41-2880ba821696: Installed remote snapshot -2026-04-22T10:09:32.727Z [INFO] server-b60e906f-adda-1903-b2a4-386a694bb5db: pipelining replication: peer="{Voter server-4e947b87-ab30-61d6-1a41-2880ba821696 4e947b87-ab30-61d6-1a41-2880ba821696}" -2026-04-22T10:09:32.727Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:09:32.732Z [INFO] server-c45dcf87-f7ea-0bad-e946-91b7beca8a51: copied to local snapshot: bytes=71 -2026-04-22T10:09:32.733Z [INFO] server-c45dcf87-f7ea-0bad-e946-91b7beca8a51: snapshot restore progress: id=2-23-1776852572725 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:32.733Z [INFO] server-c45dcf87-f7ea-0bad-e946-91b7beca8a51: removing all old logs from log store -2026-04-22T10:09:32.733Z [INFO] server-c45dcf87-f7ea-0bad-e946-91b7beca8a51: compacting logs: from=1 to=22 -2026-04-22T10:09:32.733Z [INFO] server-c45dcf87-f7ea-0bad-e946-91b7beca8a51: Installed remote snapshot -2026-04-22T10:09:32.733Z [INFO] server-b60e906f-adda-1903-b2a4-386a694bb5db: pipelining replication: peer="{Voter server-c45dcf87-f7ea-0bad-e946-91b7beca8a51 c45dcf87-f7ea-0bad-e946-91b7beca8a51}" -2026-04-22T10:09:33.732Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:09:32.727119719 +0000 UTC m=+18.750915822 (2 nodes), 1.032µs from start of poll, 2.089355433s from cluster start. Timeout at 2026-04-22 10:09:33.732128442 +0000 UTC m=+19.755924524, 1.005008702s after stability -2026-04-22T10:09:33.744Z [INFO] server-b60e906f-adda-1903-b2a4-386a694bb5db: aborting pipeline replication: peer="{Voter server-c45dcf87-f7ea-0bad-e946-91b7beca8a51 c45dcf87-f7ea-0bad-e946-91b7beca8a51}" -2026-04-22T10:09:33.744Z [INFO] server-b60e906f-adda-1903-b2a4-386a694bb5db: aborting pipeline replication: peer="{Voter server-4e947b87-ab30-61d6-1a41-2880ba821696 4e947b87-ab30-61d6-1a41-2880ba821696}" +2025-03-20T03:49:45.778Z [INFO] server-c5c39a2c-26f1-04bc-bcc3-50b7c917d903: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c5c39a2c-26f1-04bc-bcc3-50b7c917d903 Address:c5c39a2c-26f1-04bc-bcc3-50b7c917d903} {Suffrage:Voter ID:server-d90d42c6-b6c3-3baa-6f7e-6f44d4f453c3 Address:d90d42c6-b6c3-3baa-6f7e-6f44d4f453c3} {Suffrage:Voter ID:server-39ce27a0-3ba8-5e10-a9d6-bf3139c407d5 Address:39ce27a0-3ba8-5e10-a9d6-bf3139c407d5}]" +2025-03-20T03:49:45.786Z [INFO] server-c5c39a2c-26f1-04bc-bcc3-50b7c917d903: entering follower state: follower="Node at c5c39a2c-26f1-04bc-bcc3-50b7c917d903 [Follower]" leader-address= leader-id= +2025-03-20T03:49:45.789Z [INFO] server-d90d42c6-b6c3-3baa-6f7e-6f44d4f453c3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c5c39a2c-26f1-04bc-bcc3-50b7c917d903 Address:c5c39a2c-26f1-04bc-bcc3-50b7c917d903} {Suffrage:Voter ID:server-d90d42c6-b6c3-3baa-6f7e-6f44d4f453c3 Address:d90d42c6-b6c3-3baa-6f7e-6f44d4f453c3} {Suffrage:Voter ID:server-39ce27a0-3ba8-5e10-a9d6-bf3139c407d5 Address:39ce27a0-3ba8-5e10-a9d6-bf3139c407d5}]" +2025-03-20T03:49:45.789Z [INFO] server-39ce27a0-3ba8-5e10-a9d6-bf3139c407d5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c5c39a2c-26f1-04bc-bcc3-50b7c917d903 Address:c5c39a2c-26f1-04bc-bcc3-50b7c917d903} {Suffrage:Voter ID:server-d90d42c6-b6c3-3baa-6f7e-6f44d4f453c3 Address:d90d42c6-b6c3-3baa-6f7e-6f44d4f453c3} {Suffrage:Voter ID:server-39ce27a0-3ba8-5e10-a9d6-bf3139c407d5 Address:39ce27a0-3ba8-5e10-a9d6-bf3139c407d5}]" +2025-03-20T03:49:45.789Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:45.789Z [INFO] server-d90d42c6-b6c3-3baa-6f7e-6f44d4f453c3: entering follower state: follower="Node at d90d42c6-b6c3-3baa-6f7e-6f44d4f453c3 [Follower]" leader-address= leader-id= +2025-03-20T03:49:45.789Z [INFO] server-39ce27a0-3ba8-5e10-a9d6-bf3139c407d5: entering follower state: follower="Node at 39ce27a0-3ba8-5e10-a9d6-bf3139c407d5 [Follower]" leader-address= leader-id= +2025-03-20T03:49:46.714Z [WARN] server-c5c39a2c-26f1-04bc-bcc3-50b7c917d903: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:46.714Z [INFO] server-c5c39a2c-26f1-04bc-bcc3-50b7c917d903: entering candidate state: node="Node at c5c39a2c-26f1-04bc-bcc3-50b7c917d903 [Candidate]" term=2 +2025-03-20T03:49:46.714Z [INFO] server-c5c39a2c-26f1-04bc-bcc3-50b7c917d903: election won: term=2 tally=2 +2025-03-20T03:49:46.714Z [INFO] server-c5c39a2c-26f1-04bc-bcc3-50b7c917d903: entering leader state: leader="Node at c5c39a2c-26f1-04bc-bcc3-50b7c917d903 [Leader]" +2025-03-20T03:49:46.714Z [INFO] server-c5c39a2c-26f1-04bc-bcc3-50b7c917d903: added peer, starting replication: peer=server-d90d42c6-b6c3-3baa-6f7e-6f44d4f453c3 +2025-03-20T03:49:46.714Z [INFO] server-c5c39a2c-26f1-04bc-bcc3-50b7c917d903: added peer, starting replication: peer=server-39ce27a0-3ba8-5e10-a9d6-bf3139c407d5 +2025-03-20T03:49:46.714Z [INFO] server-c5c39a2c-26f1-04bc-bcc3-50b7c917d903: pipelining replication: peer="{Voter server-39ce27a0-3ba8-5e10-a9d6-bf3139c407d5 39ce27a0-3ba8-5e10-a9d6-bf3139c407d5}" +2025-03-20T03:49:46.714Z [INFO] server-c5c39a2c-26f1-04bc-bcc3-50b7c917d903: pipelining replication: peer="{Voter server-d90d42c6-b6c3-3baa-6f7e-6f44d4f453c3 d90d42c6-b6c3-3baa-6f7e-6f44d4f453c3}" +2025-03-20T03:49:47.720Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:46.71499564 +0000 UTC m=+18.389448436 (1 nodes), 925.398116ms from start of poll, 936.271709ms from cluster start. Timeout at 2025-03-20 03:49:47.720008759 +0000 UTC m=+19.394461601, 1.005013165s after stability +2025-03-20T03:49:47.722Z [INFO] server-c5c39a2c-26f1-04bc-bcc3-50b7c917d903: starting snapshot up to: index=12 +2025-03-20T03:49:47.722Z [INFO] creating new snapshot: path=/tmp/raft3981523763/snapshots/2-12-1742442587722.tmp +2025-03-20T03:49:47.723Z [INFO] server-c5c39a2c-26f1-04bc-bcc3-50b7c917d903: snapshot complete up to: index=12 +2025-03-20T03:49:47.725Z [INFO] creating new snapshot: path=/tmp/raft3981523763/snapshots/2-23-1742442587725.tmp +2025-03-20T03:49:47.727Z [INFO] server-c5c39a2c-26f1-04bc-bcc3-50b7c917d903: copied to local snapshot: bytes=71 +2025-03-20T03:49:47.727Z [INFO] server-c5c39a2c-26f1-04bc-bcc3-50b7c917d903: snapshot restore progress: id=2-23-1742442587725 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:49:47.727Z [INFO] server-c5c39a2c-26f1-04bc-bcc3-50b7c917d903: removing all old logs from log store +2025-03-20T03:49:47.727Z [INFO] server-c5c39a2c-26f1-04bc-bcc3-50b7c917d903: compacting logs: from=1 to=22 +2025-03-20T03:49:47.727Z [INFO] server-c5c39a2c-26f1-04bc-bcc3-50b7c917d903: restored user snapshot: index=23 +2025-03-20T03:49:47.727Z [ERROR] server-c5c39a2c-26f1-04bc-bcc3-50b7c917d903: failed to get log: index=22 error="log not found" +2025-03-20T03:49:47.727Z [INFO] server-c5c39a2c-26f1-04bc-bcc3-50b7c917d903: aborting pipeline replication: peer="{Voter server-39ce27a0-3ba8-5e10-a9d6-bf3139c407d5 39ce27a0-3ba8-5e10-a9d6-bf3139c407d5}" +2025-03-20T03:49:47.727Z [ERROR] server-c5c39a2c-26f1-04bc-bcc3-50b7c917d903: failed to get log: index=22 error="log not found" +2025-03-20T03:49:47.727Z [INFO] server-c5c39a2c-26f1-04bc-bcc3-50b7c917d903: aborting pipeline replication: peer="{Voter server-d90d42c6-b6c3-3baa-6f7e-6f44d4f453c3 d90d42c6-b6c3-3baa-6f7e-6f44d4f453c3}" +2025-03-20T03:49:47.735Z [ERROR] server-c5c39a2c-26f1-04bc-bcc3-50b7c917d903: failed to get log: index=22 error="log not found" +2025-03-20T03:49:47.739Z [INFO] creating new snapshot: path=/tmp/raft4222827897/snapshots/2-23-1742442587739.tmp +2025-03-20T03:49:47.740Z [INFO] server-39ce27a0-3ba8-5e10-a9d6-bf3139c407d5: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:49:47.741Z [INFO] server-39ce27a0-3ba8-5e10-a9d6-bf3139c407d5: copied to local snapshot: bytes=71 +2025-03-20T03:49:47.741Z [INFO] server-39ce27a0-3ba8-5e10-a9d6-bf3139c407d5: snapshot restore progress: id=2-23-1742442587739 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:49:47.741Z [INFO] server-39ce27a0-3ba8-5e10-a9d6-bf3139c407d5: removing all old logs from log store +2025-03-20T03:49:47.741Z [INFO] server-39ce27a0-3ba8-5e10-a9d6-bf3139c407d5: compacting logs: from=1 to=22 +2025-03-20T03:49:47.741Z [INFO] server-39ce27a0-3ba8-5e10-a9d6-bf3139c407d5: Installed remote snapshot +2025-03-20T03:49:47.741Z [INFO] server-c5c39a2c-26f1-04bc-bcc3-50b7c917d903: pipelining replication: peer="{Voter server-39ce27a0-3ba8-5e10-a9d6-bf3139c407d5 39ce27a0-3ba8-5e10-a9d6-bf3139c407d5}" +2025-03-20T03:49:47.741Z [ERROR] server-c5c39a2c-26f1-04bc-bcc3-50b7c917d903: failed to get log: index=22 error="log not found" +2025-03-20T03:49:47.742Z [INFO] creating new snapshot: path=/tmp/raft3972687028/snapshots/2-23-1742442587742.tmp +2025-03-20T03:49:47.742Z [INFO] server-d90d42c6-b6c3-3baa-6f7e-6f44d4f453c3: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:49:47.743Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:49:47.742Z [INFO] server-d90d42c6-b6c3-3baa-6f7e-6f44d4f453c3: copied to local snapshot: bytes=71 +2025-03-20T03:49:47.743Z [INFO] server-d90d42c6-b6c3-3baa-6f7e-6f44d4f453c3: snapshot restore progress: id=2-23-1742442587742 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:49:47.743Z [INFO] server-d90d42c6-b6c3-3baa-6f7e-6f44d4f453c3: removing all old logs from log store +2025-03-20T03:49:47.743Z [INFO] server-d90d42c6-b6c3-3baa-6f7e-6f44d4f453c3: compacting logs: from=1 to=22 +2025-03-20T03:49:47.743Z [INFO] server-d90d42c6-b6c3-3baa-6f7e-6f44d4f453c3: Installed remote snapshot +2025-03-20T03:49:47.743Z [INFO] server-c5c39a2c-26f1-04bc-bcc3-50b7c917d903: pipelining replication: peer="{Voter server-d90d42c6-b6c3-3baa-6f7e-6f44d4f453c3 d90d42c6-b6c3-3baa-6f7e-6f44d4f453c3}" +2025-03-20T03:49:48.748Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:49:47.74325225 +0000 UTC m=+19.417705137 (2 nodes), 2.584µs from start of poll, 1.96452841s from cluster start. Timeout at 2025-03-20 03:49:48.748273291 +0000 UTC m=+20.422726083, 1.005020946s after stability +2025-03-20T03:49:48.762Z [INFO] server-c5c39a2c-26f1-04bc-bcc3-50b7c917d903: aborting pipeline replication: peer="{Voter server-39ce27a0-3ba8-5e10-a9d6-bf3139c407d5 39ce27a0-3ba8-5e10-a9d6-bf3139c407d5}" +2025-03-20T03:49:48.762Z [INFO] server-c5c39a2c-26f1-04bc-bcc3-50b7c917d903: aborting pipeline replication: peer="{Voter server-d90d42c6-b6c3-3baa-6f7e-6f44d4f453c3 d90d42c6-b6c3-3baa-6f7e-6f44d4f453c3}" === RUN TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_true -2026-04-22T10:09:33.745Z [INFO] server-e48c12c2-128f-ded3-3904-d56098d34c0d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e48c12c2-128f-ded3-3904-d56098d34c0d Address:e48c12c2-128f-ded3-3904-d56098d34c0d} {Suffrage:Voter ID:server-6d674c41-be45-bba7-807d-9dcb068f668a Address:6d674c41-be45-bba7-807d-9dcb068f668a} {Suffrage:Voter ID:server-bc0db1a9-72c2-b3c4-0b86-2dc0c7363639 Address:bc0db1a9-72c2-b3c4-0b86-2dc0c7363639}]" -2026-04-22T10:09:33.745Z [INFO] server-6d674c41-be45-bba7-807d-9dcb068f668a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e48c12c2-128f-ded3-3904-d56098d34c0d Address:e48c12c2-128f-ded3-3904-d56098d34c0d} {Suffrage:Voter ID:server-6d674c41-be45-bba7-807d-9dcb068f668a Address:6d674c41-be45-bba7-807d-9dcb068f668a} {Suffrage:Voter ID:server-bc0db1a9-72c2-b3c4-0b86-2dc0c7363639 Address:bc0db1a9-72c2-b3c4-0b86-2dc0c7363639}]" -2026-04-22T10:09:33.745Z [INFO] server-bc0db1a9-72c2-b3c4-0b86-2dc0c7363639: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e48c12c2-128f-ded3-3904-d56098d34c0d Address:e48c12c2-128f-ded3-3904-d56098d34c0d} {Suffrage:Voter ID:server-6d674c41-be45-bba7-807d-9dcb068f668a Address:6d674c41-be45-bba7-807d-9dcb068f668a} {Suffrage:Voter ID:server-bc0db1a9-72c2-b3c4-0b86-2dc0c7363639 Address:bc0db1a9-72c2-b3c4-0b86-2dc0c7363639}]" -2026-04-22T10:09:33.745Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:33.745Z [INFO] server-6d674c41-be45-bba7-807d-9dcb068f668a: entering follower state: follower="Node at 6d674c41-be45-bba7-807d-9dcb068f668a [Follower]" leader-address= leader-id= -2026-04-22T10:09:33.745Z [INFO] server-e48c12c2-128f-ded3-3904-d56098d34c0d: entering follower state: follower="Node at e48c12c2-128f-ded3-3904-d56098d34c0d [Follower]" leader-address= leader-id= -2026-04-22T10:09:33.745Z [INFO] server-bc0db1a9-72c2-b3c4-0b86-2dc0c7363639: entering follower state: follower="Node at bc0db1a9-72c2-b3c4-0b86-2dc0c7363639 [Follower]" leader-address= leader-id= -2026-04-22T10:09:34.628Z [WARN] server-6d674c41-be45-bba7-807d-9dcb068f668a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:34.628Z [INFO] server-6d674c41-be45-bba7-807d-9dcb068f668a: entering candidate state: node="Node at 6d674c41-be45-bba7-807d-9dcb068f668a [Candidate]" term=2 -2026-04-22T10:09:34.629Z [INFO] server-6d674c41-be45-bba7-807d-9dcb068f668a: election won: term=2 tally=2 -2026-04-22T10:09:34.629Z [INFO] server-6d674c41-be45-bba7-807d-9dcb068f668a: entering leader state: leader="Node at 6d674c41-be45-bba7-807d-9dcb068f668a [Leader]" -2026-04-22T10:09:34.629Z [INFO] server-6d674c41-be45-bba7-807d-9dcb068f668a: added peer, starting replication: peer=server-e48c12c2-128f-ded3-3904-d56098d34c0d -2026-04-22T10:09:34.629Z [INFO] server-6d674c41-be45-bba7-807d-9dcb068f668a: added peer, starting replication: peer=server-bc0db1a9-72c2-b3c4-0b86-2dc0c7363639 -2026-04-22T10:09:34.629Z [INFO] server-6d674c41-be45-bba7-807d-9dcb068f668a: pipelining replication: peer="{Voter server-bc0db1a9-72c2-b3c4-0b86-2dc0c7363639 bc0db1a9-72c2-b3c4-0b86-2dc0c7363639}" -2026-04-22T10:09:34.629Z [INFO] server-6d674c41-be45-bba7-807d-9dcb068f668a: pipelining replication: peer="{Voter server-e48c12c2-128f-ded3-3904-d56098d34c0d e48c12c2-128f-ded3-3904-d56098d34c0d}" -2026-04-22T10:09:35.634Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:34.629146929 +0000 UTC m=+20.652943044 (1 nodes), 883.230492ms from start of poll, 883.478455ms from cluster start. Timeout at 2026-04-22 10:09:35.634148122 +0000 UTC m=+21.657944205, 1.005001161s after stability -2026-04-22T10:09:35.635Z [INFO] server-6d674c41-be45-bba7-807d-9dcb068f668a: starting snapshot up to: index=12 -2026-04-22T10:09:35.635Z [INFO] creating new snapshot: path=/tmp/raft3462718255/snapshots/2-12-1776852575635.tmp -2026-04-22T10:09:35.635Z [INFO] server-6d674c41-be45-bba7-807d-9dcb068f668a: snapshot complete up to: index=12 -2026-04-22T10:09:35.636Z [INFO] server-abaa14f9-a3eb-f034-af94-cb71988acdbc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-abaa14f9-a3eb-f034-af94-cb71988acdbc Address:abaa14f9-a3eb-f034-af94-cb71988acdbc} {Suffrage:Voter ID:server-bbf70462-4738-03d9-2c25-2ccdf674f0fa Address:bbf70462-4738-03d9-2c25-2ccdf674f0fa} {Suffrage:Voter ID:server-f728594b-486b-bd20-8915-1e1bc2871708 Address:f728594b-486b-bd20-8915-1e1bc2871708}]" -2026-04-22T10:09:35.636Z [INFO] server-abaa14f9-a3eb-f034-af94-cb71988acdbc: entering follower state: follower="Node at abaa14f9-a3eb-f034-af94-cb71988acdbc [Follower]" leader-address= leader-id= -2026-04-22T10:09:35.636Z [INFO] server-bbf70462-4738-03d9-2c25-2ccdf674f0fa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-abaa14f9-a3eb-f034-af94-cb71988acdbc Address:abaa14f9-a3eb-f034-af94-cb71988acdbc} {Suffrage:Voter ID:server-bbf70462-4738-03d9-2c25-2ccdf674f0fa Address:bbf70462-4738-03d9-2c25-2ccdf674f0fa} {Suffrage:Voter ID:server-f728594b-486b-bd20-8915-1e1bc2871708 Address:f728594b-486b-bd20-8915-1e1bc2871708}]" -2026-04-22T10:09:35.636Z [INFO] server-bbf70462-4738-03d9-2c25-2ccdf674f0fa: entering follower state: follower="Node at bbf70462-4738-03d9-2c25-2ccdf674f0fa [Follower]" leader-address= leader-id= -2026-04-22T10:09:35.636Z [INFO] server-f728594b-486b-bd20-8915-1e1bc2871708: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-abaa14f9-a3eb-f034-af94-cb71988acdbc Address:abaa14f9-a3eb-f034-af94-cb71988acdbc} {Suffrage:Voter ID:server-bbf70462-4738-03d9-2c25-2ccdf674f0fa Address:bbf70462-4738-03d9-2c25-2ccdf674f0fa} {Suffrage:Voter ID:server-f728594b-486b-bd20-8915-1e1bc2871708 Address:f728594b-486b-bd20-8915-1e1bc2871708}]" -2026-04-22T10:09:35.636Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:35.637Z [INFO] server-f728594b-486b-bd20-8915-1e1bc2871708: entering follower state: follower="Node at f728594b-486b-bd20-8915-1e1bc2871708 [Follower]" leader-address= leader-id= -2026-04-22T10:09:36.307Z [WARN] server-f728594b-486b-bd20-8915-1e1bc2871708: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:36.307Z [INFO] server-f728594b-486b-bd20-8915-1e1bc2871708: entering candidate state: node="Node at f728594b-486b-bd20-8915-1e1bc2871708 [Candidate]" term=2 -2026-04-22T10:09:36.308Z [INFO] server-f728594b-486b-bd20-8915-1e1bc2871708: election won: term=2 tally=2 -2026-04-22T10:09:36.308Z [INFO] server-f728594b-486b-bd20-8915-1e1bc2871708: entering leader state: leader="Node at f728594b-486b-bd20-8915-1e1bc2871708 [Leader]" -2026-04-22T10:09:36.308Z [INFO] server-f728594b-486b-bd20-8915-1e1bc2871708: added peer, starting replication: peer=server-abaa14f9-a3eb-f034-af94-cb71988acdbc -2026-04-22T10:09:36.308Z [INFO] server-f728594b-486b-bd20-8915-1e1bc2871708: added peer, starting replication: peer=server-bbf70462-4738-03d9-2c25-2ccdf674f0fa -2026-04-22T10:09:36.308Z [INFO] server-f728594b-486b-bd20-8915-1e1bc2871708: pipelining replication: peer="{Voter server-bbf70462-4738-03d9-2c25-2ccdf674f0fa bbf70462-4738-03d9-2c25-2ccdf674f0fa}" -2026-04-22T10:09:36.308Z [INFO] server-f728594b-486b-bd20-8915-1e1bc2871708: pipelining replication: peer="{Voter server-abaa14f9-a3eb-f034-af94-cb71988acdbc abaa14f9-a3eb-f034-af94-cb71988acdbc}" -2026-04-22T10:09:37.313Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:36.308092354 +0000 UTC m=+22.331888443 (1 nodes), 671.078979ms from start of poll, 671.420944ms from cluster start. Timeout at 2026-04-22 10:09:37.31309858 +0000 UTC m=+23.336894662, 1.005006219s after stability -2026-04-22T10:09:37.313Z [INFO] creating new snapshot: path=/tmp/raft3318857523/snapshots/2-13-1776852577313.tmp -2026-04-22T10:09:37.314Z [INFO] server-f728594b-486b-bd20-8915-1e1bc2871708: copied to local snapshot: bytes=71 -2026-04-22T10:09:37.314Z [INFO] server-f728594b-486b-bd20-8915-1e1bc2871708: snapshot restore progress: id=2-13-1776852577313 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:37.314Z [INFO] server-f728594b-486b-bd20-8915-1e1bc2871708: restored user snapshot: index=13 -2026-04-22T10:09:37.315Z [ERROR] server-f728594b-486b-bd20-8915-1e1bc2871708: failed to get log: index=3 error="log not found" -2026-04-22T10:09:37.315Z [ERROR] server-f728594b-486b-bd20-8915-1e1bc2871708: failed to get log: index=3 error="log not found" -2026-04-22T10:09:37.315Z [INFO] server-f728594b-486b-bd20-8915-1e1bc2871708: aborting pipeline replication: peer="{Voter server-bbf70462-4738-03d9-2c25-2ccdf674f0fa bbf70462-4738-03d9-2c25-2ccdf674f0fa}" -2026-04-22T10:09:37.315Z [INFO] server-f728594b-486b-bd20-8915-1e1bc2871708: aborting pipeline replication: peer="{Voter server-abaa14f9-a3eb-f034-af94-cb71988acdbc abaa14f9-a3eb-f034-af94-cb71988acdbc}" -2026-04-22T10:09:37.321Z [ERROR] server-f728594b-486b-bd20-8915-1e1bc2871708: failed to get log: index=3 error="log not found" -2026-04-22T10:09:37.321Z [INFO] creating new snapshot: path=/tmp/raft2747520041/snapshots/2-13-1776852577321.tmp -2026-04-22T10:09:37.322Z [INFO] server-bbf70462-4738-03d9-2c25-2ccdf674f0fa: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:37.322Z [INFO] server-bbf70462-4738-03d9-2c25-2ccdf674f0fa: copied to local snapshot: bytes=71 -2026-04-22T10:09:37.322Z [INFO] server-bbf70462-4738-03d9-2c25-2ccdf674f0fa: snapshot restore progress: id=2-13-1776852577321 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:37.323Z [INFO] server-bbf70462-4738-03d9-2c25-2ccdf674f0fa: Installed remote snapshot -2026-04-22T10:09:37.323Z [INFO] server-f728594b-486b-bd20-8915-1e1bc2871708: pipelining replication: peer="{Voter server-bbf70462-4738-03d9-2c25-2ccdf674f0fa bbf70462-4738-03d9-2c25-2ccdf674f0fa}" -2026-04-22T10:09:37.323Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:09:37.324Z [ERROR] server-f728594b-486b-bd20-8915-1e1bc2871708: failed to get log: index=3 error="log not found" -2026-04-22T10:09:37.325Z [INFO] creating new snapshot: path=/tmp/raft2751207442/snapshots/2-13-1776852577325.tmp -2026-04-22T10:09:37.325Z [INFO] server-abaa14f9-a3eb-f034-af94-cb71988acdbc: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:37.325Z [INFO] server-abaa14f9-a3eb-f034-af94-cb71988acdbc: copied to local snapshot: bytes=71 -2026-04-22T10:09:37.325Z [INFO] server-abaa14f9-a3eb-f034-af94-cb71988acdbc: snapshot restore progress: id=2-13-1776852577325 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:37.325Z [INFO] server-abaa14f9-a3eb-f034-af94-cb71988acdbc: Installed remote snapshot -2026-04-22T10:09:37.325Z [INFO] server-f728594b-486b-bd20-8915-1e1bc2871708: pipelining replication: peer="{Voter server-abaa14f9-a3eb-f034-af94-cb71988acdbc abaa14f9-a3eb-f034-af94-cb71988acdbc}" -2026-04-22T10:09:38.329Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:09:37.323262188 +0000 UTC m=+23.347058275 (2 nodes), 859ns from start of poll, 1.686590776s from cluster start. Timeout at 2026-04-22 10:09:38.328269557 +0000 UTC m=+24.352065639, 1.005007364s after stability -2026-04-22T10:09:38.340Z [INFO] server-6d674c41-be45-bba7-807d-9dcb068f668a: aborting pipeline replication: peer="{Voter server-bc0db1a9-72c2-b3c4-0b86-2dc0c7363639 bc0db1a9-72c2-b3c4-0b86-2dc0c7363639}" -2026-04-22T10:09:38.340Z [INFO] server-6d674c41-be45-bba7-807d-9dcb068f668a: aborting pipeline replication: peer="{Voter server-e48c12c2-128f-ded3-3904-d56098d34c0d e48c12c2-128f-ded3-3904-d56098d34c0d}" +2025-03-20T03:49:48.784Z [INFO] server-e9bff5b4-42d3-795e-8937-7fad9f48e35e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e9bff5b4-42d3-795e-8937-7fad9f48e35e Address:e9bff5b4-42d3-795e-8937-7fad9f48e35e} {Suffrage:Voter ID:server-cfca0fb7-ceba-66fe-a0c1-0f8700165d7a Address:cfca0fb7-ceba-66fe-a0c1-0f8700165d7a} {Suffrage:Voter ID:server-a5eeeb9f-c5a7-f5b1-961f-1825eb7c362b Address:a5eeeb9f-c5a7-f5b1-961f-1825eb7c362b}]" +2025-03-20T03:49:48.784Z [INFO] server-cfca0fb7-ceba-66fe-a0c1-0f8700165d7a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e9bff5b4-42d3-795e-8937-7fad9f48e35e Address:e9bff5b4-42d3-795e-8937-7fad9f48e35e} {Suffrage:Voter ID:server-cfca0fb7-ceba-66fe-a0c1-0f8700165d7a Address:cfca0fb7-ceba-66fe-a0c1-0f8700165d7a} {Suffrage:Voter ID:server-a5eeeb9f-c5a7-f5b1-961f-1825eb7c362b Address:a5eeeb9f-c5a7-f5b1-961f-1825eb7c362b}]" +2025-03-20T03:49:48.784Z [INFO] server-a5eeeb9f-c5a7-f5b1-961f-1825eb7c362b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e9bff5b4-42d3-795e-8937-7fad9f48e35e Address:e9bff5b4-42d3-795e-8937-7fad9f48e35e} {Suffrage:Voter ID:server-cfca0fb7-ceba-66fe-a0c1-0f8700165d7a Address:cfca0fb7-ceba-66fe-a0c1-0f8700165d7a} {Suffrage:Voter ID:server-a5eeeb9f-c5a7-f5b1-961f-1825eb7c362b Address:a5eeeb9f-c5a7-f5b1-961f-1825eb7c362b}]" +2025-03-20T03:49:48.784Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:48.784Z [INFO] server-e9bff5b4-42d3-795e-8937-7fad9f48e35e: entering follower state: follower="Node at e9bff5b4-42d3-795e-8937-7fad9f48e35e [Follower]" leader-address= leader-id= +2025-03-20T03:49:48.784Z [INFO] server-cfca0fb7-ceba-66fe-a0c1-0f8700165d7a: entering follower state: follower="Node at cfca0fb7-ceba-66fe-a0c1-0f8700165d7a [Follower]" leader-address= leader-id= +2025-03-20T03:49:48.785Z [INFO] server-a5eeeb9f-c5a7-f5b1-961f-1825eb7c362b: entering follower state: follower="Node at a5eeeb9f-c5a7-f5b1-961f-1825eb7c362b [Follower]" leader-address= leader-id= +2025-03-20T03:49:49.656Z [WARN] server-a5eeeb9f-c5a7-f5b1-961f-1825eb7c362b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:49.656Z [INFO] server-a5eeeb9f-c5a7-f5b1-961f-1825eb7c362b: entering candidate state: node="Node at a5eeeb9f-c5a7-f5b1-961f-1825eb7c362b [Candidate]" term=2 +2025-03-20T03:49:49.656Z [INFO] server-a5eeeb9f-c5a7-f5b1-961f-1825eb7c362b: election won: term=2 tally=2 +2025-03-20T03:49:49.656Z [INFO] server-a5eeeb9f-c5a7-f5b1-961f-1825eb7c362b: entering leader state: leader="Node at a5eeeb9f-c5a7-f5b1-961f-1825eb7c362b [Leader]" +2025-03-20T03:49:49.656Z [INFO] server-a5eeeb9f-c5a7-f5b1-961f-1825eb7c362b: added peer, starting replication: peer=server-e9bff5b4-42d3-795e-8937-7fad9f48e35e +2025-03-20T03:49:49.656Z [INFO] server-a5eeeb9f-c5a7-f5b1-961f-1825eb7c362b: added peer, starting replication: peer=server-cfca0fb7-ceba-66fe-a0c1-0f8700165d7a +2025-03-20T03:49:49.656Z [INFO] server-a5eeeb9f-c5a7-f5b1-961f-1825eb7c362b: pipelining replication: peer="{Voter server-cfca0fb7-ceba-66fe-a0c1-0f8700165d7a cfca0fb7-ceba-66fe-a0c1-0f8700165d7a}" +2025-03-20T03:49:49.657Z [INFO] server-a5eeeb9f-c5a7-f5b1-961f-1825eb7c362b: pipelining replication: peer="{Voter server-e9bff5b4-42d3-795e-8937-7fad9f48e35e e9bff5b4-42d3-795e-8937-7fad9f48e35e}" +2025-03-20T03:49:50.662Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:49.657217209 +0000 UTC m=+21.331670033 (1 nodes), 872.333009ms from start of poll, 873.231442ms from cluster start. Timeout at 2025-03-20 03:49:50.662222286 +0000 UTC m=+22.336677107, 1.005007074s after stability +2025-03-20T03:49:50.666Z [INFO] server-a5eeeb9f-c5a7-f5b1-961f-1825eb7c362b: starting snapshot up to: index=12 +2025-03-20T03:49:50.666Z [INFO] creating new snapshot: path=/tmp/raft2838355490/snapshots/2-12-1742442590666.tmp +2025-03-20T03:49:50.672Z [INFO] server-a5eeeb9f-c5a7-f5b1-961f-1825eb7c362b: snapshot complete up to: index=12 +2025-03-20T03:49:50.679Z [INFO] server-b9dbfdee-4a74-b888-25d8-a6e810259d00: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b9dbfdee-4a74-b888-25d8-a6e810259d00 Address:b9dbfdee-4a74-b888-25d8-a6e810259d00} {Suffrage:Voter ID:server-aafe5397-bc28-db6c-92d5-f943a036aa75 Address:aafe5397-bc28-db6c-92d5-f943a036aa75} {Suffrage:Voter ID:server-a01b1ec5-708d-74be-8a97-5b903cc77d25 Address:a01b1ec5-708d-74be-8a97-5b903cc77d25}]" +2025-03-20T03:49:50.679Z [INFO] server-aafe5397-bc28-db6c-92d5-f943a036aa75: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b9dbfdee-4a74-b888-25d8-a6e810259d00 Address:b9dbfdee-4a74-b888-25d8-a6e810259d00} {Suffrage:Voter ID:server-aafe5397-bc28-db6c-92d5-f943a036aa75 Address:aafe5397-bc28-db6c-92d5-f943a036aa75} {Suffrage:Voter ID:server-a01b1ec5-708d-74be-8a97-5b903cc77d25 Address:a01b1ec5-708d-74be-8a97-5b903cc77d25}]" +2025-03-20T03:49:50.680Z [INFO] server-a01b1ec5-708d-74be-8a97-5b903cc77d25: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b9dbfdee-4a74-b888-25d8-a6e810259d00 Address:b9dbfdee-4a74-b888-25d8-a6e810259d00} {Suffrage:Voter ID:server-aafe5397-bc28-db6c-92d5-f943a036aa75 Address:aafe5397-bc28-db6c-92d5-f943a036aa75} {Suffrage:Voter ID:server-a01b1ec5-708d-74be-8a97-5b903cc77d25 Address:a01b1ec5-708d-74be-8a97-5b903cc77d25}]" +2025-03-20T03:49:50.680Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:50.680Z [INFO] server-b9dbfdee-4a74-b888-25d8-a6e810259d00: entering follower state: follower="Node at b9dbfdee-4a74-b888-25d8-a6e810259d00 [Follower]" leader-address= leader-id= +2025-03-20T03:49:50.680Z [INFO] server-aafe5397-bc28-db6c-92d5-f943a036aa75: entering follower state: follower="Node at aafe5397-bc28-db6c-92d5-f943a036aa75 [Follower]" leader-address= leader-id= +2025-03-20T03:49:50.680Z [INFO] server-a01b1ec5-708d-74be-8a97-5b903cc77d25: entering follower state: follower="Node at a01b1ec5-708d-74be-8a97-5b903cc77d25 [Follower]" leader-address= leader-id= +2025-03-20T03:49:51.244Z [WARN] server-a01b1ec5-708d-74be-8a97-5b903cc77d25: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:51.244Z [INFO] server-a01b1ec5-708d-74be-8a97-5b903cc77d25: entering candidate state: node="Node at a01b1ec5-708d-74be-8a97-5b903cc77d25 [Candidate]" term=2 +2025-03-20T03:49:51.244Z [INFO] server-a01b1ec5-708d-74be-8a97-5b903cc77d25: election won: term=2 tally=2 +2025-03-20T03:49:51.244Z [INFO] server-a01b1ec5-708d-74be-8a97-5b903cc77d25: entering leader state: leader="Node at a01b1ec5-708d-74be-8a97-5b903cc77d25 [Leader]" +2025-03-20T03:49:51.244Z [INFO] server-a01b1ec5-708d-74be-8a97-5b903cc77d25: added peer, starting replication: peer=server-b9dbfdee-4a74-b888-25d8-a6e810259d00 +2025-03-20T03:49:51.244Z [INFO] server-a01b1ec5-708d-74be-8a97-5b903cc77d25: added peer, starting replication: peer=server-aafe5397-bc28-db6c-92d5-f943a036aa75 +2025-03-20T03:49:51.245Z [INFO] server-a01b1ec5-708d-74be-8a97-5b903cc77d25: pipelining replication: peer="{Voter server-aafe5397-bc28-db6c-92d5-f943a036aa75 aafe5397-bc28-db6c-92d5-f943a036aa75}" +2025-03-20T03:49:51.245Z [INFO] server-a01b1ec5-708d-74be-8a97-5b903cc77d25: pipelining replication: peer="{Voter server-b9dbfdee-4a74-b888-25d8-a6e810259d00 b9dbfdee-4a74-b888-25d8-a6e810259d00}" +2025-03-20T03:49:52.250Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:51.24554233 +0000 UTC m=+22.919995129 (1 nodes), 565.42258ms from start of poll, 566.336836ms from cluster start. Timeout at 2025-03-20 03:49:52.250545922 +0000 UTC m=+23.924998799, 1.00500367s after stability +2025-03-20T03:49:52.251Z [INFO] creating new snapshot: path=/tmp/raft2884369328/snapshots/2-13-1742442592251.tmp +2025-03-20T03:49:52.252Z [INFO] server-a01b1ec5-708d-74be-8a97-5b903cc77d25: copied to local snapshot: bytes=71 +2025-03-20T03:49:52.253Z [INFO] server-a01b1ec5-708d-74be-8a97-5b903cc77d25: snapshot restore progress: id=2-13-1742442592251 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:49:52.253Z [INFO] server-a01b1ec5-708d-74be-8a97-5b903cc77d25: restored user snapshot: index=13 +2025-03-20T03:49:52.253Z [ERROR] server-a01b1ec5-708d-74be-8a97-5b903cc77d25: failed to get log: index=3 error="log not found" +2025-03-20T03:49:52.253Z [INFO] server-a01b1ec5-708d-74be-8a97-5b903cc77d25: aborting pipeline replication: peer="{Voter server-aafe5397-bc28-db6c-92d5-f943a036aa75 aafe5397-bc28-db6c-92d5-f943a036aa75}" +2025-03-20T03:49:52.253Z [ERROR] server-a01b1ec5-708d-74be-8a97-5b903cc77d25: failed to get log: index=3 error="log not found" +2025-03-20T03:49:52.253Z [INFO] server-a01b1ec5-708d-74be-8a97-5b903cc77d25: aborting pipeline replication: peer="{Voter server-b9dbfdee-4a74-b888-25d8-a6e810259d00 b9dbfdee-4a74-b888-25d8-a6e810259d00}" +2025-03-20T03:49:52.261Z [ERROR] server-a01b1ec5-708d-74be-8a97-5b903cc77d25: failed to get log: index=3 error="log not found" +2025-03-20T03:49:52.262Z [INFO] creating new snapshot: path=/tmp/raft1025768667/snapshots/2-13-1742442592262.tmp +2025-03-20T03:49:52.263Z [ERROR] server-a01b1ec5-708d-74be-8a97-5b903cc77d25: failed to get log: index=3 error="log not found" +2025-03-20T03:49:52.264Z [INFO] creating new snapshot: path=/tmp/raft505632312/snapshots/2-13-1742442592264.tmp +2025-03-20T03:49:52.264Z [INFO] server-b9dbfdee-4a74-b888-25d8-a6e810259d00: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:49:52.265Z [INFO] server-b9dbfdee-4a74-b888-25d8-a6e810259d00: copied to local snapshot: bytes=71 +2025-03-20T03:49:52.265Z [INFO] server-aafe5397-bc28-db6c-92d5-f943a036aa75: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:49:52.266Z [INFO] server-aafe5397-bc28-db6c-92d5-f943a036aa75: copied to local snapshot: bytes=71 +2025-03-20T03:49:52.267Z [INFO] server-aafe5397-bc28-db6c-92d5-f943a036aa75: snapshot restore progress: id=2-13-1742442592262 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:49:52.267Z [INFO] server-aafe5397-bc28-db6c-92d5-f943a036aa75: Installed remote snapshot +2025-03-20T03:49:52.267Z [INFO] server-a01b1ec5-708d-74be-8a97-5b903cc77d25: pipelining replication: peer="{Voter server-aafe5397-bc28-db6c-92d5-f943a036aa75 aafe5397-bc28-db6c-92d5-f943a036aa75}" +2025-03-20T03:49:52.267Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:49:52.268Z [INFO] server-b9dbfdee-4a74-b888-25d8-a6e810259d00: snapshot restore progress: id=2-13-1742442592264 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:49:52.268Z [INFO] server-b9dbfdee-4a74-b888-25d8-a6e810259d00: Installed remote snapshot +2025-03-20T03:49:52.268Z [INFO] server-a01b1ec5-708d-74be-8a97-5b903cc77d25: pipelining replication: peer="{Voter server-b9dbfdee-4a74-b888-25d8-a6e810259d00 b9dbfdee-4a74-b888-25d8-a6e810259d00}" +2025-03-20T03:49:53.273Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:49:52.267873103 +0000 UTC m=+23.942325891 (2 nodes), 2.787µs from start of poll, 1.588667598s from cluster start. Timeout at 2025-03-20 03:49:53.272893814 +0000 UTC m=+24.947346894, 1.005021003s after stability +2025-03-20T03:49:53.286Z [INFO] server-a5eeeb9f-c5a7-f5b1-961f-1825eb7c362b: aborting pipeline replication: peer="{Voter server-cfca0fb7-ceba-66fe-a0c1-0f8700165d7a cfca0fb7-ceba-66fe-a0c1-0f8700165d7a}" +2025-03-20T03:49:53.286Z [INFO] server-a5eeeb9f-c5a7-f5b1-961f-1825eb7c362b: aborting pipeline replication: peer="{Voter server-e9bff5b4-42d3-795e-8937-7fad9f48e35e e9bff5b4-42d3-795e-8937-7fad9f48e35e}" === RUN TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_true -2026-04-22T10:09:38.341Z [INFO] server-c72fd751-ff91-0442-ef68-3a538c0463a2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c72fd751-ff91-0442-ef68-3a538c0463a2 Address:c72fd751-ff91-0442-ef68-3a538c0463a2} {Suffrage:Voter ID:server-cc9dabed-11fa-95fe-a765-1e07d6d7e1d1 Address:cc9dabed-11fa-95fe-a765-1e07d6d7e1d1} {Suffrage:Voter ID:server-c3d675a8-dd55-e655-f5fa-542cfc7d640c Address:c3d675a8-dd55-e655-f5fa-542cfc7d640c}]" -2026-04-22T10:09:38.341Z [INFO] server-c72fd751-ff91-0442-ef68-3a538c0463a2: entering follower state: follower="Node at c72fd751-ff91-0442-ef68-3a538c0463a2 [Follower]" leader-address= leader-id= -2026-04-22T10:09:38.341Z [INFO] server-cc9dabed-11fa-95fe-a765-1e07d6d7e1d1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c72fd751-ff91-0442-ef68-3a538c0463a2 Address:c72fd751-ff91-0442-ef68-3a538c0463a2} {Suffrage:Voter ID:server-cc9dabed-11fa-95fe-a765-1e07d6d7e1d1 Address:cc9dabed-11fa-95fe-a765-1e07d6d7e1d1} {Suffrage:Voter ID:server-c3d675a8-dd55-e655-f5fa-542cfc7d640c Address:c3d675a8-dd55-e655-f5fa-542cfc7d640c}]" -2026-04-22T10:09:38.341Z [INFO] server-cc9dabed-11fa-95fe-a765-1e07d6d7e1d1: entering follower state: follower="Node at cc9dabed-11fa-95fe-a765-1e07d6d7e1d1 [Follower]" leader-address= leader-id= -2026-04-22T10:09:38.341Z [INFO] server-c3d675a8-dd55-e655-f5fa-542cfc7d640c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c72fd751-ff91-0442-ef68-3a538c0463a2 Address:c72fd751-ff91-0442-ef68-3a538c0463a2} {Suffrage:Voter ID:server-cc9dabed-11fa-95fe-a765-1e07d6d7e1d1 Address:cc9dabed-11fa-95fe-a765-1e07d6d7e1d1} {Suffrage:Voter ID:server-c3d675a8-dd55-e655-f5fa-542cfc7d640c Address:c3d675a8-dd55-e655-f5fa-542cfc7d640c}]" -2026-04-22T10:09:38.341Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:38.341Z [INFO] server-c3d675a8-dd55-e655-f5fa-542cfc7d640c: entering follower state: follower="Node at c3d675a8-dd55-e655-f5fa-542cfc7d640c [Follower]" leader-address= leader-id= -2026-04-22T10:09:38.990Z [WARN] server-cc9dabed-11fa-95fe-a765-1e07d6d7e1d1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:38.990Z [INFO] server-cc9dabed-11fa-95fe-a765-1e07d6d7e1d1: entering candidate state: node="Node at cc9dabed-11fa-95fe-a765-1e07d6d7e1d1 [Candidate]" term=2 -2026-04-22T10:09:38.990Z [INFO] server-cc9dabed-11fa-95fe-a765-1e07d6d7e1d1: election won: term=2 tally=2 -2026-04-22T10:09:38.990Z [INFO] server-cc9dabed-11fa-95fe-a765-1e07d6d7e1d1: entering leader state: leader="Node at cc9dabed-11fa-95fe-a765-1e07d6d7e1d1 [Leader]" -2026-04-22T10:09:38.990Z [INFO] server-cc9dabed-11fa-95fe-a765-1e07d6d7e1d1: added peer, starting replication: peer=server-c72fd751-ff91-0442-ef68-3a538c0463a2 -2026-04-22T10:09:38.990Z [INFO] server-cc9dabed-11fa-95fe-a765-1e07d6d7e1d1: added peer, starting replication: peer=server-c3d675a8-dd55-e655-f5fa-542cfc7d640c -2026-04-22T10:09:38.990Z [INFO] server-cc9dabed-11fa-95fe-a765-1e07d6d7e1d1: pipelining replication: peer="{Voter server-c3d675a8-dd55-e655-f5fa-542cfc7d640c c3d675a8-dd55-e655-f5fa-542cfc7d640c}" -2026-04-22T10:09:38.991Z [INFO] server-cc9dabed-11fa-95fe-a765-1e07d6d7e1d1: pipelining replication: peer="{Voter server-c72fd751-ff91-0442-ef68-3a538c0463a2 c72fd751-ff91-0442-ef68-3a538c0463a2}" -2026-04-22T10:09:39.996Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:38.990875424 +0000 UTC m=+25.014671521 (1 nodes), 649.040722ms from start of poll, 649.351793ms from cluster start. Timeout at 2026-04-22 10:09:39.995878621 +0000 UTC m=+26.019674702, 1.005003181s after stability -2026-04-22T10:09:39.996Z [INFO] server-cc9dabed-11fa-95fe-a765-1e07d6d7e1d1: starting snapshot up to: index=12 -2026-04-22T10:09:39.996Z [INFO] creating new snapshot: path=/tmp/raft382920578/snapshots/2-12-1776852579996.tmp -2026-04-22T10:09:39.997Z [INFO] server-cc9dabed-11fa-95fe-a765-1e07d6d7e1d1: snapshot complete up to: index=12 -2026-04-22T10:09:39.998Z [INFO] server-1080493c-6516-d56e-c505-f3d641baa6d3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1080493c-6516-d56e-c505-f3d641baa6d3 Address:1080493c-6516-d56e-c505-f3d641baa6d3} {Suffrage:Voter ID:server-faadff2b-87e9-2e77-4659-543972aaaa6f Address:faadff2b-87e9-2e77-4659-543972aaaa6f} {Suffrage:Voter ID:server-d057c27f-a5c0-a855-746a-43d9dbf08668 Address:d057c27f-a5c0-a855-746a-43d9dbf08668}]" -2026-04-22T10:09:39.998Z [INFO] server-1080493c-6516-d56e-c505-f3d641baa6d3: entering follower state: follower="Node at 1080493c-6516-d56e-c505-f3d641baa6d3 [Follower]" leader-address= leader-id= -2026-04-22T10:09:39.998Z [INFO] server-faadff2b-87e9-2e77-4659-543972aaaa6f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1080493c-6516-d56e-c505-f3d641baa6d3 Address:1080493c-6516-d56e-c505-f3d641baa6d3} {Suffrage:Voter ID:server-faadff2b-87e9-2e77-4659-543972aaaa6f Address:faadff2b-87e9-2e77-4659-543972aaaa6f} {Suffrage:Voter ID:server-d057c27f-a5c0-a855-746a-43d9dbf08668 Address:d057c27f-a5c0-a855-746a-43d9dbf08668}]" -2026-04-22T10:09:39.998Z [INFO] server-faadff2b-87e9-2e77-4659-543972aaaa6f: entering follower state: follower="Node at faadff2b-87e9-2e77-4659-543972aaaa6f [Follower]" leader-address= leader-id= -2026-04-22T10:09:39.998Z [INFO] server-d057c27f-a5c0-a855-746a-43d9dbf08668: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1080493c-6516-d56e-c505-f3d641baa6d3 Address:1080493c-6516-d56e-c505-f3d641baa6d3} {Suffrage:Voter ID:server-faadff2b-87e9-2e77-4659-543972aaaa6f Address:faadff2b-87e9-2e77-4659-543972aaaa6f} {Suffrage:Voter ID:server-d057c27f-a5c0-a855-746a-43d9dbf08668 Address:d057c27f-a5c0-a855-746a-43d9dbf08668}]" -2026-04-22T10:09:39.999Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:39.999Z [INFO] server-d057c27f-a5c0-a855-746a-43d9dbf08668: entering follower state: follower="Node at d057c27f-a5c0-a855-746a-43d9dbf08668 [Follower]" leader-address= leader-id= -2026-04-22T10:09:40.623Z [WARN] server-faadff2b-87e9-2e77-4659-543972aaaa6f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:40.623Z [INFO] server-faadff2b-87e9-2e77-4659-543972aaaa6f: entering candidate state: node="Node at faadff2b-87e9-2e77-4659-543972aaaa6f [Candidate]" term=2 -2026-04-22T10:09:40.623Z [INFO] server-faadff2b-87e9-2e77-4659-543972aaaa6f: election won: term=2 tally=2 -2026-04-22T10:09:40.623Z [INFO] server-faadff2b-87e9-2e77-4659-543972aaaa6f: entering leader state: leader="Node at faadff2b-87e9-2e77-4659-543972aaaa6f [Leader]" -2026-04-22T10:09:40.623Z [INFO] server-faadff2b-87e9-2e77-4659-543972aaaa6f: added peer, starting replication: peer=server-1080493c-6516-d56e-c505-f3d641baa6d3 -2026-04-22T10:09:40.623Z [INFO] server-faadff2b-87e9-2e77-4659-543972aaaa6f: added peer, starting replication: peer=server-d057c27f-a5c0-a855-746a-43d9dbf08668 -2026-04-22T10:09:40.623Z [INFO] server-faadff2b-87e9-2e77-4659-543972aaaa6f: pipelining replication: peer="{Voter server-1080493c-6516-d56e-c505-f3d641baa6d3 1080493c-6516-d56e-c505-f3d641baa6d3}" -2026-04-22T10:09:40.623Z [INFO] server-faadff2b-87e9-2e77-4659-543972aaaa6f: pipelining replication: peer="{Voter server-d057c27f-a5c0-a855-746a-43d9dbf08668 d057c27f-a5c0-a855-746a-43d9dbf08668}" -2026-04-22T10:09:41.629Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:40.623625231 +0000 UTC m=+26.647421319 (1 nodes), 624.61151ms from start of poll, 625.078635ms from cluster start. Timeout at 2026-04-22 10:09:41.62862723 +0000 UTC m=+27.652423341, 1.005002022s after stability -2026-04-22T10:09:41.629Z [INFO] creating new snapshot: path=/tmp/raft1247540436/snapshots/2-13-1776852581629.tmp -2026-04-22T10:09:41.629Z [INFO] server-faadff2b-87e9-2e77-4659-543972aaaa6f: copied to local snapshot: bytes=71 -2026-04-22T10:09:41.630Z [INFO] server-faadff2b-87e9-2e77-4659-543972aaaa6f: snapshot restore progress: id=2-13-1776852581629 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:41.630Z [INFO] server-faadff2b-87e9-2e77-4659-543972aaaa6f: removing all old logs from log store -2026-04-22T10:09:41.630Z [INFO] server-faadff2b-87e9-2e77-4659-543972aaaa6f: compacting logs: from=1 to=2 -2026-04-22T10:09:41.630Z [INFO] server-faadff2b-87e9-2e77-4659-543972aaaa6f: restored user snapshot: index=13 -2026-04-22T10:09:41.630Z [ERROR] server-faadff2b-87e9-2e77-4659-543972aaaa6f: failed to get log: index=2 error="log not found" -2026-04-22T10:09:41.630Z [ERROR] server-faadff2b-87e9-2e77-4659-543972aaaa6f: failed to get log: index=2 error="log not found" -2026-04-22T10:09:41.630Z [INFO] server-faadff2b-87e9-2e77-4659-543972aaaa6f: aborting pipeline replication: peer="{Voter server-d057c27f-a5c0-a855-746a-43d9dbf08668 d057c27f-a5c0-a855-746a-43d9dbf08668}" -2026-04-22T10:09:41.630Z [INFO] server-faadff2b-87e9-2e77-4659-543972aaaa6f: aborting pipeline replication: peer="{Voter server-1080493c-6516-d56e-c505-f3d641baa6d3 1080493c-6516-d56e-c505-f3d641baa6d3}" -2026-04-22T10:09:41.637Z [ERROR] server-faadff2b-87e9-2e77-4659-543972aaaa6f: failed to get log: index=2 error="log not found" -2026-04-22T10:09:41.638Z [ERROR] server-faadff2b-87e9-2e77-4659-543972aaaa6f: failed to get log: index=2 error="log not found" -2026-04-22T10:09:41.638Z [INFO] creating new snapshot: path=/tmp/raft1884505931/snapshots/2-13-1776852581638.tmp -2026-04-22T10:09:41.638Z [INFO] creating new snapshot: path=/tmp/raft3226239647/snapshots/2-13-1776852581638.tmp -2026-04-22T10:09:41.638Z [INFO] server-1080493c-6516-d56e-c505-f3d641baa6d3: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:41.638Z [INFO] server-d057c27f-a5c0-a855-746a-43d9dbf08668: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:41.639Z [INFO] server-1080493c-6516-d56e-c505-f3d641baa6d3: copied to local snapshot: bytes=71 -2026-04-22T10:09:41.639Z [INFO] server-d057c27f-a5c0-a855-746a-43d9dbf08668: copied to local snapshot: bytes=71 -2026-04-22T10:09:41.639Z [INFO] server-1080493c-6516-d56e-c505-f3d641baa6d3: snapshot restore progress: id=2-13-1776852581638 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:41.639Z [INFO] server-d057c27f-a5c0-a855-746a-43d9dbf08668: snapshot restore progress: id=2-13-1776852581638 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:41.639Z [INFO] server-1080493c-6516-d56e-c505-f3d641baa6d3: removing all old logs from log store -2026-04-22T10:09:41.639Z [INFO] server-1080493c-6516-d56e-c505-f3d641baa6d3: compacting logs: from=1 to=2 -2026-04-22T10:09:41.639Z [INFO] server-d057c27f-a5c0-a855-746a-43d9dbf08668: removing all old logs from log store -2026-04-22T10:09:41.639Z [INFO] server-1080493c-6516-d56e-c505-f3d641baa6d3: Installed remote snapshot -2026-04-22T10:09:41.639Z [INFO] server-d057c27f-a5c0-a855-746a-43d9dbf08668: compacting logs: from=1 to=2 -2026-04-22T10:09:41.639Z [INFO] server-d057c27f-a5c0-a855-746a-43d9dbf08668: Installed remote snapshot -2026-04-22T10:09:41.639Z [INFO] server-faadff2b-87e9-2e77-4659-543972aaaa6f: pipelining replication: peer="{Voter server-d057c27f-a5c0-a855-746a-43d9dbf08668 d057c27f-a5c0-a855-746a-43d9dbf08668}" -2026-04-22T10:09:41.639Z [INFO] server-faadff2b-87e9-2e77-4659-543972aaaa6f: pipelining replication: peer="{Voter server-1080493c-6516-d56e-c505-f3d641baa6d3 1080493c-6516-d56e-c505-f3d641baa6d3}" -2026-04-22T10:09:41.639Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:09:42.645Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:09:41.639699917 +0000 UTC m=+27.663496014 (2 nodes), 868ns from start of poll, 1.64115333s from cluster start. Timeout at 2026-04-22 10:09:42.644703605 +0000 UTC m=+28.668499697, 1.005003683s after stability -2026-04-22T10:09:42.658Z [INFO] server-cc9dabed-11fa-95fe-a765-1e07d6d7e1d1: aborting pipeline replication: peer="{Voter server-c3d675a8-dd55-e655-f5fa-542cfc7d640c c3d675a8-dd55-e655-f5fa-542cfc7d640c}" -2026-04-22T10:09:42.658Z [INFO] server-cc9dabed-11fa-95fe-a765-1e07d6d7e1d1: aborting pipeline replication: peer="{Voter server-c72fd751-ff91-0442-ef68-3a538c0463a2 c72fd751-ff91-0442-ef68-3a538c0463a2}" +2025-03-20T03:49:53.288Z [INFO] server-745138df-d290-c7fa-704a-ae7c1cb7162f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-745138df-d290-c7fa-704a-ae7c1cb7162f Address:745138df-d290-c7fa-704a-ae7c1cb7162f} {Suffrage:Voter ID:server-00f28293-ab07-f5b2-977e-b30017fa4a55 Address:00f28293-ab07-f5b2-977e-b30017fa4a55} {Suffrage:Voter ID:server-31a07e51-4c26-93ea-7ced-5a47895bfaf8 Address:31a07e51-4c26-93ea-7ced-5a47895bfaf8}]" +2025-03-20T03:49:53.289Z [INFO] server-00f28293-ab07-f5b2-977e-b30017fa4a55: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-745138df-d290-c7fa-704a-ae7c1cb7162f Address:745138df-d290-c7fa-704a-ae7c1cb7162f} {Suffrage:Voter ID:server-00f28293-ab07-f5b2-977e-b30017fa4a55 Address:00f28293-ab07-f5b2-977e-b30017fa4a55} {Suffrage:Voter ID:server-31a07e51-4c26-93ea-7ced-5a47895bfaf8 Address:31a07e51-4c26-93ea-7ced-5a47895bfaf8}]" +2025-03-20T03:49:53.289Z [INFO] server-31a07e51-4c26-93ea-7ced-5a47895bfaf8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-745138df-d290-c7fa-704a-ae7c1cb7162f Address:745138df-d290-c7fa-704a-ae7c1cb7162f} {Suffrage:Voter ID:server-00f28293-ab07-f5b2-977e-b30017fa4a55 Address:00f28293-ab07-f5b2-977e-b30017fa4a55} {Suffrage:Voter ID:server-31a07e51-4c26-93ea-7ced-5a47895bfaf8 Address:31a07e51-4c26-93ea-7ced-5a47895bfaf8}]" +2025-03-20T03:49:53.289Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:53.289Z [INFO] server-745138df-d290-c7fa-704a-ae7c1cb7162f: entering follower state: follower="Node at 745138df-d290-c7fa-704a-ae7c1cb7162f [Follower]" leader-address= leader-id= +2025-03-20T03:49:53.289Z [INFO] server-00f28293-ab07-f5b2-977e-b30017fa4a55: entering follower state: follower="Node at 00f28293-ab07-f5b2-977e-b30017fa4a55 [Follower]" leader-address= leader-id= +2025-03-20T03:49:53.289Z [INFO] server-31a07e51-4c26-93ea-7ced-5a47895bfaf8: entering follower state: follower="Node at 31a07e51-4c26-93ea-7ced-5a47895bfaf8 [Follower]" leader-address= leader-id= +2025-03-20T03:49:53.841Z [WARN] server-31a07e51-4c26-93ea-7ced-5a47895bfaf8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:53.841Z [INFO] server-31a07e51-4c26-93ea-7ced-5a47895bfaf8: entering candidate state: node="Node at 31a07e51-4c26-93ea-7ced-5a47895bfaf8 [Candidate]" term=2 +2025-03-20T03:49:53.841Z [INFO] server-31a07e51-4c26-93ea-7ced-5a47895bfaf8: election won: term=2 tally=2 +2025-03-20T03:49:53.841Z [INFO] server-31a07e51-4c26-93ea-7ced-5a47895bfaf8: entering leader state: leader="Node at 31a07e51-4c26-93ea-7ced-5a47895bfaf8 [Leader]" +2025-03-20T03:49:53.841Z [INFO] server-31a07e51-4c26-93ea-7ced-5a47895bfaf8: added peer, starting replication: peer=server-745138df-d290-c7fa-704a-ae7c1cb7162f +2025-03-20T03:49:53.841Z [INFO] server-31a07e51-4c26-93ea-7ced-5a47895bfaf8: added peer, starting replication: peer=server-00f28293-ab07-f5b2-977e-b30017fa4a55 +2025-03-20T03:49:53.841Z [INFO] server-31a07e51-4c26-93ea-7ced-5a47895bfaf8: pipelining replication: peer="{Voter server-00f28293-ab07-f5b2-977e-b30017fa4a55 00f28293-ab07-f5b2-977e-b30017fa4a55}" +2025-03-20T03:49:53.841Z [INFO] server-31a07e51-4c26-93ea-7ced-5a47895bfaf8: pipelining replication: peer="{Voter server-745138df-d290-c7fa-704a-ae7c1cb7162f 745138df-d290-c7fa-704a-ae7c1cb7162f}" +2025-03-20T03:49:54.847Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:53.842073049 +0000 UTC m=+25.516525852 (1 nodes), 552.772428ms from start of poll, 553.219968ms from cluster start. Timeout at 2025-03-20 03:49:54.847091201 +0000 UTC m=+26.521544128, 1.005018276s after stability +2025-03-20T03:49:54.849Z [INFO] server-31a07e51-4c26-93ea-7ced-5a47895bfaf8: starting snapshot up to: index=12 +2025-03-20T03:49:54.849Z [INFO] creating new snapshot: path=/tmp/raft2915029551/snapshots/2-12-1742442594849.tmp +2025-03-20T03:49:54.851Z [INFO] server-31a07e51-4c26-93ea-7ced-5a47895bfaf8: snapshot complete up to: index=12 +2025-03-20T03:49:54.854Z [INFO] server-9736863e-fe7d-d1aa-1e3b-eca4ca637c7d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9736863e-fe7d-d1aa-1e3b-eca4ca637c7d Address:9736863e-fe7d-d1aa-1e3b-eca4ca637c7d} {Suffrage:Voter ID:server-49c1bb5a-4b75-07ac-d971-6d75efa93acc Address:49c1bb5a-4b75-07ac-d971-6d75efa93acc} {Suffrage:Voter ID:server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c Address:e5c5cd54-3179-1786-ba9b-071f3cb13d4c}]" +2025-03-20T03:49:54.855Z [INFO] server-49c1bb5a-4b75-07ac-d971-6d75efa93acc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9736863e-fe7d-d1aa-1e3b-eca4ca637c7d Address:9736863e-fe7d-d1aa-1e3b-eca4ca637c7d} {Suffrage:Voter ID:server-49c1bb5a-4b75-07ac-d971-6d75efa93acc Address:49c1bb5a-4b75-07ac-d971-6d75efa93acc} {Suffrage:Voter ID:server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c Address:e5c5cd54-3179-1786-ba9b-071f3cb13d4c}]" +2025-03-20T03:49:54.855Z [INFO] server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9736863e-fe7d-d1aa-1e3b-eca4ca637c7d Address:9736863e-fe7d-d1aa-1e3b-eca4ca637c7d} {Suffrage:Voter ID:server-49c1bb5a-4b75-07ac-d971-6d75efa93acc Address:49c1bb5a-4b75-07ac-d971-6d75efa93acc} {Suffrage:Voter ID:server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c Address:e5c5cd54-3179-1786-ba9b-071f3cb13d4c}]" +2025-03-20T03:49:54.855Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:54.855Z [INFO] server-9736863e-fe7d-d1aa-1e3b-eca4ca637c7d: entering follower state: follower="Node at 9736863e-fe7d-d1aa-1e3b-eca4ca637c7d [Follower]" leader-address= leader-id= +2025-03-20T03:49:54.855Z [INFO] server-49c1bb5a-4b75-07ac-d971-6d75efa93acc: entering follower state: follower="Node at 49c1bb5a-4b75-07ac-d971-6d75efa93acc [Follower]" leader-address= leader-id= +2025-03-20T03:49:54.855Z [INFO] server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c: entering follower state: follower="Node at e5c5cd54-3179-1786-ba9b-071f3cb13d4c [Follower]" leader-address= leader-id= +2025-03-20T03:49:55.441Z [WARN] server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:55.441Z [INFO] server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c: entering candidate state: node="Node at e5c5cd54-3179-1786-ba9b-071f3cb13d4c [Candidate]" term=2 +2025-03-20T03:49:55.441Z [INFO] server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c: election won: term=2 tally=2 +2025-03-20T03:49:55.441Z [INFO] server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c: entering leader state: leader="Node at e5c5cd54-3179-1786-ba9b-071f3cb13d4c [Leader]" +2025-03-20T03:49:55.441Z [INFO] server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c: added peer, starting replication: peer=server-9736863e-fe7d-d1aa-1e3b-eca4ca637c7d +2025-03-20T03:49:55.441Z [INFO] server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c: added peer, starting replication: peer=server-49c1bb5a-4b75-07ac-d971-6d75efa93acc +2025-03-20T03:49:55.442Z [INFO] server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c: pipelining replication: peer="{Voter server-49c1bb5a-4b75-07ac-d971-6d75efa93acc 49c1bb5a-4b75-07ac-d971-6d75efa93acc}" +2025-03-20T03:49:55.442Z [INFO] server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c: pipelining replication: peer="{Voter server-9736863e-fe7d-d1aa-1e3b-eca4ca637c7d 9736863e-fe7d-d1aa-1e3b-eca4ca637c7d}" +2025-03-20T03:49:56.448Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:55.442500567 +0000 UTC m=+27.116953367 (1 nodes), 587.27757ms from start of poll, 588.54853ms from cluster start. Timeout at 2025-03-20 03:49:56.447506582 +0000 UTC m=+28.121959506, 1.005006139s after stability +2025-03-20T03:49:56.448Z [INFO] creating new snapshot: path=/tmp/raft992868787/snapshots/2-13-1742442596448.tmp +2025-03-20T03:49:56.450Z [INFO] server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c: copied to local snapshot: bytes=71 +2025-03-20T03:49:56.451Z [INFO] server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c: snapshot restore progress: id=2-13-1742442596448 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:49:56.451Z [INFO] server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c: removing all old logs from log store +2025-03-20T03:49:56.451Z [INFO] server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c: compacting logs: from=1 to=2 +2025-03-20T03:49:56.451Z [INFO] server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c: restored user snapshot: index=13 +2025-03-20T03:49:56.451Z [ERROR] server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c: failed to get log: index=2 error="log not found" +2025-03-20T03:49:56.451Z [INFO] server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c: aborting pipeline replication: peer="{Voter server-49c1bb5a-4b75-07ac-d971-6d75efa93acc 49c1bb5a-4b75-07ac-d971-6d75efa93acc}" +2025-03-20T03:49:56.451Z [ERROR] server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c: failed to get log: index=2 error="log not found" +2025-03-20T03:49:56.451Z [INFO] server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c: aborting pipeline replication: peer="{Voter server-9736863e-fe7d-d1aa-1e3b-eca4ca637c7d 9736863e-fe7d-d1aa-1e3b-eca4ca637c7d}" +2025-03-20T03:49:56.457Z [ERROR] server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c: failed to get log: index=2 error="log not found" +2025-03-20T03:49:56.457Z [INFO] creating new snapshot: path=/tmp/raft3088405365/snapshots/2-13-1742442596457.tmp +2025-03-20T03:49:56.458Z [INFO] server-49c1bb5a-4b75-07ac-d971-6d75efa93acc: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:49:56.462Z [ERROR] server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c: failed to get log: index=2 error="log not found" +2025-03-20T03:49:56.463Z [INFO] creating new snapshot: path=/tmp/raft29591258/snapshots/2-13-1742442596463.tmp +2025-03-20T03:49:56.463Z [INFO] server-9736863e-fe7d-d1aa-1e3b-eca4ca637c7d: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:49:56.464Z [INFO] server-9736863e-fe7d-d1aa-1e3b-eca4ca637c7d: copied to local snapshot: bytes=71 +2025-03-20T03:49:56.465Z [INFO] server-9736863e-fe7d-d1aa-1e3b-eca4ca637c7d: snapshot restore progress: id=2-13-1742442596463 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:49:56.465Z [INFO] server-9736863e-fe7d-d1aa-1e3b-eca4ca637c7d: removing all old logs from log store +2025-03-20T03:49:56.465Z [INFO] server-9736863e-fe7d-d1aa-1e3b-eca4ca637c7d: compacting logs: from=1 to=2 +2025-03-20T03:49:56.465Z [INFO] server-9736863e-fe7d-d1aa-1e3b-eca4ca637c7d: Installed remote snapshot +2025-03-20T03:49:56.465Z [INFO] server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c: pipelining replication: peer="{Voter server-9736863e-fe7d-d1aa-1e3b-eca4ca637c7d 9736863e-fe7d-d1aa-1e3b-eca4ca637c7d}" +2025-03-20T03:49:56.465Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:49:56.475Z [INFO] server-49c1bb5a-4b75-07ac-d971-6d75efa93acc: copied to local snapshot: bytes=71 +2025-03-20T03:49:56.476Z [INFO] server-49c1bb5a-4b75-07ac-d971-6d75efa93acc: snapshot restore progress: id=2-13-1742442596457 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:49:56.476Z [INFO] server-49c1bb5a-4b75-07ac-d971-6d75efa93acc: removing all old logs from log store +2025-03-20T03:49:56.476Z [INFO] server-49c1bb5a-4b75-07ac-d971-6d75efa93acc: compacting logs: from=1 to=2 +2025-03-20T03:49:56.476Z [INFO] server-49c1bb5a-4b75-07ac-d971-6d75efa93acc: Installed remote snapshot +2025-03-20T03:49:56.476Z [INFO] server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c: pipelining replication: peer="{Voter server-49c1bb5a-4b75-07ac-d971-6d75efa93acc 49c1bb5a-4b75-07ac-d971-6d75efa93acc}" +2025-03-20T03:49:57.471Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:49:56.465754048 +0000 UTC m=+28.140206914 (2 nodes), 1.91µs from start of poll, 1.611802077s from cluster start. Timeout at 2025-03-20 03:49:57.470802107 +0000 UTC m=+29.145254878, 1.005047964s after stability +2025-03-20T03:49:57.483Z [INFO] server-31a07e51-4c26-93ea-7ced-5a47895bfaf8: aborting pipeline replication: peer="{Voter server-745138df-d290-c7fa-704a-ae7c1cb7162f 745138df-d290-c7fa-704a-ae7c1cb7162f}" +2025-03-20T03:49:57.483Z [INFO] server-31a07e51-4c26-93ea-7ced-5a47895bfaf8: aborting pipeline replication: peer="{Voter server-00f28293-ab07-f5b2-977e-b30017fa4a55 00f28293-ab07-f5b2-977e-b30017fa4a55}" === RUN TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_false -2026-04-22T10:09:42.659Z [INFO] server-4a32a60e-1bea-6fec-9455-46430bb3a2e7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4a32a60e-1bea-6fec-9455-46430bb3a2e7 Address:4a32a60e-1bea-6fec-9455-46430bb3a2e7} {Suffrage:Voter ID:server-f075213b-e48a-3fa2-e465-848c9d1c01f3 Address:f075213b-e48a-3fa2-e465-848c9d1c01f3} {Suffrage:Voter ID:server-ae027834-6ce4-1cc8-fb04-85373f0d9bf2 Address:ae027834-6ce4-1cc8-fb04-85373f0d9bf2}]" -2026-04-22T10:09:42.659Z [INFO] server-f075213b-e48a-3fa2-e465-848c9d1c01f3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4a32a60e-1bea-6fec-9455-46430bb3a2e7 Address:4a32a60e-1bea-6fec-9455-46430bb3a2e7} {Suffrage:Voter ID:server-f075213b-e48a-3fa2-e465-848c9d1c01f3 Address:f075213b-e48a-3fa2-e465-848c9d1c01f3} {Suffrage:Voter ID:server-ae027834-6ce4-1cc8-fb04-85373f0d9bf2 Address:ae027834-6ce4-1cc8-fb04-85373f0d9bf2}]" -2026-04-22T10:09:42.659Z [INFO] server-4a32a60e-1bea-6fec-9455-46430bb3a2e7: entering follower state: follower="Node at 4a32a60e-1bea-6fec-9455-46430bb3a2e7 [Follower]" leader-address= leader-id= -2026-04-22T10:09:42.659Z [INFO] server-f075213b-e48a-3fa2-e465-848c9d1c01f3: entering follower state: follower="Node at f075213b-e48a-3fa2-e465-848c9d1c01f3 [Follower]" leader-address= leader-id= -2026-04-22T10:09:42.659Z [INFO] server-ae027834-6ce4-1cc8-fb04-85373f0d9bf2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4a32a60e-1bea-6fec-9455-46430bb3a2e7 Address:4a32a60e-1bea-6fec-9455-46430bb3a2e7} {Suffrage:Voter ID:server-f075213b-e48a-3fa2-e465-848c9d1c01f3 Address:f075213b-e48a-3fa2-e465-848c9d1c01f3} {Suffrage:Voter ID:server-ae027834-6ce4-1cc8-fb04-85373f0d9bf2 Address:ae027834-6ce4-1cc8-fb04-85373f0d9bf2}]" -2026-04-22T10:09:42.659Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:42.659Z [INFO] server-ae027834-6ce4-1cc8-fb04-85373f0d9bf2: entering follower state: follower="Node at ae027834-6ce4-1cc8-fb04-85373f0d9bf2 [Follower]" leader-address= leader-id= -2026-04-22T10:09:43.167Z [WARN] server-4a32a60e-1bea-6fec-9455-46430bb3a2e7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:43.167Z [INFO] server-4a32a60e-1bea-6fec-9455-46430bb3a2e7: entering candidate state: node="Node at 4a32a60e-1bea-6fec-9455-46430bb3a2e7 [Candidate]" term=2 -2026-04-22T10:09:43.167Z [INFO] server-4a32a60e-1bea-6fec-9455-46430bb3a2e7: election won: term=2 tally=2 -2026-04-22T10:09:43.167Z [INFO] server-4a32a60e-1bea-6fec-9455-46430bb3a2e7: entering leader state: leader="Node at 4a32a60e-1bea-6fec-9455-46430bb3a2e7 [Leader]" -2026-04-22T10:09:43.167Z [INFO] server-4a32a60e-1bea-6fec-9455-46430bb3a2e7: added peer, starting replication: peer=server-f075213b-e48a-3fa2-e465-848c9d1c01f3 -2026-04-22T10:09:43.167Z [INFO] server-4a32a60e-1bea-6fec-9455-46430bb3a2e7: added peer, starting replication: peer=server-ae027834-6ce4-1cc8-fb04-85373f0d9bf2 -2026-04-22T10:09:43.167Z [INFO] server-4a32a60e-1bea-6fec-9455-46430bb3a2e7: pipelining replication: peer="{Voter server-ae027834-6ce4-1cc8-fb04-85373f0d9bf2 ae027834-6ce4-1cc8-fb04-85373f0d9bf2}" -2026-04-22T10:09:43.167Z [INFO] server-4a32a60e-1bea-6fec-9455-46430bb3a2e7: pipelining replication: peer="{Voter server-f075213b-e48a-3fa2-e465-848c9d1c01f3 f075213b-e48a-3fa2-e465-848c9d1c01f3}" -2026-04-22T10:09:44.173Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:43.167804558 +0000 UTC m=+29.191600645 (1 nodes), 508.427496ms from start of poll, 508.786947ms from cluster start. Timeout at 2026-04-22 10:09:44.17280549 +0000 UTC m=+30.196601573, 1.005000928s after stability -2026-04-22T10:09:44.173Z [INFO] server-4a32a60e-1bea-6fec-9455-46430bb3a2e7: starting snapshot up to: index=12 -2026-04-22T10:09:44.173Z [INFO] creating new snapshot: path=/tmp/raft2942690260/snapshots/2-12-1776852584173.tmp -2026-04-22T10:09:44.174Z [INFO] server-4a32a60e-1bea-6fec-9455-46430bb3a2e7: snapshot complete up to: index=12 -2026-04-22T10:09:44.175Z [INFO] creating new snapshot: path=/tmp/raft2942690260/snapshots/2-23-1776852584175.tmp -2026-04-22T10:09:44.176Z [INFO] server-4a32a60e-1bea-6fec-9455-46430bb3a2e7: copied to local snapshot: bytes=71 -2026-04-22T10:09:44.176Z [INFO] server-4a32a60e-1bea-6fec-9455-46430bb3a2e7: snapshot restore progress: id=2-23-1776852584175 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:44.176Z [INFO] server-4a32a60e-1bea-6fec-9455-46430bb3a2e7: restored user snapshot: index=23 -2026-04-22T10:09:44.176Z [ERROR] server-4a32a60e-1bea-6fec-9455-46430bb3a2e7: failed to get log: index=23 error="log not found" -2026-04-22T10:09:44.176Z [ERROR] server-4a32a60e-1bea-6fec-9455-46430bb3a2e7: failed to get log: index=23 error="log not found" -2026-04-22T10:09:44.176Z [INFO] server-4a32a60e-1bea-6fec-9455-46430bb3a2e7: aborting pipeline replication: peer="{Voter server-f075213b-e48a-3fa2-e465-848c9d1c01f3 f075213b-e48a-3fa2-e465-848c9d1c01f3}" -2026-04-22T10:09:44.176Z [INFO] server-4a32a60e-1bea-6fec-9455-46430bb3a2e7: aborting pipeline replication: peer="{Voter server-ae027834-6ce4-1cc8-fb04-85373f0d9bf2 ae027834-6ce4-1cc8-fb04-85373f0d9bf2}" -2026-04-22T10:09:44.184Z [ERROR] server-4a32a60e-1bea-6fec-9455-46430bb3a2e7: failed to get log: index=23 error="log not found" -2026-04-22T10:09:44.184Z [INFO] creating new snapshot: path=/tmp/raft1080222454/snapshots/2-23-1776852584184.tmp -2026-04-22T10:09:44.184Z [INFO] server-f075213b-e48a-3fa2-e465-848c9d1c01f3: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:44.185Z [INFO] server-f075213b-e48a-3fa2-e465-848c9d1c01f3: copied to local snapshot: bytes=71 -2026-04-22T10:09:44.185Z [INFO] server-f075213b-e48a-3fa2-e465-848c9d1c01f3: snapshot restore progress: id=2-23-1776852584184 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:44.185Z [ERROR] server-4a32a60e-1bea-6fec-9455-46430bb3a2e7: failed to get log: index=23 error="log not found" -2026-04-22T10:09:44.185Z [INFO] server-f075213b-e48a-3fa2-e465-848c9d1c01f3: Installed remote snapshot -2026-04-22T10:09:44.185Z [WARN] server-f075213b-e48a-3fa2-e465-848c9d1c01f3: skipping application of old log: index=22 -2026-04-22T10:09:44.185Z [INFO] server-4a32a60e-1bea-6fec-9455-46430bb3a2e7: pipelining replication: peer="{Voter server-f075213b-e48a-3fa2-e465-848c9d1c01f3 f075213b-e48a-3fa2-e465-848c9d1c01f3}" -2026-04-22T10:09:44.185Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:09:44.185Z [INFO] creating new snapshot: path=/tmp/raft3183089841/snapshots/2-23-1776852584185.tmp -2026-04-22T10:09:44.186Z [INFO] server-ae027834-6ce4-1cc8-fb04-85373f0d9bf2: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:44.186Z [INFO] server-ae027834-6ce4-1cc8-fb04-85373f0d9bf2: copied to local snapshot: bytes=71 -2026-04-22T10:09:44.186Z [INFO] server-ae027834-6ce4-1cc8-fb04-85373f0d9bf2: snapshot restore progress: id=2-23-1776852584185 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:44.186Z [INFO] server-ae027834-6ce4-1cc8-fb04-85373f0d9bf2: Installed remote snapshot -2026-04-22T10:09:44.186Z [INFO] server-4a32a60e-1bea-6fec-9455-46430bb3a2e7: pipelining replication: peer="{Voter server-ae027834-6ce4-1cc8-fb04-85373f0d9bf2 ae027834-6ce4-1cc8-fb04-85373f0d9bf2}" -2026-04-22T10:09:45.190Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:09:44.185697819 +0000 UTC m=+30.209493925 (2 nodes), 1.113µs from start of poll, 1.526680227s from cluster start. Timeout at 2026-04-22 10:09:45.190705914 +0000 UTC m=+31.214502005, 1.00500808s after stability -2026-04-22T10:09:45.202Z [INFO] server-4a32a60e-1bea-6fec-9455-46430bb3a2e7: aborting pipeline replication: peer="{Voter server-ae027834-6ce4-1cc8-fb04-85373f0d9bf2 ae027834-6ce4-1cc8-fb04-85373f0d9bf2}" -2026-04-22T10:09:45.203Z [INFO] server-4a32a60e-1bea-6fec-9455-46430bb3a2e7: aborting pipeline replication: peer="{Voter server-f075213b-e48a-3fa2-e465-848c9d1c01f3 f075213b-e48a-3fa2-e465-848c9d1c01f3}" +2025-03-20T03:49:57.484Z [INFO] server-acf62c16-cf46-dfed-755d-9dc3b98b11c5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-acf62c16-cf46-dfed-755d-9dc3b98b11c5 Address:acf62c16-cf46-dfed-755d-9dc3b98b11c5} {Suffrage:Voter ID:server-1008b222-779c-a854-9815-e188baea034b Address:1008b222-779c-a854-9815-e188baea034b} {Suffrage:Voter ID:server-12bfdb43-7ac3-54e3-b695-242f09da983b Address:12bfdb43-7ac3-54e3-b695-242f09da983b}]" +2025-03-20T03:49:57.484Z [INFO] server-1008b222-779c-a854-9815-e188baea034b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-acf62c16-cf46-dfed-755d-9dc3b98b11c5 Address:acf62c16-cf46-dfed-755d-9dc3b98b11c5} {Suffrage:Voter ID:server-1008b222-779c-a854-9815-e188baea034b Address:1008b222-779c-a854-9815-e188baea034b} {Suffrage:Voter ID:server-12bfdb43-7ac3-54e3-b695-242f09da983b Address:12bfdb43-7ac3-54e3-b695-242f09da983b}]" +2025-03-20T03:49:57.485Z [INFO] server-12bfdb43-7ac3-54e3-b695-242f09da983b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-acf62c16-cf46-dfed-755d-9dc3b98b11c5 Address:acf62c16-cf46-dfed-755d-9dc3b98b11c5} {Suffrage:Voter ID:server-1008b222-779c-a854-9815-e188baea034b Address:1008b222-779c-a854-9815-e188baea034b} {Suffrage:Voter ID:server-12bfdb43-7ac3-54e3-b695-242f09da983b Address:12bfdb43-7ac3-54e3-b695-242f09da983b}]" +2025-03-20T03:49:57.485Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:49:57.485Z [INFO] server-acf62c16-cf46-dfed-755d-9dc3b98b11c5: entering follower state: follower="Node at acf62c16-cf46-dfed-755d-9dc3b98b11c5 [Follower]" leader-address= leader-id= +2025-03-20T03:49:57.485Z [INFO] server-1008b222-779c-a854-9815-e188baea034b: entering follower state: follower="Node at 1008b222-779c-a854-9815-e188baea034b [Follower]" leader-address= leader-id= +2025-03-20T03:49:57.485Z [INFO] server-12bfdb43-7ac3-54e3-b695-242f09da983b: entering follower state: follower="Node at 12bfdb43-7ac3-54e3-b695-242f09da983b [Follower]" leader-address= leader-id= +2025-03-20T03:49:58.051Z [WARN] server-1008b222-779c-a854-9815-e188baea034b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:49:58.051Z [INFO] server-1008b222-779c-a854-9815-e188baea034b: entering candidate state: node="Node at 1008b222-779c-a854-9815-e188baea034b [Candidate]" term=2 +2025-03-20T03:49:58.056Z [INFO] server-1008b222-779c-a854-9815-e188baea034b: election won: term=2 tally=2 +2025-03-20T03:49:58.056Z [INFO] server-1008b222-779c-a854-9815-e188baea034b: entering leader state: leader="Node at 1008b222-779c-a854-9815-e188baea034b [Leader]" +2025-03-20T03:49:58.056Z [INFO] server-1008b222-779c-a854-9815-e188baea034b: added peer, starting replication: peer=server-acf62c16-cf46-dfed-755d-9dc3b98b11c5 +2025-03-20T03:49:58.056Z [INFO] server-1008b222-779c-a854-9815-e188baea034b: added peer, starting replication: peer=server-12bfdb43-7ac3-54e3-b695-242f09da983b +2025-03-20T03:49:58.057Z [INFO] server-1008b222-779c-a854-9815-e188baea034b: pipelining replication: peer="{Voter server-12bfdb43-7ac3-54e3-b695-242f09da983b 12bfdb43-7ac3-54e3-b695-242f09da983b}" +2025-03-20T03:49:58.057Z [INFO] server-1008b222-779c-a854-9815-e188baea034b: pipelining replication: peer="{Voter server-acf62c16-cf46-dfed-755d-9dc3b98b11c5 acf62c16-cf46-dfed-755d-9dc3b98b11c5}" +2025-03-20T03:49:59.062Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:49:58.057118065 +0000 UTC m=+29.731571165 (1 nodes), 572.059442ms from start of poll, 572.422243ms from cluster start. Timeout at 2025-03-20 03:49:59.062122233 +0000 UTC m=+30.736575317, 1.005004152s after stability +2025-03-20T03:49:59.063Z [INFO] server-1008b222-779c-a854-9815-e188baea034b: starting snapshot up to: index=12 +2025-03-20T03:49:59.063Z [INFO] creating new snapshot: path=/tmp/raft90261725/snapshots/2-12-1742442599063.tmp +2025-03-20T03:49:59.069Z [INFO] server-1008b222-779c-a854-9815-e188baea034b: snapshot complete up to: index=12 +2025-03-20T03:49:59.072Z [INFO] creating new snapshot: path=/tmp/raft90261725/snapshots/2-23-1742442599072.tmp +2025-03-20T03:49:59.073Z [INFO] server-1008b222-779c-a854-9815-e188baea034b: copied to local snapshot: bytes=71 +2025-03-20T03:49:59.073Z [INFO] server-1008b222-779c-a854-9815-e188baea034b: snapshot restore progress: id=2-23-1742442599072 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:49:59.073Z [INFO] server-1008b222-779c-a854-9815-e188baea034b: restored user snapshot: index=23 +2025-03-20T03:49:59.073Z [ERROR] server-1008b222-779c-a854-9815-e188baea034b: failed to get log: index=23 error="log not found" +2025-03-20T03:49:59.073Z [INFO] server-1008b222-779c-a854-9815-e188baea034b: aborting pipeline replication: peer="{Voter server-12bfdb43-7ac3-54e3-b695-242f09da983b 12bfdb43-7ac3-54e3-b695-242f09da983b}" +2025-03-20T03:49:59.073Z [ERROR] server-1008b222-779c-a854-9815-e188baea034b: failed to get log: index=23 error="log not found" +2025-03-20T03:49:59.073Z [INFO] server-1008b222-779c-a854-9815-e188baea034b: aborting pipeline replication: peer="{Voter server-acf62c16-cf46-dfed-755d-9dc3b98b11c5 acf62c16-cf46-dfed-755d-9dc3b98b11c5}" +2025-03-20T03:49:59.073Z [ERROR] server-1008b222-779c-a854-9815-e188baea034b: failed to get log: index=23 error="log not found" +2025-03-20T03:49:59.075Z [INFO] creating new snapshot: path=/tmp/raft1104386339/snapshots/2-23-1742442599075.tmp +2025-03-20T03:49:59.075Z [INFO] server-acf62c16-cf46-dfed-755d-9dc3b98b11c5: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:49:59.076Z [INFO] server-acf62c16-cf46-dfed-755d-9dc3b98b11c5: copied to local snapshot: bytes=71 +2025-03-20T03:49:59.076Z [INFO] server-acf62c16-cf46-dfed-755d-9dc3b98b11c5: snapshot restore progress: id=2-23-1742442599075 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:49:59.076Z [INFO] server-acf62c16-cf46-dfed-755d-9dc3b98b11c5: Installed remote snapshot +2025-03-20T03:49:59.084Z [ERROR] server-1008b222-779c-a854-9815-e188baea034b: failed to get log: index=23 error="log not found" +2025-03-20T03:49:59.085Z [INFO] creating new snapshot: path=/tmp/raft3812560825/snapshots/2-23-1742442599085.tmp +2025-03-20T03:49:59.085Z [INFO] server-12bfdb43-7ac3-54e3-b695-242f09da983b: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:49:59.086Z [INFO] server-12bfdb43-7ac3-54e3-b695-242f09da983b: copied to local snapshot: bytes=71 +2025-03-20T03:49:59.077Z [WARN] server-acf62c16-cf46-dfed-755d-9dc3b98b11c5: skipping application of old log: index=22 +2025-03-20T03:49:59.086Z [INFO] server-12bfdb43-7ac3-54e3-b695-242f09da983b: snapshot restore progress: id=2-23-1742442599085 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:49:59.086Z [INFO] server-1008b222-779c-a854-9815-e188baea034b: pipelining replication: peer="{Voter server-acf62c16-cf46-dfed-755d-9dc3b98b11c5 acf62c16-cf46-dfed-755d-9dc3b98b11c5}" +2025-03-20T03:49:59.086Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:49:59.086Z [INFO] server-12bfdb43-7ac3-54e3-b695-242f09da983b: Installed remote snapshot +2025-03-20T03:49:59.087Z [INFO] server-1008b222-779c-a854-9815-e188baea034b: pipelining replication: peer="{Voter server-12bfdb43-7ac3-54e3-b695-242f09da983b 12bfdb43-7ac3-54e3-b695-242f09da983b}" +2025-03-20T03:50:00.092Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:49:59.086782798 +0000 UTC m=+30.761236148 (2 nodes), 2.553µs from start of poll, 1.602087226s from cluster start. Timeout at 2025-03-20 03:50:00.091817424 +0000 UTC m=+31.766270413, 1.005034265s after stability +2025-03-20T03:50:00.137Z [INFO] server-1008b222-779c-a854-9815-e188baea034b: aborting pipeline replication: peer="{Voter server-12bfdb43-7ac3-54e3-b695-242f09da983b 12bfdb43-7ac3-54e3-b695-242f09da983b}" +2025-03-20T03:50:00.138Z [INFO] server-1008b222-779c-a854-9815-e188baea034b: aborting pipeline replication: peer="{Voter server-acf62c16-cf46-dfed-755d-9dc3b98b11c5 acf62c16-cf46-dfed-755d-9dc3b98b11c5}" === RUN TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_false -2026-04-22T10:09:45.204Z [INFO] server-d0784757-67f4-2897-a35a-71a1dfff95d4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d0784757-67f4-2897-a35a-71a1dfff95d4 Address:d0784757-67f4-2897-a35a-71a1dfff95d4} {Suffrage:Voter ID:server-46b862c5-258e-7e9f-f70e-c6369394eeb2 Address:46b862c5-258e-7e9f-f70e-c6369394eeb2} {Suffrage:Voter ID:server-7bae69ab-fd8f-bea1-167a-7301dcf276cc Address:7bae69ab-fd8f-bea1-167a-7301dcf276cc}]" -2026-04-22T10:09:45.204Z [INFO] server-d0784757-67f4-2897-a35a-71a1dfff95d4: entering follower state: follower="Node at d0784757-67f4-2897-a35a-71a1dfff95d4 [Follower]" leader-address= leader-id= -2026-04-22T10:09:45.204Z [INFO] server-46b862c5-258e-7e9f-f70e-c6369394eeb2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d0784757-67f4-2897-a35a-71a1dfff95d4 Address:d0784757-67f4-2897-a35a-71a1dfff95d4} {Suffrage:Voter ID:server-46b862c5-258e-7e9f-f70e-c6369394eeb2 Address:46b862c5-258e-7e9f-f70e-c6369394eeb2} {Suffrage:Voter ID:server-7bae69ab-fd8f-bea1-167a-7301dcf276cc Address:7bae69ab-fd8f-bea1-167a-7301dcf276cc}]" -2026-04-22T10:09:45.204Z [INFO] server-46b862c5-258e-7e9f-f70e-c6369394eeb2: entering follower state: follower="Node at 46b862c5-258e-7e9f-f70e-c6369394eeb2 [Follower]" leader-address= leader-id= -2026-04-22T10:09:45.204Z [INFO] server-7bae69ab-fd8f-bea1-167a-7301dcf276cc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d0784757-67f4-2897-a35a-71a1dfff95d4 Address:d0784757-67f4-2897-a35a-71a1dfff95d4} {Suffrage:Voter ID:server-46b862c5-258e-7e9f-f70e-c6369394eeb2 Address:46b862c5-258e-7e9f-f70e-c6369394eeb2} {Suffrage:Voter ID:server-7bae69ab-fd8f-bea1-167a-7301dcf276cc Address:7bae69ab-fd8f-bea1-167a-7301dcf276cc}]" -2026-04-22T10:09:45.204Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:45.204Z [INFO] server-7bae69ab-fd8f-bea1-167a-7301dcf276cc: entering follower state: follower="Node at 7bae69ab-fd8f-bea1-167a-7301dcf276cc [Follower]" leader-address= leader-id= -2026-04-22T10:09:45.733Z [WARN] server-d0784757-67f4-2897-a35a-71a1dfff95d4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:45.733Z [INFO] server-d0784757-67f4-2897-a35a-71a1dfff95d4: entering candidate state: node="Node at d0784757-67f4-2897-a35a-71a1dfff95d4 [Candidate]" term=2 -2026-04-22T10:09:45.733Z [INFO] server-d0784757-67f4-2897-a35a-71a1dfff95d4: election won: term=2 tally=2 -2026-04-22T10:09:45.733Z [INFO] server-d0784757-67f4-2897-a35a-71a1dfff95d4: entering leader state: leader="Node at d0784757-67f4-2897-a35a-71a1dfff95d4 [Leader]" -2026-04-22T10:09:45.733Z [INFO] server-d0784757-67f4-2897-a35a-71a1dfff95d4: added peer, starting replication: peer=server-46b862c5-258e-7e9f-f70e-c6369394eeb2 -2026-04-22T10:09:45.733Z [INFO] server-d0784757-67f4-2897-a35a-71a1dfff95d4: added peer, starting replication: peer=server-7bae69ab-fd8f-bea1-167a-7301dcf276cc -2026-04-22T10:09:45.733Z [INFO] server-d0784757-67f4-2897-a35a-71a1dfff95d4: pipelining replication: peer="{Voter server-7bae69ab-fd8f-bea1-167a-7301dcf276cc 7bae69ab-fd8f-bea1-167a-7301dcf276cc}" -2026-04-22T10:09:45.733Z [INFO] server-d0784757-67f4-2897-a35a-71a1dfff95d4: pipelining replication: peer="{Voter server-46b862c5-258e-7e9f-f70e-c6369394eeb2 46b862c5-258e-7e9f-f70e-c6369394eeb2}" -2026-04-22T10:09:46.738Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:45.733327294 +0000 UTC m=+31.757123401 (1 nodes), 528.69613ms from start of poll, 529.063332ms from cluster start. Timeout at 2026-04-22 10:09:46.738333988 +0000 UTC m=+32.762130069, 1.005006668s after stability -2026-04-22T10:09:46.739Z [INFO] server-d0784757-67f4-2897-a35a-71a1dfff95d4: starting snapshot up to: index=12 -2026-04-22T10:09:46.739Z [INFO] creating new snapshot: path=/tmp/raft497367915/snapshots/2-12-1776852586739.tmp -2026-04-22T10:09:46.739Z [INFO] server-d0784757-67f4-2897-a35a-71a1dfff95d4: snapshot complete up to: index=12 -2026-04-22T10:09:46.740Z [INFO] creating new snapshot: path=/tmp/raft497367915/snapshots/2-23-1776852586740.tmp -2026-04-22T10:09:46.740Z [INFO] server-d0784757-67f4-2897-a35a-71a1dfff95d4: copied to local snapshot: bytes=71 -2026-04-22T10:09:46.741Z [INFO] server-d0784757-67f4-2897-a35a-71a1dfff95d4: snapshot restore progress: id=2-23-1776852586740 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:46.741Z [INFO] server-d0784757-67f4-2897-a35a-71a1dfff95d4: removing all old logs from log store -2026-04-22T10:09:46.741Z [INFO] server-d0784757-67f4-2897-a35a-71a1dfff95d4: compacting logs: from=1 to=22 -2026-04-22T10:09:46.741Z [INFO] server-d0784757-67f4-2897-a35a-71a1dfff95d4: restored user snapshot: index=23 -2026-04-22T10:09:46.741Z [ERROR] server-d0784757-67f4-2897-a35a-71a1dfff95d4: failed to get log: index=22 error="log not found" -2026-04-22T10:09:46.741Z [ERROR] server-d0784757-67f4-2897-a35a-71a1dfff95d4: failed to get log: index=22 error="log not found" -2026-04-22T10:09:46.741Z [INFO] server-d0784757-67f4-2897-a35a-71a1dfff95d4: aborting pipeline replication: peer="{Voter server-46b862c5-258e-7e9f-f70e-c6369394eeb2 46b862c5-258e-7e9f-f70e-c6369394eeb2}" -2026-04-22T10:09:46.741Z [INFO] server-d0784757-67f4-2897-a35a-71a1dfff95d4: aborting pipeline replication: peer="{Voter server-7bae69ab-fd8f-bea1-167a-7301dcf276cc 7bae69ab-fd8f-bea1-167a-7301dcf276cc}" -2026-04-22T10:09:46.748Z [ERROR] server-d0784757-67f4-2897-a35a-71a1dfff95d4: failed to get log: index=22 error="log not found" -2026-04-22T10:09:46.748Z [INFO] creating new snapshot: path=/tmp/raft3782626090/snapshots/2-23-1776852586748.tmp -2026-04-22T10:09:46.749Z [INFO] server-7bae69ab-fd8f-bea1-167a-7301dcf276cc: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:46.749Z [INFO] server-7bae69ab-fd8f-bea1-167a-7301dcf276cc: copied to local snapshot: bytes=71 -2026-04-22T10:09:46.749Z [INFO] server-7bae69ab-fd8f-bea1-167a-7301dcf276cc: snapshot restore progress: id=2-23-1776852586748 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:46.749Z [INFO] server-7bae69ab-fd8f-bea1-167a-7301dcf276cc: removing all old logs from log store -2026-04-22T10:09:46.749Z [INFO] server-7bae69ab-fd8f-bea1-167a-7301dcf276cc: compacting logs: from=1 to=22 -2026-04-22T10:09:46.749Z [INFO] server-7bae69ab-fd8f-bea1-167a-7301dcf276cc: Installed remote snapshot -2026-04-22T10:09:46.749Z [WARN] server-7bae69ab-fd8f-bea1-167a-7301dcf276cc: skipping application of old log: index=22 -2026-04-22T10:09:46.749Z [INFO] server-d0784757-67f4-2897-a35a-71a1dfff95d4: pipelining replication: peer="{Voter server-7bae69ab-fd8f-bea1-167a-7301dcf276cc 7bae69ab-fd8f-bea1-167a-7301dcf276cc}" -2026-04-22T10:09:46.749Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:09:46.751Z [ERROR] server-d0784757-67f4-2897-a35a-71a1dfff95d4: failed to get log: index=22 error="log not found" -2026-04-22T10:09:46.752Z [INFO] creating new snapshot: path=/tmp/raft4186836534/snapshots/2-23-1776852586752.tmp -2026-04-22T10:09:46.752Z [INFO] server-46b862c5-258e-7e9f-f70e-c6369394eeb2: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:46.752Z [INFO] server-46b862c5-258e-7e9f-f70e-c6369394eeb2: copied to local snapshot: bytes=71 -2026-04-22T10:09:46.752Z [INFO] server-46b862c5-258e-7e9f-f70e-c6369394eeb2: snapshot restore progress: id=2-23-1776852586752 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:46.752Z [INFO] server-46b862c5-258e-7e9f-f70e-c6369394eeb2: removing all old logs from log store -2026-04-22T10:09:46.752Z [INFO] server-46b862c5-258e-7e9f-f70e-c6369394eeb2: compacting logs: from=1 to=22 -2026-04-22T10:09:46.752Z [INFO] server-46b862c5-258e-7e9f-f70e-c6369394eeb2: Installed remote snapshot -2026-04-22T10:09:46.752Z [INFO] server-d0784757-67f4-2897-a35a-71a1dfff95d4: pipelining replication: peer="{Voter server-46b862c5-258e-7e9f-f70e-c6369394eeb2 46b862c5-258e-7e9f-f70e-c6369394eeb2}" -2026-04-22T10:09:47.755Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:09:46.74981997 +0000 UTC m=+32.773616071 (2 nodes), 1.181µs from start of poll, 1.545556002s from cluster start. Timeout at 2026-04-22 10:09:47.754826817 +0000 UTC m=+33.778622899, 1.005006828s after stability -2026-04-22T10:09:47.767Z [INFO] server-d0784757-67f4-2897-a35a-71a1dfff95d4: aborting pipeline replication: peer="{Voter server-7bae69ab-fd8f-bea1-167a-7301dcf276cc 7bae69ab-fd8f-bea1-167a-7301dcf276cc}" -2026-04-22T10:09:47.767Z [INFO] server-d0784757-67f4-2897-a35a-71a1dfff95d4: aborting pipeline replication: peer="{Voter server-46b862c5-258e-7e9f-f70e-c6369394eeb2 46b862c5-258e-7e9f-f70e-c6369394eeb2}" +2025-03-20T03:50:00.146Z [INFO] server-71318f73-6671-a372-41c1-a9ff611c8d96: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-71318f73-6671-a372-41c1-a9ff611c8d96 Address:71318f73-6671-a372-41c1-a9ff611c8d96} {Suffrage:Voter ID:server-60eead5c-02bc-b1f2-c20e-271f2d962de2 Address:60eead5c-02bc-b1f2-c20e-271f2d962de2} {Suffrage:Voter ID:server-4b80d3b1-80e7-0ee5-6310-ff6554fddf04 Address:4b80d3b1-80e7-0ee5-6310-ff6554fddf04}]" +2025-03-20T03:50:00.146Z [INFO] server-60eead5c-02bc-b1f2-c20e-271f2d962de2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-71318f73-6671-a372-41c1-a9ff611c8d96 Address:71318f73-6671-a372-41c1-a9ff611c8d96} {Suffrage:Voter ID:server-60eead5c-02bc-b1f2-c20e-271f2d962de2 Address:60eead5c-02bc-b1f2-c20e-271f2d962de2} {Suffrage:Voter ID:server-4b80d3b1-80e7-0ee5-6310-ff6554fddf04 Address:4b80d3b1-80e7-0ee5-6310-ff6554fddf04}]" +2025-03-20T03:50:00.146Z [INFO] server-4b80d3b1-80e7-0ee5-6310-ff6554fddf04: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-71318f73-6671-a372-41c1-a9ff611c8d96 Address:71318f73-6671-a372-41c1-a9ff611c8d96} {Suffrage:Voter ID:server-60eead5c-02bc-b1f2-c20e-271f2d962de2 Address:60eead5c-02bc-b1f2-c20e-271f2d962de2} {Suffrage:Voter ID:server-4b80d3b1-80e7-0ee5-6310-ff6554fddf04 Address:4b80d3b1-80e7-0ee5-6310-ff6554fddf04}]" +2025-03-20T03:50:00.146Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:50:00.146Z [INFO] server-71318f73-6671-a372-41c1-a9ff611c8d96: entering follower state: follower="Node at 71318f73-6671-a372-41c1-a9ff611c8d96 [Follower]" leader-address= leader-id= +2025-03-20T03:50:00.146Z [INFO] server-4b80d3b1-80e7-0ee5-6310-ff6554fddf04: entering follower state: follower="Node at 4b80d3b1-80e7-0ee5-6310-ff6554fddf04 [Follower]" leader-address= leader-id= +2025-03-20T03:50:00.146Z [INFO] server-60eead5c-02bc-b1f2-c20e-271f2d962de2: entering follower state: follower="Node at 60eead5c-02bc-b1f2-c20e-271f2d962de2 [Follower]" leader-address= leader-id= +2025-03-20T03:50:00.688Z [WARN] server-71318f73-6671-a372-41c1-a9ff611c8d96: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:50:00.688Z [INFO] server-71318f73-6671-a372-41c1-a9ff611c8d96: entering candidate state: node="Node at 71318f73-6671-a372-41c1-a9ff611c8d96 [Candidate]" term=2 +2025-03-20T03:50:00.689Z [INFO] server-71318f73-6671-a372-41c1-a9ff611c8d96: election won: term=2 tally=2 +2025-03-20T03:50:00.689Z [INFO] server-71318f73-6671-a372-41c1-a9ff611c8d96: entering leader state: leader="Node at 71318f73-6671-a372-41c1-a9ff611c8d96 [Leader]" +2025-03-20T03:50:00.689Z [INFO] server-71318f73-6671-a372-41c1-a9ff611c8d96: added peer, starting replication: peer=server-60eead5c-02bc-b1f2-c20e-271f2d962de2 +2025-03-20T03:50:00.689Z [INFO] server-71318f73-6671-a372-41c1-a9ff611c8d96: added peer, starting replication: peer=server-4b80d3b1-80e7-0ee5-6310-ff6554fddf04 +2025-03-20T03:50:00.689Z [INFO] server-71318f73-6671-a372-41c1-a9ff611c8d96: pipelining replication: peer="{Voter server-4b80d3b1-80e7-0ee5-6310-ff6554fddf04 4b80d3b1-80e7-0ee5-6310-ff6554fddf04}" +2025-03-20T03:50:00.690Z [INFO] server-71318f73-6671-a372-41c1-a9ff611c8d96: pipelining replication: peer="{Voter server-60eead5c-02bc-b1f2-c20e-271f2d962de2 60eead5c-02bc-b1f2-c20e-271f2d962de2}" +2025-03-20T03:50:01.695Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:50:00.689653494 +0000 UTC m=+32.364106598 (1 nodes), 542.807769ms from start of poll, 543.370061ms from cluster start. Timeout at 2025-03-20 03:50:01.694660872 +0000 UTC m=+33.369117558, 1.00501096s after stability +2025-03-20T03:50:01.697Z [INFO] server-71318f73-6671-a372-41c1-a9ff611c8d96: starting snapshot up to: index=12 +2025-03-20T03:50:01.697Z [INFO] creating new snapshot: path=/tmp/raft774454920/snapshots/2-12-1742442601697.tmp +2025-03-20T03:50:01.698Z [INFO] server-71318f73-6671-a372-41c1-a9ff611c8d96: snapshot complete up to: index=12 +2025-03-20T03:50:01.700Z [INFO] creating new snapshot: path=/tmp/raft774454920/snapshots/2-23-1742442601700.tmp +2025-03-20T03:50:01.702Z [INFO] server-71318f73-6671-a372-41c1-a9ff611c8d96: copied to local snapshot: bytes=71 +2025-03-20T03:50:01.702Z [INFO] server-71318f73-6671-a372-41c1-a9ff611c8d96: snapshot restore progress: id=2-23-1742442601700 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:01.702Z [INFO] server-71318f73-6671-a372-41c1-a9ff611c8d96: removing all old logs from log store +2025-03-20T03:50:01.702Z [INFO] server-71318f73-6671-a372-41c1-a9ff611c8d96: compacting logs: from=1 to=22 +2025-03-20T03:50:01.703Z [INFO] server-71318f73-6671-a372-41c1-a9ff611c8d96: restored user snapshot: index=23 +2025-03-20T03:50:01.703Z [ERROR] server-71318f73-6671-a372-41c1-a9ff611c8d96: failed to get log: index=22 error="log not found" +2025-03-20T03:50:01.703Z [INFO] server-71318f73-6671-a372-41c1-a9ff611c8d96: aborting pipeline replication: peer="{Voter server-4b80d3b1-80e7-0ee5-6310-ff6554fddf04 4b80d3b1-80e7-0ee5-6310-ff6554fddf04}" +2025-03-20T03:50:01.703Z [ERROR] server-71318f73-6671-a372-41c1-a9ff611c8d96: failed to get log: index=22 error="log not found" +2025-03-20T03:50:01.703Z [INFO] server-71318f73-6671-a372-41c1-a9ff611c8d96: aborting pipeline replication: peer="{Voter server-60eead5c-02bc-b1f2-c20e-271f2d962de2 60eead5c-02bc-b1f2-c20e-271f2d962de2}" +2025-03-20T03:50:01.710Z [ERROR] server-71318f73-6671-a372-41c1-a9ff611c8d96: failed to get log: index=22 error="log not found" +2025-03-20T03:50:01.711Z [INFO] creating new snapshot: path=/tmp/raft1201939928/snapshots/2-23-1742442601711.tmp +2025-03-20T03:50:01.711Z [INFO] server-4b80d3b1-80e7-0ee5-6310-ff6554fddf04: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:01.712Z [INFO] server-4b80d3b1-80e7-0ee5-6310-ff6554fddf04: copied to local snapshot: bytes=71 +2025-03-20T03:50:01.712Z [INFO] server-4b80d3b1-80e7-0ee5-6310-ff6554fddf04: snapshot restore progress: id=2-23-1742442601711 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:01.712Z [ERROR] server-71318f73-6671-a372-41c1-a9ff611c8d96: failed to get log: index=22 error="log not found" +2025-03-20T03:50:01.713Z [INFO] creating new snapshot: path=/tmp/raft2073139522/snapshots/2-23-1742442601713.tmp +2025-03-20T03:50:01.713Z [INFO] server-60eead5c-02bc-b1f2-c20e-271f2d962de2: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:01.713Z [INFO] server-60eead5c-02bc-b1f2-c20e-271f2d962de2: copied to local snapshot: bytes=71 +2025-03-20T03:50:01.714Z [INFO] server-60eead5c-02bc-b1f2-c20e-271f2d962de2: snapshot restore progress: id=2-23-1742442601713 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:01.714Z [INFO] server-60eead5c-02bc-b1f2-c20e-271f2d962de2: removing all old logs from log store +2025-03-20T03:50:01.714Z [INFO] server-60eead5c-02bc-b1f2-c20e-271f2d962de2: compacting logs: from=1 to=22 +2025-03-20T03:50:01.714Z [INFO] server-60eead5c-02bc-b1f2-c20e-271f2d962de2: Installed remote snapshot +2025-03-20T03:50:01.714Z [INFO] server-4b80d3b1-80e7-0ee5-6310-ff6554fddf04: removing all old logs from log store +2025-03-20T03:50:01.714Z [INFO] server-4b80d3b1-80e7-0ee5-6310-ff6554fddf04: compacting logs: from=1 to=22 +2025-03-20T03:50:01.714Z [WARN] server-60eead5c-02bc-b1f2-c20e-271f2d962de2: skipping application of old log: index=22 +2025-03-20T03:50:01.714Z [INFO] server-71318f73-6671-a372-41c1-a9ff611c8d96: pipelining replication: peer="{Voter server-60eead5c-02bc-b1f2-c20e-271f2d962de2 60eead5c-02bc-b1f2-c20e-271f2d962de2}" +2025-03-20T03:50:01.714Z [INFO] server-4b80d3b1-80e7-0ee5-6310-ff6554fddf04: Installed remote snapshot +2025-03-20T03:50:01.714Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:50:01.715Z [INFO] server-71318f73-6671-a372-41c1-a9ff611c8d96: pipelining replication: peer="{Voter server-4b80d3b1-80e7-0ee5-6310-ff6554fddf04 4b80d3b1-80e7-0ee5-6310-ff6554fddf04}" +2025-03-20T03:50:02.721Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:50:01.714990559 +0000 UTC m=+33.389443430 (2 nodes), 2.139µs from start of poll, 1.568706893s from cluster start. Timeout at 2025-03-20 03:50:02.720015649 +0000 UTC m=+34.394468549, 1.005025119s after stability +2025-03-20T03:50:02.733Z [INFO] server-71318f73-6671-a372-41c1-a9ff611c8d96: aborting pipeline replication: peer="{Voter server-4b80d3b1-80e7-0ee5-6310-ff6554fddf04 4b80d3b1-80e7-0ee5-6310-ff6554fddf04}" +2025-03-20T03:50:02.733Z [INFO] server-71318f73-6671-a372-41c1-a9ff611c8d96: aborting pipeline replication: peer="{Voter server-60eead5c-02bc-b1f2-c20e-271f2d962de2 60eead5c-02bc-b1f2-c20e-271f2d962de2}" === RUN TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_true -2026-04-22T10:09:47.769Z [INFO] server-53b930d6-902a-8d19-e980-e348be0ad61e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-53b930d6-902a-8d19-e980-e348be0ad61e Address:53b930d6-902a-8d19-e980-e348be0ad61e} {Suffrage:Voter ID:server-cf51392d-7455-69a0-0f12-a233b2f12d74 Address:cf51392d-7455-69a0-0f12-a233b2f12d74} {Suffrage:Voter ID:server-ea2aca6b-3b7c-35b6-d187-cc0568eb2d1b Address:ea2aca6b-3b7c-35b6-d187-cc0568eb2d1b}]" -2026-04-22T10:09:47.769Z [INFO] server-53b930d6-902a-8d19-e980-e348be0ad61e: entering follower state: follower="Node at 53b930d6-902a-8d19-e980-e348be0ad61e [Follower]" leader-address= leader-id= -2026-04-22T10:09:47.769Z [INFO] server-cf51392d-7455-69a0-0f12-a233b2f12d74: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-53b930d6-902a-8d19-e980-e348be0ad61e Address:53b930d6-902a-8d19-e980-e348be0ad61e} {Suffrage:Voter ID:server-cf51392d-7455-69a0-0f12-a233b2f12d74 Address:cf51392d-7455-69a0-0f12-a233b2f12d74} {Suffrage:Voter ID:server-ea2aca6b-3b7c-35b6-d187-cc0568eb2d1b Address:ea2aca6b-3b7c-35b6-d187-cc0568eb2d1b}]" -2026-04-22T10:09:47.769Z [INFO] server-cf51392d-7455-69a0-0f12-a233b2f12d74: entering follower state: follower="Node at cf51392d-7455-69a0-0f12-a233b2f12d74 [Follower]" leader-address= leader-id= -2026-04-22T10:09:47.769Z [INFO] server-ea2aca6b-3b7c-35b6-d187-cc0568eb2d1b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-53b930d6-902a-8d19-e980-e348be0ad61e Address:53b930d6-902a-8d19-e980-e348be0ad61e} {Suffrage:Voter ID:server-cf51392d-7455-69a0-0f12-a233b2f12d74 Address:cf51392d-7455-69a0-0f12-a233b2f12d74} {Suffrage:Voter ID:server-ea2aca6b-3b7c-35b6-d187-cc0568eb2d1b Address:ea2aca6b-3b7c-35b6-d187-cc0568eb2d1b}]" -2026-04-22T10:09:47.770Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:47.770Z [INFO] server-ea2aca6b-3b7c-35b6-d187-cc0568eb2d1b: entering follower state: follower="Node at ea2aca6b-3b7c-35b6-d187-cc0568eb2d1b [Follower]" leader-address= leader-id= -2026-04-22T10:09:48.503Z [WARN] server-cf51392d-7455-69a0-0f12-a233b2f12d74: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:48.503Z [INFO] server-cf51392d-7455-69a0-0f12-a233b2f12d74: entering candidate state: node="Node at cf51392d-7455-69a0-0f12-a233b2f12d74 [Candidate]" term=2 -2026-04-22T10:09:48.503Z [INFO] server-cf51392d-7455-69a0-0f12-a233b2f12d74: election won: term=2 tally=2 -2026-04-22T10:09:48.503Z [INFO] server-cf51392d-7455-69a0-0f12-a233b2f12d74: entering leader state: leader="Node at cf51392d-7455-69a0-0f12-a233b2f12d74 [Leader]" -2026-04-22T10:09:48.503Z [INFO] server-cf51392d-7455-69a0-0f12-a233b2f12d74: added peer, starting replication: peer=server-53b930d6-902a-8d19-e980-e348be0ad61e -2026-04-22T10:09:48.503Z [INFO] server-cf51392d-7455-69a0-0f12-a233b2f12d74: added peer, starting replication: peer=server-ea2aca6b-3b7c-35b6-d187-cc0568eb2d1b -2026-04-22T10:09:48.503Z [INFO] server-cf51392d-7455-69a0-0f12-a233b2f12d74: pipelining replication: peer="{Voter server-53b930d6-902a-8d19-e980-e348be0ad61e 53b930d6-902a-8d19-e980-e348be0ad61e}" -2026-04-22T10:09:48.503Z [INFO] server-cf51392d-7455-69a0-0f12-a233b2f12d74: pipelining replication: peer="{Voter server-ea2aca6b-3b7c-35b6-d187-cc0568eb2d1b ea2aca6b-3b7c-35b6-d187-cc0568eb2d1b}" -2026-04-22T10:09:49.509Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:48.503454203 +0000 UTC m=+34.527250306 (1 nodes), 733.41144ms from start of poll, 734.072777ms from cluster start. Timeout at 2026-04-22 10:09:49.508460181 +0000 UTC m=+35.532256264, 1.005005958s after stability -2026-04-22T10:09:49.510Z [INFO] server-cf51392d-7455-69a0-0f12-a233b2f12d74: starting snapshot up to: index=12 -2026-04-22T10:09:49.510Z [INFO] creating new snapshot: path=/tmp/raft1055243269/snapshots/2-12-1776852589510.tmp -2026-04-22T10:09:49.511Z [INFO] server-cf51392d-7455-69a0-0f12-a233b2f12d74: snapshot complete up to: index=12 -2026-04-22T10:09:49.512Z [INFO] server-7cbd52b0-7c6b-a761-0e5f-e1ae8ad22021: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7cbd52b0-7c6b-a761-0e5f-e1ae8ad22021 Address:7cbd52b0-7c6b-a761-0e5f-e1ae8ad22021} {Suffrage:Voter ID:server-e3d31228-e3dc-2e66-b62f-fcb19190190f Address:e3d31228-e3dc-2e66-b62f-fcb19190190f} {Suffrage:Voter ID:server-a4c36d7f-f625-4c52-c844-32d87b5ced9e Address:a4c36d7f-f625-4c52-c844-32d87b5ced9e}]" -2026-04-22T10:09:49.512Z [INFO] server-7cbd52b0-7c6b-a761-0e5f-e1ae8ad22021: entering follower state: follower="Node at 7cbd52b0-7c6b-a761-0e5f-e1ae8ad22021 [Follower]" leader-address= leader-id= -2026-04-22T10:09:49.512Z [INFO] server-e3d31228-e3dc-2e66-b62f-fcb19190190f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7cbd52b0-7c6b-a761-0e5f-e1ae8ad22021 Address:7cbd52b0-7c6b-a761-0e5f-e1ae8ad22021} {Suffrage:Voter ID:server-e3d31228-e3dc-2e66-b62f-fcb19190190f Address:e3d31228-e3dc-2e66-b62f-fcb19190190f} {Suffrage:Voter ID:server-a4c36d7f-f625-4c52-c844-32d87b5ced9e Address:a4c36d7f-f625-4c52-c844-32d87b5ced9e}]" -2026-04-22T10:09:49.512Z [INFO] server-e3d31228-e3dc-2e66-b62f-fcb19190190f: entering follower state: follower="Node at e3d31228-e3dc-2e66-b62f-fcb19190190f [Follower]" leader-address= leader-id= -2026-04-22T10:09:49.512Z [INFO] server-a4c36d7f-f625-4c52-c844-32d87b5ced9e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7cbd52b0-7c6b-a761-0e5f-e1ae8ad22021 Address:7cbd52b0-7c6b-a761-0e5f-e1ae8ad22021} {Suffrage:Voter ID:server-e3d31228-e3dc-2e66-b62f-fcb19190190f Address:e3d31228-e3dc-2e66-b62f-fcb19190190f} {Suffrage:Voter ID:server-a4c36d7f-f625-4c52-c844-32d87b5ced9e Address:a4c36d7f-f625-4c52-c844-32d87b5ced9e}]" -2026-04-22T10:09:49.512Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:49.512Z [INFO] server-a4c36d7f-f625-4c52-c844-32d87b5ced9e: entering follower state: follower="Node at a4c36d7f-f625-4c52-c844-32d87b5ced9e [Follower]" leader-address= leader-id= -2026-04-22T10:09:50.068Z [WARN] server-e3d31228-e3dc-2e66-b62f-fcb19190190f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:50.068Z [INFO] server-e3d31228-e3dc-2e66-b62f-fcb19190190f: entering candidate state: node="Node at e3d31228-e3dc-2e66-b62f-fcb19190190f [Candidate]" term=2 -2026-04-22T10:09:50.068Z [INFO] server-e3d31228-e3dc-2e66-b62f-fcb19190190f: election won: term=2 tally=2 -2026-04-22T10:09:50.068Z [INFO] server-e3d31228-e3dc-2e66-b62f-fcb19190190f: entering leader state: leader="Node at e3d31228-e3dc-2e66-b62f-fcb19190190f [Leader]" -2026-04-22T10:09:50.068Z [INFO] server-e3d31228-e3dc-2e66-b62f-fcb19190190f: added peer, starting replication: peer=server-7cbd52b0-7c6b-a761-0e5f-e1ae8ad22021 -2026-04-22T10:09:50.068Z [INFO] server-e3d31228-e3dc-2e66-b62f-fcb19190190f: added peer, starting replication: peer=server-a4c36d7f-f625-4c52-c844-32d87b5ced9e -2026-04-22T10:09:50.068Z [INFO] server-e3d31228-e3dc-2e66-b62f-fcb19190190f: pipelining replication: peer="{Voter server-a4c36d7f-f625-4c52-c844-32d87b5ced9e a4c36d7f-f625-4c52-c844-32d87b5ced9e}" -2026-04-22T10:09:50.068Z [INFO] server-e3d31228-e3dc-2e66-b62f-fcb19190190f: pipelining replication: peer="{Voter server-7cbd52b0-7c6b-a761-0e5f-e1ae8ad22021 7cbd52b0-7c6b-a761-0e5f-e1ae8ad22021}" -2026-04-22T10:09:51.073Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:50.068372724 +0000 UTC m=+36.092168815 (1 nodes), 555.78727ms from start of poll, 556.129253ms from cluster start. Timeout at 2026-04-22 10:09:51.073377139 +0000 UTC m=+37.097173221, 1.005004406s after stability -2026-04-22T10:09:51.074Z [INFO] creating new snapshot: path=/tmp/raft3643439632/snapshots/2-14-1776852591074.tmp -2026-04-22T10:09:51.074Z [INFO] server-e3d31228-e3dc-2e66-b62f-fcb19190190f: copied to local snapshot: bytes=71 -2026-04-22T10:09:51.075Z [INFO] server-e3d31228-e3dc-2e66-b62f-fcb19190190f: snapshot restore progress: id=2-14-1776852591074 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:51.075Z [INFO] server-e3d31228-e3dc-2e66-b62f-fcb19190190f: restored user snapshot: index=14 -2026-04-22T10:09:51.075Z [ERROR] server-e3d31228-e3dc-2e66-b62f-fcb19190190f: failed to get log: index=3 error="log not found" -2026-04-22T10:09:51.075Z [INFO] server-e3d31228-e3dc-2e66-b62f-fcb19190190f: aborting pipeline replication: peer="{Voter server-a4c36d7f-f625-4c52-c844-32d87b5ced9e a4c36d7f-f625-4c52-c844-32d87b5ced9e}" -2026-04-22T10:09:51.075Z [ERROR] server-e3d31228-e3dc-2e66-b62f-fcb19190190f: failed to get log: index=3 error="log not found" -2026-04-22T10:09:51.075Z [INFO] server-e3d31228-e3dc-2e66-b62f-fcb19190190f: aborting pipeline replication: peer="{Voter server-7cbd52b0-7c6b-a761-0e5f-e1ae8ad22021 7cbd52b0-7c6b-a761-0e5f-e1ae8ad22021}" -2026-04-22T10:09:51.081Z [ERROR] server-e3d31228-e3dc-2e66-b62f-fcb19190190f: failed to get log: index=3 error="log not found" -2026-04-22T10:09:51.081Z [INFO] creating new snapshot: path=/tmp/raft2369900500/snapshots/2-14-1776852591081.tmp -2026-04-22T10:09:51.081Z [INFO] server-a4c36d7f-f625-4c52-c844-32d87b5ced9e: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:51.082Z [INFO] server-a4c36d7f-f625-4c52-c844-32d87b5ced9e: copied to local snapshot: bytes=71 -2026-04-22T10:09:51.082Z [ERROR] server-e3d31228-e3dc-2e66-b62f-fcb19190190f: failed to get log: index=3 error="log not found" -2026-04-22T10:09:51.082Z [INFO] server-a4c36d7f-f625-4c52-c844-32d87b5ced9e: snapshot restore progress: id=2-14-1776852591081 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:51.082Z [INFO] server-a4c36d7f-f625-4c52-c844-32d87b5ced9e: Installed remote snapshot -2026-04-22T10:09:51.082Z [INFO] server-e3d31228-e3dc-2e66-b62f-fcb19190190f: pipelining replication: peer="{Voter server-a4c36d7f-f625-4c52-c844-32d87b5ced9e a4c36d7f-f625-4c52-c844-32d87b5ced9e}" -2026-04-22T10:09:51.082Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:09:51.082Z [INFO] creating new snapshot: path=/tmp/raft1668314335/snapshots/2-14-1776852591082.tmp -2026-04-22T10:09:51.082Z [INFO] server-7cbd52b0-7c6b-a761-0e5f-e1ae8ad22021: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:51.083Z [INFO] server-7cbd52b0-7c6b-a761-0e5f-e1ae8ad22021: copied to local snapshot: bytes=71 -2026-04-22T10:09:51.083Z [INFO] server-7cbd52b0-7c6b-a761-0e5f-e1ae8ad22021: snapshot restore progress: id=2-14-1776852591082 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:51.083Z [INFO] server-7cbd52b0-7c6b-a761-0e5f-e1ae8ad22021: Installed remote snapshot -2026-04-22T10:09:51.083Z [INFO] server-e3d31228-e3dc-2e66-b62f-fcb19190190f: pipelining replication: peer="{Voter server-7cbd52b0-7c6b-a761-0e5f-e1ae8ad22021 7cbd52b0-7c6b-a761-0e5f-e1ae8ad22021}" -2026-04-22T10:09:52.088Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:09:51.082630747 +0000 UTC m=+37.106426834 (2 nodes), 637ns from start of poll, 1.570387272s from cluster start. Timeout at 2026-04-22 10:09:52.087635182 +0000 UTC m=+38.111431273, 1.005004439s after stability -2026-04-22T10:09:52.100Z [INFO] server-cf51392d-7455-69a0-0f12-a233b2f12d74: aborting pipeline replication: peer="{Voter server-53b930d6-902a-8d19-e980-e348be0ad61e 53b930d6-902a-8d19-e980-e348be0ad61e}" -2026-04-22T10:09:52.100Z [INFO] server-cf51392d-7455-69a0-0f12-a233b2f12d74: aborting pipeline replication: peer="{Voter server-ea2aca6b-3b7c-35b6-d187-cc0568eb2d1b ea2aca6b-3b7c-35b6-d187-cc0568eb2d1b}" +2025-03-20T03:50:02.735Z [INFO] server-05373282-805a-6852-bf2d-a5ea1d98da2b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-05373282-805a-6852-bf2d-a5ea1d98da2b Address:05373282-805a-6852-bf2d-a5ea1d98da2b} {Suffrage:Voter ID:server-f42053e6-517d-8495-870b-827b0e68bb11 Address:f42053e6-517d-8495-870b-827b0e68bb11} {Suffrage:Voter ID:server-936b6b88-7ed8-f300-c07f-d0af029c17aa Address:936b6b88-7ed8-f300-c07f-d0af029c17aa}]" +2025-03-20T03:50:02.735Z [INFO] server-f42053e6-517d-8495-870b-827b0e68bb11: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-05373282-805a-6852-bf2d-a5ea1d98da2b Address:05373282-805a-6852-bf2d-a5ea1d98da2b} {Suffrage:Voter ID:server-f42053e6-517d-8495-870b-827b0e68bb11 Address:f42053e6-517d-8495-870b-827b0e68bb11} {Suffrage:Voter ID:server-936b6b88-7ed8-f300-c07f-d0af029c17aa Address:936b6b88-7ed8-f300-c07f-d0af029c17aa}]" +2025-03-20T03:50:02.735Z [INFO] server-936b6b88-7ed8-f300-c07f-d0af029c17aa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-05373282-805a-6852-bf2d-a5ea1d98da2b Address:05373282-805a-6852-bf2d-a5ea1d98da2b} {Suffrage:Voter ID:server-f42053e6-517d-8495-870b-827b0e68bb11 Address:f42053e6-517d-8495-870b-827b0e68bb11} {Suffrage:Voter ID:server-936b6b88-7ed8-f300-c07f-d0af029c17aa Address:936b6b88-7ed8-f300-c07f-d0af029c17aa}]" +2025-03-20T03:50:02.736Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:50:02.736Z [INFO] server-05373282-805a-6852-bf2d-a5ea1d98da2b: entering follower state: follower="Node at 05373282-805a-6852-bf2d-a5ea1d98da2b [Follower]" leader-address= leader-id= +2025-03-20T03:50:02.736Z [INFO] server-f42053e6-517d-8495-870b-827b0e68bb11: entering follower state: follower="Node at f42053e6-517d-8495-870b-827b0e68bb11 [Follower]" leader-address= leader-id= +2025-03-20T03:50:02.736Z [INFO] server-936b6b88-7ed8-f300-c07f-d0af029c17aa: entering follower state: follower="Node at 936b6b88-7ed8-f300-c07f-d0af029c17aa [Follower]" leader-address= leader-id= +2025-03-20T03:50:03.248Z [WARN] server-936b6b88-7ed8-f300-c07f-d0af029c17aa: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:50:03.248Z [INFO] server-936b6b88-7ed8-f300-c07f-d0af029c17aa: entering candidate state: node="Node at 936b6b88-7ed8-f300-c07f-d0af029c17aa [Candidate]" term=2 +2025-03-20T03:50:03.248Z [INFO] server-936b6b88-7ed8-f300-c07f-d0af029c17aa: election won: term=2 tally=2 +2025-03-20T03:50:03.248Z [INFO] server-936b6b88-7ed8-f300-c07f-d0af029c17aa: entering leader state: leader="Node at 936b6b88-7ed8-f300-c07f-d0af029c17aa [Leader]" +2025-03-20T03:50:03.248Z [INFO] server-936b6b88-7ed8-f300-c07f-d0af029c17aa: added peer, starting replication: peer=server-05373282-805a-6852-bf2d-a5ea1d98da2b +2025-03-20T03:50:03.249Z [INFO] server-936b6b88-7ed8-f300-c07f-d0af029c17aa: added peer, starting replication: peer=server-f42053e6-517d-8495-870b-827b0e68bb11 +2025-03-20T03:50:03.249Z [INFO] server-936b6b88-7ed8-f300-c07f-d0af029c17aa: pipelining replication: peer="{Voter server-05373282-805a-6852-bf2d-a5ea1d98da2b 05373282-805a-6852-bf2d-a5ea1d98da2b}" +2025-03-20T03:50:03.249Z [INFO] server-936b6b88-7ed8-f300-c07f-d0af029c17aa: pipelining replication: peer="{Voter server-f42053e6-517d-8495-870b-827b0e68bb11 f42053e6-517d-8495-870b-827b0e68bb11}" +2025-03-20T03:50:04.254Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:50:03.249053076 +0000 UTC m=+34.923506170 (1 nodes), 513.014424ms from start of poll, 513.51143ms from cluster start. Timeout at 2025-03-20 03:50:04.254059707 +0000 UTC m=+35.928512685, 1.005006515s after stability +2025-03-20T03:50:04.256Z [INFO] server-936b6b88-7ed8-f300-c07f-d0af029c17aa: starting snapshot up to: index=12 +2025-03-20T03:50:04.256Z [INFO] creating new snapshot: path=/tmp/raft3473304277/snapshots/2-12-1742442604256.tmp +2025-03-20T03:50:04.257Z [INFO] server-936b6b88-7ed8-f300-c07f-d0af029c17aa: snapshot complete up to: index=12 +2025-03-20T03:50:04.262Z [INFO] server-04c85b6e-ad99-62a9-7e7b-be46be1258bf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-04c85b6e-ad99-62a9-7e7b-be46be1258bf Address:04c85b6e-ad99-62a9-7e7b-be46be1258bf} {Suffrage:Voter ID:server-80d16e54-3c85-3681-2e34-3b985620b4fa Address:80d16e54-3c85-3681-2e34-3b985620b4fa} {Suffrage:Voter ID:server-c4426023-874b-2f23-8d3e-11417740bf2d Address:c4426023-874b-2f23-8d3e-11417740bf2d}]" +2025-03-20T03:50:04.262Z [INFO] server-04c85b6e-ad99-62a9-7e7b-be46be1258bf: entering follower state: follower="Node at 04c85b6e-ad99-62a9-7e7b-be46be1258bf [Follower]" leader-address= leader-id= +2025-03-20T03:50:04.264Z [INFO] server-80d16e54-3c85-3681-2e34-3b985620b4fa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-04c85b6e-ad99-62a9-7e7b-be46be1258bf Address:04c85b6e-ad99-62a9-7e7b-be46be1258bf} {Suffrage:Voter ID:server-80d16e54-3c85-3681-2e34-3b985620b4fa Address:80d16e54-3c85-3681-2e34-3b985620b4fa} {Suffrage:Voter ID:server-c4426023-874b-2f23-8d3e-11417740bf2d Address:c4426023-874b-2f23-8d3e-11417740bf2d}]" +2025-03-20T03:50:04.273Z [INFO] server-c4426023-874b-2f23-8d3e-11417740bf2d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-04c85b6e-ad99-62a9-7e7b-be46be1258bf Address:04c85b6e-ad99-62a9-7e7b-be46be1258bf} {Suffrage:Voter ID:server-80d16e54-3c85-3681-2e34-3b985620b4fa Address:80d16e54-3c85-3681-2e34-3b985620b4fa} {Suffrage:Voter ID:server-c4426023-874b-2f23-8d3e-11417740bf2d Address:c4426023-874b-2f23-8d3e-11417740bf2d}]" +2025-03-20T03:50:04.273Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:50:04.274Z [INFO] server-80d16e54-3c85-3681-2e34-3b985620b4fa: entering follower state: follower="Node at 80d16e54-3c85-3681-2e34-3b985620b4fa [Follower]" leader-address= leader-id= +2025-03-20T03:50:04.274Z [INFO] server-c4426023-874b-2f23-8d3e-11417740bf2d: entering follower state: follower="Node at c4426023-874b-2f23-8d3e-11417740bf2d [Follower]" leader-address= leader-id= +2025-03-20T03:50:04.872Z [WARN] server-c4426023-874b-2f23-8d3e-11417740bf2d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:50:04.872Z [INFO] server-c4426023-874b-2f23-8d3e-11417740bf2d: entering candidate state: node="Node at c4426023-874b-2f23-8d3e-11417740bf2d [Candidate]" term=2 +2025-03-20T03:50:04.872Z [INFO] server-c4426023-874b-2f23-8d3e-11417740bf2d: election won: term=2 tally=2 +2025-03-20T03:50:04.872Z [INFO] server-c4426023-874b-2f23-8d3e-11417740bf2d: entering leader state: leader="Node at c4426023-874b-2f23-8d3e-11417740bf2d [Leader]" +2025-03-20T03:50:04.873Z [INFO] server-c4426023-874b-2f23-8d3e-11417740bf2d: added peer, starting replication: peer=server-04c85b6e-ad99-62a9-7e7b-be46be1258bf +2025-03-20T03:50:04.873Z [INFO] server-c4426023-874b-2f23-8d3e-11417740bf2d: added peer, starting replication: peer=server-80d16e54-3c85-3681-2e34-3b985620b4fa +2025-03-20T03:50:04.877Z [INFO] server-c4426023-874b-2f23-8d3e-11417740bf2d: pipelining replication: peer="{Voter server-80d16e54-3c85-3681-2e34-3b985620b4fa 80d16e54-3c85-3681-2e34-3b985620b4fa}" +2025-03-20T03:50:04.878Z [INFO] server-c4426023-874b-2f23-8d3e-11417740bf2d: pipelining replication: peer="{Voter server-04c85b6e-ad99-62a9-7e7b-be46be1258bf 04c85b6e-ad99-62a9-7e7b-be46be1258bf}" +2025-03-20T03:50:05.886Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:50:04.8786779 +0000 UTC m=+36.553130962 (1 nodes), 604.643591ms from start of poll, 616.59784ms from cluster start. Timeout at 2025-03-20 03:50:05.883699359 +0000 UTC m=+37.558152176, 1.005021214s after stability +2025-03-20T03:50:05.887Z [INFO] creating new snapshot: path=/tmp/raft1771674806/snapshots/2-14-1742442605887.tmp +2025-03-20T03:50:05.888Z [INFO] server-c4426023-874b-2f23-8d3e-11417740bf2d: copied to local snapshot: bytes=71 +2025-03-20T03:50:05.889Z [INFO] server-c4426023-874b-2f23-8d3e-11417740bf2d: snapshot restore progress: id=2-14-1742442605887 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:05.889Z [INFO] server-c4426023-874b-2f23-8d3e-11417740bf2d: restored user snapshot: index=14 +2025-03-20T03:50:05.889Z [ERROR] server-c4426023-874b-2f23-8d3e-11417740bf2d: failed to get log: index=3 error="log not found" +2025-03-20T03:50:05.889Z [INFO] server-c4426023-874b-2f23-8d3e-11417740bf2d: aborting pipeline replication: peer="{Voter server-80d16e54-3c85-3681-2e34-3b985620b4fa 80d16e54-3c85-3681-2e34-3b985620b4fa}" +2025-03-20T03:50:05.889Z [ERROR] server-c4426023-874b-2f23-8d3e-11417740bf2d: failed to get log: index=3 error="log not found" +2025-03-20T03:50:05.889Z [INFO] server-c4426023-874b-2f23-8d3e-11417740bf2d: aborting pipeline replication: peer="{Voter server-04c85b6e-ad99-62a9-7e7b-be46be1258bf 04c85b6e-ad99-62a9-7e7b-be46be1258bf}" +2025-03-20T03:50:05.898Z [ERROR] server-c4426023-874b-2f23-8d3e-11417740bf2d: failed to get log: index=3 error="log not found" +2025-03-20T03:50:05.898Z [ERROR] server-c4426023-874b-2f23-8d3e-11417740bf2d: failed to get log: index=3 error="log not found" +2025-03-20T03:50:05.899Z [INFO] creating new snapshot: path=/tmp/raft1959457566/snapshots/2-14-1742442605899.tmp +2025-03-20T03:50:05.899Z [INFO] creating new snapshot: path=/tmp/raft2567399480/snapshots/2-14-1742442605899.tmp +2025-03-20T03:50:05.899Z [INFO] server-80d16e54-3c85-3681-2e34-3b985620b4fa: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:05.899Z [INFO] server-04c85b6e-ad99-62a9-7e7b-be46be1258bf: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:05.900Z [INFO] server-04c85b6e-ad99-62a9-7e7b-be46be1258bf: copied to local snapshot: bytes=71 +2025-03-20T03:50:05.900Z [INFO] server-04c85b6e-ad99-62a9-7e7b-be46be1258bf: snapshot restore progress: id=2-14-1742442605899 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:05.900Z [INFO] server-80d16e54-3c85-3681-2e34-3b985620b4fa: copied to local snapshot: bytes=71 +2025-03-20T03:50:05.900Z [INFO] server-04c85b6e-ad99-62a9-7e7b-be46be1258bf: Installed remote snapshot +2025-03-20T03:50:05.901Z [INFO] server-c4426023-874b-2f23-8d3e-11417740bf2d: pipelining replication: peer="{Voter server-04c85b6e-ad99-62a9-7e7b-be46be1258bf 04c85b6e-ad99-62a9-7e7b-be46be1258bf}" +2025-03-20T03:50:05.901Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:50:05.901Z [INFO] server-80d16e54-3c85-3681-2e34-3b985620b4fa: snapshot restore progress: id=2-14-1742442605899 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:05.901Z [INFO] server-80d16e54-3c85-3681-2e34-3b985620b4fa: Installed remote snapshot +2025-03-20T03:50:05.901Z [INFO] server-c4426023-874b-2f23-8d3e-11417740bf2d: pipelining replication: peer="{Voter server-80d16e54-3c85-3681-2e34-3b985620b4fa 80d16e54-3c85-3681-2e34-3b985620b4fa}" +2025-03-20T03:50:06.906Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:50:05.901128526 +0000 UTC m=+37.575581373 (2 nodes), 2.411µs from start of poll, 1.639048251s from cluster start. Timeout at 2025-03-20 03:50:06.906167102 +0000 UTC m=+38.580619890, 1.005038517s after stability +2025-03-20T03:50:06.926Z [INFO] server-936b6b88-7ed8-f300-c07f-d0af029c17aa: aborting pipeline replication: peer="{Voter server-f42053e6-517d-8495-870b-827b0e68bb11 f42053e6-517d-8495-870b-827b0e68bb11}" +2025-03-20T03:50:06.926Z [INFO] server-936b6b88-7ed8-f300-c07f-d0af029c17aa: aborting pipeline replication: peer="{Voter server-05373282-805a-6852-bf2d-a5ea1d98da2b 05373282-805a-6852-bf2d-a5ea1d98da2b}" === RUN TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_true -2026-04-22T10:09:52.101Z [INFO] server-8b27b89f-d5e4-c93b-e9b1-f353da681c24: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8b27b89f-d5e4-c93b-e9b1-f353da681c24 Address:8b27b89f-d5e4-c93b-e9b1-f353da681c24} {Suffrage:Voter ID:server-8aff63e6-9a39-93bf-214d-da04770c33b2 Address:8aff63e6-9a39-93bf-214d-da04770c33b2} {Suffrage:Voter ID:server-a52c44bb-6097-a987-ee9a-5ac877098a02 Address:a52c44bb-6097-a987-ee9a-5ac877098a02}]" -2026-04-22T10:09:52.101Z [INFO] server-8b27b89f-d5e4-c93b-e9b1-f353da681c24: entering follower state: follower="Node at 8b27b89f-d5e4-c93b-e9b1-f353da681c24 [Follower]" leader-address= leader-id= -2026-04-22T10:09:52.101Z [INFO] server-8aff63e6-9a39-93bf-214d-da04770c33b2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8b27b89f-d5e4-c93b-e9b1-f353da681c24 Address:8b27b89f-d5e4-c93b-e9b1-f353da681c24} {Suffrage:Voter ID:server-8aff63e6-9a39-93bf-214d-da04770c33b2 Address:8aff63e6-9a39-93bf-214d-da04770c33b2} {Suffrage:Voter ID:server-a52c44bb-6097-a987-ee9a-5ac877098a02 Address:a52c44bb-6097-a987-ee9a-5ac877098a02}]" -2026-04-22T10:09:52.101Z [INFO] server-8aff63e6-9a39-93bf-214d-da04770c33b2: entering follower state: follower="Node at 8aff63e6-9a39-93bf-214d-da04770c33b2 [Follower]" leader-address= leader-id= -2026-04-22T10:09:52.101Z [INFO] server-a52c44bb-6097-a987-ee9a-5ac877098a02: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8b27b89f-d5e4-c93b-e9b1-f353da681c24 Address:8b27b89f-d5e4-c93b-e9b1-f353da681c24} {Suffrage:Voter ID:server-8aff63e6-9a39-93bf-214d-da04770c33b2 Address:8aff63e6-9a39-93bf-214d-da04770c33b2} {Suffrage:Voter ID:server-a52c44bb-6097-a987-ee9a-5ac877098a02 Address:a52c44bb-6097-a987-ee9a-5ac877098a02}]" -2026-04-22T10:09:52.101Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:52.101Z [INFO] server-a52c44bb-6097-a987-ee9a-5ac877098a02: entering follower state: follower="Node at a52c44bb-6097-a987-ee9a-5ac877098a02 [Follower]" leader-address= leader-id= -2026-04-22T10:09:52.698Z [WARN] server-8b27b89f-d5e4-c93b-e9b1-f353da681c24: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:52.698Z [INFO] server-8b27b89f-d5e4-c93b-e9b1-f353da681c24: entering candidate state: node="Node at 8b27b89f-d5e4-c93b-e9b1-f353da681c24 [Candidate]" term=2 -2026-04-22T10:09:52.699Z [INFO] server-8b27b89f-d5e4-c93b-e9b1-f353da681c24: election won: term=2 tally=2 -2026-04-22T10:09:52.699Z [INFO] server-8b27b89f-d5e4-c93b-e9b1-f353da681c24: entering leader state: leader="Node at 8b27b89f-d5e4-c93b-e9b1-f353da681c24 [Leader]" -2026-04-22T10:09:52.699Z [INFO] server-8b27b89f-d5e4-c93b-e9b1-f353da681c24: added peer, starting replication: peer=server-8aff63e6-9a39-93bf-214d-da04770c33b2 -2026-04-22T10:09:52.699Z [INFO] server-8b27b89f-d5e4-c93b-e9b1-f353da681c24: added peer, starting replication: peer=server-a52c44bb-6097-a987-ee9a-5ac877098a02 -2026-04-22T10:09:52.699Z [INFO] server-8b27b89f-d5e4-c93b-e9b1-f353da681c24: pipelining replication: peer="{Voter server-8aff63e6-9a39-93bf-214d-da04770c33b2 8aff63e6-9a39-93bf-214d-da04770c33b2}" -2026-04-22T10:09:52.699Z [INFO] server-8b27b89f-d5e4-c93b-e9b1-f353da681c24: pipelining replication: peer="{Voter server-a52c44bb-6097-a987-ee9a-5ac877098a02 a52c44bb-6097-a987-ee9a-5ac877098a02}" -2026-04-22T10:09:53.704Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:52.699046123 +0000 UTC m=+38.722842213 (1 nodes), 597.605527ms from start of poll, 597.945458ms from cluster start. Timeout at 2026-04-22 10:09:53.704054544 +0000 UTC m=+39.727850684, 1.005008471s after stability -2026-04-22T10:09:53.705Z [INFO] server-8b27b89f-d5e4-c93b-e9b1-f353da681c24: starting snapshot up to: index=12 -2026-04-22T10:09:53.705Z [INFO] creating new snapshot: path=/tmp/raft3463197649/snapshots/2-12-1776852593705.tmp -2026-04-22T10:09:53.706Z [INFO] server-8b27b89f-d5e4-c93b-e9b1-f353da681c24: snapshot complete up to: index=12 -2026-04-22T10:09:53.707Z [INFO] server-9f7439a0-2690-561a-4295-20cccd20213c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9f7439a0-2690-561a-4295-20cccd20213c Address:9f7439a0-2690-561a-4295-20cccd20213c} {Suffrage:Voter ID:server-1c6aef7e-7775-6a76-f793-c3d236133f54 Address:1c6aef7e-7775-6a76-f793-c3d236133f54} {Suffrage:Voter ID:server-8a4e42f2-0df2-c039-d5fd-6f32ecbc8053 Address:8a4e42f2-0df2-c039-d5fd-6f32ecbc8053}]" -2026-04-22T10:09:53.707Z [INFO] server-9f7439a0-2690-561a-4295-20cccd20213c: entering follower state: follower="Node at 9f7439a0-2690-561a-4295-20cccd20213c [Follower]" leader-address= leader-id= -2026-04-22T10:09:53.707Z [INFO] server-1c6aef7e-7775-6a76-f793-c3d236133f54: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9f7439a0-2690-561a-4295-20cccd20213c Address:9f7439a0-2690-561a-4295-20cccd20213c} {Suffrage:Voter ID:server-1c6aef7e-7775-6a76-f793-c3d236133f54 Address:1c6aef7e-7775-6a76-f793-c3d236133f54} {Suffrage:Voter ID:server-8a4e42f2-0df2-c039-d5fd-6f32ecbc8053 Address:8a4e42f2-0df2-c039-d5fd-6f32ecbc8053}]" -2026-04-22T10:09:53.707Z [INFO] server-1c6aef7e-7775-6a76-f793-c3d236133f54: entering follower state: follower="Node at 1c6aef7e-7775-6a76-f793-c3d236133f54 [Follower]" leader-address= leader-id= -2026-04-22T10:09:53.707Z [INFO] server-8a4e42f2-0df2-c039-d5fd-6f32ecbc8053: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9f7439a0-2690-561a-4295-20cccd20213c Address:9f7439a0-2690-561a-4295-20cccd20213c} {Suffrage:Voter ID:server-1c6aef7e-7775-6a76-f793-c3d236133f54 Address:1c6aef7e-7775-6a76-f793-c3d236133f54} {Suffrage:Voter ID:server-8a4e42f2-0df2-c039-d5fd-6f32ecbc8053 Address:8a4e42f2-0df2-c039-d5fd-6f32ecbc8053}]" -2026-04-22T10:09:53.707Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:53.707Z [INFO] server-8a4e42f2-0df2-c039-d5fd-6f32ecbc8053: entering follower state: follower="Node at 8a4e42f2-0df2-c039-d5fd-6f32ecbc8053 [Follower]" leader-address= leader-id= -2026-04-22T10:09:54.284Z [WARN] server-1c6aef7e-7775-6a76-f793-c3d236133f54: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:54.284Z [INFO] server-1c6aef7e-7775-6a76-f793-c3d236133f54: entering candidate state: node="Node at 1c6aef7e-7775-6a76-f793-c3d236133f54 [Candidate]" term=2 -2026-04-22T10:09:54.284Z [INFO] server-1c6aef7e-7775-6a76-f793-c3d236133f54: election won: term=2 tally=2 -2026-04-22T10:09:54.284Z [INFO] server-1c6aef7e-7775-6a76-f793-c3d236133f54: entering leader state: leader="Node at 1c6aef7e-7775-6a76-f793-c3d236133f54 [Leader]" -2026-04-22T10:09:54.284Z [INFO] server-1c6aef7e-7775-6a76-f793-c3d236133f54: added peer, starting replication: peer=server-9f7439a0-2690-561a-4295-20cccd20213c -2026-04-22T10:09:54.284Z [INFO] server-1c6aef7e-7775-6a76-f793-c3d236133f54: added peer, starting replication: peer=server-8a4e42f2-0df2-c039-d5fd-6f32ecbc8053 -2026-04-22T10:09:54.284Z [INFO] server-1c6aef7e-7775-6a76-f793-c3d236133f54: pipelining replication: peer="{Voter server-9f7439a0-2690-561a-4295-20cccd20213c 9f7439a0-2690-561a-4295-20cccd20213c}" -2026-04-22T10:09:54.284Z [INFO] server-1c6aef7e-7775-6a76-f793-c3d236133f54: pipelining replication: peer="{Voter server-8a4e42f2-0df2-c039-d5fd-6f32ecbc8053 8a4e42f2-0df2-c039-d5fd-6f32ecbc8053}" -2026-04-22T10:09:55.290Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:54.28471058 +0000 UTC m=+40.308506734 (1 nodes), 577.292744ms from start of poll, 577.603244ms from cluster start. Timeout at 2026-04-22 10:09:55.289712879 +0000 UTC m=+41.313509019, 1.005002285s after stability -2026-04-22T10:09:55.290Z [INFO] creating new snapshot: path=/tmp/raft3638447232/snapshots/2-14-1776852595290.tmp -2026-04-22T10:09:55.291Z [INFO] server-1c6aef7e-7775-6a76-f793-c3d236133f54: copied to local snapshot: bytes=71 -2026-04-22T10:09:55.291Z [INFO] server-1c6aef7e-7775-6a76-f793-c3d236133f54: snapshot restore progress: id=2-14-1776852595290 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:55.291Z [INFO] server-1c6aef7e-7775-6a76-f793-c3d236133f54: removing all old logs from log store -2026-04-22T10:09:55.291Z [INFO] server-1c6aef7e-7775-6a76-f793-c3d236133f54: compacting logs: from=1 to=2 -2026-04-22T10:09:55.291Z [INFO] server-1c6aef7e-7775-6a76-f793-c3d236133f54: restored user snapshot: index=14 -2026-04-22T10:09:55.291Z [ERROR] server-1c6aef7e-7775-6a76-f793-c3d236133f54: failed to get log: index=2 error="log not found" -2026-04-22T10:09:55.291Z [INFO] server-1c6aef7e-7775-6a76-f793-c3d236133f54: aborting pipeline replication: peer="{Voter server-9f7439a0-2690-561a-4295-20cccd20213c 9f7439a0-2690-561a-4295-20cccd20213c}" -2026-04-22T10:09:55.291Z [ERROR] server-1c6aef7e-7775-6a76-f793-c3d236133f54: failed to get log: index=2 error="log not found" -2026-04-22T10:09:55.291Z [INFO] server-1c6aef7e-7775-6a76-f793-c3d236133f54: aborting pipeline replication: peer="{Voter server-8a4e42f2-0df2-c039-d5fd-6f32ecbc8053 8a4e42f2-0df2-c039-d5fd-6f32ecbc8053}" -2026-04-22T10:09:55.299Z [ERROR] server-1c6aef7e-7775-6a76-f793-c3d236133f54: failed to get log: index=2 error="log not found" -2026-04-22T10:09:55.299Z [ERROR] server-1c6aef7e-7775-6a76-f793-c3d236133f54: failed to get log: index=2 error="log not found" -2026-04-22T10:09:55.299Z [INFO] creating new snapshot: path=/tmp/raft2409031489/snapshots/2-14-1776852595299.tmp -2026-04-22T10:09:55.299Z [INFO] creating new snapshot: path=/tmp/raft2495343654/snapshots/2-14-1776852595299.tmp -2026-04-22T10:09:55.299Z [INFO] server-9f7439a0-2690-561a-4295-20cccd20213c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:55.299Z [INFO] server-8a4e42f2-0df2-c039-d5fd-6f32ecbc8053: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:55.299Z [INFO] server-9f7439a0-2690-561a-4295-20cccd20213c: copied to local snapshot: bytes=71 -2026-04-22T10:09:55.299Z [INFO] server-8a4e42f2-0df2-c039-d5fd-6f32ecbc8053: copied to local snapshot: bytes=71 -2026-04-22T10:09:55.300Z [INFO] server-9f7439a0-2690-561a-4295-20cccd20213c: snapshot restore progress: id=2-14-1776852595299 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:55.300Z [INFO] server-8a4e42f2-0df2-c039-d5fd-6f32ecbc8053: snapshot restore progress: id=2-14-1776852595299 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:55.300Z [INFO] server-9f7439a0-2690-561a-4295-20cccd20213c: removing all old logs from log store -2026-04-22T10:09:55.300Z [INFO] server-9f7439a0-2690-561a-4295-20cccd20213c: compacting logs: from=1 to=2 -2026-04-22T10:09:55.300Z [INFO] server-9f7439a0-2690-561a-4295-20cccd20213c: Installed remote snapshot -2026-04-22T10:09:55.300Z [INFO] server-8a4e42f2-0df2-c039-d5fd-6f32ecbc8053: removing all old logs from log store -2026-04-22T10:09:55.300Z [INFO] server-8a4e42f2-0df2-c039-d5fd-6f32ecbc8053: compacting logs: from=1 to=2 -2026-04-22T10:09:55.300Z [INFO] server-8a4e42f2-0df2-c039-d5fd-6f32ecbc8053: Installed remote snapshot -2026-04-22T10:09:55.300Z [INFO] server-1c6aef7e-7775-6a76-f793-c3d236133f54: pipelining replication: peer="{Voter server-9f7439a0-2690-561a-4295-20cccd20213c 9f7439a0-2690-561a-4295-20cccd20213c}" -2026-04-22T10:09:55.300Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:09:55.300Z [INFO] server-1c6aef7e-7775-6a76-f793-c3d236133f54: pipelining replication: peer="{Voter server-8a4e42f2-0df2-c039-d5fd-6f32ecbc8053 8a4e42f2-0df2-c039-d5fd-6f32ecbc8053}" -2026-04-22T10:09:56.306Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:09:55.300350688 +0000 UTC m=+41.324146775 (2 nodes), 581ns from start of poll, 1.593243285s from cluster start. Timeout at 2026-04-22 10:09:56.305381477 +0000 UTC m=+42.329177595, 1.00503082s after stability -2026-04-22T10:09:56.317Z [INFO] server-8b27b89f-d5e4-c93b-e9b1-f353da681c24: aborting pipeline replication: peer="{Voter server-a52c44bb-6097-a987-ee9a-5ac877098a02 a52c44bb-6097-a987-ee9a-5ac877098a02}" -2026-04-22T10:09:56.317Z [INFO] server-8b27b89f-d5e4-c93b-e9b1-f353da681c24: aborting pipeline replication: peer="{Voter server-8aff63e6-9a39-93bf-214d-da04770c33b2 8aff63e6-9a39-93bf-214d-da04770c33b2}" +2025-03-20T03:50:06.928Z [INFO] server-f3f6985e-f049-82c1-1b4a-a97050d5c978: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f3f6985e-f049-82c1-1b4a-a97050d5c978 Address:f3f6985e-f049-82c1-1b4a-a97050d5c978} {Suffrage:Voter ID:server-bdbe2e23-1d33-3a7f-4391-eeed143295b6 Address:bdbe2e23-1d33-3a7f-4391-eeed143295b6} {Suffrage:Voter ID:server-fe36b0e8-329f-a15a-3cf0-c94caab3cef2 Address:fe36b0e8-329f-a15a-3cf0-c94caab3cef2}]" +2025-03-20T03:50:06.928Z [INFO] server-bdbe2e23-1d33-3a7f-4391-eeed143295b6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f3f6985e-f049-82c1-1b4a-a97050d5c978 Address:f3f6985e-f049-82c1-1b4a-a97050d5c978} {Suffrage:Voter ID:server-bdbe2e23-1d33-3a7f-4391-eeed143295b6 Address:bdbe2e23-1d33-3a7f-4391-eeed143295b6} {Suffrage:Voter ID:server-fe36b0e8-329f-a15a-3cf0-c94caab3cef2 Address:fe36b0e8-329f-a15a-3cf0-c94caab3cef2}]" +2025-03-20T03:50:06.929Z [INFO] server-fe36b0e8-329f-a15a-3cf0-c94caab3cef2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f3f6985e-f049-82c1-1b4a-a97050d5c978 Address:f3f6985e-f049-82c1-1b4a-a97050d5c978} {Suffrage:Voter ID:server-bdbe2e23-1d33-3a7f-4391-eeed143295b6 Address:bdbe2e23-1d33-3a7f-4391-eeed143295b6} {Suffrage:Voter ID:server-fe36b0e8-329f-a15a-3cf0-c94caab3cef2 Address:fe36b0e8-329f-a15a-3cf0-c94caab3cef2}]" +2025-03-20T03:50:06.929Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:50:06.929Z [INFO] server-f3f6985e-f049-82c1-1b4a-a97050d5c978: entering follower state: follower="Node at f3f6985e-f049-82c1-1b4a-a97050d5c978 [Follower]" leader-address= leader-id= +2025-03-20T03:50:06.929Z [INFO] server-bdbe2e23-1d33-3a7f-4391-eeed143295b6: entering follower state: follower="Node at bdbe2e23-1d33-3a7f-4391-eeed143295b6 [Follower]" leader-address= leader-id= +2025-03-20T03:50:06.934Z [INFO] server-fe36b0e8-329f-a15a-3cf0-c94caab3cef2: entering follower state: follower="Node at fe36b0e8-329f-a15a-3cf0-c94caab3cef2 [Follower]" leader-address= leader-id= +2025-03-20T03:50:07.510Z [WARN] server-fe36b0e8-329f-a15a-3cf0-c94caab3cef2: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:50:07.510Z [INFO] server-fe36b0e8-329f-a15a-3cf0-c94caab3cef2: entering candidate state: node="Node at fe36b0e8-329f-a15a-3cf0-c94caab3cef2 [Candidate]" term=2 +2025-03-20T03:50:07.511Z [INFO] server-fe36b0e8-329f-a15a-3cf0-c94caab3cef2: election won: term=2 tally=2 +2025-03-20T03:50:07.511Z [INFO] server-fe36b0e8-329f-a15a-3cf0-c94caab3cef2: entering leader state: leader="Node at fe36b0e8-329f-a15a-3cf0-c94caab3cef2 [Leader]" +2025-03-20T03:50:07.511Z [INFO] server-fe36b0e8-329f-a15a-3cf0-c94caab3cef2: added peer, starting replication: peer=server-f3f6985e-f049-82c1-1b4a-a97050d5c978 +2025-03-20T03:50:07.511Z [INFO] server-fe36b0e8-329f-a15a-3cf0-c94caab3cef2: added peer, starting replication: peer=server-bdbe2e23-1d33-3a7f-4391-eeed143295b6 +2025-03-20T03:50:07.511Z [INFO] server-fe36b0e8-329f-a15a-3cf0-c94caab3cef2: pipelining replication: peer="{Voter server-bdbe2e23-1d33-3a7f-4391-eeed143295b6 bdbe2e23-1d33-3a7f-4391-eeed143295b6}" +2025-03-20T03:50:07.511Z [WARN] server-bdbe2e23-1d33-3a7f-4391-eeed143295b6: rejecting vote request since our last term is greater: candidate=fe36b0e8-329f-a15a-3cf0-c94caab3cef2 last-term=2 last-candidate-term=1 +2025-03-20T03:50:07.512Z [INFO] server-fe36b0e8-329f-a15a-3cf0-c94caab3cef2: pipelining replication: peer="{Voter server-f3f6985e-f049-82c1-1b4a-a97050d5c978 f3f6985e-f049-82c1-1b4a-a97050d5c978}" +2025-03-20T03:50:08.521Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:50:07.511994529 +0000 UTC m=+39.186447371 (1 nodes), 582.843099ms from start of poll, 583.322045ms from cluster start. Timeout at 2025-03-20 03:50:08.517007293 +0000 UTC m=+40.191460355, 1.005012984s after stability +2025-03-20T03:50:08.523Z [INFO] server-fe36b0e8-329f-a15a-3cf0-c94caab3cef2: starting snapshot up to: index=12 +2025-03-20T03:50:08.523Z [INFO] creating new snapshot: path=/tmp/raft2353097434/snapshots/2-12-1742442608523.tmp +2025-03-20T03:50:08.524Z [INFO] server-fe36b0e8-329f-a15a-3cf0-c94caab3cef2: snapshot complete up to: index=12 +2025-03-20T03:50:08.528Z [INFO] server-0447f0b7-ede7-436f-c177-c547fb82d8ec: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0447f0b7-ede7-436f-c177-c547fb82d8ec Address:0447f0b7-ede7-436f-c177-c547fb82d8ec} {Suffrage:Voter ID:server-3ce2100e-ff0b-72f8-eee7-317c57f72604 Address:3ce2100e-ff0b-72f8-eee7-317c57f72604} {Suffrage:Voter ID:server-4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2 Address:4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2}]" +2025-03-20T03:50:08.528Z [INFO] server-3ce2100e-ff0b-72f8-eee7-317c57f72604: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0447f0b7-ede7-436f-c177-c547fb82d8ec Address:0447f0b7-ede7-436f-c177-c547fb82d8ec} {Suffrage:Voter ID:server-3ce2100e-ff0b-72f8-eee7-317c57f72604 Address:3ce2100e-ff0b-72f8-eee7-317c57f72604} {Suffrage:Voter ID:server-4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2 Address:4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2}]" +2025-03-20T03:50:08.528Z [INFO] server-0447f0b7-ede7-436f-c177-c547fb82d8ec: entering follower state: follower="Node at 0447f0b7-ede7-436f-c177-c547fb82d8ec [Follower]" leader-address= leader-id= +2025-03-20T03:50:08.529Z [INFO] server-3ce2100e-ff0b-72f8-eee7-317c57f72604: entering follower state: follower="Node at 3ce2100e-ff0b-72f8-eee7-317c57f72604 [Follower]" leader-address= leader-id= +2025-03-20T03:50:08.530Z [INFO] server-4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0447f0b7-ede7-436f-c177-c547fb82d8ec Address:0447f0b7-ede7-436f-c177-c547fb82d8ec} {Suffrage:Voter ID:server-3ce2100e-ff0b-72f8-eee7-317c57f72604 Address:3ce2100e-ff0b-72f8-eee7-317c57f72604} {Suffrage:Voter ID:server-4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2 Address:4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2}]" +2025-03-20T03:50:08.530Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:50:08.531Z [INFO] server-4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2: entering follower state: follower="Node at 4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2 [Follower]" leader-address= leader-id= +2025-03-20T03:50:09.302Z [WARN] server-4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:50:09.302Z [INFO] server-4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2: entering candidate state: node="Node at 4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2 [Candidate]" term=2 +2025-03-20T03:50:09.303Z [INFO] server-4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2: election won: term=2 tally=2 +2025-03-20T03:50:09.303Z [INFO] server-4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2: entering leader state: leader="Node at 4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2 [Leader]" +2025-03-20T03:50:09.303Z [INFO] server-4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2: added peer, starting replication: peer=server-0447f0b7-ede7-436f-c177-c547fb82d8ec +2025-03-20T03:50:09.303Z [INFO] server-4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2: added peer, starting replication: peer=server-3ce2100e-ff0b-72f8-eee7-317c57f72604 +2025-03-20T03:50:09.303Z [INFO] server-4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2: pipelining replication: peer="{Voter server-3ce2100e-ff0b-72f8-eee7-317c57f72604 3ce2100e-ff0b-72f8-eee7-317c57f72604}" +2025-03-20T03:50:09.303Z [INFO] server-4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2: pipelining replication: peer="{Voter server-0447f0b7-ede7-436f-c177-c547fb82d8ec 0447f0b7-ede7-436f-c177-c547fb82d8ec}" +2025-03-20T03:50:10.308Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:50:09.303155036 +0000 UTC m=+40.977608074 (1 nodes), 772.132139ms from start of poll, 775.790671ms from cluster start. Timeout at 2025-03-20 03:50:10.308158045 +0000 UTC m=+41.982610972, 1.005002898s after stability +2025-03-20T03:50:10.309Z [INFO] creating new snapshot: path=/tmp/raft1708097059/snapshots/2-14-1742442610309.tmp +2025-03-20T03:50:10.311Z [INFO] server-4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2: copied to local snapshot: bytes=71 +2025-03-20T03:50:10.311Z [INFO] server-4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2: snapshot restore progress: id=2-14-1742442610309 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:10.311Z [INFO] server-4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2: removing all old logs from log store +2025-03-20T03:50:10.311Z [INFO] server-4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2: compacting logs: from=1 to=2 +2025-03-20T03:50:10.311Z [INFO] server-4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2: restored user snapshot: index=14 +2025-03-20T03:50:10.312Z [ERROR] server-4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2: failed to get log: index=2 error="log not found" +2025-03-20T03:50:10.312Z [INFO] server-4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2: aborting pipeline replication: peer="{Voter server-0447f0b7-ede7-436f-c177-c547fb82d8ec 0447f0b7-ede7-436f-c177-c547fb82d8ec}" +2025-03-20T03:50:10.312Z [ERROR] server-4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2: failed to get log: index=2 error="log not found" +2025-03-20T03:50:10.312Z [INFO] server-4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2: aborting pipeline replication: peer="{Voter server-3ce2100e-ff0b-72f8-eee7-317c57f72604 3ce2100e-ff0b-72f8-eee7-317c57f72604}" +2025-03-20T03:50:10.318Z [ERROR] server-4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2: failed to get log: index=2 error="log not found" +2025-03-20T03:50:10.318Z [INFO] creating new snapshot: path=/tmp/raft1991908487/snapshots/2-14-1742442610318.tmp +2025-03-20T03:50:10.319Z [INFO] server-3ce2100e-ff0b-72f8-eee7-317c57f72604: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:10.320Z [ERROR] server-4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2: failed to get log: index=2 error="log not found" +2025-03-20T03:50:10.321Z [INFO] creating new snapshot: path=/tmp/raft2960702479/snapshots/2-14-1742442610321.tmp +2025-03-20T03:50:10.321Z [INFO] server-0447f0b7-ede7-436f-c177-c547fb82d8ec: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:10.322Z [INFO] server-0447f0b7-ede7-436f-c177-c547fb82d8ec: copied to local snapshot: bytes=71 +2025-03-20T03:50:10.323Z [INFO] server-0447f0b7-ede7-436f-c177-c547fb82d8ec: snapshot restore progress: id=2-14-1742442610321 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:10.323Z [INFO] server-0447f0b7-ede7-436f-c177-c547fb82d8ec: removing all old logs from log store +2025-03-20T03:50:10.323Z [INFO] server-0447f0b7-ede7-436f-c177-c547fb82d8ec: compacting logs: from=1 to=2 +2025-03-20T03:50:10.323Z [INFO] server-0447f0b7-ede7-436f-c177-c547fb82d8ec: Installed remote snapshot +2025-03-20T03:50:10.323Z [INFO] server-4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2: pipelining replication: peer="{Voter server-0447f0b7-ede7-436f-c177-c547fb82d8ec 0447f0b7-ede7-436f-c177-c547fb82d8ec}" +2025-03-20T03:50:10.323Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:50:10.323Z [INFO] server-3ce2100e-ff0b-72f8-eee7-317c57f72604: copied to local snapshot: bytes=71 +2025-03-20T03:50:10.324Z [INFO] server-3ce2100e-ff0b-72f8-eee7-317c57f72604: snapshot restore progress: id=2-14-1742442610318 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:10.324Z [INFO] server-3ce2100e-ff0b-72f8-eee7-317c57f72604: removing all old logs from log store +2025-03-20T03:50:10.324Z [INFO] server-3ce2100e-ff0b-72f8-eee7-317c57f72604: compacting logs: from=1 to=2 +2025-03-20T03:50:10.324Z [INFO] server-3ce2100e-ff0b-72f8-eee7-317c57f72604: Installed remote snapshot +2025-03-20T03:50:10.324Z [INFO] server-4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2: pipelining replication: peer="{Voter server-3ce2100e-ff0b-72f8-eee7-317c57f72604 3ce2100e-ff0b-72f8-eee7-317c57f72604}" +2025-03-20T03:50:11.329Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:50:10.323512305 +0000 UTC m=+41.997965157 (2 nodes), 2.122µs from start of poll, 1.796147754s from cluster start. Timeout at 2025-03-20 03:50:11.328542024 +0000 UTC m=+43.002994884, 1.005029727s after stability +2025-03-20T03:50:11.348Z [INFO] server-fe36b0e8-329f-a15a-3cf0-c94caab3cef2: aborting pipeline replication: peer="{Voter server-bdbe2e23-1d33-3a7f-4391-eeed143295b6 bdbe2e23-1d33-3a7f-4391-eeed143295b6}" +2025-03-20T03:50:11.348Z [INFO] server-fe36b0e8-329f-a15a-3cf0-c94caab3cef2: aborting pipeline replication: peer="{Voter server-f3f6985e-f049-82c1-1b4a-a97050d5c978 f3f6985e-f049-82c1-1b4a-a97050d5c978}" === RUN TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_false -2026-04-22T10:09:56.318Z [INFO] server-b44941bc-d98d-7ed8-846a-0399c47c7e63: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b44941bc-d98d-7ed8-846a-0399c47c7e63 Address:b44941bc-d98d-7ed8-846a-0399c47c7e63} {Suffrage:Voter ID:server-bbc31690-20b1-f984-7020-21c4f3d4154c Address:bbc31690-20b1-f984-7020-21c4f3d4154c} {Suffrage:Voter ID:server-741138a3-a55c-751a-0831-eb7b7c09d073 Address:741138a3-a55c-751a-0831-eb7b7c09d073}]" -2026-04-22T10:09:56.318Z [INFO] server-b44941bc-d98d-7ed8-846a-0399c47c7e63: entering follower state: follower="Node at b44941bc-d98d-7ed8-846a-0399c47c7e63 [Follower]" leader-address= leader-id= -2026-04-22T10:09:56.318Z [INFO] server-bbc31690-20b1-f984-7020-21c4f3d4154c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b44941bc-d98d-7ed8-846a-0399c47c7e63 Address:b44941bc-d98d-7ed8-846a-0399c47c7e63} {Suffrage:Voter ID:server-bbc31690-20b1-f984-7020-21c4f3d4154c Address:bbc31690-20b1-f984-7020-21c4f3d4154c} {Suffrage:Voter ID:server-741138a3-a55c-751a-0831-eb7b7c09d073 Address:741138a3-a55c-751a-0831-eb7b7c09d073}]" -2026-04-22T10:09:56.318Z [INFO] server-bbc31690-20b1-f984-7020-21c4f3d4154c: entering follower state: follower="Node at bbc31690-20b1-f984-7020-21c4f3d4154c [Follower]" leader-address= leader-id= -2026-04-22T10:09:56.318Z [INFO] server-741138a3-a55c-751a-0831-eb7b7c09d073: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b44941bc-d98d-7ed8-846a-0399c47c7e63 Address:b44941bc-d98d-7ed8-846a-0399c47c7e63} {Suffrage:Voter ID:server-bbc31690-20b1-f984-7020-21c4f3d4154c Address:bbc31690-20b1-f984-7020-21c4f3d4154c} {Suffrage:Voter ID:server-741138a3-a55c-751a-0831-eb7b7c09d073 Address:741138a3-a55c-751a-0831-eb7b7c09d073}]" -2026-04-22T10:09:56.318Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:56.318Z [INFO] server-741138a3-a55c-751a-0831-eb7b7c09d073: entering follower state: follower="Node at 741138a3-a55c-751a-0831-eb7b7c09d073 [Follower]" leader-address= leader-id= -2026-04-22T10:09:56.840Z [WARN] server-b44941bc-d98d-7ed8-846a-0399c47c7e63: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:56.840Z [INFO] server-b44941bc-d98d-7ed8-846a-0399c47c7e63: entering candidate state: node="Node at b44941bc-d98d-7ed8-846a-0399c47c7e63 [Candidate]" term=2 -2026-04-22T10:09:56.840Z [INFO] server-b44941bc-d98d-7ed8-846a-0399c47c7e63: election won: term=2 tally=2 -2026-04-22T10:09:56.840Z [INFO] server-b44941bc-d98d-7ed8-846a-0399c47c7e63: entering leader state: leader="Node at b44941bc-d98d-7ed8-846a-0399c47c7e63 [Leader]" -2026-04-22T10:09:56.840Z [INFO] server-b44941bc-d98d-7ed8-846a-0399c47c7e63: added peer, starting replication: peer=server-bbc31690-20b1-f984-7020-21c4f3d4154c -2026-04-22T10:09:56.840Z [INFO] server-b44941bc-d98d-7ed8-846a-0399c47c7e63: added peer, starting replication: peer=server-741138a3-a55c-751a-0831-eb7b7c09d073 -2026-04-22T10:09:56.841Z [INFO] server-b44941bc-d98d-7ed8-846a-0399c47c7e63: pipelining replication: peer="{Voter server-741138a3-a55c-751a-0831-eb7b7c09d073 741138a3-a55c-751a-0831-eb7b7c09d073}" -2026-04-22T10:09:56.841Z [INFO] server-b44941bc-d98d-7ed8-846a-0399c47c7e63: pipelining replication: peer="{Voter server-bbc31690-20b1-f984-7020-21c4f3d4154c bbc31690-20b1-f984-7020-21c4f3d4154c}" -2026-04-22T10:09:57.845Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:56.840932603 +0000 UTC m=+42.864728709 (1 nodes), 522.291191ms from start of poll, 522.608604ms from cluster start. Timeout at 2026-04-22 10:09:57.84593666 +0000 UTC m=+43.869732743, 1.005004034s after stability -2026-04-22T10:09:57.846Z [INFO] server-b44941bc-d98d-7ed8-846a-0399c47c7e63: starting snapshot up to: index=12 -2026-04-22T10:09:57.846Z [INFO] creating new snapshot: path=/tmp/raft2974169161/snapshots/2-12-1776852597846.tmp -2026-04-22T10:09:57.846Z [INFO] server-b44941bc-d98d-7ed8-846a-0399c47c7e63: snapshot complete up to: index=12 -2026-04-22T10:09:57.847Z [INFO] creating new snapshot: path=/tmp/raft2974169161/snapshots/2-23-1776852597847.tmp -2026-04-22T10:09:57.848Z [INFO] server-b44941bc-d98d-7ed8-846a-0399c47c7e63: copied to local snapshot: bytes=71 -2026-04-22T10:09:57.848Z [INFO] server-b44941bc-d98d-7ed8-846a-0399c47c7e63: snapshot restore progress: id=2-23-1776852597847 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:57.848Z [INFO] server-b44941bc-d98d-7ed8-846a-0399c47c7e63: restored user snapshot: index=23 -2026-04-22T10:09:57.848Z [ERROR] server-b44941bc-d98d-7ed8-846a-0399c47c7e63: failed to get log: index=23 error="log not found" -2026-04-22T10:09:57.848Z [ERROR] server-b44941bc-d98d-7ed8-846a-0399c47c7e63: failed to get log: index=23 error="log not found" -2026-04-22T10:09:57.848Z [INFO] server-b44941bc-d98d-7ed8-846a-0399c47c7e63: aborting pipeline replication: peer="{Voter server-741138a3-a55c-751a-0831-eb7b7c09d073 741138a3-a55c-751a-0831-eb7b7c09d073}" -2026-04-22T10:09:57.848Z [INFO] server-b44941bc-d98d-7ed8-846a-0399c47c7e63: aborting pipeline replication: peer="{Voter server-bbc31690-20b1-f984-7020-21c4f3d4154c bbc31690-20b1-f984-7020-21c4f3d4154c}" -2026-04-22T10:09:57.857Z [ERROR] server-b44941bc-d98d-7ed8-846a-0399c47c7e63: failed to get log: index=23 error="log not found" -2026-04-22T10:09:57.858Z [INFO] creating new snapshot: path=/tmp/raft1364874649/snapshots/2-23-1776852597858.tmp -2026-04-22T10:09:57.858Z [INFO] server-bbc31690-20b1-f984-7020-21c4f3d4154c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:57.858Z [ERROR] server-b44941bc-d98d-7ed8-846a-0399c47c7e63: failed to get log: index=23 error="log not found" -2026-04-22T10:09:57.858Z [INFO] server-bbc31690-20b1-f984-7020-21c4f3d4154c: copied to local snapshot: bytes=71 -2026-04-22T10:09:57.858Z [INFO] creating new snapshot: path=/tmp/raft2172584266/snapshots/2-23-1776852597858.tmp -2026-04-22T10:09:57.858Z [INFO] server-bbc31690-20b1-f984-7020-21c4f3d4154c: snapshot restore progress: id=2-23-1776852597858 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:57.858Z [INFO] server-bbc31690-20b1-f984-7020-21c4f3d4154c: Installed remote snapshot -2026-04-22T10:09:57.859Z [WARN] server-bbc31690-20b1-f984-7020-21c4f3d4154c: skipping application of old log: index=22 -2026-04-22T10:09:57.859Z [INFO] server-b44941bc-d98d-7ed8-846a-0399c47c7e63: pipelining replication: peer="{Voter server-bbc31690-20b1-f984-7020-21c4f3d4154c bbc31690-20b1-f984-7020-21c4f3d4154c}" -2026-04-22T10:09:57.859Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:09:57.859Z [INFO] server-741138a3-a55c-751a-0831-eb7b7c09d073: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:57.859Z [INFO] server-741138a3-a55c-751a-0831-eb7b7c09d073: copied to local snapshot: bytes=71 -2026-04-22T10:09:57.859Z [INFO] server-741138a3-a55c-751a-0831-eb7b7c09d073: snapshot restore progress: id=2-23-1776852597858 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:09:57.859Z [INFO] server-741138a3-a55c-751a-0831-eb7b7c09d073: Installed remote snapshot -2026-04-22T10:09:57.859Z [INFO] server-b44941bc-d98d-7ed8-846a-0399c47c7e63: pipelining replication: peer="{Voter server-741138a3-a55c-751a-0831-eb7b7c09d073 741138a3-a55c-751a-0831-eb7b7c09d073}" -2026-04-22T10:09:58.864Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:09:57.85910028 +0000 UTC m=+43.882896368 (2 nodes), 612ns from start of poll, 1.540776263s from cluster start. Timeout at 2026-04-22 10:09:58.864113575 +0000 UTC m=+44.887909745, 1.005013377s after stability -2026-04-22T10:09:58.876Z [INFO] server-b44941bc-d98d-7ed8-846a-0399c47c7e63: aborting pipeline replication: peer="{Voter server-741138a3-a55c-751a-0831-eb7b7c09d073 741138a3-a55c-751a-0831-eb7b7c09d073}" -2026-04-22T10:09:58.876Z [INFO] server-b44941bc-d98d-7ed8-846a-0399c47c7e63: aborting pipeline replication: peer="{Voter server-bbc31690-20b1-f984-7020-21c4f3d4154c bbc31690-20b1-f984-7020-21c4f3d4154c}" +2025-03-20T03:50:11.350Z [INFO] server-7f5cab26-f4a8-8f19-08c7-266c7374a653: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7f5cab26-f4a8-8f19-08c7-266c7374a653 Address:7f5cab26-f4a8-8f19-08c7-266c7374a653} {Suffrage:Voter ID:server-acc4a70e-5f25-19b9-e20c-0dcbdc2afe00 Address:acc4a70e-5f25-19b9-e20c-0dcbdc2afe00} {Suffrage:Voter ID:server-407e7d85-dcb1-93df-9247-38fc8b4c59f1 Address:407e7d85-dcb1-93df-9247-38fc8b4c59f1}]" +2025-03-20T03:50:11.350Z [INFO] server-acc4a70e-5f25-19b9-e20c-0dcbdc2afe00: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7f5cab26-f4a8-8f19-08c7-266c7374a653 Address:7f5cab26-f4a8-8f19-08c7-266c7374a653} {Suffrage:Voter ID:server-acc4a70e-5f25-19b9-e20c-0dcbdc2afe00 Address:acc4a70e-5f25-19b9-e20c-0dcbdc2afe00} {Suffrage:Voter ID:server-407e7d85-dcb1-93df-9247-38fc8b4c59f1 Address:407e7d85-dcb1-93df-9247-38fc8b4c59f1}]" +2025-03-20T03:50:11.351Z [INFO] server-7f5cab26-f4a8-8f19-08c7-266c7374a653: entering follower state: follower="Node at 7f5cab26-f4a8-8f19-08c7-266c7374a653 [Follower]" leader-address= leader-id= +2025-03-20T03:50:11.351Z [INFO] server-407e7d85-dcb1-93df-9247-38fc8b4c59f1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7f5cab26-f4a8-8f19-08c7-266c7374a653 Address:7f5cab26-f4a8-8f19-08c7-266c7374a653} {Suffrage:Voter ID:server-acc4a70e-5f25-19b9-e20c-0dcbdc2afe00 Address:acc4a70e-5f25-19b9-e20c-0dcbdc2afe00} {Suffrage:Voter ID:server-407e7d85-dcb1-93df-9247-38fc8b4c59f1 Address:407e7d85-dcb1-93df-9247-38fc8b4c59f1}]" +2025-03-20T03:50:11.351Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:50:11.351Z [INFO] server-407e7d85-dcb1-93df-9247-38fc8b4c59f1: entering follower state: follower="Node at 407e7d85-dcb1-93df-9247-38fc8b4c59f1 [Follower]" leader-address= leader-id= +2025-03-20T03:50:11.351Z [INFO] server-acc4a70e-5f25-19b9-e20c-0dcbdc2afe00: entering follower state: follower="Node at acc4a70e-5f25-19b9-e20c-0dcbdc2afe00 [Follower]" leader-address= leader-id= +2025-03-20T03:50:11.938Z [WARN] server-407e7d85-dcb1-93df-9247-38fc8b4c59f1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:50:11.938Z [INFO] server-407e7d85-dcb1-93df-9247-38fc8b4c59f1: entering candidate state: node="Node at 407e7d85-dcb1-93df-9247-38fc8b4c59f1 [Candidate]" term=2 +2025-03-20T03:50:11.938Z [INFO] server-407e7d85-dcb1-93df-9247-38fc8b4c59f1: election won: term=2 tally=2 +2025-03-20T03:50:11.938Z [INFO] server-407e7d85-dcb1-93df-9247-38fc8b4c59f1: entering leader state: leader="Node at 407e7d85-dcb1-93df-9247-38fc8b4c59f1 [Leader]" +2025-03-20T03:50:11.938Z [INFO] server-407e7d85-dcb1-93df-9247-38fc8b4c59f1: added peer, starting replication: peer=server-7f5cab26-f4a8-8f19-08c7-266c7374a653 +2025-03-20T03:50:11.938Z [INFO] server-407e7d85-dcb1-93df-9247-38fc8b4c59f1: added peer, starting replication: peer=server-acc4a70e-5f25-19b9-e20c-0dcbdc2afe00 +2025-03-20T03:50:11.938Z [INFO] server-407e7d85-dcb1-93df-9247-38fc8b4c59f1: pipelining replication: peer="{Voter server-7f5cab26-f4a8-8f19-08c7-266c7374a653 7f5cab26-f4a8-8f19-08c7-266c7374a653}" +2025-03-20T03:50:11.939Z [INFO] server-407e7d85-dcb1-93df-9247-38fc8b4c59f1: pipelining replication: peer="{Voter server-acc4a70e-5f25-19b9-e20c-0dcbdc2afe00 acc4a70e-5f25-19b9-e20c-0dcbdc2afe00}" +2025-03-20T03:50:12.944Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:50:11.938791744 +0000 UTC m=+43.613244527 (1 nodes), 587.575319ms from start of poll, 588.207165ms from cluster start. Timeout at 2025-03-20 03:50:12.943795359 +0000 UTC m=+44.618248336, 1.005003809s after stability +2025-03-20T03:50:12.946Z [INFO] server-407e7d85-dcb1-93df-9247-38fc8b4c59f1: starting snapshot up to: index=12 +2025-03-20T03:50:12.946Z [INFO] creating new snapshot: path=/tmp/raft3865912647/snapshots/2-12-1742442612946.tmp +2025-03-20T03:50:12.947Z [INFO] server-407e7d85-dcb1-93df-9247-38fc8b4c59f1: snapshot complete up to: index=12 +2025-03-20T03:50:12.949Z [INFO] creating new snapshot: path=/tmp/raft3865912647/snapshots/2-23-1742442612949.tmp +2025-03-20T03:50:12.949Z [INFO] server-407e7d85-dcb1-93df-9247-38fc8b4c59f1: copied to local snapshot: bytes=71 +2025-03-20T03:50:12.950Z [INFO] server-407e7d85-dcb1-93df-9247-38fc8b4c59f1: snapshot restore progress: id=2-23-1742442612949 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:12.950Z [INFO] server-407e7d85-dcb1-93df-9247-38fc8b4c59f1: restored user snapshot: index=23 +2025-03-20T03:50:12.950Z [ERROR] server-407e7d85-dcb1-93df-9247-38fc8b4c59f1: failed to get log: index=23 error="log not found" +2025-03-20T03:50:12.950Z [INFO] server-407e7d85-dcb1-93df-9247-38fc8b4c59f1: aborting pipeline replication: peer="{Voter server-acc4a70e-5f25-19b9-e20c-0dcbdc2afe00 acc4a70e-5f25-19b9-e20c-0dcbdc2afe00}" +2025-03-20T03:50:12.950Z [ERROR] server-407e7d85-dcb1-93df-9247-38fc8b4c59f1: failed to get log: index=23 error="log not found" +2025-03-20T03:50:12.950Z [INFO] server-407e7d85-dcb1-93df-9247-38fc8b4c59f1: aborting pipeline replication: peer="{Voter server-7f5cab26-f4a8-8f19-08c7-266c7374a653 7f5cab26-f4a8-8f19-08c7-266c7374a653}" +2025-03-20T03:50:12.962Z [ERROR] server-407e7d85-dcb1-93df-9247-38fc8b4c59f1: failed to get log: index=23 error="log not found" +2025-03-20T03:50:12.963Z [INFO] creating new snapshot: path=/tmp/raft3851950645/snapshots/2-23-1742442612963.tmp +2025-03-20T03:50:12.963Z [INFO] server-7f5cab26-f4a8-8f19-08c7-266c7374a653: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:12.964Z [INFO] server-7f5cab26-f4a8-8f19-08c7-266c7374a653: copied to local snapshot: bytes=71 +2025-03-20T03:50:12.964Z [INFO] server-7f5cab26-f4a8-8f19-08c7-266c7374a653: snapshot restore progress: id=2-23-1742442612963 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:12.964Z [INFO] server-7f5cab26-f4a8-8f19-08c7-266c7374a653: Installed remote snapshot +2025-03-20T03:50:12.964Z [WARN] server-7f5cab26-f4a8-8f19-08c7-266c7374a653: skipping application of old log: index=22 +2025-03-20T03:50:12.965Z [INFO] server-407e7d85-dcb1-93df-9247-38fc8b4c59f1: pipelining replication: peer="{Voter server-7f5cab26-f4a8-8f19-08c7-266c7374a653 7f5cab26-f4a8-8f19-08c7-266c7374a653}" +2025-03-20T03:50:12.965Z [ERROR] server-407e7d85-dcb1-93df-9247-38fc8b4c59f1: failed to get log: index=23 error="log not found" +2025-03-20T03:50:12.966Z [INFO] creating new snapshot: path=/tmp/raft678652905/snapshots/2-23-1742442612966.tmp +2025-03-20T03:50:12.966Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:50:12.967Z [INFO] server-acc4a70e-5f25-19b9-e20c-0dcbdc2afe00: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:12.970Z [INFO] server-acc4a70e-5f25-19b9-e20c-0dcbdc2afe00: copied to local snapshot: bytes=71 +2025-03-20T03:50:12.995Z [INFO] server-acc4a70e-5f25-19b9-e20c-0dcbdc2afe00: snapshot restore progress: id=2-23-1742442612966 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:12.995Z [INFO] server-acc4a70e-5f25-19b9-e20c-0dcbdc2afe00: Installed remote snapshot +2025-03-20T03:50:12.995Z [INFO] server-407e7d85-dcb1-93df-9247-38fc8b4c59f1: pipelining replication: peer="{Voter server-acc4a70e-5f25-19b9-e20c-0dcbdc2afe00 acc4a70e-5f25-19b9-e20c-0dcbdc2afe00}" +2025-03-20T03:50:13.972Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:50:12.966331935 +0000 UTC m=+44.640784793 (2 nodes), 2.038µs from start of poll, 1.615747431s from cluster start. Timeout at 2025-03-20 03:50:13.971352266 +0000 UTC m=+45.645805251, 1.005020458s after stability +2025-03-20T03:50:13.993Z [INFO] server-407e7d85-dcb1-93df-9247-38fc8b4c59f1: aborting pipeline replication: peer="{Voter server-acc4a70e-5f25-19b9-e20c-0dcbdc2afe00 acc4a70e-5f25-19b9-e20c-0dcbdc2afe00}" +2025-03-20T03:50:13.993Z [INFO] server-407e7d85-dcb1-93df-9247-38fc8b4c59f1: aborting pipeline replication: peer="{Voter server-7f5cab26-f4a8-8f19-08c7-266c7374a653 7f5cab26-f4a8-8f19-08c7-266c7374a653}" === RUN TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_false -2026-04-22T10:09:58.878Z [INFO] server-25913ccd-91ec-3249-33ec-48237c4fccc6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-25913ccd-91ec-3249-33ec-48237c4fccc6 Address:25913ccd-91ec-3249-33ec-48237c4fccc6} {Suffrage:Voter ID:server-e2c87d58-2999-e469-ea52-a1bda0318cd3 Address:e2c87d58-2999-e469-ea52-a1bda0318cd3} {Suffrage:Voter ID:server-cea24096-1ea1-0e49-3a31-ee1f16086f5c Address:cea24096-1ea1-0e49-3a31-ee1f16086f5c}]" -2026-04-22T10:09:58.878Z [INFO] server-25913ccd-91ec-3249-33ec-48237c4fccc6: entering follower state: follower="Node at 25913ccd-91ec-3249-33ec-48237c4fccc6 [Follower]" leader-address= leader-id= -2026-04-22T10:09:58.878Z [INFO] server-e2c87d58-2999-e469-ea52-a1bda0318cd3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-25913ccd-91ec-3249-33ec-48237c4fccc6 Address:25913ccd-91ec-3249-33ec-48237c4fccc6} {Suffrage:Voter ID:server-e2c87d58-2999-e469-ea52-a1bda0318cd3 Address:e2c87d58-2999-e469-ea52-a1bda0318cd3} {Suffrage:Voter ID:server-cea24096-1ea1-0e49-3a31-ee1f16086f5c Address:cea24096-1ea1-0e49-3a31-ee1f16086f5c}]" -2026-04-22T10:09:58.878Z [INFO] server-e2c87d58-2999-e469-ea52-a1bda0318cd3: entering follower state: follower="Node at e2c87d58-2999-e469-ea52-a1bda0318cd3 [Follower]" leader-address= leader-id= -2026-04-22T10:09:58.878Z [INFO] server-cea24096-1ea1-0e49-3a31-ee1f16086f5c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-25913ccd-91ec-3249-33ec-48237c4fccc6 Address:25913ccd-91ec-3249-33ec-48237c4fccc6} {Suffrage:Voter ID:server-e2c87d58-2999-e469-ea52-a1bda0318cd3 Address:e2c87d58-2999-e469-ea52-a1bda0318cd3} {Suffrage:Voter ID:server-cea24096-1ea1-0e49-3a31-ee1f16086f5c Address:cea24096-1ea1-0e49-3a31-ee1f16086f5c}]" -2026-04-22T10:09:58.878Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:09:58.878Z [INFO] server-cea24096-1ea1-0e49-3a31-ee1f16086f5c: entering follower state: follower="Node at cea24096-1ea1-0e49-3a31-ee1f16086f5c [Follower]" leader-address= leader-id= -2026-04-22T10:09:59.446Z [WARN] server-e2c87d58-2999-e469-ea52-a1bda0318cd3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:09:59.446Z [INFO] server-e2c87d58-2999-e469-ea52-a1bda0318cd3: entering candidate state: node="Node at e2c87d58-2999-e469-ea52-a1bda0318cd3 [Candidate]" term=2 -2026-04-22T10:09:59.446Z [INFO] server-e2c87d58-2999-e469-ea52-a1bda0318cd3: election won: term=2 tally=2 -2026-04-22T10:09:59.446Z [INFO] server-e2c87d58-2999-e469-ea52-a1bda0318cd3: entering leader state: leader="Node at e2c87d58-2999-e469-ea52-a1bda0318cd3 [Leader]" -2026-04-22T10:09:59.446Z [INFO] server-e2c87d58-2999-e469-ea52-a1bda0318cd3: added peer, starting replication: peer=server-25913ccd-91ec-3249-33ec-48237c4fccc6 -2026-04-22T10:09:59.447Z [INFO] server-e2c87d58-2999-e469-ea52-a1bda0318cd3: added peer, starting replication: peer=server-cea24096-1ea1-0e49-3a31-ee1f16086f5c -2026-04-22T10:09:59.447Z [INFO] server-e2c87d58-2999-e469-ea52-a1bda0318cd3: pipelining replication: peer="{Voter server-25913ccd-91ec-3249-33ec-48237c4fccc6 25913ccd-91ec-3249-33ec-48237c4fccc6}" -2026-04-22T10:09:59.447Z [INFO] server-e2c87d58-2999-e469-ea52-a1bda0318cd3: pipelining replication: peer="{Voter server-cea24096-1ea1-0e49-3a31-ee1f16086f5c cea24096-1ea1-0e49-3a31-ee1f16086f5c}" -2026-04-22T10:10:00.453Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:09:59.447037973 +0000 UTC m=+45.470834063 (1 nodes), 568.147711ms from start of poll, 569.081318ms from cluster start. Timeout at 2026-04-22 10:10:00.452046979 +0000 UTC m=+46.475843146, 1.005009083s after stability -2026-04-22T10:10:00.454Z [INFO] server-e2c87d58-2999-e469-ea52-a1bda0318cd3: starting snapshot up to: index=12 -2026-04-22T10:10:00.454Z [INFO] creating new snapshot: path=/tmp/raft1439203812/snapshots/2-12-1776852600454.tmp -2026-04-22T10:10:00.455Z [INFO] server-e2c87d58-2999-e469-ea52-a1bda0318cd3: snapshot complete up to: index=12 -2026-04-22T10:10:00.456Z [INFO] creating new snapshot: path=/tmp/raft1439203812/snapshots/2-23-1776852600456.tmp -2026-04-22T10:10:00.456Z [INFO] server-e2c87d58-2999-e469-ea52-a1bda0318cd3: copied to local snapshot: bytes=71 -2026-04-22T10:10:00.456Z [INFO] server-e2c87d58-2999-e469-ea52-a1bda0318cd3: snapshot restore progress: id=2-23-1776852600456 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:00.456Z [INFO] server-e2c87d58-2999-e469-ea52-a1bda0318cd3: removing all old logs from log store -2026-04-22T10:10:00.456Z [INFO] server-e2c87d58-2999-e469-ea52-a1bda0318cd3: compacting logs: from=1 to=22 -2026-04-22T10:10:00.456Z [INFO] server-e2c87d58-2999-e469-ea52-a1bda0318cd3: restored user snapshot: index=23 -2026-04-22T10:10:00.456Z [ERROR] server-e2c87d58-2999-e469-ea52-a1bda0318cd3: failed to get log: index=22 error="log not found" -2026-04-22T10:10:00.456Z [INFO] server-e2c87d58-2999-e469-ea52-a1bda0318cd3: aborting pipeline replication: peer="{Voter server-cea24096-1ea1-0e49-3a31-ee1f16086f5c cea24096-1ea1-0e49-3a31-ee1f16086f5c}" -2026-04-22T10:10:00.456Z [ERROR] server-e2c87d58-2999-e469-ea52-a1bda0318cd3: failed to get log: index=22 error="log not found" -2026-04-22T10:10:00.456Z [INFO] server-e2c87d58-2999-e469-ea52-a1bda0318cd3: aborting pipeline replication: peer="{Voter server-25913ccd-91ec-3249-33ec-48237c4fccc6 25913ccd-91ec-3249-33ec-48237c4fccc6}" -2026-04-22T10:10:00.463Z [ERROR] server-e2c87d58-2999-e469-ea52-a1bda0318cd3: failed to get log: index=22 error="log not found" -2026-04-22T10:10:00.464Z [INFO] creating new snapshot: path=/tmp/raft807479831/snapshots/2-23-1776852600464.tmp -2026-04-22T10:10:00.464Z [INFO] server-25913ccd-91ec-3249-33ec-48237c4fccc6: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:00.464Z [INFO] server-25913ccd-91ec-3249-33ec-48237c4fccc6: copied to local snapshot: bytes=71 -2026-04-22T10:10:00.465Z [INFO] server-25913ccd-91ec-3249-33ec-48237c4fccc6: snapshot restore progress: id=2-23-1776852600464 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:00.465Z [INFO] server-25913ccd-91ec-3249-33ec-48237c4fccc6: removing all old logs from log store -2026-04-22T10:10:00.465Z [INFO] server-25913ccd-91ec-3249-33ec-48237c4fccc6: compacting logs: from=1 to=22 -2026-04-22T10:10:00.465Z [INFO] server-25913ccd-91ec-3249-33ec-48237c4fccc6: Installed remote snapshot -2026-04-22T10:10:00.465Z [WARN] server-25913ccd-91ec-3249-33ec-48237c4fccc6: skipping application of old log: index=22 -2026-04-22T10:10:00.465Z [INFO] server-e2c87d58-2999-e469-ea52-a1bda0318cd3: pipelining replication: peer="{Voter server-25913ccd-91ec-3249-33ec-48237c4fccc6 25913ccd-91ec-3249-33ec-48237c4fccc6}" -2026-04-22T10:10:00.465Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:10:00.467Z [ERROR] server-e2c87d58-2999-e469-ea52-a1bda0318cd3: failed to get log: index=22 error="log not found" -2026-04-22T10:10:00.467Z [INFO] creating new snapshot: path=/tmp/raft2693525025/snapshots/2-23-1776852600467.tmp -2026-04-22T10:10:00.468Z [INFO] server-cea24096-1ea1-0e49-3a31-ee1f16086f5c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:00.468Z [INFO] server-cea24096-1ea1-0e49-3a31-ee1f16086f5c: copied to local snapshot: bytes=71 -2026-04-22T10:10:00.468Z [INFO] server-cea24096-1ea1-0e49-3a31-ee1f16086f5c: snapshot restore progress: id=2-23-1776852600467 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:00.468Z [INFO] server-cea24096-1ea1-0e49-3a31-ee1f16086f5c: removing all old logs from log store -2026-04-22T10:10:00.468Z [INFO] server-cea24096-1ea1-0e49-3a31-ee1f16086f5c: compacting logs: from=1 to=22 -2026-04-22T10:10:00.468Z [INFO] server-cea24096-1ea1-0e49-3a31-ee1f16086f5c: Installed remote snapshot -2026-04-22T10:10:00.468Z [INFO] server-e2c87d58-2999-e469-ea52-a1bda0318cd3: pipelining replication: peer="{Voter server-cea24096-1ea1-0e49-3a31-ee1f16086f5c cea24096-1ea1-0e49-3a31-ee1f16086f5c}" -2026-04-22T10:10:01.470Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:10:00.465330465 +0000 UTC m=+46.489126575 (2 nodes), 835ns from start of poll, 1.58737383s from cluster start. Timeout at 2026-04-22 10:10:01.47033791 +0000 UTC m=+47.494133993, 1.005007418s after stability -2026-04-22T10:10:01.481Z [INFO] server-e2c87d58-2999-e469-ea52-a1bda0318cd3: aborting pipeline replication: peer="{Voter server-cea24096-1ea1-0e49-3a31-ee1f16086f5c cea24096-1ea1-0e49-3a31-ee1f16086f5c}" -2026-04-22T10:10:01.481Z [INFO] server-e2c87d58-2999-e469-ea52-a1bda0318cd3: aborting pipeline replication: peer="{Voter server-25913ccd-91ec-3249-33ec-48237c4fccc6 25913ccd-91ec-3249-33ec-48237c4fccc6}" +2025-03-20T03:50:13.996Z [INFO] server-71266f4e-e98e-bb7d-0874-468647392b1b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-71266f4e-e98e-bb7d-0874-468647392b1b Address:71266f4e-e98e-bb7d-0874-468647392b1b} {Suffrage:Voter ID:server-1726b47b-0216-6d8f-d80e-62c936726a46 Address:1726b47b-0216-6d8f-d80e-62c936726a46} {Suffrage:Voter ID:server-9499e767-d28e-5ff5-a184-d80c04ef3bb6 Address:9499e767-d28e-5ff5-a184-d80c04ef3bb6}]" +2025-03-20T03:50:13.997Z [INFO] server-1726b47b-0216-6d8f-d80e-62c936726a46: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-71266f4e-e98e-bb7d-0874-468647392b1b Address:71266f4e-e98e-bb7d-0874-468647392b1b} {Suffrage:Voter ID:server-1726b47b-0216-6d8f-d80e-62c936726a46 Address:1726b47b-0216-6d8f-d80e-62c936726a46} {Suffrage:Voter ID:server-9499e767-d28e-5ff5-a184-d80c04ef3bb6 Address:9499e767-d28e-5ff5-a184-d80c04ef3bb6}]" +2025-03-20T03:50:13.997Z [INFO] server-9499e767-d28e-5ff5-a184-d80c04ef3bb6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-71266f4e-e98e-bb7d-0874-468647392b1b Address:71266f4e-e98e-bb7d-0874-468647392b1b} {Suffrage:Voter ID:server-1726b47b-0216-6d8f-d80e-62c936726a46 Address:1726b47b-0216-6d8f-d80e-62c936726a46} {Suffrage:Voter ID:server-9499e767-d28e-5ff5-a184-d80c04ef3bb6 Address:9499e767-d28e-5ff5-a184-d80c04ef3bb6}]" +2025-03-20T03:50:13.997Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:50:13.997Z [INFO] server-71266f4e-e98e-bb7d-0874-468647392b1b: entering follower state: follower="Node at 71266f4e-e98e-bb7d-0874-468647392b1b [Follower]" leader-address= leader-id= +2025-03-20T03:50:13.997Z [INFO] server-1726b47b-0216-6d8f-d80e-62c936726a46: entering follower state: follower="Node at 1726b47b-0216-6d8f-d80e-62c936726a46 [Follower]" leader-address= leader-id= +2025-03-20T03:50:13.997Z [INFO] server-9499e767-d28e-5ff5-a184-d80c04ef3bb6: entering follower state: follower="Node at 9499e767-d28e-5ff5-a184-d80c04ef3bb6 [Follower]" leader-address= leader-id= +2025-03-20T03:50:14.613Z [WARN] server-71266f4e-e98e-bb7d-0874-468647392b1b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:50:14.613Z [INFO] server-71266f4e-e98e-bb7d-0874-468647392b1b: entering candidate state: node="Node at 71266f4e-e98e-bb7d-0874-468647392b1b [Candidate]" term=2 +2025-03-20T03:50:14.614Z [INFO] server-71266f4e-e98e-bb7d-0874-468647392b1b: election won: term=2 tally=2 +2025-03-20T03:50:14.614Z [INFO] server-71266f4e-e98e-bb7d-0874-468647392b1b: entering leader state: leader="Node at 71266f4e-e98e-bb7d-0874-468647392b1b [Leader]" +2025-03-20T03:50:14.614Z [INFO] server-71266f4e-e98e-bb7d-0874-468647392b1b: added peer, starting replication: peer=server-1726b47b-0216-6d8f-d80e-62c936726a46 +2025-03-20T03:50:14.614Z [INFO] server-71266f4e-e98e-bb7d-0874-468647392b1b: added peer, starting replication: peer=server-9499e767-d28e-5ff5-a184-d80c04ef3bb6 +2025-03-20T03:50:14.614Z [INFO] server-71266f4e-e98e-bb7d-0874-468647392b1b: pipelining replication: peer="{Voter server-9499e767-d28e-5ff5-a184-d80c04ef3bb6 9499e767-d28e-5ff5-a184-d80c04ef3bb6}" +2025-03-20T03:50:14.614Z [INFO] server-71266f4e-e98e-bb7d-0874-468647392b1b: pipelining replication: peer="{Voter server-1726b47b-0216-6d8f-d80e-62c936726a46 1726b47b-0216-6d8f-d80e-62c936726a46}" +2025-03-20T03:50:15.620Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:50:14.614751733 +0000 UTC m=+46.289204609 (1 nodes), 617.508978ms from start of poll, 617.887387ms from cluster start. Timeout at 2025-03-20 03:50:15.619754648 +0000 UTC m=+47.294207517, 1.005002908s after stability +2025-03-20T03:50:15.622Z [INFO] server-71266f4e-e98e-bb7d-0874-468647392b1b: starting snapshot up to: index=12 +2025-03-20T03:50:15.622Z [INFO] creating new snapshot: path=/tmp/raft253209385/snapshots/2-12-1742442615622.tmp +2025-03-20T03:50:15.623Z [INFO] server-71266f4e-e98e-bb7d-0874-468647392b1b: snapshot complete up to: index=12 +2025-03-20T03:50:15.627Z [INFO] creating new snapshot: path=/tmp/raft253209385/snapshots/2-23-1742442615627.tmp +2025-03-20T03:50:15.628Z [INFO] server-71266f4e-e98e-bb7d-0874-468647392b1b: copied to local snapshot: bytes=71 +2025-03-20T03:50:15.628Z [INFO] server-71266f4e-e98e-bb7d-0874-468647392b1b: snapshot restore progress: id=2-23-1742442615627 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:15.628Z [INFO] server-71266f4e-e98e-bb7d-0874-468647392b1b: removing all old logs from log store +2025-03-20T03:50:15.628Z [INFO] server-71266f4e-e98e-bb7d-0874-468647392b1b: compacting logs: from=1 to=22 +2025-03-20T03:50:15.628Z [INFO] server-71266f4e-e98e-bb7d-0874-468647392b1b: restored user snapshot: index=23 +2025-03-20T03:50:15.628Z [ERROR] server-71266f4e-e98e-bb7d-0874-468647392b1b: failed to get log: index=22 error="log not found" +2025-03-20T03:50:15.628Z [INFO] server-71266f4e-e98e-bb7d-0874-468647392b1b: aborting pipeline replication: peer="{Voter server-9499e767-d28e-5ff5-a184-d80c04ef3bb6 9499e767-d28e-5ff5-a184-d80c04ef3bb6}" +2025-03-20T03:50:15.628Z [ERROR] server-71266f4e-e98e-bb7d-0874-468647392b1b: failed to get log: index=22 error="log not found" +2025-03-20T03:50:15.628Z [INFO] server-71266f4e-e98e-bb7d-0874-468647392b1b: aborting pipeline replication: peer="{Voter server-1726b47b-0216-6d8f-d80e-62c936726a46 1726b47b-0216-6d8f-d80e-62c936726a46}" +2025-03-20T03:50:15.640Z [ERROR] server-71266f4e-e98e-bb7d-0874-468647392b1b: failed to get log: index=22 error="log not found" +2025-03-20T03:50:15.641Z [INFO] creating new snapshot: path=/tmp/raft2571999857/snapshots/2-23-1742442615641.tmp +2025-03-20T03:50:15.641Z [INFO] server-9499e767-d28e-5ff5-a184-d80c04ef3bb6: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:15.642Z [ERROR] server-71266f4e-e98e-bb7d-0874-468647392b1b: failed to get log: index=22 error="log not found" +2025-03-20T03:50:15.643Z [INFO] creating new snapshot: path=/tmp/raft2965037319/snapshots/2-23-1742442615643.tmp +2025-03-20T03:50:15.643Z [INFO] server-1726b47b-0216-6d8f-d80e-62c936726a46: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:15.644Z [INFO] server-1726b47b-0216-6d8f-d80e-62c936726a46: copied to local snapshot: bytes=71 +2025-03-20T03:50:15.644Z [INFO] server-9499e767-d28e-5ff5-a184-d80c04ef3bb6: copied to local snapshot: bytes=71 +2025-03-20T03:50:15.644Z [INFO] server-9499e767-d28e-5ff5-a184-d80c04ef3bb6: snapshot restore progress: id=2-23-1742442615641 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:15.645Z [INFO] server-9499e767-d28e-5ff5-a184-d80c04ef3bb6: removing all old logs from log store +2025-03-20T03:50:15.645Z [INFO] server-9499e767-d28e-5ff5-a184-d80c04ef3bb6: compacting logs: from=1 to=22 +2025-03-20T03:50:15.645Z [INFO] server-9499e767-d28e-5ff5-a184-d80c04ef3bb6: Installed remote snapshot +2025-03-20T03:50:15.645Z [WARN] server-9499e767-d28e-5ff5-a184-d80c04ef3bb6: skipping application of old log: index=22 +2025-03-20T03:50:15.645Z [INFO] server-71266f4e-e98e-bb7d-0874-468647392b1b: pipelining replication: peer="{Voter server-9499e767-d28e-5ff5-a184-d80c04ef3bb6 9499e767-d28e-5ff5-a184-d80c04ef3bb6}" +2025-03-20T03:50:15.645Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:50:15.645Z [INFO] server-1726b47b-0216-6d8f-d80e-62c936726a46: snapshot restore progress: id=2-23-1742442615643 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:15.645Z [INFO] server-1726b47b-0216-6d8f-d80e-62c936726a46: removing all old logs from log store +2025-03-20T03:50:15.645Z [INFO] server-1726b47b-0216-6d8f-d80e-62c936726a46: compacting logs: from=1 to=22 +2025-03-20T03:50:15.645Z [INFO] server-1726b47b-0216-6d8f-d80e-62c936726a46: Installed remote snapshot +2025-03-20T03:50:15.646Z [INFO] server-71266f4e-e98e-bb7d-0874-468647392b1b: pipelining replication: peer="{Voter server-1726b47b-0216-6d8f-d80e-62c936726a46 1726b47b-0216-6d8f-d80e-62c936726a46}" +2025-03-20T03:50:16.650Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:50:15.645477926 +0000 UTC m=+47.319931038 (2 nodes), 2.695µs from start of poll, 1.648613816s from cluster start. Timeout at 2025-03-20 03:50:16.650547182 +0000 UTC m=+48.325000000, 1.005068962s after stability +2025-03-20T03:50:16.664Z [INFO] server-71266f4e-e98e-bb7d-0874-468647392b1b: aborting pipeline replication: peer="{Voter server-9499e767-d28e-5ff5-a184-d80c04ef3bb6 9499e767-d28e-5ff5-a184-d80c04ef3bb6}" +2025-03-20T03:50:16.664Z [INFO] server-71266f4e-e98e-bb7d-0874-468647392b1b: aborting pipeline replication: peer="{Voter server-1726b47b-0216-6d8f-d80e-62c936726a46 1726b47b-0216-6d8f-d80e-62c936726a46}" === RUN TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_true -2026-04-22T10:10:01.487Z [INFO] server-c0f2d1a6-c5a1-dafe-63dc-6c4ae05d2700: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c0f2d1a6-c5a1-dafe-63dc-6c4ae05d2700 Address:c0f2d1a6-c5a1-dafe-63dc-6c4ae05d2700} {Suffrage:Voter ID:server-1b7504ce-4f7e-2638-6547-ddd88374b9f1 Address:1b7504ce-4f7e-2638-6547-ddd88374b9f1} {Suffrage:Voter ID:server-1e36660f-0e8c-db77-7038-b66ec1360c48 Address:1e36660f-0e8c-db77-7038-b66ec1360c48}]" -2026-04-22T10:10:01.487Z [INFO] server-c0f2d1a6-c5a1-dafe-63dc-6c4ae05d2700: entering follower state: follower="Node at c0f2d1a6-c5a1-dafe-63dc-6c4ae05d2700 [Follower]" leader-address= leader-id= -2026-04-22T10:10:01.487Z [INFO] server-1b7504ce-4f7e-2638-6547-ddd88374b9f1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c0f2d1a6-c5a1-dafe-63dc-6c4ae05d2700 Address:c0f2d1a6-c5a1-dafe-63dc-6c4ae05d2700} {Suffrage:Voter ID:server-1b7504ce-4f7e-2638-6547-ddd88374b9f1 Address:1b7504ce-4f7e-2638-6547-ddd88374b9f1} {Suffrage:Voter ID:server-1e36660f-0e8c-db77-7038-b66ec1360c48 Address:1e36660f-0e8c-db77-7038-b66ec1360c48}]" -2026-04-22T10:10:01.487Z [INFO] server-1b7504ce-4f7e-2638-6547-ddd88374b9f1: entering follower state: follower="Node at 1b7504ce-4f7e-2638-6547-ddd88374b9f1 [Follower]" leader-address= leader-id= -2026-04-22T10:10:01.487Z [INFO] server-1e36660f-0e8c-db77-7038-b66ec1360c48: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c0f2d1a6-c5a1-dafe-63dc-6c4ae05d2700 Address:c0f2d1a6-c5a1-dafe-63dc-6c4ae05d2700} {Suffrage:Voter ID:server-1b7504ce-4f7e-2638-6547-ddd88374b9f1 Address:1b7504ce-4f7e-2638-6547-ddd88374b9f1} {Suffrage:Voter ID:server-1e36660f-0e8c-db77-7038-b66ec1360c48 Address:1e36660f-0e8c-db77-7038-b66ec1360c48}]" -2026-04-22T10:10:01.487Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:01.487Z [INFO] server-1e36660f-0e8c-db77-7038-b66ec1360c48: entering follower state: follower="Node at 1e36660f-0e8c-db77-7038-b66ec1360c48 [Follower]" leader-address= leader-id= -2026-04-22T10:10:02.256Z [WARN] server-c0f2d1a6-c5a1-dafe-63dc-6c4ae05d2700: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:02.256Z [INFO] server-c0f2d1a6-c5a1-dafe-63dc-6c4ae05d2700: entering candidate state: node="Node at c0f2d1a6-c5a1-dafe-63dc-6c4ae05d2700 [Candidate]" term=2 -2026-04-22T10:10:02.256Z [INFO] server-c0f2d1a6-c5a1-dafe-63dc-6c4ae05d2700: election won: term=2 tally=2 -2026-04-22T10:10:02.256Z [INFO] server-c0f2d1a6-c5a1-dafe-63dc-6c4ae05d2700: entering leader state: leader="Node at c0f2d1a6-c5a1-dafe-63dc-6c4ae05d2700 [Leader]" -2026-04-22T10:10:02.256Z [INFO] server-c0f2d1a6-c5a1-dafe-63dc-6c4ae05d2700: added peer, starting replication: peer=server-1b7504ce-4f7e-2638-6547-ddd88374b9f1 -2026-04-22T10:10:02.256Z [INFO] server-c0f2d1a6-c5a1-dafe-63dc-6c4ae05d2700: added peer, starting replication: peer=server-1e36660f-0e8c-db77-7038-b66ec1360c48 -2026-04-22T10:10:02.256Z [INFO] server-c0f2d1a6-c5a1-dafe-63dc-6c4ae05d2700: pipelining replication: peer="{Voter server-1b7504ce-4f7e-2638-6547-ddd88374b9f1 1b7504ce-4f7e-2638-6547-ddd88374b9f1}" -2026-04-22T10:10:02.257Z [INFO] server-c0f2d1a6-c5a1-dafe-63dc-6c4ae05d2700: pipelining replication: peer="{Voter server-1e36660f-0e8c-db77-7038-b66ec1360c48 1e36660f-0e8c-db77-7038-b66ec1360c48}" -2026-04-22T10:10:03.262Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:02.256898726 +0000 UTC m=+48.280694902 (1 nodes), 769.095536ms from start of poll, 769.396362ms from cluster start. Timeout at 2026-04-22 10:10:03.261900574 +0000 UTC m=+49.285696713, 1.005001811s after stability -2026-04-22T10:10:03.262Z [INFO] server-c0f2d1a6-c5a1-dafe-63dc-6c4ae05d2700: starting snapshot up to: index=12 -2026-04-22T10:10:03.262Z [INFO] creating new snapshot: path=/tmp/raft1662794114/snapshots/2-12-1776852603262.tmp -2026-04-22T10:10:03.263Z [INFO] server-c0f2d1a6-c5a1-dafe-63dc-6c4ae05d2700: snapshot complete up to: index=12 -2026-04-22T10:10:03.264Z [INFO] server-ba90a3a8-f980-75cd-6ce0-79e747342937: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ba90a3a8-f980-75cd-6ce0-79e747342937 Address:ba90a3a8-f980-75cd-6ce0-79e747342937} {Suffrage:Voter ID:server-d305b1ab-13b7-318b-7a98-ec07f25053a5 Address:d305b1ab-13b7-318b-7a98-ec07f25053a5} {Suffrage:Voter ID:server-c21f2676-5a5a-d8e8-b600-a6a367b12daa Address:c21f2676-5a5a-d8e8-b600-a6a367b12daa}]" -2026-04-22T10:10:03.264Z [INFO] server-ba90a3a8-f980-75cd-6ce0-79e747342937: entering follower state: follower="Node at ba90a3a8-f980-75cd-6ce0-79e747342937 [Follower]" leader-address= leader-id= -2026-04-22T10:10:03.264Z [INFO] server-d305b1ab-13b7-318b-7a98-ec07f25053a5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ba90a3a8-f980-75cd-6ce0-79e747342937 Address:ba90a3a8-f980-75cd-6ce0-79e747342937} {Suffrage:Voter ID:server-d305b1ab-13b7-318b-7a98-ec07f25053a5 Address:d305b1ab-13b7-318b-7a98-ec07f25053a5} {Suffrage:Voter ID:server-c21f2676-5a5a-d8e8-b600-a6a367b12daa Address:c21f2676-5a5a-d8e8-b600-a6a367b12daa}]" -2026-04-22T10:10:03.264Z [INFO] server-d305b1ab-13b7-318b-7a98-ec07f25053a5: entering follower state: follower="Node at d305b1ab-13b7-318b-7a98-ec07f25053a5 [Follower]" leader-address= leader-id= -2026-04-22T10:10:03.264Z [INFO] server-c21f2676-5a5a-d8e8-b600-a6a367b12daa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ba90a3a8-f980-75cd-6ce0-79e747342937 Address:ba90a3a8-f980-75cd-6ce0-79e747342937} {Suffrage:Voter ID:server-d305b1ab-13b7-318b-7a98-ec07f25053a5 Address:d305b1ab-13b7-318b-7a98-ec07f25053a5} {Suffrage:Voter ID:server-c21f2676-5a5a-d8e8-b600-a6a367b12daa Address:c21f2676-5a5a-d8e8-b600-a6a367b12daa}]" -2026-04-22T10:10:03.264Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:03.264Z [INFO] server-c21f2676-5a5a-d8e8-b600-a6a367b12daa: entering follower state: follower="Node at c21f2676-5a5a-d8e8-b600-a6a367b12daa [Follower]" leader-address= leader-id= -2026-04-22T10:10:03.943Z [WARN] server-d305b1ab-13b7-318b-7a98-ec07f25053a5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:03.943Z [INFO] server-d305b1ab-13b7-318b-7a98-ec07f25053a5: entering candidate state: node="Node at d305b1ab-13b7-318b-7a98-ec07f25053a5 [Candidate]" term=2 -2026-04-22T10:10:03.943Z [INFO] server-d305b1ab-13b7-318b-7a98-ec07f25053a5: election won: term=2 tally=2 -2026-04-22T10:10:03.943Z [INFO] server-d305b1ab-13b7-318b-7a98-ec07f25053a5: entering leader state: leader="Node at d305b1ab-13b7-318b-7a98-ec07f25053a5 [Leader]" -2026-04-22T10:10:03.943Z [INFO] server-d305b1ab-13b7-318b-7a98-ec07f25053a5: added peer, starting replication: peer=server-ba90a3a8-f980-75cd-6ce0-79e747342937 -2026-04-22T10:10:03.943Z [INFO] server-d305b1ab-13b7-318b-7a98-ec07f25053a5: added peer, starting replication: peer=server-c21f2676-5a5a-d8e8-b600-a6a367b12daa -2026-04-22T10:10:03.944Z [INFO] server-d305b1ab-13b7-318b-7a98-ec07f25053a5: pipelining replication: peer="{Voter server-ba90a3a8-f980-75cd-6ce0-79e747342937 ba90a3a8-f980-75cd-6ce0-79e747342937}" -2026-04-22T10:10:03.944Z [INFO] server-d305b1ab-13b7-318b-7a98-ec07f25053a5: pipelining replication: peer="{Voter server-c21f2676-5a5a-d8e8-b600-a6a367b12daa c21f2676-5a5a-d8e8-b600-a6a367b12daa}" -2026-04-22T10:10:04.949Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:03.943934524 +0000 UTC m=+49.967730613 (1 nodes), 679.160736ms from start of poll, 679.451875ms from cluster start. Timeout at 2026-04-22 10:10:04.948937667 +0000 UTC m=+50.972733749, 1.005003136s after stability -2026-04-22T10:10:04.949Z [INFO] creating new snapshot: path=/tmp/raft3173064347/snapshots/2-15-1776852604949.tmp -2026-04-22T10:10:04.949Z [INFO] server-d305b1ab-13b7-318b-7a98-ec07f25053a5: copied to local snapshot: bytes=71 -2026-04-22T10:10:04.949Z [INFO] server-d305b1ab-13b7-318b-7a98-ec07f25053a5: snapshot restore progress: id=2-15-1776852604949 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:04.950Z [INFO] server-d305b1ab-13b7-318b-7a98-ec07f25053a5: restored user snapshot: index=15 -2026-04-22T10:10:04.950Z [ERROR] server-d305b1ab-13b7-318b-7a98-ec07f25053a5: failed to get log: index=3 error="log not found" -2026-04-22T10:10:04.950Z [INFO] server-d305b1ab-13b7-318b-7a98-ec07f25053a5: aborting pipeline replication: peer="{Voter server-ba90a3a8-f980-75cd-6ce0-79e747342937 ba90a3a8-f980-75cd-6ce0-79e747342937}" -2026-04-22T10:10:04.950Z [ERROR] server-d305b1ab-13b7-318b-7a98-ec07f25053a5: failed to get log: index=3 error="log not found" -2026-04-22T10:10:04.950Z [INFO] server-d305b1ab-13b7-318b-7a98-ec07f25053a5: aborting pipeline replication: peer="{Voter server-c21f2676-5a5a-d8e8-b600-a6a367b12daa c21f2676-5a5a-d8e8-b600-a6a367b12daa}" -2026-04-22T10:10:04.956Z [ERROR] server-d305b1ab-13b7-318b-7a98-ec07f25053a5: failed to get log: index=3 error="log not found" -2026-04-22T10:10:04.956Z [INFO] creating new snapshot: path=/tmp/raft1966396183/snapshots/2-15-1776852604956.tmp -2026-04-22T10:10:04.956Z [INFO] server-ba90a3a8-f980-75cd-6ce0-79e747342937: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:04.956Z [INFO] server-ba90a3a8-f980-75cd-6ce0-79e747342937: copied to local snapshot: bytes=71 -2026-04-22T10:10:04.957Z [INFO] server-ba90a3a8-f980-75cd-6ce0-79e747342937: snapshot restore progress: id=2-15-1776852604956 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:04.957Z [INFO] server-ba90a3a8-f980-75cd-6ce0-79e747342937: Installed remote snapshot -2026-04-22T10:10:04.957Z [INFO] server-d305b1ab-13b7-318b-7a98-ec07f25053a5: pipelining replication: peer="{Voter server-ba90a3a8-f980-75cd-6ce0-79e747342937 ba90a3a8-f980-75cd-6ce0-79e747342937}" -2026-04-22T10:10:04.957Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:10:04.959Z [ERROR] server-d305b1ab-13b7-318b-7a98-ec07f25053a5: failed to get log: index=3 error="log not found" -2026-04-22T10:10:04.960Z [INFO] creating new snapshot: path=/tmp/raft3599686553/snapshots/2-15-1776852604960.tmp -2026-04-22T10:10:04.960Z [INFO] server-c21f2676-5a5a-d8e8-b600-a6a367b12daa: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:04.960Z [INFO] server-c21f2676-5a5a-d8e8-b600-a6a367b12daa: copied to local snapshot: bytes=71 -2026-04-22T10:10:04.960Z [INFO] server-c21f2676-5a5a-d8e8-b600-a6a367b12daa: snapshot restore progress: id=2-15-1776852604960 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:04.960Z [INFO] server-c21f2676-5a5a-d8e8-b600-a6a367b12daa: Installed remote snapshot -2026-04-22T10:10:04.960Z [INFO] server-d305b1ab-13b7-318b-7a98-ec07f25053a5: pipelining replication: peer="{Voter server-c21f2676-5a5a-d8e8-b600-a6a367b12daa c21f2676-5a5a-d8e8-b600-a6a367b12daa}" -2026-04-22T10:10:05.962Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:10:04.957250922 +0000 UTC m=+50.981047015 (2 nodes), 714ns from start of poll, 1.692768277s from cluster start. Timeout at 2026-04-22 10:10:05.962266763 +0000 UTC m=+51.986062896, 1.005015881s after stability -2026-04-22T10:10:05.974Z [INFO] server-c0f2d1a6-c5a1-dafe-63dc-6c4ae05d2700: aborting pipeline replication: peer="{Voter server-1e36660f-0e8c-db77-7038-b66ec1360c48 1e36660f-0e8c-db77-7038-b66ec1360c48}" -2026-04-22T10:10:05.974Z [INFO] server-c0f2d1a6-c5a1-dafe-63dc-6c4ae05d2700: aborting pipeline replication: peer="{Voter server-1b7504ce-4f7e-2638-6547-ddd88374b9f1 1b7504ce-4f7e-2638-6547-ddd88374b9f1}" +2025-03-20T03:50:16.667Z [INFO] server-f39cd79e-da9c-f2f0-eb5a-695a9c42227a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f39cd79e-da9c-f2f0-eb5a-695a9c42227a Address:f39cd79e-da9c-f2f0-eb5a-695a9c42227a} {Suffrage:Voter ID:server-2906e968-a1d0-c451-d3ce-a00c2894847c Address:2906e968-a1d0-c451-d3ce-a00c2894847c} {Suffrage:Voter ID:server-28352af3-9f7d-97d1-3fda-621942e1dc38 Address:28352af3-9f7d-97d1-3fda-621942e1dc38}]" +2025-03-20T03:50:16.667Z [INFO] server-2906e968-a1d0-c451-d3ce-a00c2894847c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f39cd79e-da9c-f2f0-eb5a-695a9c42227a Address:f39cd79e-da9c-f2f0-eb5a-695a9c42227a} {Suffrage:Voter ID:server-2906e968-a1d0-c451-d3ce-a00c2894847c Address:2906e968-a1d0-c451-d3ce-a00c2894847c} {Suffrage:Voter ID:server-28352af3-9f7d-97d1-3fda-621942e1dc38 Address:28352af3-9f7d-97d1-3fda-621942e1dc38}]" +2025-03-20T03:50:16.667Z [INFO] server-28352af3-9f7d-97d1-3fda-621942e1dc38: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f39cd79e-da9c-f2f0-eb5a-695a9c42227a Address:f39cd79e-da9c-f2f0-eb5a-695a9c42227a} {Suffrage:Voter ID:server-2906e968-a1d0-c451-d3ce-a00c2894847c Address:2906e968-a1d0-c451-d3ce-a00c2894847c} {Suffrage:Voter ID:server-28352af3-9f7d-97d1-3fda-621942e1dc38 Address:28352af3-9f7d-97d1-3fda-621942e1dc38}]" +2025-03-20T03:50:16.667Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:50:16.667Z [INFO] server-f39cd79e-da9c-f2f0-eb5a-695a9c42227a: entering follower state: follower="Node at f39cd79e-da9c-f2f0-eb5a-695a9c42227a [Follower]" leader-address= leader-id= +2025-03-20T03:50:16.667Z [INFO] server-2906e968-a1d0-c451-d3ce-a00c2894847c: entering follower state: follower="Node at 2906e968-a1d0-c451-d3ce-a00c2894847c [Follower]" leader-address= leader-id= +2025-03-20T03:50:16.667Z [INFO] server-28352af3-9f7d-97d1-3fda-621942e1dc38: entering follower state: follower="Node at 28352af3-9f7d-97d1-3fda-621942e1dc38 [Follower]" leader-address= leader-id= +2025-03-20T03:50:17.170Z [WARN] server-2906e968-a1d0-c451-d3ce-a00c2894847c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:50:17.170Z [INFO] server-2906e968-a1d0-c451-d3ce-a00c2894847c: entering candidate state: node="Node at 2906e968-a1d0-c451-d3ce-a00c2894847c [Candidate]" term=2 +2025-03-20T03:50:17.170Z [INFO] server-2906e968-a1d0-c451-d3ce-a00c2894847c: election won: term=2 tally=2 +2025-03-20T03:50:17.170Z [INFO] server-2906e968-a1d0-c451-d3ce-a00c2894847c: entering leader state: leader="Node at 2906e968-a1d0-c451-d3ce-a00c2894847c [Leader]" +2025-03-20T03:50:17.170Z [INFO] server-2906e968-a1d0-c451-d3ce-a00c2894847c: added peer, starting replication: peer=server-f39cd79e-da9c-f2f0-eb5a-695a9c42227a +2025-03-20T03:50:17.170Z [INFO] server-2906e968-a1d0-c451-d3ce-a00c2894847c: added peer, starting replication: peer=server-28352af3-9f7d-97d1-3fda-621942e1dc38 +2025-03-20T03:50:17.171Z [INFO] server-2906e968-a1d0-c451-d3ce-a00c2894847c: pipelining replication: peer="{Voter server-28352af3-9f7d-97d1-3fda-621942e1dc38 28352af3-9f7d-97d1-3fda-621942e1dc38}" +2025-03-20T03:50:17.171Z [INFO] server-2906e968-a1d0-c451-d3ce-a00c2894847c: pipelining replication: peer="{Voter server-f39cd79e-da9c-f2f0-eb5a-695a9c42227a f39cd79e-da9c-f2f0-eb5a-695a9c42227a}" +2025-03-20T03:50:18.178Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:50:17.171516433 +0000 UTC m=+48.845969290 (1 nodes), 503.932118ms from start of poll, 504.371212ms from cluster start. Timeout at 2025-03-20 03:50:18.176521807 +0000 UTC m=+49.850975512, 1.005006222s after stability +2025-03-20T03:50:18.179Z [INFO] server-2906e968-a1d0-c451-d3ce-a00c2894847c: starting snapshot up to: index=12 +2025-03-20T03:50:18.179Z [INFO] creating new snapshot: path=/tmp/raft1911948163/snapshots/2-12-1742442618179.tmp +2025-03-20T03:50:18.180Z [INFO] server-2906e968-a1d0-c451-d3ce-a00c2894847c: snapshot complete up to: index=12 +2025-03-20T03:50:18.183Z [INFO] server-8fc4c748-473e-84cf-e592-676b7ab9a049: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8fc4c748-473e-84cf-e592-676b7ab9a049 Address:8fc4c748-473e-84cf-e592-676b7ab9a049} {Suffrage:Voter ID:server-4588dbe5-3659-fdb6-7992-ea346fa55b28 Address:4588dbe5-3659-fdb6-7992-ea346fa55b28} {Suffrage:Voter ID:server-29f478ab-7a47-a793-ea2c-01b0bd11c7a5 Address:29f478ab-7a47-a793-ea2c-01b0bd11c7a5}]" +2025-03-20T03:50:18.183Z [INFO] server-4588dbe5-3659-fdb6-7992-ea346fa55b28: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8fc4c748-473e-84cf-e592-676b7ab9a049 Address:8fc4c748-473e-84cf-e592-676b7ab9a049} {Suffrage:Voter ID:server-4588dbe5-3659-fdb6-7992-ea346fa55b28 Address:4588dbe5-3659-fdb6-7992-ea346fa55b28} {Suffrage:Voter ID:server-29f478ab-7a47-a793-ea2c-01b0bd11c7a5 Address:29f478ab-7a47-a793-ea2c-01b0bd11c7a5}]" +2025-03-20T03:50:18.183Z [INFO] server-29f478ab-7a47-a793-ea2c-01b0bd11c7a5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8fc4c748-473e-84cf-e592-676b7ab9a049 Address:8fc4c748-473e-84cf-e592-676b7ab9a049} {Suffrage:Voter ID:server-4588dbe5-3659-fdb6-7992-ea346fa55b28 Address:4588dbe5-3659-fdb6-7992-ea346fa55b28} {Suffrage:Voter ID:server-29f478ab-7a47-a793-ea2c-01b0bd11c7a5 Address:29f478ab-7a47-a793-ea2c-01b0bd11c7a5}]" +2025-03-20T03:50:18.183Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:50:18.183Z [INFO] server-8fc4c748-473e-84cf-e592-676b7ab9a049: entering follower state: follower="Node at 8fc4c748-473e-84cf-e592-676b7ab9a049 [Follower]" leader-address= leader-id= +2025-03-20T03:50:18.183Z [INFO] server-4588dbe5-3659-fdb6-7992-ea346fa55b28: entering follower state: follower="Node at 4588dbe5-3659-fdb6-7992-ea346fa55b28 [Follower]" leader-address= leader-id= +2025-03-20T03:50:18.183Z [INFO] server-29f478ab-7a47-a793-ea2c-01b0bd11c7a5: entering follower state: follower="Node at 29f478ab-7a47-a793-ea2c-01b0bd11c7a5 [Follower]" leader-address= leader-id= +2025-03-20T03:50:18.699Z [WARN] server-29f478ab-7a47-a793-ea2c-01b0bd11c7a5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:50:18.699Z [INFO] server-29f478ab-7a47-a793-ea2c-01b0bd11c7a5: entering candidate state: node="Node at 29f478ab-7a47-a793-ea2c-01b0bd11c7a5 [Candidate]" term=2 +2025-03-20T03:50:18.699Z [INFO] server-29f478ab-7a47-a793-ea2c-01b0bd11c7a5: election won: term=2 tally=2 +2025-03-20T03:50:18.699Z [INFO] server-29f478ab-7a47-a793-ea2c-01b0bd11c7a5: entering leader state: leader="Node at 29f478ab-7a47-a793-ea2c-01b0bd11c7a5 [Leader]" +2025-03-20T03:50:18.699Z [INFO] server-29f478ab-7a47-a793-ea2c-01b0bd11c7a5: added peer, starting replication: peer=server-8fc4c748-473e-84cf-e592-676b7ab9a049 +2025-03-20T03:50:18.699Z [INFO] server-29f478ab-7a47-a793-ea2c-01b0bd11c7a5: added peer, starting replication: peer=server-4588dbe5-3659-fdb6-7992-ea346fa55b28 +2025-03-20T03:50:18.699Z [INFO] server-29f478ab-7a47-a793-ea2c-01b0bd11c7a5: pipelining replication: peer="{Voter server-4588dbe5-3659-fdb6-7992-ea346fa55b28 4588dbe5-3659-fdb6-7992-ea346fa55b28}" +2025-03-20T03:50:18.699Z [INFO] server-29f478ab-7a47-a793-ea2c-01b0bd11c7a5: pipelining replication: peer="{Voter server-8fc4c748-473e-84cf-e592-676b7ab9a049 8fc4c748-473e-84cf-e592-676b7ab9a049}" +2025-03-20T03:50:19.706Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:50:18.699783271 +0000 UTC m=+50.374236384 (1 nodes), 516.404491ms from start of poll, 516.838139ms from cluster start. Timeout at 2025-03-20 03:50:19.704815798 +0000 UTC m=+51.379268577, 1.005032193s after stability +2025-03-20T03:50:19.706Z [INFO] creating new snapshot: path=/tmp/raft131989513/snapshots/2-15-1742442619706.tmp +2025-03-20T03:50:19.707Z [INFO] server-29f478ab-7a47-a793-ea2c-01b0bd11c7a5: copied to local snapshot: bytes=71 +2025-03-20T03:50:19.708Z [INFO] server-29f478ab-7a47-a793-ea2c-01b0bd11c7a5: snapshot restore progress: id=2-15-1742442619706 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:19.708Z [INFO] server-29f478ab-7a47-a793-ea2c-01b0bd11c7a5: restored user snapshot: index=15 +2025-03-20T03:50:19.708Z [ERROR] server-29f478ab-7a47-a793-ea2c-01b0bd11c7a5: failed to get log: index=3 error="log not found" +2025-03-20T03:50:19.708Z [INFO] server-29f478ab-7a47-a793-ea2c-01b0bd11c7a5: aborting pipeline replication: peer="{Voter server-8fc4c748-473e-84cf-e592-676b7ab9a049 8fc4c748-473e-84cf-e592-676b7ab9a049}" +2025-03-20T03:50:19.708Z [ERROR] server-29f478ab-7a47-a793-ea2c-01b0bd11c7a5: failed to get log: index=3 error="log not found" +2025-03-20T03:50:19.708Z [INFO] server-29f478ab-7a47-a793-ea2c-01b0bd11c7a5: aborting pipeline replication: peer="{Voter server-4588dbe5-3659-fdb6-7992-ea346fa55b28 4588dbe5-3659-fdb6-7992-ea346fa55b28}" +2025-03-20T03:50:19.715Z [ERROR] server-29f478ab-7a47-a793-ea2c-01b0bd11c7a5: failed to get log: index=3 error="log not found" +2025-03-20T03:50:19.716Z [INFO] creating new snapshot: path=/tmp/raft3717657466/snapshots/2-15-1742442619716.tmp +2025-03-20T03:50:19.716Z [INFO] server-4588dbe5-3659-fdb6-7992-ea346fa55b28: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:19.717Z [INFO] server-4588dbe5-3659-fdb6-7992-ea346fa55b28: copied to local snapshot: bytes=71 +2025-03-20T03:50:19.718Z [ERROR] server-29f478ab-7a47-a793-ea2c-01b0bd11c7a5: failed to get log: index=3 error="log not found" +2025-03-20T03:50:19.718Z [INFO] server-4588dbe5-3659-fdb6-7992-ea346fa55b28: snapshot restore progress: id=2-15-1742442619716 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:19.718Z [INFO] server-4588dbe5-3659-fdb6-7992-ea346fa55b28: Installed remote snapshot +2025-03-20T03:50:19.719Z [INFO] server-29f478ab-7a47-a793-ea2c-01b0bd11c7a5: pipelining replication: peer="{Voter server-4588dbe5-3659-fdb6-7992-ea346fa55b28 4588dbe5-3659-fdb6-7992-ea346fa55b28}" +2025-03-20T03:50:19.719Z [INFO] creating new snapshot: path=/tmp/raft2465243543/snapshots/2-15-1742442619719.tmp +2025-03-20T03:50:19.719Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:50:19.719Z [INFO] server-8fc4c748-473e-84cf-e592-676b7ab9a049: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:19.720Z [INFO] server-8fc4c748-473e-84cf-e592-676b7ab9a049: copied to local snapshot: bytes=71 +2025-03-20T03:50:19.720Z [INFO] server-8fc4c748-473e-84cf-e592-676b7ab9a049: snapshot restore progress: id=2-15-1742442619719 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:19.720Z [INFO] server-8fc4c748-473e-84cf-e592-676b7ab9a049: Installed remote snapshot +2025-03-20T03:50:19.720Z [INFO] server-29f478ab-7a47-a793-ea2c-01b0bd11c7a5: pipelining replication: peer="{Voter server-8fc4c748-473e-84cf-e592-676b7ab9a049 8fc4c748-473e-84cf-e592-676b7ab9a049}" +2025-03-20T03:50:20.725Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:50:19.719186074 +0000 UTC m=+51.393638965 (2 nodes), 2.116µs from start of poll, 1.53624072s from cluster start. Timeout at 2025-03-20 03:50:20.724215615 +0000 UTC m=+52.398668756, 1.005029791s after stability +2025-03-20T03:50:20.741Z [INFO] server-2906e968-a1d0-c451-d3ce-a00c2894847c: aborting pipeline replication: peer="{Voter server-f39cd79e-da9c-f2f0-eb5a-695a9c42227a f39cd79e-da9c-f2f0-eb5a-695a9c42227a}" +2025-03-20T03:50:20.741Z [INFO] server-2906e968-a1d0-c451-d3ce-a00c2894847c: aborting pipeline replication: peer="{Voter server-28352af3-9f7d-97d1-3fda-621942e1dc38 28352af3-9f7d-97d1-3fda-621942e1dc38}" === RUN TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_true -2026-04-22T10:10:05.975Z [INFO] server-a74df70e-ac27-7834-7d95-a4925533c173: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a74df70e-ac27-7834-7d95-a4925533c173 Address:a74df70e-ac27-7834-7d95-a4925533c173} {Suffrage:Voter ID:server-a65a933e-c447-f688-a674-8b3eb17ad0fe Address:a65a933e-c447-f688-a674-8b3eb17ad0fe} {Suffrage:Voter ID:server-99044550-26ba-b3c3-83f7-0d4bb4fb805d Address:99044550-26ba-b3c3-83f7-0d4bb4fb805d}]" -2026-04-22T10:10:05.975Z [INFO] server-a74df70e-ac27-7834-7d95-a4925533c173: entering follower state: follower="Node at a74df70e-ac27-7834-7d95-a4925533c173 [Follower]" leader-address= leader-id= -2026-04-22T10:10:05.975Z [INFO] server-a65a933e-c447-f688-a674-8b3eb17ad0fe: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a74df70e-ac27-7834-7d95-a4925533c173 Address:a74df70e-ac27-7834-7d95-a4925533c173} {Suffrage:Voter ID:server-a65a933e-c447-f688-a674-8b3eb17ad0fe Address:a65a933e-c447-f688-a674-8b3eb17ad0fe} {Suffrage:Voter ID:server-99044550-26ba-b3c3-83f7-0d4bb4fb805d Address:99044550-26ba-b3c3-83f7-0d4bb4fb805d}]" -2026-04-22T10:10:05.975Z [INFO] server-a65a933e-c447-f688-a674-8b3eb17ad0fe: entering follower state: follower="Node at a65a933e-c447-f688-a674-8b3eb17ad0fe [Follower]" leader-address= leader-id= -2026-04-22T10:10:05.975Z [INFO] server-99044550-26ba-b3c3-83f7-0d4bb4fb805d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a74df70e-ac27-7834-7d95-a4925533c173 Address:a74df70e-ac27-7834-7d95-a4925533c173} {Suffrage:Voter ID:server-a65a933e-c447-f688-a674-8b3eb17ad0fe Address:a65a933e-c447-f688-a674-8b3eb17ad0fe} {Suffrage:Voter ID:server-99044550-26ba-b3c3-83f7-0d4bb4fb805d Address:99044550-26ba-b3c3-83f7-0d4bb4fb805d}]" -2026-04-22T10:10:05.975Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:05.976Z [INFO] server-99044550-26ba-b3c3-83f7-0d4bb4fb805d: entering follower state: follower="Node at 99044550-26ba-b3c3-83f7-0d4bb4fb805d [Follower]" leader-address= leader-id= -2026-04-22T10:10:06.641Z [WARN] server-a74df70e-ac27-7834-7d95-a4925533c173: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:06.641Z [INFO] server-a74df70e-ac27-7834-7d95-a4925533c173: entering candidate state: node="Node at a74df70e-ac27-7834-7d95-a4925533c173 [Candidate]" term=2 -2026-04-22T10:10:06.641Z [INFO] server-a74df70e-ac27-7834-7d95-a4925533c173: election won: term=2 tally=2 -2026-04-22T10:10:06.641Z [INFO] server-a74df70e-ac27-7834-7d95-a4925533c173: entering leader state: leader="Node at a74df70e-ac27-7834-7d95-a4925533c173 [Leader]" -2026-04-22T10:10:06.641Z [INFO] server-a74df70e-ac27-7834-7d95-a4925533c173: added peer, starting replication: peer=server-a65a933e-c447-f688-a674-8b3eb17ad0fe -2026-04-22T10:10:06.641Z [INFO] server-a74df70e-ac27-7834-7d95-a4925533c173: added peer, starting replication: peer=server-99044550-26ba-b3c3-83f7-0d4bb4fb805d -2026-04-22T10:10:06.641Z [INFO] server-a74df70e-ac27-7834-7d95-a4925533c173: pipelining replication: peer="{Voter server-99044550-26ba-b3c3-83f7-0d4bb4fb805d 99044550-26ba-b3c3-83f7-0d4bb4fb805d}" -2026-04-22T10:10:06.641Z [INFO] server-a74df70e-ac27-7834-7d95-a4925533c173: pipelining replication: peer="{Voter server-a65a933e-c447-f688-a674-8b3eb17ad0fe a65a933e-c447-f688-a674-8b3eb17ad0fe}" -2026-04-22T10:10:07.646Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:06.64180989 +0000 UTC m=+52.665605978 (1 nodes), 665.799391ms from start of poll, 666.107018ms from cluster start. Timeout at 2026-04-22 10:10:07.646815333 +0000 UTC m=+53.670611461, 1.005005483s after stability -2026-04-22T10:10:07.647Z [INFO] server-a74df70e-ac27-7834-7d95-a4925533c173: starting snapshot up to: index=12 -2026-04-22T10:10:07.647Z [INFO] creating new snapshot: path=/tmp/raft3131355993/snapshots/2-12-1776852607647.tmp -2026-04-22T10:10:07.648Z [INFO] server-a74df70e-ac27-7834-7d95-a4925533c173: snapshot complete up to: index=12 -2026-04-22T10:10:07.649Z [INFO] server-a7d1dfc8-d842-46b0-38ae-67983b35e470: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a7d1dfc8-d842-46b0-38ae-67983b35e470 Address:a7d1dfc8-d842-46b0-38ae-67983b35e470} {Suffrage:Voter ID:server-1c09e58b-6cb6-b4fc-4c18-d04ed88a9716 Address:1c09e58b-6cb6-b4fc-4c18-d04ed88a9716} {Suffrage:Voter ID:server-d7661377-f98d-d220-70e1-1063ecec9952 Address:d7661377-f98d-d220-70e1-1063ecec9952}]" -2026-04-22T10:10:07.649Z [INFO] server-a7d1dfc8-d842-46b0-38ae-67983b35e470: entering follower state: follower="Node at a7d1dfc8-d842-46b0-38ae-67983b35e470 [Follower]" leader-address= leader-id= -2026-04-22T10:10:07.650Z [INFO] server-1c09e58b-6cb6-b4fc-4c18-d04ed88a9716: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a7d1dfc8-d842-46b0-38ae-67983b35e470 Address:a7d1dfc8-d842-46b0-38ae-67983b35e470} {Suffrage:Voter ID:server-1c09e58b-6cb6-b4fc-4c18-d04ed88a9716 Address:1c09e58b-6cb6-b4fc-4c18-d04ed88a9716} {Suffrage:Voter ID:server-d7661377-f98d-d220-70e1-1063ecec9952 Address:d7661377-f98d-d220-70e1-1063ecec9952}]" -2026-04-22T10:10:07.650Z [INFO] server-1c09e58b-6cb6-b4fc-4c18-d04ed88a9716: entering follower state: follower="Node at 1c09e58b-6cb6-b4fc-4c18-d04ed88a9716 [Follower]" leader-address= leader-id= -2026-04-22T10:10:07.650Z [INFO] server-d7661377-f98d-d220-70e1-1063ecec9952: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a7d1dfc8-d842-46b0-38ae-67983b35e470 Address:a7d1dfc8-d842-46b0-38ae-67983b35e470} {Suffrage:Voter ID:server-1c09e58b-6cb6-b4fc-4c18-d04ed88a9716 Address:1c09e58b-6cb6-b4fc-4c18-d04ed88a9716} {Suffrage:Voter ID:server-d7661377-f98d-d220-70e1-1063ecec9952 Address:d7661377-f98d-d220-70e1-1063ecec9952}]" -2026-04-22T10:10:07.650Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:07.650Z [INFO] server-d7661377-f98d-d220-70e1-1063ecec9952: entering follower state: follower="Node at d7661377-f98d-d220-70e1-1063ecec9952 [Follower]" leader-address= leader-id= -2026-04-22T10:10:08.251Z [WARN] server-a7d1dfc8-d842-46b0-38ae-67983b35e470: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:08.251Z [INFO] server-a7d1dfc8-d842-46b0-38ae-67983b35e470: entering candidate state: node="Node at a7d1dfc8-d842-46b0-38ae-67983b35e470 [Candidate]" term=2 -2026-04-22T10:10:08.251Z [INFO] server-a7d1dfc8-d842-46b0-38ae-67983b35e470: election won: term=2 tally=2 -2026-04-22T10:10:08.251Z [INFO] server-a7d1dfc8-d842-46b0-38ae-67983b35e470: entering leader state: leader="Node at a7d1dfc8-d842-46b0-38ae-67983b35e470 [Leader]" -2026-04-22T10:10:08.251Z [INFO] server-a7d1dfc8-d842-46b0-38ae-67983b35e470: added peer, starting replication: peer=server-1c09e58b-6cb6-b4fc-4c18-d04ed88a9716 -2026-04-22T10:10:08.251Z [INFO] server-a7d1dfc8-d842-46b0-38ae-67983b35e470: added peer, starting replication: peer=server-d7661377-f98d-d220-70e1-1063ecec9952 -2026-04-22T10:10:08.251Z [INFO] server-a7d1dfc8-d842-46b0-38ae-67983b35e470: pipelining replication: peer="{Voter server-d7661377-f98d-d220-70e1-1063ecec9952 d7661377-f98d-d220-70e1-1063ecec9952}" -2026-04-22T10:10:08.251Z [INFO] server-a7d1dfc8-d842-46b0-38ae-67983b35e470: pipelining replication: peer="{Voter server-1c09e58b-6cb6-b4fc-4c18-d04ed88a9716 1c09e58b-6cb6-b4fc-4c18-d04ed88a9716}" -2026-04-22T10:10:09.256Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:08.251251381 +0000 UTC m=+54.275047530 (1 nodes), 600.784251ms from start of poll, 601.643602ms from cluster start. Timeout at 2026-04-22 10:10:09.25626573 +0000 UTC m=+55.280061863, 1.005014333s after stability -2026-04-22T10:10:09.256Z [INFO] creating new snapshot: path=/tmp/raft740047648/snapshots/2-15-1776852609256.tmp -2026-04-22T10:10:09.257Z [INFO] server-a7d1dfc8-d842-46b0-38ae-67983b35e470: copied to local snapshot: bytes=71 -2026-04-22T10:10:09.257Z [INFO] server-a7d1dfc8-d842-46b0-38ae-67983b35e470: snapshot restore progress: id=2-15-1776852609256 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:09.257Z [INFO] server-a7d1dfc8-d842-46b0-38ae-67983b35e470: removing all old logs from log store -2026-04-22T10:10:09.257Z [INFO] server-a7d1dfc8-d842-46b0-38ae-67983b35e470: compacting logs: from=1 to=2 -2026-04-22T10:10:09.257Z [INFO] server-a7d1dfc8-d842-46b0-38ae-67983b35e470: restored user snapshot: index=15 -2026-04-22T10:10:09.257Z [ERROR] server-a7d1dfc8-d842-46b0-38ae-67983b35e470: failed to get log: index=2 error="log not found" -2026-04-22T10:10:09.257Z [ERROR] server-a7d1dfc8-d842-46b0-38ae-67983b35e470: failed to get log: index=2 error="log not found" -2026-04-22T10:10:09.257Z [INFO] server-a7d1dfc8-d842-46b0-38ae-67983b35e470: aborting pipeline replication: peer="{Voter server-1c09e58b-6cb6-b4fc-4c18-d04ed88a9716 1c09e58b-6cb6-b4fc-4c18-d04ed88a9716}" -2026-04-22T10:10:09.257Z [INFO] server-a7d1dfc8-d842-46b0-38ae-67983b35e470: aborting pipeline replication: peer="{Voter server-d7661377-f98d-d220-70e1-1063ecec9952 d7661377-f98d-d220-70e1-1063ecec9952}" -2026-04-22T10:10:09.263Z [ERROR] server-a7d1dfc8-d842-46b0-38ae-67983b35e470: failed to get log: index=2 error="log not found" -2026-04-22T10:10:09.264Z [INFO] creating new snapshot: path=/tmp/raft1970142833/snapshots/2-15-1776852609264.tmp -2026-04-22T10:10:09.264Z [INFO] server-1c09e58b-6cb6-b4fc-4c18-d04ed88a9716: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:09.264Z [INFO] server-1c09e58b-6cb6-b4fc-4c18-d04ed88a9716: copied to local snapshot: bytes=71 -2026-04-22T10:10:09.264Z [INFO] server-1c09e58b-6cb6-b4fc-4c18-d04ed88a9716: snapshot restore progress: id=2-15-1776852609264 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:09.264Z [INFO] server-1c09e58b-6cb6-b4fc-4c18-d04ed88a9716: removing all old logs from log store -2026-04-22T10:10:09.264Z [INFO] server-1c09e58b-6cb6-b4fc-4c18-d04ed88a9716: compacting logs: from=1 to=2 -2026-04-22T10:10:09.264Z [INFO] server-1c09e58b-6cb6-b4fc-4c18-d04ed88a9716: Installed remote snapshot -2026-04-22T10:10:09.264Z [INFO] server-a7d1dfc8-d842-46b0-38ae-67983b35e470: pipelining replication: peer="{Voter server-1c09e58b-6cb6-b4fc-4c18-d04ed88a9716 1c09e58b-6cb6-b4fc-4c18-d04ed88a9716}" -2026-04-22T10:10:09.265Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:10:09.267Z [ERROR] server-a7d1dfc8-d842-46b0-38ae-67983b35e470: failed to get log: index=2 error="log not found" -2026-04-22T10:10:09.267Z [INFO] creating new snapshot: path=/tmp/raft1819537510/snapshots/2-15-1776852609267.tmp -2026-04-22T10:10:09.267Z [INFO] server-d7661377-f98d-d220-70e1-1063ecec9952: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:09.268Z [INFO] server-d7661377-f98d-d220-70e1-1063ecec9952: copied to local snapshot: bytes=71 -2026-04-22T10:10:09.268Z [INFO] server-d7661377-f98d-d220-70e1-1063ecec9952: snapshot restore progress: id=2-15-1776852609267 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:09.268Z [INFO] server-d7661377-f98d-d220-70e1-1063ecec9952: removing all old logs from log store -2026-04-22T10:10:09.268Z [INFO] server-d7661377-f98d-d220-70e1-1063ecec9952: compacting logs: from=1 to=2 -2026-04-22T10:10:09.268Z [INFO] server-d7661377-f98d-d220-70e1-1063ecec9952: Installed remote snapshot -2026-04-22T10:10:09.268Z [INFO] server-a7d1dfc8-d842-46b0-38ae-67983b35e470: pipelining replication: peer="{Voter server-d7661377-f98d-d220-70e1-1063ecec9952 d7661377-f98d-d220-70e1-1063ecec9952}" -2026-04-22T10:10:10.271Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:10:09.265036493 +0000 UTC m=+55.288832604 (2 nodes), 772ns from start of poll, 1.615428676s from cluster start. Timeout at 2026-04-22 10:10:10.270055596 +0000 UTC m=+56.293851759, 1.005019155s after stability -2026-04-22T10:10:10.283Z [INFO] server-a74df70e-ac27-7834-7d95-a4925533c173: aborting pipeline replication: peer="{Voter server-99044550-26ba-b3c3-83f7-0d4bb4fb805d 99044550-26ba-b3c3-83f7-0d4bb4fb805d}" -2026-04-22T10:10:10.283Z [INFO] server-a74df70e-ac27-7834-7d95-a4925533c173: aborting pipeline replication: peer="{Voter server-a65a933e-c447-f688-a674-8b3eb17ad0fe a65a933e-c447-f688-a674-8b3eb17ad0fe}" +2025-03-20T03:50:20.743Z [INFO] server-cd5dc3e0-d8a8-0cb2-aca5-0038b0bc7979: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cd5dc3e0-d8a8-0cb2-aca5-0038b0bc7979 Address:cd5dc3e0-d8a8-0cb2-aca5-0038b0bc7979} {Suffrage:Voter ID:server-bfc455e1-c23e-c2f0-4b80-6992045f85b9 Address:bfc455e1-c23e-c2f0-4b80-6992045f85b9} {Suffrage:Voter ID:server-b40cefb6-e851-7659-4cd6-ca016e864fdd Address:b40cefb6-e851-7659-4cd6-ca016e864fdd}]" +2025-03-20T03:50:20.743Z [INFO] server-bfc455e1-c23e-c2f0-4b80-6992045f85b9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cd5dc3e0-d8a8-0cb2-aca5-0038b0bc7979 Address:cd5dc3e0-d8a8-0cb2-aca5-0038b0bc7979} {Suffrage:Voter ID:server-bfc455e1-c23e-c2f0-4b80-6992045f85b9 Address:bfc455e1-c23e-c2f0-4b80-6992045f85b9} {Suffrage:Voter ID:server-b40cefb6-e851-7659-4cd6-ca016e864fdd Address:b40cefb6-e851-7659-4cd6-ca016e864fdd}]" +2025-03-20T03:50:20.743Z [INFO] server-cd5dc3e0-d8a8-0cb2-aca5-0038b0bc7979: entering follower state: follower="Node at cd5dc3e0-d8a8-0cb2-aca5-0038b0bc7979 [Follower]" leader-address= leader-id= +2025-03-20T03:50:20.743Z [INFO] server-b40cefb6-e851-7659-4cd6-ca016e864fdd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cd5dc3e0-d8a8-0cb2-aca5-0038b0bc7979 Address:cd5dc3e0-d8a8-0cb2-aca5-0038b0bc7979} {Suffrage:Voter ID:server-bfc455e1-c23e-c2f0-4b80-6992045f85b9 Address:bfc455e1-c23e-c2f0-4b80-6992045f85b9} {Suffrage:Voter ID:server-b40cefb6-e851-7659-4cd6-ca016e864fdd Address:b40cefb6-e851-7659-4cd6-ca016e864fdd}]" +2025-03-20T03:50:20.743Z [INFO] server-bfc455e1-c23e-c2f0-4b80-6992045f85b9: entering follower state: follower="Node at bfc455e1-c23e-c2f0-4b80-6992045f85b9 [Follower]" leader-address= leader-id= +2025-03-20T03:50:20.743Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:50:20.743Z [INFO] server-b40cefb6-e851-7659-4cd6-ca016e864fdd: entering follower state: follower="Node at b40cefb6-e851-7659-4cd6-ca016e864fdd [Follower]" leader-address= leader-id= +2025-03-20T03:50:21.313Z [WARN] server-b40cefb6-e851-7659-4cd6-ca016e864fdd: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:50:21.313Z [INFO] server-b40cefb6-e851-7659-4cd6-ca016e864fdd: entering candidate state: node="Node at b40cefb6-e851-7659-4cd6-ca016e864fdd [Candidate]" term=2 +2025-03-20T03:50:21.313Z [INFO] server-b40cefb6-e851-7659-4cd6-ca016e864fdd: election won: term=2 tally=2 +2025-03-20T03:50:21.313Z [INFO] server-b40cefb6-e851-7659-4cd6-ca016e864fdd: entering leader state: leader="Node at b40cefb6-e851-7659-4cd6-ca016e864fdd [Leader]" +2025-03-20T03:50:21.313Z [INFO] server-b40cefb6-e851-7659-4cd6-ca016e864fdd: added peer, starting replication: peer=server-cd5dc3e0-d8a8-0cb2-aca5-0038b0bc7979 +2025-03-20T03:50:21.313Z [INFO] server-b40cefb6-e851-7659-4cd6-ca016e864fdd: added peer, starting replication: peer=server-bfc455e1-c23e-c2f0-4b80-6992045f85b9 +2025-03-20T03:50:21.313Z [INFO] server-b40cefb6-e851-7659-4cd6-ca016e864fdd: pipelining replication: peer="{Voter server-bfc455e1-c23e-c2f0-4b80-6992045f85b9 bfc455e1-c23e-c2f0-4b80-6992045f85b9}" +2025-03-20T03:50:21.313Z [INFO] server-b40cefb6-e851-7659-4cd6-ca016e864fdd: pipelining replication: peer="{Voter server-cd5dc3e0-d8a8-0cb2-aca5-0038b0bc7979 cd5dc3e0-d8a8-0cb2-aca5-0038b0bc7979}" +2025-03-20T03:50:22.318Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:50:21.313836364 +0000 UTC m=+52.988290207 (1 nodes), 570.045237ms from start of poll, 570.471935ms from cluster start. Timeout at 2025-03-20 03:50:22.318845257 +0000 UTC m=+53.993298248, 1.005008041s after stability +2025-03-20T03:50:22.320Z [INFO] server-b40cefb6-e851-7659-4cd6-ca016e864fdd: starting snapshot up to: index=12 +2025-03-20T03:50:22.320Z [INFO] creating new snapshot: path=/tmp/raft2914431399/snapshots/2-12-1742442622320.tmp +2025-03-20T03:50:22.321Z [INFO] server-b40cefb6-e851-7659-4cd6-ca016e864fdd: snapshot complete up to: index=12 +2025-03-20T03:50:22.334Z [INFO] server-cd38a9e4-f93a-b249-77dc-db998253e2f9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cd38a9e4-f93a-b249-77dc-db998253e2f9 Address:cd38a9e4-f93a-b249-77dc-db998253e2f9} {Suffrage:Voter ID:server-2585b7d3-2635-ea9d-6ada-0e0ed216a860 Address:2585b7d3-2635-ea9d-6ada-0e0ed216a860} {Suffrage:Voter ID:server-4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f Address:4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f}]" +2025-03-20T03:50:22.334Z [INFO] server-2585b7d3-2635-ea9d-6ada-0e0ed216a860: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cd38a9e4-f93a-b249-77dc-db998253e2f9 Address:cd38a9e4-f93a-b249-77dc-db998253e2f9} {Suffrage:Voter ID:server-2585b7d3-2635-ea9d-6ada-0e0ed216a860 Address:2585b7d3-2635-ea9d-6ada-0e0ed216a860} {Suffrage:Voter ID:server-4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f Address:4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f}]" +2025-03-20T03:50:22.334Z [INFO] server-4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cd38a9e4-f93a-b249-77dc-db998253e2f9 Address:cd38a9e4-f93a-b249-77dc-db998253e2f9} {Suffrage:Voter ID:server-2585b7d3-2635-ea9d-6ada-0e0ed216a860 Address:2585b7d3-2635-ea9d-6ada-0e0ed216a860} {Suffrage:Voter ID:server-4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f Address:4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f}]" +2025-03-20T03:50:22.334Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:50:22.334Z [INFO] server-cd38a9e4-f93a-b249-77dc-db998253e2f9: entering follower state: follower="Node at cd38a9e4-f93a-b249-77dc-db998253e2f9 [Follower]" leader-address= leader-id= +2025-03-20T03:50:22.334Z [INFO] server-2585b7d3-2635-ea9d-6ada-0e0ed216a860: entering follower state: follower="Node at 2585b7d3-2635-ea9d-6ada-0e0ed216a860 [Follower]" leader-address= leader-id= +2025-03-20T03:50:22.334Z [INFO] server-4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f: entering follower state: follower="Node at 4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f [Follower]" leader-address= leader-id= +2025-03-20T03:50:22.851Z [WARN] server-2585b7d3-2635-ea9d-6ada-0e0ed216a860: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:50:22.851Z [INFO] server-2585b7d3-2635-ea9d-6ada-0e0ed216a860: entering candidate state: node="Node at 2585b7d3-2635-ea9d-6ada-0e0ed216a860 [Candidate]" term=2 +2025-03-20T03:50:22.851Z [INFO] server-2585b7d3-2635-ea9d-6ada-0e0ed216a860: election won: term=2 tally=2 +2025-03-20T03:50:22.851Z [INFO] server-2585b7d3-2635-ea9d-6ada-0e0ed216a860: entering leader state: leader="Node at 2585b7d3-2635-ea9d-6ada-0e0ed216a860 [Leader]" +2025-03-20T03:50:22.851Z [INFO] server-2585b7d3-2635-ea9d-6ada-0e0ed216a860: added peer, starting replication: peer=server-cd38a9e4-f93a-b249-77dc-db998253e2f9 +2025-03-20T03:50:22.851Z [INFO] server-2585b7d3-2635-ea9d-6ada-0e0ed216a860: added peer, starting replication: peer=server-4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f +2025-03-20T03:50:22.852Z [INFO] server-2585b7d3-2635-ea9d-6ada-0e0ed216a860: pipelining replication: peer="{Voter server-4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f 4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f}" +2025-03-20T03:50:22.852Z [WARN] server-4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f: rejecting vote request since our last term is greater: candidate=2585b7d3-2635-ea9d-6ada-0e0ed216a860 last-term=2 last-candidate-term=1 +2025-03-20T03:50:22.852Z [INFO] server-2585b7d3-2635-ea9d-6ada-0e0ed216a860: pipelining replication: peer="{Voter server-cd38a9e4-f93a-b249-77dc-db998253e2f9 cd38a9e4-f93a-b249-77dc-db998253e2f9}" +2025-03-20T03:50:23.868Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:50:22.852449692 +0000 UTC m=+54.526902559 (1 nodes), 517.804961ms from start of poll, 518.35734ms from cluster start. Timeout at 2025-03-20 03:50:23.857485607 +0000 UTC m=+55.531938491, 1.005035932s after stability +2025-03-20T03:50:23.869Z [INFO] creating new snapshot: path=/tmp/raft645160594/snapshots/2-15-1742442623869.tmp +2025-03-20T03:50:23.870Z [INFO] server-2585b7d3-2635-ea9d-6ada-0e0ed216a860: copied to local snapshot: bytes=71 +2025-03-20T03:50:23.871Z [INFO] server-2585b7d3-2635-ea9d-6ada-0e0ed216a860: snapshot restore progress: id=2-15-1742442623869 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:23.871Z [INFO] server-2585b7d3-2635-ea9d-6ada-0e0ed216a860: removing all old logs from log store +2025-03-20T03:50:23.871Z [INFO] server-2585b7d3-2635-ea9d-6ada-0e0ed216a860: compacting logs: from=1 to=2 +2025-03-20T03:50:23.871Z [INFO] server-2585b7d3-2635-ea9d-6ada-0e0ed216a860: restored user snapshot: index=15 +2025-03-20T03:50:23.871Z [ERROR] server-2585b7d3-2635-ea9d-6ada-0e0ed216a860: failed to get log: index=2 error="log not found" +2025-03-20T03:50:23.871Z [INFO] server-2585b7d3-2635-ea9d-6ada-0e0ed216a860: aborting pipeline replication: peer="{Voter server-4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f 4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f}" +2025-03-20T03:50:23.871Z [ERROR] server-2585b7d3-2635-ea9d-6ada-0e0ed216a860: failed to get log: index=2 error="log not found" +2025-03-20T03:50:23.871Z [INFO] server-2585b7d3-2635-ea9d-6ada-0e0ed216a860: aborting pipeline replication: peer="{Voter server-cd38a9e4-f93a-b249-77dc-db998253e2f9 cd38a9e4-f93a-b249-77dc-db998253e2f9}" +2025-03-20T03:50:23.871Z [ERROR] server-2585b7d3-2635-ea9d-6ada-0e0ed216a860: failed to get log: index=2 error="log not found" +2025-03-20T03:50:23.871Z [INFO] creating new snapshot: path=/tmp/raft2642728165/snapshots/2-15-1742442623871.tmp +2025-03-20T03:50:23.872Z [INFO] server-4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:23.872Z [INFO] server-4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f: copied to local snapshot: bytes=71 +2025-03-20T03:50:23.873Z [INFO] server-4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f: snapshot restore progress: id=2-15-1742442623871 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:23.873Z [INFO] server-4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f: removing all old logs from log store +2025-03-20T03:50:23.873Z [INFO] server-4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f: compacting logs: from=1 to=2 +2025-03-20T03:50:23.873Z [INFO] server-4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f: Installed remote snapshot +2025-03-20T03:50:23.873Z [INFO] server-2585b7d3-2635-ea9d-6ada-0e0ed216a860: pipelining replication: peer="{Voter server-4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f 4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f}" +2025-03-20T03:50:23.873Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:50:23.880Z [ERROR] server-2585b7d3-2635-ea9d-6ada-0e0ed216a860: failed to get log: index=2 error="log not found" +2025-03-20T03:50:23.881Z [INFO] creating new snapshot: path=/tmp/raft1276272923/snapshots/2-15-1742442623881.tmp +2025-03-20T03:50:23.881Z [INFO] server-cd38a9e4-f93a-b249-77dc-db998253e2f9: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:23.882Z [INFO] server-cd38a9e4-f93a-b249-77dc-db998253e2f9: copied to local snapshot: bytes=71 +2025-03-20T03:50:23.882Z [INFO] server-cd38a9e4-f93a-b249-77dc-db998253e2f9: snapshot restore progress: id=2-15-1742442623881 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:23.882Z [INFO] server-cd38a9e4-f93a-b249-77dc-db998253e2f9: removing all old logs from log store +2025-03-20T03:50:23.882Z [INFO] server-cd38a9e4-f93a-b249-77dc-db998253e2f9: compacting logs: from=1 to=2 +2025-03-20T03:50:23.882Z [INFO] server-cd38a9e4-f93a-b249-77dc-db998253e2f9: Installed remote snapshot +2025-03-20T03:50:23.882Z [INFO] server-2585b7d3-2635-ea9d-6ada-0e0ed216a860: pipelining replication: peer="{Voter server-cd38a9e4-f93a-b249-77dc-db998253e2f9 cd38a9e4-f93a-b249-77dc-db998253e2f9}" +2025-03-20T03:50:24.887Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:50:23.873610143 +0000 UTC m=+55.548062984 (2 nodes), 1.919µs from start of poll, 1.539517765s from cluster start. Timeout at 2025-03-20 03:50:24.878676276 +0000 UTC m=+56.553129339, 1.005066355s after stability +2025-03-20T03:50:24.934Z [INFO] server-b40cefb6-e851-7659-4cd6-ca016e864fdd: aborting pipeline replication: peer="{Voter server-cd5dc3e0-d8a8-0cb2-aca5-0038b0bc7979 cd5dc3e0-d8a8-0cb2-aca5-0038b0bc7979}" +2025-03-20T03:50:24.946Z [INFO] server-b40cefb6-e851-7659-4cd6-ca016e864fdd: aborting pipeline replication: peer="{Voter server-bfc455e1-c23e-c2f0-4b80-6992045f85b9 bfc455e1-c23e-c2f0-4b80-6992045f85b9}" +2025-03-20T03:50:25.482Z [ERROR] server-b40cefb6-e851-7659-4cd6-ca016e864fdd: failed to heartbeat to: peer=cd5dc3e0-d8a8-0cb2-aca5-0038b0bc7979 backoff time=10ms error="command timed out" === RUN TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_false -2026-04-22T10:10:10.284Z [INFO] server-6deb9e7b-cfc7-3256-b519-f4245a571e10: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6deb9e7b-cfc7-3256-b519-f4245a571e10 Address:6deb9e7b-cfc7-3256-b519-f4245a571e10} {Suffrage:Voter ID:server-9a2c530f-a845-e0a9-6b7c-390938e35697 Address:9a2c530f-a845-e0a9-6b7c-390938e35697} {Suffrage:Voter ID:server-7d14a275-45e0-08d8-9a64-3280fb0444e9 Address:7d14a275-45e0-08d8-9a64-3280fb0444e9}]" -2026-04-22T10:10:10.284Z [INFO] server-6deb9e7b-cfc7-3256-b519-f4245a571e10: entering follower state: follower="Node at 6deb9e7b-cfc7-3256-b519-f4245a571e10 [Follower]" leader-address= leader-id= -2026-04-22T10:10:10.284Z [INFO] server-9a2c530f-a845-e0a9-6b7c-390938e35697: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6deb9e7b-cfc7-3256-b519-f4245a571e10 Address:6deb9e7b-cfc7-3256-b519-f4245a571e10} {Suffrage:Voter ID:server-9a2c530f-a845-e0a9-6b7c-390938e35697 Address:9a2c530f-a845-e0a9-6b7c-390938e35697} {Suffrage:Voter ID:server-7d14a275-45e0-08d8-9a64-3280fb0444e9 Address:7d14a275-45e0-08d8-9a64-3280fb0444e9}]" -2026-04-22T10:10:10.284Z [INFO] server-7d14a275-45e0-08d8-9a64-3280fb0444e9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6deb9e7b-cfc7-3256-b519-f4245a571e10 Address:6deb9e7b-cfc7-3256-b519-f4245a571e10} {Suffrage:Voter ID:server-9a2c530f-a845-e0a9-6b7c-390938e35697 Address:9a2c530f-a845-e0a9-6b7c-390938e35697} {Suffrage:Voter ID:server-7d14a275-45e0-08d8-9a64-3280fb0444e9 Address:7d14a275-45e0-08d8-9a64-3280fb0444e9}]" -2026-04-22T10:10:10.284Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:10.284Z [INFO] server-9a2c530f-a845-e0a9-6b7c-390938e35697: entering follower state: follower="Node at 9a2c530f-a845-e0a9-6b7c-390938e35697 [Follower]" leader-address= leader-id= -2026-04-22T10:10:10.284Z [INFO] server-7d14a275-45e0-08d8-9a64-3280fb0444e9: entering follower state: follower="Node at 7d14a275-45e0-08d8-9a64-3280fb0444e9 [Follower]" leader-address= leader-id= -2026-04-22T10:10:10.876Z [WARN] server-6deb9e7b-cfc7-3256-b519-f4245a571e10: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:10.876Z [INFO] server-6deb9e7b-cfc7-3256-b519-f4245a571e10: entering candidate state: node="Node at 6deb9e7b-cfc7-3256-b519-f4245a571e10 [Candidate]" term=2 -2026-04-22T10:10:10.876Z [INFO] server-6deb9e7b-cfc7-3256-b519-f4245a571e10: election won: term=2 tally=2 -2026-04-22T10:10:10.876Z [INFO] server-6deb9e7b-cfc7-3256-b519-f4245a571e10: entering leader state: leader="Node at 6deb9e7b-cfc7-3256-b519-f4245a571e10 [Leader]" -2026-04-22T10:10:10.876Z [INFO] server-6deb9e7b-cfc7-3256-b519-f4245a571e10: added peer, starting replication: peer=server-9a2c530f-a845-e0a9-6b7c-390938e35697 -2026-04-22T10:10:10.876Z [INFO] server-6deb9e7b-cfc7-3256-b519-f4245a571e10: added peer, starting replication: peer=server-7d14a275-45e0-08d8-9a64-3280fb0444e9 -2026-04-22T10:10:10.876Z [INFO] server-6deb9e7b-cfc7-3256-b519-f4245a571e10: pipelining replication: peer="{Voter server-9a2c530f-a845-e0a9-6b7c-390938e35697 9a2c530f-a845-e0a9-6b7c-390938e35697}" -2026-04-22T10:10:10.876Z [INFO] server-6deb9e7b-cfc7-3256-b519-f4245a571e10: pipelining replication: peer="{Voter server-7d14a275-45e0-08d8-9a64-3280fb0444e9 7d14a275-45e0-08d8-9a64-3280fb0444e9}" -2026-04-22T10:10:11.882Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:10.876530042 +0000 UTC m=+56.900326133 (1 nodes), 592.14139ms from start of poll, 592.406649ms from cluster start. Timeout at 2026-04-22 10:10:11.881534196 +0000 UTC m=+57.905330278, 1.005004145s after stability -2026-04-22T10:10:11.882Z [INFO] server-6deb9e7b-cfc7-3256-b519-f4245a571e10: starting snapshot up to: index=12 -2026-04-22T10:10:11.882Z [INFO] creating new snapshot: path=/tmp/raft80188224/snapshots/2-12-1776852611882.tmp -2026-04-22T10:10:11.882Z [INFO] server-6deb9e7b-cfc7-3256-b519-f4245a571e10: snapshot complete up to: index=12 -2026-04-22T10:10:11.883Z [INFO] creating new snapshot: path=/tmp/raft80188224/snapshots/2-113-1776852611883.tmp -2026-04-22T10:10:11.883Z [INFO] server-6deb9e7b-cfc7-3256-b519-f4245a571e10: copied to local snapshot: bytes=71 -2026-04-22T10:10:11.884Z [INFO] server-6deb9e7b-cfc7-3256-b519-f4245a571e10: snapshot restore progress: id=2-113-1776852611883 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:11.884Z [INFO] server-6deb9e7b-cfc7-3256-b519-f4245a571e10: restored user snapshot: index=113 -2026-04-22T10:10:11.884Z [ERROR] server-6deb9e7b-cfc7-3256-b519-f4245a571e10: failed to get log: index=23 error="log not found" -2026-04-22T10:10:11.884Z [INFO] server-6deb9e7b-cfc7-3256-b519-f4245a571e10: aborting pipeline replication: peer="{Voter server-7d14a275-45e0-08d8-9a64-3280fb0444e9 7d14a275-45e0-08d8-9a64-3280fb0444e9}" -2026-04-22T10:10:11.884Z [ERROR] server-6deb9e7b-cfc7-3256-b519-f4245a571e10: failed to get log: index=23 error="log not found" -2026-04-22T10:10:11.884Z [INFO] server-6deb9e7b-cfc7-3256-b519-f4245a571e10: aborting pipeline replication: peer="{Voter server-9a2c530f-a845-e0a9-6b7c-390938e35697 9a2c530f-a845-e0a9-6b7c-390938e35697}" -2026-04-22T10:10:11.891Z [ERROR] server-6deb9e7b-cfc7-3256-b519-f4245a571e10: failed to get log: index=23 error="log not found" -2026-04-22T10:10:11.892Z [ERROR] server-6deb9e7b-cfc7-3256-b519-f4245a571e10: failed to get log: index=23 error="log not found" -2026-04-22T10:10:11.892Z [INFO] creating new snapshot: path=/tmp/raft1946938039/snapshots/2-113-1776852611892.tmp -2026-04-22T10:10:11.892Z [INFO] server-9a2c530f-a845-e0a9-6b7c-390938e35697: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:11.892Z [INFO] creating new snapshot: path=/tmp/raft4196342661/snapshots/2-113-1776852611892.tmp -2026-04-22T10:10:11.892Z [INFO] server-7d14a275-45e0-08d8-9a64-3280fb0444e9: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:11.892Z [INFO] server-9a2c530f-a845-e0a9-6b7c-390938e35697: copied to local snapshot: bytes=71 -2026-04-22T10:10:11.893Z [INFO] server-7d14a275-45e0-08d8-9a64-3280fb0444e9: copied to local snapshot: bytes=71 -2026-04-22T10:10:11.893Z [INFO] server-9a2c530f-a845-e0a9-6b7c-390938e35697: snapshot restore progress: id=2-113-1776852611892 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:11.893Z [INFO] server-7d14a275-45e0-08d8-9a64-3280fb0444e9: snapshot restore progress: id=2-113-1776852611892 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:11.893Z [INFO] server-9a2c530f-a845-e0a9-6b7c-390938e35697: Installed remote snapshot -2026-04-22T10:10:11.893Z [INFO] server-7d14a275-45e0-08d8-9a64-3280fb0444e9: Installed remote snapshot -2026-04-22T10:10:11.893Z [WARN] server-7d14a275-45e0-08d8-9a64-3280fb0444e9: skipping application of old log: index=22 -2026-04-22T10:10:11.893Z [WARN] server-9a2c530f-a845-e0a9-6b7c-390938e35697: skipping application of old log: index=113 -2026-04-22T10:10:11.893Z [INFO] server-6deb9e7b-cfc7-3256-b519-f4245a571e10: pipelining replication: peer="{Voter server-7d14a275-45e0-08d8-9a64-3280fb0444e9 7d14a275-45e0-08d8-9a64-3280fb0444e9}" -2026-04-22T10:10:11.893Z [INFO] server-6deb9e7b-cfc7-3256-b519-f4245a571e10: pipelining replication: peer="{Voter server-9a2c530f-a845-e0a9-6b7c-390938e35697 9a2c530f-a845-e0a9-6b7c-390938e35697}" -2026-04-22T10:10:11.893Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:10:12.899Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:10:11.893367689 +0000 UTC m=+57.917163789 (2 nodes), 661ns from start of poll, 1.609244305s from cluster start. Timeout at 2026-04-22 10:10:12.898377136 +0000 UTC m=+58.922173326, 1.005009537s after stability -2026-04-22T10:10:12.910Z [INFO] server-6deb9e7b-cfc7-3256-b519-f4245a571e10: aborting pipeline replication: peer="{Voter server-7d14a275-45e0-08d8-9a64-3280fb0444e9 7d14a275-45e0-08d8-9a64-3280fb0444e9}" -2026-04-22T10:10:12.910Z [INFO] server-6deb9e7b-cfc7-3256-b519-f4245a571e10: aborting pipeline replication: peer="{Voter server-9a2c530f-a845-e0a9-6b7c-390938e35697 9a2c530f-a845-e0a9-6b7c-390938e35697}" +2025-03-20T03:50:25.554Z [INFO] server-99675687-de75-dd49-22c4-188300c11019: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-99675687-de75-dd49-22c4-188300c11019 Address:99675687-de75-dd49-22c4-188300c11019} {Suffrage:Voter ID:server-0a2f1a27-4e24-37a8-33be-7c21b8cab5d3 Address:0a2f1a27-4e24-37a8-33be-7c21b8cab5d3} {Suffrage:Voter ID:server-49bdad95-c367-f8ee-d27a-49eed1a059b8 Address:49bdad95-c367-f8ee-d27a-49eed1a059b8}]" +2025-03-20T03:50:25.555Z [INFO] server-99675687-de75-dd49-22c4-188300c11019: entering follower state: follower="Node at 99675687-de75-dd49-22c4-188300c11019 [Follower]" leader-address= leader-id= +2025-03-20T03:50:25.574Z [INFO] server-0a2f1a27-4e24-37a8-33be-7c21b8cab5d3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-99675687-de75-dd49-22c4-188300c11019 Address:99675687-de75-dd49-22c4-188300c11019} {Suffrage:Voter ID:server-0a2f1a27-4e24-37a8-33be-7c21b8cab5d3 Address:0a2f1a27-4e24-37a8-33be-7c21b8cab5d3} {Suffrage:Voter ID:server-49bdad95-c367-f8ee-d27a-49eed1a059b8 Address:49bdad95-c367-f8ee-d27a-49eed1a059b8}]" +2025-03-20T03:50:25.574Z [INFO] server-0a2f1a27-4e24-37a8-33be-7c21b8cab5d3: entering follower state: follower="Node at 0a2f1a27-4e24-37a8-33be-7c21b8cab5d3 [Follower]" leader-address= leader-id= +2025-03-20T03:50:25.574Z [INFO] server-49bdad95-c367-f8ee-d27a-49eed1a059b8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-99675687-de75-dd49-22c4-188300c11019 Address:99675687-de75-dd49-22c4-188300c11019} {Suffrage:Voter ID:server-0a2f1a27-4e24-37a8-33be-7c21b8cab5d3 Address:0a2f1a27-4e24-37a8-33be-7c21b8cab5d3} {Suffrage:Voter ID:server-49bdad95-c367-f8ee-d27a-49eed1a059b8 Address:49bdad95-c367-f8ee-d27a-49eed1a059b8}]" +2025-03-20T03:50:25.581Z [INFO] server-49bdad95-c367-f8ee-d27a-49eed1a059b8: entering follower state: follower="Node at 49bdad95-c367-f8ee-d27a-49eed1a059b8 [Follower]" leader-address= leader-id= +2025-03-20T03:50:25.583Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:50:26.082Z [WARN] server-0a2f1a27-4e24-37a8-33be-7c21b8cab5d3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:50:26.082Z [INFO] server-0a2f1a27-4e24-37a8-33be-7c21b8cab5d3: entering candidate state: node="Node at 0a2f1a27-4e24-37a8-33be-7c21b8cab5d3 [Candidate]" term=2 +2025-03-20T03:50:26.084Z [INFO] server-0a2f1a27-4e24-37a8-33be-7c21b8cab5d3: election won: term=2 tally=2 +2025-03-20T03:50:26.084Z [INFO] server-0a2f1a27-4e24-37a8-33be-7c21b8cab5d3: entering leader state: leader="Node at 0a2f1a27-4e24-37a8-33be-7c21b8cab5d3 [Leader]" +2025-03-20T03:50:26.084Z [INFO] server-0a2f1a27-4e24-37a8-33be-7c21b8cab5d3: added peer, starting replication: peer=server-99675687-de75-dd49-22c4-188300c11019 +2025-03-20T03:50:26.084Z [INFO] server-0a2f1a27-4e24-37a8-33be-7c21b8cab5d3: added peer, starting replication: peer=server-49bdad95-c367-f8ee-d27a-49eed1a059b8 +2025-03-20T03:50:26.084Z [INFO] server-0a2f1a27-4e24-37a8-33be-7c21b8cab5d3: pipelining replication: peer="{Voter server-49bdad95-c367-f8ee-d27a-49eed1a059b8 49bdad95-c367-f8ee-d27a-49eed1a059b8}" +2025-03-20T03:50:26.085Z [INFO] server-0a2f1a27-4e24-37a8-33be-7c21b8cab5d3: pipelining replication: peer="{Voter server-99675687-de75-dd49-22c4-188300c11019 99675687-de75-dd49-22c4-188300c11019}" +2025-03-20T03:50:27.095Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:50:26.089536136 +0000 UTC m=+57.763988948 (1 nodes), 505.50283ms from start of poll, 536.614788ms from cluster start. Timeout at 2025-03-20 03:50:27.094541099 +0000 UTC m=+58.768994459, 1.005005511s after stability +2025-03-20T03:50:27.110Z [INFO] server-0a2f1a27-4e24-37a8-33be-7c21b8cab5d3: starting snapshot up to: index=12 +2025-03-20T03:50:27.110Z [INFO] creating new snapshot: path=/tmp/raft3012684618/snapshots/2-12-1742442627110.tmp +2025-03-20T03:50:27.118Z [INFO] server-0a2f1a27-4e24-37a8-33be-7c21b8cab5d3: snapshot complete up to: index=12 +2025-03-20T03:50:27.141Z [INFO] creating new snapshot: path=/tmp/raft3012684618/snapshots/2-113-1742442627141.tmp +2025-03-20T03:50:27.142Z [INFO] server-0a2f1a27-4e24-37a8-33be-7c21b8cab5d3: copied to local snapshot: bytes=71 +2025-03-20T03:50:27.143Z [INFO] server-0a2f1a27-4e24-37a8-33be-7c21b8cab5d3: snapshot restore progress: id=2-113-1742442627141 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:27.143Z [INFO] server-0a2f1a27-4e24-37a8-33be-7c21b8cab5d3: restored user snapshot: index=113 +2025-03-20T03:50:27.143Z [ERROR] server-0a2f1a27-4e24-37a8-33be-7c21b8cab5d3: failed to get log: index=23 error="log not found" +2025-03-20T03:50:27.143Z [INFO] server-0a2f1a27-4e24-37a8-33be-7c21b8cab5d3: aborting pipeline replication: peer="{Voter server-49bdad95-c367-f8ee-d27a-49eed1a059b8 49bdad95-c367-f8ee-d27a-49eed1a059b8}" +2025-03-20T03:50:27.143Z [ERROR] server-0a2f1a27-4e24-37a8-33be-7c21b8cab5d3: failed to get log: index=23 error="log not found" +2025-03-20T03:50:27.143Z [INFO] server-0a2f1a27-4e24-37a8-33be-7c21b8cab5d3: aborting pipeline replication: peer="{Voter server-99675687-de75-dd49-22c4-188300c11019 99675687-de75-dd49-22c4-188300c11019}" +2025-03-20T03:50:27.150Z [ERROR] server-0a2f1a27-4e24-37a8-33be-7c21b8cab5d3: failed to get log: index=23 error="log not found" +2025-03-20T03:50:27.151Z [INFO] creating new snapshot: path=/tmp/raft1867248614/snapshots/2-113-1742442627151.tmp +2025-03-20T03:50:27.151Z [INFO] server-99675687-de75-dd49-22c4-188300c11019: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:27.152Z [ERROR] server-0a2f1a27-4e24-37a8-33be-7c21b8cab5d3: failed to get log: index=23 error="log not found" +2025-03-20T03:50:27.152Z [INFO] server-99675687-de75-dd49-22c4-188300c11019: copied to local snapshot: bytes=71 +2025-03-20T03:50:27.152Z [INFO] creating new snapshot: path=/tmp/raft3595940051/snapshots/2-113-1742442627152.tmp +2025-03-20T03:50:27.152Z [INFO] server-99675687-de75-dd49-22c4-188300c11019: snapshot restore progress: id=2-113-1742442627151 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:27.152Z [INFO] server-99675687-de75-dd49-22c4-188300c11019: Installed remote snapshot +2025-03-20T03:50:27.153Z [INFO] server-0a2f1a27-4e24-37a8-33be-7c21b8cab5d3: pipelining replication: peer="{Voter server-99675687-de75-dd49-22c4-188300c11019 99675687-de75-dd49-22c4-188300c11019}" +2025-03-20T03:50:27.153Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:50:27.153Z [INFO] server-49bdad95-c367-f8ee-d27a-49eed1a059b8: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:27.153Z [INFO] server-49bdad95-c367-f8ee-d27a-49eed1a059b8: copied to local snapshot: bytes=71 +2025-03-20T03:50:27.154Z [INFO] server-49bdad95-c367-f8ee-d27a-49eed1a059b8: snapshot restore progress: id=2-113-1742442627152 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:27.154Z [INFO] server-49bdad95-c367-f8ee-d27a-49eed1a059b8: Installed remote snapshot +2025-03-20T03:50:27.155Z [INFO] server-0a2f1a27-4e24-37a8-33be-7c21b8cab5d3: pipelining replication: peer="{Voter server-49bdad95-c367-f8ee-d27a-49eed1a059b8 49bdad95-c367-f8ee-d27a-49eed1a059b8}" +2025-03-20T03:50:28.158Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:50:27.153252901 +0000 UTC m=+58.827705690 (2 nodes), 2.226µs from start of poll, 1.60033153s from cluster start. Timeout at 2025-03-20 03:50:28.158264712 +0000 UTC m=+59.832717527, 1.005011837s after stability +2025-03-20T03:50:28.175Z [INFO] server-0a2f1a27-4e24-37a8-33be-7c21b8cab5d3: aborting pipeline replication: peer="{Voter server-49bdad95-c367-f8ee-d27a-49eed1a059b8 49bdad95-c367-f8ee-d27a-49eed1a059b8}" +2025-03-20T03:50:28.175Z [INFO] server-0a2f1a27-4e24-37a8-33be-7c21b8cab5d3: aborting pipeline replication: peer="{Voter server-99675687-de75-dd49-22c4-188300c11019 99675687-de75-dd49-22c4-188300c11019}" === RUN TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_false -2026-04-22T10:10:12.911Z [INFO] server-e13c70ee-7e16-bab4-8eef-e4b69e0fb9fe: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e13c70ee-7e16-bab4-8eef-e4b69e0fb9fe Address:e13c70ee-7e16-bab4-8eef-e4b69e0fb9fe} {Suffrage:Voter ID:server-a95198fd-9b56-6a10-b4f3-9921c1a451f7 Address:a95198fd-9b56-6a10-b4f3-9921c1a451f7} {Suffrage:Voter ID:server-6fdfb9b0-1513-6fff-c565-d2c7ffc713ba Address:6fdfb9b0-1513-6fff-c565-d2c7ffc713ba}]" -2026-04-22T10:10:12.911Z [INFO] server-e13c70ee-7e16-bab4-8eef-e4b69e0fb9fe: entering follower state: follower="Node at e13c70ee-7e16-bab4-8eef-e4b69e0fb9fe [Follower]" leader-address= leader-id= -2026-04-22T10:10:12.912Z [INFO] server-a95198fd-9b56-6a10-b4f3-9921c1a451f7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e13c70ee-7e16-bab4-8eef-e4b69e0fb9fe Address:e13c70ee-7e16-bab4-8eef-e4b69e0fb9fe} {Suffrage:Voter ID:server-a95198fd-9b56-6a10-b4f3-9921c1a451f7 Address:a95198fd-9b56-6a10-b4f3-9921c1a451f7} {Suffrage:Voter ID:server-6fdfb9b0-1513-6fff-c565-d2c7ffc713ba Address:6fdfb9b0-1513-6fff-c565-d2c7ffc713ba}]" -2026-04-22T10:10:12.912Z [INFO] server-6fdfb9b0-1513-6fff-c565-d2c7ffc713ba: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e13c70ee-7e16-bab4-8eef-e4b69e0fb9fe Address:e13c70ee-7e16-bab4-8eef-e4b69e0fb9fe} {Suffrage:Voter ID:server-a95198fd-9b56-6a10-b4f3-9921c1a451f7 Address:a95198fd-9b56-6a10-b4f3-9921c1a451f7} {Suffrage:Voter ID:server-6fdfb9b0-1513-6fff-c565-d2c7ffc713ba Address:6fdfb9b0-1513-6fff-c565-d2c7ffc713ba}]" -2026-04-22T10:10:12.912Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:12.912Z [INFO] server-a95198fd-9b56-6a10-b4f3-9921c1a451f7: entering follower state: follower="Node at a95198fd-9b56-6a10-b4f3-9921c1a451f7 [Follower]" leader-address= leader-id= -2026-04-22T10:10:12.912Z [INFO] server-6fdfb9b0-1513-6fff-c565-d2c7ffc713ba: entering follower state: follower="Node at 6fdfb9b0-1513-6fff-c565-d2c7ffc713ba [Follower]" leader-address= leader-id= -2026-04-22T10:10:13.452Z [WARN] server-6fdfb9b0-1513-6fff-c565-d2c7ffc713ba: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:13.452Z [INFO] server-6fdfb9b0-1513-6fff-c565-d2c7ffc713ba: entering candidate state: node="Node at 6fdfb9b0-1513-6fff-c565-d2c7ffc713ba [Candidate]" term=2 -2026-04-22T10:10:13.452Z [INFO] server-6fdfb9b0-1513-6fff-c565-d2c7ffc713ba: election won: term=2 tally=2 -2026-04-22T10:10:13.452Z [INFO] server-6fdfb9b0-1513-6fff-c565-d2c7ffc713ba: entering leader state: leader="Node at 6fdfb9b0-1513-6fff-c565-d2c7ffc713ba [Leader]" -2026-04-22T10:10:13.452Z [INFO] server-6fdfb9b0-1513-6fff-c565-d2c7ffc713ba: added peer, starting replication: peer=server-e13c70ee-7e16-bab4-8eef-e4b69e0fb9fe -2026-04-22T10:10:13.452Z [INFO] server-6fdfb9b0-1513-6fff-c565-d2c7ffc713ba: added peer, starting replication: peer=server-a95198fd-9b56-6a10-b4f3-9921c1a451f7 -2026-04-22T10:10:13.452Z [INFO] server-6fdfb9b0-1513-6fff-c565-d2c7ffc713ba: pipelining replication: peer="{Voter server-e13c70ee-7e16-bab4-8eef-e4b69e0fb9fe e13c70ee-7e16-bab4-8eef-e4b69e0fb9fe}" -2026-04-22T10:10:13.452Z [INFO] server-6fdfb9b0-1513-6fff-c565-d2c7ffc713ba: pipelining replication: peer="{Voter server-a95198fd-9b56-6a10-b4f3-9921c1a451f7 a95198fd-9b56-6a10-b4f3-9921c1a451f7}" -2026-04-22T10:10:14.459Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:13.452845554 +0000 UTC m=+59.476641645 (1 nodes), 540.724399ms from start of poll, 540.992188ms from cluster start. Timeout at 2026-04-22 10:10:14.457853617 +0000 UTC m=+60.481649754, 1.005008109s after stability -2026-04-22T10:10:14.460Z [INFO] server-6fdfb9b0-1513-6fff-c565-d2c7ffc713ba: starting snapshot up to: index=12 -2026-04-22T10:10:14.460Z [INFO] creating new snapshot: path=/tmp/raft343419281/snapshots/2-12-1776852614460.tmp -2026-04-22T10:10:14.460Z [INFO] server-6fdfb9b0-1513-6fff-c565-d2c7ffc713ba: snapshot complete up to: index=12 -2026-04-22T10:10:14.461Z [INFO] creating new snapshot: path=/tmp/raft343419281/snapshots/2-113-1776852614461.tmp -2026-04-22T10:10:14.461Z [INFO] server-6fdfb9b0-1513-6fff-c565-d2c7ffc713ba: copied to local snapshot: bytes=71 -2026-04-22T10:10:14.462Z [INFO] server-6fdfb9b0-1513-6fff-c565-d2c7ffc713ba: snapshot restore progress: id=2-113-1776852614461 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:14.462Z [INFO] server-6fdfb9b0-1513-6fff-c565-d2c7ffc713ba: removing all old logs from log store -2026-04-22T10:10:14.462Z [INFO] server-6fdfb9b0-1513-6fff-c565-d2c7ffc713ba: compacting logs: from=1 to=22 -2026-04-22T10:10:14.462Z [INFO] server-6fdfb9b0-1513-6fff-c565-d2c7ffc713ba: restored user snapshot: index=113 -2026-04-22T10:10:14.462Z [ERROR] server-6fdfb9b0-1513-6fff-c565-d2c7ffc713ba: failed to get log: index=22 error="log not found" -2026-04-22T10:10:14.462Z [INFO] server-6fdfb9b0-1513-6fff-c565-d2c7ffc713ba: aborting pipeline replication: peer="{Voter server-e13c70ee-7e16-bab4-8eef-e4b69e0fb9fe e13c70ee-7e16-bab4-8eef-e4b69e0fb9fe}" -2026-04-22T10:10:14.462Z [ERROR] server-6fdfb9b0-1513-6fff-c565-d2c7ffc713ba: failed to get log: index=22 error="log not found" -2026-04-22T10:10:14.462Z [INFO] server-6fdfb9b0-1513-6fff-c565-d2c7ffc713ba: aborting pipeline replication: peer="{Voter server-a95198fd-9b56-6a10-b4f3-9921c1a451f7 a95198fd-9b56-6a10-b4f3-9921c1a451f7}" -2026-04-22T10:10:14.468Z [ERROR] server-6fdfb9b0-1513-6fff-c565-d2c7ffc713ba: failed to get log: index=22 error="log not found" -2026-04-22T10:10:14.469Z [INFO] creating new snapshot: path=/tmp/raft2687155162/snapshots/2-113-1776852614469.tmp -2026-04-22T10:10:14.469Z [INFO] server-e13c70ee-7e16-bab4-8eef-e4b69e0fb9fe: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:14.469Z [INFO] server-e13c70ee-7e16-bab4-8eef-e4b69e0fb9fe: copied to local snapshot: bytes=71 -2026-04-22T10:10:14.469Z [INFO] server-e13c70ee-7e16-bab4-8eef-e4b69e0fb9fe: snapshot restore progress: id=2-113-1776852614469 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:14.469Z [INFO] server-e13c70ee-7e16-bab4-8eef-e4b69e0fb9fe: removing all old logs from log store -2026-04-22T10:10:14.469Z [INFO] server-e13c70ee-7e16-bab4-8eef-e4b69e0fb9fe: compacting logs: from=1 to=22 -2026-04-22T10:10:14.470Z [INFO] server-e13c70ee-7e16-bab4-8eef-e4b69e0fb9fe: Installed remote snapshot -2026-04-22T10:10:14.470Z [WARN] server-e13c70ee-7e16-bab4-8eef-e4b69e0fb9fe: skipping application of old log: index=22 -2026-04-22T10:10:14.470Z [INFO] server-6fdfb9b0-1513-6fff-c565-d2c7ffc713ba: pipelining replication: peer="{Voter server-e13c70ee-7e16-bab4-8eef-e4b69e0fb9fe e13c70ee-7e16-bab4-8eef-e4b69e0fb9fe}" -2026-04-22T10:10:14.470Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:10:14.471Z [ERROR] server-6fdfb9b0-1513-6fff-c565-d2c7ffc713ba: failed to get log: index=22 error="log not found" -2026-04-22T10:10:14.471Z [INFO] creating new snapshot: path=/tmp/raft758770256/snapshots/2-113-1776852614471.tmp -2026-04-22T10:10:14.471Z [INFO] server-a95198fd-9b56-6a10-b4f3-9921c1a451f7: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:14.472Z [INFO] server-a95198fd-9b56-6a10-b4f3-9921c1a451f7: copied to local snapshot: bytes=71 -2026-04-22T10:10:14.472Z [INFO] server-a95198fd-9b56-6a10-b4f3-9921c1a451f7: snapshot restore progress: id=2-113-1776852614471 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:14.472Z [INFO] server-a95198fd-9b56-6a10-b4f3-9921c1a451f7: removing all old logs from log store -2026-04-22T10:10:14.472Z [INFO] server-a95198fd-9b56-6a10-b4f3-9921c1a451f7: compacting logs: from=1 to=22 -2026-04-22T10:10:14.472Z [INFO] server-a95198fd-9b56-6a10-b4f3-9921c1a451f7: Installed remote snapshot -2026-04-22T10:10:14.472Z [INFO] server-6fdfb9b0-1513-6fff-c565-d2c7ffc713ba: pipelining replication: peer="{Voter server-a95198fd-9b56-6a10-b4f3-9921c1a451f7 a95198fd-9b56-6a10-b4f3-9921c1a451f7}" -2026-04-22T10:10:15.476Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:10:14.470201545 +0000 UTC m=+60.493997646 (2 nodes), 834ns from start of poll, 1.558348189s from cluster start. Timeout at 2026-04-22 10:10:15.475221414 +0000 UTC m=+61.499017551, 1.005019905s after stability -2026-04-22T10:10:15.488Z [INFO] server-6fdfb9b0-1513-6fff-c565-d2c7ffc713ba: aborting pipeline replication: peer="{Voter server-a95198fd-9b56-6a10-b4f3-9921c1a451f7 a95198fd-9b56-6a10-b4f3-9921c1a451f7}" -2026-04-22T10:10:15.488Z [INFO] server-6fdfb9b0-1513-6fff-c565-d2c7ffc713ba: aborting pipeline replication: peer="{Voter server-e13c70ee-7e16-bab4-8eef-e4b69e0fb9fe e13c70ee-7e16-bab4-8eef-e4b69e0fb9fe}" +2025-03-20T03:50:28.179Z [INFO] server-f27e94bd-3a17-3d87-930c-81661070b232: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f27e94bd-3a17-3d87-930c-81661070b232 Address:f27e94bd-3a17-3d87-930c-81661070b232} {Suffrage:Voter ID:server-5ec4c8ff-462a-0413-b7bc-7f0541805e20 Address:5ec4c8ff-462a-0413-b7bc-7f0541805e20} {Suffrage:Voter ID:server-2fcd6d21-cc82-177c-c44d-92cd0f790da4 Address:2fcd6d21-cc82-177c-c44d-92cd0f790da4}]" +2025-03-20T03:50:28.179Z [INFO] server-5ec4c8ff-462a-0413-b7bc-7f0541805e20: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f27e94bd-3a17-3d87-930c-81661070b232 Address:f27e94bd-3a17-3d87-930c-81661070b232} {Suffrage:Voter ID:server-5ec4c8ff-462a-0413-b7bc-7f0541805e20 Address:5ec4c8ff-462a-0413-b7bc-7f0541805e20} {Suffrage:Voter ID:server-2fcd6d21-cc82-177c-c44d-92cd0f790da4 Address:2fcd6d21-cc82-177c-c44d-92cd0f790da4}]" +2025-03-20T03:50:28.179Z [INFO] server-2fcd6d21-cc82-177c-c44d-92cd0f790da4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f27e94bd-3a17-3d87-930c-81661070b232 Address:f27e94bd-3a17-3d87-930c-81661070b232} {Suffrage:Voter ID:server-5ec4c8ff-462a-0413-b7bc-7f0541805e20 Address:5ec4c8ff-462a-0413-b7bc-7f0541805e20} {Suffrage:Voter ID:server-2fcd6d21-cc82-177c-c44d-92cd0f790da4 Address:2fcd6d21-cc82-177c-c44d-92cd0f790da4}]" +2025-03-20T03:50:28.179Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:50:28.179Z [INFO] server-f27e94bd-3a17-3d87-930c-81661070b232: entering follower state: follower="Node at f27e94bd-3a17-3d87-930c-81661070b232 [Follower]" leader-address= leader-id= +2025-03-20T03:50:28.179Z [INFO] server-5ec4c8ff-462a-0413-b7bc-7f0541805e20: entering follower state: follower="Node at 5ec4c8ff-462a-0413-b7bc-7f0541805e20 [Follower]" leader-address= leader-id= +2025-03-20T03:50:28.179Z [INFO] server-2fcd6d21-cc82-177c-c44d-92cd0f790da4: entering follower state: follower="Node at 2fcd6d21-cc82-177c-c44d-92cd0f790da4 [Follower]" leader-address= leader-id= +2025-03-20T03:50:28.698Z [WARN] server-2fcd6d21-cc82-177c-c44d-92cd0f790da4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:50:28.699Z [INFO] server-2fcd6d21-cc82-177c-c44d-92cd0f790da4: entering candidate state: node="Node at 2fcd6d21-cc82-177c-c44d-92cd0f790da4 [Candidate]" term=2 +2025-03-20T03:50:28.700Z [INFO] server-2fcd6d21-cc82-177c-c44d-92cd0f790da4: election won: term=2 tally=2 +2025-03-20T03:50:28.700Z [INFO] server-2fcd6d21-cc82-177c-c44d-92cd0f790da4: entering leader state: leader="Node at 2fcd6d21-cc82-177c-c44d-92cd0f790da4 [Leader]" +2025-03-20T03:50:28.700Z [INFO] server-2fcd6d21-cc82-177c-c44d-92cd0f790da4: added peer, starting replication: peer=server-f27e94bd-3a17-3d87-930c-81661070b232 +2025-03-20T03:50:28.700Z [INFO] server-2fcd6d21-cc82-177c-c44d-92cd0f790da4: added peer, starting replication: peer=server-5ec4c8ff-462a-0413-b7bc-7f0541805e20 +2025-03-20T03:50:28.700Z [INFO] server-2fcd6d21-cc82-177c-c44d-92cd0f790da4: pipelining replication: peer="{Voter server-5ec4c8ff-462a-0413-b7bc-7f0541805e20 5ec4c8ff-462a-0413-b7bc-7f0541805e20}" +2025-03-20T03:50:28.700Z [INFO] server-2fcd6d21-cc82-177c-c44d-92cd0f790da4: pipelining replication: peer="{Voter server-f27e94bd-3a17-3d87-930c-81661070b232 f27e94bd-3a17-3d87-930c-81661070b232}" +2025-03-20T03:50:29.706Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:50:28.700690581 +0000 UTC m=+60.375143448 (1 nodes), 521.088713ms from start of poll, 521.637922ms from cluster start. Timeout at 2025-03-20 03:50:29.705697986 +0000 UTC m=+61.380151280, 1.005007832s after stability +2025-03-20T03:50:29.708Z [INFO] server-2fcd6d21-cc82-177c-c44d-92cd0f790da4: starting snapshot up to: index=12 +2025-03-20T03:50:29.708Z [INFO] creating new snapshot: path=/tmp/raft4205723962/snapshots/2-12-1742442629708.tmp +2025-03-20T03:50:29.714Z [INFO] server-2fcd6d21-cc82-177c-c44d-92cd0f790da4: snapshot complete up to: index=12 +2025-03-20T03:50:29.716Z [INFO] creating new snapshot: path=/tmp/raft4205723962/snapshots/2-113-1742442629716.tmp +2025-03-20T03:50:29.718Z [INFO] server-2fcd6d21-cc82-177c-c44d-92cd0f790da4: copied to local snapshot: bytes=71 +2025-03-20T03:50:29.718Z [INFO] server-2fcd6d21-cc82-177c-c44d-92cd0f790da4: snapshot restore progress: id=2-113-1742442629716 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:29.718Z [INFO] server-2fcd6d21-cc82-177c-c44d-92cd0f790da4: removing all old logs from log store +2025-03-20T03:50:29.718Z [INFO] server-2fcd6d21-cc82-177c-c44d-92cd0f790da4: compacting logs: from=1 to=22 +2025-03-20T03:50:29.718Z [INFO] server-2fcd6d21-cc82-177c-c44d-92cd0f790da4: restored user snapshot: index=113 +2025-03-20T03:50:29.718Z [ERROR] server-2fcd6d21-cc82-177c-c44d-92cd0f790da4: failed to get log: index=22 error="log not found" +2025-03-20T03:50:29.718Z [INFO] server-2fcd6d21-cc82-177c-c44d-92cd0f790da4: aborting pipeline replication: peer="{Voter server-5ec4c8ff-462a-0413-b7bc-7f0541805e20 5ec4c8ff-462a-0413-b7bc-7f0541805e20}" +2025-03-20T03:50:29.718Z [ERROR] server-2fcd6d21-cc82-177c-c44d-92cd0f790da4: failed to get log: index=22 error="log not found" +2025-03-20T03:50:29.718Z [INFO] server-2fcd6d21-cc82-177c-c44d-92cd0f790da4: aborting pipeline replication: peer="{Voter server-f27e94bd-3a17-3d87-930c-81661070b232 f27e94bd-3a17-3d87-930c-81661070b232}" +2025-03-20T03:50:29.726Z [ERROR] server-2fcd6d21-cc82-177c-c44d-92cd0f790da4: failed to get log: index=22 error="log not found" +2025-03-20T03:50:29.727Z [ERROR] server-2fcd6d21-cc82-177c-c44d-92cd0f790da4: failed to get log: index=22 error="log not found" +2025-03-20T03:50:29.727Z [INFO] creating new snapshot: path=/tmp/raft3488503481/snapshots/2-113-1742442629727.tmp +2025-03-20T03:50:29.727Z [INFO] server-f27e94bd-3a17-3d87-930c-81661070b232: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:29.728Z [INFO] server-f27e94bd-3a17-3d87-930c-81661070b232: copied to local snapshot: bytes=71 +2025-03-20T03:50:29.728Z [INFO] server-f27e94bd-3a17-3d87-930c-81661070b232: snapshot restore progress: id=2-113-1742442629727 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:29.728Z [INFO] server-f27e94bd-3a17-3d87-930c-81661070b232: removing all old logs from log store +2025-03-20T03:50:29.728Z [INFO] server-f27e94bd-3a17-3d87-930c-81661070b232: compacting logs: from=1 to=22 +2025-03-20T03:50:29.728Z [INFO] server-f27e94bd-3a17-3d87-930c-81661070b232: Installed remote snapshot +2025-03-20T03:50:29.729Z [WARN] server-f27e94bd-3a17-3d87-930c-81661070b232: skipping application of old log: index=22 +2025-03-20T03:50:29.729Z [INFO] server-2fcd6d21-cc82-177c-c44d-92cd0f790da4: pipelining replication: peer="{Voter server-f27e94bd-3a17-3d87-930c-81661070b232 f27e94bd-3a17-3d87-930c-81661070b232}" +2025-03-20T03:50:29.729Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:50:29.729Z [INFO] creating new snapshot: path=/tmp/raft1225549121/snapshots/2-113-1742442629729.tmp +2025-03-20T03:50:29.730Z [INFO] server-5ec4c8ff-462a-0413-b7bc-7f0541805e20: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:29.731Z [INFO] server-5ec4c8ff-462a-0413-b7bc-7f0541805e20: copied to local snapshot: bytes=71 +2025-03-20T03:50:29.732Z [INFO] server-5ec4c8ff-462a-0413-b7bc-7f0541805e20: snapshot restore progress: id=2-113-1742442629729 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:29.732Z [INFO] server-5ec4c8ff-462a-0413-b7bc-7f0541805e20: removing all old logs from log store +2025-03-20T03:50:29.732Z [INFO] server-5ec4c8ff-462a-0413-b7bc-7f0541805e20: compacting logs: from=1 to=22 +2025-03-20T03:50:29.732Z [INFO] server-5ec4c8ff-462a-0413-b7bc-7f0541805e20: Installed remote snapshot +2025-03-20T03:50:29.732Z [INFO] server-2fcd6d21-cc82-177c-c44d-92cd0f790da4: pipelining replication: peer="{Voter server-5ec4c8ff-462a-0413-b7bc-7f0541805e20 5ec4c8ff-462a-0413-b7bc-7f0541805e20}" +2025-03-20T03:50:31.036Z [ERROR] server-4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2: failed to heartbeat to: peer=3ce2100e-ff0b-72f8-eee7-317c57f72604 backoff time=10ms error="command timed out" +2025-03-20T03:50:31.194Z [WARN] server-80d16e54-3c85-3681-2e34-3b985620b4fa: heartbeat timeout reached, starting election: last-leader-addr=c4426023-874b-2f23-8d3e-11417740bf2d last-leader-id=server-c4426023-874b-2f23-8d3e-11417740bf2d +2025-03-20T03:50:31.194Z [INFO] server-80d16e54-3c85-3681-2e34-3b985620b4fa: entering candidate state: node="Node at 80d16e54-3c85-3681-2e34-3b985620b4fa [Candidate]" term=3 +2025-03-20T03:50:31.220Z [WARN] server-4588dbe5-3659-fdb6-7992-ea346fa55b28: heartbeat timeout reached, starting election: last-leader-addr=29f478ab-7a47-a793-ea2c-01b0bd11c7a5 last-leader-id=server-29f478ab-7a47-a793-ea2c-01b0bd11c7a5 +2025-03-20T03:50:31.220Z [INFO] server-4588dbe5-3659-fdb6-7992-ea346fa55b28: entering candidate state: node="Node at 4588dbe5-3659-fdb6-7992-ea346fa55b28 [Candidate]" term=3 +2025-03-20T03:50:31.370Z [WARN] server-04c85b6e-ad99-62a9-7e7b-be46be1258bf: heartbeat timeout reached, starting election: last-leader-addr=c4426023-874b-2f23-8d3e-11417740bf2d last-leader-id=server-c4426023-874b-2f23-8d3e-11417740bf2d +2025-03-20T03:50:31.370Z [INFO] server-04c85b6e-ad99-62a9-7e7b-be46be1258bf: entering candidate state: node="Node at 04c85b6e-ad99-62a9-7e7b-be46be1258bf [Candidate]" term=3 +2025-03-20T03:50:31.582Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:50:29.729368679 +0000 UTC m=+61.403821526 (2 nodes), 2.089µs from start of poll, 1.550316s from cluster start. Timeout at 2025-03-20 03:50:30.734428432 +0000 UTC m=+62.408881249, 1.005059723s after stability +2025-03-20T03:50:31.583Z [INFO] server-80d16e54-3c85-3681-2e34-3b985620b4fa: duplicate requestVote for same term: term=3 +2025-03-20T03:50:31.583Z [WARN] server-4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f: heartbeat timeout reached, starting election: last-leader-addr=2585b7d3-2635-ea9d-6ada-0e0ed216a860 last-leader-id=server-2585b7d3-2635-ea9d-6ada-0e0ed216a860 +2025-03-20T03:50:31.583Z [INFO] server-4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f: entering candidate state: node="Node at 4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f [Candidate]" term=3 +2025-03-20T03:50:31.583Z [WARN] server-cd38a9e4-f93a-b249-77dc-db998253e2f9: heartbeat timeout reached, starting election: last-leader-addr=2585b7d3-2635-ea9d-6ada-0e0ed216a860 last-leader-id=server-2585b7d3-2635-ea9d-6ada-0e0ed216a860 +2025-03-20T03:50:31.588Z [INFO] server-cd38a9e4-f93a-b249-77dc-db998253e2f9: entering candidate state: node="Node at cd38a9e4-f93a-b249-77dc-db998253e2f9 [Candidate]" term=3 +2025-03-20T03:50:31.586Z [WARN] server-2585b7d3-2635-ea9d-6ada-0e0ed216a860: failed to contact: server-id=server-cd38a9e4-f93a-b249-77dc-db998253e2f9 time=837.046833ms +2025-03-20T03:50:31.586Z [WARN] server-aafe5397-bc28-db6c-92d5-f943a036aa75: heartbeat timeout reached, starting election: last-leader-addr=a01b1ec5-708d-74be-8a97-5b903cc77d25 last-leader-id=server-a01b1ec5-708d-74be-8a97-5b903cc77d25 +2025-03-20T03:50:31.588Z [INFO] server-aafe5397-bc28-db6c-92d5-f943a036aa75: entering candidate state: node="Node at aafe5397-bc28-db6c-92d5-f943a036aa75 [Candidate]" term=3 +2025-03-20T03:50:31.587Z [WARN] server-49c1bb5a-4b75-07ac-d971-6d75efa93acc: heartbeat timeout reached, starting election: last-leader-addr=e5c5cd54-3179-1786-ba9b-071f3cb13d4c last-leader-id=server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c +2025-03-20T03:50:31.588Z [INFO] server-49c1bb5a-4b75-07ac-d971-6d75efa93acc: entering candidate state: node="Node at 49c1bb5a-4b75-07ac-d971-6d75efa93acc [Candidate]" term=3 +2025-03-20T03:50:31.587Z [INFO] server-4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2: aborting pipeline replication: peer="{Voter server-3ce2100e-ff0b-72f8-eee7-317c57f72604 3ce2100e-ff0b-72f8-eee7-317c57f72604}" +2025-03-20T03:50:31.586Z [ERROR] server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c: failed to heartbeat to: peer=9736863e-fe7d-d1aa-1e3b-eca4ca637c7d backoff time=10ms error="command timed out" +2025-03-20T03:50:31.586Z [INFO] server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c: aborting pipeline replication: peer="{Voter server-9736863e-fe7d-d1aa-1e3b-eca4ca637c7d 9736863e-fe7d-d1aa-1e3b-eca4ca637c7d}" +2025-03-20T03:50:31.586Z [ERROR] server-c4426023-874b-2f23-8d3e-11417740bf2d: peer has newer term, stopping replication: peer="{Voter server-80d16e54-3c85-3681-2e34-3b985620b4fa 80d16e54-3c85-3681-2e34-3b985620b4fa}" +2025-03-20T03:50:31.588Z [INFO] server-c4426023-874b-2f23-8d3e-11417740bf2d: aborting pipeline replication: peer="{Voter server-80d16e54-3c85-3681-2e34-3b985620b4fa 80d16e54-3c85-3681-2e34-3b985620b4fa}" +2025-03-20T03:50:31.587Z [ERROR] server-29f478ab-7a47-a793-ea2c-01b0bd11c7a5: peer has newer term, stopping replication: peer="{Voter server-4588dbe5-3659-fdb6-7992-ea346fa55b28 4588dbe5-3659-fdb6-7992-ea346fa55b28}" +2025-03-20T03:50:31.588Z [INFO] server-29f478ab-7a47-a793-ea2c-01b0bd11c7a5: aborting pipeline replication: peer="{Voter server-4588dbe5-3659-fdb6-7992-ea346fa55b28 4588dbe5-3659-fdb6-7992-ea346fa55b28}" +2025-03-20T03:50:31.587Z [WARN] server-8fc4c748-473e-84cf-e592-676b7ab9a049: rejecting vote request since we have a leader: from=4588dbe5-3659-fdb6-7992-ea346fa55b28 leader=29f478ab-7a47-a793-ea2c-01b0bd11c7a5 leader-id=server-29f478ab-7a47-a793-ea2c-01b0bd11c7a5 +2025-03-20T03:50:31.588Z [INFO] server-29f478ab-7a47-a793-ea2c-01b0bd11c7a5: entering follower state: follower="Node at 29f478ab-7a47-a793-ea2c-01b0bd11c7a5 [Follower]" leader-address= leader-id= +2025-03-20T03:50:31.614Z [ERROR] server-2585b7d3-2635-ea9d-6ada-0e0ed216a860: peer has newer term, stopping replication: peer="{Voter server-4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f 4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f}" +2025-03-20T03:50:31.614Z [INFO] server-2585b7d3-2635-ea9d-6ada-0e0ed216a860: aborting pipeline replication: peer="{Voter server-4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f 4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f}" +2025-03-20T03:50:31.614Z [ERROR] server-2585b7d3-2635-ea9d-6ada-0e0ed216a860: peer has newer term, stopping replication: peer="{Voter server-cd38a9e4-f93a-b249-77dc-db998253e2f9 cd38a9e4-f93a-b249-77dc-db998253e2f9}" +2025-03-20T03:50:31.615Z [INFO] server-2585b7d3-2635-ea9d-6ada-0e0ed216a860: aborting pipeline replication: peer="{Voter server-cd38a9e4-f93a-b249-77dc-db998253e2f9 cd38a9e4-f93a-b249-77dc-db998253e2f9}" +2025-03-20T03:50:31.615Z [INFO] server-cd38a9e4-f93a-b249-77dc-db998253e2f9: duplicate requestVote for same term: term=3 +2025-03-20T03:50:31.615Z [INFO] server-2585b7d3-2635-ea9d-6ada-0e0ed216a860: entering follower state: follower="Node at 2585b7d3-2635-ea9d-6ada-0e0ed216a860 [Follower]" leader-address= leader-id= +2025-03-20T03:50:31.615Z [INFO] server-4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f: election won: term=3 tally=2 +2025-03-20T03:50:31.615Z [INFO] server-4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f: entering leader state: leader="Node at 4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f [Leader]" +2025-03-20T03:50:31.615Z [INFO] server-4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f: added peer, starting replication: peer=server-cd38a9e4-f93a-b249-77dc-db998253e2f9 +2025-03-20T03:50:31.615Z [INFO] server-4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f: added peer, starting replication: peer=server-2585b7d3-2635-ea9d-6ada-0e0ed216a860 +2025-03-20T03:50:31.615Z [INFO] server-4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f: pipelining replication: peer="{Voter server-2585b7d3-2635-ea9d-6ada-0e0ed216a860 2585b7d3-2635-ea9d-6ada-0e0ed216a860}" +2025-03-20T03:50:31.615Z [WARN] server-2585b7d3-2635-ea9d-6ada-0e0ed216a860: rejecting vote request since we have a leader: from=cd38a9e4-f93a-b249-77dc-db998253e2f9 leader=4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f leader-id=server-4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f +2025-03-20T03:50:31.615Z [ERROR] server-a01b1ec5-708d-74be-8a97-5b903cc77d25: peer has newer term, stopping replication: peer="{Voter server-aafe5397-bc28-db6c-92d5-f943a036aa75 aafe5397-bc28-db6c-92d5-f943a036aa75}" +2025-03-20T03:50:31.615Z [INFO] server-a01b1ec5-708d-74be-8a97-5b903cc77d25: aborting pipeline replication: peer="{Voter server-aafe5397-bc28-db6c-92d5-f943a036aa75 aafe5397-bc28-db6c-92d5-f943a036aa75}" +2025-03-20T03:50:31.615Z [INFO] server-cd38a9e4-f93a-b249-77dc-db998253e2f9: entering follower state: follower="Node at cd38a9e4-f93a-b249-77dc-db998253e2f9 [Follower]" leader-address=4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f leader-id=server-4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f +2025-03-20T03:50:31.615Z [INFO] server-4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f: pipelining replication: peer="{Voter server-cd38a9e4-f93a-b249-77dc-db998253e2f9 cd38a9e4-f93a-b249-77dc-db998253e2f9}" +2025-03-20T03:50:31.615Z [WARN] server-4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f: rejecting vote request since we have a leader: from=cd38a9e4-f93a-b249-77dc-db998253e2f9 leader=4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f leader-id=server-4ec633ae-2a56-a3ca-cc58-3f3fc74fcb2f +2025-03-20T03:50:31.616Z [ERROR] server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c: peer has newer term, stopping replication: peer="{Voter server-49c1bb5a-4b75-07ac-d971-6d75efa93acc 49c1bb5a-4b75-07ac-d971-6d75efa93acc}" +2025-03-20T03:50:31.616Z [INFO] server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c: aborting pipeline replication: peer="{Voter server-49c1bb5a-4b75-07ac-d971-6d75efa93acc 49c1bb5a-4b75-07ac-d971-6d75efa93acc}" +2025-03-20T03:50:31.616Z [INFO] server-a01b1ec5-708d-74be-8a97-5b903cc77d25: entering follower state: follower="Node at a01b1ec5-708d-74be-8a97-5b903cc77d25 [Follower]" leader-address= leader-id= +2025-03-20T03:50:31.616Z [INFO] server-aafe5397-bc28-db6c-92d5-f943a036aa75: election won: term=3 tally=2 +2025-03-20T03:50:31.616Z [INFO] server-aafe5397-bc28-db6c-92d5-f943a036aa75: entering leader state: leader="Node at aafe5397-bc28-db6c-92d5-f943a036aa75 [Leader]" +2025-03-20T03:50:31.616Z [INFO] server-aafe5397-bc28-db6c-92d5-f943a036aa75: added peer, starting replication: peer=server-b9dbfdee-4a74-b888-25d8-a6e810259d00 +2025-03-20T03:50:31.616Z [INFO] server-aafe5397-bc28-db6c-92d5-f943a036aa75: added peer, starting replication: peer=server-a01b1ec5-708d-74be-8a97-5b903cc77d25 +2025-03-20T03:50:31.616Z [INFO] server-aafe5397-bc28-db6c-92d5-f943a036aa75: pipelining replication: peer="{Voter server-a01b1ec5-708d-74be-8a97-5b903cc77d25 a01b1ec5-708d-74be-8a97-5b903cc77d25}" +2025-03-20T03:50:31.616Z [WARN] server-8fc4c748-473e-84cf-e592-676b7ab9a049: heartbeat timeout reached, starting election: last-leader-addr=29f478ab-7a47-a793-ea2c-01b0bd11c7a5 last-leader-id=server-29f478ab-7a47-a793-ea2c-01b0bd11c7a5 +2025-03-20T03:50:31.616Z [INFO] server-8fc4c748-473e-84cf-e592-676b7ab9a049: entering candidate state: node="Node at 8fc4c748-473e-84cf-e592-676b7ab9a049 [Candidate]" term=3 +2025-03-20T03:50:31.616Z [INFO] server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c: entering follower state: follower="Node at e5c5cd54-3179-1786-ba9b-071f3cb13d4c [Follower]" leader-address= leader-id= +2025-03-20T03:50:31.616Z [INFO] server-49c1bb5a-4b75-07ac-d971-6d75efa93acc: election won: term=3 tally=2 +2025-03-20T03:50:31.616Z [INFO] server-49c1bb5a-4b75-07ac-d971-6d75efa93acc: entering leader state: leader="Node at 49c1bb5a-4b75-07ac-d971-6d75efa93acc [Leader]" +2025-03-20T03:50:31.616Z [INFO] server-49c1bb5a-4b75-07ac-d971-6d75efa93acc: added peer, starting replication: peer=server-9736863e-fe7d-d1aa-1e3b-eca4ca637c7d +2025-03-20T03:50:31.616Z [INFO] server-49c1bb5a-4b75-07ac-d971-6d75efa93acc: added peer, starting replication: peer=server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c +2025-03-20T03:50:31.617Z [INFO] server-49c1bb5a-4b75-07ac-d971-6d75efa93acc: pipelining replication: peer="{Voter server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c e5c5cd54-3179-1786-ba9b-071f3cb13d4c}" +2025-03-20T03:50:31.617Z [INFO] server-a01b1ec5-708d-74be-8a97-5b903cc77d25: aborting pipeline replication: peer="{Voter server-b9dbfdee-4a74-b888-25d8-a6e810259d00 b9dbfdee-4a74-b888-25d8-a6e810259d00}" +2025-03-20T03:50:31.617Z [INFO] server-4588dbe5-3659-fdb6-7992-ea346fa55b28: election won: term=3 tally=2 +2025-03-20T03:50:31.617Z [INFO] server-4588dbe5-3659-fdb6-7992-ea346fa55b28: entering leader state: leader="Node at 4588dbe5-3659-fdb6-7992-ea346fa55b28 [Leader]" +2025-03-20T03:50:31.617Z [INFO] server-4588dbe5-3659-fdb6-7992-ea346fa55b28: added peer, starting replication: peer=server-8fc4c748-473e-84cf-e592-676b7ab9a049 +2025-03-20T03:50:31.617Z [INFO] server-4588dbe5-3659-fdb6-7992-ea346fa55b28: added peer, starting replication: peer=server-29f478ab-7a47-a793-ea2c-01b0bd11c7a5 +2025-03-20T03:50:31.617Z [INFO] server-4588dbe5-3659-fdb6-7992-ea346fa55b28: pipelining replication: peer="{Voter server-29f478ab-7a47-a793-ea2c-01b0bd11c7a5 29f478ab-7a47-a793-ea2c-01b0bd11c7a5}" +2025-03-20T03:50:31.617Z [WARN] server-29f478ab-7a47-a793-ea2c-01b0bd11c7a5: rejecting vote request since we have a leader: from=8fc4c748-473e-84cf-e592-676b7ab9a049 leader=4588dbe5-3659-fdb6-7992-ea346fa55b28 leader-id=server-4588dbe5-3659-fdb6-7992-ea346fa55b28 +2025-03-20T03:50:31.617Z [INFO] server-04c85b6e-ad99-62a9-7e7b-be46be1258bf: duplicate requestVote for same term: term=3 +2025-03-20T03:50:31.618Z [WARN] server-4588dbe5-3659-fdb6-7992-ea346fa55b28: rejecting vote request since we have a leader: from=8fc4c748-473e-84cf-e592-676b7ab9a049 leader=4588dbe5-3659-fdb6-7992-ea346fa55b28 leader-id=server-4588dbe5-3659-fdb6-7992-ea346fa55b28 +2025-03-20T03:50:31.618Z [INFO] server-8fc4c748-473e-84cf-e592-676b7ab9a049: entering follower state: follower="Node at 8fc4c748-473e-84cf-e592-676b7ab9a049 [Follower]" leader-address=4588dbe5-3659-fdb6-7992-ea346fa55b28 leader-id=server-4588dbe5-3659-fdb6-7992-ea346fa55b28 +2025-03-20T03:50:31.619Z [WARN] server-9736863e-fe7d-d1aa-1e3b-eca4ca637c7d: heartbeat timeout reached, starting election: last-leader-addr=e5c5cd54-3179-1786-ba9b-071f3cb13d4c last-leader-id=server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c +2025-03-20T03:50:31.621Z [INFO] server-9736863e-fe7d-d1aa-1e3b-eca4ca637c7d: entering candidate state: node="Node at 9736863e-fe7d-d1aa-1e3b-eca4ca637c7d [Candidate]" term=3 +2025-03-20T03:50:31.621Z [INFO] server-9736863e-fe7d-d1aa-1e3b-eca4ca637c7d: duplicate requestVote for same term: term=3 +2025-03-20T03:50:31.621Z [INFO] server-9736863e-fe7d-d1aa-1e3b-eca4ca637c7d: entering follower state: follower="Node at 9736863e-fe7d-d1aa-1e3b-eca4ca637c7d [Follower]" leader-address=49c1bb5a-4b75-07ac-d971-6d75efa93acc leader-id=server-49c1bb5a-4b75-07ac-d971-6d75efa93acc +2025-03-20T03:50:31.621Z [INFO] server-4c5f8e1b-8dd7-cf2e-db18-d78370c05ab2: pipelining replication: peer="{Voter server-3ce2100e-ff0b-72f8-eee7-317c57f72604 3ce2100e-ff0b-72f8-eee7-317c57f72604}" +2025-03-20T03:50:31.619Z [INFO] server-4588dbe5-3659-fdb6-7992-ea346fa55b28: pipelining replication: peer="{Voter server-8fc4c748-473e-84cf-e592-676b7ab9a049 8fc4c748-473e-84cf-e592-676b7ab9a049}" +2025-03-20T03:50:31.619Z [INFO] server-29f478ab-7a47-a793-ea2c-01b0bd11c7a5: aborting pipeline replication: peer="{Voter server-8fc4c748-473e-84cf-e592-676b7ab9a049 8fc4c748-473e-84cf-e592-676b7ab9a049}" +2025-03-20T03:50:31.619Z [INFO] server-c4426023-874b-2f23-8d3e-11417740bf2d: entering follower state: follower="Node at c4426023-874b-2f23-8d3e-11417740bf2d [Follower]" leader-address= leader-id= +2025-03-20T03:50:31.621Z [INFO] server-c4426023-874b-2f23-8d3e-11417740bf2d: duplicate requestVote for same term: term=3 +2025-03-20T03:50:31.621Z [WARN] server-49c1bb5a-4b75-07ac-d971-6d75efa93acc: rejecting vote request since we have a leader: from=9736863e-fe7d-d1aa-1e3b-eca4ca637c7d leader=49c1bb5a-4b75-07ac-d971-6d75efa93acc leader-id=server-49c1bb5a-4b75-07ac-d971-6d75efa93acc +2025-03-20T03:50:31.621Z [WARN] server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c: rejecting vote request since we have a leader: from=9736863e-fe7d-d1aa-1e3b-eca4ca637c7d leader=49c1bb5a-4b75-07ac-d971-6d75efa93acc leader-id=server-49c1bb5a-4b75-07ac-d971-6d75efa93acc +2025-03-20T03:50:31.621Z [ERROR] server-e5c5cd54-3179-1786-ba9b-071f3cb13d4c: peer has newer term, stopping replication: peer="{Voter server-9736863e-fe7d-d1aa-1e3b-eca4ca637c7d 9736863e-fe7d-d1aa-1e3b-eca4ca637c7d}" +2025-03-20T03:50:31.619Z [WARN] server-b9dbfdee-4a74-b888-25d8-a6e810259d00: rejecting vote request since we have a leader: from=aafe5397-bc28-db6c-92d5-f943a036aa75 leader=a01b1ec5-708d-74be-8a97-5b903cc77d25 leader-id=server-a01b1ec5-708d-74be-8a97-5b903cc77d25 +2025-03-20T03:50:31.621Z [INFO] server-aafe5397-bc28-db6c-92d5-f943a036aa75: pipelining replication: peer="{Voter server-b9dbfdee-4a74-b888-25d8-a6e810259d00 b9dbfdee-4a74-b888-25d8-a6e810259d00}" +2025-03-20T03:50:31.622Z [INFO] server-49c1bb5a-4b75-07ac-d971-6d75efa93acc: pipelining replication: peer="{Voter server-9736863e-fe7d-d1aa-1e3b-eca4ca637c7d 9736863e-fe7d-d1aa-1e3b-eca4ca637c7d}" +2025-03-20T03:50:31.622Z [INFO] server-80d16e54-3c85-3681-2e34-3b985620b4fa: election won: term=3 tally=2 +2025-03-20T03:50:31.622Z [INFO] server-80d16e54-3c85-3681-2e34-3b985620b4fa: entering leader state: leader="Node at 80d16e54-3c85-3681-2e34-3b985620b4fa [Leader]" +2025-03-20T03:50:31.624Z [INFO] server-80d16e54-3c85-3681-2e34-3b985620b4fa: added peer, starting replication: peer=server-04c85b6e-ad99-62a9-7e7b-be46be1258bf +2025-03-20T03:50:31.624Z [INFO] server-80d16e54-3c85-3681-2e34-3b985620b4fa: added peer, starting replication: peer=server-c4426023-874b-2f23-8d3e-11417740bf2d +2025-03-20T03:50:31.624Z [INFO] server-80d16e54-3c85-3681-2e34-3b985620b4fa: pipelining replication: peer="{Voter server-c4426023-874b-2f23-8d3e-11417740bf2d c4426023-874b-2f23-8d3e-11417740bf2d}" +2025-03-20T03:50:31.620Z [ERROR] server-c4426023-874b-2f23-8d3e-11417740bf2d: peer has newer term, stopping replication: peer="{Voter server-80d16e54-3c85-3681-2e34-3b985620b4fa 80d16e54-3c85-3681-2e34-3b985620b4fa}" +2025-03-20T03:50:31.624Z [INFO] server-04c85b6e-ad99-62a9-7e7b-be46be1258bf: entering follower state: follower="Node at 04c85b6e-ad99-62a9-7e7b-be46be1258bf [Follower]" leader-address=80d16e54-3c85-3681-2e34-3b985620b4fa leader-id=server-80d16e54-3c85-3681-2e34-3b985620b4fa +2025-03-20T03:50:31.625Z [INFO] server-80d16e54-3c85-3681-2e34-3b985620b4fa: pipelining replication: peer="{Voter server-04c85b6e-ad99-62a9-7e7b-be46be1258bf 04c85b6e-ad99-62a9-7e7b-be46be1258bf}" +2025-03-20T03:50:31.620Z [ERROR] server-c4426023-874b-2f23-8d3e-11417740bf2d: peer has newer term, stopping replication: peer="{Voter server-04c85b6e-ad99-62a9-7e7b-be46be1258bf 04c85b6e-ad99-62a9-7e7b-be46be1258bf}" +2025-03-20T03:50:31.625Z [INFO] server-c4426023-874b-2f23-8d3e-11417740bf2d: aborting pipeline replication: peer="{Voter server-04c85b6e-ad99-62a9-7e7b-be46be1258bf 04c85b6e-ad99-62a9-7e7b-be46be1258bf}" +2025-03-20T03:50:31.629Z [INFO] server-2fcd6d21-cc82-177c-c44d-92cd0f790da4: aborting pipeline replication: peer="{Voter server-5ec4c8ff-462a-0413-b7bc-7f0541805e20 5ec4c8ff-462a-0413-b7bc-7f0541805e20}" +2025-03-20T03:50:31.629Z [INFO] server-2fcd6d21-cc82-177c-c44d-92cd0f790da4: aborting pipeline replication: peer="{Voter server-f27e94bd-3a17-3d87-930c-81661070b232 f27e94bd-3a17-3d87-930c-81661070b232}" === RUN TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_true -2026-04-22T10:10:15.489Z [INFO] server-a7da2b12-aa24-9080-774f-30aae561cb18: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a7da2b12-aa24-9080-774f-30aae561cb18 Address:a7da2b12-aa24-9080-774f-30aae561cb18} {Suffrage:Voter ID:server-3ce9b743-7fde-59be-af7f-b9908fef017e Address:3ce9b743-7fde-59be-af7f-b9908fef017e} {Suffrage:Voter ID:server-194a846a-8e2c-b051-91e7-5b82addb1490 Address:194a846a-8e2c-b051-91e7-5b82addb1490}]" -2026-04-22T10:10:15.489Z [INFO] server-a7da2b12-aa24-9080-774f-30aae561cb18: entering follower state: follower="Node at a7da2b12-aa24-9080-774f-30aae561cb18 [Follower]" leader-address= leader-id= -2026-04-22T10:10:15.489Z [INFO] server-3ce9b743-7fde-59be-af7f-b9908fef017e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a7da2b12-aa24-9080-774f-30aae561cb18 Address:a7da2b12-aa24-9080-774f-30aae561cb18} {Suffrage:Voter ID:server-3ce9b743-7fde-59be-af7f-b9908fef017e Address:3ce9b743-7fde-59be-af7f-b9908fef017e} {Suffrage:Voter ID:server-194a846a-8e2c-b051-91e7-5b82addb1490 Address:194a846a-8e2c-b051-91e7-5b82addb1490}]" -2026-04-22T10:10:15.489Z [INFO] server-3ce9b743-7fde-59be-af7f-b9908fef017e: entering follower state: follower="Node at 3ce9b743-7fde-59be-af7f-b9908fef017e [Follower]" leader-address= leader-id= -2026-04-22T10:10:15.489Z [INFO] server-194a846a-8e2c-b051-91e7-5b82addb1490: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a7da2b12-aa24-9080-774f-30aae561cb18 Address:a7da2b12-aa24-9080-774f-30aae561cb18} {Suffrage:Voter ID:server-3ce9b743-7fde-59be-af7f-b9908fef017e Address:3ce9b743-7fde-59be-af7f-b9908fef017e} {Suffrage:Voter ID:server-194a846a-8e2c-b051-91e7-5b82addb1490 Address:194a846a-8e2c-b051-91e7-5b82addb1490}]" -2026-04-22T10:10:15.489Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:15.489Z [INFO] server-194a846a-8e2c-b051-91e7-5b82addb1490: entering follower state: follower="Node at 194a846a-8e2c-b051-91e7-5b82addb1490 [Follower]" leader-address= leader-id= -2026-04-22T10:10:16.128Z [WARN] server-3ce9b743-7fde-59be-af7f-b9908fef017e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:16.128Z [INFO] server-3ce9b743-7fde-59be-af7f-b9908fef017e: entering candidate state: node="Node at 3ce9b743-7fde-59be-af7f-b9908fef017e [Candidate]" term=2 -2026-04-22T10:10:16.128Z [INFO] server-3ce9b743-7fde-59be-af7f-b9908fef017e: election won: term=2 tally=2 -2026-04-22T10:10:16.128Z [INFO] server-3ce9b743-7fde-59be-af7f-b9908fef017e: entering leader state: leader="Node at 3ce9b743-7fde-59be-af7f-b9908fef017e [Leader]" -2026-04-22T10:10:16.128Z [INFO] server-3ce9b743-7fde-59be-af7f-b9908fef017e: added peer, starting replication: peer=server-a7da2b12-aa24-9080-774f-30aae561cb18 -2026-04-22T10:10:16.128Z [INFO] server-3ce9b743-7fde-59be-af7f-b9908fef017e: added peer, starting replication: peer=server-194a846a-8e2c-b051-91e7-5b82addb1490 -2026-04-22T10:10:16.128Z [INFO] server-3ce9b743-7fde-59be-af7f-b9908fef017e: pipelining replication: peer="{Voter server-194a846a-8e2c-b051-91e7-5b82addb1490 194a846a-8e2c-b051-91e7-5b82addb1490}" -2026-04-22T10:10:16.128Z [INFO] server-3ce9b743-7fde-59be-af7f-b9908fef017e: pipelining replication: peer="{Voter server-a7da2b12-aa24-9080-774f-30aae561cb18 a7da2b12-aa24-9080-774f-30aae561cb18}" -2026-04-22T10:10:17.134Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:16.128742814 +0000 UTC m=+62.152538925 (1 nodes), 638.889887ms from start of poll, 639.278986ms from cluster start. Timeout at 2026-04-22 10:10:17.133750335 +0000 UTC m=+63.157546475, 1.00500755s after stability -2026-04-22T10:10:17.135Z [INFO] server-3ce9b743-7fde-59be-af7f-b9908fef017e: starting snapshot up to: index=12 -2026-04-22T10:10:17.136Z [INFO] creating new snapshot: path=/tmp/raft1231272908/snapshots/2-12-1776852617135.tmp -2026-04-22T10:10:17.136Z [INFO] server-3ce9b743-7fde-59be-af7f-b9908fef017e: snapshot complete up to: index=12 -2026-04-22T10:10:17.137Z [INFO] server-1416f794-abdc-e710-3c71-735b000aae94: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1416f794-abdc-e710-3c71-735b000aae94 Address:1416f794-abdc-e710-3c71-735b000aae94} {Suffrage:Voter ID:server-112470c4-e61e-b9ab-221f-b74693553f4d Address:112470c4-e61e-b9ab-221f-b74693553f4d} {Suffrage:Voter ID:server-da057de3-7d9d-9080-7b5a-c692af39c8c9 Address:da057de3-7d9d-9080-7b5a-c692af39c8c9}]" -2026-04-22T10:10:17.137Z [INFO] server-1416f794-abdc-e710-3c71-735b000aae94: entering follower state: follower="Node at 1416f794-abdc-e710-3c71-735b000aae94 [Follower]" leader-address= leader-id= -2026-04-22T10:10:17.137Z [INFO] server-112470c4-e61e-b9ab-221f-b74693553f4d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1416f794-abdc-e710-3c71-735b000aae94 Address:1416f794-abdc-e710-3c71-735b000aae94} {Suffrage:Voter ID:server-112470c4-e61e-b9ab-221f-b74693553f4d Address:112470c4-e61e-b9ab-221f-b74693553f4d} {Suffrage:Voter ID:server-da057de3-7d9d-9080-7b5a-c692af39c8c9 Address:da057de3-7d9d-9080-7b5a-c692af39c8c9}]" -2026-04-22T10:10:17.137Z [INFO] server-112470c4-e61e-b9ab-221f-b74693553f4d: entering follower state: follower="Node at 112470c4-e61e-b9ab-221f-b74693553f4d [Follower]" leader-address= leader-id= -2026-04-22T10:10:17.137Z [INFO] server-da057de3-7d9d-9080-7b5a-c692af39c8c9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1416f794-abdc-e710-3c71-735b000aae94 Address:1416f794-abdc-e710-3c71-735b000aae94} {Suffrage:Voter ID:server-112470c4-e61e-b9ab-221f-b74693553f4d Address:112470c4-e61e-b9ab-221f-b74693553f4d} {Suffrage:Voter ID:server-da057de3-7d9d-9080-7b5a-c692af39c8c9 Address:da057de3-7d9d-9080-7b5a-c692af39c8c9}]" -2026-04-22T10:10:17.137Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:17.137Z [INFO] server-da057de3-7d9d-9080-7b5a-c692af39c8c9: entering follower state: follower="Node at da057de3-7d9d-9080-7b5a-c692af39c8c9 [Follower]" leader-address= leader-id= -2026-04-22T10:10:17.676Z [WARN] server-112470c4-e61e-b9ab-221f-b74693553f4d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:17.676Z [INFO] server-112470c4-e61e-b9ab-221f-b74693553f4d: entering candidate state: node="Node at 112470c4-e61e-b9ab-221f-b74693553f4d [Candidate]" term=2 -2026-04-22T10:10:17.676Z [INFO] server-112470c4-e61e-b9ab-221f-b74693553f4d: election won: term=2 tally=2 -2026-04-22T10:10:17.677Z [INFO] server-112470c4-e61e-b9ab-221f-b74693553f4d: entering leader state: leader="Node at 112470c4-e61e-b9ab-221f-b74693553f4d [Leader]" -2026-04-22T10:10:17.677Z [INFO] server-112470c4-e61e-b9ab-221f-b74693553f4d: added peer, starting replication: peer=server-1416f794-abdc-e710-3c71-735b000aae94 -2026-04-22T10:10:17.677Z [INFO] server-112470c4-e61e-b9ab-221f-b74693553f4d: added peer, starting replication: peer=server-da057de3-7d9d-9080-7b5a-c692af39c8c9 -2026-04-22T10:10:17.677Z [INFO] server-112470c4-e61e-b9ab-221f-b74693553f4d: pipelining replication: peer="{Voter server-da057de3-7d9d-9080-7b5a-c692af39c8c9 da057de3-7d9d-9080-7b5a-c692af39c8c9}" -2026-04-22T10:10:17.677Z [INFO] server-112470c4-e61e-b9ab-221f-b74693553f4d: pipelining replication: peer="{Voter server-1416f794-abdc-e710-3c71-735b000aae94 1416f794-abdc-e710-3c71-735b000aae94}" -2026-04-22T10:10:18.682Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:17.677133496 +0000 UTC m=+63.700929583 (1 nodes), 539.459903ms from start of poll, 539.743435ms from cluster start. Timeout at 2026-04-22 10:10:18.682137928 +0000 UTC m=+64.705934011, 1.005004428s after stability -2026-04-22T10:10:18.684Z [INFO] creating new snapshot: path=/tmp/raft4109923467/snapshots/2-113-1776852618684.tmp -2026-04-22T10:10:18.685Z [INFO] server-112470c4-e61e-b9ab-221f-b74693553f4d: copied to local snapshot: bytes=71 -2026-04-22T10:10:18.685Z [INFO] server-112470c4-e61e-b9ab-221f-b74693553f4d: snapshot restore progress: id=2-113-1776852618684 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:18.685Z [INFO] server-112470c4-e61e-b9ab-221f-b74693553f4d: restored user snapshot: index=113 -2026-04-22T10:10:18.685Z [ERROR] server-112470c4-e61e-b9ab-221f-b74693553f4d: failed to get log: index=3 error="log not found" -2026-04-22T10:10:18.685Z [ERROR] server-112470c4-e61e-b9ab-221f-b74693553f4d: failed to get log: index=3 error="log not found" -2026-04-22T10:10:18.685Z [INFO] server-112470c4-e61e-b9ab-221f-b74693553f4d: aborting pipeline replication: peer="{Voter server-1416f794-abdc-e710-3c71-735b000aae94 1416f794-abdc-e710-3c71-735b000aae94}" -2026-04-22T10:10:18.685Z [INFO] server-112470c4-e61e-b9ab-221f-b74693553f4d: aborting pipeline replication: peer="{Voter server-da057de3-7d9d-9080-7b5a-c692af39c8c9 da057de3-7d9d-9080-7b5a-c692af39c8c9}" -2026-04-22T10:10:18.691Z [ERROR] server-112470c4-e61e-b9ab-221f-b74693553f4d: failed to get log: index=3 error="log not found" -2026-04-22T10:10:18.692Z [INFO] creating new snapshot: path=/tmp/raft2695396732/snapshots/2-113-1776852618692.tmp -2026-04-22T10:10:18.693Z [INFO] server-1416f794-abdc-e710-3c71-735b000aae94: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:18.693Z [INFO] server-1416f794-abdc-e710-3c71-735b000aae94: copied to local snapshot: bytes=71 -2026-04-22T10:10:18.694Z [INFO] server-1416f794-abdc-e710-3c71-735b000aae94: snapshot restore progress: id=2-113-1776852618692 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:18.694Z [INFO] server-1416f794-abdc-e710-3c71-735b000aae94: Installed remote snapshot -2026-04-22T10:10:18.694Z [INFO] server-112470c4-e61e-b9ab-221f-b74693553f4d: pipelining replication: peer="{Voter server-1416f794-abdc-e710-3c71-735b000aae94 1416f794-abdc-e710-3c71-735b000aae94}" -2026-04-22T10:10:18.694Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:10:18.694Z [ERROR] server-112470c4-e61e-b9ab-221f-b74693553f4d: failed to get log: index=3 error="log not found" -2026-04-22T10:10:18.694Z [INFO] creating new snapshot: path=/tmp/raft3514705329/snapshots/2-113-1776852618694.tmp -2026-04-22T10:10:18.694Z [INFO] server-da057de3-7d9d-9080-7b5a-c692af39c8c9: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:18.695Z [INFO] server-da057de3-7d9d-9080-7b5a-c692af39c8c9: copied to local snapshot: bytes=71 -2026-04-22T10:10:18.695Z [INFO] server-da057de3-7d9d-9080-7b5a-c692af39c8c9: snapshot restore progress: id=2-113-1776852618694 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:18.695Z [INFO] server-da057de3-7d9d-9080-7b5a-c692af39c8c9: Installed remote snapshot -2026-04-22T10:10:18.695Z [INFO] server-112470c4-e61e-b9ab-221f-b74693553f4d: pipelining replication: peer="{Voter server-da057de3-7d9d-9080-7b5a-c692af39c8c9 da057de3-7d9d-9080-7b5a-c692af39c8c9}" -2026-04-22T10:10:19.699Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:10:18.694238633 +0000 UTC m=+64.718034726 (2 nodes), 616ns from start of poll, 1.556848578s from cluster start. Timeout at 2026-04-22 10:10:19.69924831 +0000 UTC m=+65.723044392, 1.005009666s after stability -2026-04-22T10:10:19.710Z [INFO] server-3ce9b743-7fde-59be-af7f-b9908fef017e: aborting pipeline replication: peer="{Voter server-194a846a-8e2c-b051-91e7-5b82addb1490 194a846a-8e2c-b051-91e7-5b82addb1490}" -2026-04-22T10:10:19.710Z [INFO] server-3ce9b743-7fde-59be-af7f-b9908fef017e: aborting pipeline replication: peer="{Voter server-a7da2b12-aa24-9080-774f-30aae561cb18 a7da2b12-aa24-9080-774f-30aae561cb18}" +2025-03-20T03:50:31.637Z [INFO] server-68bc4e90-b8a0-7c64-c8cc-99553dd98d88: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-68bc4e90-b8a0-7c64-c8cc-99553dd98d88 Address:68bc4e90-b8a0-7c64-c8cc-99553dd98d88} {Suffrage:Voter ID:server-e3e8ff80-80cc-f75f-eee3-7588faded053 Address:e3e8ff80-80cc-f75f-eee3-7588faded053} {Suffrage:Voter ID:server-9d9c7eeb-1017-0596-e0f9-d0727e5d804d Address:9d9c7eeb-1017-0596-e0f9-d0727e5d804d}]" +2025-03-20T03:50:31.637Z [INFO] server-e3e8ff80-80cc-f75f-eee3-7588faded053: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-68bc4e90-b8a0-7c64-c8cc-99553dd98d88 Address:68bc4e90-b8a0-7c64-c8cc-99553dd98d88} {Suffrage:Voter ID:server-e3e8ff80-80cc-f75f-eee3-7588faded053 Address:e3e8ff80-80cc-f75f-eee3-7588faded053} {Suffrage:Voter ID:server-9d9c7eeb-1017-0596-e0f9-d0727e5d804d Address:9d9c7eeb-1017-0596-e0f9-d0727e5d804d}]" +2025-03-20T03:50:31.637Z [INFO] server-9d9c7eeb-1017-0596-e0f9-d0727e5d804d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-68bc4e90-b8a0-7c64-c8cc-99553dd98d88 Address:68bc4e90-b8a0-7c64-c8cc-99553dd98d88} {Suffrage:Voter ID:server-e3e8ff80-80cc-f75f-eee3-7588faded053 Address:e3e8ff80-80cc-f75f-eee3-7588faded053} {Suffrage:Voter ID:server-9d9c7eeb-1017-0596-e0f9-d0727e5d804d Address:9d9c7eeb-1017-0596-e0f9-d0727e5d804d}]" +2025-03-20T03:50:31.637Z [INFO] server-68bc4e90-b8a0-7c64-c8cc-99553dd98d88: entering follower state: follower="Node at 68bc4e90-b8a0-7c64-c8cc-99553dd98d88 [Follower]" leader-address= leader-id= +2025-03-20T03:50:31.637Z [INFO] server-e3e8ff80-80cc-f75f-eee3-7588faded053: entering follower state: follower="Node at e3e8ff80-80cc-f75f-eee3-7588faded053 [Follower]" leader-address= leader-id= +2025-03-20T03:50:31.637Z [INFO] server-9d9c7eeb-1017-0596-e0f9-d0727e5d804d: entering follower state: follower="Node at 9d9c7eeb-1017-0596-e0f9-d0727e5d804d [Follower]" leader-address= leader-id= +2025-03-20T03:50:31.646Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:50:32.174Z [WARN] server-e3e8ff80-80cc-f75f-eee3-7588faded053: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:50:32.174Z [INFO] server-e3e8ff80-80cc-f75f-eee3-7588faded053: entering candidate state: node="Node at e3e8ff80-80cc-f75f-eee3-7588faded053 [Candidate]" term=2 +2025-03-20T03:50:32.175Z [INFO] server-e3e8ff80-80cc-f75f-eee3-7588faded053: election won: term=2 tally=2 +2025-03-20T03:50:32.175Z [INFO] server-e3e8ff80-80cc-f75f-eee3-7588faded053: entering leader state: leader="Node at e3e8ff80-80cc-f75f-eee3-7588faded053 [Leader]" +2025-03-20T03:50:32.175Z [INFO] server-e3e8ff80-80cc-f75f-eee3-7588faded053: added peer, starting replication: peer=server-68bc4e90-b8a0-7c64-c8cc-99553dd98d88 +2025-03-20T03:50:32.175Z [INFO] server-e3e8ff80-80cc-f75f-eee3-7588faded053: added peer, starting replication: peer=server-9d9c7eeb-1017-0596-e0f9-d0727e5d804d +2025-03-20T03:50:32.175Z [INFO] server-e3e8ff80-80cc-f75f-eee3-7588faded053: pipelining replication: peer="{Voter server-68bc4e90-b8a0-7c64-c8cc-99553dd98d88 68bc4e90-b8a0-7c64-c8cc-99553dd98d88}" +2025-03-20T03:50:32.175Z [INFO] server-e3e8ff80-80cc-f75f-eee3-7588faded053: pipelining replication: peer="{Voter server-9d9c7eeb-1017-0596-e0f9-d0727e5d804d 9d9c7eeb-1017-0596-e0f9-d0727e5d804d}" +2025-03-20T03:50:33.181Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:50:32.175220049 +0000 UTC m=+63.849672937 (1 nodes), 528.639389ms from start of poll, 538.311174ms from cluster start. Timeout at 2025-03-20 03:50:33.180225393 +0000 UTC m=+64.854678444, 1.005005507s after stability +2025-03-20T03:50:33.182Z [INFO] server-e3e8ff80-80cc-f75f-eee3-7588faded053: starting snapshot up to: index=12 +2025-03-20T03:50:33.182Z [INFO] creating new snapshot: path=/tmp/raft1977844188/snapshots/2-12-1742442633182.tmp +2025-03-20T03:50:33.187Z [INFO] server-e3e8ff80-80cc-f75f-eee3-7588faded053: snapshot complete up to: index=12 +2025-03-20T03:50:33.189Z [INFO] server-59d5204d-7f1e-c021-b2ce-029b065a6169: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-59d5204d-7f1e-c021-b2ce-029b065a6169 Address:59d5204d-7f1e-c021-b2ce-029b065a6169} {Suffrage:Voter ID:server-ff76e54a-22c5-c70b-20f2-51d2e1c97b92 Address:ff76e54a-22c5-c70b-20f2-51d2e1c97b92} {Suffrage:Voter ID:server-d7e09412-edf8-6d85-328f-014c9f5d7829 Address:d7e09412-edf8-6d85-328f-014c9f5d7829}]" +2025-03-20T03:50:33.189Z [INFO] server-59d5204d-7f1e-c021-b2ce-029b065a6169: entering follower state: follower="Node at 59d5204d-7f1e-c021-b2ce-029b065a6169 [Follower]" leader-address= leader-id= +2025-03-20T03:50:33.189Z [INFO] server-ff76e54a-22c5-c70b-20f2-51d2e1c97b92: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-59d5204d-7f1e-c021-b2ce-029b065a6169 Address:59d5204d-7f1e-c021-b2ce-029b065a6169} {Suffrage:Voter ID:server-ff76e54a-22c5-c70b-20f2-51d2e1c97b92 Address:ff76e54a-22c5-c70b-20f2-51d2e1c97b92} {Suffrage:Voter ID:server-d7e09412-edf8-6d85-328f-014c9f5d7829 Address:d7e09412-edf8-6d85-328f-014c9f5d7829}]" +2025-03-20T03:50:33.190Z [INFO] server-ff76e54a-22c5-c70b-20f2-51d2e1c97b92: entering follower state: follower="Node at ff76e54a-22c5-c70b-20f2-51d2e1c97b92 [Follower]" leader-address= leader-id= +2025-03-20T03:50:33.190Z [INFO] server-d7e09412-edf8-6d85-328f-014c9f5d7829: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-59d5204d-7f1e-c021-b2ce-029b065a6169 Address:59d5204d-7f1e-c021-b2ce-029b065a6169} {Suffrage:Voter ID:server-ff76e54a-22c5-c70b-20f2-51d2e1c97b92 Address:ff76e54a-22c5-c70b-20f2-51d2e1c97b92} {Suffrage:Voter ID:server-d7e09412-edf8-6d85-328f-014c9f5d7829 Address:d7e09412-edf8-6d85-328f-014c9f5d7829}]" +2025-03-20T03:50:33.190Z [INFO] server-d7e09412-edf8-6d85-328f-014c9f5d7829: entering follower state: follower="Node at d7e09412-edf8-6d85-328f-014c9f5d7829 [Follower]" leader-address= leader-id= +2025-03-20T03:50:33.190Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:50:33.822Z [WARN] server-59d5204d-7f1e-c021-b2ce-029b065a6169: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:50:33.822Z [INFO] server-59d5204d-7f1e-c021-b2ce-029b065a6169: entering candidate state: node="Node at 59d5204d-7f1e-c021-b2ce-029b065a6169 [Candidate]" term=2 +2025-03-20T03:50:33.822Z [INFO] server-59d5204d-7f1e-c021-b2ce-029b065a6169: election won: term=2 tally=2 +2025-03-20T03:50:33.822Z [INFO] server-59d5204d-7f1e-c021-b2ce-029b065a6169: entering leader state: leader="Node at 59d5204d-7f1e-c021-b2ce-029b065a6169 [Leader]" +2025-03-20T03:50:33.823Z [INFO] server-59d5204d-7f1e-c021-b2ce-029b065a6169: added peer, starting replication: peer=server-ff76e54a-22c5-c70b-20f2-51d2e1c97b92 +2025-03-20T03:50:33.823Z [INFO] server-59d5204d-7f1e-c021-b2ce-029b065a6169: added peer, starting replication: peer=server-d7e09412-edf8-6d85-328f-014c9f5d7829 +2025-03-20T03:50:33.823Z [INFO] server-59d5204d-7f1e-c021-b2ce-029b065a6169: pipelining replication: peer="{Voter server-ff76e54a-22c5-c70b-20f2-51d2e1c97b92 ff76e54a-22c5-c70b-20f2-51d2e1c97b92}" +2025-03-20T03:50:33.823Z [INFO] server-59d5204d-7f1e-c021-b2ce-029b065a6169: pipelining replication: peer="{Voter server-d7e09412-edf8-6d85-328f-014c9f5d7829 d7e09412-edf8-6d85-328f-014c9f5d7829}" +2025-03-20T03:50:34.828Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:50:33.823025321 +0000 UTC m=+65.497478161 (1 nodes), 632.390137ms from start of poll, 633.603136ms from cluster start. Timeout at 2025-03-20 03:50:34.828034009 +0000 UTC m=+66.502486806, 1.005008645s after stability +2025-03-20T03:50:34.831Z [INFO] creating new snapshot: path=/tmp/raft2125212828/snapshots/2-113-1742442634831.tmp +2025-03-20T03:50:34.832Z [INFO] server-59d5204d-7f1e-c021-b2ce-029b065a6169: copied to local snapshot: bytes=71 +2025-03-20T03:50:34.832Z [INFO] server-59d5204d-7f1e-c021-b2ce-029b065a6169: snapshot restore progress: id=2-113-1742442634831 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:34.832Z [INFO] server-59d5204d-7f1e-c021-b2ce-029b065a6169: restored user snapshot: index=113 +2025-03-20T03:50:34.832Z [ERROR] server-59d5204d-7f1e-c021-b2ce-029b065a6169: failed to get log: index=3 error="log not found" +2025-03-20T03:50:34.832Z [INFO] server-59d5204d-7f1e-c021-b2ce-029b065a6169: aborting pipeline replication: peer="{Voter server-d7e09412-edf8-6d85-328f-014c9f5d7829 d7e09412-edf8-6d85-328f-014c9f5d7829}" +2025-03-20T03:50:34.832Z [ERROR] server-59d5204d-7f1e-c021-b2ce-029b065a6169: failed to get log: index=3 error="log not found" +2025-03-20T03:50:34.832Z [INFO] server-59d5204d-7f1e-c021-b2ce-029b065a6169: aborting pipeline replication: peer="{Voter server-ff76e54a-22c5-c70b-20f2-51d2e1c97b92 ff76e54a-22c5-c70b-20f2-51d2e1c97b92}" +2025-03-20T03:50:34.842Z [ERROR] server-59d5204d-7f1e-c021-b2ce-029b065a6169: failed to get log: index=3 error="log not found" +2025-03-20T03:50:34.842Z [ERROR] server-59d5204d-7f1e-c021-b2ce-029b065a6169: failed to get log: index=3 error="log not found" +2025-03-20T03:50:34.843Z [INFO] creating new snapshot: path=/tmp/raft1867461722/snapshots/2-113-1742442634843.tmp +2025-03-20T03:50:34.843Z [INFO] creating new snapshot: path=/tmp/raft2120221436/snapshots/2-113-1742442634843.tmp +2025-03-20T03:50:34.843Z [INFO] server-d7e09412-edf8-6d85-328f-014c9f5d7829: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:34.843Z [INFO] server-ff76e54a-22c5-c70b-20f2-51d2e1c97b92: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:34.843Z [INFO] server-ff76e54a-22c5-c70b-20f2-51d2e1c97b92: copied to local snapshot: bytes=71 +2025-03-20T03:50:34.844Z [INFO] server-ff76e54a-22c5-c70b-20f2-51d2e1c97b92: snapshot restore progress: id=2-113-1742442634843 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:34.844Z [INFO] server-ff76e54a-22c5-c70b-20f2-51d2e1c97b92: Installed remote snapshot +2025-03-20T03:50:34.844Z [INFO] server-59d5204d-7f1e-c021-b2ce-029b065a6169: pipelining replication: peer="{Voter server-ff76e54a-22c5-c70b-20f2-51d2e1c97b92 ff76e54a-22c5-c70b-20f2-51d2e1c97b92}" +2025-03-20T03:50:34.844Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:50:34.845Z [INFO] server-d7e09412-edf8-6d85-328f-014c9f5d7829: copied to local snapshot: bytes=71 +2025-03-20T03:50:34.848Z [INFO] server-d7e09412-edf8-6d85-328f-014c9f5d7829: snapshot restore progress: id=2-113-1742442634843 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:34.850Z [INFO] server-d7e09412-edf8-6d85-328f-014c9f5d7829: Installed remote snapshot +2025-03-20T03:50:34.850Z [INFO] server-59d5204d-7f1e-c021-b2ce-029b065a6169: pipelining replication: peer="{Voter server-d7e09412-edf8-6d85-328f-014c9f5d7829 d7e09412-edf8-6d85-328f-014c9f5d7829}" +2025-03-20T03:50:35.849Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:50:34.844596975 +0000 UTC m=+66.519049814 (2 nodes), 1.946µs from start of poll, 1.655174789s from cluster start. Timeout at 2025-03-20 03:50:35.849641534 +0000 UTC m=+67.524094472, 1.005044658s after stability +2025-03-20T03:50:35.864Z [INFO] server-e3e8ff80-80cc-f75f-eee3-7588faded053: aborting pipeline replication: peer="{Voter server-68bc4e90-b8a0-7c64-c8cc-99553dd98d88 68bc4e90-b8a0-7c64-c8cc-99553dd98d88}" +2025-03-20T03:50:35.864Z [INFO] server-e3e8ff80-80cc-f75f-eee3-7588faded053: aborting pipeline replication: peer="{Voter server-9d9c7eeb-1017-0596-e0f9-d0727e5d804d 9d9c7eeb-1017-0596-e0f9-d0727e5d804d}" === RUN TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_true -2026-04-22T10:10:19.711Z [INFO] server-07973ad8-9170-e79e-dcf0-990c1e550bc3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-07973ad8-9170-e79e-dcf0-990c1e550bc3 Address:07973ad8-9170-e79e-dcf0-990c1e550bc3} {Suffrage:Voter ID:server-76a82980-c888-4e8c-0cb6-2966d429191a Address:76a82980-c888-4e8c-0cb6-2966d429191a} {Suffrage:Voter ID:server-2309cd97-aa43-957c-864f-7bff16a24e84 Address:2309cd97-aa43-957c-864f-7bff16a24e84}]" -2026-04-22T10:10:19.711Z [INFO] server-07973ad8-9170-e79e-dcf0-990c1e550bc3: entering follower state: follower="Node at 07973ad8-9170-e79e-dcf0-990c1e550bc3 [Follower]" leader-address= leader-id= -2026-04-22T10:10:19.711Z [INFO] server-76a82980-c888-4e8c-0cb6-2966d429191a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-07973ad8-9170-e79e-dcf0-990c1e550bc3 Address:07973ad8-9170-e79e-dcf0-990c1e550bc3} {Suffrage:Voter ID:server-76a82980-c888-4e8c-0cb6-2966d429191a Address:76a82980-c888-4e8c-0cb6-2966d429191a} {Suffrage:Voter ID:server-2309cd97-aa43-957c-864f-7bff16a24e84 Address:2309cd97-aa43-957c-864f-7bff16a24e84}]" -2026-04-22T10:10:19.711Z [INFO] server-76a82980-c888-4e8c-0cb6-2966d429191a: entering follower state: follower="Node at 76a82980-c888-4e8c-0cb6-2966d429191a [Follower]" leader-address= leader-id= -2026-04-22T10:10:19.711Z [INFO] server-2309cd97-aa43-957c-864f-7bff16a24e84: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-07973ad8-9170-e79e-dcf0-990c1e550bc3 Address:07973ad8-9170-e79e-dcf0-990c1e550bc3} {Suffrage:Voter ID:server-76a82980-c888-4e8c-0cb6-2966d429191a Address:76a82980-c888-4e8c-0cb6-2966d429191a} {Suffrage:Voter ID:server-2309cd97-aa43-957c-864f-7bff16a24e84 Address:2309cd97-aa43-957c-864f-7bff16a24e84}]" -2026-04-22T10:10:19.711Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:19.711Z [INFO] server-2309cd97-aa43-957c-864f-7bff16a24e84: entering follower state: follower="Node at 2309cd97-aa43-957c-864f-7bff16a24e84 [Follower]" leader-address= leader-id= -2026-04-22T10:10:20.230Z [WARN] server-2309cd97-aa43-957c-864f-7bff16a24e84: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:20.230Z [INFO] server-2309cd97-aa43-957c-864f-7bff16a24e84: entering candidate state: node="Node at 2309cd97-aa43-957c-864f-7bff16a24e84 [Candidate]" term=2 -2026-04-22T10:10:20.230Z [INFO] server-2309cd97-aa43-957c-864f-7bff16a24e84: election won: term=2 tally=2 -2026-04-22T10:10:20.230Z [INFO] server-2309cd97-aa43-957c-864f-7bff16a24e84: entering leader state: leader="Node at 2309cd97-aa43-957c-864f-7bff16a24e84 [Leader]" -2026-04-22T10:10:20.230Z [INFO] server-2309cd97-aa43-957c-864f-7bff16a24e84: added peer, starting replication: peer=server-07973ad8-9170-e79e-dcf0-990c1e550bc3 -2026-04-22T10:10:20.230Z [INFO] server-2309cd97-aa43-957c-864f-7bff16a24e84: added peer, starting replication: peer=server-76a82980-c888-4e8c-0cb6-2966d429191a -2026-04-22T10:10:20.230Z [INFO] server-2309cd97-aa43-957c-864f-7bff16a24e84: pipelining replication: peer="{Voter server-76a82980-c888-4e8c-0cb6-2966d429191a 76a82980-c888-4e8c-0cb6-2966d429191a}" -2026-04-22T10:10:20.230Z [INFO] server-2309cd97-aa43-957c-864f-7bff16a24e84: pipelining replication: peer="{Voter server-07973ad8-9170-e79e-dcf0-990c1e550bc3 07973ad8-9170-e79e-dcf0-990c1e550bc3}" -2026-04-22T10:10:21.235Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:20.230355809 +0000 UTC m=+66.254151906 (1 nodes), 518.406684ms from start of poll, 518.759413ms from cluster start. Timeout at 2026-04-22 10:10:21.235446651 +0000 UTC m=+67.259242835, 1.005090929s after stability -2026-04-22T10:10:21.237Z [INFO] server-2309cd97-aa43-957c-864f-7bff16a24e84: starting snapshot up to: index=12 -2026-04-22T10:10:21.237Z [INFO] creating new snapshot: path=/tmp/raft3482561811/snapshots/2-12-1776852621237.tmp -2026-04-22T10:10:21.237Z [INFO] server-2309cd97-aa43-957c-864f-7bff16a24e84: snapshot complete up to: index=12 -2026-04-22T10:10:21.238Z [INFO] server-e61722ad-1c51-ff91-373d-9654c034430e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e61722ad-1c51-ff91-373d-9654c034430e Address:e61722ad-1c51-ff91-373d-9654c034430e} {Suffrage:Voter ID:server-a468eb11-0a8c-79e7-346d-5159613d0dbe Address:a468eb11-0a8c-79e7-346d-5159613d0dbe} {Suffrage:Voter ID:server-77538438-2d54-053a-ca33-1e628c71a620 Address:77538438-2d54-053a-ca33-1e628c71a620}]" -2026-04-22T10:10:21.238Z [INFO] server-e61722ad-1c51-ff91-373d-9654c034430e: entering follower state: follower="Node at e61722ad-1c51-ff91-373d-9654c034430e [Follower]" leader-address= leader-id= -2026-04-22T10:10:21.238Z [INFO] server-a468eb11-0a8c-79e7-346d-5159613d0dbe: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e61722ad-1c51-ff91-373d-9654c034430e Address:e61722ad-1c51-ff91-373d-9654c034430e} {Suffrage:Voter ID:server-a468eb11-0a8c-79e7-346d-5159613d0dbe Address:a468eb11-0a8c-79e7-346d-5159613d0dbe} {Suffrage:Voter ID:server-77538438-2d54-053a-ca33-1e628c71a620 Address:77538438-2d54-053a-ca33-1e628c71a620}]" -2026-04-22T10:10:21.238Z [INFO] server-77538438-2d54-053a-ca33-1e628c71a620: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e61722ad-1c51-ff91-373d-9654c034430e Address:e61722ad-1c51-ff91-373d-9654c034430e} {Suffrage:Voter ID:server-a468eb11-0a8c-79e7-346d-5159613d0dbe Address:a468eb11-0a8c-79e7-346d-5159613d0dbe} {Suffrage:Voter ID:server-77538438-2d54-053a-ca33-1e628c71a620 Address:77538438-2d54-053a-ca33-1e628c71a620}]" -2026-04-22T10:10:21.238Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:21.238Z [INFO] server-a468eb11-0a8c-79e7-346d-5159613d0dbe: entering follower state: follower="Node at a468eb11-0a8c-79e7-346d-5159613d0dbe [Follower]" leader-address= leader-id= -2026-04-22T10:10:21.238Z [INFO] server-77538438-2d54-053a-ca33-1e628c71a620: entering follower state: follower="Node at 77538438-2d54-053a-ca33-1e628c71a620 [Follower]" leader-address= leader-id= -2026-04-22T10:10:21.808Z [WARN] server-77538438-2d54-053a-ca33-1e628c71a620: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:21.808Z [INFO] server-77538438-2d54-053a-ca33-1e628c71a620: entering candidate state: node="Node at 77538438-2d54-053a-ca33-1e628c71a620 [Candidate]" term=2 -2026-04-22T10:10:21.808Z [INFO] server-77538438-2d54-053a-ca33-1e628c71a620: election won: term=2 tally=2 -2026-04-22T10:10:21.808Z [INFO] server-77538438-2d54-053a-ca33-1e628c71a620: entering leader state: leader="Node at 77538438-2d54-053a-ca33-1e628c71a620 [Leader]" -2026-04-22T10:10:21.808Z [INFO] server-77538438-2d54-053a-ca33-1e628c71a620: added peer, starting replication: peer=server-e61722ad-1c51-ff91-373d-9654c034430e -2026-04-22T10:10:21.808Z [INFO] server-77538438-2d54-053a-ca33-1e628c71a620: added peer, starting replication: peer=server-a468eb11-0a8c-79e7-346d-5159613d0dbe -2026-04-22T10:10:21.808Z [INFO] server-77538438-2d54-053a-ca33-1e628c71a620: pipelining replication: peer="{Voter server-a468eb11-0a8c-79e7-346d-5159613d0dbe a468eb11-0a8c-79e7-346d-5159613d0dbe}" -2026-04-22T10:10:21.808Z [INFO] server-77538438-2d54-053a-ca33-1e628c71a620: pipelining replication: peer="{Voter server-e61722ad-1c51-ff91-373d-9654c034430e e61722ad-1c51-ff91-373d-9654c034430e}" -2026-04-22T10:10:22.813Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:21.808510491 +0000 UTC m=+67.832306579 (1 nodes), 569.533184ms from start of poll, 569.757124ms from cluster start. Timeout at 2026-04-22 10:10:22.813532944 +0000 UTC m=+68.837329062, 1.005022483s after stability -2026-04-22T10:10:22.813Z [INFO] creating new snapshot: path=/tmp/raft2452288288/snapshots/2-113-1776852622813.tmp -2026-04-22T10:10:22.814Z [INFO] server-77538438-2d54-053a-ca33-1e628c71a620: copied to local snapshot: bytes=71 -2026-04-22T10:10:22.814Z [INFO] server-77538438-2d54-053a-ca33-1e628c71a620: snapshot restore progress: id=2-113-1776852622813 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:22.814Z [INFO] server-77538438-2d54-053a-ca33-1e628c71a620: removing all old logs from log store -2026-04-22T10:10:22.814Z [INFO] server-77538438-2d54-053a-ca33-1e628c71a620: compacting logs: from=1 to=2 -2026-04-22T10:10:22.814Z [INFO] server-77538438-2d54-053a-ca33-1e628c71a620: restored user snapshot: index=113 -2026-04-22T10:10:22.814Z [ERROR] server-77538438-2d54-053a-ca33-1e628c71a620: failed to get log: index=2 error="log not found" -2026-04-22T10:10:22.814Z [INFO] server-77538438-2d54-053a-ca33-1e628c71a620: aborting pipeline replication: peer="{Voter server-a468eb11-0a8c-79e7-346d-5159613d0dbe a468eb11-0a8c-79e7-346d-5159613d0dbe}" -2026-04-22T10:10:22.814Z [ERROR] server-77538438-2d54-053a-ca33-1e628c71a620: failed to get log: index=2 error="log not found" -2026-04-22T10:10:22.814Z [INFO] server-77538438-2d54-053a-ca33-1e628c71a620: aborting pipeline replication: peer="{Voter server-e61722ad-1c51-ff91-373d-9654c034430e e61722ad-1c51-ff91-373d-9654c034430e}" -2026-04-22T10:10:22.821Z [ERROR] server-77538438-2d54-053a-ca33-1e628c71a620: failed to get log: index=2 error="log not found" -2026-04-22T10:10:22.821Z [INFO] creating new snapshot: path=/tmp/raft2572710521/snapshots/2-113-1776852622821.tmp -2026-04-22T10:10:22.821Z [INFO] server-e61722ad-1c51-ff91-373d-9654c034430e: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:22.821Z [ERROR] server-77538438-2d54-053a-ca33-1e628c71a620: failed to get log: index=2 error="log not found" -2026-04-22T10:10:22.822Z [INFO] creating new snapshot: path=/tmp/raft1937850857/snapshots/2-113-1776852622822.tmp -2026-04-22T10:10:22.822Z [INFO] server-e61722ad-1c51-ff91-373d-9654c034430e: copied to local snapshot: bytes=71 -2026-04-22T10:10:22.822Z [INFO] server-a468eb11-0a8c-79e7-346d-5159613d0dbe: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:22.822Z [INFO] server-e61722ad-1c51-ff91-373d-9654c034430e: snapshot restore progress: id=2-113-1776852622821 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:22.822Z [INFO] server-e61722ad-1c51-ff91-373d-9654c034430e: removing all old logs from log store -2026-04-22T10:10:22.822Z [INFO] server-e61722ad-1c51-ff91-373d-9654c034430e: compacting logs: from=1 to=2 -2026-04-22T10:10:22.822Z [INFO] server-e61722ad-1c51-ff91-373d-9654c034430e: Installed remote snapshot -2026-04-22T10:10:22.822Z [INFO] server-77538438-2d54-053a-ca33-1e628c71a620: pipelining replication: peer="{Voter server-e61722ad-1c51-ff91-373d-9654c034430e e61722ad-1c51-ff91-373d-9654c034430e}" -2026-04-22T10:10:22.822Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:10:22.822Z [INFO] server-a468eb11-0a8c-79e7-346d-5159613d0dbe: copied to local snapshot: bytes=71 -2026-04-22T10:10:22.822Z [INFO] server-a468eb11-0a8c-79e7-346d-5159613d0dbe: snapshot restore progress: id=2-113-1776852622822 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:22.822Z [INFO] server-a468eb11-0a8c-79e7-346d-5159613d0dbe: removing all old logs from log store -2026-04-22T10:10:22.822Z [INFO] server-a468eb11-0a8c-79e7-346d-5159613d0dbe: compacting logs: from=1 to=2 -2026-04-22T10:10:22.822Z [INFO] server-a468eb11-0a8c-79e7-346d-5159613d0dbe: Installed remote snapshot -2026-04-22T10:10:22.822Z [INFO] server-77538438-2d54-053a-ca33-1e628c71a620: pipelining replication: peer="{Voter server-a468eb11-0a8c-79e7-346d-5159613d0dbe a468eb11-0a8c-79e7-346d-5159613d0dbe}" -2026-04-22T10:10:23.828Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:10:22.822594655 +0000 UTC m=+68.846390764 (2 nodes), 806ns from start of poll, 1.583841309s from cluster start. Timeout at 2026-04-22 10:10:23.827601802 +0000 UTC m=+69.851397918, 1.005007154s after stability -2026-04-22T10:10:23.845Z [INFO] server-2309cd97-aa43-957c-864f-7bff16a24e84: aborting pipeline replication: peer="{Voter server-07973ad8-9170-e79e-dcf0-990c1e550bc3 07973ad8-9170-e79e-dcf0-990c1e550bc3}" -2026-04-22T10:10:23.845Z [INFO] server-2309cd97-aa43-957c-864f-7bff16a24e84: aborting pipeline replication: peer="{Voter server-76a82980-c888-4e8c-0cb6-2966d429191a 76a82980-c888-4e8c-0cb6-2966d429191a}" +2025-03-20T03:50:35.866Z [INFO] server-c090ee41-e15d-f14a-b01b-a0a8de320b69: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c090ee41-e15d-f14a-b01b-a0a8de320b69 Address:c090ee41-e15d-f14a-b01b-a0a8de320b69} {Suffrage:Voter ID:server-15fe07e5-8bed-b7ee-b717-19b856448d04 Address:15fe07e5-8bed-b7ee-b717-19b856448d04} {Suffrage:Voter ID:server-967de951-b9f8-d7d3-2628-4a47ce01c070 Address:967de951-b9f8-d7d3-2628-4a47ce01c070}]" +2025-03-20T03:50:35.866Z [INFO] server-c090ee41-e15d-f14a-b01b-a0a8de320b69: entering follower state: follower="Node at c090ee41-e15d-f14a-b01b-a0a8de320b69 [Follower]" leader-address= leader-id= +2025-03-20T03:50:35.866Z [INFO] server-15fe07e5-8bed-b7ee-b717-19b856448d04: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c090ee41-e15d-f14a-b01b-a0a8de320b69 Address:c090ee41-e15d-f14a-b01b-a0a8de320b69} {Suffrage:Voter ID:server-15fe07e5-8bed-b7ee-b717-19b856448d04 Address:15fe07e5-8bed-b7ee-b717-19b856448d04} {Suffrage:Voter ID:server-967de951-b9f8-d7d3-2628-4a47ce01c070 Address:967de951-b9f8-d7d3-2628-4a47ce01c070}]" +2025-03-20T03:50:35.867Z [INFO] server-967de951-b9f8-d7d3-2628-4a47ce01c070: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c090ee41-e15d-f14a-b01b-a0a8de320b69 Address:c090ee41-e15d-f14a-b01b-a0a8de320b69} {Suffrage:Voter ID:server-15fe07e5-8bed-b7ee-b717-19b856448d04 Address:15fe07e5-8bed-b7ee-b717-19b856448d04} {Suffrage:Voter ID:server-967de951-b9f8-d7d3-2628-4a47ce01c070 Address:967de951-b9f8-d7d3-2628-4a47ce01c070}]" +2025-03-20T03:50:35.867Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:50:35.867Z [INFO] server-15fe07e5-8bed-b7ee-b717-19b856448d04: entering follower state: follower="Node at 15fe07e5-8bed-b7ee-b717-19b856448d04 [Follower]" leader-address= leader-id= +2025-03-20T03:50:35.867Z [INFO] server-967de951-b9f8-d7d3-2628-4a47ce01c070: entering follower state: follower="Node at 967de951-b9f8-d7d3-2628-4a47ce01c070 [Follower]" leader-address= leader-id= +2025-03-20T03:50:36.398Z [WARN] server-967de951-b9f8-d7d3-2628-4a47ce01c070: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:50:36.398Z [INFO] server-967de951-b9f8-d7d3-2628-4a47ce01c070: entering candidate state: node="Node at 967de951-b9f8-d7d3-2628-4a47ce01c070 [Candidate]" term=2 +2025-03-20T03:50:36.398Z [INFO] server-967de951-b9f8-d7d3-2628-4a47ce01c070: election won: term=2 tally=2 +2025-03-20T03:50:36.398Z [INFO] server-967de951-b9f8-d7d3-2628-4a47ce01c070: entering leader state: leader="Node at 967de951-b9f8-d7d3-2628-4a47ce01c070 [Leader]" +2025-03-20T03:50:36.399Z [INFO] server-967de951-b9f8-d7d3-2628-4a47ce01c070: added peer, starting replication: peer=server-c090ee41-e15d-f14a-b01b-a0a8de320b69 +2025-03-20T03:50:36.399Z [INFO] server-967de951-b9f8-d7d3-2628-4a47ce01c070: added peer, starting replication: peer=server-15fe07e5-8bed-b7ee-b717-19b856448d04 +2025-03-20T03:50:36.399Z [INFO] server-967de951-b9f8-d7d3-2628-4a47ce01c070: pipelining replication: peer="{Voter server-c090ee41-e15d-f14a-b01b-a0a8de320b69 c090ee41-e15d-f14a-b01b-a0a8de320b69}" +2025-03-20T03:50:36.400Z [INFO] server-967de951-b9f8-d7d3-2628-4a47ce01c070: pipelining replication: peer="{Voter server-15fe07e5-8bed-b7ee-b717-19b856448d04 15fe07e5-8bed-b7ee-b717-19b856448d04}" +2025-03-20T03:50:37.406Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:50:36.399232751 +0000 UTC m=+68.073685866 (1 nodes), 531.994279ms from start of poll, 532.942836ms from cluster start. Timeout at 2025-03-20 03:50:37.404257308 +0000 UTC m=+69.078710336, 1.00502447s after stability +2025-03-20T03:50:37.407Z [INFO] server-967de951-b9f8-d7d3-2628-4a47ce01c070: starting snapshot up to: index=12 +2025-03-20T03:50:37.408Z [INFO] creating new snapshot: path=/tmp/raft4245852889/snapshots/2-12-1742442637408.tmp +2025-03-20T03:50:37.408Z [INFO] server-967de951-b9f8-d7d3-2628-4a47ce01c070: snapshot complete up to: index=12 +2025-03-20T03:50:37.411Z [INFO] server-6bdb146f-7390-8f0b-1f95-0701fe533ece: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6bdb146f-7390-8f0b-1f95-0701fe533ece Address:6bdb146f-7390-8f0b-1f95-0701fe533ece} {Suffrage:Voter ID:server-13fd62e2-66e5-4431-7236-8f7f1ff227cc Address:13fd62e2-66e5-4431-7236-8f7f1ff227cc} {Suffrage:Voter ID:server-6445ca0f-1046-5d1a-9b55-0eef9eaa38fd Address:6445ca0f-1046-5d1a-9b55-0eef9eaa38fd}]" +2025-03-20T03:50:37.411Z [INFO] server-13fd62e2-66e5-4431-7236-8f7f1ff227cc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6bdb146f-7390-8f0b-1f95-0701fe533ece Address:6bdb146f-7390-8f0b-1f95-0701fe533ece} {Suffrage:Voter ID:server-13fd62e2-66e5-4431-7236-8f7f1ff227cc Address:13fd62e2-66e5-4431-7236-8f7f1ff227cc} {Suffrage:Voter ID:server-6445ca0f-1046-5d1a-9b55-0eef9eaa38fd Address:6445ca0f-1046-5d1a-9b55-0eef9eaa38fd}]" +2025-03-20T03:50:37.411Z [INFO] server-6bdb146f-7390-8f0b-1f95-0701fe533ece: entering follower state: follower="Node at 6bdb146f-7390-8f0b-1f95-0701fe533ece [Follower]" leader-address= leader-id= +2025-03-20T03:50:37.411Z [INFO] server-13fd62e2-66e5-4431-7236-8f7f1ff227cc: entering follower state: follower="Node at 13fd62e2-66e5-4431-7236-8f7f1ff227cc [Follower]" leader-address= leader-id= +2025-03-20T03:50:37.414Z [INFO] server-6445ca0f-1046-5d1a-9b55-0eef9eaa38fd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6bdb146f-7390-8f0b-1f95-0701fe533ece Address:6bdb146f-7390-8f0b-1f95-0701fe533ece} {Suffrage:Voter ID:server-13fd62e2-66e5-4431-7236-8f7f1ff227cc Address:13fd62e2-66e5-4431-7236-8f7f1ff227cc} {Suffrage:Voter ID:server-6445ca0f-1046-5d1a-9b55-0eef9eaa38fd Address:6445ca0f-1046-5d1a-9b55-0eef9eaa38fd}]" +2025-03-20T03:50:37.414Z [INFO] server-6445ca0f-1046-5d1a-9b55-0eef9eaa38fd: entering follower state: follower="Node at 6445ca0f-1046-5d1a-9b55-0eef9eaa38fd [Follower]" leader-address= leader-id= +2025-03-20T03:50:37.427Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:50:37.926Z [WARN] server-13fd62e2-66e5-4431-7236-8f7f1ff227cc: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:50:37.926Z [INFO] server-13fd62e2-66e5-4431-7236-8f7f1ff227cc: entering candidate state: node="Node at 13fd62e2-66e5-4431-7236-8f7f1ff227cc [Candidate]" term=2 +2025-03-20T03:50:37.927Z [INFO] server-13fd62e2-66e5-4431-7236-8f7f1ff227cc: election won: term=2 tally=2 +2025-03-20T03:50:37.927Z [INFO] server-13fd62e2-66e5-4431-7236-8f7f1ff227cc: entering leader state: leader="Node at 13fd62e2-66e5-4431-7236-8f7f1ff227cc [Leader]" +2025-03-20T03:50:37.927Z [INFO] server-13fd62e2-66e5-4431-7236-8f7f1ff227cc: added peer, starting replication: peer=server-6bdb146f-7390-8f0b-1f95-0701fe533ece +2025-03-20T03:50:37.927Z [INFO] server-13fd62e2-66e5-4431-7236-8f7f1ff227cc: added peer, starting replication: peer=server-6445ca0f-1046-5d1a-9b55-0eef9eaa38fd +2025-03-20T03:50:37.927Z [INFO] server-13fd62e2-66e5-4431-7236-8f7f1ff227cc: pipelining replication: peer="{Voter server-6445ca0f-1046-5d1a-9b55-0eef9eaa38fd 6445ca0f-1046-5d1a-9b55-0eef9eaa38fd}" +2025-03-20T03:50:37.927Z [WARN] server-6445ca0f-1046-5d1a-9b55-0eef9eaa38fd: rejecting vote request since our last term is greater: candidate=13fd62e2-66e5-4431-7236-8f7f1ff227cc last-term=2 last-candidate-term=1 +2025-03-20T03:50:37.927Z [INFO] server-13fd62e2-66e5-4431-7236-8f7f1ff227cc: pipelining replication: peer="{Voter server-6bdb146f-7390-8f0b-1f95-0701fe533ece 6bdb146f-7390-8f0b-1f95-0701fe533ece}" +2025-03-20T03:50:38.933Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:50:37.927536502 +0000 UTC m=+69.601989356 (1 nodes), 499.633191ms from start of poll, 516.399488ms from cluster start. Timeout at 2025-03-20 03:50:38.932613751 +0000 UTC m=+70.607066593, 1.005077237s after stability +2025-03-20T03:50:38.934Z [INFO] creating new snapshot: path=/tmp/raft1526284784/snapshots/2-113-1742442638934.tmp +2025-03-20T03:50:38.936Z [INFO] server-13fd62e2-66e5-4431-7236-8f7f1ff227cc: copied to local snapshot: bytes=71 +2025-03-20T03:50:38.937Z [INFO] server-13fd62e2-66e5-4431-7236-8f7f1ff227cc: snapshot restore progress: id=2-113-1742442638934 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:38.937Z [INFO] server-13fd62e2-66e5-4431-7236-8f7f1ff227cc: removing all old logs from log store +2025-03-20T03:50:38.937Z [INFO] server-13fd62e2-66e5-4431-7236-8f7f1ff227cc: compacting logs: from=1 to=2 +2025-03-20T03:50:38.937Z [INFO] server-13fd62e2-66e5-4431-7236-8f7f1ff227cc: restored user snapshot: index=113 +2025-03-20T03:50:38.937Z [ERROR] server-13fd62e2-66e5-4431-7236-8f7f1ff227cc: failed to get log: index=2 error="log not found" +2025-03-20T03:50:38.937Z [INFO] server-13fd62e2-66e5-4431-7236-8f7f1ff227cc: aborting pipeline replication: peer="{Voter server-6bdb146f-7390-8f0b-1f95-0701fe533ece 6bdb146f-7390-8f0b-1f95-0701fe533ece}" +2025-03-20T03:50:38.937Z [ERROR] server-13fd62e2-66e5-4431-7236-8f7f1ff227cc: failed to get log: index=2 error="log not found" +2025-03-20T03:50:38.937Z [INFO] server-13fd62e2-66e5-4431-7236-8f7f1ff227cc: aborting pipeline replication: peer="{Voter server-6445ca0f-1046-5d1a-9b55-0eef9eaa38fd 6445ca0f-1046-5d1a-9b55-0eef9eaa38fd}" +2025-03-20T03:50:38.947Z [ERROR] server-13fd62e2-66e5-4431-7236-8f7f1ff227cc: failed to get log: index=2 error="log not found" +2025-03-20T03:50:38.949Z [ERROR] server-13fd62e2-66e5-4431-7236-8f7f1ff227cc: failed to get log: index=2 error="log not found" +2025-03-20T03:50:38.950Z [INFO] creating new snapshot: path=/tmp/raft3662529133/snapshots/2-113-1742442638950.tmp +2025-03-20T03:50:38.950Z [INFO] server-6445ca0f-1046-5d1a-9b55-0eef9eaa38fd: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:38.951Z [INFO] server-6445ca0f-1046-5d1a-9b55-0eef9eaa38fd: copied to local snapshot: bytes=71 +2025-03-20T03:50:38.951Z [INFO] server-6445ca0f-1046-5d1a-9b55-0eef9eaa38fd: snapshot restore progress: id=2-113-1742442638950 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:38.951Z [INFO] server-6445ca0f-1046-5d1a-9b55-0eef9eaa38fd: removing all old logs from log store +2025-03-20T03:50:38.951Z [INFO] server-6445ca0f-1046-5d1a-9b55-0eef9eaa38fd: compacting logs: from=1 to=2 +2025-03-20T03:50:38.951Z [INFO] server-6445ca0f-1046-5d1a-9b55-0eef9eaa38fd: Installed remote snapshot +2025-03-20T03:50:38.951Z [INFO] server-13fd62e2-66e5-4431-7236-8f7f1ff227cc: pipelining replication: peer="{Voter server-6445ca0f-1046-5d1a-9b55-0eef9eaa38fd 6445ca0f-1046-5d1a-9b55-0eef9eaa38fd}" +2025-03-20T03:50:38.951Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:50:38.955Z [INFO] creating new snapshot: path=/tmp/raft610068493/snapshots/2-113-1742442638955.tmp +2025-03-20T03:50:38.955Z [INFO] server-6bdb146f-7390-8f0b-1f95-0701fe533ece: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:38.956Z [INFO] server-6bdb146f-7390-8f0b-1f95-0701fe533ece: copied to local snapshot: bytes=71 +2025-03-20T03:50:38.956Z [INFO] server-6bdb146f-7390-8f0b-1f95-0701fe533ece: snapshot restore progress: id=2-113-1742442638955 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:38.956Z [INFO] server-6bdb146f-7390-8f0b-1f95-0701fe533ece: removing all old logs from log store +2025-03-20T03:50:38.956Z [INFO] server-6bdb146f-7390-8f0b-1f95-0701fe533ece: compacting logs: from=1 to=2 +2025-03-20T03:50:38.956Z [INFO] server-6bdb146f-7390-8f0b-1f95-0701fe533ece: Installed remote snapshot +2025-03-20T03:50:38.956Z [INFO] server-13fd62e2-66e5-4431-7236-8f7f1ff227cc: pipelining replication: peer="{Voter server-6bdb146f-7390-8f0b-1f95-0701fe533ece 6bdb146f-7390-8f0b-1f95-0701fe533ece}" +2025-03-20T03:50:39.957Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:50:38.951987818 +0000 UTC m=+70.626440887 (2 nodes), 3.173µs from start of poll, 1.540851019s from cluster start. Timeout at 2025-03-20 03:50:39.956994134 +0000 UTC m=+71.631447110, 1.005006223s after stability +2025-03-20T03:50:39.970Z [INFO] server-967de951-b9f8-d7d3-2628-4a47ce01c070: aborting pipeline replication: peer="{Voter server-15fe07e5-8bed-b7ee-b717-19b856448d04 15fe07e5-8bed-b7ee-b717-19b856448d04}" +2025-03-20T03:50:39.970Z [INFO] server-967de951-b9f8-d7d3-2628-4a47ce01c070: aborting pipeline replication: peer="{Voter server-c090ee41-e15d-f14a-b01b-a0a8de320b69 c090ee41-e15d-f14a-b01b-a0a8de320b69}" === RUN TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_false -2026-04-22T10:10:23.849Z [INFO] server-8f138bdf-0682-e997-6507-1e5b6e93e1a7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8f138bdf-0682-e997-6507-1e5b6e93e1a7 Address:8f138bdf-0682-e997-6507-1e5b6e93e1a7} {Suffrage:Voter ID:server-07a4df22-d471-72be-eb48-a813d8a0e9a6 Address:07a4df22-d471-72be-eb48-a813d8a0e9a6} {Suffrage:Voter ID:server-ece219b1-1681-20d8-c65f-784bafa7039c Address:ece219b1-1681-20d8-c65f-784bafa7039c}]" -2026-04-22T10:10:23.849Z [INFO] server-8f138bdf-0682-e997-6507-1e5b6e93e1a7: entering follower state: follower="Node at 8f138bdf-0682-e997-6507-1e5b6e93e1a7 [Follower]" leader-address= leader-id= -2026-04-22T10:10:23.849Z [INFO] server-07a4df22-d471-72be-eb48-a813d8a0e9a6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8f138bdf-0682-e997-6507-1e5b6e93e1a7 Address:8f138bdf-0682-e997-6507-1e5b6e93e1a7} {Suffrage:Voter ID:server-07a4df22-d471-72be-eb48-a813d8a0e9a6 Address:07a4df22-d471-72be-eb48-a813d8a0e9a6} {Suffrage:Voter ID:server-ece219b1-1681-20d8-c65f-784bafa7039c Address:ece219b1-1681-20d8-c65f-784bafa7039c}]" -2026-04-22T10:10:23.849Z [INFO] server-07a4df22-d471-72be-eb48-a813d8a0e9a6: entering follower state: follower="Node at 07a4df22-d471-72be-eb48-a813d8a0e9a6 [Follower]" leader-address= leader-id= -2026-04-22T10:10:23.849Z [INFO] server-ece219b1-1681-20d8-c65f-784bafa7039c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8f138bdf-0682-e997-6507-1e5b6e93e1a7 Address:8f138bdf-0682-e997-6507-1e5b6e93e1a7} {Suffrage:Voter ID:server-07a4df22-d471-72be-eb48-a813d8a0e9a6 Address:07a4df22-d471-72be-eb48-a813d8a0e9a6} {Suffrage:Voter ID:server-ece219b1-1681-20d8-c65f-784bafa7039c Address:ece219b1-1681-20d8-c65f-784bafa7039c}]" -2026-04-22T10:10:23.849Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:23.849Z [INFO] server-ece219b1-1681-20d8-c65f-784bafa7039c: entering follower state: follower="Node at ece219b1-1681-20d8-c65f-784bafa7039c [Follower]" leader-address= leader-id= -2026-04-22T10:10:24.357Z [WARN] server-8f138bdf-0682-e997-6507-1e5b6e93e1a7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:24.357Z [INFO] server-8f138bdf-0682-e997-6507-1e5b6e93e1a7: entering candidate state: node="Node at 8f138bdf-0682-e997-6507-1e5b6e93e1a7 [Candidate]" term=2 -2026-04-22T10:10:24.357Z [INFO] server-8f138bdf-0682-e997-6507-1e5b6e93e1a7: election won: term=2 tally=2 -2026-04-22T10:10:24.357Z [INFO] server-8f138bdf-0682-e997-6507-1e5b6e93e1a7: entering leader state: leader="Node at 8f138bdf-0682-e997-6507-1e5b6e93e1a7 [Leader]" -2026-04-22T10:10:24.357Z [INFO] server-8f138bdf-0682-e997-6507-1e5b6e93e1a7: added peer, starting replication: peer=server-07a4df22-d471-72be-eb48-a813d8a0e9a6 -2026-04-22T10:10:24.357Z [INFO] server-8f138bdf-0682-e997-6507-1e5b6e93e1a7: added peer, starting replication: peer=server-ece219b1-1681-20d8-c65f-784bafa7039c -2026-04-22T10:10:24.357Z [INFO] server-8f138bdf-0682-e997-6507-1e5b6e93e1a7: pipelining replication: peer="{Voter server-ece219b1-1681-20d8-c65f-784bafa7039c ece219b1-1681-20d8-c65f-784bafa7039c}" -2026-04-22T10:10:24.357Z [INFO] server-8f138bdf-0682-e997-6507-1e5b6e93e1a7: pipelining replication: peer="{Voter server-07a4df22-d471-72be-eb48-a813d8a0e9a6 07a4df22-d471-72be-eb48-a813d8a0e9a6}" -2026-04-22T10:10:25.362Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:24.357659732 +0000 UTC m=+70.381455845 (1 nodes), 507.73771ms from start of poll, 508.04482ms from cluster start. Timeout at 2026-04-22 10:10:25.362668258 +0000 UTC m=+71.386464404, 1.005008559s after stability -2026-04-22T10:10:25.363Z [INFO] server-8f138bdf-0682-e997-6507-1e5b6e93e1a7: starting snapshot up to: index=12 -2026-04-22T10:10:25.363Z [INFO] creating new snapshot: path=/tmp/raft364147191/snapshots/2-12-1776852625363.tmp -2026-04-22T10:10:25.364Z [INFO] server-8f138bdf-0682-e997-6507-1e5b6e93e1a7: snapshot complete up to: index=12 -2026-04-22T10:10:25.364Z [INFO] creating new snapshot: path=/tmp/raft364147191/snapshots/2-1013-1776852625364.tmp -2026-04-22T10:10:25.365Z [INFO] server-8f138bdf-0682-e997-6507-1e5b6e93e1a7: copied to local snapshot: bytes=71 -2026-04-22T10:10:25.365Z [INFO] server-8f138bdf-0682-e997-6507-1e5b6e93e1a7: snapshot restore progress: id=2-1013-1776852625364 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:25.365Z [INFO] server-8f138bdf-0682-e997-6507-1e5b6e93e1a7: restored user snapshot: index=1013 -2026-04-22T10:10:25.365Z [ERROR] server-8f138bdf-0682-e997-6507-1e5b6e93e1a7: failed to get log: index=23 error="log not found" -2026-04-22T10:10:25.365Z [ERROR] server-8f138bdf-0682-e997-6507-1e5b6e93e1a7: failed to get log: index=23 error="log not found" -2026-04-22T10:10:25.365Z [INFO] server-8f138bdf-0682-e997-6507-1e5b6e93e1a7: aborting pipeline replication: peer="{Voter server-07a4df22-d471-72be-eb48-a813d8a0e9a6 07a4df22-d471-72be-eb48-a813d8a0e9a6}" -2026-04-22T10:10:25.365Z [INFO] server-8f138bdf-0682-e997-6507-1e5b6e93e1a7: aborting pipeline replication: peer="{Voter server-ece219b1-1681-20d8-c65f-784bafa7039c ece219b1-1681-20d8-c65f-784bafa7039c}" -2026-04-22T10:10:25.372Z [ERROR] server-8f138bdf-0682-e997-6507-1e5b6e93e1a7: failed to get log: index=23 error="log not found" -2026-04-22T10:10:25.372Z [INFO] creating new snapshot: path=/tmp/raft3137963848/snapshots/2-1013-1776852625372.tmp -2026-04-22T10:10:25.372Z [INFO] server-ece219b1-1681-20d8-c65f-784bafa7039c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:25.373Z [INFO] server-ece219b1-1681-20d8-c65f-784bafa7039c: copied to local snapshot: bytes=71 -2026-04-22T10:10:25.373Z [INFO] server-ece219b1-1681-20d8-c65f-784bafa7039c: snapshot restore progress: id=2-1013-1776852625372 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:25.373Z [INFO] server-ece219b1-1681-20d8-c65f-784bafa7039c: Installed remote snapshot -2026-04-22T10:10:25.373Z [WARN] server-ece219b1-1681-20d8-c65f-784bafa7039c: skipping application of old log: index=1013 -2026-04-22T10:10:25.373Z [INFO] server-8f138bdf-0682-e997-6507-1e5b6e93e1a7: pipelining replication: peer="{Voter server-ece219b1-1681-20d8-c65f-784bafa7039c ece219b1-1681-20d8-c65f-784bafa7039c}" -2026-04-22T10:10:25.373Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:10:25.374Z [ERROR] server-8f138bdf-0682-e997-6507-1e5b6e93e1a7: failed to get log: index=23 error="log not found" -2026-04-22T10:10:25.375Z [INFO] creating new snapshot: path=/tmp/raft3639915264/snapshots/2-1013-1776852625375.tmp -2026-04-22T10:10:25.375Z [INFO] server-07a4df22-d471-72be-eb48-a813d8a0e9a6: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:25.375Z [INFO] server-07a4df22-d471-72be-eb48-a813d8a0e9a6: copied to local snapshot: bytes=71 -2026-04-22T10:10:25.375Z [INFO] server-07a4df22-d471-72be-eb48-a813d8a0e9a6: snapshot restore progress: id=2-1013-1776852625375 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:25.375Z [INFO] server-07a4df22-d471-72be-eb48-a813d8a0e9a6: Installed remote snapshot -2026-04-22T10:10:25.375Z [INFO] server-8f138bdf-0682-e997-6507-1e5b6e93e1a7: pipelining replication: peer="{Voter server-07a4df22-d471-72be-eb48-a813d8a0e9a6 07a4df22-d471-72be-eb48-a813d8a0e9a6}" -2026-04-22T10:10:26.379Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:10:25.37369818 +0000 UTC m=+71.397494282 (2 nodes), 877ns from start of poll, 1.524083257s from cluster start. Timeout at 2026-04-22 10:10:26.378711614 +0000 UTC m=+72.402507838, 1.005013556s after stability -2026-04-22T10:10:26.392Z [INFO] server-8f138bdf-0682-e997-6507-1e5b6e93e1a7: aborting pipeline replication: peer="{Voter server-ece219b1-1681-20d8-c65f-784bafa7039c ece219b1-1681-20d8-c65f-784bafa7039c}" -2026-04-22T10:10:26.392Z [INFO] server-8f138bdf-0682-e997-6507-1e5b6e93e1a7: aborting pipeline replication: peer="{Voter server-07a4df22-d471-72be-eb48-a813d8a0e9a6 07a4df22-d471-72be-eb48-a813d8a0e9a6}" +2025-03-20T03:50:39.972Z [INFO] server-96bd8fd5-3635-5fb2-a8d2-5167a017157a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-96bd8fd5-3635-5fb2-a8d2-5167a017157a Address:96bd8fd5-3635-5fb2-a8d2-5167a017157a} {Suffrage:Voter ID:server-4c95b148-d9b9-a73c-3a7c-3aa96785b708 Address:4c95b148-d9b9-a73c-3a7c-3aa96785b708} {Suffrage:Voter ID:server-613869b2-c687-9545-9135-85ae9fb90f02 Address:613869b2-c687-9545-9135-85ae9fb90f02}]" +2025-03-20T03:50:39.972Z [INFO] server-4c95b148-d9b9-a73c-3a7c-3aa96785b708: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-96bd8fd5-3635-5fb2-a8d2-5167a017157a Address:96bd8fd5-3635-5fb2-a8d2-5167a017157a} {Suffrage:Voter ID:server-4c95b148-d9b9-a73c-3a7c-3aa96785b708 Address:4c95b148-d9b9-a73c-3a7c-3aa96785b708} {Suffrage:Voter ID:server-613869b2-c687-9545-9135-85ae9fb90f02 Address:613869b2-c687-9545-9135-85ae9fb90f02}]" +2025-03-20T03:50:39.972Z [INFO] server-613869b2-c687-9545-9135-85ae9fb90f02: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-96bd8fd5-3635-5fb2-a8d2-5167a017157a Address:96bd8fd5-3635-5fb2-a8d2-5167a017157a} {Suffrage:Voter ID:server-4c95b148-d9b9-a73c-3a7c-3aa96785b708 Address:4c95b148-d9b9-a73c-3a7c-3aa96785b708} {Suffrage:Voter ID:server-613869b2-c687-9545-9135-85ae9fb90f02 Address:613869b2-c687-9545-9135-85ae9fb90f02}]" +2025-03-20T03:50:39.972Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:50:39.972Z [INFO] server-96bd8fd5-3635-5fb2-a8d2-5167a017157a: entering follower state: follower="Node at 96bd8fd5-3635-5fb2-a8d2-5167a017157a [Follower]" leader-address= leader-id= +2025-03-20T03:50:39.972Z [INFO] server-4c95b148-d9b9-a73c-3a7c-3aa96785b708: entering follower state: follower="Node at 4c95b148-d9b9-a73c-3a7c-3aa96785b708 [Follower]" leader-address= leader-id= +2025-03-20T03:50:39.972Z [INFO] server-613869b2-c687-9545-9135-85ae9fb90f02: entering follower state: follower="Node at 613869b2-c687-9545-9135-85ae9fb90f02 [Follower]" leader-address= leader-id= +2025-03-20T03:50:40.497Z [WARN] server-96bd8fd5-3635-5fb2-a8d2-5167a017157a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:50:40.497Z [INFO] server-96bd8fd5-3635-5fb2-a8d2-5167a017157a: entering candidate state: node="Node at 96bd8fd5-3635-5fb2-a8d2-5167a017157a [Candidate]" term=2 +2025-03-20T03:50:40.497Z [INFO] server-96bd8fd5-3635-5fb2-a8d2-5167a017157a: election won: term=2 tally=2 +2025-03-20T03:50:40.497Z [INFO] server-96bd8fd5-3635-5fb2-a8d2-5167a017157a: entering leader state: leader="Node at 96bd8fd5-3635-5fb2-a8d2-5167a017157a [Leader]" +2025-03-20T03:50:40.497Z [INFO] server-96bd8fd5-3635-5fb2-a8d2-5167a017157a: added peer, starting replication: peer=server-4c95b148-d9b9-a73c-3a7c-3aa96785b708 +2025-03-20T03:50:40.497Z [INFO] server-96bd8fd5-3635-5fb2-a8d2-5167a017157a: added peer, starting replication: peer=server-613869b2-c687-9545-9135-85ae9fb90f02 +2025-03-20T03:50:40.497Z [INFO] server-96bd8fd5-3635-5fb2-a8d2-5167a017157a: pipelining replication: peer="{Voter server-613869b2-c687-9545-9135-85ae9fb90f02 613869b2-c687-9545-9135-85ae9fb90f02}" +2025-03-20T03:50:40.497Z [INFO] server-96bd8fd5-3635-5fb2-a8d2-5167a017157a: pipelining replication: peer="{Voter server-4c95b148-d9b9-a73c-3a7c-3aa96785b708 4c95b148-d9b9-a73c-3a7c-3aa96785b708}" +2025-03-20T03:50:41.504Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:50:40.497448826 +0000 UTC m=+72.171901640 (1 nodes), 524.922305ms from start of poll, 525.301309ms from cluster start. Timeout at 2025-03-20 03:50:41.502464306 +0000 UTC m=+73.176917337, 1.005015697s after stability +2025-03-20T03:50:41.506Z [INFO] server-96bd8fd5-3635-5fb2-a8d2-5167a017157a: starting snapshot up to: index=12 +2025-03-20T03:50:41.506Z [INFO] creating new snapshot: path=/tmp/raft2638828251/snapshots/2-12-1742442641506.tmp +2025-03-20T03:50:41.507Z [INFO] server-96bd8fd5-3635-5fb2-a8d2-5167a017157a: snapshot complete up to: index=12 +2025-03-20T03:50:41.509Z [INFO] creating new snapshot: path=/tmp/raft2638828251/snapshots/2-1013-1742442641509.tmp +2025-03-20T03:50:41.510Z [INFO] server-96bd8fd5-3635-5fb2-a8d2-5167a017157a: copied to local snapshot: bytes=71 +2025-03-20T03:50:41.510Z [INFO] server-96bd8fd5-3635-5fb2-a8d2-5167a017157a: snapshot restore progress: id=2-1013-1742442641509 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:41.510Z [INFO] server-96bd8fd5-3635-5fb2-a8d2-5167a017157a: restored user snapshot: index=1013 +2025-03-20T03:50:41.510Z [ERROR] server-96bd8fd5-3635-5fb2-a8d2-5167a017157a: failed to get log: index=23 error="log not found" +2025-03-20T03:50:41.510Z [INFO] server-96bd8fd5-3635-5fb2-a8d2-5167a017157a: aborting pipeline replication: peer="{Voter server-4c95b148-d9b9-a73c-3a7c-3aa96785b708 4c95b148-d9b9-a73c-3a7c-3aa96785b708}" +2025-03-20T03:50:41.510Z [ERROR] server-96bd8fd5-3635-5fb2-a8d2-5167a017157a: failed to get log: index=23 error="log not found" +2025-03-20T03:50:41.510Z [INFO] server-96bd8fd5-3635-5fb2-a8d2-5167a017157a: aborting pipeline replication: peer="{Voter server-613869b2-c687-9545-9135-85ae9fb90f02 613869b2-c687-9545-9135-85ae9fb90f02}" +2025-03-20T03:50:41.517Z [ERROR] server-96bd8fd5-3635-5fb2-a8d2-5167a017157a: failed to get log: index=23 error="log not found" +2025-03-20T03:50:41.518Z [INFO] creating new snapshot: path=/tmp/raft2672715872/snapshots/2-1013-1742442641518.tmp +2025-03-20T03:50:41.518Z [ERROR] server-96bd8fd5-3635-5fb2-a8d2-5167a017157a: failed to get log: index=23 error="log not found" +2025-03-20T03:50:41.519Z [INFO] server-613869b2-c687-9545-9135-85ae9fb90f02: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:41.518Z [INFO] creating new snapshot: path=/tmp/raft2408600005/snapshots/2-1013-1742442641518.tmp +2025-03-20T03:50:41.519Z [INFO] server-613869b2-c687-9545-9135-85ae9fb90f02: copied to local snapshot: bytes=71 +2025-03-20T03:50:41.519Z [INFO] server-4c95b148-d9b9-a73c-3a7c-3aa96785b708: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:41.520Z [INFO] server-4c95b148-d9b9-a73c-3a7c-3aa96785b708: copied to local snapshot: bytes=71 +2025-03-20T03:50:41.520Z [INFO] server-613869b2-c687-9545-9135-85ae9fb90f02: snapshot restore progress: id=2-1013-1742442641518 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:41.520Z [INFO] server-613869b2-c687-9545-9135-85ae9fb90f02: Installed remote snapshot +2025-03-20T03:50:41.520Z [INFO] server-4c95b148-d9b9-a73c-3a7c-3aa96785b708: snapshot restore progress: id=2-1013-1742442641518 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:41.520Z [INFO] server-4c95b148-d9b9-a73c-3a7c-3aa96785b708: Installed remote snapshot +2025-03-20T03:50:41.520Z [WARN] server-613869b2-c687-9545-9135-85ae9fb90f02: skipping application of old log: index=22 +2025-03-20T03:50:41.520Z [INFO] server-96bd8fd5-3635-5fb2-a8d2-5167a017157a: pipelining replication: peer="{Voter server-613869b2-c687-9545-9135-85ae9fb90f02 613869b2-c687-9545-9135-85ae9fb90f02}" +2025-03-20T03:50:41.520Z [WARN] server-4c95b148-d9b9-a73c-3a7c-3aa96785b708: skipping application of old log: index=1013 +2025-03-20T03:50:41.520Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:50:41.520Z [INFO] server-96bd8fd5-3635-5fb2-a8d2-5167a017157a: pipelining replication: peer="{Voter server-4c95b148-d9b9-a73c-3a7c-3aa96785b708 4c95b148-d9b9-a73c-3a7c-3aa96785b708}" +2025-03-20T03:50:42.526Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:50:41.520920471 +0000 UTC m=+73.195373879 (2 nodes), 2.788µs from start of poll, 1.548773548s from cluster start. Timeout at 2025-03-20 03:50:42.525956883 +0000 UTC m=+74.200409713, 1.005035834s after stability +2025-03-20T03:50:42.540Z [INFO] server-96bd8fd5-3635-5fb2-a8d2-5167a017157a: aborting pipeline replication: peer="{Voter server-613869b2-c687-9545-9135-85ae9fb90f02 613869b2-c687-9545-9135-85ae9fb90f02}" +2025-03-20T03:50:42.540Z [INFO] server-96bd8fd5-3635-5fb2-a8d2-5167a017157a: aborting pipeline replication: peer="{Voter server-4c95b148-d9b9-a73c-3a7c-3aa96785b708 4c95b148-d9b9-a73c-3a7c-3aa96785b708}" === RUN TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_false -2026-04-22T10:10:26.393Z [INFO] server-693f3dbf-8ac8-16ec-75dc-1be133385b0e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-693f3dbf-8ac8-16ec-75dc-1be133385b0e Address:693f3dbf-8ac8-16ec-75dc-1be133385b0e} {Suffrage:Voter ID:server-18b0c58b-eb00-040a-86d9-bb62bea4b584 Address:18b0c58b-eb00-040a-86d9-bb62bea4b584} {Suffrage:Voter ID:server-f1c0d81b-c7ec-45d7-0326-772e73b3c628 Address:f1c0d81b-c7ec-45d7-0326-772e73b3c628}]" -2026-04-22T10:10:26.394Z [INFO] server-693f3dbf-8ac8-16ec-75dc-1be133385b0e: entering follower state: follower="Node at 693f3dbf-8ac8-16ec-75dc-1be133385b0e [Follower]" leader-address= leader-id= -2026-04-22T10:10:26.394Z [INFO] server-18b0c58b-eb00-040a-86d9-bb62bea4b584: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-693f3dbf-8ac8-16ec-75dc-1be133385b0e Address:693f3dbf-8ac8-16ec-75dc-1be133385b0e} {Suffrage:Voter ID:server-18b0c58b-eb00-040a-86d9-bb62bea4b584 Address:18b0c58b-eb00-040a-86d9-bb62bea4b584} {Suffrage:Voter ID:server-f1c0d81b-c7ec-45d7-0326-772e73b3c628 Address:f1c0d81b-c7ec-45d7-0326-772e73b3c628}]" -2026-04-22T10:10:26.394Z [INFO] server-18b0c58b-eb00-040a-86d9-bb62bea4b584: entering follower state: follower="Node at 18b0c58b-eb00-040a-86d9-bb62bea4b584 [Follower]" leader-address= leader-id= -2026-04-22T10:10:26.394Z [INFO] server-f1c0d81b-c7ec-45d7-0326-772e73b3c628: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-693f3dbf-8ac8-16ec-75dc-1be133385b0e Address:693f3dbf-8ac8-16ec-75dc-1be133385b0e} {Suffrage:Voter ID:server-18b0c58b-eb00-040a-86d9-bb62bea4b584 Address:18b0c58b-eb00-040a-86d9-bb62bea4b584} {Suffrage:Voter ID:server-f1c0d81b-c7ec-45d7-0326-772e73b3c628 Address:f1c0d81b-c7ec-45d7-0326-772e73b3c628}]" -2026-04-22T10:10:26.394Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:26.394Z [INFO] server-f1c0d81b-c7ec-45d7-0326-772e73b3c628: entering follower state: follower="Node at f1c0d81b-c7ec-45d7-0326-772e73b3c628 [Follower]" leader-address= leader-id= -2026-04-22T10:10:26.995Z [WARN] server-693f3dbf-8ac8-16ec-75dc-1be133385b0e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:26.995Z [INFO] server-693f3dbf-8ac8-16ec-75dc-1be133385b0e: entering candidate state: node="Node at 693f3dbf-8ac8-16ec-75dc-1be133385b0e [Candidate]" term=2 -2026-04-22T10:10:26.995Z [INFO] server-693f3dbf-8ac8-16ec-75dc-1be133385b0e: election won: term=2 tally=2 -2026-04-22T10:10:26.995Z [INFO] server-693f3dbf-8ac8-16ec-75dc-1be133385b0e: entering leader state: leader="Node at 693f3dbf-8ac8-16ec-75dc-1be133385b0e [Leader]" -2026-04-22T10:10:26.995Z [INFO] server-693f3dbf-8ac8-16ec-75dc-1be133385b0e: added peer, starting replication: peer=server-18b0c58b-eb00-040a-86d9-bb62bea4b584 -2026-04-22T10:10:26.995Z [INFO] server-693f3dbf-8ac8-16ec-75dc-1be133385b0e: added peer, starting replication: peer=server-f1c0d81b-c7ec-45d7-0326-772e73b3c628 -2026-04-22T10:10:26.995Z [INFO] server-693f3dbf-8ac8-16ec-75dc-1be133385b0e: pipelining replication: peer="{Voter server-f1c0d81b-c7ec-45d7-0326-772e73b3c628 f1c0d81b-c7ec-45d7-0326-772e73b3c628}" -2026-04-22T10:10:26.995Z [INFO] server-693f3dbf-8ac8-16ec-75dc-1be133385b0e: pipelining replication: peer="{Voter server-18b0c58b-eb00-040a-86d9-bb62bea4b584 18b0c58b-eb00-040a-86d9-bb62bea4b584}" -2026-04-22T10:10:28.000Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:26.995467489 +0000 UTC m=+73.019263606 (1 nodes), 600.85111ms from start of poll, 601.7643ms from cluster start. Timeout at 2026-04-22 10:10:28.000473985 +0000 UTC m=+74.024270101, 1.005006495s after stability -2026-04-22T10:10:28.001Z [INFO] server-693f3dbf-8ac8-16ec-75dc-1be133385b0e: starting snapshot up to: index=12 -2026-04-22T10:10:28.001Z [INFO] creating new snapshot: path=/tmp/raft3083185278/snapshots/2-12-1776852628001.tmp -2026-04-22T10:10:28.001Z [INFO] server-693f3dbf-8ac8-16ec-75dc-1be133385b0e: snapshot complete up to: index=12 -2026-04-22T10:10:28.002Z [INFO] creating new snapshot: path=/tmp/raft3083185278/snapshots/2-1013-1776852628002.tmp -2026-04-22T10:10:28.003Z [INFO] server-693f3dbf-8ac8-16ec-75dc-1be133385b0e: copied to local snapshot: bytes=71 -2026-04-22T10:10:28.003Z [INFO] server-693f3dbf-8ac8-16ec-75dc-1be133385b0e: snapshot restore progress: id=2-1013-1776852628002 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:28.003Z [INFO] server-693f3dbf-8ac8-16ec-75dc-1be133385b0e: removing all old logs from log store -2026-04-22T10:10:28.003Z [INFO] server-693f3dbf-8ac8-16ec-75dc-1be133385b0e: compacting logs: from=1 to=22 -2026-04-22T10:10:28.003Z [INFO] server-693f3dbf-8ac8-16ec-75dc-1be133385b0e: restored user snapshot: index=1013 -2026-04-22T10:10:28.003Z [ERROR] server-693f3dbf-8ac8-16ec-75dc-1be133385b0e: failed to get log: index=22 error="log not found" -2026-04-22T10:10:28.003Z [ERROR] server-693f3dbf-8ac8-16ec-75dc-1be133385b0e: failed to get log: index=22 error="log not found" -2026-04-22T10:10:28.003Z [INFO] server-693f3dbf-8ac8-16ec-75dc-1be133385b0e: aborting pipeline replication: peer="{Voter server-18b0c58b-eb00-040a-86d9-bb62bea4b584 18b0c58b-eb00-040a-86d9-bb62bea4b584}" -2026-04-22T10:10:28.003Z [INFO] server-693f3dbf-8ac8-16ec-75dc-1be133385b0e: aborting pipeline replication: peer="{Voter server-f1c0d81b-c7ec-45d7-0326-772e73b3c628 f1c0d81b-c7ec-45d7-0326-772e73b3c628}" -2026-04-22T10:10:28.009Z [ERROR] server-693f3dbf-8ac8-16ec-75dc-1be133385b0e: failed to get log: index=22 error="log not found" -2026-04-22T10:10:28.010Z [INFO] creating new snapshot: path=/tmp/raft1807093734/snapshots/2-1013-1776852628010.tmp -2026-04-22T10:10:28.010Z [INFO] server-18b0c58b-eb00-040a-86d9-bb62bea4b584: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:28.010Z [INFO] server-18b0c58b-eb00-040a-86d9-bb62bea4b584: copied to local snapshot: bytes=71 -2026-04-22T10:10:28.010Z [INFO] server-18b0c58b-eb00-040a-86d9-bb62bea4b584: snapshot restore progress: id=2-1013-1776852628010 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:28.010Z [INFO] server-18b0c58b-eb00-040a-86d9-bb62bea4b584: removing all old logs from log store -2026-04-22T10:10:28.010Z [INFO] server-18b0c58b-eb00-040a-86d9-bb62bea4b584: compacting logs: from=1 to=22 -2026-04-22T10:10:28.010Z [INFO] server-18b0c58b-eb00-040a-86d9-bb62bea4b584: Installed remote snapshot -2026-04-22T10:10:28.010Z [WARN] server-18b0c58b-eb00-040a-86d9-bb62bea4b584: skipping application of old log: index=22 -2026-04-22T10:10:28.010Z [INFO] server-693f3dbf-8ac8-16ec-75dc-1be133385b0e: pipelining replication: peer="{Voter server-18b0c58b-eb00-040a-86d9-bb62bea4b584 18b0c58b-eb00-040a-86d9-bb62bea4b584}" -2026-04-22T10:10:28.010Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:10:28.013Z [ERROR] server-693f3dbf-8ac8-16ec-75dc-1be133385b0e: failed to get log: index=22 error="log not found" -2026-04-22T10:10:28.013Z [INFO] creating new snapshot: path=/tmp/raft718060485/snapshots/2-1013-1776852628013.tmp -2026-04-22T10:10:28.013Z [INFO] server-f1c0d81b-c7ec-45d7-0326-772e73b3c628: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:28.014Z [INFO] server-f1c0d81b-c7ec-45d7-0326-772e73b3c628: copied to local snapshot: bytes=71 -2026-04-22T10:10:28.014Z [INFO] server-f1c0d81b-c7ec-45d7-0326-772e73b3c628: snapshot restore progress: id=2-1013-1776852628013 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:28.014Z [INFO] server-f1c0d81b-c7ec-45d7-0326-772e73b3c628: removing all old logs from log store -2026-04-22T10:10:28.014Z [INFO] server-f1c0d81b-c7ec-45d7-0326-772e73b3c628: compacting logs: from=1 to=22 -2026-04-22T10:10:28.014Z [INFO] server-f1c0d81b-c7ec-45d7-0326-772e73b3c628: Installed remote snapshot -2026-04-22T10:10:28.015Z [INFO] server-693f3dbf-8ac8-16ec-75dc-1be133385b0e: pipelining replication: peer="{Voter server-f1c0d81b-c7ec-45d7-0326-772e73b3c628 f1c0d81b-c7ec-45d7-0326-772e73b3c628}" -2026-04-22T10:10:29.016Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:10:28.010863136 +0000 UTC m=+74.034659222 (2 nodes), 629ns from start of poll, 1.617159916s from cluster start. Timeout at 2026-04-22 10:10:29.015866278 +0000 UTC m=+75.039662361, 1.005003139s after stability -2026-04-22T10:10:29.028Z [INFO] server-693f3dbf-8ac8-16ec-75dc-1be133385b0e: aborting pipeline replication: peer="{Voter server-f1c0d81b-c7ec-45d7-0326-772e73b3c628 f1c0d81b-c7ec-45d7-0326-772e73b3c628}" -2026-04-22T10:10:29.028Z [INFO] server-693f3dbf-8ac8-16ec-75dc-1be133385b0e: aborting pipeline replication: peer="{Voter server-18b0c58b-eb00-040a-86d9-bb62bea4b584 18b0c58b-eb00-040a-86d9-bb62bea4b584}" +2025-03-20T03:50:42.543Z [INFO] server-4f5d09fc-8ffd-e584-a695-6b568cac6a2b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4f5d09fc-8ffd-e584-a695-6b568cac6a2b Address:4f5d09fc-8ffd-e584-a695-6b568cac6a2b} {Suffrage:Voter ID:server-fdd83ef2-f1f1-b58f-1226-4a2c943d9094 Address:fdd83ef2-f1f1-b58f-1226-4a2c943d9094} {Suffrage:Voter ID:server-eb8eee47-c07a-2ee7-cdc0-8576948081a0 Address:eb8eee47-c07a-2ee7-cdc0-8576948081a0}]" +2025-03-20T03:50:42.543Z [INFO] server-fdd83ef2-f1f1-b58f-1226-4a2c943d9094: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4f5d09fc-8ffd-e584-a695-6b568cac6a2b Address:4f5d09fc-8ffd-e584-a695-6b568cac6a2b} {Suffrage:Voter ID:server-fdd83ef2-f1f1-b58f-1226-4a2c943d9094 Address:fdd83ef2-f1f1-b58f-1226-4a2c943d9094} {Suffrage:Voter ID:server-eb8eee47-c07a-2ee7-cdc0-8576948081a0 Address:eb8eee47-c07a-2ee7-cdc0-8576948081a0}]" +2025-03-20T03:50:42.543Z [INFO] server-eb8eee47-c07a-2ee7-cdc0-8576948081a0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4f5d09fc-8ffd-e584-a695-6b568cac6a2b Address:4f5d09fc-8ffd-e584-a695-6b568cac6a2b} {Suffrage:Voter ID:server-fdd83ef2-f1f1-b58f-1226-4a2c943d9094 Address:fdd83ef2-f1f1-b58f-1226-4a2c943d9094} {Suffrage:Voter ID:server-eb8eee47-c07a-2ee7-cdc0-8576948081a0 Address:eb8eee47-c07a-2ee7-cdc0-8576948081a0}]" +2025-03-20T03:50:42.543Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:50:42.543Z [INFO] server-4f5d09fc-8ffd-e584-a695-6b568cac6a2b: entering follower state: follower="Node at 4f5d09fc-8ffd-e584-a695-6b568cac6a2b [Follower]" leader-address= leader-id= +2025-03-20T03:50:42.543Z [INFO] server-fdd83ef2-f1f1-b58f-1226-4a2c943d9094: entering follower state: follower="Node at fdd83ef2-f1f1-b58f-1226-4a2c943d9094 [Follower]" leader-address= leader-id= +2025-03-20T03:50:42.543Z [INFO] server-eb8eee47-c07a-2ee7-cdc0-8576948081a0: entering follower state: follower="Node at eb8eee47-c07a-2ee7-cdc0-8576948081a0 [Follower]" leader-address= leader-id= +2025-03-20T03:50:43.078Z [WARN] server-4f5d09fc-8ffd-e584-a695-6b568cac6a2b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:50:43.078Z [INFO] server-4f5d09fc-8ffd-e584-a695-6b568cac6a2b: entering candidate state: node="Node at 4f5d09fc-8ffd-e584-a695-6b568cac6a2b [Candidate]" term=2 +2025-03-20T03:50:43.078Z [INFO] server-4f5d09fc-8ffd-e584-a695-6b568cac6a2b: election won: term=2 tally=2 +2025-03-20T03:50:43.078Z [INFO] server-4f5d09fc-8ffd-e584-a695-6b568cac6a2b: entering leader state: leader="Node at 4f5d09fc-8ffd-e584-a695-6b568cac6a2b [Leader]" +2025-03-20T03:50:43.078Z [INFO] server-4f5d09fc-8ffd-e584-a695-6b568cac6a2b: added peer, starting replication: peer=server-fdd83ef2-f1f1-b58f-1226-4a2c943d9094 +2025-03-20T03:50:43.078Z [INFO] server-4f5d09fc-8ffd-e584-a695-6b568cac6a2b: added peer, starting replication: peer=server-eb8eee47-c07a-2ee7-cdc0-8576948081a0 +2025-03-20T03:50:43.079Z [INFO] server-4f5d09fc-8ffd-e584-a695-6b568cac6a2b: pipelining replication: peer="{Voter server-eb8eee47-c07a-2ee7-cdc0-8576948081a0 eb8eee47-c07a-2ee7-cdc0-8576948081a0}" +2025-03-20T03:50:43.079Z [INFO] server-4f5d09fc-8ffd-e584-a695-6b568cac6a2b: pipelining replication: peer="{Voter server-fdd83ef2-f1f1-b58f-1226-4a2c943d9094 fdd83ef2-f1f1-b58f-1226-4a2c943d9094}" +2025-03-20T03:50:44.084Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:50:43.079410751 +0000 UTC m=+74.753863725 (1 nodes), 535.883555ms from start of poll, 536.314012ms from cluster start. Timeout at 2025-03-20 03:50:44.084418069 +0000 UTC m=+75.758870838, 1.005007113s after stability +2025-03-20T03:50:44.086Z [INFO] server-4f5d09fc-8ffd-e584-a695-6b568cac6a2b: starting snapshot up to: index=12 +2025-03-20T03:50:44.086Z [INFO] creating new snapshot: path=/tmp/raft801834933/snapshots/2-12-1742442644086.tmp +2025-03-20T03:50:44.087Z [INFO] server-4f5d09fc-8ffd-e584-a695-6b568cac6a2b: snapshot complete up to: index=12 +2025-03-20T03:50:44.090Z [INFO] creating new snapshot: path=/tmp/raft801834933/snapshots/2-1013-1742442644090.tmp +2025-03-20T03:50:44.092Z [INFO] server-4f5d09fc-8ffd-e584-a695-6b568cac6a2b: copied to local snapshot: bytes=71 +2025-03-20T03:50:44.092Z [INFO] server-4f5d09fc-8ffd-e584-a695-6b568cac6a2b: snapshot restore progress: id=2-1013-1742442644090 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:44.092Z [INFO] server-4f5d09fc-8ffd-e584-a695-6b568cac6a2b: removing all old logs from log store +2025-03-20T03:50:44.092Z [INFO] server-4f5d09fc-8ffd-e584-a695-6b568cac6a2b: compacting logs: from=1 to=22 +2025-03-20T03:50:44.092Z [INFO] server-4f5d09fc-8ffd-e584-a695-6b568cac6a2b: restored user snapshot: index=1013 +2025-03-20T03:50:44.092Z [ERROR] server-4f5d09fc-8ffd-e584-a695-6b568cac6a2b: failed to get log: index=22 error="log not found" +2025-03-20T03:50:44.092Z [ERROR] server-4f5d09fc-8ffd-e584-a695-6b568cac6a2b: failed to get log: index=22 error="log not found" +2025-03-20T03:50:44.092Z [INFO] server-4f5d09fc-8ffd-e584-a695-6b568cac6a2b: aborting pipeline replication: peer="{Voter server-eb8eee47-c07a-2ee7-cdc0-8576948081a0 eb8eee47-c07a-2ee7-cdc0-8576948081a0}" +2025-03-20T03:50:44.092Z [INFO] server-4f5d09fc-8ffd-e584-a695-6b568cac6a2b: aborting pipeline replication: peer="{Voter server-fdd83ef2-f1f1-b58f-1226-4a2c943d9094 fdd83ef2-f1f1-b58f-1226-4a2c943d9094}" +2025-03-20T03:50:44.100Z [ERROR] server-4f5d09fc-8ffd-e584-a695-6b568cac6a2b: failed to get log: index=22 error="log not found" +2025-03-20T03:50:44.101Z [INFO] creating new snapshot: path=/tmp/raft876999471/snapshots/2-1013-1742442644101.tmp +2025-03-20T03:50:44.101Z [INFO] server-fdd83ef2-f1f1-b58f-1226-4a2c943d9094: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:44.101Z [ERROR] server-4f5d09fc-8ffd-e584-a695-6b568cac6a2b: failed to get log: index=22 error="log not found" +2025-03-20T03:50:44.102Z [INFO] server-fdd83ef2-f1f1-b58f-1226-4a2c943d9094: copied to local snapshot: bytes=71 +2025-03-20T03:50:44.102Z [INFO] server-fdd83ef2-f1f1-b58f-1226-4a2c943d9094: snapshot restore progress: id=2-1013-1742442644101 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:44.102Z [INFO] server-fdd83ef2-f1f1-b58f-1226-4a2c943d9094: removing all old logs from log store +2025-03-20T03:50:44.102Z [INFO] server-fdd83ef2-f1f1-b58f-1226-4a2c943d9094: compacting logs: from=1 to=22 +2025-03-20T03:50:44.102Z [INFO] server-fdd83ef2-f1f1-b58f-1226-4a2c943d9094: Installed remote snapshot +2025-03-20T03:50:44.102Z [WARN] server-fdd83ef2-f1f1-b58f-1226-4a2c943d9094: skipping application of old log: index=22 +2025-03-20T03:50:44.102Z [INFO] server-4f5d09fc-8ffd-e584-a695-6b568cac6a2b: pipelining replication: peer="{Voter server-fdd83ef2-f1f1-b58f-1226-4a2c943d9094 fdd83ef2-f1f1-b58f-1226-4a2c943d9094}" +2025-03-20T03:50:44.102Z [INFO] creating new snapshot: path=/tmp/raft1593199920/snapshots/2-1013-1742442644102.tmp +2025-03-20T03:50:44.103Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:50:44.108Z [INFO] server-eb8eee47-c07a-2ee7-cdc0-8576948081a0: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:44.110Z [INFO] server-eb8eee47-c07a-2ee7-cdc0-8576948081a0: copied to local snapshot: bytes=71 +2025-03-20T03:50:44.110Z [INFO] server-eb8eee47-c07a-2ee7-cdc0-8576948081a0: snapshot restore progress: id=2-1013-1742442644102 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:44.110Z [INFO] server-eb8eee47-c07a-2ee7-cdc0-8576948081a0: removing all old logs from log store +2025-03-20T03:50:44.110Z [INFO] server-eb8eee47-c07a-2ee7-cdc0-8576948081a0: compacting logs: from=1 to=22 +2025-03-20T03:50:44.110Z [INFO] server-eb8eee47-c07a-2ee7-cdc0-8576948081a0: Installed remote snapshot +2025-03-20T03:50:44.111Z [INFO] server-4f5d09fc-8ffd-e584-a695-6b568cac6a2b: pipelining replication: peer="{Voter server-eb8eee47-c07a-2ee7-cdc0-8576948081a0 eb8eee47-c07a-2ee7-cdc0-8576948081a0}" +2025-03-20T03:50:45.108Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:50:44.103019308 +0000 UTC m=+75.777472093 (2 nodes), 2.009µs from start of poll, 1.55992238s from cluster start. Timeout at 2025-03-20 03:50:45.108066734 +0000 UTC m=+76.782519590, 1.005047497s after stability +2025-03-20T03:50:45.121Z [INFO] server-4f5d09fc-8ffd-e584-a695-6b568cac6a2b: aborting pipeline replication: peer="{Voter server-eb8eee47-c07a-2ee7-cdc0-8576948081a0 eb8eee47-c07a-2ee7-cdc0-8576948081a0}" +2025-03-20T03:50:45.121Z [INFO] server-4f5d09fc-8ffd-e584-a695-6b568cac6a2b: aborting pipeline replication: peer="{Voter server-fdd83ef2-f1f1-b58f-1226-4a2c943d9094 fdd83ef2-f1f1-b58f-1226-4a2c943d9094}" === RUN TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_true -2026-04-22T10:10:29.033Z [INFO] server-9e30c983-0aa8-13c7-5d6e-153fde24f5f3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9e30c983-0aa8-13c7-5d6e-153fde24f5f3 Address:9e30c983-0aa8-13c7-5d6e-153fde24f5f3} {Suffrage:Voter ID:server-54b3284e-4caa-7ca7-0c3c-a7c933c43379 Address:54b3284e-4caa-7ca7-0c3c-a7c933c43379} {Suffrage:Voter ID:server-fb05d6cf-79f8-cc0f-a861-81fc30896f50 Address:fb05d6cf-79f8-cc0f-a861-81fc30896f50}]" -2026-04-22T10:10:29.033Z [INFO] server-9e30c983-0aa8-13c7-5d6e-153fde24f5f3: entering follower state: follower="Node at 9e30c983-0aa8-13c7-5d6e-153fde24f5f3 [Follower]" leader-address= leader-id= -2026-04-22T10:10:29.033Z [INFO] server-54b3284e-4caa-7ca7-0c3c-a7c933c43379: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9e30c983-0aa8-13c7-5d6e-153fde24f5f3 Address:9e30c983-0aa8-13c7-5d6e-153fde24f5f3} {Suffrage:Voter ID:server-54b3284e-4caa-7ca7-0c3c-a7c933c43379 Address:54b3284e-4caa-7ca7-0c3c-a7c933c43379} {Suffrage:Voter ID:server-fb05d6cf-79f8-cc0f-a861-81fc30896f50 Address:fb05d6cf-79f8-cc0f-a861-81fc30896f50}]" -2026-04-22T10:10:29.033Z [INFO] server-54b3284e-4caa-7ca7-0c3c-a7c933c43379: entering follower state: follower="Node at 54b3284e-4caa-7ca7-0c3c-a7c933c43379 [Follower]" leader-address= leader-id= -2026-04-22T10:10:29.033Z [INFO] server-fb05d6cf-79f8-cc0f-a861-81fc30896f50: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9e30c983-0aa8-13c7-5d6e-153fde24f5f3 Address:9e30c983-0aa8-13c7-5d6e-153fde24f5f3} {Suffrage:Voter ID:server-54b3284e-4caa-7ca7-0c3c-a7c933c43379 Address:54b3284e-4caa-7ca7-0c3c-a7c933c43379} {Suffrage:Voter ID:server-fb05d6cf-79f8-cc0f-a861-81fc30896f50 Address:fb05d6cf-79f8-cc0f-a861-81fc30896f50}]" -2026-04-22T10:10:29.033Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:29.033Z [INFO] server-fb05d6cf-79f8-cc0f-a861-81fc30896f50: entering follower state: follower="Node at fb05d6cf-79f8-cc0f-a861-81fc30896f50 [Follower]" leader-address= leader-id= -2026-04-22T10:10:29.570Z [WARN] server-fb05d6cf-79f8-cc0f-a861-81fc30896f50: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:29.570Z [INFO] server-fb05d6cf-79f8-cc0f-a861-81fc30896f50: entering candidate state: node="Node at fb05d6cf-79f8-cc0f-a861-81fc30896f50 [Candidate]" term=2 -2026-04-22T10:10:29.570Z [INFO] server-fb05d6cf-79f8-cc0f-a861-81fc30896f50: election won: term=2 tally=2 -2026-04-22T10:10:29.570Z [INFO] server-fb05d6cf-79f8-cc0f-a861-81fc30896f50: entering leader state: leader="Node at fb05d6cf-79f8-cc0f-a861-81fc30896f50 [Leader]" -2026-04-22T10:10:29.570Z [INFO] server-fb05d6cf-79f8-cc0f-a861-81fc30896f50: added peer, starting replication: peer=server-9e30c983-0aa8-13c7-5d6e-153fde24f5f3 -2026-04-22T10:10:29.570Z [INFO] server-fb05d6cf-79f8-cc0f-a861-81fc30896f50: added peer, starting replication: peer=server-54b3284e-4caa-7ca7-0c3c-a7c933c43379 -2026-04-22T10:10:29.571Z [INFO] server-fb05d6cf-79f8-cc0f-a861-81fc30896f50: pipelining replication: peer="{Voter server-54b3284e-4caa-7ca7-0c3c-a7c933c43379 54b3284e-4caa-7ca7-0c3c-a7c933c43379}" -2026-04-22T10:10:29.571Z [INFO] server-fb05d6cf-79f8-cc0f-a861-81fc30896f50: pipelining replication: peer="{Voter server-9e30c983-0aa8-13c7-5d6e-153fde24f5f3 9e30c983-0aa8-13c7-5d6e-153fde24f5f3}" -2026-04-22T10:10:30.576Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:29.570988637 +0000 UTC m=+75.594784722 (1 nodes), 537.252637ms from start of poll, 538.129476ms from cluster start. Timeout at 2026-04-22 10:10:30.575990752 +0000 UTC m=+76.599786832, 1.00500211s after stability -2026-04-22T10:10:30.576Z [INFO] server-fb05d6cf-79f8-cc0f-a861-81fc30896f50: starting snapshot up to: index=12 -2026-04-22T10:10:30.576Z [INFO] creating new snapshot: path=/tmp/raft4151621761/snapshots/2-12-1776852630576.tmp -2026-04-22T10:10:30.577Z [INFO] server-fb05d6cf-79f8-cc0f-a861-81fc30896f50: snapshot complete up to: index=12 -2026-04-22T10:10:30.578Z [INFO] server-21c703c0-a4d0-9d9f-ba03-528c4cba67a2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-21c703c0-a4d0-9d9f-ba03-528c4cba67a2 Address:21c703c0-a4d0-9d9f-ba03-528c4cba67a2} {Suffrage:Voter ID:server-fa73ec27-99bd-601c-41e2-869b7ac99f7d Address:fa73ec27-99bd-601c-41e2-869b7ac99f7d} {Suffrage:Voter ID:server-1bb58cda-7edd-04e7-e576-7ea1fa87ffea Address:1bb58cda-7edd-04e7-e576-7ea1fa87ffea}]" -2026-04-22T10:10:30.578Z [INFO] server-21c703c0-a4d0-9d9f-ba03-528c4cba67a2: entering follower state: follower="Node at 21c703c0-a4d0-9d9f-ba03-528c4cba67a2 [Follower]" leader-address= leader-id= -2026-04-22T10:10:30.578Z [INFO] server-fa73ec27-99bd-601c-41e2-869b7ac99f7d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-21c703c0-a4d0-9d9f-ba03-528c4cba67a2 Address:21c703c0-a4d0-9d9f-ba03-528c4cba67a2} {Suffrage:Voter ID:server-fa73ec27-99bd-601c-41e2-869b7ac99f7d Address:fa73ec27-99bd-601c-41e2-869b7ac99f7d} {Suffrage:Voter ID:server-1bb58cda-7edd-04e7-e576-7ea1fa87ffea Address:1bb58cda-7edd-04e7-e576-7ea1fa87ffea}]" -2026-04-22T10:10:30.578Z [INFO] server-fa73ec27-99bd-601c-41e2-869b7ac99f7d: entering follower state: follower="Node at fa73ec27-99bd-601c-41e2-869b7ac99f7d [Follower]" leader-address= leader-id= -2026-04-22T10:10:30.578Z [INFO] server-1bb58cda-7edd-04e7-e576-7ea1fa87ffea: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-21c703c0-a4d0-9d9f-ba03-528c4cba67a2 Address:21c703c0-a4d0-9d9f-ba03-528c4cba67a2} {Suffrage:Voter ID:server-fa73ec27-99bd-601c-41e2-869b7ac99f7d Address:fa73ec27-99bd-601c-41e2-869b7ac99f7d} {Suffrage:Voter ID:server-1bb58cda-7edd-04e7-e576-7ea1fa87ffea Address:1bb58cda-7edd-04e7-e576-7ea1fa87ffea}]" -2026-04-22T10:10:30.578Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:30.578Z [INFO] server-1bb58cda-7edd-04e7-e576-7ea1fa87ffea: entering follower state: follower="Node at 1bb58cda-7edd-04e7-e576-7ea1fa87ffea [Follower]" leader-address= leader-id= -2026-04-22T10:10:31.212Z [WARN] server-fa73ec27-99bd-601c-41e2-869b7ac99f7d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:31.212Z [INFO] server-fa73ec27-99bd-601c-41e2-869b7ac99f7d: entering candidate state: node="Node at fa73ec27-99bd-601c-41e2-869b7ac99f7d [Candidate]" term=2 -2026-04-22T10:10:31.212Z [INFO] server-fa73ec27-99bd-601c-41e2-869b7ac99f7d: election won: term=2 tally=2 -2026-04-22T10:10:31.212Z [INFO] server-fa73ec27-99bd-601c-41e2-869b7ac99f7d: entering leader state: leader="Node at fa73ec27-99bd-601c-41e2-869b7ac99f7d [Leader]" -2026-04-22T10:10:31.212Z [INFO] server-fa73ec27-99bd-601c-41e2-869b7ac99f7d: added peer, starting replication: peer=server-21c703c0-a4d0-9d9f-ba03-528c4cba67a2 -2026-04-22T10:10:31.212Z [INFO] server-fa73ec27-99bd-601c-41e2-869b7ac99f7d: added peer, starting replication: peer=server-1bb58cda-7edd-04e7-e576-7ea1fa87ffea -2026-04-22T10:10:31.212Z [INFO] server-fa73ec27-99bd-601c-41e2-869b7ac99f7d: pipelining replication: peer="{Voter server-21c703c0-a4d0-9d9f-ba03-528c4cba67a2 21c703c0-a4d0-9d9f-ba03-528c4cba67a2}" -2026-04-22T10:10:31.212Z [INFO] server-fa73ec27-99bd-601c-41e2-869b7ac99f7d: pipelining replication: peer="{Voter server-1bb58cda-7edd-04e7-e576-7ea1fa87ffea 1bb58cda-7edd-04e7-e576-7ea1fa87ffea}" -2026-04-22T10:10:32.218Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:31.212371216 +0000 UTC m=+77.236167307 (1 nodes), 633.818247ms from start of poll, 634.135217ms from cluster start. Timeout at 2026-04-22 10:10:32.217377657 +0000 UTC m=+78.241173740, 1.005006433s after stability -2026-04-22T10:10:32.218Z [INFO] creating new snapshot: path=/tmp/raft972038166/snapshots/2-1013-1776852632218.tmp -2026-04-22T10:10:32.219Z [INFO] server-fa73ec27-99bd-601c-41e2-869b7ac99f7d: copied to local snapshot: bytes=71 -2026-04-22T10:10:32.219Z [INFO] server-fa73ec27-99bd-601c-41e2-869b7ac99f7d: snapshot restore progress: id=2-1013-1776852632218 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:32.219Z [INFO] server-fa73ec27-99bd-601c-41e2-869b7ac99f7d: restored user snapshot: index=1013 -2026-04-22T10:10:32.219Z [ERROR] server-fa73ec27-99bd-601c-41e2-869b7ac99f7d: failed to get log: index=3 error="log not found" -2026-04-22T10:10:32.219Z [INFO] server-fa73ec27-99bd-601c-41e2-869b7ac99f7d: aborting pipeline replication: peer="{Voter server-1bb58cda-7edd-04e7-e576-7ea1fa87ffea 1bb58cda-7edd-04e7-e576-7ea1fa87ffea}" -2026-04-22T10:10:32.219Z [ERROR] server-fa73ec27-99bd-601c-41e2-869b7ac99f7d: failed to get log: index=3 error="log not found" -2026-04-22T10:10:32.219Z [INFO] server-fa73ec27-99bd-601c-41e2-869b7ac99f7d: aborting pipeline replication: peer="{Voter server-21c703c0-a4d0-9d9f-ba03-528c4cba67a2 21c703c0-a4d0-9d9f-ba03-528c4cba67a2}" -2026-04-22T10:10:32.225Z [ERROR] server-fa73ec27-99bd-601c-41e2-869b7ac99f7d: failed to get log: index=3 error="log not found" -2026-04-22T10:10:32.225Z [INFO] creating new snapshot: path=/tmp/raft1926815847/snapshots/2-1013-1776852632225.tmp -2026-04-22T10:10:32.225Z [INFO] server-1bb58cda-7edd-04e7-e576-7ea1fa87ffea: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:32.226Z [INFO] server-1bb58cda-7edd-04e7-e576-7ea1fa87ffea: copied to local snapshot: bytes=71 -2026-04-22T10:10:32.226Z [INFO] server-1bb58cda-7edd-04e7-e576-7ea1fa87ffea: snapshot restore progress: id=2-1013-1776852632225 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:32.226Z [INFO] server-1bb58cda-7edd-04e7-e576-7ea1fa87ffea: Installed remote snapshot -2026-04-22T10:10:32.226Z [INFO] server-fa73ec27-99bd-601c-41e2-869b7ac99f7d: pipelining replication: peer="{Voter server-1bb58cda-7edd-04e7-e576-7ea1fa87ffea 1bb58cda-7edd-04e7-e576-7ea1fa87ffea}" -2026-04-22T10:10:32.226Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:10:32.230Z [ERROR] server-fa73ec27-99bd-601c-41e2-869b7ac99f7d: failed to get log: index=3 error="log not found" -2026-04-22T10:10:32.230Z [INFO] creating new snapshot: path=/tmp/raft64231691/snapshots/2-1013-1776852632230.tmp -2026-04-22T10:10:32.230Z [INFO] server-21c703c0-a4d0-9d9f-ba03-528c4cba67a2: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:32.230Z [INFO] server-21c703c0-a4d0-9d9f-ba03-528c4cba67a2: copied to local snapshot: bytes=71 -2026-04-22T10:10:32.231Z [INFO] server-21c703c0-a4d0-9d9f-ba03-528c4cba67a2: snapshot restore progress: id=2-1013-1776852632230 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:32.231Z [INFO] server-21c703c0-a4d0-9d9f-ba03-528c4cba67a2: Installed remote snapshot -2026-04-22T10:10:32.231Z [INFO] server-fa73ec27-99bd-601c-41e2-869b7ac99f7d: pipelining replication: peer="{Voter server-21c703c0-a4d0-9d9f-ba03-528c4cba67a2 21c703c0-a4d0-9d9f-ba03-528c4cba67a2}" -2026-04-22T10:10:33.231Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:10:32.226524437 +0000 UTC m=+78.250320523 (2 nodes), 566ns from start of poll, 1.648288433s from cluster start. Timeout at 2026-04-22 10:10:33.231535163 +0000 UTC m=+79.255331244, 1.005010721s after stability -2026-04-22T10:10:33.242Z [INFO] server-fb05d6cf-79f8-cc0f-a861-81fc30896f50: aborting pipeline replication: peer="{Voter server-9e30c983-0aa8-13c7-5d6e-153fde24f5f3 9e30c983-0aa8-13c7-5d6e-153fde24f5f3}" -2026-04-22T10:10:33.242Z [INFO] server-fb05d6cf-79f8-cc0f-a861-81fc30896f50: aborting pipeline replication: peer="{Voter server-54b3284e-4caa-7ca7-0c3c-a7c933c43379 54b3284e-4caa-7ca7-0c3c-a7c933c43379}" +2025-03-20T03:50:45.124Z [INFO] server-074bffd4-e9f7-0711-435a-a19b6cbd01f0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-074bffd4-e9f7-0711-435a-a19b6cbd01f0 Address:074bffd4-e9f7-0711-435a-a19b6cbd01f0} {Suffrage:Voter ID:server-723c4485-a815-6322-5021-b1b9b7baa7d3 Address:723c4485-a815-6322-5021-b1b9b7baa7d3} {Suffrage:Voter ID:server-91b7aee0-11d5-79eb-935d-2a9bb43a36e7 Address:91b7aee0-11d5-79eb-935d-2a9bb43a36e7}]" +2025-03-20T03:50:45.124Z [INFO] server-074bffd4-e9f7-0711-435a-a19b6cbd01f0: entering follower state: follower="Node at 074bffd4-e9f7-0711-435a-a19b6cbd01f0 [Follower]" leader-address= leader-id= +2025-03-20T03:50:45.125Z [INFO] server-723c4485-a815-6322-5021-b1b9b7baa7d3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-074bffd4-e9f7-0711-435a-a19b6cbd01f0 Address:074bffd4-e9f7-0711-435a-a19b6cbd01f0} {Suffrage:Voter ID:server-723c4485-a815-6322-5021-b1b9b7baa7d3 Address:723c4485-a815-6322-5021-b1b9b7baa7d3} {Suffrage:Voter ID:server-91b7aee0-11d5-79eb-935d-2a9bb43a36e7 Address:91b7aee0-11d5-79eb-935d-2a9bb43a36e7}]" +2025-03-20T03:50:45.125Z [INFO] server-91b7aee0-11d5-79eb-935d-2a9bb43a36e7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-074bffd4-e9f7-0711-435a-a19b6cbd01f0 Address:074bffd4-e9f7-0711-435a-a19b6cbd01f0} {Suffrage:Voter ID:server-723c4485-a815-6322-5021-b1b9b7baa7d3 Address:723c4485-a815-6322-5021-b1b9b7baa7d3} {Suffrage:Voter ID:server-91b7aee0-11d5-79eb-935d-2a9bb43a36e7 Address:91b7aee0-11d5-79eb-935d-2a9bb43a36e7}]" +2025-03-20T03:50:45.125Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:50:45.125Z [INFO] server-723c4485-a815-6322-5021-b1b9b7baa7d3: entering follower state: follower="Node at 723c4485-a815-6322-5021-b1b9b7baa7d3 [Follower]" leader-address= leader-id= +2025-03-20T03:50:45.125Z [INFO] server-91b7aee0-11d5-79eb-935d-2a9bb43a36e7: entering follower state: follower="Node at 91b7aee0-11d5-79eb-935d-2a9bb43a36e7 [Follower]" leader-address= leader-id= +2025-03-20T03:50:45.779Z [WARN] server-074bffd4-e9f7-0711-435a-a19b6cbd01f0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:50:45.779Z [INFO] server-074bffd4-e9f7-0711-435a-a19b6cbd01f0: entering candidate state: node="Node at 074bffd4-e9f7-0711-435a-a19b6cbd01f0 [Candidate]" term=2 +2025-03-20T03:50:45.779Z [INFO] server-074bffd4-e9f7-0711-435a-a19b6cbd01f0: election won: term=2 tally=2 +2025-03-20T03:50:45.779Z [INFO] server-074bffd4-e9f7-0711-435a-a19b6cbd01f0: entering leader state: leader="Node at 074bffd4-e9f7-0711-435a-a19b6cbd01f0 [Leader]" +2025-03-20T03:50:45.779Z [INFO] server-074bffd4-e9f7-0711-435a-a19b6cbd01f0: added peer, starting replication: peer=server-723c4485-a815-6322-5021-b1b9b7baa7d3 +2025-03-20T03:50:45.779Z [INFO] server-074bffd4-e9f7-0711-435a-a19b6cbd01f0: added peer, starting replication: peer=server-91b7aee0-11d5-79eb-935d-2a9bb43a36e7 +2025-03-20T03:50:45.779Z [INFO] server-074bffd4-e9f7-0711-435a-a19b6cbd01f0: pipelining replication: peer="{Voter server-91b7aee0-11d5-79eb-935d-2a9bb43a36e7 91b7aee0-11d5-79eb-935d-2a9bb43a36e7}" +2025-03-20T03:50:45.779Z [INFO] server-074bffd4-e9f7-0711-435a-a19b6cbd01f0: pipelining replication: peer="{Voter server-723c4485-a815-6322-5021-b1b9b7baa7d3 723c4485-a815-6322-5021-b1b9b7baa7d3}" +2025-03-20T03:50:46.785Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:50:45.779754417 +0000 UTC m=+77.454207266 (1 nodes), 654.155974ms from start of poll, 655.098854ms from cluster start. Timeout at 2025-03-20 03:50:46.78476036 +0000 UTC m=+78.459213241, 1.005005975s after stability +2025-03-20T03:50:46.796Z [INFO] server-074bffd4-e9f7-0711-435a-a19b6cbd01f0: starting snapshot up to: index=12 +2025-03-20T03:50:46.796Z [INFO] creating new snapshot: path=/tmp/raft3771612336/snapshots/2-12-1742442646796.tmp +2025-03-20T03:50:46.796Z [INFO] server-074bffd4-e9f7-0711-435a-a19b6cbd01f0: snapshot complete up to: index=12 +2025-03-20T03:50:46.799Z [INFO] server-4c84b052-ff71-cb01-879a-b2549fce77cc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4c84b052-ff71-cb01-879a-b2549fce77cc Address:4c84b052-ff71-cb01-879a-b2549fce77cc} {Suffrage:Voter ID:server-80957441-6717-cb86-8d07-d5085f91a7f1 Address:80957441-6717-cb86-8d07-d5085f91a7f1} {Suffrage:Voter ID:server-ee495797-9082-d5fa-4a0c-fc55d270764b Address:ee495797-9082-d5fa-4a0c-fc55d270764b}]" +2025-03-20T03:50:46.799Z [INFO] server-80957441-6717-cb86-8d07-d5085f91a7f1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4c84b052-ff71-cb01-879a-b2549fce77cc Address:4c84b052-ff71-cb01-879a-b2549fce77cc} {Suffrage:Voter ID:server-80957441-6717-cb86-8d07-d5085f91a7f1 Address:80957441-6717-cb86-8d07-d5085f91a7f1} {Suffrage:Voter ID:server-ee495797-9082-d5fa-4a0c-fc55d270764b Address:ee495797-9082-d5fa-4a0c-fc55d270764b}]" +2025-03-20T03:50:46.799Z [INFO] server-4c84b052-ff71-cb01-879a-b2549fce77cc: entering follower state: follower="Node at 4c84b052-ff71-cb01-879a-b2549fce77cc [Follower]" leader-address= leader-id= +2025-03-20T03:50:46.799Z [INFO] server-ee495797-9082-d5fa-4a0c-fc55d270764b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4c84b052-ff71-cb01-879a-b2549fce77cc Address:4c84b052-ff71-cb01-879a-b2549fce77cc} {Suffrage:Voter ID:server-80957441-6717-cb86-8d07-d5085f91a7f1 Address:80957441-6717-cb86-8d07-d5085f91a7f1} {Suffrage:Voter ID:server-ee495797-9082-d5fa-4a0c-fc55d270764b Address:ee495797-9082-d5fa-4a0c-fc55d270764b}]" +2025-03-20T03:50:46.799Z [INFO] server-80957441-6717-cb86-8d07-d5085f91a7f1: entering follower state: follower="Node at 80957441-6717-cb86-8d07-d5085f91a7f1 [Follower]" leader-address= leader-id= +2025-03-20T03:50:46.799Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:50:46.799Z [INFO] server-ee495797-9082-d5fa-4a0c-fc55d270764b: entering follower state: follower="Node at ee495797-9082-d5fa-4a0c-fc55d270764b [Follower]" leader-address= leader-id= +2025-03-20T03:50:47.386Z [WARN] server-80957441-6717-cb86-8d07-d5085f91a7f1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:50:47.386Z [INFO] server-80957441-6717-cb86-8d07-d5085f91a7f1: entering candidate state: node="Node at 80957441-6717-cb86-8d07-d5085f91a7f1 [Candidate]" term=2 +2025-03-20T03:50:47.386Z [INFO] server-80957441-6717-cb86-8d07-d5085f91a7f1: election won: term=2 tally=2 +2025-03-20T03:50:47.386Z [INFO] server-80957441-6717-cb86-8d07-d5085f91a7f1: entering leader state: leader="Node at 80957441-6717-cb86-8d07-d5085f91a7f1 [Leader]" +2025-03-20T03:50:47.386Z [INFO] server-80957441-6717-cb86-8d07-d5085f91a7f1: added peer, starting replication: peer=server-4c84b052-ff71-cb01-879a-b2549fce77cc +2025-03-20T03:50:47.386Z [INFO] server-80957441-6717-cb86-8d07-d5085f91a7f1: added peer, starting replication: peer=server-ee495797-9082-d5fa-4a0c-fc55d270764b +2025-03-20T03:50:47.386Z [INFO] server-80957441-6717-cb86-8d07-d5085f91a7f1: pipelining replication: peer="{Voter server-ee495797-9082-d5fa-4a0c-fc55d270764b ee495797-9082-d5fa-4a0c-fc55d270764b}" +2025-03-20T03:50:47.386Z [INFO] server-80957441-6717-cb86-8d07-d5085f91a7f1: pipelining replication: peer="{Voter server-4c84b052-ff71-cb01-879a-b2549fce77cc 4c84b052-ff71-cb01-879a-b2549fce77cc}" +2025-03-20T03:50:47.386Z [WARN] server-4c84b052-ff71-cb01-879a-b2549fce77cc: rejecting vote request since our last term is greater: candidate=80957441-6717-cb86-8d07-d5085f91a7f1 last-term=2 last-candidate-term=1 +2025-03-20T03:50:48.392Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:50:47.38692972 +0000 UTC m=+79.061382757 (1 nodes), 587.187997ms from start of poll, 587.751361ms from cluster start. Timeout at 2025-03-20 03:50:48.3919444 +0000 UTC m=+80.066397273, 1.005014516s after stability +2025-03-20T03:50:48.392Z [INFO] creating new snapshot: path=/tmp/raft4034008271/snapshots/2-1013-1742442648392.tmp +2025-03-20T03:50:48.393Z [INFO] server-80957441-6717-cb86-8d07-d5085f91a7f1: copied to local snapshot: bytes=71 +2025-03-20T03:50:48.394Z [INFO] server-80957441-6717-cb86-8d07-d5085f91a7f1: snapshot restore progress: id=2-1013-1742442648392 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:48.394Z [INFO] server-80957441-6717-cb86-8d07-d5085f91a7f1: restored user snapshot: index=1013 +2025-03-20T03:50:48.394Z [ERROR] server-80957441-6717-cb86-8d07-d5085f91a7f1: failed to get log: index=3 error="log not found" +2025-03-20T03:50:48.395Z [INFO] server-80957441-6717-cb86-8d07-d5085f91a7f1: aborting pipeline replication: peer="{Voter server-ee495797-9082-d5fa-4a0c-fc55d270764b ee495797-9082-d5fa-4a0c-fc55d270764b}" +2025-03-20T03:50:48.394Z [ERROR] server-80957441-6717-cb86-8d07-d5085f91a7f1: failed to get log: index=3 error="log not found" +2025-03-20T03:50:48.395Z [INFO] server-80957441-6717-cb86-8d07-d5085f91a7f1: aborting pipeline replication: peer="{Voter server-4c84b052-ff71-cb01-879a-b2549fce77cc 4c84b052-ff71-cb01-879a-b2549fce77cc}" +2025-03-20T03:50:48.402Z [ERROR] server-80957441-6717-cb86-8d07-d5085f91a7f1: failed to get log: index=3 error="log not found" +2025-03-20T03:50:48.403Z [INFO] creating new snapshot: path=/tmp/raft2049951741/snapshots/2-1013-1742442648403.tmp +2025-03-20T03:50:48.403Z [INFO] server-ee495797-9082-d5fa-4a0c-fc55d270764b: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:48.404Z [INFO] server-ee495797-9082-d5fa-4a0c-fc55d270764b: copied to local snapshot: bytes=71 +2025-03-20T03:50:48.404Z [ERROR] server-80957441-6717-cb86-8d07-d5085f91a7f1: failed to get log: index=3 error="log not found" +2025-03-20T03:50:48.405Z [INFO] creating new snapshot: path=/tmp/raft3326040862/snapshots/2-1013-1742442648405.tmp +2025-03-20T03:50:48.405Z [INFO] server-ee495797-9082-d5fa-4a0c-fc55d270764b: snapshot restore progress: id=2-1013-1742442648403 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:48.405Z [INFO] server-ee495797-9082-d5fa-4a0c-fc55d270764b: Installed remote snapshot +2025-03-20T03:50:48.405Z [INFO] server-80957441-6717-cb86-8d07-d5085f91a7f1: pipelining replication: peer="{Voter server-ee495797-9082-d5fa-4a0c-fc55d270764b ee495797-9082-d5fa-4a0c-fc55d270764b}" +2025-03-20T03:50:48.405Z [INFO] server-4c84b052-ff71-cb01-879a-b2549fce77cc: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:48.405Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:50:48.406Z [INFO] server-4c84b052-ff71-cb01-879a-b2549fce77cc: copied to local snapshot: bytes=71 +2025-03-20T03:50:48.406Z [INFO] server-4c84b052-ff71-cb01-879a-b2549fce77cc: snapshot restore progress: id=2-1013-1742442648405 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:48.406Z [INFO] server-4c84b052-ff71-cb01-879a-b2549fce77cc: Installed remote snapshot +2025-03-20T03:50:48.407Z [INFO] server-80957441-6717-cb86-8d07-d5085f91a7f1: pipelining replication: peer="{Voter server-4c84b052-ff71-cb01-879a-b2549fce77cc 4c84b052-ff71-cb01-879a-b2549fce77cc}" +2025-03-20T03:50:49.411Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:50:48.405869298 +0000 UTC m=+80.080322154 (2 nodes), 1.951µs from start of poll, 1.606690758s from cluster start. Timeout at 2025-03-20 03:50:49.410929712 +0000 UTC m=+81.085382689, 1.005060535s after stability +2025-03-20T03:50:49.437Z [WARN] server-ee495797-9082-d5fa-4a0c-fc55d270764b: skipping application of old log: index=1014 +2025-03-20T03:50:49.443Z [INFO] server-074bffd4-e9f7-0711-435a-a19b6cbd01f0: aborting pipeline replication: peer="{Voter server-91b7aee0-11d5-79eb-935d-2a9bb43a36e7 91b7aee0-11d5-79eb-935d-2a9bb43a36e7}" +2025-03-20T03:50:49.443Z [INFO] server-074bffd4-e9f7-0711-435a-a19b6cbd01f0: aborting pipeline replication: peer="{Voter server-723c4485-a815-6322-5021-b1b9b7baa7d3 723c4485-a815-6322-5021-b1b9b7baa7d3}" === RUN TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_true -2026-04-22T10:10:33.244Z [INFO] server-63c00e54-84bd-a944-ed8e-85e1f6e1a860: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-63c00e54-84bd-a944-ed8e-85e1f6e1a860 Address:63c00e54-84bd-a944-ed8e-85e1f6e1a860} {Suffrage:Voter ID:server-ad91fa6a-c384-fb2d-ecd5-d4de76a404b0 Address:ad91fa6a-c384-fb2d-ecd5-d4de76a404b0} {Suffrage:Voter ID:server-f08abc32-a7e8-0985-6a9f-52928273156f Address:f08abc32-a7e8-0985-6a9f-52928273156f}]" -2026-04-22T10:10:33.244Z [INFO] server-63c00e54-84bd-a944-ed8e-85e1f6e1a860: entering follower state: follower="Node at 63c00e54-84bd-a944-ed8e-85e1f6e1a860 [Follower]" leader-address= leader-id= -2026-04-22T10:10:33.245Z [INFO] server-ad91fa6a-c384-fb2d-ecd5-d4de76a404b0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-63c00e54-84bd-a944-ed8e-85e1f6e1a860 Address:63c00e54-84bd-a944-ed8e-85e1f6e1a860} {Suffrage:Voter ID:server-ad91fa6a-c384-fb2d-ecd5-d4de76a404b0 Address:ad91fa6a-c384-fb2d-ecd5-d4de76a404b0} {Suffrage:Voter ID:server-f08abc32-a7e8-0985-6a9f-52928273156f Address:f08abc32-a7e8-0985-6a9f-52928273156f}]" -2026-04-22T10:10:33.245Z [INFO] server-ad91fa6a-c384-fb2d-ecd5-d4de76a404b0: entering follower state: follower="Node at ad91fa6a-c384-fb2d-ecd5-d4de76a404b0 [Follower]" leader-address= leader-id= -2026-04-22T10:10:33.245Z [INFO] server-f08abc32-a7e8-0985-6a9f-52928273156f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-63c00e54-84bd-a944-ed8e-85e1f6e1a860 Address:63c00e54-84bd-a944-ed8e-85e1f6e1a860} {Suffrage:Voter ID:server-ad91fa6a-c384-fb2d-ecd5-d4de76a404b0 Address:ad91fa6a-c384-fb2d-ecd5-d4de76a404b0} {Suffrage:Voter ID:server-f08abc32-a7e8-0985-6a9f-52928273156f Address:f08abc32-a7e8-0985-6a9f-52928273156f}]" -2026-04-22T10:10:33.245Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:33.245Z [INFO] server-f08abc32-a7e8-0985-6a9f-52928273156f: entering follower state: follower="Node at f08abc32-a7e8-0985-6a9f-52928273156f [Follower]" leader-address= leader-id= -2026-04-22T10:10:33.752Z [WARN] server-f08abc32-a7e8-0985-6a9f-52928273156f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:33.753Z [INFO] server-f08abc32-a7e8-0985-6a9f-52928273156f: entering candidate state: node="Node at f08abc32-a7e8-0985-6a9f-52928273156f [Candidate]" term=2 -2026-04-22T10:10:33.753Z [INFO] server-f08abc32-a7e8-0985-6a9f-52928273156f: election won: term=2 tally=2 -2026-04-22T10:10:33.753Z [INFO] server-f08abc32-a7e8-0985-6a9f-52928273156f: entering leader state: leader="Node at f08abc32-a7e8-0985-6a9f-52928273156f [Leader]" -2026-04-22T10:10:33.753Z [INFO] server-f08abc32-a7e8-0985-6a9f-52928273156f: added peer, starting replication: peer=server-63c00e54-84bd-a944-ed8e-85e1f6e1a860 -2026-04-22T10:10:33.753Z [INFO] server-f08abc32-a7e8-0985-6a9f-52928273156f: added peer, starting replication: peer=server-ad91fa6a-c384-fb2d-ecd5-d4de76a404b0 -2026-04-22T10:10:33.753Z [INFO] server-f08abc32-a7e8-0985-6a9f-52928273156f: pipelining replication: peer="{Voter server-63c00e54-84bd-a944-ed8e-85e1f6e1a860 63c00e54-84bd-a944-ed8e-85e1f6e1a860}" -2026-04-22T10:10:33.753Z [INFO] server-f08abc32-a7e8-0985-6a9f-52928273156f: pipelining replication: peer="{Voter server-ad91fa6a-c384-fb2d-ecd5-d4de76a404b0 ad91fa6a-c384-fb2d-ecd5-d4de76a404b0}" -2026-04-22T10:10:34.758Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:33.753188901 +0000 UTC m=+79.776985007 (1 nodes), 507.936385ms from start of poll, 508.767238ms from cluster start. Timeout at 2026-04-22 10:10:34.758191958 +0000 UTC m=+80.781988041, 1.005003034s after stability -2026-04-22T10:10:34.759Z [INFO] server-f08abc32-a7e8-0985-6a9f-52928273156f: starting snapshot up to: index=12 -2026-04-22T10:10:34.759Z [INFO] creating new snapshot: path=/tmp/raft3859420460/snapshots/2-12-1776852634759.tmp -2026-04-22T10:10:34.764Z [INFO] server-f08abc32-a7e8-0985-6a9f-52928273156f: snapshot complete up to: index=12 -2026-04-22T10:10:34.765Z [INFO] server-81157623-98e1-92d4-c3b3-80583eb268f7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-81157623-98e1-92d4-c3b3-80583eb268f7 Address:81157623-98e1-92d4-c3b3-80583eb268f7} {Suffrage:Voter ID:server-8b9d8cd2-e8dd-b436-e2f3-d207e2fc8988 Address:8b9d8cd2-e8dd-b436-e2f3-d207e2fc8988} {Suffrage:Voter ID:server-63f5c036-ea37-45b4-ccbb-8c472e7db78f Address:63f5c036-ea37-45b4-ccbb-8c472e7db78f}]" -2026-04-22T10:10:34.765Z [INFO] server-81157623-98e1-92d4-c3b3-80583eb268f7: entering follower state: follower="Node at 81157623-98e1-92d4-c3b3-80583eb268f7 [Follower]" leader-address= leader-id= -2026-04-22T10:10:34.766Z [INFO] server-8b9d8cd2-e8dd-b436-e2f3-d207e2fc8988: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-81157623-98e1-92d4-c3b3-80583eb268f7 Address:81157623-98e1-92d4-c3b3-80583eb268f7} {Suffrage:Voter ID:server-8b9d8cd2-e8dd-b436-e2f3-d207e2fc8988 Address:8b9d8cd2-e8dd-b436-e2f3-d207e2fc8988} {Suffrage:Voter ID:server-63f5c036-ea37-45b4-ccbb-8c472e7db78f Address:63f5c036-ea37-45b4-ccbb-8c472e7db78f}]" -2026-04-22T10:10:34.766Z [INFO] server-8b9d8cd2-e8dd-b436-e2f3-d207e2fc8988: entering follower state: follower="Node at 8b9d8cd2-e8dd-b436-e2f3-d207e2fc8988 [Follower]" leader-address= leader-id= -2026-04-22T10:10:34.766Z [INFO] server-63f5c036-ea37-45b4-ccbb-8c472e7db78f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-81157623-98e1-92d4-c3b3-80583eb268f7 Address:81157623-98e1-92d4-c3b3-80583eb268f7} {Suffrage:Voter ID:server-8b9d8cd2-e8dd-b436-e2f3-d207e2fc8988 Address:8b9d8cd2-e8dd-b436-e2f3-d207e2fc8988} {Suffrage:Voter ID:server-63f5c036-ea37-45b4-ccbb-8c472e7db78f Address:63f5c036-ea37-45b4-ccbb-8c472e7db78f}]" -2026-04-22T10:10:34.766Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:34.766Z [INFO] server-63f5c036-ea37-45b4-ccbb-8c472e7db78f: entering follower state: follower="Node at 63f5c036-ea37-45b4-ccbb-8c472e7db78f [Follower]" leader-address= leader-id= -2026-04-22T10:10:35.458Z [WARN] server-63f5c036-ea37-45b4-ccbb-8c472e7db78f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:35.458Z [INFO] server-63f5c036-ea37-45b4-ccbb-8c472e7db78f: entering candidate state: node="Node at 63f5c036-ea37-45b4-ccbb-8c472e7db78f [Candidate]" term=2 -2026-04-22T10:10:35.458Z [INFO] server-63f5c036-ea37-45b4-ccbb-8c472e7db78f: election won: term=2 tally=2 -2026-04-22T10:10:35.458Z [INFO] server-63f5c036-ea37-45b4-ccbb-8c472e7db78f: entering leader state: leader="Node at 63f5c036-ea37-45b4-ccbb-8c472e7db78f [Leader]" -2026-04-22T10:10:35.458Z [INFO] server-63f5c036-ea37-45b4-ccbb-8c472e7db78f: added peer, starting replication: peer=server-81157623-98e1-92d4-c3b3-80583eb268f7 -2026-04-22T10:10:35.458Z [INFO] server-63f5c036-ea37-45b4-ccbb-8c472e7db78f: added peer, starting replication: peer=server-8b9d8cd2-e8dd-b436-e2f3-d207e2fc8988 -2026-04-22T10:10:35.458Z [INFO] server-63f5c036-ea37-45b4-ccbb-8c472e7db78f: pipelining replication: peer="{Voter server-8b9d8cd2-e8dd-b436-e2f3-d207e2fc8988 8b9d8cd2-e8dd-b436-e2f3-d207e2fc8988}" -2026-04-22T10:10:35.458Z [INFO] server-63f5c036-ea37-45b4-ccbb-8c472e7db78f: pipelining replication: peer="{Voter server-81157623-98e1-92d4-c3b3-80583eb268f7 81157623-98e1-92d4-c3b3-80583eb268f7}" -2026-04-22T10:10:36.463Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:35.458773697 +0000 UTC m=+81.482569788 (1 nodes), 692.560636ms from start of poll, 692.972228ms from cluster start. Timeout at 2026-04-22 10:10:36.463775067 +0000 UTC m=+82.487571204, 1.005001416s after stability -2026-04-22T10:10:36.464Z [INFO] creating new snapshot: path=/tmp/raft2679765030/snapshots/2-1013-1776852636464.tmp -2026-04-22T10:10:36.464Z [INFO] server-63f5c036-ea37-45b4-ccbb-8c472e7db78f: copied to local snapshot: bytes=71 -2026-04-22T10:10:36.464Z [INFO] server-63f5c036-ea37-45b4-ccbb-8c472e7db78f: snapshot restore progress: id=2-1013-1776852636464 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:36.464Z [INFO] server-63f5c036-ea37-45b4-ccbb-8c472e7db78f: removing all old logs from log store -2026-04-22T10:10:36.464Z [INFO] server-63f5c036-ea37-45b4-ccbb-8c472e7db78f: compacting logs: from=1 to=2 -2026-04-22T10:10:36.464Z [INFO] server-63f5c036-ea37-45b4-ccbb-8c472e7db78f: restored user snapshot: index=1013 -2026-04-22T10:10:36.464Z [ERROR] server-63f5c036-ea37-45b4-ccbb-8c472e7db78f: failed to get log: index=2 error="log not found" -2026-04-22T10:10:36.464Z [INFO] server-63f5c036-ea37-45b4-ccbb-8c472e7db78f: aborting pipeline replication: peer="{Voter server-8b9d8cd2-e8dd-b436-e2f3-d207e2fc8988 8b9d8cd2-e8dd-b436-e2f3-d207e2fc8988}" -2026-04-22T10:10:36.464Z [ERROR] server-63f5c036-ea37-45b4-ccbb-8c472e7db78f: failed to get log: index=2 error="log not found" -2026-04-22T10:10:36.464Z [INFO] server-63f5c036-ea37-45b4-ccbb-8c472e7db78f: aborting pipeline replication: peer="{Voter server-81157623-98e1-92d4-c3b3-80583eb268f7 81157623-98e1-92d4-c3b3-80583eb268f7}" -2026-04-22T10:10:36.472Z [ERROR] server-63f5c036-ea37-45b4-ccbb-8c472e7db78f: failed to get log: index=2 error="log not found" -2026-04-22T10:10:36.472Z [INFO] creating new snapshot: path=/tmp/raft1717177076/snapshots/2-1013-1776852636472.tmp -2026-04-22T10:10:36.473Z [INFO] server-8b9d8cd2-e8dd-b436-e2f3-d207e2fc8988: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:36.473Z [INFO] server-8b9d8cd2-e8dd-b436-e2f3-d207e2fc8988: copied to local snapshot: bytes=71 -2026-04-22T10:10:36.473Z [INFO] server-8b9d8cd2-e8dd-b436-e2f3-d207e2fc8988: snapshot restore progress: id=2-1013-1776852636472 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:36.473Z [INFO] server-8b9d8cd2-e8dd-b436-e2f3-d207e2fc8988: removing all old logs from log store -2026-04-22T10:10:36.473Z [INFO] server-8b9d8cd2-e8dd-b436-e2f3-d207e2fc8988: compacting logs: from=1 to=2 -2026-04-22T10:10:36.473Z [INFO] server-8b9d8cd2-e8dd-b436-e2f3-d207e2fc8988: Installed remote snapshot -2026-04-22T10:10:36.473Z [INFO] server-63f5c036-ea37-45b4-ccbb-8c472e7db78f: pipelining replication: peer="{Voter server-8b9d8cd2-e8dd-b436-e2f3-d207e2fc8988 8b9d8cd2-e8dd-b436-e2f3-d207e2fc8988}" -2026-04-22T10:10:36.473Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:10:36.474Z [ERROR] server-63f5c036-ea37-45b4-ccbb-8c472e7db78f: failed to get log: index=2 error="log not found" -2026-04-22T10:10:36.475Z [INFO] creating new snapshot: path=/tmp/raft1709363962/snapshots/2-1013-1776852636475.tmp -2026-04-22T10:10:36.475Z [INFO] server-81157623-98e1-92d4-c3b3-80583eb268f7: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:36.476Z [INFO] server-81157623-98e1-92d4-c3b3-80583eb268f7: copied to local snapshot: bytes=71 -2026-04-22T10:10:36.476Z [INFO] server-81157623-98e1-92d4-c3b3-80583eb268f7: snapshot restore progress: id=2-1013-1776852636475 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:36.476Z [INFO] server-81157623-98e1-92d4-c3b3-80583eb268f7: removing all old logs from log store -2026-04-22T10:10:36.476Z [INFO] server-81157623-98e1-92d4-c3b3-80583eb268f7: compacting logs: from=1 to=2 -2026-04-22T10:10:36.476Z [INFO] server-81157623-98e1-92d4-c3b3-80583eb268f7: Installed remote snapshot -2026-04-22T10:10:36.476Z [INFO] server-63f5c036-ea37-45b4-ccbb-8c472e7db78f: pipelining replication: peer="{Voter server-81157623-98e1-92d4-c3b3-80583eb268f7 81157623-98e1-92d4-c3b3-80583eb268f7}" -2026-04-22T10:10:37.478Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:10:36.473755961 +0000 UTC m=+82.497552047 (2 nodes), 612ns from start of poll, 1.707954487s from cluster start. Timeout at 2026-04-22 10:10:37.478775223 +0000 UTC m=+83.502571328, 1.005019281s after stability -2026-04-22T10:10:37.490Z [INFO] server-f08abc32-a7e8-0985-6a9f-52928273156f: aborting pipeline replication: peer="{Voter server-ad91fa6a-c384-fb2d-ecd5-d4de76a404b0 ad91fa6a-c384-fb2d-ecd5-d4de76a404b0}" -2026-04-22T10:10:37.490Z [INFO] server-f08abc32-a7e8-0985-6a9f-52928273156f: aborting pipeline replication: peer="{Voter server-63c00e54-84bd-a944-ed8e-85e1f6e1a860 63c00e54-84bd-a944-ed8e-85e1f6e1a860}" +2025-03-20T03:50:49.445Z [INFO] server-69a284ca-c41e-b0a5-6e3b-9db93a068c12: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-69a284ca-c41e-b0a5-6e3b-9db93a068c12 Address:69a284ca-c41e-b0a5-6e3b-9db93a068c12} {Suffrage:Voter ID:server-a50052c9-9ce3-a9ae-6d48-09f39ab12bee Address:a50052c9-9ce3-a9ae-6d48-09f39ab12bee} {Suffrage:Voter ID:server-ef9b76c4-8a4f-f328-6653-49791f8ba48f Address:ef9b76c4-8a4f-f328-6653-49791f8ba48f}]" +2025-03-20T03:50:49.445Z [INFO] server-69a284ca-c41e-b0a5-6e3b-9db93a068c12: entering follower state: follower="Node at 69a284ca-c41e-b0a5-6e3b-9db93a068c12 [Follower]" leader-address= leader-id= +2025-03-20T03:50:49.445Z [INFO] server-a50052c9-9ce3-a9ae-6d48-09f39ab12bee: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-69a284ca-c41e-b0a5-6e3b-9db93a068c12 Address:69a284ca-c41e-b0a5-6e3b-9db93a068c12} {Suffrage:Voter ID:server-a50052c9-9ce3-a9ae-6d48-09f39ab12bee Address:a50052c9-9ce3-a9ae-6d48-09f39ab12bee} {Suffrage:Voter ID:server-ef9b76c4-8a4f-f328-6653-49791f8ba48f Address:ef9b76c4-8a4f-f328-6653-49791f8ba48f}]" +2025-03-20T03:50:49.445Z [INFO] server-a50052c9-9ce3-a9ae-6d48-09f39ab12bee: entering follower state: follower="Node at a50052c9-9ce3-a9ae-6d48-09f39ab12bee [Follower]" leader-address= leader-id= +2025-03-20T03:50:49.445Z [INFO] server-ef9b76c4-8a4f-f328-6653-49791f8ba48f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-69a284ca-c41e-b0a5-6e3b-9db93a068c12 Address:69a284ca-c41e-b0a5-6e3b-9db93a068c12} {Suffrage:Voter ID:server-a50052c9-9ce3-a9ae-6d48-09f39ab12bee Address:a50052c9-9ce3-a9ae-6d48-09f39ab12bee} {Suffrage:Voter ID:server-ef9b76c4-8a4f-f328-6653-49791f8ba48f Address:ef9b76c4-8a4f-f328-6653-49791f8ba48f}]" +2025-03-20T03:50:49.445Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:50:49.445Z [INFO] server-ef9b76c4-8a4f-f328-6653-49791f8ba48f: entering follower state: follower="Node at ef9b76c4-8a4f-f328-6653-49791f8ba48f [Follower]" leader-address= leader-id= +2025-03-20T03:50:50.050Z [WARN] server-ef9b76c4-8a4f-f328-6653-49791f8ba48f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:50:50.050Z [INFO] server-ef9b76c4-8a4f-f328-6653-49791f8ba48f: entering candidate state: node="Node at ef9b76c4-8a4f-f328-6653-49791f8ba48f [Candidate]" term=2 +2025-03-20T03:50:50.051Z [INFO] server-ef9b76c4-8a4f-f328-6653-49791f8ba48f: election won: term=2 tally=2 +2025-03-20T03:50:50.051Z [INFO] server-ef9b76c4-8a4f-f328-6653-49791f8ba48f: entering leader state: leader="Node at ef9b76c4-8a4f-f328-6653-49791f8ba48f [Leader]" +2025-03-20T03:50:50.051Z [INFO] server-ef9b76c4-8a4f-f328-6653-49791f8ba48f: added peer, starting replication: peer=server-69a284ca-c41e-b0a5-6e3b-9db93a068c12 +2025-03-20T03:50:50.051Z [INFO] server-ef9b76c4-8a4f-f328-6653-49791f8ba48f: added peer, starting replication: peer=server-a50052c9-9ce3-a9ae-6d48-09f39ab12bee +2025-03-20T03:50:50.051Z [INFO] server-ef9b76c4-8a4f-f328-6653-49791f8ba48f: pipelining replication: peer="{Voter server-a50052c9-9ce3-a9ae-6d48-09f39ab12bee a50052c9-9ce3-a9ae-6d48-09f39ab12bee}" +2025-03-20T03:50:50.051Z [INFO] server-ef9b76c4-8a4f-f328-6653-49791f8ba48f: pipelining replication: peer="{Voter server-69a284ca-c41e-b0a5-6e3b-9db93a068c12 69a284ca-c41e-b0a5-6e3b-9db93a068c12}" +2025-03-20T03:50:51.057Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:50:50.05181244 +0000 UTC m=+81.726265399 (1 nodes), 606.0969ms from start of poll, 606.570436ms from cluster start. Timeout at 2025-03-20 03:50:51.056816416 +0000 UTC m=+82.731269303, 1.005003904s after stability +2025-03-20T03:50:51.059Z [INFO] server-ef9b76c4-8a4f-f328-6653-49791f8ba48f: starting snapshot up to: index=12 +2025-03-20T03:50:51.059Z [INFO] creating new snapshot: path=/tmp/raft1249139076/snapshots/2-12-1742442651059.tmp +2025-03-20T03:50:51.060Z [INFO] server-ef9b76c4-8a4f-f328-6653-49791f8ba48f: snapshot complete up to: index=12 +2025-03-20T03:50:51.063Z [INFO] server-1b9b1c7d-427c-3914-2cc7-805f5eed1622: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1b9b1c7d-427c-3914-2cc7-805f5eed1622 Address:1b9b1c7d-427c-3914-2cc7-805f5eed1622} {Suffrage:Voter ID:server-5ca48ed1-fc92-47a4-a415-598de2fa7d08 Address:5ca48ed1-fc92-47a4-a415-598de2fa7d08} {Suffrage:Voter ID:server-5ee7177c-8ebb-300e-2d1b-8e68022d551c Address:5ee7177c-8ebb-300e-2d1b-8e68022d551c}]" +2025-03-20T03:50:51.064Z [INFO] server-1b9b1c7d-427c-3914-2cc7-805f5eed1622: entering follower state: follower="Node at 1b9b1c7d-427c-3914-2cc7-805f5eed1622 [Follower]" leader-address= leader-id= +2025-03-20T03:50:51.064Z [INFO] server-5ca48ed1-fc92-47a4-a415-598de2fa7d08: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1b9b1c7d-427c-3914-2cc7-805f5eed1622 Address:1b9b1c7d-427c-3914-2cc7-805f5eed1622} {Suffrage:Voter ID:server-5ca48ed1-fc92-47a4-a415-598de2fa7d08 Address:5ca48ed1-fc92-47a4-a415-598de2fa7d08} {Suffrage:Voter ID:server-5ee7177c-8ebb-300e-2d1b-8e68022d551c Address:5ee7177c-8ebb-300e-2d1b-8e68022d551c}]" +2025-03-20T03:50:51.064Z [INFO] server-5ee7177c-8ebb-300e-2d1b-8e68022d551c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1b9b1c7d-427c-3914-2cc7-805f5eed1622 Address:1b9b1c7d-427c-3914-2cc7-805f5eed1622} {Suffrage:Voter ID:server-5ca48ed1-fc92-47a4-a415-598de2fa7d08 Address:5ca48ed1-fc92-47a4-a415-598de2fa7d08} {Suffrage:Voter ID:server-5ee7177c-8ebb-300e-2d1b-8e68022d551c Address:5ee7177c-8ebb-300e-2d1b-8e68022d551c}]" +2025-03-20T03:50:51.065Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:50:51.065Z [INFO] server-5ee7177c-8ebb-300e-2d1b-8e68022d551c: entering follower state: follower="Node at 5ee7177c-8ebb-300e-2d1b-8e68022d551c [Follower]" leader-address= leader-id= +2025-03-20T03:50:51.065Z [INFO] server-5ca48ed1-fc92-47a4-a415-598de2fa7d08: entering follower state: follower="Node at 5ca48ed1-fc92-47a4-a415-598de2fa7d08 [Follower]" leader-address= leader-id= +2025-03-20T03:50:51.617Z [WARN] server-5ee7177c-8ebb-300e-2d1b-8e68022d551c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:50:51.617Z [INFO] server-5ee7177c-8ebb-300e-2d1b-8e68022d551c: entering candidate state: node="Node at 5ee7177c-8ebb-300e-2d1b-8e68022d551c [Candidate]" term=2 +2025-03-20T03:50:51.617Z [INFO] server-5ee7177c-8ebb-300e-2d1b-8e68022d551c: election won: term=2 tally=2 +2025-03-20T03:50:51.617Z [INFO] server-5ee7177c-8ebb-300e-2d1b-8e68022d551c: entering leader state: leader="Node at 5ee7177c-8ebb-300e-2d1b-8e68022d551c [Leader]" +2025-03-20T03:50:51.617Z [INFO] server-5ee7177c-8ebb-300e-2d1b-8e68022d551c: added peer, starting replication: peer=server-1b9b1c7d-427c-3914-2cc7-805f5eed1622 +2025-03-20T03:50:51.617Z [INFO] server-5ee7177c-8ebb-300e-2d1b-8e68022d551c: added peer, starting replication: peer=server-5ca48ed1-fc92-47a4-a415-598de2fa7d08 +2025-03-20T03:50:51.617Z [INFO] server-5ee7177c-8ebb-300e-2d1b-8e68022d551c: pipelining replication: peer="{Voter server-1b9b1c7d-427c-3914-2cc7-805f5eed1622 1b9b1c7d-427c-3914-2cc7-805f5eed1622}" +2025-03-20T03:50:51.617Z [INFO] server-5ee7177c-8ebb-300e-2d1b-8e68022d551c: pipelining replication: peer="{Voter server-5ca48ed1-fc92-47a4-a415-598de2fa7d08 5ca48ed1-fc92-47a4-a415-598de2fa7d08}" +2025-03-20T03:50:52.623Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:50:51.617752806 +0000 UTC m=+83.292205769 (1 nodes), 552.672351ms from start of poll, 553.944299ms from cluster start. Timeout at 2025-03-20 03:50:52.622759973 +0000 UTC m=+84.297213348, 1.005007579s after stability +2025-03-20T03:50:52.623Z [INFO] creating new snapshot: path=/tmp/raft2576542711/snapshots/2-1013-1742442652623.tmp +2025-03-20T03:50:52.624Z [INFO] server-5ee7177c-8ebb-300e-2d1b-8e68022d551c: copied to local snapshot: bytes=71 +2025-03-20T03:50:52.624Z [INFO] server-5ee7177c-8ebb-300e-2d1b-8e68022d551c: snapshot restore progress: id=2-1013-1742442652623 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:52.624Z [INFO] server-5ee7177c-8ebb-300e-2d1b-8e68022d551c: removing all old logs from log store +2025-03-20T03:50:52.624Z [INFO] server-5ee7177c-8ebb-300e-2d1b-8e68022d551c: compacting logs: from=1 to=2 +2025-03-20T03:50:52.624Z [INFO] server-5ee7177c-8ebb-300e-2d1b-8e68022d551c: restored user snapshot: index=1013 +2025-03-20T03:50:52.624Z [ERROR] server-5ee7177c-8ebb-300e-2d1b-8e68022d551c: failed to get log: index=2 error="log not found" +2025-03-20T03:50:52.624Z [INFO] server-5ee7177c-8ebb-300e-2d1b-8e68022d551c: aborting pipeline replication: peer="{Voter server-5ca48ed1-fc92-47a4-a415-598de2fa7d08 5ca48ed1-fc92-47a4-a415-598de2fa7d08}" +2025-03-20T03:50:52.624Z [ERROR] server-5ee7177c-8ebb-300e-2d1b-8e68022d551c: failed to get log: index=2 error="log not found" +2025-03-20T03:50:52.624Z [INFO] server-5ee7177c-8ebb-300e-2d1b-8e68022d551c: aborting pipeline replication: peer="{Voter server-1b9b1c7d-427c-3914-2cc7-805f5eed1622 1b9b1c7d-427c-3914-2cc7-805f5eed1622}" +2025-03-20T03:50:52.630Z [ERROR] server-5ee7177c-8ebb-300e-2d1b-8e68022d551c: failed to get log: index=2 error="log not found" +2025-03-20T03:50:52.631Z [INFO] creating new snapshot: path=/tmp/raft3092658595/snapshots/2-1013-1742442652631.tmp +2025-03-20T03:50:52.631Z [ERROR] server-5ee7177c-8ebb-300e-2d1b-8e68022d551c: failed to get log: index=2 error="log not found" +2025-03-20T03:50:52.631Z [INFO] server-1b9b1c7d-427c-3914-2cc7-805f5eed1622: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:52.632Z [INFO] creating new snapshot: path=/tmp/raft3293783940/snapshots/2-1013-1742442652632.tmp +2025-03-20T03:50:52.632Z [INFO] server-1b9b1c7d-427c-3914-2cc7-805f5eed1622: copied to local snapshot: bytes=71 +2025-03-20T03:50:52.632Z [INFO] server-5ca48ed1-fc92-47a4-a415-598de2fa7d08: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:52.633Z [INFO] server-1b9b1c7d-427c-3914-2cc7-805f5eed1622: snapshot restore progress: id=2-1013-1742442652631 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:52.633Z [INFO] server-5ca48ed1-fc92-47a4-a415-598de2fa7d08: copied to local snapshot: bytes=71 +2025-03-20T03:50:52.633Z [INFO] server-1b9b1c7d-427c-3914-2cc7-805f5eed1622: removing all old logs from log store +2025-03-20T03:50:52.633Z [INFO] server-1b9b1c7d-427c-3914-2cc7-805f5eed1622: compacting logs: from=1 to=2 +2025-03-20T03:50:52.633Z [INFO] server-1b9b1c7d-427c-3914-2cc7-805f5eed1622: Installed remote snapshot +2025-03-20T03:50:52.633Z [INFO] server-5ee7177c-8ebb-300e-2d1b-8e68022d551c: pipelining replication: peer="{Voter server-1b9b1c7d-427c-3914-2cc7-805f5eed1622 1b9b1c7d-427c-3914-2cc7-805f5eed1622}" +2025-03-20T03:50:52.633Z [INFO] server-5ca48ed1-fc92-47a4-a415-598de2fa7d08: snapshot restore progress: id=2-1013-1742442652632 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:52.633Z [INFO] server-5ca48ed1-fc92-47a4-a415-598de2fa7d08: removing all old logs from log store +2025-03-20T03:50:52.633Z [INFO] server-5ca48ed1-fc92-47a4-a415-598de2fa7d08: compacting logs: from=1 to=2 +2025-03-20T03:50:52.633Z [INFO] server-5ca48ed1-fc92-47a4-a415-598de2fa7d08: Installed remote snapshot +2025-03-20T03:50:52.633Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:50:52.634Z [INFO] server-5ee7177c-8ebb-300e-2d1b-8e68022d551c: pipelining replication: peer="{Voter server-5ca48ed1-fc92-47a4-a415-598de2fa7d08 5ca48ed1-fc92-47a4-a415-598de2fa7d08}" +2025-03-20T03:50:53.640Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:50:52.634671879 +0000 UTC m=+84.309124706 (2 nodes), 2.078µs from start of poll, 1.570863236s from cluster start. Timeout at 2025-03-20 03:50:53.639770331 +0000 UTC m=+85.314223187, 1.005098481s after stability +2025-03-20T03:50:53.653Z [INFO] server-ef9b76c4-8a4f-f328-6653-49791f8ba48f: aborting pipeline replication: peer="{Voter server-a50052c9-9ce3-a9ae-6d48-09f39ab12bee a50052c9-9ce3-a9ae-6d48-09f39ab12bee}" +2025-03-20T03:50:53.653Z [INFO] server-ef9b76c4-8a4f-f328-6653-49791f8ba48f: aborting pipeline replication: peer="{Voter server-69a284ca-c41e-b0a5-6e3b-9db93a068c12 69a284ca-c41e-b0a5-6e3b-9db93a068c12}" === RUN TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_false -2026-04-22T10:10:37.492Z [INFO] server-31d596f7-e985-37c8-1baf-8a4e413a2803: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31d596f7-e985-37c8-1baf-8a4e413a2803 Address:31d596f7-e985-37c8-1baf-8a4e413a2803} {Suffrage:Voter ID:server-abf15593-c66d-f22e-48cb-c85ff73f494a Address:abf15593-c66d-f22e-48cb-c85ff73f494a} {Suffrage:Voter ID:server-4c1aac57-541c-bd52-df89-5fc87a5a399c Address:4c1aac57-541c-bd52-df89-5fc87a5a399c}]" -2026-04-22T10:10:37.492Z [INFO] server-31d596f7-e985-37c8-1baf-8a4e413a2803: entering follower state: follower="Node at 31d596f7-e985-37c8-1baf-8a4e413a2803 [Follower]" leader-address= leader-id= -2026-04-22T10:10:37.492Z [INFO] server-abf15593-c66d-f22e-48cb-c85ff73f494a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31d596f7-e985-37c8-1baf-8a4e413a2803 Address:31d596f7-e985-37c8-1baf-8a4e413a2803} {Suffrage:Voter ID:server-abf15593-c66d-f22e-48cb-c85ff73f494a Address:abf15593-c66d-f22e-48cb-c85ff73f494a} {Suffrage:Voter ID:server-4c1aac57-541c-bd52-df89-5fc87a5a399c Address:4c1aac57-541c-bd52-df89-5fc87a5a399c}]" -2026-04-22T10:10:37.492Z [INFO] server-abf15593-c66d-f22e-48cb-c85ff73f494a: entering follower state: follower="Node at abf15593-c66d-f22e-48cb-c85ff73f494a [Follower]" leader-address= leader-id= -2026-04-22T10:10:37.493Z [INFO] server-4c1aac57-541c-bd52-df89-5fc87a5a399c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31d596f7-e985-37c8-1baf-8a4e413a2803 Address:31d596f7-e985-37c8-1baf-8a4e413a2803} {Suffrage:Voter ID:server-abf15593-c66d-f22e-48cb-c85ff73f494a Address:abf15593-c66d-f22e-48cb-c85ff73f494a} {Suffrage:Voter ID:server-4c1aac57-541c-bd52-df89-5fc87a5a399c Address:4c1aac57-541c-bd52-df89-5fc87a5a399c}]" -2026-04-22T10:10:37.493Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:37.493Z [INFO] server-4c1aac57-541c-bd52-df89-5fc87a5a399c: entering follower state: follower="Node at 4c1aac57-541c-bd52-df89-5fc87a5a399c [Follower]" leader-address= leader-id= -2026-04-22T10:10:38.140Z [WARN] server-abf15593-c66d-f22e-48cb-c85ff73f494a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:38.140Z [INFO] server-abf15593-c66d-f22e-48cb-c85ff73f494a: entering candidate state: node="Node at abf15593-c66d-f22e-48cb-c85ff73f494a [Candidate]" term=2 -2026-04-22T10:10:38.140Z [INFO] server-abf15593-c66d-f22e-48cb-c85ff73f494a: election won: term=2 tally=2 -2026-04-22T10:10:38.140Z [INFO] server-abf15593-c66d-f22e-48cb-c85ff73f494a: entering leader state: leader="Node at abf15593-c66d-f22e-48cb-c85ff73f494a [Leader]" -2026-04-22T10:10:38.140Z [INFO] server-abf15593-c66d-f22e-48cb-c85ff73f494a: added peer, starting replication: peer=server-31d596f7-e985-37c8-1baf-8a4e413a2803 -2026-04-22T10:10:38.140Z [INFO] server-abf15593-c66d-f22e-48cb-c85ff73f494a: added peer, starting replication: peer=server-4c1aac57-541c-bd52-df89-5fc87a5a399c -2026-04-22T10:10:38.140Z [INFO] server-abf15593-c66d-f22e-48cb-c85ff73f494a: pipelining replication: peer="{Voter server-31d596f7-e985-37c8-1baf-8a4e413a2803 31d596f7-e985-37c8-1baf-8a4e413a2803}" -2026-04-22T10:10:38.140Z [INFO] server-abf15593-c66d-f22e-48cb-c85ff73f494a: pipelining replication: peer="{Voter server-4c1aac57-541c-bd52-df89-5fc87a5a399c 4c1aac57-541c-bd52-df89-5fc87a5a399c}" -2026-04-22T10:10:39.145Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:38.140307031 +0000 UTC m=+84.164103124 (1 nodes), 647.235162ms from start of poll, 648.113243ms from cluster start. Timeout at 2026-04-22 10:10:39.145312763 +0000 UTC m=+85.169108844, 1.00500572s after stability -2026-04-22T10:10:39.145Z [INFO] server-abf15593-c66d-f22e-48cb-c85ff73f494a: starting snapshot up to: index=12 -2026-04-22T10:10:39.145Z [INFO] creating new snapshot: path=/tmp/raft571035003/snapshots/2-12-1776852639145.tmp -2026-04-22T10:10:39.146Z [INFO] server-abf15593-c66d-f22e-48cb-c85ff73f494a: snapshot complete up to: index=12 -2026-04-22T10:10:39.146Z [INFO] creating new snapshot: path=/tmp/raft571035003/snapshots/2-10013-1776852639146.tmp -2026-04-22T10:10:39.147Z [INFO] server-abf15593-c66d-f22e-48cb-c85ff73f494a: copied to local snapshot: bytes=71 -2026-04-22T10:10:39.147Z [INFO] server-abf15593-c66d-f22e-48cb-c85ff73f494a: snapshot restore progress: id=2-10013-1776852639146 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:39.147Z [INFO] server-abf15593-c66d-f22e-48cb-c85ff73f494a: restored user snapshot: index=10013 -2026-04-22T10:10:39.147Z [ERROR] server-abf15593-c66d-f22e-48cb-c85ff73f494a: failed to get log: index=23 error="log not found" -2026-04-22T10:10:39.147Z [INFO] server-abf15593-c66d-f22e-48cb-c85ff73f494a: aborting pipeline replication: peer="{Voter server-4c1aac57-541c-bd52-df89-5fc87a5a399c 4c1aac57-541c-bd52-df89-5fc87a5a399c}" -2026-04-22T10:10:39.147Z [ERROR] server-abf15593-c66d-f22e-48cb-c85ff73f494a: failed to get log: index=23 error="log not found" -2026-04-22T10:10:39.147Z [INFO] server-abf15593-c66d-f22e-48cb-c85ff73f494a: aborting pipeline replication: peer="{Voter server-31d596f7-e985-37c8-1baf-8a4e413a2803 31d596f7-e985-37c8-1baf-8a4e413a2803}" -2026-04-22T10:10:39.154Z [ERROR] server-abf15593-c66d-f22e-48cb-c85ff73f494a: failed to get log: index=23 error="log not found" -2026-04-22T10:10:39.154Z [INFO] creating new snapshot: path=/tmp/raft594710046/snapshots/2-10013-1776852639154.tmp -2026-04-22T10:10:39.154Z [INFO] server-31d596f7-e985-37c8-1baf-8a4e413a2803: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:39.155Z [INFO] server-31d596f7-e985-37c8-1baf-8a4e413a2803: copied to local snapshot: bytes=71 -2026-04-22T10:10:39.155Z [INFO] server-31d596f7-e985-37c8-1baf-8a4e413a2803: snapshot restore progress: id=2-10013-1776852639154 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:39.155Z [INFO] server-31d596f7-e985-37c8-1baf-8a4e413a2803: Installed remote snapshot -2026-04-22T10:10:39.155Z [WARN] server-31d596f7-e985-37c8-1baf-8a4e413a2803: skipping application of old log: index=22 -2026-04-22T10:10:39.155Z [INFO] server-abf15593-c66d-f22e-48cb-c85ff73f494a: pipelining replication: peer="{Voter server-31d596f7-e985-37c8-1baf-8a4e413a2803 31d596f7-e985-37c8-1baf-8a4e413a2803}" -2026-04-22T10:10:39.155Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:10:39.156Z [ERROR] server-abf15593-c66d-f22e-48cb-c85ff73f494a: failed to get log: index=23 error="log not found" -2026-04-22T10:10:39.156Z [INFO] creating new snapshot: path=/tmp/raft1956719663/snapshots/2-10013-1776852639156.tmp -2026-04-22T10:10:39.156Z [INFO] server-4c1aac57-541c-bd52-df89-5fc87a5a399c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:39.157Z [INFO] server-4c1aac57-541c-bd52-df89-5fc87a5a399c: copied to local snapshot: bytes=71 -2026-04-22T10:10:39.157Z [INFO] server-4c1aac57-541c-bd52-df89-5fc87a5a399c: snapshot restore progress: id=2-10013-1776852639156 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:39.157Z [INFO] server-4c1aac57-541c-bd52-df89-5fc87a5a399c: Installed remote snapshot -2026-04-22T10:10:39.157Z [INFO] server-abf15593-c66d-f22e-48cb-c85ff73f494a: pipelining replication: peer="{Voter server-4c1aac57-541c-bd52-df89-5fc87a5a399c 4c1aac57-541c-bd52-df89-5fc87a5a399c}" -2026-04-22T10:10:40.161Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:10:39.155553594 +0000 UTC m=+85.179349692 (2 nodes), 1.085µs from start of poll, 1.663359811s from cluster start. Timeout at 2026-04-22 10:10:40.160587306 +0000 UTC m=+86.184383421, 1.005033729s after stability -2026-04-22T10:10:40.177Z [INFO] server-abf15593-c66d-f22e-48cb-c85ff73f494a: aborting pipeline replication: peer="{Voter server-31d596f7-e985-37c8-1baf-8a4e413a2803 31d596f7-e985-37c8-1baf-8a4e413a2803}" -2026-04-22T10:10:40.177Z [INFO] server-abf15593-c66d-f22e-48cb-c85ff73f494a: aborting pipeline replication: peer="{Voter server-4c1aac57-541c-bd52-df89-5fc87a5a399c 4c1aac57-541c-bd52-df89-5fc87a5a399c}" +2025-03-20T03:50:53.656Z [INFO] server-a78a38ca-a971-8877-7d3b-823e9a85d905: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a78a38ca-a971-8877-7d3b-823e9a85d905 Address:a78a38ca-a971-8877-7d3b-823e9a85d905} {Suffrage:Voter ID:server-fbead2ed-26cb-02cc-a4cf-c86ee57770e4 Address:fbead2ed-26cb-02cc-a4cf-c86ee57770e4} {Suffrage:Voter ID:server-f9087526-fbba-a579-c499-dcc79bb0f730 Address:f9087526-fbba-a579-c499-dcc79bb0f730}]" +2025-03-20T03:50:53.656Z [INFO] server-fbead2ed-26cb-02cc-a4cf-c86ee57770e4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a78a38ca-a971-8877-7d3b-823e9a85d905 Address:a78a38ca-a971-8877-7d3b-823e9a85d905} {Suffrage:Voter ID:server-fbead2ed-26cb-02cc-a4cf-c86ee57770e4 Address:fbead2ed-26cb-02cc-a4cf-c86ee57770e4} {Suffrage:Voter ID:server-f9087526-fbba-a579-c499-dcc79bb0f730 Address:f9087526-fbba-a579-c499-dcc79bb0f730}]" +2025-03-20T03:50:53.656Z [INFO] server-f9087526-fbba-a579-c499-dcc79bb0f730: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a78a38ca-a971-8877-7d3b-823e9a85d905 Address:a78a38ca-a971-8877-7d3b-823e9a85d905} {Suffrage:Voter ID:server-fbead2ed-26cb-02cc-a4cf-c86ee57770e4 Address:fbead2ed-26cb-02cc-a4cf-c86ee57770e4} {Suffrage:Voter ID:server-f9087526-fbba-a579-c499-dcc79bb0f730 Address:f9087526-fbba-a579-c499-dcc79bb0f730}]" +2025-03-20T03:50:53.656Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:50:53.656Z [INFO] server-a78a38ca-a971-8877-7d3b-823e9a85d905: entering follower state: follower="Node at a78a38ca-a971-8877-7d3b-823e9a85d905 [Follower]" leader-address= leader-id= +2025-03-20T03:50:53.657Z [INFO] server-fbead2ed-26cb-02cc-a4cf-c86ee57770e4: entering follower state: follower="Node at fbead2ed-26cb-02cc-a4cf-c86ee57770e4 [Follower]" leader-address= leader-id= +2025-03-20T03:50:53.657Z [INFO] server-f9087526-fbba-a579-c499-dcc79bb0f730: entering follower state: follower="Node at f9087526-fbba-a579-c499-dcc79bb0f730 [Follower]" leader-address= leader-id= +2025-03-20T03:50:54.254Z [WARN] server-fbead2ed-26cb-02cc-a4cf-c86ee57770e4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:50:54.254Z [INFO] server-fbead2ed-26cb-02cc-a4cf-c86ee57770e4: entering candidate state: node="Node at fbead2ed-26cb-02cc-a4cf-c86ee57770e4 [Candidate]" term=2 +2025-03-20T03:50:54.254Z [INFO] server-fbead2ed-26cb-02cc-a4cf-c86ee57770e4: election won: term=2 tally=2 +2025-03-20T03:50:54.254Z [INFO] server-fbead2ed-26cb-02cc-a4cf-c86ee57770e4: entering leader state: leader="Node at fbead2ed-26cb-02cc-a4cf-c86ee57770e4 [Leader]" +2025-03-20T03:50:54.254Z [INFO] server-fbead2ed-26cb-02cc-a4cf-c86ee57770e4: added peer, starting replication: peer=server-a78a38ca-a971-8877-7d3b-823e9a85d905 +2025-03-20T03:50:54.254Z [INFO] server-fbead2ed-26cb-02cc-a4cf-c86ee57770e4: added peer, starting replication: peer=server-f9087526-fbba-a579-c499-dcc79bb0f730 +2025-03-20T03:50:54.254Z [INFO] server-fbead2ed-26cb-02cc-a4cf-c86ee57770e4: pipelining replication: peer="{Voter server-f9087526-fbba-a579-c499-dcc79bb0f730 f9087526-fbba-a579-c499-dcc79bb0f730}" +2025-03-20T03:50:54.255Z [INFO] server-fbead2ed-26cb-02cc-a4cf-c86ee57770e4: pipelining replication: peer="{Voter server-a78a38ca-a971-8877-7d3b-823e9a85d905 a78a38ca-a971-8877-7d3b-823e9a85d905}" +2025-03-20T03:50:55.261Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:50:54.256280005 +0000 UTC m=+85.930733118 (1 nodes), 599.709408ms from start of poll, 600.224291ms from cluster start. Timeout at 2025-03-20 03:50:55.261305881 +0000 UTC m=+86.935758825, 1.005025707s after stability +2025-03-20T03:50:55.264Z [INFO] server-fbead2ed-26cb-02cc-a4cf-c86ee57770e4: starting snapshot up to: index=12 +2025-03-20T03:50:55.266Z [INFO] creating new snapshot: path=/tmp/raft529500636/snapshots/2-12-1742442655266.tmp +2025-03-20T03:50:55.267Z [INFO] server-fbead2ed-26cb-02cc-a4cf-c86ee57770e4: snapshot complete up to: index=12 +2025-03-20T03:50:55.272Z [INFO] creating new snapshot: path=/tmp/raft529500636/snapshots/2-10013-1742442655272.tmp +2025-03-20T03:50:55.275Z [INFO] server-fbead2ed-26cb-02cc-a4cf-c86ee57770e4: copied to local snapshot: bytes=71 +2025-03-20T03:50:55.276Z [INFO] server-fbead2ed-26cb-02cc-a4cf-c86ee57770e4: snapshot restore progress: id=2-10013-1742442655272 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:55.276Z [INFO] server-fbead2ed-26cb-02cc-a4cf-c86ee57770e4: restored user snapshot: index=10013 +2025-03-20T03:50:55.276Z [ERROR] server-fbead2ed-26cb-02cc-a4cf-c86ee57770e4: failed to get log: index=23 error="log not found" +2025-03-20T03:50:55.276Z [INFO] server-fbead2ed-26cb-02cc-a4cf-c86ee57770e4: aborting pipeline replication: peer="{Voter server-f9087526-fbba-a579-c499-dcc79bb0f730 f9087526-fbba-a579-c499-dcc79bb0f730}" +2025-03-20T03:50:55.276Z [ERROR] server-fbead2ed-26cb-02cc-a4cf-c86ee57770e4: failed to get log: index=23 error="log not found" +2025-03-20T03:50:55.276Z [INFO] server-fbead2ed-26cb-02cc-a4cf-c86ee57770e4: aborting pipeline replication: peer="{Voter server-a78a38ca-a971-8877-7d3b-823e9a85d905 a78a38ca-a971-8877-7d3b-823e9a85d905}" +2025-03-20T03:50:55.283Z [ERROR] server-fbead2ed-26cb-02cc-a4cf-c86ee57770e4: failed to get log: index=23 error="log not found" +2025-03-20T03:50:55.284Z [INFO] creating new snapshot: path=/tmp/raft2618368769/snapshots/2-10013-1742442655284.tmp +2025-03-20T03:50:55.284Z [INFO] server-f9087526-fbba-a579-c499-dcc79bb0f730: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:55.286Z [ERROR] server-fbead2ed-26cb-02cc-a4cf-c86ee57770e4: failed to get log: index=23 error="log not found" +2025-03-20T03:50:55.287Z [INFO] creating new snapshot: path=/tmp/raft186385528/snapshots/2-10013-1742442655287.tmp +2025-03-20T03:50:55.287Z [INFO] server-a78a38ca-a971-8877-7d3b-823e9a85d905: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:55.288Z [INFO] server-a78a38ca-a971-8877-7d3b-823e9a85d905: copied to local snapshot: bytes=71 +2025-03-20T03:50:55.289Z [INFO] server-f9087526-fbba-a579-c499-dcc79bb0f730: copied to local snapshot: bytes=71 +2025-03-20T03:50:55.290Z [INFO] server-f9087526-fbba-a579-c499-dcc79bb0f730: snapshot restore progress: id=2-10013-1742442655284 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:55.290Z [INFO] server-f9087526-fbba-a579-c499-dcc79bb0f730: Installed remote snapshot +2025-03-20T03:50:55.290Z [WARN] server-f9087526-fbba-a579-c499-dcc79bb0f730: skipping application of old log: index=22 +2025-03-20T03:50:55.290Z [INFO] server-fbead2ed-26cb-02cc-a4cf-c86ee57770e4: pipelining replication: peer="{Voter server-f9087526-fbba-a579-c499-dcc79bb0f730 f9087526-fbba-a579-c499-dcc79bb0f730}" +2025-03-20T03:50:55.290Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:50:55.291Z [INFO] server-a78a38ca-a971-8877-7d3b-823e9a85d905: snapshot restore progress: id=2-10013-1742442655287 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:55.299Z [INFO] server-a78a38ca-a971-8877-7d3b-823e9a85d905: Installed remote snapshot +2025-03-20T03:50:55.299Z [INFO] server-fbead2ed-26cb-02cc-a4cf-c86ee57770e4: pipelining replication: peer="{Voter server-a78a38ca-a971-8877-7d3b-823e9a85d905 a78a38ca-a971-8877-7d3b-823e9a85d905}" +2025-03-20T03:50:56.296Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:50:55.2907459 +0000 UTC m=+86.965198711 (2 nodes), 2.306µs from start of poll, 1.634689884s from cluster start. Timeout at 2025-03-20 03:50:56.295986624 +0000 UTC m=+87.970439723, 1.005241012s after stability +2025-03-20T03:50:56.314Z [INFO] server-fbead2ed-26cb-02cc-a4cf-c86ee57770e4: aborting pipeline replication: peer="{Voter server-f9087526-fbba-a579-c499-dcc79bb0f730 f9087526-fbba-a579-c499-dcc79bb0f730}" +2025-03-20T03:50:56.315Z [INFO] server-fbead2ed-26cb-02cc-a4cf-c86ee57770e4: aborting pipeline replication: peer="{Voter server-a78a38ca-a971-8877-7d3b-823e9a85d905 a78a38ca-a971-8877-7d3b-823e9a85d905}" === RUN TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_false -2026-04-22T10:10:40.178Z [INFO] server-37346471-9e74-b1cb-0cb6-0d343376117e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-37346471-9e74-b1cb-0cb6-0d343376117e Address:37346471-9e74-b1cb-0cb6-0d343376117e} {Suffrage:Voter ID:server-a25f9558-a32c-26a8-7e96-7570f4c7966a Address:a25f9558-a32c-26a8-7e96-7570f4c7966a} {Suffrage:Voter ID:server-f1715582-af1c-1f4d-40ae-b6b574ddd4e6 Address:f1715582-af1c-1f4d-40ae-b6b574ddd4e6}]" -2026-04-22T10:10:40.178Z [INFO] server-37346471-9e74-b1cb-0cb6-0d343376117e: entering follower state: follower="Node at 37346471-9e74-b1cb-0cb6-0d343376117e [Follower]" leader-address= leader-id= -2026-04-22T10:10:40.179Z [INFO] server-a25f9558-a32c-26a8-7e96-7570f4c7966a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-37346471-9e74-b1cb-0cb6-0d343376117e Address:37346471-9e74-b1cb-0cb6-0d343376117e} {Suffrage:Voter ID:server-a25f9558-a32c-26a8-7e96-7570f4c7966a Address:a25f9558-a32c-26a8-7e96-7570f4c7966a} {Suffrage:Voter ID:server-f1715582-af1c-1f4d-40ae-b6b574ddd4e6 Address:f1715582-af1c-1f4d-40ae-b6b574ddd4e6}]" -2026-04-22T10:10:40.179Z [INFO] server-a25f9558-a32c-26a8-7e96-7570f4c7966a: entering follower state: follower="Node at a25f9558-a32c-26a8-7e96-7570f4c7966a [Follower]" leader-address= leader-id= -2026-04-22T10:10:40.179Z [INFO] server-f1715582-af1c-1f4d-40ae-b6b574ddd4e6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-37346471-9e74-b1cb-0cb6-0d343376117e Address:37346471-9e74-b1cb-0cb6-0d343376117e} {Suffrage:Voter ID:server-a25f9558-a32c-26a8-7e96-7570f4c7966a Address:a25f9558-a32c-26a8-7e96-7570f4c7966a} {Suffrage:Voter ID:server-f1715582-af1c-1f4d-40ae-b6b574ddd4e6 Address:f1715582-af1c-1f4d-40ae-b6b574ddd4e6}]" -2026-04-22T10:10:40.179Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:40.179Z [INFO] server-f1715582-af1c-1f4d-40ae-b6b574ddd4e6: entering follower state: follower="Node at f1715582-af1c-1f4d-40ae-b6b574ddd4e6 [Follower]" leader-address= leader-id= -2026-04-22T10:10:40.937Z [WARN] server-f1715582-af1c-1f4d-40ae-b6b574ddd4e6: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:40.937Z [INFO] server-f1715582-af1c-1f4d-40ae-b6b574ddd4e6: entering candidate state: node="Node at f1715582-af1c-1f4d-40ae-b6b574ddd4e6 [Candidate]" term=2 -2026-04-22T10:10:40.937Z [INFO] server-f1715582-af1c-1f4d-40ae-b6b574ddd4e6: election won: term=2 tally=2 -2026-04-22T10:10:40.937Z [INFO] server-f1715582-af1c-1f4d-40ae-b6b574ddd4e6: entering leader state: leader="Node at f1715582-af1c-1f4d-40ae-b6b574ddd4e6 [Leader]" -2026-04-22T10:10:40.937Z [INFO] server-f1715582-af1c-1f4d-40ae-b6b574ddd4e6: added peer, starting replication: peer=server-37346471-9e74-b1cb-0cb6-0d343376117e -2026-04-22T10:10:40.937Z [INFO] server-f1715582-af1c-1f4d-40ae-b6b574ddd4e6: added peer, starting replication: peer=server-a25f9558-a32c-26a8-7e96-7570f4c7966a -2026-04-22T10:10:40.937Z [INFO] server-f1715582-af1c-1f4d-40ae-b6b574ddd4e6: pipelining replication: peer="{Voter server-37346471-9e74-b1cb-0cb6-0d343376117e 37346471-9e74-b1cb-0cb6-0d343376117e}" -2026-04-22T10:10:40.938Z [INFO] server-f1715582-af1c-1f4d-40ae-b6b574ddd4e6: pipelining replication: peer="{Voter server-a25f9558-a32c-26a8-7e96-7570f4c7966a a25f9558-a32c-26a8-7e96-7570f4c7966a}" -2026-04-22T10:10:41.943Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:40.937932904 +0000 UTC m=+86.961728992 (1 nodes), 758.77344ms from start of poll, 759.078169ms from cluster start. Timeout at 2026-04-22 10:10:41.942934769 +0000 UTC m=+87.966730905, 1.005001913s after stability -2026-04-22T10:10:41.944Z [INFO] server-f1715582-af1c-1f4d-40ae-b6b574ddd4e6: starting snapshot up to: index=12 -2026-04-22T10:10:41.944Z [INFO] creating new snapshot: path=/tmp/raft1111434963/snapshots/2-12-1776852641944.tmp -2026-04-22T10:10:41.944Z [INFO] server-f1715582-af1c-1f4d-40ae-b6b574ddd4e6: snapshot complete up to: index=12 -2026-04-22T10:10:41.945Z [INFO] creating new snapshot: path=/tmp/raft1111434963/snapshots/2-10013-1776852641945.tmp -2026-04-22T10:10:41.945Z [INFO] server-f1715582-af1c-1f4d-40ae-b6b574ddd4e6: copied to local snapshot: bytes=71 -2026-04-22T10:10:41.946Z [INFO] server-f1715582-af1c-1f4d-40ae-b6b574ddd4e6: snapshot restore progress: id=2-10013-1776852641945 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:41.946Z [INFO] server-f1715582-af1c-1f4d-40ae-b6b574ddd4e6: removing all old logs from log store -2026-04-22T10:10:41.946Z [INFO] server-f1715582-af1c-1f4d-40ae-b6b574ddd4e6: compacting logs: from=1 to=22 -2026-04-22T10:10:41.946Z [INFO] server-f1715582-af1c-1f4d-40ae-b6b574ddd4e6: restored user snapshot: index=10013 -2026-04-22T10:10:41.946Z [ERROR] server-f1715582-af1c-1f4d-40ae-b6b574ddd4e6: failed to get log: index=22 error="log not found" -2026-04-22T10:10:41.946Z [INFO] server-f1715582-af1c-1f4d-40ae-b6b574ddd4e6: aborting pipeline replication: peer="{Voter server-37346471-9e74-b1cb-0cb6-0d343376117e 37346471-9e74-b1cb-0cb6-0d343376117e}" -2026-04-22T10:10:41.946Z [ERROR] server-f1715582-af1c-1f4d-40ae-b6b574ddd4e6: failed to get log: index=22 error="log not found" -2026-04-22T10:10:41.946Z [INFO] server-f1715582-af1c-1f4d-40ae-b6b574ddd4e6: aborting pipeline replication: peer="{Voter server-a25f9558-a32c-26a8-7e96-7570f4c7966a a25f9558-a32c-26a8-7e96-7570f4c7966a}" -2026-04-22T10:10:41.952Z [ERROR] server-f1715582-af1c-1f4d-40ae-b6b574ddd4e6: failed to get log: index=22 error="log not found" -2026-04-22T10:10:41.953Z [INFO] creating new snapshot: path=/tmp/raft3715421644/snapshots/2-10013-1776852641953.tmp -2026-04-22T10:10:41.953Z [INFO] server-37346471-9e74-b1cb-0cb6-0d343376117e: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:41.953Z [INFO] server-37346471-9e74-b1cb-0cb6-0d343376117e: copied to local snapshot: bytes=71 -2026-04-22T10:10:41.954Z [INFO] server-37346471-9e74-b1cb-0cb6-0d343376117e: snapshot restore progress: id=2-10013-1776852641953 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:41.954Z [INFO] server-37346471-9e74-b1cb-0cb6-0d343376117e: removing all old logs from log store -2026-04-22T10:10:41.954Z [INFO] server-37346471-9e74-b1cb-0cb6-0d343376117e: compacting logs: from=1 to=22 -2026-04-22T10:10:41.954Z [INFO] server-37346471-9e74-b1cb-0cb6-0d343376117e: Installed remote snapshot -2026-04-22T10:10:41.954Z [WARN] server-37346471-9e74-b1cb-0cb6-0d343376117e: skipping application of old log: index=22 -2026-04-22T10:10:41.954Z [INFO] server-f1715582-af1c-1f4d-40ae-b6b574ddd4e6: pipelining replication: peer="{Voter server-37346471-9e74-b1cb-0cb6-0d343376117e 37346471-9e74-b1cb-0cb6-0d343376117e}" -2026-04-22T10:10:41.954Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:10:41.957Z [ERROR] server-f1715582-af1c-1f4d-40ae-b6b574ddd4e6: failed to get log: index=22 error="log not found" -2026-04-22T10:10:41.958Z [INFO] creating new snapshot: path=/tmp/raft210572504/snapshots/2-10013-1776852641958.tmp -2026-04-22T10:10:41.958Z [INFO] server-a25f9558-a32c-26a8-7e96-7570f4c7966a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:41.958Z [INFO] server-a25f9558-a32c-26a8-7e96-7570f4c7966a: copied to local snapshot: bytes=71 -2026-04-22T10:10:41.958Z [INFO] server-a25f9558-a32c-26a8-7e96-7570f4c7966a: snapshot restore progress: id=2-10013-1776852641958 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:41.958Z [INFO] server-a25f9558-a32c-26a8-7e96-7570f4c7966a: removing all old logs from log store -2026-04-22T10:10:41.958Z [INFO] server-a25f9558-a32c-26a8-7e96-7570f4c7966a: compacting logs: from=1 to=22 -2026-04-22T10:10:41.958Z [INFO] server-a25f9558-a32c-26a8-7e96-7570f4c7966a: Installed remote snapshot -2026-04-22T10:10:41.958Z [INFO] server-f1715582-af1c-1f4d-40ae-b6b574ddd4e6: pipelining replication: peer="{Voter server-a25f9558-a32c-26a8-7e96-7570f4c7966a a25f9558-a32c-26a8-7e96-7570f4c7966a}" -2026-04-22T10:10:42.960Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:10:41.954355363 +0000 UTC m=+87.978151452 (2 nodes), 687ns from start of poll, 1.775500629s from cluster start. Timeout at 2026-04-22 10:10:42.959375143 +0000 UTC m=+88.983171333, 1.005019881s after stability -2026-04-22T10:10:42.971Z [INFO] server-f1715582-af1c-1f4d-40ae-b6b574ddd4e6: aborting pipeline replication: peer="{Voter server-a25f9558-a32c-26a8-7e96-7570f4c7966a a25f9558-a32c-26a8-7e96-7570f4c7966a}" -2026-04-22T10:10:42.971Z [INFO] server-f1715582-af1c-1f4d-40ae-b6b574ddd4e6: aborting pipeline replication: peer="{Voter server-37346471-9e74-b1cb-0cb6-0d343376117e 37346471-9e74-b1cb-0cb6-0d343376117e}" +2025-03-20T03:50:56.324Z [INFO] server-726a475a-46fc-ef5f-4277-9b04ab82843c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-726a475a-46fc-ef5f-4277-9b04ab82843c Address:726a475a-46fc-ef5f-4277-9b04ab82843c} {Suffrage:Voter ID:server-33906d2a-66ab-6e9c-465a-30ec94c4fe0d Address:33906d2a-66ab-6e9c-465a-30ec94c4fe0d} {Suffrage:Voter ID:server-1cb4b734-39ca-a85c-e86e-b0f5b5337985 Address:1cb4b734-39ca-a85c-e86e-b0f5b5337985}]" +2025-03-20T03:50:56.324Z [INFO] server-33906d2a-66ab-6e9c-465a-30ec94c4fe0d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-726a475a-46fc-ef5f-4277-9b04ab82843c Address:726a475a-46fc-ef5f-4277-9b04ab82843c} {Suffrage:Voter ID:server-33906d2a-66ab-6e9c-465a-30ec94c4fe0d Address:33906d2a-66ab-6e9c-465a-30ec94c4fe0d} {Suffrage:Voter ID:server-1cb4b734-39ca-a85c-e86e-b0f5b5337985 Address:1cb4b734-39ca-a85c-e86e-b0f5b5337985}]" +2025-03-20T03:50:56.324Z [INFO] server-1cb4b734-39ca-a85c-e86e-b0f5b5337985: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-726a475a-46fc-ef5f-4277-9b04ab82843c Address:726a475a-46fc-ef5f-4277-9b04ab82843c} {Suffrage:Voter ID:server-33906d2a-66ab-6e9c-465a-30ec94c4fe0d Address:33906d2a-66ab-6e9c-465a-30ec94c4fe0d} {Suffrage:Voter ID:server-1cb4b734-39ca-a85c-e86e-b0f5b5337985 Address:1cb4b734-39ca-a85c-e86e-b0f5b5337985}]" +2025-03-20T03:50:56.325Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:50:56.325Z [INFO] server-726a475a-46fc-ef5f-4277-9b04ab82843c: entering follower state: follower="Node at 726a475a-46fc-ef5f-4277-9b04ab82843c [Follower]" leader-address= leader-id= +2025-03-20T03:50:56.325Z [INFO] server-33906d2a-66ab-6e9c-465a-30ec94c4fe0d: entering follower state: follower="Node at 33906d2a-66ab-6e9c-465a-30ec94c4fe0d [Follower]" leader-address= leader-id= +2025-03-20T03:50:56.325Z [INFO] server-1cb4b734-39ca-a85c-e86e-b0f5b5337985: entering follower state: follower="Node at 1cb4b734-39ca-a85c-e86e-b0f5b5337985 [Follower]" leader-address= leader-id= +2025-03-20T03:50:56.947Z [WARN] server-33906d2a-66ab-6e9c-465a-30ec94c4fe0d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:50:56.949Z [INFO] server-33906d2a-66ab-6e9c-465a-30ec94c4fe0d: entering candidate state: node="Node at 33906d2a-66ab-6e9c-465a-30ec94c4fe0d [Candidate]" term=2 +2025-03-20T03:50:56.950Z [INFO] server-33906d2a-66ab-6e9c-465a-30ec94c4fe0d: election won: term=2 tally=2 +2025-03-20T03:50:56.951Z [INFO] server-33906d2a-66ab-6e9c-465a-30ec94c4fe0d: entering leader state: leader="Node at 33906d2a-66ab-6e9c-465a-30ec94c4fe0d [Leader]" +2025-03-20T03:50:56.951Z [INFO] server-33906d2a-66ab-6e9c-465a-30ec94c4fe0d: added peer, starting replication: peer=server-726a475a-46fc-ef5f-4277-9b04ab82843c +2025-03-20T03:50:56.952Z [INFO] server-33906d2a-66ab-6e9c-465a-30ec94c4fe0d: added peer, starting replication: peer=server-1cb4b734-39ca-a85c-e86e-b0f5b5337985 +2025-03-20T03:50:56.952Z [INFO] server-33906d2a-66ab-6e9c-465a-30ec94c4fe0d: pipelining replication: peer="{Voter server-1cb4b734-39ca-a85c-e86e-b0f5b5337985 1cb4b734-39ca-a85c-e86e-b0f5b5337985}" +2025-03-20T03:50:56.952Z [INFO] server-33906d2a-66ab-6e9c-465a-30ec94c4fe0d: pipelining replication: peer="{Voter server-726a475a-46fc-ef5f-4277-9b04ab82843c 726a475a-46fc-ef5f-4277-9b04ab82843c}" +2025-03-20T03:50:57.958Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:50:56.95142153 +0000 UTC m=+88.625874365 (1 nodes), 626.378814ms from start of poll, 627.150221ms from cluster start. Timeout at 2025-03-20 03:50:57.956428986 +0000 UTC m=+89.630881870, 1.005007505s after stability +2025-03-20T03:50:57.963Z [INFO] server-33906d2a-66ab-6e9c-465a-30ec94c4fe0d: starting snapshot up to: index=12 +2025-03-20T03:50:57.966Z [INFO] creating new snapshot: path=/tmp/raft614760165/snapshots/2-12-1742442657965.tmp +2025-03-20T03:50:57.999Z [INFO] server-33906d2a-66ab-6e9c-465a-30ec94c4fe0d: snapshot complete up to: index=12 +2025-03-20T03:50:58.062Z [INFO] creating new snapshot: path=/tmp/raft614760165/snapshots/2-10013-1742442658062.tmp +2025-03-20T03:50:58.064Z [INFO] server-33906d2a-66ab-6e9c-465a-30ec94c4fe0d: copied to local snapshot: bytes=71 +2025-03-20T03:50:58.079Z [INFO] server-33906d2a-66ab-6e9c-465a-30ec94c4fe0d: snapshot restore progress: id=2-10013-1742442658062 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:58.079Z [INFO] server-33906d2a-66ab-6e9c-465a-30ec94c4fe0d: removing all old logs from log store +2025-03-20T03:50:58.079Z [INFO] server-33906d2a-66ab-6e9c-465a-30ec94c4fe0d: compacting logs: from=1 to=22 +2025-03-20T03:50:58.079Z [INFO] server-33906d2a-66ab-6e9c-465a-30ec94c4fe0d: restored user snapshot: index=10013 +2025-03-20T03:50:58.079Z [ERROR] server-33906d2a-66ab-6e9c-465a-30ec94c4fe0d: failed to get log: index=22 error="log not found" +2025-03-20T03:50:58.079Z [INFO] server-33906d2a-66ab-6e9c-465a-30ec94c4fe0d: aborting pipeline replication: peer="{Voter server-1cb4b734-39ca-a85c-e86e-b0f5b5337985 1cb4b734-39ca-a85c-e86e-b0f5b5337985}" +2025-03-20T03:50:58.079Z [ERROR] server-33906d2a-66ab-6e9c-465a-30ec94c4fe0d: failed to get log: index=22 error="log not found" +2025-03-20T03:50:58.079Z [INFO] server-33906d2a-66ab-6e9c-465a-30ec94c4fe0d: aborting pipeline replication: peer="{Voter server-726a475a-46fc-ef5f-4277-9b04ab82843c 726a475a-46fc-ef5f-4277-9b04ab82843c}" +2025-03-20T03:50:58.089Z [ERROR] server-33906d2a-66ab-6e9c-465a-30ec94c4fe0d: failed to get log: index=22 error="log not found" +2025-03-20T03:50:58.089Z [ERROR] server-33906d2a-66ab-6e9c-465a-30ec94c4fe0d: failed to get log: index=22 error="log not found" +2025-03-20T03:50:58.089Z [INFO] creating new snapshot: path=/tmp/raft2266886488/snapshots/2-10013-1742442658089.tmp +2025-03-20T03:50:58.089Z [INFO] creating new snapshot: path=/tmp/raft2770259885/snapshots/2-10013-1742442658089.tmp +2025-03-20T03:50:58.090Z [INFO] server-1cb4b734-39ca-a85c-e86e-b0f5b5337985: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:58.090Z [INFO] server-726a475a-46fc-ef5f-4277-9b04ab82843c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:58.091Z [INFO] server-726a475a-46fc-ef5f-4277-9b04ab82843c: copied to local snapshot: bytes=71 +2025-03-20T03:50:58.091Z [INFO] server-1cb4b734-39ca-a85c-e86e-b0f5b5337985: copied to local snapshot: bytes=71 +2025-03-20T03:50:58.091Z [INFO] server-1cb4b734-39ca-a85c-e86e-b0f5b5337985: snapshot restore progress: id=2-10013-1742442658089 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:58.091Z [INFO] server-726a475a-46fc-ef5f-4277-9b04ab82843c: snapshot restore progress: id=2-10013-1742442658089 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:50:58.091Z [INFO] server-726a475a-46fc-ef5f-4277-9b04ab82843c: removing all old logs from log store +2025-03-20T03:50:58.091Z [INFO] server-726a475a-46fc-ef5f-4277-9b04ab82843c: compacting logs: from=1 to=22 +2025-03-20T03:50:58.091Z [INFO] server-726a475a-46fc-ef5f-4277-9b04ab82843c: Installed remote snapshot +2025-03-20T03:50:58.091Z [INFO] server-1cb4b734-39ca-a85c-e86e-b0f5b5337985: removing all old logs from log store +2025-03-20T03:50:58.091Z [INFO] server-1cb4b734-39ca-a85c-e86e-b0f5b5337985: compacting logs: from=1 to=22 +2025-03-20T03:50:58.091Z [INFO] server-1cb4b734-39ca-a85c-e86e-b0f5b5337985: Installed remote snapshot +2025-03-20T03:50:58.093Z [INFO] server-33906d2a-66ab-6e9c-465a-30ec94c4fe0d: pipelining replication: peer="{Voter server-726a475a-46fc-ef5f-4277-9b04ab82843c 726a475a-46fc-ef5f-4277-9b04ab82843c}" +2025-03-20T03:50:58.093Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:50:58.093Z [INFO] server-33906d2a-66ab-6e9c-465a-30ec94c4fe0d: pipelining replication: peer="{Voter server-1cb4b734-39ca-a85c-e86e-b0f5b5337985 1cb4b734-39ca-a85c-e86e-b0f5b5337985}" +2025-03-20T03:50:59.098Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:50:58.093612066 +0000 UTC m=+89.768065124 (2 nodes), 2.804µs from start of poll, 1.76934098s from cluster start. Timeout at 2025-03-20 03:50:59.098669264 +0000 UTC m=+90.773122027, 1.005056903s after stability +2025-03-20T03:50:59.119Z [INFO] server-33906d2a-66ab-6e9c-465a-30ec94c4fe0d: aborting pipeline replication: peer="{Voter server-1cb4b734-39ca-a85c-e86e-b0f5b5337985 1cb4b734-39ca-a85c-e86e-b0f5b5337985}" +2025-03-20T03:50:59.119Z [INFO] server-33906d2a-66ab-6e9c-465a-30ec94c4fe0d: aborting pipeline replication: peer="{Voter server-726a475a-46fc-ef5f-4277-9b04ab82843c 726a475a-46fc-ef5f-4277-9b04ab82843c}" === RUN TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_true -2026-04-22T10:10:42.972Z [INFO] server-74f249c8-e935-8345-8467-911569067747: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-74f249c8-e935-8345-8467-911569067747 Address:74f249c8-e935-8345-8467-911569067747} {Suffrage:Voter ID:server-443b1416-14d3-8558-0989-055ac897e976 Address:443b1416-14d3-8558-0989-055ac897e976} {Suffrage:Voter ID:server-c5e5714b-11dd-6633-9f56-4e373ebb14c3 Address:c5e5714b-11dd-6633-9f56-4e373ebb14c3}]" -2026-04-22T10:10:42.972Z [INFO] server-74f249c8-e935-8345-8467-911569067747: entering follower state: follower="Node at 74f249c8-e935-8345-8467-911569067747 [Follower]" leader-address= leader-id= -2026-04-22T10:10:42.972Z [INFO] server-443b1416-14d3-8558-0989-055ac897e976: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-74f249c8-e935-8345-8467-911569067747 Address:74f249c8-e935-8345-8467-911569067747} {Suffrage:Voter ID:server-443b1416-14d3-8558-0989-055ac897e976 Address:443b1416-14d3-8558-0989-055ac897e976} {Suffrage:Voter ID:server-c5e5714b-11dd-6633-9f56-4e373ebb14c3 Address:c5e5714b-11dd-6633-9f56-4e373ebb14c3}]" -2026-04-22T10:10:42.972Z [INFO] server-443b1416-14d3-8558-0989-055ac897e976: entering follower state: follower="Node at 443b1416-14d3-8558-0989-055ac897e976 [Follower]" leader-address= leader-id= -2026-04-22T10:10:42.972Z [INFO] server-c5e5714b-11dd-6633-9f56-4e373ebb14c3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-74f249c8-e935-8345-8467-911569067747 Address:74f249c8-e935-8345-8467-911569067747} {Suffrage:Voter ID:server-443b1416-14d3-8558-0989-055ac897e976 Address:443b1416-14d3-8558-0989-055ac897e976} {Suffrage:Voter ID:server-c5e5714b-11dd-6633-9f56-4e373ebb14c3 Address:c5e5714b-11dd-6633-9f56-4e373ebb14c3}]" -2026-04-22T10:10:42.973Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:42.973Z [INFO] server-c5e5714b-11dd-6633-9f56-4e373ebb14c3: entering follower state: follower="Node at c5e5714b-11dd-6633-9f56-4e373ebb14c3 [Follower]" leader-address= leader-id= -2026-04-22T10:10:43.551Z [WARN] server-443b1416-14d3-8558-0989-055ac897e976: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:43.551Z [INFO] server-443b1416-14d3-8558-0989-055ac897e976: entering candidate state: node="Node at 443b1416-14d3-8558-0989-055ac897e976 [Candidate]" term=2 -2026-04-22T10:10:43.551Z [INFO] server-443b1416-14d3-8558-0989-055ac897e976: election won: term=2 tally=2 -2026-04-22T10:10:43.551Z [INFO] server-443b1416-14d3-8558-0989-055ac897e976: entering leader state: leader="Node at 443b1416-14d3-8558-0989-055ac897e976 [Leader]" -2026-04-22T10:10:43.551Z [INFO] server-443b1416-14d3-8558-0989-055ac897e976: added peer, starting replication: peer=server-74f249c8-e935-8345-8467-911569067747 -2026-04-22T10:10:43.551Z [INFO] server-443b1416-14d3-8558-0989-055ac897e976: added peer, starting replication: peer=server-c5e5714b-11dd-6633-9f56-4e373ebb14c3 -2026-04-22T10:10:43.551Z [INFO] server-443b1416-14d3-8558-0989-055ac897e976: pipelining replication: peer="{Voter server-74f249c8-e935-8345-8467-911569067747 74f249c8-e935-8345-8467-911569067747}" -2026-04-22T10:10:43.551Z [INFO] server-443b1416-14d3-8558-0989-055ac897e976: pipelining replication: peer="{Voter server-c5e5714b-11dd-6633-9f56-4e373ebb14c3 c5e5714b-11dd-6633-9f56-4e373ebb14c3}" -2026-04-22T10:10:44.557Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:43.551621209 +0000 UTC m=+89.575417298 (1 nodes), 578.605051ms from start of poll, 578.855412ms from cluster start. Timeout at 2026-04-22 10:10:44.556623603 +0000 UTC m=+90.580419684, 1.005002386s after stability -2026-04-22T10:10:44.557Z [INFO] server-443b1416-14d3-8558-0989-055ac897e976: starting snapshot up to: index=12 -2026-04-22T10:10:44.557Z [INFO] creating new snapshot: path=/tmp/raft1634090500/snapshots/2-12-1776852644557.tmp -2026-04-22T10:10:44.558Z [INFO] server-443b1416-14d3-8558-0989-055ac897e976: snapshot complete up to: index=12 -2026-04-22T10:10:44.559Z [INFO] server-3c500fa8-e4d3-a9b3-7919-005ff7e42c45: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3c500fa8-e4d3-a9b3-7919-005ff7e42c45 Address:3c500fa8-e4d3-a9b3-7919-005ff7e42c45} {Suffrage:Voter ID:server-1a378a13-bc14-6333-bf01-60a05f46d57c Address:1a378a13-bc14-6333-bf01-60a05f46d57c} {Suffrage:Voter ID:server-65ab38b4-c201-2589-5555-dcbb8e7c2ea2 Address:65ab38b4-c201-2589-5555-dcbb8e7c2ea2}]" -2026-04-22T10:10:44.559Z [INFO] server-3c500fa8-e4d3-a9b3-7919-005ff7e42c45: entering follower state: follower="Node at 3c500fa8-e4d3-a9b3-7919-005ff7e42c45 [Follower]" leader-address= leader-id= -2026-04-22T10:10:44.559Z [INFO] server-1a378a13-bc14-6333-bf01-60a05f46d57c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3c500fa8-e4d3-a9b3-7919-005ff7e42c45 Address:3c500fa8-e4d3-a9b3-7919-005ff7e42c45} {Suffrage:Voter ID:server-1a378a13-bc14-6333-bf01-60a05f46d57c Address:1a378a13-bc14-6333-bf01-60a05f46d57c} {Suffrage:Voter ID:server-65ab38b4-c201-2589-5555-dcbb8e7c2ea2 Address:65ab38b4-c201-2589-5555-dcbb8e7c2ea2}]" -2026-04-22T10:10:44.559Z [INFO] server-1a378a13-bc14-6333-bf01-60a05f46d57c: entering follower state: follower="Node at 1a378a13-bc14-6333-bf01-60a05f46d57c [Follower]" leader-address= leader-id= -2026-04-22T10:10:44.559Z [INFO] server-65ab38b4-c201-2589-5555-dcbb8e7c2ea2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3c500fa8-e4d3-a9b3-7919-005ff7e42c45 Address:3c500fa8-e4d3-a9b3-7919-005ff7e42c45} {Suffrage:Voter ID:server-1a378a13-bc14-6333-bf01-60a05f46d57c Address:1a378a13-bc14-6333-bf01-60a05f46d57c} {Suffrage:Voter ID:server-65ab38b4-c201-2589-5555-dcbb8e7c2ea2 Address:65ab38b4-c201-2589-5555-dcbb8e7c2ea2}]" -2026-04-22T10:10:44.559Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:44.559Z [INFO] server-65ab38b4-c201-2589-5555-dcbb8e7c2ea2: entering follower state: follower="Node at 65ab38b4-c201-2589-5555-dcbb8e7c2ea2 [Follower]" leader-address= leader-id= -2026-04-22T10:10:45.083Z [WARN] server-65ab38b4-c201-2589-5555-dcbb8e7c2ea2: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:45.083Z [INFO] server-65ab38b4-c201-2589-5555-dcbb8e7c2ea2: entering candidate state: node="Node at 65ab38b4-c201-2589-5555-dcbb8e7c2ea2 [Candidate]" term=2 -2026-04-22T10:10:45.083Z [INFO] server-65ab38b4-c201-2589-5555-dcbb8e7c2ea2: election won: term=2 tally=2 -2026-04-22T10:10:45.083Z [INFO] server-65ab38b4-c201-2589-5555-dcbb8e7c2ea2: entering leader state: leader="Node at 65ab38b4-c201-2589-5555-dcbb8e7c2ea2 [Leader]" -2026-04-22T10:10:45.083Z [INFO] server-65ab38b4-c201-2589-5555-dcbb8e7c2ea2: added peer, starting replication: peer=server-3c500fa8-e4d3-a9b3-7919-005ff7e42c45 -2026-04-22T10:10:45.083Z [INFO] server-65ab38b4-c201-2589-5555-dcbb8e7c2ea2: added peer, starting replication: peer=server-1a378a13-bc14-6333-bf01-60a05f46d57c -2026-04-22T10:10:45.083Z [INFO] server-65ab38b4-c201-2589-5555-dcbb8e7c2ea2: pipelining replication: peer="{Voter server-1a378a13-bc14-6333-bf01-60a05f46d57c 1a378a13-bc14-6333-bf01-60a05f46d57c}" -2026-04-22T10:10:45.083Z [INFO] server-65ab38b4-c201-2589-5555-dcbb8e7c2ea2: pipelining replication: peer="{Voter server-3c500fa8-e4d3-a9b3-7919-005ff7e42c45 3c500fa8-e4d3-a9b3-7919-005ff7e42c45}" -2026-04-22T10:10:46.088Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:45.083346273 +0000 UTC m=+91.107142362 (1 nodes), 523.987539ms from start of poll, 524.293942ms from cluster start. Timeout at 2026-04-22 10:10:46.088355301 +0000 UTC m=+92.112151438, 1.005009076s after stability -2026-04-22T10:10:46.088Z [INFO] creating new snapshot: path=/tmp/raft2779065373/snapshots/2-10013-1776852646088.tmp -2026-04-22T10:10:46.089Z [INFO] server-65ab38b4-c201-2589-5555-dcbb8e7c2ea2: copied to local snapshot: bytes=71 -2026-04-22T10:10:46.089Z [INFO] server-65ab38b4-c201-2589-5555-dcbb8e7c2ea2: snapshot restore progress: id=2-10013-1776852646088 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:46.089Z [INFO] server-65ab38b4-c201-2589-5555-dcbb8e7c2ea2: restored user snapshot: index=10013 -2026-04-22T10:10:46.089Z [ERROR] server-65ab38b4-c201-2589-5555-dcbb8e7c2ea2: failed to get log: index=3 error="log not found" -2026-04-22T10:10:46.089Z [INFO] server-65ab38b4-c201-2589-5555-dcbb8e7c2ea2: aborting pipeline replication: peer="{Voter server-3c500fa8-e4d3-a9b3-7919-005ff7e42c45 3c500fa8-e4d3-a9b3-7919-005ff7e42c45}" -2026-04-22T10:10:46.089Z [ERROR] server-65ab38b4-c201-2589-5555-dcbb8e7c2ea2: failed to get log: index=3 error="log not found" -2026-04-22T10:10:46.089Z [INFO] server-65ab38b4-c201-2589-5555-dcbb8e7c2ea2: aborting pipeline replication: peer="{Voter server-1a378a13-bc14-6333-bf01-60a05f46d57c 1a378a13-bc14-6333-bf01-60a05f46d57c}" -2026-04-22T10:10:46.097Z [ERROR] server-65ab38b4-c201-2589-5555-dcbb8e7c2ea2: failed to get log: index=3 error="log not found" -2026-04-22T10:10:46.097Z [INFO] creating new snapshot: path=/tmp/raft1277979547/snapshots/2-10013-1776852646097.tmp -2026-04-22T10:10:46.098Z [INFO] server-1a378a13-bc14-6333-bf01-60a05f46d57c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:46.098Z [INFO] server-1a378a13-bc14-6333-bf01-60a05f46d57c: copied to local snapshot: bytes=71 -2026-04-22T10:10:46.098Z [INFO] server-1a378a13-bc14-6333-bf01-60a05f46d57c: snapshot restore progress: id=2-10013-1776852646097 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:46.098Z [INFO] server-1a378a13-bc14-6333-bf01-60a05f46d57c: Installed remote snapshot -2026-04-22T10:10:46.098Z [INFO] server-65ab38b4-c201-2589-5555-dcbb8e7c2ea2: pipelining replication: peer="{Voter server-1a378a13-bc14-6333-bf01-60a05f46d57c 1a378a13-bc14-6333-bf01-60a05f46d57c}" -2026-04-22T10:10:46.098Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:10:46.099Z [ERROR] server-65ab38b4-c201-2589-5555-dcbb8e7c2ea2: failed to get log: index=3 error="log not found" -2026-04-22T10:10:46.100Z [INFO] creating new snapshot: path=/tmp/raft3895872293/snapshots/2-10013-1776852646100.tmp -2026-04-22T10:10:46.100Z [INFO] server-3c500fa8-e4d3-a9b3-7919-005ff7e42c45: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:46.100Z [INFO] server-3c500fa8-e4d3-a9b3-7919-005ff7e42c45: copied to local snapshot: bytes=71 -2026-04-22T10:10:46.101Z [INFO] server-3c500fa8-e4d3-a9b3-7919-005ff7e42c45: snapshot restore progress: id=2-10013-1776852646100 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:46.101Z [INFO] server-3c500fa8-e4d3-a9b3-7919-005ff7e42c45: Installed remote snapshot -2026-04-22T10:10:46.101Z [INFO] server-65ab38b4-c201-2589-5555-dcbb8e7c2ea2: pipelining replication: peer="{Voter server-3c500fa8-e4d3-a9b3-7919-005ff7e42c45 3c500fa8-e4d3-a9b3-7919-005ff7e42c45}" -2026-04-22T10:10:47.103Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:10:46.098738324 +0000 UTC m=+92.122534461 (2 nodes), 1.068µs from start of poll, 1.539686041s from cluster start. Timeout at 2026-04-22 10:10:47.103742241 +0000 UTC m=+93.127538347, 1.005003886s after stability -2026-04-22T10:10:47.115Z [INFO] server-443b1416-14d3-8558-0989-055ac897e976: aborting pipeline replication: peer="{Voter server-c5e5714b-11dd-6633-9f56-4e373ebb14c3 c5e5714b-11dd-6633-9f56-4e373ebb14c3}" -2026-04-22T10:10:47.115Z [INFO] server-443b1416-14d3-8558-0989-055ac897e976: aborting pipeline replication: peer="{Voter server-74f249c8-e935-8345-8467-911569067747 74f249c8-e935-8345-8467-911569067747}" +2025-03-20T03:50:59.124Z [INFO] server-638c3501-326a-9a2d-69c4-6182828f1358: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-638c3501-326a-9a2d-69c4-6182828f1358 Address:638c3501-326a-9a2d-69c4-6182828f1358} {Suffrage:Voter ID:server-52969345-71c3-56e9-3062-dd1a63c02d21 Address:52969345-71c3-56e9-3062-dd1a63c02d21} {Suffrage:Voter ID:server-564dec58-95a9-cdc6-750b-f0d00c049bf2 Address:564dec58-95a9-cdc6-750b-f0d00c049bf2}]" +2025-03-20T03:50:59.124Z [INFO] server-52969345-71c3-56e9-3062-dd1a63c02d21: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-638c3501-326a-9a2d-69c4-6182828f1358 Address:638c3501-326a-9a2d-69c4-6182828f1358} {Suffrage:Voter ID:server-52969345-71c3-56e9-3062-dd1a63c02d21 Address:52969345-71c3-56e9-3062-dd1a63c02d21} {Suffrage:Voter ID:server-564dec58-95a9-cdc6-750b-f0d00c049bf2 Address:564dec58-95a9-cdc6-750b-f0d00c049bf2}]" +2025-03-20T03:50:59.125Z [INFO] server-564dec58-95a9-cdc6-750b-f0d00c049bf2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-638c3501-326a-9a2d-69c4-6182828f1358 Address:638c3501-326a-9a2d-69c4-6182828f1358} {Suffrage:Voter ID:server-52969345-71c3-56e9-3062-dd1a63c02d21 Address:52969345-71c3-56e9-3062-dd1a63c02d21} {Suffrage:Voter ID:server-564dec58-95a9-cdc6-750b-f0d00c049bf2 Address:564dec58-95a9-cdc6-750b-f0d00c049bf2}]" +2025-03-20T03:50:59.125Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:50:59.125Z [INFO] server-638c3501-326a-9a2d-69c4-6182828f1358: entering follower state: follower="Node at 638c3501-326a-9a2d-69c4-6182828f1358 [Follower]" leader-address= leader-id= +2025-03-20T03:50:59.125Z [INFO] server-52969345-71c3-56e9-3062-dd1a63c02d21: entering follower state: follower="Node at 52969345-71c3-56e9-3062-dd1a63c02d21 [Follower]" leader-address= leader-id= +2025-03-20T03:50:59.125Z [INFO] server-564dec58-95a9-cdc6-750b-f0d00c049bf2: entering follower state: follower="Node at 564dec58-95a9-cdc6-750b-f0d00c049bf2 [Follower]" leader-address= leader-id= +2025-03-20T03:50:59.851Z [WARN] server-638c3501-326a-9a2d-69c4-6182828f1358: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:50:59.851Z [INFO] server-638c3501-326a-9a2d-69c4-6182828f1358: entering candidate state: node="Node at 638c3501-326a-9a2d-69c4-6182828f1358 [Candidate]" term=2 +2025-03-20T03:50:59.852Z [INFO] server-638c3501-326a-9a2d-69c4-6182828f1358: election won: term=2 tally=2 +2025-03-20T03:50:59.852Z [INFO] server-638c3501-326a-9a2d-69c4-6182828f1358: entering leader state: leader="Node at 638c3501-326a-9a2d-69c4-6182828f1358 [Leader]" +2025-03-20T03:50:59.852Z [INFO] server-638c3501-326a-9a2d-69c4-6182828f1358: added peer, starting replication: peer=server-52969345-71c3-56e9-3062-dd1a63c02d21 +2025-03-20T03:50:59.852Z [INFO] server-638c3501-326a-9a2d-69c4-6182828f1358: added peer, starting replication: peer=server-564dec58-95a9-cdc6-750b-f0d00c049bf2 +2025-03-20T03:50:59.852Z [INFO] server-638c3501-326a-9a2d-69c4-6182828f1358: pipelining replication: peer="{Voter server-564dec58-95a9-cdc6-750b-f0d00c049bf2 564dec58-95a9-cdc6-750b-f0d00c049bf2}" +2025-03-20T03:50:59.852Z [INFO] server-638c3501-326a-9a2d-69c4-6182828f1358: pipelining replication: peer="{Voter server-52969345-71c3-56e9-3062-dd1a63c02d21 52969345-71c3-56e9-3062-dd1a63c02d21}" +2025-03-20T03:51:00.858Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:50:59.852459758 +0000 UTC m=+91.526912848 (1 nodes), 727.338157ms from start of poll, 727.814794ms from cluster start. Timeout at 2025-03-20 03:51:00.857468306 +0000 UTC m=+92.531921339, 1.005008491s after stability +2025-03-20T03:51:00.860Z [INFO] server-638c3501-326a-9a2d-69c4-6182828f1358: starting snapshot up to: index=12 +2025-03-20T03:51:00.860Z [INFO] creating new snapshot: path=/tmp/raft706293288/snapshots/2-12-1742442660860.tmp +2025-03-20T03:51:00.861Z [INFO] server-638c3501-326a-9a2d-69c4-6182828f1358: snapshot complete up to: index=12 +2025-03-20T03:51:00.867Z [INFO] server-35739739-db0b-8b0d-2f65-77894a11de0e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-35739739-db0b-8b0d-2f65-77894a11de0e Address:35739739-db0b-8b0d-2f65-77894a11de0e} {Suffrage:Voter ID:server-f7256e23-4820-22b5-fd9b-88b198f05ba2 Address:f7256e23-4820-22b5-fd9b-88b198f05ba2} {Suffrage:Voter ID:server-269b24b5-640d-a061-d0bf-0fd88c7a34ec Address:269b24b5-640d-a061-d0bf-0fd88c7a34ec}]" +2025-03-20T03:51:00.867Z [INFO] server-f7256e23-4820-22b5-fd9b-88b198f05ba2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-35739739-db0b-8b0d-2f65-77894a11de0e Address:35739739-db0b-8b0d-2f65-77894a11de0e} {Suffrage:Voter ID:server-f7256e23-4820-22b5-fd9b-88b198f05ba2 Address:f7256e23-4820-22b5-fd9b-88b198f05ba2} {Suffrage:Voter ID:server-269b24b5-640d-a061-d0bf-0fd88c7a34ec Address:269b24b5-640d-a061-d0bf-0fd88c7a34ec}]" +2025-03-20T03:51:00.867Z [INFO] server-269b24b5-640d-a061-d0bf-0fd88c7a34ec: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-35739739-db0b-8b0d-2f65-77894a11de0e Address:35739739-db0b-8b0d-2f65-77894a11de0e} {Suffrage:Voter ID:server-f7256e23-4820-22b5-fd9b-88b198f05ba2 Address:f7256e23-4820-22b5-fd9b-88b198f05ba2} {Suffrage:Voter ID:server-269b24b5-640d-a061-d0bf-0fd88c7a34ec Address:269b24b5-640d-a061-d0bf-0fd88c7a34ec}]" +2025-03-20T03:51:00.867Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:00.867Z [INFO] server-35739739-db0b-8b0d-2f65-77894a11de0e: entering follower state: follower="Node at 35739739-db0b-8b0d-2f65-77894a11de0e [Follower]" leader-address= leader-id= +2025-03-20T03:51:00.867Z [INFO] server-f7256e23-4820-22b5-fd9b-88b198f05ba2: entering follower state: follower="Node at f7256e23-4820-22b5-fd9b-88b198f05ba2 [Follower]" leader-address= leader-id= +2025-03-20T03:51:00.867Z [INFO] server-269b24b5-640d-a061-d0bf-0fd88c7a34ec: entering follower state: follower="Node at 269b24b5-640d-a061-d0bf-0fd88c7a34ec [Follower]" leader-address= leader-id= +2025-03-20T03:51:01.475Z [WARN] server-269b24b5-640d-a061-d0bf-0fd88c7a34ec: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:01.475Z [INFO] server-269b24b5-640d-a061-d0bf-0fd88c7a34ec: entering candidate state: node="Node at 269b24b5-640d-a061-d0bf-0fd88c7a34ec [Candidate]" term=2 +2025-03-20T03:51:01.475Z [INFO] server-269b24b5-640d-a061-d0bf-0fd88c7a34ec: election won: term=2 tally=2 +2025-03-20T03:51:01.475Z [INFO] server-269b24b5-640d-a061-d0bf-0fd88c7a34ec: entering leader state: leader="Node at 269b24b5-640d-a061-d0bf-0fd88c7a34ec [Leader]" +2025-03-20T03:51:01.475Z [INFO] server-269b24b5-640d-a061-d0bf-0fd88c7a34ec: added peer, starting replication: peer=server-35739739-db0b-8b0d-2f65-77894a11de0e +2025-03-20T03:51:01.475Z [INFO] server-269b24b5-640d-a061-d0bf-0fd88c7a34ec: added peer, starting replication: peer=server-f7256e23-4820-22b5-fd9b-88b198f05ba2 +2025-03-20T03:51:01.476Z [INFO] server-269b24b5-640d-a061-d0bf-0fd88c7a34ec: pipelining replication: peer="{Voter server-35739739-db0b-8b0d-2f65-77894a11de0e 35739739-db0b-8b0d-2f65-77894a11de0e}" +2025-03-20T03:51:01.479Z [INFO] server-269b24b5-640d-a061-d0bf-0fd88c7a34ec: pipelining replication: peer="{Voter server-f7256e23-4820-22b5-fd9b-88b198f05ba2 f7256e23-4820-22b5-fd9b-88b198f05ba2}" +2025-03-20T03:51:02.483Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:01.477965206 +0000 UTC m=+93.152418231 (1 nodes), 610.599819ms from start of poll, 611.117533ms from cluster start. Timeout at 2025-03-20 03:51:02.483039119 +0000 UTC m=+94.157492171, 1.00507394s after stability +2025-03-20T03:51:02.483Z [INFO] creating new snapshot: path=/tmp/raft3009060404/snapshots/2-10013-1742442662483.tmp +2025-03-20T03:51:02.485Z [INFO] server-269b24b5-640d-a061-d0bf-0fd88c7a34ec: copied to local snapshot: bytes=71 +2025-03-20T03:51:02.485Z [INFO] server-269b24b5-640d-a061-d0bf-0fd88c7a34ec: snapshot restore progress: id=2-10013-1742442662483 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:51:02.485Z [INFO] server-269b24b5-640d-a061-d0bf-0fd88c7a34ec: restored user snapshot: index=10013 +2025-03-20T03:51:02.486Z [ERROR] server-269b24b5-640d-a061-d0bf-0fd88c7a34ec: failed to get log: index=3 error="log not found" +2025-03-20T03:51:02.486Z [INFO] server-269b24b5-640d-a061-d0bf-0fd88c7a34ec: aborting pipeline replication: peer="{Voter server-35739739-db0b-8b0d-2f65-77894a11de0e 35739739-db0b-8b0d-2f65-77894a11de0e}" +2025-03-20T03:51:02.486Z [ERROR] server-269b24b5-640d-a061-d0bf-0fd88c7a34ec: failed to get log: index=3 error="log not found" +2025-03-20T03:51:02.486Z [INFO] server-269b24b5-640d-a061-d0bf-0fd88c7a34ec: aborting pipeline replication: peer="{Voter server-f7256e23-4820-22b5-fd9b-88b198f05ba2 f7256e23-4820-22b5-fd9b-88b198f05ba2}" +2025-03-20T03:51:02.492Z [ERROR] server-269b24b5-640d-a061-d0bf-0fd88c7a34ec: failed to get log: index=3 error="log not found" +2025-03-20T03:51:02.492Z [INFO] creating new snapshot: path=/tmp/raft2408352755/snapshots/2-10013-1742442662492.tmp +2025-03-20T03:51:02.493Z [INFO] server-35739739-db0b-8b0d-2f65-77894a11de0e: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:51:02.494Z [INFO] server-35739739-db0b-8b0d-2f65-77894a11de0e: copied to local snapshot: bytes=71 +2025-03-20T03:51:02.494Z [INFO] server-35739739-db0b-8b0d-2f65-77894a11de0e: snapshot restore progress: id=2-10013-1742442662492 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:51:02.494Z [INFO] server-35739739-db0b-8b0d-2f65-77894a11de0e: Installed remote snapshot +2025-03-20T03:51:02.494Z [INFO] server-269b24b5-640d-a061-d0bf-0fd88c7a34ec: pipelining replication: peer="{Voter server-35739739-db0b-8b0d-2f65-77894a11de0e 35739739-db0b-8b0d-2f65-77894a11de0e}" +2025-03-20T03:51:02.495Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:51:02.496Z [ERROR] server-269b24b5-640d-a061-d0bf-0fd88c7a34ec: failed to get log: index=3 error="log not found" +2025-03-20T03:51:02.500Z [INFO] creating new snapshot: path=/tmp/raft3927522771/snapshots/2-10013-1742442662500.tmp +2025-03-20T03:51:02.500Z [INFO] server-f7256e23-4820-22b5-fd9b-88b198f05ba2: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:51:02.503Z [INFO] server-f7256e23-4820-22b5-fd9b-88b198f05ba2: copied to local snapshot: bytes=71 +2025-03-20T03:51:02.503Z [INFO] server-f7256e23-4820-22b5-fd9b-88b198f05ba2: snapshot restore progress: id=2-10013-1742442662500 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:51:02.503Z [INFO] server-f7256e23-4820-22b5-fd9b-88b198f05ba2: Installed remote snapshot +2025-03-20T03:51:02.504Z [INFO] server-269b24b5-640d-a061-d0bf-0fd88c7a34ec: pipelining replication: peer="{Voter server-f7256e23-4820-22b5-fd9b-88b198f05ba2 f7256e23-4820-22b5-fd9b-88b198f05ba2}" +2025-03-20T03:51:03.505Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:51:02.499497077 +0000 UTC m=+94.173949897 (2 nodes), 2.616µs from start of poll, 1.632649199s from cluster start. Timeout at 2025-03-20 03:51:03.504612098 +0000 UTC m=+95.179065060, 1.005115163s after stability +2025-03-20T03:51:03.519Z [INFO] server-638c3501-326a-9a2d-69c4-6182828f1358: aborting pipeline replication: peer="{Voter server-564dec58-95a9-cdc6-750b-f0d00c049bf2 564dec58-95a9-cdc6-750b-f0d00c049bf2}" +2025-03-20T03:51:03.519Z [INFO] server-638c3501-326a-9a2d-69c4-6182828f1358: aborting pipeline replication: peer="{Voter server-52969345-71c3-56e9-3062-dd1a63c02d21 52969345-71c3-56e9-3062-dd1a63c02d21}" === RUN TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_true -2026-04-22T10:10:47.116Z [INFO] server-f8154d60-a386-e233-83d9-9476b3ab25da: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f8154d60-a386-e233-83d9-9476b3ab25da Address:f8154d60-a386-e233-83d9-9476b3ab25da} {Suffrage:Voter ID:server-1b83c46c-5f2c-7d10-7fd5-24f75fc8daef Address:1b83c46c-5f2c-7d10-7fd5-24f75fc8daef} {Suffrage:Voter ID:server-9916af56-683b-66aa-86e8-0651da84b291 Address:9916af56-683b-66aa-86e8-0651da84b291}]" -2026-04-22T10:10:47.116Z [INFO] server-f8154d60-a386-e233-83d9-9476b3ab25da: entering follower state: follower="Node at f8154d60-a386-e233-83d9-9476b3ab25da [Follower]" leader-address= leader-id= -2026-04-22T10:10:47.116Z [INFO] server-1b83c46c-5f2c-7d10-7fd5-24f75fc8daef: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f8154d60-a386-e233-83d9-9476b3ab25da Address:f8154d60-a386-e233-83d9-9476b3ab25da} {Suffrage:Voter ID:server-1b83c46c-5f2c-7d10-7fd5-24f75fc8daef Address:1b83c46c-5f2c-7d10-7fd5-24f75fc8daef} {Suffrage:Voter ID:server-9916af56-683b-66aa-86e8-0651da84b291 Address:9916af56-683b-66aa-86e8-0651da84b291}]" -2026-04-22T10:10:47.116Z [INFO] server-1b83c46c-5f2c-7d10-7fd5-24f75fc8daef: entering follower state: follower="Node at 1b83c46c-5f2c-7d10-7fd5-24f75fc8daef [Follower]" leader-address= leader-id= -2026-04-22T10:10:47.116Z [INFO] server-9916af56-683b-66aa-86e8-0651da84b291: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f8154d60-a386-e233-83d9-9476b3ab25da Address:f8154d60-a386-e233-83d9-9476b3ab25da} {Suffrage:Voter ID:server-1b83c46c-5f2c-7d10-7fd5-24f75fc8daef Address:1b83c46c-5f2c-7d10-7fd5-24f75fc8daef} {Suffrage:Voter ID:server-9916af56-683b-66aa-86e8-0651da84b291 Address:9916af56-683b-66aa-86e8-0651da84b291}]" -2026-04-22T10:10:47.116Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:47.116Z [INFO] server-9916af56-683b-66aa-86e8-0651da84b291: entering follower state: follower="Node at 9916af56-683b-66aa-86e8-0651da84b291 [Follower]" leader-address= leader-id= -2026-04-22T10:10:47.629Z [WARN] server-f8154d60-a386-e233-83d9-9476b3ab25da: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:47.629Z [INFO] server-f8154d60-a386-e233-83d9-9476b3ab25da: entering candidate state: node="Node at f8154d60-a386-e233-83d9-9476b3ab25da [Candidate]" term=2 -2026-04-22T10:10:47.629Z [INFO] server-f8154d60-a386-e233-83d9-9476b3ab25da: election won: term=2 tally=2 -2026-04-22T10:10:47.629Z [INFO] server-f8154d60-a386-e233-83d9-9476b3ab25da: entering leader state: leader="Node at f8154d60-a386-e233-83d9-9476b3ab25da [Leader]" -2026-04-22T10:10:47.629Z [INFO] server-f8154d60-a386-e233-83d9-9476b3ab25da: added peer, starting replication: peer=server-1b83c46c-5f2c-7d10-7fd5-24f75fc8daef -2026-04-22T10:10:47.629Z [INFO] server-f8154d60-a386-e233-83d9-9476b3ab25da: added peer, starting replication: peer=server-9916af56-683b-66aa-86e8-0651da84b291 -2026-04-22T10:10:47.630Z [INFO] server-f8154d60-a386-e233-83d9-9476b3ab25da: pipelining replication: peer="{Voter server-9916af56-683b-66aa-86e8-0651da84b291 9916af56-683b-66aa-86e8-0651da84b291}" -2026-04-22T10:10:47.630Z [INFO] server-f8154d60-a386-e233-83d9-9476b3ab25da: pipelining replication: peer="{Voter server-1b83c46c-5f2c-7d10-7fd5-24f75fc8daef 1b83c46c-5f2c-7d10-7fd5-24f75fc8daef}" -2026-04-22T10:10:48.635Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:47.629979707 +0000 UTC m=+93.653775795 (1 nodes), 513.463844ms from start of poll, 513.790615ms from cluster start. Timeout at 2026-04-22 10:10:48.634982067 +0000 UTC m=+94.658778150, 1.005002355s after stability -2026-04-22T10:10:48.636Z [INFO] server-f8154d60-a386-e233-83d9-9476b3ab25da: starting snapshot up to: index=12 -2026-04-22T10:10:48.636Z [INFO] creating new snapshot: path=/tmp/raft1258858585/snapshots/2-12-1776852648636.tmp -2026-04-22T10:10:48.636Z [INFO] server-f8154d60-a386-e233-83d9-9476b3ab25da: snapshot complete up to: index=12 -2026-04-22T10:10:48.637Z [INFO] server-b1362830-6f1a-5a35-4371-c4ed5a83e34b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b1362830-6f1a-5a35-4371-c4ed5a83e34b Address:b1362830-6f1a-5a35-4371-c4ed5a83e34b} {Suffrage:Voter ID:server-9887f76b-e10b-919a-2172-a42816a2d600 Address:9887f76b-e10b-919a-2172-a42816a2d600} {Suffrage:Voter ID:server-e6284e92-4356-237c-64a9-d2a644732297 Address:e6284e92-4356-237c-64a9-d2a644732297}]" -2026-04-22T10:10:48.637Z [INFO] server-b1362830-6f1a-5a35-4371-c4ed5a83e34b: entering follower state: follower="Node at b1362830-6f1a-5a35-4371-c4ed5a83e34b [Follower]" leader-address= leader-id= -2026-04-22T10:10:48.637Z [INFO] server-9887f76b-e10b-919a-2172-a42816a2d600: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b1362830-6f1a-5a35-4371-c4ed5a83e34b Address:b1362830-6f1a-5a35-4371-c4ed5a83e34b} {Suffrage:Voter ID:server-9887f76b-e10b-919a-2172-a42816a2d600 Address:9887f76b-e10b-919a-2172-a42816a2d600} {Suffrage:Voter ID:server-e6284e92-4356-237c-64a9-d2a644732297 Address:e6284e92-4356-237c-64a9-d2a644732297}]" -2026-04-22T10:10:48.637Z [INFO] server-9887f76b-e10b-919a-2172-a42816a2d600: entering follower state: follower="Node at 9887f76b-e10b-919a-2172-a42816a2d600 [Follower]" leader-address= leader-id= -2026-04-22T10:10:48.637Z [INFO] server-e6284e92-4356-237c-64a9-d2a644732297: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b1362830-6f1a-5a35-4371-c4ed5a83e34b Address:b1362830-6f1a-5a35-4371-c4ed5a83e34b} {Suffrage:Voter ID:server-9887f76b-e10b-919a-2172-a42816a2d600 Address:9887f76b-e10b-919a-2172-a42816a2d600} {Suffrage:Voter ID:server-e6284e92-4356-237c-64a9-d2a644732297 Address:e6284e92-4356-237c-64a9-d2a644732297}]" -2026-04-22T10:10:48.637Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:48.637Z [INFO] server-e6284e92-4356-237c-64a9-d2a644732297: entering follower state: follower="Node at e6284e92-4356-237c-64a9-d2a644732297 [Follower]" leader-address= leader-id= -2026-04-22T10:10:49.352Z [WARN] server-e6284e92-4356-237c-64a9-d2a644732297: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:49.352Z [INFO] server-e6284e92-4356-237c-64a9-d2a644732297: entering candidate state: node="Node at e6284e92-4356-237c-64a9-d2a644732297 [Candidate]" term=2 -2026-04-22T10:10:49.352Z [INFO] server-e6284e92-4356-237c-64a9-d2a644732297: election won: term=2 tally=2 -2026-04-22T10:10:49.352Z [INFO] server-e6284e92-4356-237c-64a9-d2a644732297: entering leader state: leader="Node at e6284e92-4356-237c-64a9-d2a644732297 [Leader]" -2026-04-22T10:10:49.352Z [INFO] server-e6284e92-4356-237c-64a9-d2a644732297: added peer, starting replication: peer=server-b1362830-6f1a-5a35-4371-c4ed5a83e34b -2026-04-22T10:10:49.352Z [INFO] server-e6284e92-4356-237c-64a9-d2a644732297: added peer, starting replication: peer=server-9887f76b-e10b-919a-2172-a42816a2d600 -2026-04-22T10:10:49.352Z [INFO] server-e6284e92-4356-237c-64a9-d2a644732297: pipelining replication: peer="{Voter server-9887f76b-e10b-919a-2172-a42816a2d600 9887f76b-e10b-919a-2172-a42816a2d600}" -2026-04-22T10:10:49.352Z [INFO] server-e6284e92-4356-237c-64a9-d2a644732297: pipelining replication: peer="{Voter server-b1362830-6f1a-5a35-4371-c4ed5a83e34b b1362830-6f1a-5a35-4371-c4ed5a83e34b}" -2026-04-22T10:10:50.358Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:49.352636971 +0000 UTC m=+95.376433121 (1 nodes), 714.8054ms from start of poll, 715.163109ms from cluster start. Timeout at 2026-04-22 10:10:50.35764784 +0000 UTC m=+96.381443974, 1.005010853s after stability -2026-04-22T10:10:50.358Z [INFO] creating new snapshot: path=/tmp/raft2240097003/snapshots/2-10013-1776852650358.tmp -2026-04-22T10:10:50.358Z [INFO] server-e6284e92-4356-237c-64a9-d2a644732297: copied to local snapshot: bytes=71 -2026-04-22T10:10:50.359Z [INFO] server-e6284e92-4356-237c-64a9-d2a644732297: snapshot restore progress: id=2-10013-1776852650358 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:50.359Z [INFO] server-e6284e92-4356-237c-64a9-d2a644732297: removing all old logs from log store -2026-04-22T10:10:50.359Z [INFO] server-e6284e92-4356-237c-64a9-d2a644732297: compacting logs: from=1 to=2 -2026-04-22T10:10:50.359Z [INFO] server-e6284e92-4356-237c-64a9-d2a644732297: restored user snapshot: index=10013 -2026-04-22T10:10:50.359Z [ERROR] server-e6284e92-4356-237c-64a9-d2a644732297: failed to get log: index=2 error="log not found" -2026-04-22T10:10:50.359Z [INFO] server-e6284e92-4356-237c-64a9-d2a644732297: aborting pipeline replication: peer="{Voter server-b1362830-6f1a-5a35-4371-c4ed5a83e34b b1362830-6f1a-5a35-4371-c4ed5a83e34b}" -2026-04-22T10:10:50.359Z [ERROR] server-e6284e92-4356-237c-64a9-d2a644732297: failed to get log: index=2 error="log not found" -2026-04-22T10:10:50.359Z [INFO] server-e6284e92-4356-237c-64a9-d2a644732297: aborting pipeline replication: peer="{Voter server-9887f76b-e10b-919a-2172-a42816a2d600 9887f76b-e10b-919a-2172-a42816a2d600}" -2026-04-22T10:10:50.365Z [ERROR] server-e6284e92-4356-237c-64a9-d2a644732297: failed to get log: index=2 error="log not found" -2026-04-22T10:10:50.366Z [INFO] creating new snapshot: path=/tmp/raft1510242677/snapshots/2-10013-1776852650366.tmp -2026-04-22T10:10:50.366Z [INFO] server-b1362830-6f1a-5a35-4371-c4ed5a83e34b: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:50.366Z [INFO] server-b1362830-6f1a-5a35-4371-c4ed5a83e34b: copied to local snapshot: bytes=71 -2026-04-22T10:10:50.367Z [INFO] server-b1362830-6f1a-5a35-4371-c4ed5a83e34b: snapshot restore progress: id=2-10013-1776852650366 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:50.367Z [INFO] server-b1362830-6f1a-5a35-4371-c4ed5a83e34b: removing all old logs from log store -2026-04-22T10:10:50.367Z [INFO] server-b1362830-6f1a-5a35-4371-c4ed5a83e34b: compacting logs: from=1 to=2 -2026-04-22T10:10:50.367Z [INFO] server-b1362830-6f1a-5a35-4371-c4ed5a83e34b: Installed remote snapshot -2026-04-22T10:10:50.367Z [INFO] server-e6284e92-4356-237c-64a9-d2a644732297: pipelining replication: peer="{Voter server-b1362830-6f1a-5a35-4371-c4ed5a83e34b b1362830-6f1a-5a35-4371-c4ed5a83e34b}" -2026-04-22T10:10:50.367Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:10:50.369Z [ERROR] server-e6284e92-4356-237c-64a9-d2a644732297: failed to get log: index=2 error="log not found" -2026-04-22T10:10:50.369Z [INFO] creating new snapshot: path=/tmp/raft4061300982/snapshots/2-10013-1776852650369.tmp -2026-04-22T10:10:50.370Z [INFO] server-9887f76b-e10b-919a-2172-a42816a2d600: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:50.370Z [INFO] server-9887f76b-e10b-919a-2172-a42816a2d600: copied to local snapshot: bytes=71 -2026-04-22T10:10:50.370Z [INFO] server-9887f76b-e10b-919a-2172-a42816a2d600: snapshot restore progress: id=2-10013-1776852650369 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-04-22T10:10:50.370Z [INFO] server-9887f76b-e10b-919a-2172-a42816a2d600: removing all old logs from log store -2026-04-22T10:10:50.370Z [INFO] server-9887f76b-e10b-919a-2172-a42816a2d600: compacting logs: from=1 to=2 -2026-04-22T10:10:50.370Z [INFO] server-9887f76b-e10b-919a-2172-a42816a2d600: Installed remote snapshot -2026-04-22T10:10:50.370Z [INFO] server-e6284e92-4356-237c-64a9-d2a644732297: pipelining replication: peer="{Voter server-9887f76b-e10b-919a-2172-a42816a2d600 9887f76b-e10b-919a-2172-a42816a2d600}" -2026-04-22T10:10:51.373Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:10:50.367839391 +0000 UTC m=+96.391635479 (2 nodes), 619ns from start of poll, 1.730365467s from cluster start. Timeout at 2026-04-22 10:10:51.37286229 +0000 UTC m=+97.396658474, 1.005022995s after stability -2026-04-22T10:10:51.385Z [INFO] server-f8154d60-a386-e233-83d9-9476b3ab25da: aborting pipeline replication: peer="{Voter server-9916af56-683b-66aa-86e8-0651da84b291 9916af56-683b-66aa-86e8-0651da84b291}" -2026-04-22T10:10:51.385Z [INFO] server-f8154d60-a386-e233-83d9-9476b3ab25da: aborting pipeline replication: peer="{Voter server-1b83c46c-5f2c-7d10-7fd5-24f75fc8daef 1b83c46c-5f2c-7d10-7fd5-24f75fc8daef}" ---- PASS: TestRaft_UserRestore (83.44s) - --- PASS: TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_false (2.69s) - --- PASS: TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_false (3.11s) - --- PASS: TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_true (4.60s) - --- PASS: TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_true (4.32s) - --- PASS: TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_false (2.55s) - --- PASS: TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_false (2.56s) - --- PASS: TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_true (4.33s) - --- PASS: TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_true (4.22s) - --- PASS: TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_false (2.56s) - --- PASS: TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_false (2.61s) - --- PASS: TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_true (4.49s) - --- PASS: TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_true (4.31s) +2025-03-20T03:51:03.522Z [INFO] server-9fc05167-09f7-86e9-39cc-f95008e7359c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9fc05167-09f7-86e9-39cc-f95008e7359c Address:9fc05167-09f7-86e9-39cc-f95008e7359c} {Suffrage:Voter ID:server-66140f42-c071-3c3d-0a50-d307f5e697cb Address:66140f42-c071-3c3d-0a50-d307f5e697cb} {Suffrage:Voter ID:server-2ef6731e-fdc9-70de-114b-da6c9bc3b00f Address:2ef6731e-fdc9-70de-114b-da6c9bc3b00f}]" +2025-03-20T03:51:03.523Z [INFO] server-66140f42-c071-3c3d-0a50-d307f5e697cb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9fc05167-09f7-86e9-39cc-f95008e7359c Address:9fc05167-09f7-86e9-39cc-f95008e7359c} {Suffrage:Voter ID:server-66140f42-c071-3c3d-0a50-d307f5e697cb Address:66140f42-c071-3c3d-0a50-d307f5e697cb} {Suffrage:Voter ID:server-2ef6731e-fdc9-70de-114b-da6c9bc3b00f Address:2ef6731e-fdc9-70de-114b-da6c9bc3b00f}]" +2025-03-20T03:51:03.523Z [INFO] server-2ef6731e-fdc9-70de-114b-da6c9bc3b00f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9fc05167-09f7-86e9-39cc-f95008e7359c Address:9fc05167-09f7-86e9-39cc-f95008e7359c} {Suffrage:Voter ID:server-66140f42-c071-3c3d-0a50-d307f5e697cb Address:66140f42-c071-3c3d-0a50-d307f5e697cb} {Suffrage:Voter ID:server-2ef6731e-fdc9-70de-114b-da6c9bc3b00f Address:2ef6731e-fdc9-70de-114b-da6c9bc3b00f}]" +2025-03-20T03:51:03.523Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:03.523Z [INFO] server-9fc05167-09f7-86e9-39cc-f95008e7359c: entering follower state: follower="Node at 9fc05167-09f7-86e9-39cc-f95008e7359c [Follower]" leader-address= leader-id= +2025-03-20T03:51:03.523Z [INFO] server-66140f42-c071-3c3d-0a50-d307f5e697cb: entering follower state: follower="Node at 66140f42-c071-3c3d-0a50-d307f5e697cb [Follower]" leader-address= leader-id= +2025-03-20T03:51:03.523Z [INFO] server-2ef6731e-fdc9-70de-114b-da6c9bc3b00f: entering follower state: follower="Node at 2ef6731e-fdc9-70de-114b-da6c9bc3b00f [Follower]" leader-address= leader-id= +2025-03-20T03:51:04.158Z [WARN] server-66140f42-c071-3c3d-0a50-d307f5e697cb: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:04.158Z [INFO] server-66140f42-c071-3c3d-0a50-d307f5e697cb: entering candidate state: node="Node at 66140f42-c071-3c3d-0a50-d307f5e697cb [Candidate]" term=2 +2025-03-20T03:51:04.158Z [INFO] server-66140f42-c071-3c3d-0a50-d307f5e697cb: election won: term=2 tally=2 +2025-03-20T03:51:04.158Z [INFO] server-66140f42-c071-3c3d-0a50-d307f5e697cb: entering leader state: leader="Node at 66140f42-c071-3c3d-0a50-d307f5e697cb [Leader]" +2025-03-20T03:51:04.158Z [INFO] server-66140f42-c071-3c3d-0a50-d307f5e697cb: added peer, starting replication: peer=server-9fc05167-09f7-86e9-39cc-f95008e7359c +2025-03-20T03:51:04.159Z [INFO] server-66140f42-c071-3c3d-0a50-d307f5e697cb: added peer, starting replication: peer=server-2ef6731e-fdc9-70de-114b-da6c9bc3b00f +2025-03-20T03:51:04.159Z [INFO] server-66140f42-c071-3c3d-0a50-d307f5e697cb: pipelining replication: peer="{Voter server-2ef6731e-fdc9-70de-114b-da6c9bc3b00f 2ef6731e-fdc9-70de-114b-da6c9bc3b00f}" +2025-03-20T03:51:04.159Z [INFO] server-66140f42-c071-3c3d-0a50-d307f5e697cb: pipelining replication: peer="{Voter server-9fc05167-09f7-86e9-39cc-f95008e7359c 9fc05167-09f7-86e9-39cc-f95008e7359c}" +2025-03-20T03:51:05.165Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:04.159631963 +0000 UTC m=+95.834085015 (1 nodes), 636.358035ms from start of poll, 636.873897ms from cluster start. Timeout at 2025-03-20 03:51:05.164636884 +0000 UTC m=+96.839089714, 1.005004699s after stability +2025-03-20T03:51:05.167Z [INFO] server-66140f42-c071-3c3d-0a50-d307f5e697cb: starting snapshot up to: index=12 +2025-03-20T03:51:05.167Z [INFO] creating new snapshot: path=/tmp/raft2669047005/snapshots/2-12-1742442665167.tmp +2025-03-20T03:51:05.168Z [INFO] server-66140f42-c071-3c3d-0a50-d307f5e697cb: snapshot complete up to: index=12 +2025-03-20T03:51:05.172Z [INFO] server-55d0e794-2c7f-d637-863f-3dd8e5f546dd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-55d0e794-2c7f-d637-863f-3dd8e5f546dd Address:55d0e794-2c7f-d637-863f-3dd8e5f546dd} {Suffrage:Voter ID:server-9933ab53-1120-1d4d-7df8-3651703c69e5 Address:9933ab53-1120-1d4d-7df8-3651703c69e5} {Suffrage:Voter ID:server-bce8afbb-7d8d-8d76-c7e4-92ac04c81828 Address:bce8afbb-7d8d-8d76-c7e4-92ac04c81828}]" +2025-03-20T03:51:05.172Z [INFO] server-9933ab53-1120-1d4d-7df8-3651703c69e5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-55d0e794-2c7f-d637-863f-3dd8e5f546dd Address:55d0e794-2c7f-d637-863f-3dd8e5f546dd} {Suffrage:Voter ID:server-9933ab53-1120-1d4d-7df8-3651703c69e5 Address:9933ab53-1120-1d4d-7df8-3651703c69e5} {Suffrage:Voter ID:server-bce8afbb-7d8d-8d76-c7e4-92ac04c81828 Address:bce8afbb-7d8d-8d76-c7e4-92ac04c81828}]" +2025-03-20T03:51:05.172Z [INFO] server-bce8afbb-7d8d-8d76-c7e4-92ac04c81828: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-55d0e794-2c7f-d637-863f-3dd8e5f546dd Address:55d0e794-2c7f-d637-863f-3dd8e5f546dd} {Suffrage:Voter ID:server-9933ab53-1120-1d4d-7df8-3651703c69e5 Address:9933ab53-1120-1d4d-7df8-3651703c69e5} {Suffrage:Voter ID:server-bce8afbb-7d8d-8d76-c7e4-92ac04c81828 Address:bce8afbb-7d8d-8d76-c7e4-92ac04c81828}]" +2025-03-20T03:51:05.172Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:05.172Z [INFO] server-55d0e794-2c7f-d637-863f-3dd8e5f546dd: entering follower state: follower="Node at 55d0e794-2c7f-d637-863f-3dd8e5f546dd [Follower]" leader-address= leader-id= +2025-03-20T03:51:05.173Z [INFO] server-9933ab53-1120-1d4d-7df8-3651703c69e5: entering follower state: follower="Node at 9933ab53-1120-1d4d-7df8-3651703c69e5 [Follower]" leader-address= leader-id= +2025-03-20T03:51:05.173Z [INFO] server-bce8afbb-7d8d-8d76-c7e4-92ac04c81828: entering follower state: follower="Node at bce8afbb-7d8d-8d76-c7e4-92ac04c81828 [Follower]" leader-address= leader-id= +2025-03-20T03:51:05.831Z [WARN] server-55d0e794-2c7f-d637-863f-3dd8e5f546dd: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:05.831Z [INFO] server-55d0e794-2c7f-d637-863f-3dd8e5f546dd: entering candidate state: node="Node at 55d0e794-2c7f-d637-863f-3dd8e5f546dd [Candidate]" term=2 +2025-03-20T03:51:05.831Z [INFO] server-55d0e794-2c7f-d637-863f-3dd8e5f546dd: election won: term=2 tally=2 +2025-03-20T03:51:05.831Z [INFO] server-55d0e794-2c7f-d637-863f-3dd8e5f546dd: entering leader state: leader="Node at 55d0e794-2c7f-d637-863f-3dd8e5f546dd [Leader]" +2025-03-20T03:51:05.831Z [INFO] server-55d0e794-2c7f-d637-863f-3dd8e5f546dd: added peer, starting replication: peer=server-9933ab53-1120-1d4d-7df8-3651703c69e5 +2025-03-20T03:51:05.831Z [INFO] server-55d0e794-2c7f-d637-863f-3dd8e5f546dd: added peer, starting replication: peer=server-bce8afbb-7d8d-8d76-c7e4-92ac04c81828 +2025-03-20T03:51:05.832Z [INFO] server-55d0e794-2c7f-d637-863f-3dd8e5f546dd: pipelining replication: peer="{Voter server-bce8afbb-7d8d-8d76-c7e4-92ac04c81828 bce8afbb-7d8d-8d76-c7e4-92ac04c81828}" +2025-03-20T03:51:05.833Z [INFO] server-55d0e794-2c7f-d637-863f-3dd8e5f546dd: pipelining replication: peer="{Voter server-9933ab53-1120-1d4d-7df8-3651703c69e5 9933ab53-1120-1d4d-7df8-3651703c69e5}" +2025-03-20T03:51:06.839Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:05.83301724 +0000 UTC m=+97.507470075 (1 nodes), 660.099792ms from start of poll, 661.577011ms from cluster start. Timeout at 2025-03-20 03:51:06.838063675 +0000 UTC m=+98.512516762, 1.005046687s after stability +2025-03-20T03:51:06.839Z [INFO] creating new snapshot: path=/tmp/raft1033518525/snapshots/2-10013-1742442666839.tmp +2025-03-20T03:51:06.841Z [INFO] server-55d0e794-2c7f-d637-863f-3dd8e5f546dd: copied to local snapshot: bytes=71 +2025-03-20T03:51:06.841Z [INFO] server-55d0e794-2c7f-d637-863f-3dd8e5f546dd: snapshot restore progress: id=2-10013-1742442666839 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:51:06.841Z [INFO] server-55d0e794-2c7f-d637-863f-3dd8e5f546dd: removing all old logs from log store +2025-03-20T03:51:06.841Z [INFO] server-55d0e794-2c7f-d637-863f-3dd8e5f546dd: compacting logs: from=1 to=2 +2025-03-20T03:51:06.841Z [INFO] server-55d0e794-2c7f-d637-863f-3dd8e5f546dd: restored user snapshot: index=10013 +2025-03-20T03:51:06.841Z [ERROR] server-55d0e794-2c7f-d637-863f-3dd8e5f546dd: failed to get log: index=2 error="log not found" +2025-03-20T03:51:06.841Z [INFO] server-55d0e794-2c7f-d637-863f-3dd8e5f546dd: aborting pipeline replication: peer="{Voter server-bce8afbb-7d8d-8d76-c7e4-92ac04c81828 bce8afbb-7d8d-8d76-c7e4-92ac04c81828}" +2025-03-20T03:51:06.841Z [ERROR] server-55d0e794-2c7f-d637-863f-3dd8e5f546dd: failed to get log: index=2 error="log not found" +2025-03-20T03:51:06.841Z [INFO] server-55d0e794-2c7f-d637-863f-3dd8e5f546dd: aborting pipeline replication: peer="{Voter server-9933ab53-1120-1d4d-7df8-3651703c69e5 9933ab53-1120-1d4d-7df8-3651703c69e5}" +2025-03-20T03:51:06.848Z [ERROR] server-55d0e794-2c7f-d637-863f-3dd8e5f546dd: failed to get log: index=2 error="log not found" +2025-03-20T03:51:06.848Z [INFO] creating new snapshot: path=/tmp/raft3623533049/snapshots/2-10013-1742442666848.tmp +2025-03-20T03:51:06.849Z [ERROR] server-55d0e794-2c7f-d637-863f-3dd8e5f546dd: failed to get log: index=2 error="log not found" +2025-03-20T03:51:06.853Z [INFO] server-bce8afbb-7d8d-8d76-c7e4-92ac04c81828: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:51:06.856Z [INFO] server-bce8afbb-7d8d-8d76-c7e4-92ac04c81828: copied to local snapshot: bytes=71 +2025-03-20T03:51:06.859Z [INFO] server-bce8afbb-7d8d-8d76-c7e4-92ac04c81828: snapshot restore progress: id=2-10013-1742442666848 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:51:06.859Z [INFO] server-bce8afbb-7d8d-8d76-c7e4-92ac04c81828: removing all old logs from log store +2025-03-20T03:51:06.859Z [INFO] server-bce8afbb-7d8d-8d76-c7e4-92ac04c81828: compacting logs: from=1 to=2 +2025-03-20T03:51:06.859Z [INFO] server-bce8afbb-7d8d-8d76-c7e4-92ac04c81828: Installed remote snapshot +2025-03-20T03:51:06.859Z [INFO] server-55d0e794-2c7f-d637-863f-3dd8e5f546dd: pipelining replication: peer="{Voter server-bce8afbb-7d8d-8d76-c7e4-92ac04c81828 bce8afbb-7d8d-8d76-c7e4-92ac04c81828}" +2025-03-20T03:51:06.859Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:51:06.878Z [INFO] creating new snapshot: path=/tmp/raft2877350572/snapshots/2-10013-1742442666878.tmp +2025-03-20T03:51:06.878Z [INFO] server-9933ab53-1120-1d4d-7df8-3651703c69e5: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-03-20T03:51:06.880Z [INFO] server-9933ab53-1120-1d4d-7df8-3651703c69e5: copied to local snapshot: bytes=71 +2025-03-20T03:51:06.885Z [INFO] server-9933ab53-1120-1d4d-7df8-3651703c69e5: snapshot restore progress: id=2-10013-1742442666878 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-03-20T03:51:06.885Z [INFO] server-9933ab53-1120-1d4d-7df8-3651703c69e5: removing all old logs from log store +2025-03-20T03:51:06.885Z [INFO] server-9933ab53-1120-1d4d-7df8-3651703c69e5: compacting logs: from=1 to=2 +2025-03-20T03:51:06.885Z [INFO] server-9933ab53-1120-1d4d-7df8-3651703c69e5: Installed remote snapshot +2025-03-20T03:51:06.886Z [INFO] server-55d0e794-2c7f-d637-863f-3dd8e5f546dd: pipelining replication: peer="{Voter server-9933ab53-1120-1d4d-7df8-3651703c69e5 9933ab53-1120-1d4d-7df8-3651703c69e5}" +2025-03-20T03:51:07.868Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:51:06.860232931 +0000 UTC m=+98.534685905 (2 nodes), 2.228µs from start of poll, 1.688792841s from cluster start. Timeout at 2025-03-20 03:51:07.865353972 +0000 UTC m=+99.539806832, 1.005120927s after stability +2025-03-20T03:51:07.882Z [INFO] server-66140f42-c071-3c3d-0a50-d307f5e697cb: aborting pipeline replication: peer="{Voter server-2ef6731e-fdc9-70de-114b-da6c9bc3b00f 2ef6731e-fdc9-70de-114b-da6c9bc3b00f}" +2025-03-20T03:51:07.883Z [INFO] server-66140f42-c071-3c3d-0a50-d307f5e697cb: aborting pipeline replication: peer="{Voter server-9fc05167-09f7-86e9-39cc-f95008e7359c 9fc05167-09f7-86e9-39cc-f95008e7359c}" +--- PASS: TestRaft_UserRestore (84.82s) + --- PASS: TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_false (2.72s) + --- PASS: TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_false (3.01s) + --- PASS: TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_true (4.50s) + --- PASS: TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_true (4.20s) + --- PASS: TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_false (2.66s) + --- PASS: TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_false (2.59s) + --- PASS: TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_true (4.19s) + --- PASS: TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_true (4.42s) + --- PASS: TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_false (2.65s) + --- PASS: TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_false (2.67s) + --- PASS: TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_true (4.08s) + --- PASS: TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_true (4.80s) --- PASS: TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_false (2.63s) - --- PASS: TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_false (2.58s) - --- PASS: TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_true (4.22s) - --- PASS: TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_true (4.14s) - --- PASS: TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_false (2.54s) - --- PASS: TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_false (2.64s) - --- PASS: TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_true (4.21s) - --- PASS: TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_true (4.25s) - --- PASS: TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_false (2.69s) - --- PASS: TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_false (2.79s) - --- PASS: TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_true (4.14s) - --- PASS: TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_true (4.27s) + --- PASS: TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_false (3.45s) + --- PASS: TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_true (4.23s) + --- PASS: TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_true (4.11s) + --- PASS: TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_false (2.57s) + --- PASS: TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_false (2.58s) + --- PASS: TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_true (4.32s) + --- PASS: TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_true (4.21s) + --- PASS: TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_false (2.66s) + --- PASS: TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_false (2.80s) + --- PASS: TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_true (4.40s) + --- PASS: TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_true (4.36s) === RUN TestRaft_SendSnapshotFollower -2026-04-22T10:10:51.388Z [INFO] server-878ba533-b1f1-6e69-d77a-be501715c960: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-878ba533-b1f1-6e69-d77a-be501715c960 Address:878ba533-b1f1-6e69-d77a-be501715c960} {Suffrage:Voter ID:server-c9fba947-d710-cff4-09c9-eb1a8f0178c7 Address:c9fba947-d710-cff4-09c9-eb1a8f0178c7} {Suffrage:Voter ID:server-0dfc14f3-5266-5ec1-4865-bdb2041a37fb Address:0dfc14f3-5266-5ec1-4865-bdb2041a37fb}]" -2026-04-22T10:10:51.388Z [INFO] server-878ba533-b1f1-6e69-d77a-be501715c960: entering follower state: follower="Node at 878ba533-b1f1-6e69-d77a-be501715c960 [Follower]" leader-address= leader-id= -2026-04-22T10:10:51.388Z [INFO] server-c9fba947-d710-cff4-09c9-eb1a8f0178c7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-878ba533-b1f1-6e69-d77a-be501715c960 Address:878ba533-b1f1-6e69-d77a-be501715c960} {Suffrage:Voter ID:server-c9fba947-d710-cff4-09c9-eb1a8f0178c7 Address:c9fba947-d710-cff4-09c9-eb1a8f0178c7} {Suffrage:Voter ID:server-0dfc14f3-5266-5ec1-4865-bdb2041a37fb Address:0dfc14f3-5266-5ec1-4865-bdb2041a37fb}]" -2026-04-22T10:10:51.388Z [INFO] server-c9fba947-d710-cff4-09c9-eb1a8f0178c7: entering follower state: follower="Node at c9fba947-d710-cff4-09c9-eb1a8f0178c7 [Follower]" leader-address= leader-id= -2026-04-22T10:10:51.388Z [INFO] server-0dfc14f3-5266-5ec1-4865-bdb2041a37fb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-878ba533-b1f1-6e69-d77a-be501715c960 Address:878ba533-b1f1-6e69-d77a-be501715c960} {Suffrage:Voter ID:server-c9fba947-d710-cff4-09c9-eb1a8f0178c7 Address:c9fba947-d710-cff4-09c9-eb1a8f0178c7} {Suffrage:Voter ID:server-0dfc14f3-5266-5ec1-4865-bdb2041a37fb Address:0dfc14f3-5266-5ec1-4865-bdb2041a37fb}]" -2026-04-22T10:10:51.388Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:10:51.388Z [INFO] server-0dfc14f3-5266-5ec1-4865-bdb2041a37fb: entering follower state: follower="Node at 0dfc14f3-5266-5ec1-4865-bdb2041a37fb [Follower]" leader-address= leader-id= -2026-04-22T10:10:51.457Z [WARN] server-c9fba947-d710-cff4-09c9-eb1a8f0178c7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:51.457Z [INFO] server-c9fba947-d710-cff4-09c9-eb1a8f0178c7: entering candidate state: node="Node at c9fba947-d710-cff4-09c9-eb1a8f0178c7 [Candidate]" term=2 -2026-04-22T10:10:51.457Z [INFO] server-c9fba947-d710-cff4-09c9-eb1a8f0178c7: election won: term=2 tally=2 -2026-04-22T10:10:51.457Z [INFO] server-c9fba947-d710-cff4-09c9-eb1a8f0178c7: entering leader state: leader="Node at c9fba947-d710-cff4-09c9-eb1a8f0178c7 [Leader]" -2026-04-22T10:10:51.457Z [INFO] server-c9fba947-d710-cff4-09c9-eb1a8f0178c7: added peer, starting replication: peer=server-878ba533-b1f1-6e69-d77a-be501715c960 -2026-04-22T10:10:51.457Z [INFO] server-c9fba947-d710-cff4-09c9-eb1a8f0178c7: added peer, starting replication: peer=server-0dfc14f3-5266-5ec1-4865-bdb2041a37fb -2026-04-22T10:10:51.457Z [INFO] server-c9fba947-d710-cff4-09c9-eb1a8f0178c7: pipelining replication: peer="{Voter server-0dfc14f3-5266-5ec1-4865-bdb2041a37fb 0dfc14f3-5266-5ec1-4865-bdb2041a37fb}" -2026-04-22T10:10:51.457Z [INFO] server-c9fba947-d710-cff4-09c9-eb1a8f0178c7: pipelining replication: peer="{Voter server-878ba533-b1f1-6e69-d77a-be501715c960 878ba533-b1f1-6e69-d77a-be501715c960}" -2026-04-22T10:10:51.563Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:10:51.457825045 +0000 UTC m=+97.481621138 (2 nodes), 69.41838ms from start of poll, 69.745681ms from cluster start. Timeout at 2026-04-22 10:10:51.562831654 +0000 UTC m=+97.586627791, 105.006653ms after stability -2026-04-22T10:10:51.563Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:51.668Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:51.563286703 +0000 UTC m=+97.587082880 (1 nodes), 1.38µs from start of poll, 175.207423ms from cluster start. Timeout at 2026-04-22 10:10:51.668298574 +0000 UTC m=+97.692094734, 105.011854ms after stability -2026-04-22T10:10:51.668Z [ERROR] server-c9fba947-d710-cff4-09c9-eb1a8f0178c7: failed to pipeline appendEntries: peer="{Voter server-878ba533-b1f1-6e69-d77a-be501715c960 878ba533-b1f1-6e69-d77a-be501715c960}" error="append pipeline closed" -2026-04-22T10:10:51.668Z [INFO] server-c9fba947-d710-cff4-09c9-eb1a8f0178c7: aborting pipeline replication: peer="{Voter server-878ba533-b1f1-6e69-d77a-be501715c960 878ba533-b1f1-6e69-d77a-be501715c960}" -2026-04-22T10:10:51.668Z [ERROR] server-c9fba947-d710-cff4-09c9-eb1a8f0178c7: failed to appendEntries to: peer="{Voter server-878ba533-b1f1-6e69-d77a-be501715c960 878ba533-b1f1-6e69-d77a-be501715c960}" error="failed to connect to peer: 878ba533-b1f1-6e69-d77a-be501715c960" +2025-03-20T03:51:07.887Z [INFO] server-ee531a2e-628d-9fb0-0c4d-e6863d19d6ec: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ee531a2e-628d-9fb0-0c4d-e6863d19d6ec Address:ee531a2e-628d-9fb0-0c4d-e6863d19d6ec} {Suffrage:Voter ID:server-2165f328-d003-d713-6d28-e04f6b518276 Address:2165f328-d003-d713-6d28-e04f6b518276} {Suffrage:Voter ID:server-a1f0ef21-6866-8bf4-0ec1-690dca424179 Address:a1f0ef21-6866-8bf4-0ec1-690dca424179}]" +2025-03-20T03:51:07.887Z [INFO] server-2165f328-d003-d713-6d28-e04f6b518276: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ee531a2e-628d-9fb0-0c4d-e6863d19d6ec Address:ee531a2e-628d-9fb0-0c4d-e6863d19d6ec} {Suffrage:Voter ID:server-2165f328-d003-d713-6d28-e04f6b518276 Address:2165f328-d003-d713-6d28-e04f6b518276} {Suffrage:Voter ID:server-a1f0ef21-6866-8bf4-0ec1-690dca424179 Address:a1f0ef21-6866-8bf4-0ec1-690dca424179}]" +2025-03-20T03:51:07.887Z [INFO] server-a1f0ef21-6866-8bf4-0ec1-690dca424179: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ee531a2e-628d-9fb0-0c4d-e6863d19d6ec Address:ee531a2e-628d-9fb0-0c4d-e6863d19d6ec} {Suffrage:Voter ID:server-2165f328-d003-d713-6d28-e04f6b518276 Address:2165f328-d003-d713-6d28-e04f6b518276} {Suffrage:Voter ID:server-a1f0ef21-6866-8bf4-0ec1-690dca424179 Address:a1f0ef21-6866-8bf4-0ec1-690dca424179}]" +2025-03-20T03:51:07.887Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:51:07.887Z [INFO] server-ee531a2e-628d-9fb0-0c4d-e6863d19d6ec: entering follower state: follower="Node at ee531a2e-628d-9fb0-0c4d-e6863d19d6ec [Follower]" leader-address= leader-id= +2025-03-20T03:51:07.887Z [INFO] server-2165f328-d003-d713-6d28-e04f6b518276: entering follower state: follower="Node at 2165f328-d003-d713-6d28-e04f6b518276 [Follower]" leader-address= leader-id= +2025-03-20T03:51:07.887Z [INFO] server-a1f0ef21-6866-8bf4-0ec1-690dca424179: entering follower state: follower="Node at a1f0ef21-6866-8bf4-0ec1-690dca424179 [Follower]" leader-address= leader-id= +2025-03-20T03:51:07.942Z [WARN] server-ee531a2e-628d-9fb0-0c4d-e6863d19d6ec: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:07.942Z [INFO] server-ee531a2e-628d-9fb0-0c4d-e6863d19d6ec: entering candidate state: node="Node at ee531a2e-628d-9fb0-0c4d-e6863d19d6ec [Candidate]" term=2 +2025-03-20T03:51:07.942Z [INFO] server-ee531a2e-628d-9fb0-0c4d-e6863d19d6ec: election won: term=2 tally=2 +2025-03-20T03:51:07.942Z [INFO] server-ee531a2e-628d-9fb0-0c4d-e6863d19d6ec: entering leader state: leader="Node at ee531a2e-628d-9fb0-0c4d-e6863d19d6ec [Leader]" +2025-03-20T03:51:07.942Z [INFO] server-ee531a2e-628d-9fb0-0c4d-e6863d19d6ec: added peer, starting replication: peer=server-2165f328-d003-d713-6d28-e04f6b518276 +2025-03-20T03:51:07.942Z [INFO] server-ee531a2e-628d-9fb0-0c4d-e6863d19d6ec: added peer, starting replication: peer=server-a1f0ef21-6866-8bf4-0ec1-690dca424179 +2025-03-20T03:51:07.945Z [INFO] server-ee531a2e-628d-9fb0-0c4d-e6863d19d6ec: pipelining replication: peer="{Voter server-a1f0ef21-6866-8bf4-0ec1-690dca424179 a1f0ef21-6866-8bf4-0ec1-690dca424179}" +2025-03-20T03:51:07.947Z [INFO] server-ee531a2e-628d-9fb0-0c4d-e6863d19d6ec: pipelining replication: peer="{Voter server-2165f328-d003-d713-6d28-e04f6b518276 2165f328-d003-d713-6d28-e04f6b518276}" +2025-03-20T03:51:08.050Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:51:07.943193505 +0000 UTC m=+99.617646646 (2 nodes), 55.652692ms from start of poll, 56.154133ms from cluster start. Timeout at 2025-03-20 03:51:08.048198281 +0000 UTC m=+99.722651123, 105.004477ms after stability +2025-03-20T03:51:08.050Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:08.155Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:08.050359909 +0000 UTC m=+99.724812817 (1 nodes), 2.263µs from start of poll, 163.320304ms from cluster start. Timeout at 2025-03-20 03:51:08.155411126 +0000 UTC m=+99.829864136, 105.051319ms after stability +2025-03-20T03:51:08.155Z [ERROR] server-ee531a2e-628d-9fb0-0c4d-e6863d19d6ec: failed to pipeline appendEntries: peer="{Voter server-2165f328-d003-d713-6d28-e04f6b518276 2165f328-d003-d713-6d28-e04f6b518276}" error="append pipeline closed" +2025-03-20T03:51:08.155Z [INFO] server-ee531a2e-628d-9fb0-0c4d-e6863d19d6ec: aborting pipeline replication: peer="{Voter server-2165f328-d003-d713-6d28-e04f6b518276 2165f328-d003-d713-6d28-e04f6b518276}" +2025-03-20T03:51:08.155Z [ERROR] server-ee531a2e-628d-9fb0-0c4d-e6863d19d6ec: failed to appendEntries to: peer="{Voter server-2165f328-d003-d713-6d28-e04f6b518276 2165f328-d003-d713-6d28-e04f6b518276}" error="failed to connect to peer: 2165f328-d003-d713-6d28-e04f6b518276" +2025-03-20T03:51:08.158Z [ERROR] server-ee531a2e-628d-9fb0-0c4d-e6863d19d6ec: failed to heartbeat to: peer=2165f328-d003-d713-6d28-e04f6b518276 backoff time=10ms error="failed to connect to peer: 2165f328-d003-d713-6d28-e04f6b518276" +2025-03-20T03:51:08.166Z [ERROR] server-ee531a2e-628d-9fb0-0c4d-e6863d19d6ec: failed to appendEntries to: peer="{Voter server-2165f328-d003-d713-6d28-e04f6b518276 2165f328-d003-d713-6d28-e04f6b518276}" error="failed to connect to peer: 2165f328-d003-d713-6d28-e04f6b518276" raft_test.go:1627: [INFO] Finished apply without behind follower -2026-04-22T10:10:51.670Z [ERROR] server-878ba533-b1f1-6e69-d77a-be501715c960: failed to take snapshot: error="nothing new to snapshot" -2026-04-22T10:10:51.670Z [INFO] server-c9fba947-d710-cff4-09c9-eb1a8f0178c7: starting snapshot up to: index=102 -2026-04-22T10:10:51.670Z [INFO] creating new snapshot: path=/tmp/raft3782968894/snapshots/2-102-1776852651670.tmp -2026-04-22T10:10:51.671Z [INFO] server-c9fba947-d710-cff4-09c9-eb1a8f0178c7: compacting logs: from=1 to=92 -2026-04-22T10:10:51.671Z [INFO] server-c9fba947-d710-cff4-09c9-eb1a8f0178c7: snapshot complete up to: index=102 -2026-04-22T10:10:51.671Z [INFO] server-0dfc14f3-5266-5ec1-4865-bdb2041a37fb: starting snapshot up to: index=97 -2026-04-22T10:10:51.671Z [INFO] creating new snapshot: path=/tmp/raft2904758255/snapshots/2-97-1776852651671.tmp -2026-04-22T10:10:51.671Z [INFO] server-0dfc14f3-5266-5ec1-4865-bdb2041a37fb: compacting logs: from=1 to=92 -2026-04-22T10:10:51.671Z [INFO] server-0dfc14f3-5266-5ec1-4865-bdb2041a37fb: snapshot complete up to: index=97 -2026-04-22T10:10:51.679Z [ERROR] server-c9fba947-d710-cff4-09c9-eb1a8f0178c7: failed to get log: index=2 error="log not found" -2026-04-22T10:10:51.679Z [INFO] creating new snapshot: path=/tmp/raft2335437208/snapshots/2-102-1776852651679.tmp -2026-04-22T10:10:51.679Z [INFO] server-878ba533-b1f1-6e69-d77a-be501715c960: snapshot network transfer progress: read-bytes=693 percent-complete=100.00% -2026-04-22T10:10:51.679Z [INFO] server-878ba533-b1f1-6e69-d77a-be501715c960: copied to local snapshot: bytes=693 -2026-04-22T10:10:51.680Z [INFO] server-878ba533-b1f1-6e69-d77a-be501715c960: snapshot restore progress: id=2-102-1776852651679 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% -2026-04-22T10:10:51.680Z [INFO] server-878ba533-b1f1-6e69-d77a-be501715c960: Installed remote snapshot -2026-04-22T10:10:51.680Z [WARN] server-878ba533-b1f1-6e69-d77a-be501715c960: skipping application of old log: index=102 -2026-04-22T10:10:51.680Z [INFO] server-c9fba947-d710-cff4-09c9-eb1a8f0178c7: pipelining replication: peer="{Voter server-878ba533-b1f1-6e69-d77a-be501715c960 878ba533-b1f1-6e69-d77a-be501715c960}" -2026-04-22T10:10:51.682Z [INFO] server-c9fba947-d710-cff4-09c9-eb1a8f0178c7: aborting pipeline replication: peer="{Voter server-0dfc14f3-5266-5ec1-4865-bdb2041a37fb 0dfc14f3-5266-5ec1-4865-bdb2041a37fb}" -2026-04-22T10:10:51.682Z [INFO] server-c9fba947-d710-cff4-09c9-eb1a8f0178c7: aborting pipeline replication: peer="{Voter server-878ba533-b1f1-6e69-d77a-be501715c960 878ba533-b1f1-6e69-d77a-be501715c960}" ---- PASS: TestRaft_SendSnapshotFollower (0.30s) +2025-03-20T03:51:08.168Z [INFO] server-ee531a2e-628d-9fb0-0c4d-e6863d19d6ec: starting snapshot up to: index=102 +2025-03-20T03:51:08.168Z [INFO] creating new snapshot: path=/tmp/raft675886414/snapshots/2-102-1742442668168.tmp +2025-03-20T03:51:08.169Z [INFO] server-ee531a2e-628d-9fb0-0c4d-e6863d19d6ec: compacting logs: from=1 to=92 +2025-03-20T03:51:08.169Z [INFO] server-ee531a2e-628d-9fb0-0c4d-e6863d19d6ec: snapshot complete up to: index=102 +2025-03-20T03:51:08.170Z [ERROR] server-2165f328-d003-d713-6d28-e04f6b518276: failed to take snapshot: error="nothing new to snapshot" +2025-03-20T03:51:08.170Z [INFO] server-a1f0ef21-6866-8bf4-0ec1-690dca424179: starting snapshot up to: index=95 +2025-03-20T03:51:08.170Z [INFO] creating new snapshot: path=/tmp/raft3347335338/snapshots/2-95-1742442668170.tmp +2025-03-20T03:51:08.171Z [INFO] server-a1f0ef21-6866-8bf4-0ec1-690dca424179: compacting logs: from=1 to=92 +2025-03-20T03:51:08.171Z [INFO] server-a1f0ef21-6866-8bf4-0ec1-690dca424179: snapshot complete up to: index=95 +2025-03-20T03:51:08.176Z [ERROR] server-ee531a2e-628d-9fb0-0c4d-e6863d19d6ec: failed to get log: index=2 error="log not found" +2025-03-20T03:51:08.177Z [INFO] creating new snapshot: path=/tmp/raft946542385/snapshots/2-102-1742442668177.tmp +2025-03-20T03:51:08.177Z [INFO] server-2165f328-d003-d713-6d28-e04f6b518276: snapshot network transfer progress: read-bytes=693 percent-complete=100.00% +2025-03-20T03:51:08.178Z [INFO] server-2165f328-d003-d713-6d28-e04f6b518276: copied to local snapshot: bytes=693 +2025-03-20T03:51:08.179Z [INFO] server-2165f328-d003-d713-6d28-e04f6b518276: snapshot restore progress: id=2-102-1742442668177 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% +2025-03-20T03:51:08.179Z [INFO] server-2165f328-d003-d713-6d28-e04f6b518276: Installed remote snapshot +2025-03-20T03:51:08.179Z [INFO] server-ee531a2e-628d-9fb0-0c4d-e6863d19d6ec: aborting pipeline replication: peer="{Voter server-a1f0ef21-6866-8bf4-0ec1-690dca424179 a1f0ef21-6866-8bf4-0ec1-690dca424179}" +2025-03-20T03:51:08.679Z [ERROR] server-ee531a2e-628d-9fb0-0c4d-e6863d19d6ec: failed to appendEntries to: peer="{Voter server-2165f328-d003-d713-6d28-e04f6b518276 2165f328-d003-d713-6d28-e04f6b518276}" error="command timed out" +2025-03-20T03:51:08.690Z [ERROR] server-ee531a2e-628d-9fb0-0c4d-e6863d19d6ec: failed to heartbeat to: peer=2165f328-d003-d713-6d28-e04f6b518276 backoff time=10ms error="command timed out" +--- PASS: TestRaft_SendSnapshotFollower (0.81s) === RUN TestRaft_SendSnapshotAndLogsFollower -2026-04-22T10:10:51.683Z [INFO] server-a0de610f-f807-67e4-bb1b-e2528b9dfa1e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a0de610f-f807-67e4-bb1b-e2528b9dfa1e Address:a0de610f-f807-67e4-bb1b-e2528b9dfa1e} {Suffrage:Voter ID:server-88e001bf-6957-d893-ba34-36567703f67e Address:88e001bf-6957-d893-ba34-36567703f67e} {Suffrage:Voter ID:server-a69afb66-fdb1-dd1e-2501-87b7a3e933fb Address:a69afb66-fdb1-dd1e-2501-87b7a3e933fb}]" -2026-04-22T10:10:51.683Z [INFO] server-a0de610f-f807-67e4-bb1b-e2528b9dfa1e: entering follower state: follower="Node at a0de610f-f807-67e4-bb1b-e2528b9dfa1e [Follower]" leader-address= leader-id= -2026-04-22T10:10:51.683Z [INFO] server-88e001bf-6957-d893-ba34-36567703f67e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a0de610f-f807-67e4-bb1b-e2528b9dfa1e Address:a0de610f-f807-67e4-bb1b-e2528b9dfa1e} {Suffrage:Voter ID:server-88e001bf-6957-d893-ba34-36567703f67e Address:88e001bf-6957-d893-ba34-36567703f67e} {Suffrage:Voter ID:server-a69afb66-fdb1-dd1e-2501-87b7a3e933fb Address:a69afb66-fdb1-dd1e-2501-87b7a3e933fb}]" -2026-04-22T10:10:51.683Z [INFO] server-88e001bf-6957-d893-ba34-36567703f67e: entering follower state: follower="Node at 88e001bf-6957-d893-ba34-36567703f67e [Follower]" leader-address= leader-id= -2026-04-22T10:10:51.684Z [INFO] server-a69afb66-fdb1-dd1e-2501-87b7a3e933fb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a0de610f-f807-67e4-bb1b-e2528b9dfa1e Address:a0de610f-f807-67e4-bb1b-e2528b9dfa1e} {Suffrage:Voter ID:server-88e001bf-6957-d893-ba34-36567703f67e Address:88e001bf-6957-d893-ba34-36567703f67e} {Suffrage:Voter ID:server-a69afb66-fdb1-dd1e-2501-87b7a3e933fb Address:a69afb66-fdb1-dd1e-2501-87b7a3e933fb}]" -2026-04-22T10:10:51.684Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:10:51.684Z [INFO] server-a69afb66-fdb1-dd1e-2501-87b7a3e933fb: entering follower state: follower="Node at a69afb66-fdb1-dd1e-2501-87b7a3e933fb [Follower]" leader-address= leader-id= -2026-04-22T10:10:51.746Z [WARN] server-a0de610f-f807-67e4-bb1b-e2528b9dfa1e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:51.746Z [INFO] server-a0de610f-f807-67e4-bb1b-e2528b9dfa1e: entering candidate state: node="Node at a0de610f-f807-67e4-bb1b-e2528b9dfa1e [Candidate]" term=2 -2026-04-22T10:10:51.746Z [INFO] server-a0de610f-f807-67e4-bb1b-e2528b9dfa1e: election won: term=2 tally=2 -2026-04-22T10:10:51.746Z [INFO] server-a0de610f-f807-67e4-bb1b-e2528b9dfa1e: entering leader state: leader="Node at a0de610f-f807-67e4-bb1b-e2528b9dfa1e [Leader]" -2026-04-22T10:10:51.746Z [INFO] server-a0de610f-f807-67e4-bb1b-e2528b9dfa1e: added peer, starting replication: peer=server-88e001bf-6957-d893-ba34-36567703f67e -2026-04-22T10:10:51.746Z [INFO] server-a0de610f-f807-67e4-bb1b-e2528b9dfa1e: added peer, starting replication: peer=server-a69afb66-fdb1-dd1e-2501-87b7a3e933fb -2026-04-22T10:10:51.746Z [INFO] server-a0de610f-f807-67e4-bb1b-e2528b9dfa1e: pipelining replication: peer="{Voter server-88e001bf-6957-d893-ba34-36567703f67e 88e001bf-6957-d893-ba34-36567703f67e}" -2026-04-22T10:10:51.746Z [INFO] server-a0de610f-f807-67e4-bb1b-e2528b9dfa1e: pipelining replication: peer="{Voter server-a69afb66-fdb1-dd1e-2501-87b7a3e933fb a69afb66-fdb1-dd1e-2501-87b7a3e933fb}" -2026-04-22T10:10:51.853Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:10:51.746709489 +0000 UTC m=+97.770505576 (2 nodes), 62.646146ms from start of poll, 62.942778ms from cluster start. Timeout at 2026-04-22 10:10:51.851719474 +0000 UTC m=+97.875515611, 105.010035ms after stability -2026-04-22T10:10:51.853Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:51.959Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:51.853081605 +0000 UTC m=+97.876877711 (1 nodes), 1.06µs from start of poll, 169.314913ms from cluster start. Timeout at 2026-04-22 10:10:51.958115849 +0000 UTC m=+97.981912029, 105.034318ms after stability -2026-04-22T10:10:51.959Z [ERROR] server-a0de610f-f807-67e4-bb1b-e2528b9dfa1e: failed to pipeline appendEntries: peer="{Voter server-88e001bf-6957-d893-ba34-36567703f67e 88e001bf-6957-d893-ba34-36567703f67e}" error="append pipeline closed" -2026-04-22T10:10:51.959Z [INFO] server-a0de610f-f807-67e4-bb1b-e2528b9dfa1e: aborting pipeline replication: peer="{Voter server-88e001bf-6957-d893-ba34-36567703f67e 88e001bf-6957-d893-ba34-36567703f67e}" -2026-04-22T10:10:51.959Z [ERROR] server-a0de610f-f807-67e4-bb1b-e2528b9dfa1e: failed to appendEntries to: peer="{Voter server-88e001bf-6957-d893-ba34-36567703f67e 88e001bf-6957-d893-ba34-36567703f67e}" error="failed to connect to peer: 88e001bf-6957-d893-ba34-36567703f67e" +2025-03-20T03:51:08.692Z [INFO] server-c6183e4b-3f71-a3fb-e31d-15b18b9da21c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c6183e4b-3f71-a3fb-e31d-15b18b9da21c Address:c6183e4b-3f71-a3fb-e31d-15b18b9da21c} {Suffrage:Voter ID:server-39ab6176-617e-bfe3-460f-7cd195faebc7 Address:39ab6176-617e-bfe3-460f-7cd195faebc7} {Suffrage:Voter ID:server-a5228793-bafa-bec0-0ea6-78d8e80fd074 Address:a5228793-bafa-bec0-0ea6-78d8e80fd074}]" +2025-03-20T03:51:08.692Z [INFO] server-39ab6176-617e-bfe3-460f-7cd195faebc7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c6183e4b-3f71-a3fb-e31d-15b18b9da21c Address:c6183e4b-3f71-a3fb-e31d-15b18b9da21c} {Suffrage:Voter ID:server-39ab6176-617e-bfe3-460f-7cd195faebc7 Address:39ab6176-617e-bfe3-460f-7cd195faebc7} {Suffrage:Voter ID:server-a5228793-bafa-bec0-0ea6-78d8e80fd074 Address:a5228793-bafa-bec0-0ea6-78d8e80fd074}]" +2025-03-20T03:51:08.692Z [INFO] server-a5228793-bafa-bec0-0ea6-78d8e80fd074: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c6183e4b-3f71-a3fb-e31d-15b18b9da21c Address:c6183e4b-3f71-a3fb-e31d-15b18b9da21c} {Suffrage:Voter ID:server-39ab6176-617e-bfe3-460f-7cd195faebc7 Address:39ab6176-617e-bfe3-460f-7cd195faebc7} {Suffrage:Voter ID:server-a5228793-bafa-bec0-0ea6-78d8e80fd074 Address:a5228793-bafa-bec0-0ea6-78d8e80fd074}]" +2025-03-20T03:51:08.692Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:51:08.693Z [INFO] server-c6183e4b-3f71-a3fb-e31d-15b18b9da21c: entering follower state: follower="Node at c6183e4b-3f71-a3fb-e31d-15b18b9da21c [Follower]" leader-address= leader-id= +2025-03-20T03:51:08.693Z [INFO] server-39ab6176-617e-bfe3-460f-7cd195faebc7: entering follower state: follower="Node at 39ab6176-617e-bfe3-460f-7cd195faebc7 [Follower]" leader-address= leader-id= +2025-03-20T03:51:08.693Z [INFO] server-a5228793-bafa-bec0-0ea6-78d8e80fd074: entering follower state: follower="Node at a5228793-bafa-bec0-0ea6-78d8e80fd074 [Follower]" leader-address= leader-id= +2025-03-20T03:51:08.759Z [WARN] server-c6183e4b-3f71-a3fb-e31d-15b18b9da21c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:08.759Z [INFO] server-c6183e4b-3f71-a3fb-e31d-15b18b9da21c: entering candidate state: node="Node at c6183e4b-3f71-a3fb-e31d-15b18b9da21c [Candidate]" term=2 +2025-03-20T03:51:08.759Z [INFO] server-c6183e4b-3f71-a3fb-e31d-15b18b9da21c: election won: term=2 tally=2 +2025-03-20T03:51:08.759Z [INFO] server-c6183e4b-3f71-a3fb-e31d-15b18b9da21c: entering leader state: leader="Node at c6183e4b-3f71-a3fb-e31d-15b18b9da21c [Leader]" +2025-03-20T03:51:08.759Z [INFO] server-c6183e4b-3f71-a3fb-e31d-15b18b9da21c: added peer, starting replication: peer=server-39ab6176-617e-bfe3-460f-7cd195faebc7 +2025-03-20T03:51:08.759Z [INFO] server-c6183e4b-3f71-a3fb-e31d-15b18b9da21c: added peer, starting replication: peer=server-a5228793-bafa-bec0-0ea6-78d8e80fd074 +2025-03-20T03:51:08.759Z [INFO] server-c6183e4b-3f71-a3fb-e31d-15b18b9da21c: pipelining replication: peer="{Voter server-a5228793-bafa-bec0-0ea6-78d8e80fd074 a5228793-bafa-bec0-0ea6-78d8e80fd074}" +2025-03-20T03:51:08.759Z [INFO] server-c6183e4b-3f71-a3fb-e31d-15b18b9da21c: pipelining replication: peer="{Voter server-39ab6176-617e-bfe3-460f-7cd195faebc7 39ab6176-617e-bfe3-460f-7cd195faebc7}" +2025-03-20T03:51:08.865Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:51:08.759890668 +0000 UTC m=+100.434343608 (2 nodes), 67.008291ms from start of poll, 67.457248ms from cluster start. Timeout at 2025-03-20 03:51:08.864897703 +0000 UTC m=+100.539350550, 105.006942ms after stability +2025-03-20T03:51:08.865Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:08.971Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:08.865655548 +0000 UTC m=+100.540108393 (1 nodes), 3.802µs from start of poll, 173.222033ms from cluster start. Timeout at 2025-03-20 03:51:08.970731659 +0000 UTC m=+100.645184679, 105.076286ms after stability +2025-03-20T03:51:08.971Z [ERROR] server-c6183e4b-3f71-a3fb-e31d-15b18b9da21c: failed to pipeline appendEntries: peer="{Voter server-39ab6176-617e-bfe3-460f-7cd195faebc7 39ab6176-617e-bfe3-460f-7cd195faebc7}" error="append pipeline closed" +2025-03-20T03:51:08.974Z [INFO] server-c6183e4b-3f71-a3fb-e31d-15b18b9da21c: aborting pipeline replication: peer="{Voter server-39ab6176-617e-bfe3-460f-7cd195faebc7 39ab6176-617e-bfe3-460f-7cd195faebc7}" +2025-03-20T03:51:08.975Z [ERROR] server-c6183e4b-3f71-a3fb-e31d-15b18b9da21c: failed to appendEntries to: peer="{Voter server-39ab6176-617e-bfe3-460f-7cd195faebc7 39ab6176-617e-bfe3-460f-7cd195faebc7}" error="failed to connect to peer: 39ab6176-617e-bfe3-460f-7cd195faebc7" +2025-03-20T03:51:08.978Z [ERROR] server-c6183e4b-3f71-a3fb-e31d-15b18b9da21c: failed to heartbeat to: peer=39ab6176-617e-bfe3-460f-7cd195faebc7 backoff time=10ms error="failed to connect to peer: 39ab6176-617e-bfe3-460f-7cd195faebc7" raft_test.go:1669: [INFO] Finished apply without behind follower -2026-04-22T10:10:51.963Z [INFO] server-a0de610f-f807-67e4-bb1b-e2528b9dfa1e: starting snapshot up to: index=102 -2026-04-22T10:10:51.963Z [INFO] creating new snapshot: path=/tmp/raft3180086824/snapshots/2-102-1776852651963.tmp -2026-04-22T10:10:51.963Z [INFO] server-a0de610f-f807-67e4-bb1b-e2528b9dfa1e: compacting logs: from=1 to=92 -2026-04-22T10:10:51.963Z [INFO] server-a0de610f-f807-67e4-bb1b-e2528b9dfa1e: snapshot complete up to: index=102 -2026-04-22T10:10:51.964Z [ERROR] server-88e001bf-6957-d893-ba34-36567703f67e: failed to take snapshot: error="nothing new to snapshot" -2026-04-22T10:10:51.964Z [INFO] server-a69afb66-fdb1-dd1e-2501-87b7a3e933fb: starting snapshot up to: index=76 -2026-04-22T10:10:51.964Z [INFO] creating new snapshot: path=/tmp/raft3022706013/snapshots/2-76-1776852651964.tmp -2026-04-22T10:10:51.964Z [INFO] server-a69afb66-fdb1-dd1e-2501-87b7a3e933fb: compacting logs: from=1 to=76 -2026-04-22T10:10:51.964Z [INFO] server-a69afb66-fdb1-dd1e-2501-87b7a3e933fb: snapshot complete up to: index=76 -2026-04-22T10:10:51.964Z [ERROR] server-a0de610f-f807-67e4-bb1b-e2528b9dfa1e: failed to heartbeat to: peer=88e001bf-6957-d893-ba34-36567703f67e backoff time=10ms error="failed to connect to peer: 88e001bf-6957-d893-ba34-36567703f67e" +2025-03-20T03:51:08.982Z [INFO] server-c6183e4b-3f71-a3fb-e31d-15b18b9da21c: starting snapshot up to: index=102 +2025-03-20T03:51:08.982Z [INFO] creating new snapshot: path=/tmp/raft3216958840/snapshots/2-102-1742442668982.tmp +2025-03-20T03:51:08.983Z [INFO] server-c6183e4b-3f71-a3fb-e31d-15b18b9da21c: compacting logs: from=1 to=92 +2025-03-20T03:51:08.983Z [INFO] server-c6183e4b-3f71-a3fb-e31d-15b18b9da21c: snapshot complete up to: index=102 +2025-03-20T03:51:08.983Z [ERROR] server-39ab6176-617e-bfe3-460f-7cd195faebc7: failed to take snapshot: error="nothing new to snapshot" +2025-03-20T03:51:08.984Z [INFO] server-a5228793-bafa-bec0-0ea6-78d8e80fd074: starting snapshot up to: index=97 +2025-03-20T03:51:08.984Z [INFO] creating new snapshot: path=/tmp/raft9318909/snapshots/2-97-1742442668984.tmp +2025-03-20T03:51:08.984Z [INFO] server-a5228793-bafa-bec0-0ea6-78d8e80fd074: compacting logs: from=1 to=92 +2025-03-20T03:51:08.984Z [INFO] server-a5228793-bafa-bec0-0ea6-78d8e80fd074: snapshot complete up to: index=97 +2025-03-20T03:51:08.985Z [ERROR] server-c6183e4b-3f71-a3fb-e31d-15b18b9da21c: failed to get log: index=2 error="log not found" +2025-03-20T03:51:08.985Z [ERROR] server-c6183e4b-3f71-a3fb-e31d-15b18b9da21c: failed to install snapshot: id=2-102-1742442668982 error="failed to connect to peer: 39ab6176-617e-bfe3-460f-7cd195faebc7" +2025-03-20T03:51:08.985Z [ERROR] server-c6183e4b-3f71-a3fb-e31d-15b18b9da21c: failed to send snapshot to: peer="{Voter server-39ab6176-617e-bfe3-460f-7cd195faebc7 39ab6176-617e-bfe3-460f-7cd195faebc7}" error="failed to connect to peer: 39ab6176-617e-bfe3-460f-7cd195faebc7" raft_test.go:1690: [INFO] Finished apply without behind follower -2026-04-22T10:10:51.969Z [ERROR] server-a0de610f-f807-67e4-bb1b-e2528b9dfa1e: failed to get log: index=2 error="log not found" -2026-04-22T10:10:51.970Z [INFO] creating new snapshot: path=/tmp/raft3849518958/snapshots/2-102-1776852651970.tmp -2026-04-22T10:10:51.970Z [INFO] server-88e001bf-6957-d893-ba34-36567703f67e: snapshot network transfer progress: read-bytes=693 percent-complete=100.00% -2026-04-22T10:10:51.970Z [INFO] server-88e001bf-6957-d893-ba34-36567703f67e: copied to local snapshot: bytes=693 -2026-04-22T10:10:51.970Z [INFO] server-88e001bf-6957-d893-ba34-36567703f67e: snapshot restore progress: id=2-102-1776852651970 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% -2026-04-22T10:10:51.970Z [INFO] server-88e001bf-6957-d893-ba34-36567703f67e: Installed remote snapshot -2026-04-22T10:10:51.971Z [INFO] server-a0de610f-f807-67e4-bb1b-e2528b9dfa1e: pipelining replication: peer="{Voter server-88e001bf-6957-d893-ba34-36567703f67e 88e001bf-6957-d893-ba34-36567703f67e}" -2026-04-22T10:10:51.979Z [INFO] server-a0de610f-f807-67e4-bb1b-e2528b9dfa1e: aborting pipeline replication: peer="{Voter server-88e001bf-6957-d893-ba34-36567703f67e 88e001bf-6957-d893-ba34-36567703f67e}" -2026-04-22T10:10:51.979Z [INFO] server-a0de610f-f807-67e4-bb1b-e2528b9dfa1e: aborting pipeline replication: peer="{Voter server-a69afb66-fdb1-dd1e-2501-87b7a3e933fb a69afb66-fdb1-dd1e-2501-87b7a3e933fb}" ---- PASS: TestRaft_SendSnapshotAndLogsFollower (0.30s) +2025-03-20T03:51:08.996Z [ERROR] server-c6183e4b-3f71-a3fb-e31d-15b18b9da21c: failed to get log: index=2 error="log not found" +2025-03-20T03:51:08.997Z [INFO] creating new snapshot: path=/tmp/raft1027195716/snapshots/2-102-1742442668997.tmp +2025-03-20T03:51:08.997Z [INFO] server-39ab6176-617e-bfe3-460f-7cd195faebc7: snapshot network transfer progress: read-bytes=693 percent-complete=100.00% +2025-03-20T03:51:08.998Z [INFO] server-39ab6176-617e-bfe3-460f-7cd195faebc7: copied to local snapshot: bytes=693 +2025-03-20T03:51:08.998Z [INFO] server-39ab6176-617e-bfe3-460f-7cd195faebc7: snapshot restore progress: id=2-102-1742442668997 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% +2025-03-20T03:51:08.998Z [INFO] server-39ab6176-617e-bfe3-460f-7cd195faebc7: Installed remote snapshot +2025-03-20T03:51:08.999Z [INFO] server-c6183e4b-3f71-a3fb-e31d-15b18b9da21c: pipelining replication: peer="{Voter server-39ab6176-617e-bfe3-460f-7cd195faebc7 39ab6176-617e-bfe3-460f-7cd195faebc7}" +2025-03-20T03:51:09.013Z [INFO] server-c6183e4b-3f71-a3fb-e31d-15b18b9da21c: aborting pipeline replication: peer="{Voter server-a5228793-bafa-bec0-0ea6-78d8e80fd074 a5228793-bafa-bec0-0ea6-78d8e80fd074}" +2025-03-20T03:51:09.013Z [INFO] server-c6183e4b-3f71-a3fb-e31d-15b18b9da21c: aborting pipeline replication: peer="{Voter server-39ab6176-617e-bfe3-460f-7cd195faebc7 39ab6176-617e-bfe3-460f-7cd195faebc7}" +--- PASS: TestRaft_SendSnapshotAndLogsFollower (0.32s) === RUN TestRaft_ReJoinFollower -2026-04-22T10:10:51.980Z [INFO] server-0bba6b8e-771d-10bf-63ee-ecea10f4e685: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0bba6b8e-771d-10bf-63ee-ecea10f4e685 Address:0bba6b8e-771d-10bf-63ee-ecea10f4e685} {Suffrage:Voter ID:server-4256db95-139b-8e62-3f6c-f66df7e3a560 Address:4256db95-139b-8e62-3f6c-f66df7e3a560} {Suffrage:Voter ID:server-17e61f28-b457-5169-0688-59198711421d Address:17e61f28-b457-5169-0688-59198711421d}]" -2026-04-22T10:10:51.980Z [INFO] server-0bba6b8e-771d-10bf-63ee-ecea10f4e685: entering follower state: follower="Node at 0bba6b8e-771d-10bf-63ee-ecea10f4e685 [Follower]" leader-address= leader-id= -2026-04-22T10:10:51.980Z [INFO] server-4256db95-139b-8e62-3f6c-f66df7e3a560: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0bba6b8e-771d-10bf-63ee-ecea10f4e685 Address:0bba6b8e-771d-10bf-63ee-ecea10f4e685} {Suffrage:Voter ID:server-4256db95-139b-8e62-3f6c-f66df7e3a560 Address:4256db95-139b-8e62-3f6c-f66df7e3a560} {Suffrage:Voter ID:server-17e61f28-b457-5169-0688-59198711421d Address:17e61f28-b457-5169-0688-59198711421d}]" -2026-04-22T10:10:51.980Z [INFO] server-4256db95-139b-8e62-3f6c-f66df7e3a560: entering follower state: follower="Node at 4256db95-139b-8e62-3f6c-f66df7e3a560 [Follower]" leader-address= leader-id= -2026-04-22T10:10:51.980Z [INFO] server-17e61f28-b457-5169-0688-59198711421d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0bba6b8e-771d-10bf-63ee-ecea10f4e685 Address:0bba6b8e-771d-10bf-63ee-ecea10f4e685} {Suffrage:Voter ID:server-4256db95-139b-8e62-3f6c-f66df7e3a560 Address:4256db95-139b-8e62-3f6c-f66df7e3a560} {Suffrage:Voter ID:server-17e61f28-b457-5169-0688-59198711421d Address:17e61f28-b457-5169-0688-59198711421d}]" -2026-04-22T10:10:51.980Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:51.980Z [INFO] server-17e61f28-b457-5169-0688-59198711421d: entering follower state: follower="Node at 17e61f28-b457-5169-0688-59198711421d [Follower]" leader-address= leader-id= -2026-04-22T10:10:52.035Z [WARN] server-0bba6b8e-771d-10bf-63ee-ecea10f4e685: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:52.035Z [INFO] server-0bba6b8e-771d-10bf-63ee-ecea10f4e685: entering candidate state: node="Node at 0bba6b8e-771d-10bf-63ee-ecea10f4e685 [Candidate]" term=2 -2026-04-22T10:10:52.035Z [INFO] server-0bba6b8e-771d-10bf-63ee-ecea10f4e685: election won: term=2 tally=2 -2026-04-22T10:10:52.035Z [INFO] server-0bba6b8e-771d-10bf-63ee-ecea10f4e685: entering leader state: leader="Node at 0bba6b8e-771d-10bf-63ee-ecea10f4e685 [Leader]" -2026-04-22T10:10:52.035Z [INFO] server-0bba6b8e-771d-10bf-63ee-ecea10f4e685: added peer, starting replication: peer=server-4256db95-139b-8e62-3f6c-f66df7e3a560 -2026-04-22T10:10:52.035Z [INFO] server-0bba6b8e-771d-10bf-63ee-ecea10f4e685: added peer, starting replication: peer=server-17e61f28-b457-5169-0688-59198711421d -2026-04-22T10:10:52.035Z [INFO] server-0bba6b8e-771d-10bf-63ee-ecea10f4e685: pipelining replication: peer="{Voter server-4256db95-139b-8e62-3f6c-f66df7e3a560 4256db95-139b-8e62-3f6c-f66df7e3a560}" -2026-04-22T10:10:52.035Z [INFO] server-0bba6b8e-771d-10bf-63ee-ecea10f4e685: pipelining replication: peer="{Voter server-17e61f28-b457-5169-0688-59198711421d 17e61f28-b457-5169-0688-59198711421d}" -2026-04-22T10:10:52.140Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:52.035215717 +0000 UTC m=+98.059011826 (1 nodes), 54.84582ms from start of poll, 55.15796ms from cluster start. Timeout at 2026-04-22 10:10:52.140222151 +0000 UTC m=+98.164018288, 105.006462ms after stability -2026-04-22T10:10:52.146Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:10:52.251Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:10:52.146130655 +0000 UTC m=+98.169929445 (2 nodes), 4.734µs from start of poll, 166.075579ms from cluster start. Timeout at 2026-04-22 10:10:52.25114217 +0000 UTC m=+98.274938313, 105.008868ms after stability -2026-04-22T10:10:52.252Z [INFO] server-0bba6b8e-771d-10bf-63ee-ecea10f4e685: updating configuration: command=RemoveServer server-id=server-4256db95-139b-8e62-3f6c-f66df7e3a560 server-addr= servers="[{Suffrage:Voter ID:server-0bba6b8e-771d-10bf-63ee-ecea10f4e685 Address:0bba6b8e-771d-10bf-63ee-ecea10f4e685} {Suffrage:Voter ID:server-17e61f28-b457-5169-0688-59198711421d Address:17e61f28-b457-5169-0688-59198711421d}]" -2026-04-22T10:10:52.252Z [INFO] server-0bba6b8e-771d-10bf-63ee-ecea10f4e685: removed peer, stopping replication: peer=server-4256db95-139b-8e62-3f6c-f66df7e3a560 last-index=3 -2026-04-22T10:10:52.252Z [INFO] server-0bba6b8e-771d-10bf-63ee-ecea10f4e685: aborting pipeline replication: peer="{Voter server-4256db95-139b-8e62-3f6c-f66df7e3a560 4256db95-139b-8e62-3f6c-f66df7e3a560}" -2026-04-22T10:10:52.357Z [INFO] server-0bba6b8e-771d-10bf-63ee-ecea10f4e685: updating configuration: command=AddVoter server-id=server-4256db95-139b-8e62-3f6c-f66df7e3a560 server-addr=4256db95-139b-8e62-3f6c-f66df7e3a560 servers="[{Suffrage:Voter ID:server-0bba6b8e-771d-10bf-63ee-ecea10f4e685 Address:0bba6b8e-771d-10bf-63ee-ecea10f4e685} {Suffrage:Voter ID:server-17e61f28-b457-5169-0688-59198711421d Address:17e61f28-b457-5169-0688-59198711421d} {Suffrage:Voter ID:server-4256db95-139b-8e62-3f6c-f66df7e3a560 Address:4256db95-139b-8e62-3f6c-f66df7e3a560}]" -2026-04-22T10:10:52.357Z [INFO] server-0bba6b8e-771d-10bf-63ee-ecea10f4e685: added peer, starting replication: peer=server-4256db95-139b-8e62-3f6c-f66df7e3a560 -2026-04-22T10:10:52.358Z [WARN] server-4256db95-139b-8e62-3f6c-f66df7e3a560: failed to get previous log: previous-index=4 last-index=3 error="log not found" -2026-04-22T10:10:52.358Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:52.358Z [WARN] server-0bba6b8e-771d-10bf-63ee-ecea10f4e685: appendEntries rejected, sending older logs: peer="{Voter server-4256db95-139b-8e62-3f6c-f66df7e3a560 4256db95-139b-8e62-3f6c-f66df7e3a560}" next=4 -2026-04-22T10:10:52.358Z [INFO] server-0bba6b8e-771d-10bf-63ee-ecea10f4e685: pipelining replication: peer="{Voter server-4256db95-139b-8e62-3f6c-f66df7e3a560 4256db95-139b-8e62-3f6c-f66df7e3a560}" -2026-04-22T10:10:52.463Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:52.358184856 +0000 UTC m=+98.381981006 (1 nodes), 1.039µs from start of poll, 378.12714ms from cluster start. Timeout at 2026-04-22 10:10:52.46321697 +0000 UTC m=+98.487029308, 105.048302ms after stability -2026-04-22T10:10:52.463Z [INFO] server-0bba6b8e-771d-10bf-63ee-ecea10f4e685: aborting pipeline replication: peer="{Voter server-17e61f28-b457-5169-0688-59198711421d 17e61f28-b457-5169-0688-59198711421d}" -2026-04-22T10:10:52.463Z [INFO] server-0bba6b8e-771d-10bf-63ee-ecea10f4e685: aborting pipeline replication: peer="{Voter server-4256db95-139b-8e62-3f6c-f66df7e3a560 4256db95-139b-8e62-3f6c-f66df7e3a560}" ---- PASS: TestRaft_ReJoinFollower (0.48s) +2025-03-20T03:51:09.017Z [INFO] server-ca7b3183-f744-6363-2d8e-fdb70e0c441f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ca7b3183-f744-6363-2d8e-fdb70e0c441f Address:ca7b3183-f744-6363-2d8e-fdb70e0c441f} {Suffrage:Voter ID:server-b65e6408-4ffb-cb20-e5dc-37645596d168 Address:b65e6408-4ffb-cb20-e5dc-37645596d168} {Suffrage:Voter ID:server-6fc79211-181f-7692-db39-2b5a817ddf68 Address:6fc79211-181f-7692-db39-2b5a817ddf68}]" +2025-03-20T03:51:09.018Z [INFO] server-b65e6408-4ffb-cb20-e5dc-37645596d168: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ca7b3183-f744-6363-2d8e-fdb70e0c441f Address:ca7b3183-f744-6363-2d8e-fdb70e0c441f} {Suffrage:Voter ID:server-b65e6408-4ffb-cb20-e5dc-37645596d168 Address:b65e6408-4ffb-cb20-e5dc-37645596d168} {Suffrage:Voter ID:server-6fc79211-181f-7692-db39-2b5a817ddf68 Address:6fc79211-181f-7692-db39-2b5a817ddf68}]" +2025-03-20T03:51:09.019Z [INFO] server-6fc79211-181f-7692-db39-2b5a817ddf68: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ca7b3183-f744-6363-2d8e-fdb70e0c441f Address:ca7b3183-f744-6363-2d8e-fdb70e0c441f} {Suffrage:Voter ID:server-b65e6408-4ffb-cb20-e5dc-37645596d168 Address:b65e6408-4ffb-cb20-e5dc-37645596d168} {Suffrage:Voter ID:server-6fc79211-181f-7692-db39-2b5a817ddf68 Address:6fc79211-181f-7692-db39-2b5a817ddf68}]" +2025-03-20T03:51:09.019Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:09.023Z [INFO] server-ca7b3183-f744-6363-2d8e-fdb70e0c441f: entering follower state: follower="Node at ca7b3183-f744-6363-2d8e-fdb70e0c441f [Follower]" leader-address= leader-id= +2025-03-20T03:51:09.023Z [INFO] server-b65e6408-4ffb-cb20-e5dc-37645596d168: entering follower state: follower="Node at b65e6408-4ffb-cb20-e5dc-37645596d168 [Follower]" leader-address= leader-id= +2025-03-20T03:51:09.024Z [INFO] server-6fc79211-181f-7692-db39-2b5a817ddf68: entering follower state: follower="Node at 6fc79211-181f-7692-db39-2b5a817ddf68 [Follower]" leader-address= leader-id= +2025-03-20T03:51:09.081Z [WARN] server-b65e6408-4ffb-cb20-e5dc-37645596d168: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:09.081Z [INFO] server-b65e6408-4ffb-cb20-e5dc-37645596d168: entering candidate state: node="Node at b65e6408-4ffb-cb20-e5dc-37645596d168 [Candidate]" term=2 +2025-03-20T03:51:09.081Z [INFO] server-b65e6408-4ffb-cb20-e5dc-37645596d168: election won: term=2 tally=2 +2025-03-20T03:51:09.081Z [INFO] server-b65e6408-4ffb-cb20-e5dc-37645596d168: entering leader state: leader="Node at b65e6408-4ffb-cb20-e5dc-37645596d168 [Leader]" +2025-03-20T03:51:09.081Z [INFO] server-b65e6408-4ffb-cb20-e5dc-37645596d168: added peer, starting replication: peer=server-ca7b3183-f744-6363-2d8e-fdb70e0c441f +2025-03-20T03:51:09.081Z [INFO] server-b65e6408-4ffb-cb20-e5dc-37645596d168: added peer, starting replication: peer=server-6fc79211-181f-7692-db39-2b5a817ddf68 +2025-03-20T03:51:09.082Z [INFO] server-b65e6408-4ffb-cb20-e5dc-37645596d168: pipelining replication: peer="{Voter server-ca7b3183-f744-6363-2d8e-fdb70e0c441f ca7b3183-f744-6363-2d8e-fdb70e0c441f}" +2025-03-20T03:51:09.082Z [INFO] server-b65e6408-4ffb-cb20-e5dc-37645596d168: pipelining replication: peer="{Voter server-6fc79211-181f-7692-db39-2b5a817ddf68 6fc79211-181f-7692-db39-2b5a817ddf68}" +2025-03-20T03:51:09.188Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:09.081937455 +0000 UTC m=+100.756391100 (1 nodes), 62.281735ms from start of poll, 65.494194ms from cluster start. Timeout at 2025-03-20 03:51:09.186944525 +0000 UTC m=+100.861397481, 105.006381ms after stability +2025-03-20T03:51:09.194Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:51:09.300Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:51:09.194362546 +0000 UTC m=+100.868815699 (2 nodes), 3.012µs from start of poll, 177.918793ms from cluster start. Timeout at 2025-03-20 03:51:09.299435838 +0000 UTC m=+100.973888951, 105.073252ms after stability +2025-03-20T03:51:09.300Z [INFO] server-b65e6408-4ffb-cb20-e5dc-37645596d168: updating configuration: command=RemoveServer server-id=server-ca7b3183-f744-6363-2d8e-fdb70e0c441f server-addr= servers="[{Suffrage:Voter ID:server-b65e6408-4ffb-cb20-e5dc-37645596d168 Address:b65e6408-4ffb-cb20-e5dc-37645596d168} {Suffrage:Voter ID:server-6fc79211-181f-7692-db39-2b5a817ddf68 Address:6fc79211-181f-7692-db39-2b5a817ddf68}]" +2025-03-20T03:51:09.300Z [INFO] server-b65e6408-4ffb-cb20-e5dc-37645596d168: removed peer, stopping replication: peer=server-ca7b3183-f744-6363-2d8e-fdb70e0c441f last-index=3 +2025-03-20T03:51:09.300Z [INFO] server-b65e6408-4ffb-cb20-e5dc-37645596d168: aborting pipeline replication: peer="{Voter server-ca7b3183-f744-6363-2d8e-fdb70e0c441f ca7b3183-f744-6363-2d8e-fdb70e0c441f}" +2025-03-20T03:51:09.393Z [WARN] server-ca7b3183-f744-6363-2d8e-fdb70e0c441f: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +2025-03-20T03:51:09.406Z [INFO] server-b65e6408-4ffb-cb20-e5dc-37645596d168: updating configuration: command=AddVoter server-id=server-ca7b3183-f744-6363-2d8e-fdb70e0c441f server-addr=ca7b3183-f744-6363-2d8e-fdb70e0c441f servers="[{Suffrage:Voter ID:server-b65e6408-4ffb-cb20-e5dc-37645596d168 Address:b65e6408-4ffb-cb20-e5dc-37645596d168} {Suffrage:Voter ID:server-6fc79211-181f-7692-db39-2b5a817ddf68 Address:6fc79211-181f-7692-db39-2b5a817ddf68} {Suffrage:Voter ID:server-ca7b3183-f744-6363-2d8e-fdb70e0c441f Address:ca7b3183-f744-6363-2d8e-fdb70e0c441f}]" +2025-03-20T03:51:09.407Z [INFO] server-b65e6408-4ffb-cb20-e5dc-37645596d168: added peer, starting replication: peer=server-ca7b3183-f744-6363-2d8e-fdb70e0c441f +2025-03-20T03:51:09.407Z [WARN] server-ca7b3183-f744-6363-2d8e-fdb70e0c441f: failed to get previous log: previous-index=4 last-index=3 error="log not found" +2025-03-20T03:51:09.407Z [WARN] server-b65e6408-4ffb-cb20-e5dc-37645596d168: appendEntries rejected, sending older logs: peer="{Voter server-ca7b3183-f744-6363-2d8e-fdb70e0c441f ca7b3183-f744-6363-2d8e-fdb70e0c441f}" next=4 +2025-03-20T03:51:09.407Z [INFO] server-b65e6408-4ffb-cb20-e5dc-37645596d168: pipelining replication: peer="{Voter server-ca7b3183-f744-6363-2d8e-fdb70e0c441f ca7b3183-f744-6363-2d8e-fdb70e0c441f}" +2025-03-20T03:51:09.407Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:09.513Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:09.407555288 +0000 UTC m=+101.082008075 (1 nodes), 1.515µs from start of poll, 391.111169ms from cluster start. Timeout at 2025-03-20 03:51:09.512723269 +0000 UTC m=+101.187176408, 105.168333ms after stability +2025-03-20T03:51:09.513Z [INFO] server-b65e6408-4ffb-cb20-e5dc-37645596d168: aborting pipeline replication: peer="{Voter server-6fc79211-181f-7692-db39-2b5a817ddf68 6fc79211-181f-7692-db39-2b5a817ddf68}" +2025-03-20T03:51:09.513Z [INFO] server-b65e6408-4ffb-cb20-e5dc-37645596d168: aborting pipeline replication: peer="{Voter server-ca7b3183-f744-6363-2d8e-fdb70e0c441f ca7b3183-f744-6363-2d8e-fdb70e0c441f}" +--- PASS: TestRaft_ReJoinFollower (0.50s) === RUN TestRaft_LeaderLeaseExpire -2026-04-22T10:10:52.464Z [INFO] server-3ed41163-ca1e-01be-5350-08301687ee2d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3ed41163-ca1e-01be-5350-08301687ee2d Address:3ed41163-ca1e-01be-5350-08301687ee2d} {Suffrage:Voter ID:server-0c19192d-eea8-56e5-8fea-c839cdea3de4 Address:0c19192d-eea8-56e5-8fea-c839cdea3de4}]" -2026-04-22T10:10:52.464Z [INFO] server-3ed41163-ca1e-01be-5350-08301687ee2d: entering follower state: follower="Node at 3ed41163-ca1e-01be-5350-08301687ee2d [Follower]" leader-address= leader-id= -2026-04-22T10:10:52.464Z [INFO] server-0c19192d-eea8-56e5-8fea-c839cdea3de4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3ed41163-ca1e-01be-5350-08301687ee2d Address:3ed41163-ca1e-01be-5350-08301687ee2d} {Suffrage:Voter ID:server-0c19192d-eea8-56e5-8fea-c839cdea3de4 Address:0c19192d-eea8-56e5-8fea-c839cdea3de4}]" -2026-04-22T10:10:52.464Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:52.464Z [INFO] server-0c19192d-eea8-56e5-8fea-c839cdea3de4: entering follower state: follower="Node at 0c19192d-eea8-56e5-8fea-c839cdea3de4 [Follower]" leader-address= leader-id= -2026-04-22T10:10:52.523Z [WARN] server-3ed41163-ca1e-01be-5350-08301687ee2d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:52.523Z [INFO] server-3ed41163-ca1e-01be-5350-08301687ee2d: entering candidate state: node="Node at 3ed41163-ca1e-01be-5350-08301687ee2d [Candidate]" term=2 -2026-04-22T10:10:52.523Z [INFO] server-3ed41163-ca1e-01be-5350-08301687ee2d: election won: term=2 tally=2 -2026-04-22T10:10:52.523Z [INFO] server-3ed41163-ca1e-01be-5350-08301687ee2d: entering leader state: leader="Node at 3ed41163-ca1e-01be-5350-08301687ee2d [Leader]" -2026-04-22T10:10:52.523Z [INFO] server-3ed41163-ca1e-01be-5350-08301687ee2d: added peer, starting replication: peer=server-0c19192d-eea8-56e5-8fea-c839cdea3de4 -2026-04-22T10:10:52.523Z [INFO] server-3ed41163-ca1e-01be-5350-08301687ee2d: pipelining replication: peer="{Voter server-0c19192d-eea8-56e5-8fea-c839cdea3de4 0c19192d-eea8-56e5-8fea-c839cdea3de4}" -2026-04-22T10:10:52.629Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:52.523814881 +0000 UTC m=+98.547610996 (1 nodes), 59.155431ms from start of poll, 59.358929ms from cluster start. Timeout at 2026-04-22 10:10:52.628818531 +0000 UTC m=+98.652614625, 105.003629ms after stability -2026-04-22T10:10:52.635Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:10:52.741Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:10:52.635914176 +0000 UTC m=+98.659710277 (1 nodes), 584ns from start of poll, 171.45821ms from cluster start. Timeout at 2026-04-22 10:10:52.740932897 +0000 UTC m=+98.764729033, 105.018756ms after stability - raft_test.go:1799: [INFO] Disconnecting Node at 0c19192d-eea8-56e5-8fea-c839cdea3de4 [Follower] -2026-04-22T10:10:52.742Z [ERROR] server-3ed41163-ca1e-01be-5350-08301687ee2d: failed to pipeline appendEntries: peer="{Voter server-0c19192d-eea8-56e5-8fea-c839cdea3de4 0c19192d-eea8-56e5-8fea-c839cdea3de4}" error="append pipeline closed" -2026-04-22T10:10:52.742Z [INFO] server-3ed41163-ca1e-01be-5350-08301687ee2d: aborting pipeline replication: peer="{Voter server-0c19192d-eea8-56e5-8fea-c839cdea3de4 0c19192d-eea8-56e5-8fea-c839cdea3de4}" -2026-04-22T10:10:52.743Z [ERROR] server-3ed41163-ca1e-01be-5350-08301687ee2d: failed to heartbeat to: peer=0c19192d-eea8-56e5-8fea-c839cdea3de4 backoff time=10ms error="failed to connect to peer: 0c19192d-eea8-56e5-8fea-c839cdea3de4" -2026-04-22T10:10:52.751Z [ERROR] server-3ed41163-ca1e-01be-5350-08301687ee2d: failed to appendEntries to: peer="{Voter server-0c19192d-eea8-56e5-8fea-c839cdea3de4 0c19192d-eea8-56e5-8fea-c839cdea3de4}" error="failed to connect to peer: 0c19192d-eea8-56e5-8fea-c839cdea3de4" -2026-04-22T10:10:52.762Z [ERROR] server-3ed41163-ca1e-01be-5350-08301687ee2d: failed to heartbeat to: peer=0c19192d-eea8-56e5-8fea-c839cdea3de4 backoff time=10ms error="failed to connect to peer: 0c19192d-eea8-56e5-8fea-c839cdea3de4" -2026-04-22T10:10:52.766Z [ERROR] server-3ed41163-ca1e-01be-5350-08301687ee2d: failed to appendEntries to: peer="{Voter server-0c19192d-eea8-56e5-8fea-c839cdea3de4 0c19192d-eea8-56e5-8fea-c839cdea3de4}" error="failed to connect to peer: 0c19192d-eea8-56e5-8fea-c839cdea3de4" -2026-04-22T10:10:52.781Z [ERROR] server-3ed41163-ca1e-01be-5350-08301687ee2d: failed to heartbeat to: peer=0c19192d-eea8-56e5-8fea-c839cdea3de4 backoff time=10ms error="failed to connect to peer: 0c19192d-eea8-56e5-8fea-c839cdea3de4" -2026-04-22T10:10:52.784Z [ERROR] server-3ed41163-ca1e-01be-5350-08301687ee2d: failed to appendEntries to: peer="{Voter server-0c19192d-eea8-56e5-8fea-c839cdea3de4 0c19192d-eea8-56e5-8fea-c839cdea3de4}" error="failed to connect to peer: 0c19192d-eea8-56e5-8fea-c839cdea3de4" -2026-04-22T10:10:52.784Z [WARN] server-3ed41163-ca1e-01be-5350-08301687ee2d: failed to contact: server-id=server-0c19192d-eea8-56e5-8fea-c839cdea3de4 time=50.089949ms -2026-04-22T10:10:52.784Z [WARN] server-3ed41163-ca1e-01be-5350-08301687ee2d: failed to contact quorum of nodes, stepping down -2026-04-22T10:10:52.784Z [INFO] server-3ed41163-ca1e-01be-5350-08301687ee2d: entering follower state: follower="Node at 3ed41163-ca1e-01be-5350-08301687ee2d [Follower]" leader-address= leader-id= -2026-04-22T10:10:52.784Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:52.833Z [WARN] server-0c19192d-eea8-56e5-8fea-c839cdea3de4: heartbeat timeout reached, starting election: last-leader-addr=3ed41163-ca1e-01be-5350-08301687ee2d last-leader-id=server-3ed41163-ca1e-01be-5350-08301687ee2d -2026-04-22T10:10:52.833Z [INFO] server-0c19192d-eea8-56e5-8fea-c839cdea3de4: entering candidate state: node="Node at 0c19192d-eea8-56e5-8fea-c839cdea3de4 [Candidate]" term=3 -2026-04-22T10:10:52.833Z [ERROR] server-0c19192d-eea8-56e5-8fea-c839cdea3de4: failed to make requestVote RPC: target="{Voter server-3ed41163-ca1e-01be-5350-08301687ee2d 3ed41163-ca1e-01be-5350-08301687ee2d}" error="failed to connect to peer: 3ed41163-ca1e-01be-5350-08301687ee2d" term=3 -2026-04-22T10:10:52.837Z [WARN] server-3ed41163-ca1e-01be-5350-08301687ee2d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:52.837Z [INFO] server-3ed41163-ca1e-01be-5350-08301687ee2d: entering candidate state: node="Node at 3ed41163-ca1e-01be-5350-08301687ee2d [Candidate]" term=3 -2026-04-22T10:10:52.837Z [ERROR] server-3ed41163-ca1e-01be-5350-08301687ee2d: failed to make requestVote RPC: target="{Voter server-0c19192d-eea8-56e5-8fea-c839cdea3de4 0c19192d-eea8-56e5-8fea-c839cdea3de4}" error="failed to connect to peer: 0c19192d-eea8-56e5-8fea-c839cdea3de4" term=3 -2026-04-22T10:10:52.910Z [WARN] server-0c19192d-eea8-56e5-8fea-c839cdea3de4: Election timeout reached, restarting election -2026-04-22T10:10:52.910Z [INFO] server-0c19192d-eea8-56e5-8fea-c839cdea3de4: entering candidate state: node="Node at 0c19192d-eea8-56e5-8fea-c839cdea3de4 [Candidate]" term=4 -2026-04-22T10:10:52.910Z [ERROR] server-0c19192d-eea8-56e5-8fea-c839cdea3de4: failed to make requestVote RPC: target="{Voter server-3ed41163-ca1e-01be-5350-08301687ee2d 3ed41163-ca1e-01be-5350-08301687ee2d}" error="failed to connect to peer: 3ed41163-ca1e-01be-5350-08301687ee2d" term=4 -2026-04-22T10:10:52.934Z [WARN] server-3ed41163-ca1e-01be-5350-08301687ee2d: Election timeout reached, restarting election -2026-04-22T10:10:52.934Z [INFO] server-3ed41163-ca1e-01be-5350-08301687ee2d: entering candidate state: node="Node at 3ed41163-ca1e-01be-5350-08301687ee2d [Candidate]" term=4 -2026-04-22T10:10:52.934Z [ERROR] server-3ed41163-ca1e-01be-5350-08301687ee2d: failed to make requestVote RPC: target="{Voter server-0c19192d-eea8-56e5-8fea-c839cdea3de4 0c19192d-eea8-56e5-8fea-c839cdea3de4}" error="failed to connect to peer: 0c19192d-eea8-56e5-8fea-c839cdea3de4" term=4 -2026-04-22T10:10:52.942Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:52.837160566 +0000 UTC m=+98.860956716 (0 nodes), 52.637139ms from start of poll, 372.704649ms from cluster start. Timeout at 2026-04-22 10:10:52.942179118 +0000 UTC m=+98.965975345, 105.018629ms after stability -2026-04-22T10:10:52.962Z [WARN] server-0c19192d-eea8-56e5-8fea-c839cdea3de4: Election timeout reached, restarting election -2026-04-22T10:10:52.962Z [INFO] server-0c19192d-eea8-56e5-8fea-c839cdea3de4: entering candidate state: node="Node at 0c19192d-eea8-56e5-8fea-c839cdea3de4 [Candidate]" term=5 -2026-04-22T10:10:52.962Z [ERROR] server-0c19192d-eea8-56e5-8fea-c839cdea3de4: failed to make requestVote RPC: target="{Voter server-3ed41163-ca1e-01be-5350-08301687ee2d 3ed41163-ca1e-01be-5350-08301687ee2d}" error="failed to connect to peer: 3ed41163-ca1e-01be-5350-08301687ee2d" term=5 -2026-04-22T10:10:52.989Z [WARN] server-3ed41163-ca1e-01be-5350-08301687ee2d: Election timeout reached, restarting election -2026-04-22T10:10:52.989Z [INFO] server-3ed41163-ca1e-01be-5350-08301687ee2d: entering candidate state: node="Node at 3ed41163-ca1e-01be-5350-08301687ee2d [Candidate]" term=5 -2026-04-22T10:10:52.989Z [ERROR] server-3ed41163-ca1e-01be-5350-08301687ee2d: failed to make requestVote RPC: target="{Voter server-0c19192d-eea8-56e5-8fea-c839cdea3de4 0c19192d-eea8-56e5-8fea-c839cdea3de4}" error="failed to connect to peer: 0c19192d-eea8-56e5-8fea-c839cdea3de4" term=5 -2026-04-22T10:10:53.032Z [WARN] server-0c19192d-eea8-56e5-8fea-c839cdea3de4: Election timeout reached, restarting election -2026-04-22T10:10:53.032Z [INFO] server-0c19192d-eea8-56e5-8fea-c839cdea3de4: entering candidate state: node="Node at 0c19192d-eea8-56e5-8fea-c839cdea3de4 [Candidate]" term=6 -2026-04-22T10:10:53.032Z [ERROR] server-0c19192d-eea8-56e5-8fea-c839cdea3de4: failed to make requestVote RPC: target="{Voter server-3ed41163-ca1e-01be-5350-08301687ee2d 3ed41163-ca1e-01be-5350-08301687ee2d}" error="failed to connect to peer: 3ed41163-ca1e-01be-5350-08301687ee2d" term=6 -2026-04-22T10:10:53.046Z [WARN] server-3ed41163-ca1e-01be-5350-08301687ee2d: Election timeout reached, restarting election -2026-04-22T10:10:53.046Z [INFO] server-3ed41163-ca1e-01be-5350-08301687ee2d: entering candidate state: node="Node at 3ed41163-ca1e-01be-5350-08301687ee2d [Candidate]" term=6 -2026-04-22T10:10:53.046Z [ERROR] server-3ed41163-ca1e-01be-5350-08301687ee2d: failed to make requestVote RPC: target="{Voter server-0c19192d-eea8-56e5-8fea-c839cdea3de4 0c19192d-eea8-56e5-8fea-c839cdea3de4}" error="failed to connect to peer: 0c19192d-eea8-56e5-8fea-c839cdea3de4" term=6 ---- PASS: TestRaft_LeaderLeaseExpire (0.58s) +2025-03-20T03:51:09.517Z [INFO] server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e Address:9abd7df2-304f-3b7d-c2ab-cbe968a7817e} {Suffrage:Voter ID:server-f49f0bd9-4faf-2546-4781-849927a81645 Address:f49f0bd9-4faf-2546-4781-849927a81645}]" +2025-03-20T03:51:09.517Z [INFO] server-f49f0bd9-4faf-2546-4781-849927a81645: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e Address:9abd7df2-304f-3b7d-c2ab-cbe968a7817e} {Suffrage:Voter ID:server-f49f0bd9-4faf-2546-4781-849927a81645 Address:f49f0bd9-4faf-2546-4781-849927a81645}]" +2025-03-20T03:51:09.517Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:09.517Z [INFO] server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e: entering follower state: follower="Node at 9abd7df2-304f-3b7d-c2ab-cbe968a7817e [Follower]" leader-address= leader-id= +2025-03-20T03:51:09.517Z [INFO] server-f49f0bd9-4faf-2546-4781-849927a81645: entering follower state: follower="Node at f49f0bd9-4faf-2546-4781-849927a81645 [Follower]" leader-address= leader-id= +2025-03-20T03:51:09.577Z [WARN] server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:09.577Z [INFO] server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e: entering candidate state: node="Node at 9abd7df2-304f-3b7d-c2ab-cbe968a7817e [Candidate]" term=2 +2025-03-20T03:51:09.577Z [INFO] server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e: election won: term=2 tally=2 +2025-03-20T03:51:09.577Z [INFO] server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e: entering leader state: leader="Node at 9abd7df2-304f-3b7d-c2ab-cbe968a7817e [Leader]" +2025-03-20T03:51:09.577Z [INFO] server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e: added peer, starting replication: peer=server-f49f0bd9-4faf-2546-4781-849927a81645 +2025-03-20T03:51:09.577Z [INFO] server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e: pipelining replication: peer="{Voter server-f49f0bd9-4faf-2546-4781-849927a81645 f49f0bd9-4faf-2546-4781-849927a81645}" +2025-03-20T03:51:09.682Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:09.577276442 +0000 UTC m=+101.251729365 (1 nodes), 59.41699ms from start of poll, 59.79225ms from cluster start. Timeout at 2025-03-20 03:51:09.682293065 +0000 UTC m=+101.356745835, 105.01647ms after stability +2025-03-20T03:51:09.688Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:51:09.793Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:51:09.688333844 +0000 UTC m=+101.362786789 (1 nodes), 1.931µs from start of poll, 170.849674ms from cluster start. Timeout at 2025-03-20 03:51:09.793428271 +0000 UTC m=+101.467881118, 105.094329ms after stability + raft_test.go:1799: [INFO] Disconnecting Node at f49f0bd9-4faf-2546-4781-849927a81645 [Follower] +2025-03-20T03:51:09.795Z [ERROR] server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e: failed to heartbeat to: peer=f49f0bd9-4faf-2546-4781-849927a81645 backoff time=10ms error="failed to connect to peer: f49f0bd9-4faf-2546-4781-849927a81645" +2025-03-20T03:51:09.803Z [ERROR] server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e: failed to pipeline appendEntries: peer="{Voter server-f49f0bd9-4faf-2546-4781-849927a81645 f49f0bd9-4faf-2546-4781-849927a81645}" error="append pipeline closed" +2025-03-20T03:51:09.803Z [INFO] server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e: aborting pipeline replication: peer="{Voter server-f49f0bd9-4faf-2546-4781-849927a81645 f49f0bd9-4faf-2546-4781-849927a81645}" +2025-03-20T03:51:09.811Z [ERROR] server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e: failed to appendEntries to: peer="{Voter server-f49f0bd9-4faf-2546-4781-849927a81645 f49f0bd9-4faf-2546-4781-849927a81645}" error="failed to connect to peer: f49f0bd9-4faf-2546-4781-849927a81645" +2025-03-20T03:51:09.815Z [ERROR] server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e: failed to heartbeat to: peer=f49f0bd9-4faf-2546-4781-849927a81645 backoff time=10ms error="failed to connect to peer: f49f0bd9-4faf-2546-4781-849927a81645" +2025-03-20T03:51:09.830Z [ERROR] server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e: failed to appendEntries to: peer="{Voter server-f49f0bd9-4faf-2546-4781-849927a81645 f49f0bd9-4faf-2546-4781-849927a81645}" error="failed to connect to peer: f49f0bd9-4faf-2546-4781-849927a81645" +2025-03-20T03:51:09.835Z [ERROR] server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e: failed to heartbeat to: peer=f49f0bd9-4faf-2546-4781-849927a81645 backoff time=10ms error="failed to connect to peer: f49f0bd9-4faf-2546-4781-849927a81645" +2025-03-20T03:51:09.839Z [WARN] server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e: failed to contact: server-id=server-f49f0bd9-4faf-2546-4781-849927a81645 time=51.419281ms +2025-03-20T03:51:09.839Z [WARN] server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e: failed to contact quorum of nodes, stepping down +2025-03-20T03:51:09.839Z [INFO] server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e: entering follower state: follower="Node at 9abd7df2-304f-3b7d-c2ab-cbe968a7817e [Follower]" leader-address= leader-id= +2025-03-20T03:51:09.839Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:09.904Z [WARN] server-f49f0bd9-4faf-2546-4781-849927a81645: heartbeat timeout reached, starting election: last-leader-addr=9abd7df2-304f-3b7d-c2ab-cbe968a7817e last-leader-id=server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e +2025-03-20T03:51:09.904Z [INFO] server-f49f0bd9-4faf-2546-4781-849927a81645: entering candidate state: node="Node at f49f0bd9-4faf-2546-4781-849927a81645 [Candidate]" term=3 +2025-03-20T03:51:09.904Z [ERROR] server-f49f0bd9-4faf-2546-4781-849927a81645: failed to make requestVote RPC: target="{Voter server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e 9abd7df2-304f-3b7d-c2ab-cbe968a7817e}" error="failed to connect to peer: 9abd7df2-304f-3b7d-c2ab-cbe968a7817e" term=3 +2025-03-20T03:51:09.908Z [WARN] server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:09.908Z [INFO] server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e: entering candidate state: node="Node at 9abd7df2-304f-3b7d-c2ab-cbe968a7817e [Candidate]" term=3 +2025-03-20T03:51:09.908Z [ERROR] server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e: failed to make requestVote RPC: target="{Voter server-f49f0bd9-4faf-2546-4781-849927a81645 f49f0bd9-4faf-2546-4781-849927a81645}" error="failed to connect to peer: f49f0bd9-4faf-2546-4781-849927a81645" term=3 +2025-03-20T03:51:09.961Z [WARN] server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e: Election timeout reached, restarting election +2025-03-20T03:51:09.961Z [INFO] server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e: entering candidate state: node="Node at 9abd7df2-304f-3b7d-c2ab-cbe968a7817e [Candidate]" term=4 +2025-03-20T03:51:09.961Z [ERROR] server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e: failed to make requestVote RPC: target="{Voter server-f49f0bd9-4faf-2546-4781-849927a81645 f49f0bd9-4faf-2546-4781-849927a81645}" error="failed to connect to peer: f49f0bd9-4faf-2546-4781-849927a81645" term=4 +2025-03-20T03:51:09.991Z [WARN] server-f49f0bd9-4faf-2546-4781-849927a81645: Election timeout reached, restarting election +2025-03-20T03:51:09.991Z [INFO] server-f49f0bd9-4faf-2546-4781-849927a81645: entering candidate state: node="Node at f49f0bd9-4faf-2546-4781-849927a81645 [Candidate]" term=4 +2025-03-20T03:51:09.991Z [ERROR] server-f49f0bd9-4faf-2546-4781-849927a81645: failed to make requestVote RPC: target="{Voter server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e 9abd7df2-304f-3b7d-c2ab-cbe968a7817e}" error="failed to connect to peer: 9abd7df2-304f-3b7d-c2ab-cbe968a7817e" term=4 +2025-03-20T03:51:10.015Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:09.908646576 +0000 UTC m=+101.583099449 (0 nodes), 69.147766ms from start of poll, 391.162334ms from cluster start. Timeout at 2025-03-20 03:51:10.01372626 +0000 UTC m=+101.688179158, 105.079709ms after stability +2025-03-20T03:51:10.040Z [WARN] server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e: Election timeout reached, restarting election +2025-03-20T03:51:10.040Z [INFO] server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e: entering candidate state: node="Node at 9abd7df2-304f-3b7d-c2ab-cbe968a7817e [Candidate]" term=5 +2025-03-20T03:51:10.041Z [ERROR] server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e: failed to make requestVote RPC: target="{Voter server-f49f0bd9-4faf-2546-4781-849927a81645 f49f0bd9-4faf-2546-4781-849927a81645}" error="failed to connect to peer: f49f0bd9-4faf-2546-4781-849927a81645" term=5 +2025-03-20T03:51:10.045Z [WARN] server-f49f0bd9-4faf-2546-4781-849927a81645: Election timeout reached, restarting election +2025-03-20T03:51:10.045Z [INFO] server-f49f0bd9-4faf-2546-4781-849927a81645: entering candidate state: node="Node at f49f0bd9-4faf-2546-4781-849927a81645 [Candidate]" term=5 +2025-03-20T03:51:10.045Z [ERROR] server-f49f0bd9-4faf-2546-4781-849927a81645: failed to make requestVote RPC: target="{Voter server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e 9abd7df2-304f-3b7d-c2ab-cbe968a7817e}" error="failed to connect to peer: 9abd7df2-304f-3b7d-c2ab-cbe968a7817e" term=5 +2025-03-20T03:51:10.109Z [WARN] server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e: Election timeout reached, restarting election +2025-03-20T03:51:10.109Z [INFO] server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e: entering candidate state: node="Node at 9abd7df2-304f-3b7d-c2ab-cbe968a7817e [Candidate]" term=6 +2025-03-20T03:51:10.109Z [ERROR] server-9abd7df2-304f-3b7d-c2ab-cbe968a7817e: failed to make requestVote RPC: target="{Voter server-f49f0bd9-4faf-2546-4781-849927a81645 f49f0bd9-4faf-2546-4781-849927a81645}" error="failed to connect to peer: f49f0bd9-4faf-2546-4781-849927a81645" term=6 +--- PASS: TestRaft_LeaderLeaseExpire (0.61s) === RUN TestRaft_Barrier -2026-04-22T10:10:53.048Z [INFO] server-bc8b56ec-3caa-7235-ef45-e0293dd4bc2c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bc8b56ec-3caa-7235-ef45-e0293dd4bc2c Address:bc8b56ec-3caa-7235-ef45-e0293dd4bc2c} {Suffrage:Voter ID:server-c4334f10-15ef-328b-d767-8b1455a04f84 Address:c4334f10-15ef-328b-d767-8b1455a04f84} {Suffrage:Voter ID:server-f9c50256-5530-39e2-8527-8b41cefa7f55 Address:f9c50256-5530-39e2-8527-8b41cefa7f55}]" -2026-04-22T10:10:53.048Z [INFO] server-bc8b56ec-3caa-7235-ef45-e0293dd4bc2c: entering follower state: follower="Node at bc8b56ec-3caa-7235-ef45-e0293dd4bc2c [Follower]" leader-address= leader-id= -2026-04-22T10:10:53.048Z [INFO] server-c4334f10-15ef-328b-d767-8b1455a04f84: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bc8b56ec-3caa-7235-ef45-e0293dd4bc2c Address:bc8b56ec-3caa-7235-ef45-e0293dd4bc2c} {Suffrage:Voter ID:server-c4334f10-15ef-328b-d767-8b1455a04f84 Address:c4334f10-15ef-328b-d767-8b1455a04f84} {Suffrage:Voter ID:server-f9c50256-5530-39e2-8527-8b41cefa7f55 Address:f9c50256-5530-39e2-8527-8b41cefa7f55}]" -2026-04-22T10:10:53.049Z [INFO] server-c4334f10-15ef-328b-d767-8b1455a04f84: entering follower state: follower="Node at c4334f10-15ef-328b-d767-8b1455a04f84 [Follower]" leader-address= leader-id= -2026-04-22T10:10:53.049Z [INFO] server-f9c50256-5530-39e2-8527-8b41cefa7f55: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bc8b56ec-3caa-7235-ef45-e0293dd4bc2c Address:bc8b56ec-3caa-7235-ef45-e0293dd4bc2c} {Suffrage:Voter ID:server-c4334f10-15ef-328b-d767-8b1455a04f84 Address:c4334f10-15ef-328b-d767-8b1455a04f84} {Suffrage:Voter ID:server-f9c50256-5530-39e2-8527-8b41cefa7f55 Address:f9c50256-5530-39e2-8527-8b41cefa7f55}]" -2026-04-22T10:10:53.049Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:53.049Z [INFO] server-f9c50256-5530-39e2-8527-8b41cefa7f55: entering follower state: follower="Node at f9c50256-5530-39e2-8527-8b41cefa7f55 [Follower]" leader-address= leader-id= -2026-04-22T10:10:53.099Z [WARN] server-f9c50256-5530-39e2-8527-8b41cefa7f55: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:53.099Z [INFO] server-f9c50256-5530-39e2-8527-8b41cefa7f55: entering candidate state: node="Node at f9c50256-5530-39e2-8527-8b41cefa7f55 [Candidate]" term=2 -2026-04-22T10:10:53.099Z [INFO] server-f9c50256-5530-39e2-8527-8b41cefa7f55: election won: term=2 tally=2 -2026-04-22T10:10:53.099Z [INFO] server-f9c50256-5530-39e2-8527-8b41cefa7f55: entering leader state: leader="Node at f9c50256-5530-39e2-8527-8b41cefa7f55 [Leader]" -2026-04-22T10:10:53.099Z [INFO] server-f9c50256-5530-39e2-8527-8b41cefa7f55: added peer, starting replication: peer=server-bc8b56ec-3caa-7235-ef45-e0293dd4bc2c -2026-04-22T10:10:53.099Z [INFO] server-f9c50256-5530-39e2-8527-8b41cefa7f55: added peer, starting replication: peer=server-c4334f10-15ef-328b-d767-8b1455a04f84 -2026-04-22T10:10:53.099Z [INFO] server-f9c50256-5530-39e2-8527-8b41cefa7f55: pipelining replication: peer="{Voter server-bc8b56ec-3caa-7235-ef45-e0293dd4bc2c bc8b56ec-3caa-7235-ef45-e0293dd4bc2c}" -2026-04-22T10:10:53.099Z [INFO] server-f9c50256-5530-39e2-8527-8b41cefa7f55: pipelining replication: peer="{Voter server-c4334f10-15ef-328b-d767-8b1455a04f84 c4334f10-15ef-328b-d767-8b1455a04f84}" -2026-04-22T10:10:53.206Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:53.099856083 +0000 UTC m=+99.123652172 (1 nodes), 50.743695ms from start of poll, 51.008784ms from cluster start. Timeout at 2026-04-22 10:10:53.204857965 +0000 UTC m=+99.228654105, 105.001933ms after stability -2026-04-22T10:10:53.222Z [INFO] server-f9c50256-5530-39e2-8527-8b41cefa7f55: aborting pipeline replication: peer="{Voter server-bc8b56ec-3caa-7235-ef45-e0293dd4bc2c bc8b56ec-3caa-7235-ef45-e0293dd4bc2c}" -2026-04-22T10:10:53.222Z [INFO] server-f9c50256-5530-39e2-8527-8b41cefa7f55: aborting pipeline replication: peer="{Voter server-c4334f10-15ef-328b-d767-8b1455a04f84 c4334f10-15ef-328b-d767-8b1455a04f84}" ---- PASS: TestRaft_Barrier (0.17s) +2025-03-20T03:51:10.123Z [INFO] server-f6330971-d8ca-ba0c-276a-e6a561e30936: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f6330971-d8ca-ba0c-276a-e6a561e30936 Address:f6330971-d8ca-ba0c-276a-e6a561e30936} {Suffrage:Voter ID:server-57187d1e-43d5-a7d7-6aab-de10003a1d7b Address:57187d1e-43d5-a7d7-6aab-de10003a1d7b} {Suffrage:Voter ID:server-e38e87e2-e0e3-4d2e-1b3e-fad3b9e6bc89 Address:e38e87e2-e0e3-4d2e-1b3e-fad3b9e6bc89}]" +2025-03-20T03:51:10.123Z [INFO] server-57187d1e-43d5-a7d7-6aab-de10003a1d7b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f6330971-d8ca-ba0c-276a-e6a561e30936 Address:f6330971-d8ca-ba0c-276a-e6a561e30936} {Suffrage:Voter ID:server-57187d1e-43d5-a7d7-6aab-de10003a1d7b Address:57187d1e-43d5-a7d7-6aab-de10003a1d7b} {Suffrage:Voter ID:server-e38e87e2-e0e3-4d2e-1b3e-fad3b9e6bc89 Address:e38e87e2-e0e3-4d2e-1b3e-fad3b9e6bc89}]" +2025-03-20T03:51:10.123Z [INFO] server-e38e87e2-e0e3-4d2e-1b3e-fad3b9e6bc89: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f6330971-d8ca-ba0c-276a-e6a561e30936 Address:f6330971-d8ca-ba0c-276a-e6a561e30936} {Suffrage:Voter ID:server-57187d1e-43d5-a7d7-6aab-de10003a1d7b Address:57187d1e-43d5-a7d7-6aab-de10003a1d7b} {Suffrage:Voter ID:server-e38e87e2-e0e3-4d2e-1b3e-fad3b9e6bc89 Address:e38e87e2-e0e3-4d2e-1b3e-fad3b9e6bc89}]" +2025-03-20T03:51:10.123Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:10.123Z [INFO] server-f6330971-d8ca-ba0c-276a-e6a561e30936: entering follower state: follower="Node at f6330971-d8ca-ba0c-276a-e6a561e30936 [Follower]" leader-address= leader-id= +2025-03-20T03:51:10.123Z [INFO] server-57187d1e-43d5-a7d7-6aab-de10003a1d7b: entering follower state: follower="Node at 57187d1e-43d5-a7d7-6aab-de10003a1d7b [Follower]" leader-address= leader-id= +2025-03-20T03:51:10.123Z [INFO] server-e38e87e2-e0e3-4d2e-1b3e-fad3b9e6bc89: entering follower state: follower="Node at e38e87e2-e0e3-4d2e-1b3e-fad3b9e6bc89 [Follower]" leader-address= leader-id= +2025-03-20T03:51:10.194Z [WARN] server-f6330971-d8ca-ba0c-276a-e6a561e30936: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:10.194Z [INFO] server-f6330971-d8ca-ba0c-276a-e6a561e30936: entering candidate state: node="Node at f6330971-d8ca-ba0c-276a-e6a561e30936 [Candidate]" term=2 +2025-03-20T03:51:10.194Z [INFO] server-f6330971-d8ca-ba0c-276a-e6a561e30936: election won: term=2 tally=2 +2025-03-20T03:51:10.194Z [INFO] server-f6330971-d8ca-ba0c-276a-e6a561e30936: entering leader state: leader="Node at f6330971-d8ca-ba0c-276a-e6a561e30936 [Leader]" +2025-03-20T03:51:10.194Z [INFO] server-f6330971-d8ca-ba0c-276a-e6a561e30936: added peer, starting replication: peer=server-57187d1e-43d5-a7d7-6aab-de10003a1d7b +2025-03-20T03:51:10.194Z [INFO] server-f6330971-d8ca-ba0c-276a-e6a561e30936: added peer, starting replication: peer=server-e38e87e2-e0e3-4d2e-1b3e-fad3b9e6bc89 +2025-03-20T03:51:10.194Z [INFO] server-f6330971-d8ca-ba0c-276a-e6a561e30936: pipelining replication: peer="{Voter server-e38e87e2-e0e3-4d2e-1b3e-fad3b9e6bc89 e38e87e2-e0e3-4d2e-1b3e-fad3b9e6bc89}" +2025-03-20T03:51:10.194Z [INFO] server-f6330971-d8ca-ba0c-276a-e6a561e30936: pipelining replication: peer="{Voter server-57187d1e-43d5-a7d7-6aab-de10003a1d7b 57187d1e-43d5-a7d7-6aab-de10003a1d7b}" +2025-03-20T03:51:10.299Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:10.194438159 +0000 UTC m=+101.868891189 (1 nodes), 71.029741ms from start of poll, 71.595972ms from cluster start. Timeout at 2025-03-20 03:51:10.299441369 +0000 UTC m=+101.973894308, 105.003119ms after stability +2025-03-20T03:51:10.325Z [INFO] server-f6330971-d8ca-ba0c-276a-e6a561e30936: aborting pipeline replication: peer="{Voter server-e38e87e2-e0e3-4d2e-1b3e-fad3b9e6bc89 e38e87e2-e0e3-4d2e-1b3e-fad3b9e6bc89}" +2025-03-20T03:51:10.325Z [INFO] server-f6330971-d8ca-ba0c-276a-e6a561e30936: aborting pipeline replication: peer="{Voter server-57187d1e-43d5-a7d7-6aab-de10003a1d7b 57187d1e-43d5-a7d7-6aab-de10003a1d7b}" +2025-03-20T03:51:10.825Z [ERROR] server-f6330971-d8ca-ba0c-276a-e6a561e30936: failed to heartbeat to: peer=57187d1e-43d5-a7d7-6aab-de10003a1d7b backoff time=10ms error="command timed out" +--- PASS: TestRaft_Barrier (0.70s) === RUN TestRaft_VerifyLeader -2026-04-22T10:10:53.223Z [INFO] server-d3f15a9d-76f2-d9ff-fbbf-c885d1c91572: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d3f15a9d-76f2-d9ff-fbbf-c885d1c91572 Address:d3f15a9d-76f2-d9ff-fbbf-c885d1c91572} {Suffrage:Voter ID:server-94a46356-ce81-0174-d4ff-31db9ca2cc9b Address:94a46356-ce81-0174-d4ff-31db9ca2cc9b} {Suffrage:Voter ID:server-771c181c-f7e9-7f10-6fe1-fc6333e5e63f Address:771c181c-f7e9-7f10-6fe1-fc6333e5e63f}]" -2026-04-22T10:10:53.223Z [INFO] server-d3f15a9d-76f2-d9ff-fbbf-c885d1c91572: entering follower state: follower="Node at d3f15a9d-76f2-d9ff-fbbf-c885d1c91572 [Follower]" leader-address= leader-id= -2026-04-22T10:10:53.223Z [INFO] server-94a46356-ce81-0174-d4ff-31db9ca2cc9b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d3f15a9d-76f2-d9ff-fbbf-c885d1c91572 Address:d3f15a9d-76f2-d9ff-fbbf-c885d1c91572} {Suffrage:Voter ID:server-94a46356-ce81-0174-d4ff-31db9ca2cc9b Address:94a46356-ce81-0174-d4ff-31db9ca2cc9b} {Suffrage:Voter ID:server-771c181c-f7e9-7f10-6fe1-fc6333e5e63f Address:771c181c-f7e9-7f10-6fe1-fc6333e5e63f}]" -2026-04-22T10:10:53.223Z [INFO] server-94a46356-ce81-0174-d4ff-31db9ca2cc9b: entering follower state: follower="Node at 94a46356-ce81-0174-d4ff-31db9ca2cc9b [Follower]" leader-address= leader-id= -2026-04-22T10:10:53.223Z [INFO] server-771c181c-f7e9-7f10-6fe1-fc6333e5e63f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d3f15a9d-76f2-d9ff-fbbf-c885d1c91572 Address:d3f15a9d-76f2-d9ff-fbbf-c885d1c91572} {Suffrage:Voter ID:server-94a46356-ce81-0174-d4ff-31db9ca2cc9b Address:94a46356-ce81-0174-d4ff-31db9ca2cc9b} {Suffrage:Voter ID:server-771c181c-f7e9-7f10-6fe1-fc6333e5e63f Address:771c181c-f7e9-7f10-6fe1-fc6333e5e63f}]" -2026-04-22T10:10:53.223Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:53.223Z [INFO] server-771c181c-f7e9-7f10-6fe1-fc6333e5e63f: entering follower state: follower="Node at 771c181c-f7e9-7f10-6fe1-fc6333e5e63f [Follower]" leader-address= leader-id= -2026-04-22T10:10:53.284Z [WARN] server-94a46356-ce81-0174-d4ff-31db9ca2cc9b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:53.284Z [INFO] server-94a46356-ce81-0174-d4ff-31db9ca2cc9b: entering candidate state: node="Node at 94a46356-ce81-0174-d4ff-31db9ca2cc9b [Candidate]" term=2 -2026-04-22T10:10:53.284Z [INFO] server-94a46356-ce81-0174-d4ff-31db9ca2cc9b: election won: term=2 tally=2 -2026-04-22T10:10:53.284Z [INFO] server-94a46356-ce81-0174-d4ff-31db9ca2cc9b: entering leader state: leader="Node at 94a46356-ce81-0174-d4ff-31db9ca2cc9b [Leader]" -2026-04-22T10:10:53.284Z [INFO] server-94a46356-ce81-0174-d4ff-31db9ca2cc9b: added peer, starting replication: peer=server-d3f15a9d-76f2-d9ff-fbbf-c885d1c91572 -2026-04-22T10:10:53.284Z [INFO] server-94a46356-ce81-0174-d4ff-31db9ca2cc9b: added peer, starting replication: peer=server-771c181c-f7e9-7f10-6fe1-fc6333e5e63f -2026-04-22T10:10:53.284Z [INFO] server-94a46356-ce81-0174-d4ff-31db9ca2cc9b: pipelining replication: peer="{Voter server-d3f15a9d-76f2-d9ff-fbbf-c885d1c91572 d3f15a9d-76f2-d9ff-fbbf-c885d1c91572}" -2026-04-22T10:10:53.284Z [INFO] server-94a46356-ce81-0174-d4ff-31db9ca2cc9b: pipelining replication: peer="{Voter server-771c181c-f7e9-7f10-6fe1-fc6333e5e63f 771c181c-f7e9-7f10-6fe1-fc6333e5e63f}" -2026-04-22T10:10:53.390Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:53.284732067 +0000 UTC m=+99.308528154 (1 nodes), 60.957535ms from start of poll, 61.23912ms from cluster start. Timeout at 2026-04-22 10:10:53.389736113 +0000 UTC m=+99.413532246, 105.004092ms after stability -2026-04-22T10:10:53.390Z [INFO] server-94a46356-ce81-0174-d4ff-31db9ca2cc9b: aborting pipeline replication: peer="{Voter server-771c181c-f7e9-7f10-6fe1-fc6333e5e63f 771c181c-f7e9-7f10-6fe1-fc6333e5e63f}" -2026-04-22T10:10:53.390Z [INFO] server-94a46356-ce81-0174-d4ff-31db9ca2cc9b: aborting pipeline replication: peer="{Voter server-d3f15a9d-76f2-d9ff-fbbf-c885d1c91572 d3f15a9d-76f2-d9ff-fbbf-c885d1c91572}" ---- PASS: TestRaft_VerifyLeader (0.17s) +2025-03-20T03:51:10.827Z [INFO] server-31d551e3-d450-72c1-37ec-bce86189c677: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31d551e3-d450-72c1-37ec-bce86189c677 Address:31d551e3-d450-72c1-37ec-bce86189c677} {Suffrage:Voter ID:server-4983cfdc-e10b-7f9e-b744-e8ea662e390c Address:4983cfdc-e10b-7f9e-b744-e8ea662e390c} {Suffrage:Voter ID:server-55687050-4714-e44f-39ce-61639383dcbb Address:55687050-4714-e44f-39ce-61639383dcbb}]" +2025-03-20T03:51:10.827Z [INFO] server-4983cfdc-e10b-7f9e-b744-e8ea662e390c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31d551e3-d450-72c1-37ec-bce86189c677 Address:31d551e3-d450-72c1-37ec-bce86189c677} {Suffrage:Voter ID:server-4983cfdc-e10b-7f9e-b744-e8ea662e390c Address:4983cfdc-e10b-7f9e-b744-e8ea662e390c} {Suffrage:Voter ID:server-55687050-4714-e44f-39ce-61639383dcbb Address:55687050-4714-e44f-39ce-61639383dcbb}]" +2025-03-20T03:51:10.827Z [INFO] server-55687050-4714-e44f-39ce-61639383dcbb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31d551e3-d450-72c1-37ec-bce86189c677 Address:31d551e3-d450-72c1-37ec-bce86189c677} {Suffrage:Voter ID:server-4983cfdc-e10b-7f9e-b744-e8ea662e390c Address:4983cfdc-e10b-7f9e-b744-e8ea662e390c} {Suffrage:Voter ID:server-55687050-4714-e44f-39ce-61639383dcbb Address:55687050-4714-e44f-39ce-61639383dcbb}]" +2025-03-20T03:51:10.828Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:10.828Z [INFO] server-31d551e3-d450-72c1-37ec-bce86189c677: entering follower state: follower="Node at 31d551e3-d450-72c1-37ec-bce86189c677 [Follower]" leader-address= leader-id= +2025-03-20T03:51:10.828Z [INFO] server-4983cfdc-e10b-7f9e-b744-e8ea662e390c: entering follower state: follower="Node at 4983cfdc-e10b-7f9e-b744-e8ea662e390c [Follower]" leader-address= leader-id= +2025-03-20T03:51:10.828Z [INFO] server-55687050-4714-e44f-39ce-61639383dcbb: entering follower state: follower="Node at 55687050-4714-e44f-39ce-61639383dcbb [Follower]" leader-address= leader-id= +2025-03-20T03:51:10.903Z [WARN] server-31d551e3-d450-72c1-37ec-bce86189c677: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:10.903Z [INFO] server-31d551e3-d450-72c1-37ec-bce86189c677: entering candidate state: node="Node at 31d551e3-d450-72c1-37ec-bce86189c677 [Candidate]" term=2 +2025-03-20T03:51:10.904Z [INFO] server-31d551e3-d450-72c1-37ec-bce86189c677: election won: term=2 tally=2 +2025-03-20T03:51:10.904Z [INFO] server-31d551e3-d450-72c1-37ec-bce86189c677: entering leader state: leader="Node at 31d551e3-d450-72c1-37ec-bce86189c677 [Leader]" +2025-03-20T03:51:10.904Z [INFO] server-31d551e3-d450-72c1-37ec-bce86189c677: added peer, starting replication: peer=server-4983cfdc-e10b-7f9e-b744-e8ea662e390c +2025-03-20T03:51:10.904Z [INFO] server-31d551e3-d450-72c1-37ec-bce86189c677: added peer, starting replication: peer=server-55687050-4714-e44f-39ce-61639383dcbb +2025-03-20T03:51:10.904Z [INFO] server-31d551e3-d450-72c1-37ec-bce86189c677: pipelining replication: peer="{Voter server-55687050-4714-e44f-39ce-61639383dcbb 55687050-4714-e44f-39ce-61639383dcbb}" +2025-03-20T03:51:10.904Z [INFO] server-31d551e3-d450-72c1-37ec-bce86189c677: pipelining replication: peer="{Voter server-4983cfdc-e10b-7f9e-b744-e8ea662e390c 4983cfdc-e10b-7f9e-b744-e8ea662e390c}" +2025-03-20T03:51:11.009Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:10.904197429 +0000 UTC m=+102.578650257 (1 nodes), 76.152124ms from start of poll, 76.57674ms from cluster start. Timeout at 2025-03-20 03:51:11.009208589 +0000 UTC m=+102.683661591, 105.011334ms after stability +2025-03-20T03:51:11.009Z [INFO] server-31d551e3-d450-72c1-37ec-bce86189c677: aborting pipeline replication: peer="{Voter server-4983cfdc-e10b-7f9e-b744-e8ea662e390c 4983cfdc-e10b-7f9e-b744-e8ea662e390c}" +2025-03-20T03:51:11.009Z [INFO] server-31d551e3-d450-72c1-37ec-bce86189c677: aborting pipeline replication: peer="{Voter server-55687050-4714-e44f-39ce-61639383dcbb 55687050-4714-e44f-39ce-61639383dcbb}" +--- PASS: TestRaft_VerifyLeader (0.18s) === RUN TestRaft_VerifyLeader_Single -2026-04-22T10:10:53.391Z [INFO] server-6b0c8df9-eb4e-e6c5-71b1-6dc84bd7da19: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6b0c8df9-eb4e-e6c5-71b1-6dc84bd7da19 Address:6b0c8df9-eb4e-e6c5-71b1-6dc84bd7da19}]" -2026-04-22T10:10:53.391Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:53.391Z [INFO] server-6b0c8df9-eb4e-e6c5-71b1-6dc84bd7da19: entering follower state: follower="Node at 6b0c8df9-eb4e-e6c5-71b1-6dc84bd7da19 [Follower]" leader-address= leader-id= -2026-04-22T10:10:53.465Z [WARN] server-6b0c8df9-eb4e-e6c5-71b1-6dc84bd7da19: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:53.465Z [INFO] server-6b0c8df9-eb4e-e6c5-71b1-6dc84bd7da19: entering candidate state: node="Node at 6b0c8df9-eb4e-e6c5-71b1-6dc84bd7da19 [Candidate]" term=2 -2026-04-22T10:10:53.466Z [INFO] server-6b0c8df9-eb4e-e6c5-71b1-6dc84bd7da19: election won: term=2 tally=1 -2026-04-22T10:10:53.466Z [INFO] server-6b0c8df9-eb4e-e6c5-71b1-6dc84bd7da19: entering leader state: leader="Node at 6b0c8df9-eb4e-e6c5-71b1-6dc84bd7da19 [Leader]" -2026-04-22T10:10:53.571Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:53.466102731 +0000 UTC m=+99.489898881 (1 nodes), 74.961374ms from start of poll, 75.059268ms from cluster start. Timeout at 2026-04-22 10:10:53.571113034 +0000 UTC m=+99.594909168, 105.010287ms after stability ---- PASS: TestRaft_VerifyLeader_Single (0.18s) +2025-03-20T03:51:11.011Z [INFO] server-ff2aa3dd-6059-2d21-48bc-2e21561bd5b5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff2aa3dd-6059-2d21-48bc-2e21561bd5b5 Address:ff2aa3dd-6059-2d21-48bc-2e21561bd5b5}]" +2025-03-20T03:51:11.011Z [INFO] server-ff2aa3dd-6059-2d21-48bc-2e21561bd5b5: entering follower state: follower="Node at ff2aa3dd-6059-2d21-48bc-2e21561bd5b5 [Follower]" leader-address= leader-id= +2025-03-20T03:51:11.012Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:11.107Z [WARN] server-ff2aa3dd-6059-2d21-48bc-2e21561bd5b5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:11.108Z [INFO] server-ff2aa3dd-6059-2d21-48bc-2e21561bd5b5: entering candidate state: node="Node at ff2aa3dd-6059-2d21-48bc-2e21561bd5b5 [Candidate]" term=2 +2025-03-20T03:51:11.108Z [INFO] server-ff2aa3dd-6059-2d21-48bc-2e21561bd5b5: election won: term=2 tally=1 +2025-03-20T03:51:11.108Z [INFO] server-ff2aa3dd-6059-2d21-48bc-2e21561bd5b5: entering leader state: leader="Node at ff2aa3dd-6059-2d21-48bc-2e21561bd5b5 [Leader]" +2025-03-20T03:51:11.213Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:11.108255045 +0000 UTC m=+102.782707920 (1 nodes), 96.064249ms from start of poll, 96.992433ms from cluster start. Timeout at 2025-03-20 03:51:11.21326042 +0000 UTC m=+102.887713288, 105.005368ms after stability +--- PASS: TestRaft_VerifyLeader_Single (0.20s) === RUN TestRaft_VerifyLeader_Fail -2026-04-22T10:10:53.572Z [INFO] server-8e8d2309-46e5-3dd4-9056-3f6a2ce8ee18: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8e8d2309-46e5-3dd4-9056-3f6a2ce8ee18 Address:8e8d2309-46e5-3dd4-9056-3f6a2ce8ee18} {Suffrage:Voter ID:server-5d7e175b-3019-ada3-d2b3-6219bc2cb083 Address:5d7e175b-3019-ada3-d2b3-6219bc2cb083}]" -2026-04-22T10:10:53.572Z [INFO] server-8e8d2309-46e5-3dd4-9056-3f6a2ce8ee18: entering follower state: follower="Node at 8e8d2309-46e5-3dd4-9056-3f6a2ce8ee18 [Follower]" leader-address= leader-id= -2026-04-22T10:10:53.572Z [INFO] server-5d7e175b-3019-ada3-d2b3-6219bc2cb083: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8e8d2309-46e5-3dd4-9056-3f6a2ce8ee18 Address:8e8d2309-46e5-3dd4-9056-3f6a2ce8ee18} {Suffrage:Voter ID:server-5d7e175b-3019-ada3-d2b3-6219bc2cb083 Address:5d7e175b-3019-ada3-d2b3-6219bc2cb083}]" -2026-04-22T10:10:53.572Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:53.572Z [INFO] server-5d7e175b-3019-ada3-d2b3-6219bc2cb083: entering follower state: follower="Node at 5d7e175b-3019-ada3-d2b3-6219bc2cb083 [Follower]" leader-address= leader-id= -2026-04-22T10:10:53.662Z [WARN] server-8e8d2309-46e5-3dd4-9056-3f6a2ce8ee18: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:53.662Z [INFO] server-8e8d2309-46e5-3dd4-9056-3f6a2ce8ee18: entering candidate state: node="Node at 8e8d2309-46e5-3dd4-9056-3f6a2ce8ee18 [Candidate]" term=2 -2026-04-22T10:10:53.662Z [INFO] server-8e8d2309-46e5-3dd4-9056-3f6a2ce8ee18: election won: term=2 tally=2 -2026-04-22T10:10:53.662Z [INFO] server-8e8d2309-46e5-3dd4-9056-3f6a2ce8ee18: entering leader state: leader="Node at 8e8d2309-46e5-3dd4-9056-3f6a2ce8ee18 [Leader]" -2026-04-22T10:10:53.662Z [INFO] server-8e8d2309-46e5-3dd4-9056-3f6a2ce8ee18: added peer, starting replication: peer=server-5d7e175b-3019-ada3-d2b3-6219bc2cb083 -2026-04-22T10:10:53.663Z [INFO] server-8e8d2309-46e5-3dd4-9056-3f6a2ce8ee18: pipelining replication: peer="{Voter server-5d7e175b-3019-ada3-d2b3-6219bc2cb083 5d7e175b-3019-ada3-d2b3-6219bc2cb083}" -2026-04-22T10:10:53.768Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:53.662910744 +0000 UTC m=+99.686706847 (1 nodes), 90.770779ms from start of poll, 90.954097ms from cluster start. Timeout at 2026-04-22 10:10:53.767912038 +0000 UTC m=+99.791708122, 105.001275ms after stability -2026-04-22T10:10:53.768Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:10:53.874Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:10:53.768822708 +0000 UTC m=+99.792618795 (1 nodes), 549ns from start of poll, 196.866045ms from cluster start. Timeout at 2026-04-22 10:10:53.873837886 +0000 UTC m=+99.897634123, 105.015328ms after stability -2026-04-22T10:10:53.882Z [ERROR] server-8e8d2309-46e5-3dd4-9056-3f6a2ce8ee18: peer has newer term, stopping replication: peer="{Voter server-5d7e175b-3019-ada3-d2b3-6219bc2cb083 5d7e175b-3019-ada3-d2b3-6219bc2cb083}" -2026-04-22T10:10:53.882Z [INFO] server-8e8d2309-46e5-3dd4-9056-3f6a2ce8ee18: aborting pipeline replication: peer="{Voter server-5d7e175b-3019-ada3-d2b3-6219bc2cb083 5d7e175b-3019-ada3-d2b3-6219bc2cb083}" -2026-04-22T10:10:53.882Z [INFO] server-8e8d2309-46e5-3dd4-9056-3f6a2ce8ee18: entering follower state: follower="Node at 8e8d2309-46e5-3dd4-9056-3f6a2ce8ee18 [Follower]" leader-address= leader-id= ---- PASS: TestRaft_VerifyLeader_Fail (0.31s) +2025-03-20T03:51:11.214Z [INFO] server-96929157-4a55-060b-86fa-71dd1e75306a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-96929157-4a55-060b-86fa-71dd1e75306a Address:96929157-4a55-060b-86fa-71dd1e75306a} {Suffrage:Voter ID:server-4d01c0ec-b710-d133-ea50-2970a78956a8 Address:4d01c0ec-b710-d133-ea50-2970a78956a8}]" +2025-03-20T03:51:11.214Z [INFO] server-4d01c0ec-b710-d133-ea50-2970a78956a8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-96929157-4a55-060b-86fa-71dd1e75306a Address:96929157-4a55-060b-86fa-71dd1e75306a} {Suffrage:Voter ID:server-4d01c0ec-b710-d133-ea50-2970a78956a8 Address:4d01c0ec-b710-d133-ea50-2970a78956a8}]" +2025-03-20T03:51:11.215Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:11.215Z [INFO] server-96929157-4a55-060b-86fa-71dd1e75306a: entering follower state: follower="Node at 96929157-4a55-060b-86fa-71dd1e75306a [Follower]" leader-address= leader-id= +2025-03-20T03:51:11.215Z [INFO] server-4d01c0ec-b710-d133-ea50-2970a78956a8: entering follower state: follower="Node at 4d01c0ec-b710-d133-ea50-2970a78956a8 [Follower]" leader-address= leader-id= +2025-03-20T03:51:11.291Z [WARN] server-96929157-4a55-060b-86fa-71dd1e75306a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:11.291Z [INFO] server-96929157-4a55-060b-86fa-71dd1e75306a: entering candidate state: node="Node at 96929157-4a55-060b-86fa-71dd1e75306a [Candidate]" term=2 +2025-03-20T03:51:11.291Z [INFO] server-96929157-4a55-060b-86fa-71dd1e75306a: election won: term=2 tally=2 +2025-03-20T03:51:11.291Z [INFO] server-96929157-4a55-060b-86fa-71dd1e75306a: entering leader state: leader="Node at 96929157-4a55-060b-86fa-71dd1e75306a [Leader]" +2025-03-20T03:51:11.291Z [INFO] server-96929157-4a55-060b-86fa-71dd1e75306a: added peer, starting replication: peer=server-4d01c0ec-b710-d133-ea50-2970a78956a8 +2025-03-20T03:51:11.292Z [INFO] server-96929157-4a55-060b-86fa-71dd1e75306a: pipelining replication: peer="{Voter server-4d01c0ec-b710-d133-ea50-2970a78956a8 4d01c0ec-b710-d133-ea50-2970a78956a8}" +2025-03-20T03:51:11.397Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:11.292196419 +0000 UTC m=+102.966649282 (1 nodes), 77.15355ms from start of poll, 77.483741ms from cluster start. Timeout at 2025-03-20 03:51:11.397203691 +0000 UTC m=+103.071656591, 105.007309ms after stability +2025-03-20T03:51:11.397Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:51:11.503Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:51:11.397744341 +0000 UTC m=+103.072197237 (1 nodes), 2.243µs from start of poll, 183.031696ms from cluster start. Timeout at 2025-03-20 03:51:11.502829533 +0000 UTC m=+103.177282469, 105.085232ms after stability +2025-03-20T03:51:11.509Z [ERROR] server-96929157-4a55-060b-86fa-71dd1e75306a: peer has newer term, stopping replication: peer="{Voter server-4d01c0ec-b710-d133-ea50-2970a78956a8 4d01c0ec-b710-d133-ea50-2970a78956a8}" +2025-03-20T03:51:11.509Z [INFO] server-96929157-4a55-060b-86fa-71dd1e75306a: aborting pipeline replication: peer="{Voter server-4d01c0ec-b710-d133-ea50-2970a78956a8 4d01c0ec-b710-d133-ea50-2970a78956a8}" +2025-03-20T03:51:11.509Z [INFO] server-96929157-4a55-060b-86fa-71dd1e75306a: entering follower state: follower="Node at 96929157-4a55-060b-86fa-71dd1e75306a [Follower]" leader-address= leader-id= +--- PASS: TestRaft_VerifyLeader_Fail (0.30s) === RUN TestRaft_VerifyLeader_PartialConnect -2026-04-22T10:10:53.883Z [INFO] server-7b6b812b-4bb9-4a38-0720-899dd6c2d7d6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b6b812b-4bb9-4a38-0720-899dd6c2d7d6 Address:7b6b812b-4bb9-4a38-0720-899dd6c2d7d6} {Suffrage:Voter ID:server-873c0c5a-f7aa-bc11-17e3-fdcb955be0aa Address:873c0c5a-f7aa-bc11-17e3-fdcb955be0aa} {Suffrage:Voter ID:server-8bcd5410-fcb9-4916-e395-9140369fec2a Address:8bcd5410-fcb9-4916-e395-9140369fec2a}]" -2026-04-22T10:10:53.883Z [INFO] server-7b6b812b-4bb9-4a38-0720-899dd6c2d7d6: entering follower state: follower="Node at 7b6b812b-4bb9-4a38-0720-899dd6c2d7d6 [Follower]" leader-address= leader-id= -2026-04-22T10:10:53.883Z [INFO] server-873c0c5a-f7aa-bc11-17e3-fdcb955be0aa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b6b812b-4bb9-4a38-0720-899dd6c2d7d6 Address:7b6b812b-4bb9-4a38-0720-899dd6c2d7d6} {Suffrage:Voter ID:server-873c0c5a-f7aa-bc11-17e3-fdcb955be0aa Address:873c0c5a-f7aa-bc11-17e3-fdcb955be0aa} {Suffrage:Voter ID:server-8bcd5410-fcb9-4916-e395-9140369fec2a Address:8bcd5410-fcb9-4916-e395-9140369fec2a}]" -2026-04-22T10:10:53.883Z [INFO] server-873c0c5a-f7aa-bc11-17e3-fdcb955be0aa: entering follower state: follower="Node at 873c0c5a-f7aa-bc11-17e3-fdcb955be0aa [Follower]" leader-address= leader-id= -2026-04-22T10:10:53.883Z [INFO] server-8bcd5410-fcb9-4916-e395-9140369fec2a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b6b812b-4bb9-4a38-0720-899dd6c2d7d6 Address:7b6b812b-4bb9-4a38-0720-899dd6c2d7d6} {Suffrage:Voter ID:server-873c0c5a-f7aa-bc11-17e3-fdcb955be0aa Address:873c0c5a-f7aa-bc11-17e3-fdcb955be0aa} {Suffrage:Voter ID:server-8bcd5410-fcb9-4916-e395-9140369fec2a Address:8bcd5410-fcb9-4916-e395-9140369fec2a}]" -2026-04-22T10:10:53.883Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:53.883Z [INFO] server-8bcd5410-fcb9-4916-e395-9140369fec2a: entering follower state: follower="Node at 8bcd5410-fcb9-4916-e395-9140369fec2a [Follower]" leader-address= leader-id= -2026-04-22T10:10:53.941Z [WARN] server-7b6b812b-4bb9-4a38-0720-899dd6c2d7d6: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:53.941Z [INFO] server-7b6b812b-4bb9-4a38-0720-899dd6c2d7d6: entering candidate state: node="Node at 7b6b812b-4bb9-4a38-0720-899dd6c2d7d6 [Candidate]" term=2 -2026-04-22T10:10:53.941Z [INFO] server-7b6b812b-4bb9-4a38-0720-899dd6c2d7d6: election won: term=2 tally=2 -2026-04-22T10:10:53.941Z [INFO] server-7b6b812b-4bb9-4a38-0720-899dd6c2d7d6: entering leader state: leader="Node at 7b6b812b-4bb9-4a38-0720-899dd6c2d7d6 [Leader]" -2026-04-22T10:10:53.941Z [INFO] server-7b6b812b-4bb9-4a38-0720-899dd6c2d7d6: added peer, starting replication: peer=server-873c0c5a-f7aa-bc11-17e3-fdcb955be0aa -2026-04-22T10:10:53.941Z [INFO] server-7b6b812b-4bb9-4a38-0720-899dd6c2d7d6: added peer, starting replication: peer=server-8bcd5410-fcb9-4916-e395-9140369fec2a -2026-04-22T10:10:53.941Z [INFO] server-7b6b812b-4bb9-4a38-0720-899dd6c2d7d6: pipelining replication: peer="{Voter server-8bcd5410-fcb9-4916-e395-9140369fec2a 8bcd5410-fcb9-4916-e395-9140369fec2a}" -2026-04-22T10:10:53.941Z [INFO] server-7b6b812b-4bb9-4a38-0720-899dd6c2d7d6: pipelining replication: peer="{Voter server-873c0c5a-f7aa-bc11-17e3-fdcb955be0aa 873c0c5a-f7aa-bc11-17e3-fdcb955be0aa}" -2026-04-22T10:10:54.047Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:53.941564522 +0000 UTC m=+99.965360675 (1 nodes), 57.871882ms from start of poll, 58.178032ms from cluster start. Timeout at 2026-04-22 10:10:54.046565806 +0000 UTC m=+100.070361889, 105.001214ms after stability -2026-04-22T10:10:54.053Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:10:54.159Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:10:54.053573408 +0000 UTC m=+100.077369495 (2 nodes), 714ns from start of poll, 170.186852ms from cluster start. Timeout at 2026-04-22 10:10:54.158598009 +0000 UTC m=+100.182394117, 105.024622ms after stability - raft_test.go:1968: [INFO] Disconnecting Node at 873c0c5a-f7aa-bc11-17e3-fdcb955be0aa [Follower] -2026-04-22T10:10:54.159Z [ERROR] server-7b6b812b-4bb9-4a38-0720-899dd6c2d7d6: failed to heartbeat to: peer=873c0c5a-f7aa-bc11-17e3-fdcb955be0aa backoff time=10ms error="failed to connect to peer: 873c0c5a-f7aa-bc11-17e3-fdcb955be0aa" -2026-04-22T10:10:54.159Z [INFO] server-7b6b812b-4bb9-4a38-0720-899dd6c2d7d6: aborting pipeline replication: peer="{Voter server-8bcd5410-fcb9-4916-e395-9140369fec2a 8bcd5410-fcb9-4916-e395-9140369fec2a}" -2026-04-22T10:10:54.159Z [INFO] server-7b6b812b-4bb9-4a38-0720-899dd6c2d7d6: aborting pipeline replication: peer="{Voter server-873c0c5a-f7aa-bc11-17e3-fdcb955be0aa 873c0c5a-f7aa-bc11-17e3-fdcb955be0aa}" +2025-03-20T03:51:11.511Z [INFO] server-f35bbbc2-5f90-6523-17f3-9abd623f65a0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f35bbbc2-5f90-6523-17f3-9abd623f65a0 Address:f35bbbc2-5f90-6523-17f3-9abd623f65a0} {Suffrage:Voter ID:server-78126716-a533-0f36-e02d-56eb606d2b75 Address:78126716-a533-0f36-e02d-56eb606d2b75} {Suffrage:Voter ID:server-ab80983a-2d71-c183-078b-2be428891adb Address:ab80983a-2d71-c183-078b-2be428891adb}]" +2025-03-20T03:51:11.511Z [INFO] server-78126716-a533-0f36-e02d-56eb606d2b75: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f35bbbc2-5f90-6523-17f3-9abd623f65a0 Address:f35bbbc2-5f90-6523-17f3-9abd623f65a0} {Suffrage:Voter ID:server-78126716-a533-0f36-e02d-56eb606d2b75 Address:78126716-a533-0f36-e02d-56eb606d2b75} {Suffrage:Voter ID:server-ab80983a-2d71-c183-078b-2be428891adb Address:ab80983a-2d71-c183-078b-2be428891adb}]" +2025-03-20T03:51:11.511Z [INFO] server-ab80983a-2d71-c183-078b-2be428891adb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f35bbbc2-5f90-6523-17f3-9abd623f65a0 Address:f35bbbc2-5f90-6523-17f3-9abd623f65a0} {Suffrage:Voter ID:server-78126716-a533-0f36-e02d-56eb606d2b75 Address:78126716-a533-0f36-e02d-56eb606d2b75} {Suffrage:Voter ID:server-ab80983a-2d71-c183-078b-2be428891adb Address:ab80983a-2d71-c183-078b-2be428891adb}]" +2025-03-20T03:51:11.511Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:11.511Z [INFO] server-f35bbbc2-5f90-6523-17f3-9abd623f65a0: entering follower state: follower="Node at f35bbbc2-5f90-6523-17f3-9abd623f65a0 [Follower]" leader-address= leader-id= +2025-03-20T03:51:11.511Z [INFO] server-78126716-a533-0f36-e02d-56eb606d2b75: entering follower state: follower="Node at 78126716-a533-0f36-e02d-56eb606d2b75 [Follower]" leader-address= leader-id= +2025-03-20T03:51:11.511Z [INFO] server-ab80983a-2d71-c183-078b-2be428891adb: entering follower state: follower="Node at ab80983a-2d71-c183-078b-2be428891adb [Follower]" leader-address= leader-id= +2025-03-20T03:51:11.570Z [WARN] server-ab80983a-2d71-c183-078b-2be428891adb: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:11.570Z [INFO] server-ab80983a-2d71-c183-078b-2be428891adb: entering candidate state: node="Node at ab80983a-2d71-c183-078b-2be428891adb [Candidate]" term=2 +2025-03-20T03:51:11.570Z [INFO] server-ab80983a-2d71-c183-078b-2be428891adb: election won: term=2 tally=2 +2025-03-20T03:51:11.570Z [INFO] server-ab80983a-2d71-c183-078b-2be428891adb: entering leader state: leader="Node at ab80983a-2d71-c183-078b-2be428891adb [Leader]" +2025-03-20T03:51:11.570Z [INFO] server-ab80983a-2d71-c183-078b-2be428891adb: added peer, starting replication: peer=server-f35bbbc2-5f90-6523-17f3-9abd623f65a0 +2025-03-20T03:51:11.570Z [INFO] server-ab80983a-2d71-c183-078b-2be428891adb: added peer, starting replication: peer=server-78126716-a533-0f36-e02d-56eb606d2b75 +2025-03-20T03:51:11.571Z [INFO] server-ab80983a-2d71-c183-078b-2be428891adb: pipelining replication: peer="{Voter server-78126716-a533-0f36-e02d-56eb606d2b75 78126716-a533-0f36-e02d-56eb606d2b75}" +2025-03-20T03:51:11.571Z [INFO] server-ab80983a-2d71-c183-078b-2be428891adb: pipelining replication: peer="{Voter server-f35bbbc2-5f90-6523-17f3-9abd623f65a0 f35bbbc2-5f90-6523-17f3-9abd623f65a0}" +2025-03-20T03:51:11.678Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:11.571140207 +0000 UTC m=+103.245593156 (1 nodes), 59.414232ms from start of poll, 59.837029ms from cluster start. Timeout at 2025-03-20 03:51:11.676220883 +0000 UTC m=+103.350673905, 105.080749ms after stability +2025-03-20T03:51:11.683Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:51:11.789Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:51:11.683492301 +0000 UTC m=+103.357945272 (2 nodes), 2.429µs from start of poll, 172.189145ms from cluster start. Timeout at 2025-03-20 03:51:11.788599162 +0000 UTC m=+103.463052004, 105.106732ms after stability + raft_test.go:1968: [INFO] Disconnecting Node at f35bbbc2-5f90-6523-17f3-9abd623f65a0 [Follower] +2025-03-20T03:51:11.789Z [ERROR] server-ab80983a-2d71-c183-078b-2be428891adb: failed to heartbeat to: peer=f35bbbc2-5f90-6523-17f3-9abd623f65a0 backoff time=10ms error="failed to connect to peer: f35bbbc2-5f90-6523-17f3-9abd623f65a0" +2025-03-20T03:51:11.789Z [INFO] server-ab80983a-2d71-c183-078b-2be428891adb: aborting pipeline replication: peer="{Voter server-78126716-a533-0f36-e02d-56eb606d2b75 78126716-a533-0f36-e02d-56eb606d2b75}" +2025-03-20T03:51:11.789Z [INFO] server-ab80983a-2d71-c183-078b-2be428891adb: aborting pipeline replication: peer="{Voter server-f35bbbc2-5f90-6523-17f3-9abd623f65a0 f35bbbc2-5f90-6523-17f3-9abd623f65a0}" --- PASS: TestRaft_VerifyLeader_PartialConnect (0.28s) === RUN TestRaft_NotifyCh -2026-04-22T10:10:54.160Z [INFO] server-4b67986a-8b5a-31fa-0df2-a5d62360c478: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4b67986a-8b5a-31fa-0df2-a5d62360c478 Address:4b67986a-8b5a-31fa-0df2-a5d62360c478}]" -2026-04-22T10:10:54.160Z [INFO] server-4b67986a-8b5a-31fa-0df2-a5d62360c478: entering follower state: follower="Node at 4b67986a-8b5a-31fa-0df2-a5d62360c478 [Follower]" leader-address= leader-id= -2026-04-22T10:10:54.260Z [WARN] server-4b67986a-8b5a-31fa-0df2-a5d62360c478: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:54.260Z [INFO] server-4b67986a-8b5a-31fa-0df2-a5d62360c478: entering candidate state: node="Node at 4b67986a-8b5a-31fa-0df2-a5d62360c478 [Candidate]" term=2 -2026-04-22T10:10:54.261Z [INFO] server-4b67986a-8b5a-31fa-0df2-a5d62360c478: election won: term=2 tally=1 -2026-04-22T10:10:54.261Z [INFO] server-4b67986a-8b5a-31fa-0df2-a5d62360c478: entering leader state: leader="Node at 4b67986a-8b5a-31fa-0df2-a5d62360c478 [Leader]" +2025-03-20T03:51:11.790Z [INFO] server-0b385025-4a9c-6760-3df4-88b36ac81b12: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0b385025-4a9c-6760-3df4-88b36ac81b12 Address:0b385025-4a9c-6760-3df4-88b36ac81b12}]" +2025-03-20T03:51:11.790Z [INFO] server-0b385025-4a9c-6760-3df4-88b36ac81b12: entering follower state: follower="Node at 0b385025-4a9c-6760-3df4-88b36ac81b12 [Follower]" leader-address= leader-id= +2025-03-20T03:51:11.884Z [WARN] server-0b385025-4a9c-6760-3df4-88b36ac81b12: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:11.885Z [INFO] server-0b385025-4a9c-6760-3df4-88b36ac81b12: entering candidate state: node="Node at 0b385025-4a9c-6760-3df4-88b36ac81b12 [Candidate]" term=2 +2025-03-20T03:51:11.885Z [INFO] server-0b385025-4a9c-6760-3df4-88b36ac81b12: election won: term=2 tally=1 +2025-03-20T03:51:11.885Z [INFO] server-0b385025-4a9c-6760-3df4-88b36ac81b12: entering leader state: leader="Node at 0b385025-4a9c-6760-3df4-88b36ac81b12 [Leader]" --- PASS: TestRaft_NotifyCh (0.10s) === RUN TestRaft_AppendEntry -2026-04-22T10:10:54.261Z [INFO] server-ba7e9a42-0427-82b2-4356-d332b1f3f78c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ba7e9a42-0427-82b2-4356-d332b1f3f78c Address:ba7e9a42-0427-82b2-4356-d332b1f3f78c} {Suffrage:Voter ID:server-67ba9f51-83b1-d19a-f5fc-8cf7d97a457c Address:67ba9f51-83b1-d19a-f5fc-8cf7d97a457c} {Suffrage:Voter ID:server-7c832aa0-a138-6137-8701-be41e24a5c01 Address:7c832aa0-a138-6137-8701-be41e24a5c01}]" -2026-04-22T10:10:54.261Z [INFO] server-ba7e9a42-0427-82b2-4356-d332b1f3f78c: entering follower state: follower="Node at ba7e9a42-0427-82b2-4356-d332b1f3f78c [Follower]" leader-address= leader-id= -2026-04-22T10:10:54.261Z [INFO] server-67ba9f51-83b1-d19a-f5fc-8cf7d97a457c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ba7e9a42-0427-82b2-4356-d332b1f3f78c Address:ba7e9a42-0427-82b2-4356-d332b1f3f78c} {Suffrage:Voter ID:server-67ba9f51-83b1-d19a-f5fc-8cf7d97a457c Address:67ba9f51-83b1-d19a-f5fc-8cf7d97a457c} {Suffrage:Voter ID:server-7c832aa0-a138-6137-8701-be41e24a5c01 Address:7c832aa0-a138-6137-8701-be41e24a5c01}]" -2026-04-22T10:10:54.261Z [INFO] server-67ba9f51-83b1-d19a-f5fc-8cf7d97a457c: entering follower state: follower="Node at 67ba9f51-83b1-d19a-f5fc-8cf7d97a457c [Follower]" leader-address= leader-id= -2026-04-22T10:10:54.261Z [INFO] server-7c832aa0-a138-6137-8701-be41e24a5c01: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ba7e9a42-0427-82b2-4356-d332b1f3f78c Address:ba7e9a42-0427-82b2-4356-d332b1f3f78c} {Suffrage:Voter ID:server-67ba9f51-83b1-d19a-f5fc-8cf7d97a457c Address:67ba9f51-83b1-d19a-f5fc-8cf7d97a457c} {Suffrage:Voter ID:server-7c832aa0-a138-6137-8701-be41e24a5c01 Address:7c832aa0-a138-6137-8701-be41e24a5c01}]" -2026-04-22T10:10:54.261Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:10:54.261Z [INFO] server-7c832aa0-a138-6137-8701-be41e24a5c01: entering follower state: follower="Node at 7c832aa0-a138-6137-8701-be41e24a5c01 [Follower]" leader-address= leader-id= -2026-04-22T10:10:54.323Z [WARN] server-7c832aa0-a138-6137-8701-be41e24a5c01: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:54.323Z [INFO] server-7c832aa0-a138-6137-8701-be41e24a5c01: entering candidate state: node="Node at 7c832aa0-a138-6137-8701-be41e24a5c01 [Candidate]" term=2 -2026-04-22T10:10:54.323Z [INFO] server-7c832aa0-a138-6137-8701-be41e24a5c01: election won: term=2 tally=2 -2026-04-22T10:10:54.323Z [INFO] server-7c832aa0-a138-6137-8701-be41e24a5c01: entering leader state: leader="Node at 7c832aa0-a138-6137-8701-be41e24a5c01 [Leader]" -2026-04-22T10:10:54.323Z [INFO] server-7c832aa0-a138-6137-8701-be41e24a5c01: added peer, starting replication: peer=server-ba7e9a42-0427-82b2-4356-d332b1f3f78c -2026-04-22T10:10:54.323Z [INFO] server-7c832aa0-a138-6137-8701-be41e24a5c01: added peer, starting replication: peer=server-67ba9f51-83b1-d19a-f5fc-8cf7d97a457c -2026-04-22T10:10:54.323Z [INFO] server-7c832aa0-a138-6137-8701-be41e24a5c01: pipelining replication: peer="{Voter server-ba7e9a42-0427-82b2-4356-d332b1f3f78c ba7e9a42-0427-82b2-4356-d332b1f3f78c}" -2026-04-22T10:10:54.323Z [INFO] server-7c832aa0-a138-6137-8701-be41e24a5c01: pipelining replication: peer="{Voter server-67ba9f51-83b1-d19a-f5fc-8cf7d97a457c 67ba9f51-83b1-d19a-f5fc-8cf7d97a457c}" -2026-04-22T10:10:54.428Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:10:54.323617079 +0000 UTC m=+100.347413229 (2 nodes), 61.634165ms from start of poll, 61.847962ms from cluster start. Timeout at 2026-04-22 10:10:54.428620839 +0000 UTC m=+100.452416922, 105.003693ms after stability -2026-04-22T10:10:54.428Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:54.534Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:54.428755878 +0000 UTC m=+100.452551966 (1 nodes), 486ns from start of poll, 166.986699ms from cluster start. Timeout at 2026-04-22 10:10:54.5337669 +0000 UTC m=+100.557580044, 105.028078ms after stability -2026-04-22T10:10:54.534Z [WARN] server-ba7e9a42-0427-82b2-4356-d332b1f3f78c: clearing log suffix: from=1 to=2 -2026-04-22T10:10:54.534Z [WARN] server-ba7e9a42-0427-82b2-4356-d332b1f3f78c: skipping application of old log: index=1 -2026-04-22T10:10:54.534Z [WARN] server-ba7e9a42-0427-82b2-4356-d332b1f3f78c: skipping application of old log: index=1 -2026-04-22T10:10:54.534Z [INFO] server-7c832aa0-a138-6137-8701-be41e24a5c01: aborting pipeline replication: peer="{Voter server-67ba9f51-83b1-d19a-f5fc-8cf7d97a457c 67ba9f51-83b1-d19a-f5fc-8cf7d97a457c}" -2026-04-22T10:10:54.534Z [INFO] server-7c832aa0-a138-6137-8701-be41e24a5c01: aborting pipeline replication: peer="{Voter server-ba7e9a42-0427-82b2-4356-d332b1f3f78c ba7e9a42-0427-82b2-4356-d332b1f3f78c}" -2026-04-22T10:10:55.034Z [ERROR] server-7c832aa0-a138-6137-8701-be41e24a5c01: failed to heartbeat to: peer=ba7e9a42-0427-82b2-4356-d332b1f3f78c backoff time=10ms error="command timed out" ---- PASS: TestRaft_AppendEntry (0.77s) +2025-03-20T03:51:11.887Z [INFO] server-966c86d1-99d3-5ac0-d232-a59d6acedf9b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-966c86d1-99d3-5ac0-d232-a59d6acedf9b Address:966c86d1-99d3-5ac0-d232-a59d6acedf9b} {Suffrage:Voter ID:server-3e260c7f-f658-796c-b0d1-99be96c3f02d Address:3e260c7f-f658-796c-b0d1-99be96c3f02d} {Suffrage:Voter ID:server-fb0597d9-42d1-be15-b546-a97e3a3f3ea2 Address:fb0597d9-42d1-be15-b546-a97e3a3f3ea2}]" +2025-03-20T03:51:11.888Z [INFO] server-3e260c7f-f658-796c-b0d1-99be96c3f02d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-966c86d1-99d3-5ac0-d232-a59d6acedf9b Address:966c86d1-99d3-5ac0-d232-a59d6acedf9b} {Suffrage:Voter ID:server-3e260c7f-f658-796c-b0d1-99be96c3f02d Address:3e260c7f-f658-796c-b0d1-99be96c3f02d} {Suffrage:Voter ID:server-fb0597d9-42d1-be15-b546-a97e3a3f3ea2 Address:fb0597d9-42d1-be15-b546-a97e3a3f3ea2}]" +2025-03-20T03:51:11.888Z [INFO] server-fb0597d9-42d1-be15-b546-a97e3a3f3ea2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-966c86d1-99d3-5ac0-d232-a59d6acedf9b Address:966c86d1-99d3-5ac0-d232-a59d6acedf9b} {Suffrage:Voter ID:server-3e260c7f-f658-796c-b0d1-99be96c3f02d Address:3e260c7f-f658-796c-b0d1-99be96c3f02d} {Suffrage:Voter ID:server-fb0597d9-42d1-be15-b546-a97e3a3f3ea2 Address:fb0597d9-42d1-be15-b546-a97e3a3f3ea2}]" +2025-03-20T03:51:11.888Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:51:11.888Z [INFO] server-966c86d1-99d3-5ac0-d232-a59d6acedf9b: entering follower state: follower="Node at 966c86d1-99d3-5ac0-d232-a59d6acedf9b [Follower]" leader-address= leader-id= +2025-03-20T03:51:11.888Z [INFO] server-3e260c7f-f658-796c-b0d1-99be96c3f02d: entering follower state: follower="Node at 3e260c7f-f658-796c-b0d1-99be96c3f02d [Follower]" leader-address= leader-id= +2025-03-20T03:51:11.888Z [INFO] server-fb0597d9-42d1-be15-b546-a97e3a3f3ea2: entering follower state: follower="Node at fb0597d9-42d1-be15-b546-a97e3a3f3ea2 [Follower]" leader-address= leader-id= +2025-03-20T03:51:11.944Z [WARN] server-966c86d1-99d3-5ac0-d232-a59d6acedf9b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:11.944Z [INFO] server-966c86d1-99d3-5ac0-d232-a59d6acedf9b: entering candidate state: node="Node at 966c86d1-99d3-5ac0-d232-a59d6acedf9b [Candidate]" term=2 +2025-03-20T03:51:11.944Z [INFO] server-966c86d1-99d3-5ac0-d232-a59d6acedf9b: election won: term=2 tally=2 +2025-03-20T03:51:11.944Z [INFO] server-966c86d1-99d3-5ac0-d232-a59d6acedf9b: entering leader state: leader="Node at 966c86d1-99d3-5ac0-d232-a59d6acedf9b [Leader]" +2025-03-20T03:51:11.944Z [INFO] server-966c86d1-99d3-5ac0-d232-a59d6acedf9b: added peer, starting replication: peer=server-3e260c7f-f658-796c-b0d1-99be96c3f02d +2025-03-20T03:51:11.944Z [INFO] server-966c86d1-99d3-5ac0-d232-a59d6acedf9b: added peer, starting replication: peer=server-fb0597d9-42d1-be15-b546-a97e3a3f3ea2 +2025-03-20T03:51:11.944Z [INFO] server-966c86d1-99d3-5ac0-d232-a59d6acedf9b: pipelining replication: peer="{Voter server-fb0597d9-42d1-be15-b546-a97e3a3f3ea2 fb0597d9-42d1-be15-b546-a97e3a3f3ea2}" +2025-03-20T03:51:11.944Z [WARN] server-fb0597d9-42d1-be15-b546-a97e3a3f3ea2: rejecting vote request since our last term is greater: candidate=966c86d1-99d3-5ac0-d232-a59d6acedf9b last-term=2 last-candidate-term=1 +2025-03-20T03:51:11.945Z [INFO] server-966c86d1-99d3-5ac0-d232-a59d6acedf9b: pipelining replication: peer="{Voter server-3e260c7f-f658-796c-b0d1-99be96c3f02d 3e260c7f-f658-796c-b0d1-99be96c3f02d}" +2025-03-20T03:51:12.050Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:51:11.94492173 +0000 UTC m=+103.619374663 (2 nodes), 56.691728ms from start of poll, 57.144079ms from cluster start. Timeout at 2025-03-20 03:51:12.049928716 +0000 UTC m=+103.724381809, 105.007146ms after stability +2025-03-20T03:51:12.050Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:12.156Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:12.050266144 +0000 UTC m=+103.724718922 (1 nodes), 2.666µs from start of poll, 162.488338ms from cluster start. Timeout at 2025-03-20 03:51:12.155358748 +0000 UTC m=+103.829811667, 105.092745ms after stability +2025-03-20T03:51:12.156Z [WARN] server-3e260c7f-f658-796c-b0d1-99be96c3f02d: clearing log suffix: from=1 to=2 +2025-03-20T03:51:12.156Z [WARN] server-3e260c7f-f658-796c-b0d1-99be96c3f02d: skipping application of old log: index=1 +2025-03-20T03:51:12.156Z [WARN] server-3e260c7f-f658-796c-b0d1-99be96c3f02d: skipping application of old log: index=1 +2025-03-20T03:51:12.156Z [INFO] server-966c86d1-99d3-5ac0-d232-a59d6acedf9b: aborting pipeline replication: peer="{Voter server-fb0597d9-42d1-be15-b546-a97e3a3f3ea2 fb0597d9-42d1-be15-b546-a97e3a3f3ea2}" +2025-03-20T03:51:12.156Z [INFO] server-966c86d1-99d3-5ac0-d232-a59d6acedf9b: aborting pipeline replication: peer="{Voter server-3e260c7f-f658-796c-b0d1-99be96c3f02d 3e260c7f-f658-796c-b0d1-99be96c3f02d}" +--- PASS: TestRaft_AppendEntry (0.27s) === RUN TestRaft_VotingGrant_WhenLeaderAvailable -2026-04-22T10:10:55.035Z [INFO] server-70c7850e-9f1f-fe79-ffda-59de795c7c55: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-70c7850e-9f1f-fe79-ffda-59de795c7c55 Address:70c7850e-9f1f-fe79-ffda-59de795c7c55} {Suffrage:Voter ID:server-d0b39596-ef96-6c73-8b6c-c2409efae439 Address:d0b39596-ef96-6c73-8b6c-c2409efae439} {Suffrage:Voter ID:server-565fc759-3c12-801d-646c-0f57dba12edf Address:565fc759-3c12-801d-646c-0f57dba12edf}]" -2026-04-22T10:10:55.035Z [INFO] server-70c7850e-9f1f-fe79-ffda-59de795c7c55: entering follower state: follower="Node at 70c7850e-9f1f-fe79-ffda-59de795c7c55 [Follower]" leader-address= leader-id= -2026-04-22T10:10:55.035Z [INFO] server-d0b39596-ef96-6c73-8b6c-c2409efae439: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-70c7850e-9f1f-fe79-ffda-59de795c7c55 Address:70c7850e-9f1f-fe79-ffda-59de795c7c55} {Suffrage:Voter ID:server-d0b39596-ef96-6c73-8b6c-c2409efae439 Address:d0b39596-ef96-6c73-8b6c-c2409efae439} {Suffrage:Voter ID:server-565fc759-3c12-801d-646c-0f57dba12edf Address:565fc759-3c12-801d-646c-0f57dba12edf}]" -2026-04-22T10:10:55.035Z [INFO] server-d0b39596-ef96-6c73-8b6c-c2409efae439: entering follower state: follower="Node at d0b39596-ef96-6c73-8b6c-c2409efae439 [Follower]" leader-address= leader-id= -2026-04-22T10:10:55.035Z [INFO] server-565fc759-3c12-801d-646c-0f57dba12edf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-70c7850e-9f1f-fe79-ffda-59de795c7c55 Address:70c7850e-9f1f-fe79-ffda-59de795c7c55} {Suffrage:Voter ID:server-d0b39596-ef96-6c73-8b6c-c2409efae439 Address:d0b39596-ef96-6c73-8b6c-c2409efae439} {Suffrage:Voter ID:server-565fc759-3c12-801d-646c-0f57dba12edf Address:565fc759-3c12-801d-646c-0f57dba12edf}]" -2026-04-22T10:10:55.035Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:10:55.035Z [INFO] server-565fc759-3c12-801d-646c-0f57dba12edf: entering follower state: follower="Node at 565fc759-3c12-801d-646c-0f57dba12edf [Follower]" leader-address= leader-id= -2026-04-22T10:10:55.113Z [WARN] server-70c7850e-9f1f-fe79-ffda-59de795c7c55: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:55.113Z [INFO] server-70c7850e-9f1f-fe79-ffda-59de795c7c55: entering candidate state: node="Node at 70c7850e-9f1f-fe79-ffda-59de795c7c55 [Candidate]" term=2 -2026-04-22T10:10:55.113Z [INFO] server-70c7850e-9f1f-fe79-ffda-59de795c7c55: election won: term=2 tally=2 -2026-04-22T10:10:55.113Z [INFO] server-70c7850e-9f1f-fe79-ffda-59de795c7c55: entering leader state: leader="Node at 70c7850e-9f1f-fe79-ffda-59de795c7c55 [Leader]" -2026-04-22T10:10:55.113Z [INFO] server-70c7850e-9f1f-fe79-ffda-59de795c7c55: added peer, starting replication: peer=server-d0b39596-ef96-6c73-8b6c-c2409efae439 -2026-04-22T10:10:55.113Z [INFO] server-70c7850e-9f1f-fe79-ffda-59de795c7c55: added peer, starting replication: peer=server-565fc759-3c12-801d-646c-0f57dba12edf -2026-04-22T10:10:55.113Z [INFO] server-70c7850e-9f1f-fe79-ffda-59de795c7c55: pipelining replication: peer="{Voter server-565fc759-3c12-801d-646c-0f57dba12edf 565fc759-3c12-801d-646c-0f57dba12edf}" -2026-04-22T10:10:55.113Z [INFO] server-70c7850e-9f1f-fe79-ffda-59de795c7c55: pipelining replication: peer="{Voter server-d0b39596-ef96-6c73-8b6c-c2409efae439 d0b39596-ef96-6c73-8b6c-c2409efae439}" -2026-04-22T10:10:55.218Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:10:55.113562877 +0000 UTC m=+101.137358976 (2 nodes), 78.002093ms from start of poll, 78.32627ms from cluster start. Timeout at 2026-04-22 10:10:55.218570753 +0000 UTC m=+101.242366940, 105.007964ms after stability -2026-04-22T10:10:55.218Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:55.324Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:55.218772097 +0000 UTC m=+101.242568284 (1 nodes), 1.738µs from start of poll, 183.535578ms from cluster start. Timeout at 2026-04-22 10:10:55.323779757 +0000 UTC m=+101.347575860, 105.007576ms after stability -2026-04-22T10:10:55.324Z [WARN] server-565fc759-3c12-801d-646c-0f57dba12edf: rejecting vote request since we have a leader: from=d0b39596-ef96-6c73-8b6c-c2409efae439 leader=70c7850e-9f1f-fe79-ffda-59de795c7c55 leader-id=server-70c7850e-9f1f-fe79-ffda-59de795c7c55 -2026-04-22T10:10:55.324Z [INFO] server-70c7850e-9f1f-fe79-ffda-59de795c7c55: aborting pipeline replication: peer="{Voter server-565fc759-3c12-801d-646c-0f57dba12edf 565fc759-3c12-801d-646c-0f57dba12edf}" -2026-04-22T10:10:55.324Z [INFO] server-70c7850e-9f1f-fe79-ffda-59de795c7c55: aborting pipeline replication: peer="{Voter server-d0b39596-ef96-6c73-8b6c-c2409efae439 d0b39596-ef96-6c73-8b6c-c2409efae439}" +2025-03-20T03:51:12.158Z [INFO] server-2387dbe6-15ac-136a-5105-cb0700c9c95c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2387dbe6-15ac-136a-5105-cb0700c9c95c Address:2387dbe6-15ac-136a-5105-cb0700c9c95c} {Suffrage:Voter ID:server-468a9456-20d1-ca72-4567-e9213e646ab7 Address:468a9456-20d1-ca72-4567-e9213e646ab7} {Suffrage:Voter ID:server-c19bd395-5155-4c75-b514-65bbb9e3db91 Address:c19bd395-5155-4c75-b514-65bbb9e3db91}]" +2025-03-20T03:51:12.158Z [INFO] server-2387dbe6-15ac-136a-5105-cb0700c9c95c: entering follower state: follower="Node at 2387dbe6-15ac-136a-5105-cb0700c9c95c [Follower]" leader-address= leader-id= +2025-03-20T03:51:12.158Z [INFO] server-468a9456-20d1-ca72-4567-e9213e646ab7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2387dbe6-15ac-136a-5105-cb0700c9c95c Address:2387dbe6-15ac-136a-5105-cb0700c9c95c} {Suffrage:Voter ID:server-468a9456-20d1-ca72-4567-e9213e646ab7 Address:468a9456-20d1-ca72-4567-e9213e646ab7} {Suffrage:Voter ID:server-c19bd395-5155-4c75-b514-65bbb9e3db91 Address:c19bd395-5155-4c75-b514-65bbb9e3db91}]" +2025-03-20T03:51:12.158Z [INFO] server-c19bd395-5155-4c75-b514-65bbb9e3db91: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2387dbe6-15ac-136a-5105-cb0700c9c95c Address:2387dbe6-15ac-136a-5105-cb0700c9c95c} {Suffrage:Voter ID:server-468a9456-20d1-ca72-4567-e9213e646ab7 Address:468a9456-20d1-ca72-4567-e9213e646ab7} {Suffrage:Voter ID:server-c19bd395-5155-4c75-b514-65bbb9e3db91 Address:c19bd395-5155-4c75-b514-65bbb9e3db91}]" +2025-03-20T03:51:12.158Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:51:12.158Z [INFO] server-468a9456-20d1-ca72-4567-e9213e646ab7: entering follower state: follower="Node at 468a9456-20d1-ca72-4567-e9213e646ab7 [Follower]" leader-address= leader-id= +2025-03-20T03:51:12.158Z [INFO] server-c19bd395-5155-4c75-b514-65bbb9e3db91: entering follower state: follower="Node at c19bd395-5155-4c75-b514-65bbb9e3db91 [Follower]" leader-address= leader-id= +2025-03-20T03:51:12.227Z [WARN] server-2387dbe6-15ac-136a-5105-cb0700c9c95c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:12.227Z [INFO] server-2387dbe6-15ac-136a-5105-cb0700c9c95c: entering candidate state: node="Node at 2387dbe6-15ac-136a-5105-cb0700c9c95c [Candidate]" term=2 +2025-03-20T03:51:12.227Z [INFO] server-2387dbe6-15ac-136a-5105-cb0700c9c95c: election won: term=2 tally=2 +2025-03-20T03:51:12.227Z [INFO] server-2387dbe6-15ac-136a-5105-cb0700c9c95c: entering leader state: leader="Node at 2387dbe6-15ac-136a-5105-cb0700c9c95c [Leader]" +2025-03-20T03:51:12.227Z [INFO] server-2387dbe6-15ac-136a-5105-cb0700c9c95c: added peer, starting replication: peer=server-468a9456-20d1-ca72-4567-e9213e646ab7 +2025-03-20T03:51:12.227Z [INFO] server-2387dbe6-15ac-136a-5105-cb0700c9c95c: added peer, starting replication: peer=server-c19bd395-5155-4c75-b514-65bbb9e3db91 +2025-03-20T03:51:12.227Z [INFO] server-2387dbe6-15ac-136a-5105-cb0700c9c95c: pipelining replication: peer="{Voter server-c19bd395-5155-4c75-b514-65bbb9e3db91 c19bd395-5155-4c75-b514-65bbb9e3db91}" +2025-03-20T03:51:12.227Z [INFO] server-2387dbe6-15ac-136a-5105-cb0700c9c95c: pipelining replication: peer="{Voter server-468a9456-20d1-ca72-4567-e9213e646ab7 468a9456-20d1-ca72-4567-e9213e646ab7}" +2025-03-20T03:51:12.335Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:51:12.227929269 +0000 UTC m=+103.902382224 (2 nodes), 69.058742ms from start of poll, 69.454979ms from cluster start. Timeout at 2025-03-20 03:51:12.332937196 +0000 UTC m=+104.007390121, 105.007897ms after stability +2025-03-20T03:51:12.335Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:12.441Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:12.335470968 +0000 UTC m=+104.009923761 (1 nodes), 1.768µs from start of poll, 176.996516ms from cluster start. Timeout at 2025-03-20 03:51:12.440536252 +0000 UTC m=+104.114989137, 105.065376ms after stability +2025-03-20T03:51:12.441Z [WARN] server-c19bd395-5155-4c75-b514-65bbb9e3db91: rejecting vote request since we have a leader: from=468a9456-20d1-ca72-4567-e9213e646ab7 leader=2387dbe6-15ac-136a-5105-cb0700c9c95c leader-id=server-2387dbe6-15ac-136a-5105-cb0700c9c95c +2025-03-20T03:51:12.442Z [INFO] server-2387dbe6-15ac-136a-5105-cb0700c9c95c: aborting pipeline replication: peer="{Voter server-c19bd395-5155-4c75-b514-65bbb9e3db91 c19bd395-5155-4c75-b514-65bbb9e3db91}" +2025-03-20T03:51:12.442Z [INFO] server-2387dbe6-15ac-136a-5105-cb0700c9c95c: aborting pipeline replication: peer="{Voter server-468a9456-20d1-ca72-4567-e9213e646ab7 468a9456-20d1-ca72-4567-e9213e646ab7}" --- PASS: TestRaft_VotingGrant_WhenLeaderAvailable (0.29s) === RUN TestRaft_ProtocolVersion_RejectRPC -2026-04-22T10:10:55.325Z [INFO] server-8a66f41d-ce0d-4584-7df4-28294371d022: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8a66f41d-ce0d-4584-7df4-28294371d022 Address:8a66f41d-ce0d-4584-7df4-28294371d022} {Suffrage:Voter ID:server-f8c4b8b0-968f-723f-4b6a-2a3c10d3400e Address:f8c4b8b0-968f-723f-4b6a-2a3c10d3400e} {Suffrage:Voter ID:server-6d52832f-1b94-3729-3fff-62ad9b1cfebd Address:6d52832f-1b94-3729-3fff-62ad9b1cfebd}]" -2026-04-22T10:10:55.325Z [INFO] server-8a66f41d-ce0d-4584-7df4-28294371d022: entering follower state: follower="Node at 8a66f41d-ce0d-4584-7df4-28294371d022 [Follower]" leader-address= leader-id= -2026-04-22T10:10:55.325Z [INFO] server-f8c4b8b0-968f-723f-4b6a-2a3c10d3400e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8a66f41d-ce0d-4584-7df4-28294371d022 Address:8a66f41d-ce0d-4584-7df4-28294371d022} {Suffrage:Voter ID:server-f8c4b8b0-968f-723f-4b6a-2a3c10d3400e Address:f8c4b8b0-968f-723f-4b6a-2a3c10d3400e} {Suffrage:Voter ID:server-6d52832f-1b94-3729-3fff-62ad9b1cfebd Address:6d52832f-1b94-3729-3fff-62ad9b1cfebd}]" -2026-04-22T10:10:55.325Z [INFO] server-f8c4b8b0-968f-723f-4b6a-2a3c10d3400e: entering follower state: follower="Node at f8c4b8b0-968f-723f-4b6a-2a3c10d3400e [Follower]" leader-address= leader-id= -2026-04-22T10:10:55.325Z [INFO] server-6d52832f-1b94-3729-3fff-62ad9b1cfebd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8a66f41d-ce0d-4584-7df4-28294371d022 Address:8a66f41d-ce0d-4584-7df4-28294371d022} {Suffrage:Voter ID:server-f8c4b8b0-968f-723f-4b6a-2a3c10d3400e Address:f8c4b8b0-968f-723f-4b6a-2a3c10d3400e} {Suffrage:Voter ID:server-6d52832f-1b94-3729-3fff-62ad9b1cfebd Address:6d52832f-1b94-3729-3fff-62ad9b1cfebd}]" -2026-04-22T10:10:55.325Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:10:55.325Z [INFO] server-6d52832f-1b94-3729-3fff-62ad9b1cfebd: entering follower state: follower="Node at 6d52832f-1b94-3729-3fff-62ad9b1cfebd [Follower]" leader-address= leader-id= -2026-04-22T10:10:55.393Z [WARN] server-6d52832f-1b94-3729-3fff-62ad9b1cfebd: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:55.393Z [INFO] server-6d52832f-1b94-3729-3fff-62ad9b1cfebd: entering candidate state: node="Node at 6d52832f-1b94-3729-3fff-62ad9b1cfebd [Candidate]" term=2 -2026-04-22T10:10:55.393Z [INFO] server-6d52832f-1b94-3729-3fff-62ad9b1cfebd: election won: term=2 tally=2 -2026-04-22T10:10:55.393Z [INFO] server-6d52832f-1b94-3729-3fff-62ad9b1cfebd: entering leader state: leader="Node at 6d52832f-1b94-3729-3fff-62ad9b1cfebd [Leader]" -2026-04-22T10:10:55.393Z [INFO] server-6d52832f-1b94-3729-3fff-62ad9b1cfebd: added peer, starting replication: peer=server-8a66f41d-ce0d-4584-7df4-28294371d022 -2026-04-22T10:10:55.393Z [INFO] server-6d52832f-1b94-3729-3fff-62ad9b1cfebd: added peer, starting replication: peer=server-f8c4b8b0-968f-723f-4b6a-2a3c10d3400e -2026-04-22T10:10:55.393Z [INFO] server-6d52832f-1b94-3729-3fff-62ad9b1cfebd: pipelining replication: peer="{Voter server-8a66f41d-ce0d-4584-7df4-28294371d022 8a66f41d-ce0d-4584-7df4-28294371d022}" -2026-04-22T10:10:55.394Z [INFO] server-6d52832f-1b94-3729-3fff-62ad9b1cfebd: pipelining replication: peer="{Voter server-f8c4b8b0-968f-723f-4b6a-2a3c10d3400e f8c4b8b0-968f-723f-4b6a-2a3c10d3400e}" -2026-04-22T10:10:55.499Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:10:55.393782559 +0000 UTC m=+101.417578654 (2 nodes), 68.408043ms from start of poll, 68.686154ms from cluster start. Timeout at 2026-04-22 10:10:55.498785766 +0000 UTC m=+101.522581871, 105.003217ms after stability -2026-04-22T10:10:55.499Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:55.604Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:55.49911049 +0000 UTC m=+101.522906578 (1 nodes), 474ns from start of poll, 174.014078ms from cluster start. Timeout at 2026-04-22 10:10:55.604147604 +0000 UTC m=+101.627943740, 105.037162ms after stability -2026-04-22T10:10:55.604Z [INFO] server-6d52832f-1b94-3729-3fff-62ad9b1cfebd: aborting pipeline replication: peer="{Voter server-f8c4b8b0-968f-723f-4b6a-2a3c10d3400e f8c4b8b0-968f-723f-4b6a-2a3c10d3400e}" -2026-04-22T10:10:55.604Z [INFO] server-6d52832f-1b94-3729-3fff-62ad9b1cfebd: aborting pipeline replication: peer="{Voter server-8a66f41d-ce0d-4584-7df4-28294371d022 8a66f41d-ce0d-4584-7df4-28294371d022}" ---- PASS: TestRaft_ProtocolVersion_RejectRPC (0.28s) +2025-03-20T03:51:12.443Z [INFO] server-570f4a74-0279-41e6-ac61-8784b8b8c7e0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-570f4a74-0279-41e6-ac61-8784b8b8c7e0 Address:570f4a74-0279-41e6-ac61-8784b8b8c7e0} {Suffrage:Voter ID:server-9609ce51-cd9f-8c82-5027-1487a531aeee Address:9609ce51-cd9f-8c82-5027-1487a531aeee} {Suffrage:Voter ID:server-09e95070-b7c6-a9dc-bc25-c27f1df82b09 Address:09e95070-b7c6-a9dc-bc25-c27f1df82b09}]" +2025-03-20T03:51:12.444Z [INFO] server-9609ce51-cd9f-8c82-5027-1487a531aeee: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-570f4a74-0279-41e6-ac61-8784b8b8c7e0 Address:570f4a74-0279-41e6-ac61-8784b8b8c7e0} {Suffrage:Voter ID:server-9609ce51-cd9f-8c82-5027-1487a531aeee Address:9609ce51-cd9f-8c82-5027-1487a531aeee} {Suffrage:Voter ID:server-09e95070-b7c6-a9dc-bc25-c27f1df82b09 Address:09e95070-b7c6-a9dc-bc25-c27f1df82b09}]" +2025-03-20T03:51:12.444Z [INFO] server-09e95070-b7c6-a9dc-bc25-c27f1df82b09: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-570f4a74-0279-41e6-ac61-8784b8b8c7e0 Address:570f4a74-0279-41e6-ac61-8784b8b8c7e0} {Suffrage:Voter ID:server-9609ce51-cd9f-8c82-5027-1487a531aeee Address:9609ce51-cd9f-8c82-5027-1487a531aeee} {Suffrage:Voter ID:server-09e95070-b7c6-a9dc-bc25-c27f1df82b09 Address:09e95070-b7c6-a9dc-bc25-c27f1df82b09}]" +2025-03-20T03:51:12.444Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:51:12.444Z [INFO] server-570f4a74-0279-41e6-ac61-8784b8b8c7e0: entering follower state: follower="Node at 570f4a74-0279-41e6-ac61-8784b8b8c7e0 [Follower]" leader-address= leader-id= +2025-03-20T03:51:12.444Z [INFO] server-9609ce51-cd9f-8c82-5027-1487a531aeee: entering follower state: follower="Node at 9609ce51-cd9f-8c82-5027-1487a531aeee [Follower]" leader-address= leader-id= +2025-03-20T03:51:12.444Z [INFO] server-09e95070-b7c6-a9dc-bc25-c27f1df82b09: entering follower state: follower="Node at 09e95070-b7c6-a9dc-bc25-c27f1df82b09 [Follower]" leader-address= leader-id= +2025-03-20T03:51:12.527Z [WARN] server-9609ce51-cd9f-8c82-5027-1487a531aeee: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:12.527Z [INFO] server-9609ce51-cd9f-8c82-5027-1487a531aeee: entering candidate state: node="Node at 9609ce51-cd9f-8c82-5027-1487a531aeee [Candidate]" term=2 +2025-03-20T03:51:12.527Z [INFO] server-9609ce51-cd9f-8c82-5027-1487a531aeee: election won: term=2 tally=2 +2025-03-20T03:51:12.527Z [INFO] server-9609ce51-cd9f-8c82-5027-1487a531aeee: entering leader state: leader="Node at 9609ce51-cd9f-8c82-5027-1487a531aeee [Leader]" +2025-03-20T03:51:12.527Z [INFO] server-9609ce51-cd9f-8c82-5027-1487a531aeee: added peer, starting replication: peer=server-570f4a74-0279-41e6-ac61-8784b8b8c7e0 +2025-03-20T03:51:12.527Z [INFO] server-9609ce51-cd9f-8c82-5027-1487a531aeee: added peer, starting replication: peer=server-09e95070-b7c6-a9dc-bc25-c27f1df82b09 +2025-03-20T03:51:12.528Z [INFO] server-9609ce51-cd9f-8c82-5027-1487a531aeee: pipelining replication: peer="{Voter server-09e95070-b7c6-a9dc-bc25-c27f1df82b09 09e95070-b7c6-a9dc-bc25-c27f1df82b09}" +2025-03-20T03:51:12.528Z [INFO] server-9609ce51-cd9f-8c82-5027-1487a531aeee: pipelining replication: peer="{Voter server-570f4a74-0279-41e6-ac61-8784b8b8c7e0 570f4a74-0279-41e6-ac61-8784b8b8c7e0}" +2025-03-20T03:51:12.646Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:51:12.528111665 +0000 UTC m=+104.202564610 (2 nodes), 83.796102ms from start of poll, 84.409118ms from cluster start. Timeout at 2025-03-20 03:51:12.633212449 +0000 UTC m=+104.307665301, 105.100691ms after stability +2025-03-20T03:51:12.647Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:12.757Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:12.647118176 +0000 UTC m=+104.321571473 (1 nodes), 7.124µs from start of poll, 203.415981ms from cluster start. Timeout at 2025-03-20 03:51:12.752185354 +0000 UTC m=+104.426638393, 105.06692ms after stability +2025-03-20T03:51:12.757Z [INFO] server-9609ce51-cd9f-8c82-5027-1487a531aeee: aborting pipeline replication: peer="{Voter server-09e95070-b7c6-a9dc-bc25-c27f1df82b09 09e95070-b7c6-a9dc-bc25-c27f1df82b09}" +2025-03-20T03:51:12.757Z [INFO] server-9609ce51-cd9f-8c82-5027-1487a531aeee: aborting pipeline replication: peer="{Voter server-570f4a74-0279-41e6-ac61-8784b8b8c7e0 570f4a74-0279-41e6-ac61-8784b8b8c7e0}" +--- PASS: TestRaft_ProtocolVersion_RejectRPC (0.32s) === RUN TestRaft_ProtocolVersion_Upgrade_1_2 raft_test.go:2151: Skipping unreliable test --- SKIP: TestRaft_ProtocolVersion_Upgrade_1_2 (0.00s) @@ -3595,267 +3716,281 @@ raft_test.go:2196: Skipping unreliable test --- SKIP: TestRaft_ProtocolVersion_Upgrade_2_3 (0.00s) === RUN TestRaft_LeaderID_Propagated -2026-04-22T10:10:55.605Z [INFO] server-85089f2d-ee17-0ff7-a070-ffc4b1f5bc5b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-85089f2d-ee17-0ff7-a070-ffc4b1f5bc5b Address:85089f2d-ee17-0ff7-a070-ffc4b1f5bc5b} {Suffrage:Voter ID:server-da481f95-dbde-6efb-7b4b-5c8f29e96189 Address:da481f95-dbde-6efb-7b4b-5c8f29e96189} {Suffrage:Voter ID:server-6c4867b0-4c18-c379-81c4-6a93ed6ebb2c Address:6c4867b0-4c18-c379-81c4-6a93ed6ebb2c}]" -2026-04-22T10:10:55.605Z [INFO] server-85089f2d-ee17-0ff7-a070-ffc4b1f5bc5b: entering follower state: follower="Node at 85089f2d-ee17-0ff7-a070-ffc4b1f5bc5b [Follower]" leader-address= leader-id= -2026-04-22T10:10:55.605Z [INFO] server-da481f95-dbde-6efb-7b4b-5c8f29e96189: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-85089f2d-ee17-0ff7-a070-ffc4b1f5bc5b Address:85089f2d-ee17-0ff7-a070-ffc4b1f5bc5b} {Suffrage:Voter ID:server-da481f95-dbde-6efb-7b4b-5c8f29e96189 Address:da481f95-dbde-6efb-7b4b-5c8f29e96189} {Suffrage:Voter ID:server-6c4867b0-4c18-c379-81c4-6a93ed6ebb2c Address:6c4867b0-4c18-c379-81c4-6a93ed6ebb2c}]" -2026-04-22T10:10:55.605Z [INFO] server-da481f95-dbde-6efb-7b4b-5c8f29e96189: entering follower state: follower="Node at da481f95-dbde-6efb-7b4b-5c8f29e96189 [Follower]" leader-address= leader-id= -2026-04-22T10:10:55.605Z [INFO] server-6c4867b0-4c18-c379-81c4-6a93ed6ebb2c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-85089f2d-ee17-0ff7-a070-ffc4b1f5bc5b Address:85089f2d-ee17-0ff7-a070-ffc4b1f5bc5b} {Suffrage:Voter ID:server-da481f95-dbde-6efb-7b4b-5c8f29e96189 Address:da481f95-dbde-6efb-7b4b-5c8f29e96189} {Suffrage:Voter ID:server-6c4867b0-4c18-c379-81c4-6a93ed6ebb2c Address:6c4867b0-4c18-c379-81c4-6a93ed6ebb2c}]" -2026-04-22T10:10:55.605Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:55.605Z [INFO] server-6c4867b0-4c18-c379-81c4-6a93ed6ebb2c: entering follower state: follower="Node at 6c4867b0-4c18-c379-81c4-6a93ed6ebb2c [Follower]" leader-address= leader-id= -2026-04-22T10:10:55.668Z [WARN] server-da481f95-dbde-6efb-7b4b-5c8f29e96189: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:55.668Z [INFO] server-da481f95-dbde-6efb-7b4b-5c8f29e96189: entering candidate state: node="Node at da481f95-dbde-6efb-7b4b-5c8f29e96189 [Candidate]" term=2 -2026-04-22T10:10:55.668Z [INFO] server-da481f95-dbde-6efb-7b4b-5c8f29e96189: election won: term=2 tally=2 -2026-04-22T10:10:55.668Z [INFO] server-da481f95-dbde-6efb-7b4b-5c8f29e96189: entering leader state: leader="Node at da481f95-dbde-6efb-7b4b-5c8f29e96189 [Leader]" -2026-04-22T10:10:55.668Z [INFO] server-da481f95-dbde-6efb-7b4b-5c8f29e96189: added peer, starting replication: peer=server-85089f2d-ee17-0ff7-a070-ffc4b1f5bc5b -2026-04-22T10:10:55.668Z [INFO] server-da481f95-dbde-6efb-7b4b-5c8f29e96189: added peer, starting replication: peer=server-6c4867b0-4c18-c379-81c4-6a93ed6ebb2c -2026-04-22T10:10:55.669Z [INFO] server-da481f95-dbde-6efb-7b4b-5c8f29e96189: pipelining replication: peer="{Voter server-85089f2d-ee17-0ff7-a070-ffc4b1f5bc5b 85089f2d-ee17-0ff7-a070-ffc4b1f5bc5b}" -2026-04-22T10:10:55.669Z [INFO] server-da481f95-dbde-6efb-7b4b-5c8f29e96189: pipelining replication: peer="{Voter server-6c4867b0-4c18-c379-81c4-6a93ed6ebb2c 6c4867b0-4c18-c379-81c4-6a93ed6ebb2c}" -2026-04-22T10:10:55.774Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:55.669214364 +0000 UTC m=+101.693010457 (1 nodes), 63.419478ms from start of poll, 63.755006ms from cluster start. Timeout at 2026-04-22 10:10:55.774221735 +0000 UTC m=+101.798017875, 105.007418ms after stability -2026-04-22T10:10:55.774Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:55.880Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:55.77466758 +0000 UTC m=+101.798463674 (1 nodes), 548ns from start of poll, 169.208223ms from cluster start. Timeout at 2026-04-22 10:10:55.879700108 +0000 UTC m=+101.903496311, 105.032637ms after stability -2026-04-22T10:10:55.880Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:55.985Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:55.880355888 +0000 UTC m=+101.904152145 (1 nodes), 4.02µs from start of poll, 274.896694ms from cluster start. Timeout at 2026-04-22 10:10:55.985363339 +0000 UTC m=+102.009159438, 105.007293ms after stability -2026-04-22T10:10:55.985Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:56.090Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:55.985620673 +0000 UTC m=+102.009416761 (1 nodes), 447ns from start of poll, 380.16131ms from cluster start. Timeout at 2026-04-22 10:10:56.090666345 +0000 UTC m=+102.114462426, 105.045665ms after stability -2026-04-22T10:10:56.090Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:56.196Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:56.090852879 +0000 UTC m=+102.114648965 (1 nodes), 574ns from start of poll, 485.393514ms from cluster start. Timeout at 2026-04-22 10:10:56.19588425 +0000 UTC m=+102.219680363, 105.031398ms after stability -2026-04-22T10:10:56.196Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:56.302Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:56.196462042 +0000 UTC m=+102.220258129 (1 nodes), 502ns from start of poll, 591.002678ms from cluster start. Timeout at 2026-04-22 10:10:56.301473161 +0000 UTC m=+102.325269277, 105.011148ms after stability -2026-04-22T10:10:56.408Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:56.514Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:56.408308797 +0000 UTC m=+102.432104885 (1 nodes), 629ns from start of poll, 802.849434ms from cluster start. Timeout at 2026-04-22 10:10:56.513352733 +0000 UTC m=+102.537148893, 105.044008ms after stability -2026-04-22T10:10:56.514Z [INFO] server-da481f95-dbde-6efb-7b4b-5c8f29e96189: aborting pipeline replication: peer="{Voter server-6c4867b0-4c18-c379-81c4-6a93ed6ebb2c 6c4867b0-4c18-c379-81c4-6a93ed6ebb2c}" -2026-04-22T10:10:56.514Z [INFO] server-da481f95-dbde-6efb-7b4b-5c8f29e96189: aborting pipeline replication: peer="{Voter server-85089f2d-ee17-0ff7-a070-ffc4b1f5bc5b 85089f2d-ee17-0ff7-a070-ffc4b1f5bc5b}" ---- PASS: TestRaft_LeaderID_Propagated (0.91s) +2025-03-20T03:51:12.766Z [INFO] server-bc390438-72e5-a439-1847-b3eb3be666ed: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bc390438-72e5-a439-1847-b3eb3be666ed Address:bc390438-72e5-a439-1847-b3eb3be666ed} {Suffrage:Voter ID:server-c946fb51-23c0-b345-2ae5-72a1612ee5e4 Address:c946fb51-23c0-b345-2ae5-72a1612ee5e4} {Suffrage:Voter ID:server-407b101d-45d1-193f-6596-ce28ee9a8d98 Address:407b101d-45d1-193f-6596-ce28ee9a8d98}]" +2025-03-20T03:51:12.767Z [INFO] server-c946fb51-23c0-b345-2ae5-72a1612ee5e4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bc390438-72e5-a439-1847-b3eb3be666ed Address:bc390438-72e5-a439-1847-b3eb3be666ed} {Suffrage:Voter ID:server-c946fb51-23c0-b345-2ae5-72a1612ee5e4 Address:c946fb51-23c0-b345-2ae5-72a1612ee5e4} {Suffrage:Voter ID:server-407b101d-45d1-193f-6596-ce28ee9a8d98 Address:407b101d-45d1-193f-6596-ce28ee9a8d98}]" +2025-03-20T03:51:12.767Z [INFO] server-407b101d-45d1-193f-6596-ce28ee9a8d98: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bc390438-72e5-a439-1847-b3eb3be666ed Address:bc390438-72e5-a439-1847-b3eb3be666ed} {Suffrage:Voter ID:server-c946fb51-23c0-b345-2ae5-72a1612ee5e4 Address:c946fb51-23c0-b345-2ae5-72a1612ee5e4} {Suffrage:Voter ID:server-407b101d-45d1-193f-6596-ce28ee9a8d98 Address:407b101d-45d1-193f-6596-ce28ee9a8d98}]" +2025-03-20T03:51:12.767Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:12.767Z [INFO] server-bc390438-72e5-a439-1847-b3eb3be666ed: entering follower state: follower="Node at bc390438-72e5-a439-1847-b3eb3be666ed [Follower]" leader-address= leader-id= +2025-03-20T03:51:12.767Z [INFO] server-c946fb51-23c0-b345-2ae5-72a1612ee5e4: entering follower state: follower="Node at c946fb51-23c0-b345-2ae5-72a1612ee5e4 [Follower]" leader-address= leader-id= +2025-03-20T03:51:12.767Z [INFO] server-407b101d-45d1-193f-6596-ce28ee9a8d98: entering follower state: follower="Node at 407b101d-45d1-193f-6596-ce28ee9a8d98 [Follower]" leader-address= leader-id= +2025-03-20T03:51:12.835Z [WARN] server-407b101d-45d1-193f-6596-ce28ee9a8d98: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:12.835Z [INFO] server-407b101d-45d1-193f-6596-ce28ee9a8d98: entering candidate state: node="Node at 407b101d-45d1-193f-6596-ce28ee9a8d98 [Candidate]" term=2 +2025-03-20T03:51:12.836Z [INFO] server-407b101d-45d1-193f-6596-ce28ee9a8d98: election won: term=2 tally=2 +2025-03-20T03:51:12.836Z [INFO] server-407b101d-45d1-193f-6596-ce28ee9a8d98: entering leader state: leader="Node at 407b101d-45d1-193f-6596-ce28ee9a8d98 [Leader]" +2025-03-20T03:51:12.836Z [INFO] server-407b101d-45d1-193f-6596-ce28ee9a8d98: added peer, starting replication: peer=server-bc390438-72e5-a439-1847-b3eb3be666ed +2025-03-20T03:51:12.836Z [INFO] server-407b101d-45d1-193f-6596-ce28ee9a8d98: added peer, starting replication: peer=server-c946fb51-23c0-b345-2ae5-72a1612ee5e4 +2025-03-20T03:51:12.837Z [INFO] server-407b101d-45d1-193f-6596-ce28ee9a8d98: pipelining replication: peer="{Voter server-bc390438-72e5-a439-1847-b3eb3be666ed bc390438-72e5-a439-1847-b3eb3be666ed}" +2025-03-20T03:51:12.837Z [INFO] server-407b101d-45d1-193f-6596-ce28ee9a8d98: pipelining replication: peer="{Voter server-c946fb51-23c0-b345-2ae5-72a1612ee5e4 c946fb51-23c0-b345-2ae5-72a1612ee5e4}" +2025-03-20T03:51:12.942Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:12.836948059 +0000 UTC m=+104.511400889 (1 nodes), 69.578835ms from start of poll, 70.160531ms from cluster start. Timeout at 2025-03-20 03:51:12.941950629 +0000 UTC m=+104.616403557, 105.002668ms after stability +2025-03-20T03:51:12.942Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:13.047Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:12.942131859 +0000 UTC m=+104.616584809 (1 nodes), 2.055µs from start of poll, 175.344451ms from cluster start. Timeout at 2025-03-20 03:51:13.047184264 +0000 UTC m=+104.721637214, 105.052405ms after stability +2025-03-20T03:51:13.048Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:13.154Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:13.04807178 +0000 UTC m=+104.722524632 (1 nodes), 1.97µs from start of poll, 281.284274ms from cluster start. Timeout at 2025-03-20 03:51:13.153201463 +0000 UTC m=+104.827654375, 105.129743ms after stability +2025-03-20T03:51:13.155Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:13.260Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:13.155040319 +0000 UTC m=+104.829493234 (1 nodes), 1.916µs from start of poll, 388.252876ms from cluster start. Timeout at 2025-03-20 03:51:13.260144976 +0000 UTC m=+104.934597921, 105.104687ms after stability +2025-03-20T03:51:13.261Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:13.383Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:13.26115254 +0000 UTC m=+104.935605415 (1 nodes), 1.933µs from start of poll, 494.365057ms from cluster start. Timeout at 2025-03-20 03:51:13.36765037 +0000 UTC m=+105.042103737, 106.498322ms after stability +2025-03-20T03:51:13.384Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:13.494Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:13.384078519 +0000 UTC m=+105.058531802 (1 nodes), 3.264µs from start of poll, 617.291444ms from cluster start. Timeout at 2025-03-20 03:51:13.489208131 +0000 UTC m=+105.163660897, 105.129095ms after stability +2025-03-20T03:51:13.600Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:13.705Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:13.600766379 +0000 UTC m=+105.275219300 (1 nodes), 1.941µs from start of poll, 833.978942ms from cluster start. Timeout at 2025-03-20 03:51:13.705821089 +0000 UTC m=+105.380274588, 105.055288ms after stability +2025-03-20T03:51:13.706Z [INFO] server-407b101d-45d1-193f-6596-ce28ee9a8d98: aborting pipeline replication: peer="{Voter server-c946fb51-23c0-b345-2ae5-72a1612ee5e4 c946fb51-23c0-b345-2ae5-72a1612ee5e4}" +2025-03-20T03:51:13.706Z [INFO] server-407b101d-45d1-193f-6596-ce28ee9a8d98: aborting pipeline replication: peer="{Voter server-bc390438-72e5-a439-1847-b3eb3be666ed bc390438-72e5-a439-1847-b3eb3be666ed}" +2025-03-20T03:51:14.206Z [ERROR] server-407b101d-45d1-193f-6596-ce28ee9a8d98: failed to heartbeat to: peer=bc390438-72e5-a439-1847-b3eb3be666ed backoff time=10ms error="command timed out" +--- PASS: TestRaft_LeaderID_Propagated (1.45s) === RUN TestRaft_LeadershipTransferInProgress --- PASS: TestRaft_LeadershipTransferInProgress (0.00s) === RUN TestRaft_LeadershipTransferPickServer --- PASS: TestRaft_LeadershipTransferPickServer (0.00s) === RUN TestRaft_LeadershipTransfer -2026-04-22T10:10:56.516Z [INFO] server-45631c7f-dc65-64c7-2244-a218e8ab65b1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-45631c7f-dc65-64c7-2244-a218e8ab65b1 Address:45631c7f-dc65-64c7-2244-a218e8ab65b1} {Suffrage:Voter ID:server-f83b9a4d-5666-f038-68bc-8d477737c76e Address:f83b9a4d-5666-f038-68bc-8d477737c76e} {Suffrage:Voter ID:server-d341e058-e483-b2d3-0509-4356f6738cd5 Address:d341e058-e483-b2d3-0509-4356f6738cd5}]" -2026-04-22T10:10:56.516Z [INFO] server-45631c7f-dc65-64c7-2244-a218e8ab65b1: entering follower state: follower="Node at 45631c7f-dc65-64c7-2244-a218e8ab65b1 [Follower]" leader-address= leader-id= -2026-04-22T10:10:56.516Z [INFO] server-f83b9a4d-5666-f038-68bc-8d477737c76e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-45631c7f-dc65-64c7-2244-a218e8ab65b1 Address:45631c7f-dc65-64c7-2244-a218e8ab65b1} {Suffrage:Voter ID:server-f83b9a4d-5666-f038-68bc-8d477737c76e Address:f83b9a4d-5666-f038-68bc-8d477737c76e} {Suffrage:Voter ID:server-d341e058-e483-b2d3-0509-4356f6738cd5 Address:d341e058-e483-b2d3-0509-4356f6738cd5}]" -2026-04-22T10:10:56.516Z [INFO] server-f83b9a4d-5666-f038-68bc-8d477737c76e: entering follower state: follower="Node at f83b9a4d-5666-f038-68bc-8d477737c76e [Follower]" leader-address= leader-id= -2026-04-22T10:10:56.516Z [INFO] server-d341e058-e483-b2d3-0509-4356f6738cd5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-45631c7f-dc65-64c7-2244-a218e8ab65b1 Address:45631c7f-dc65-64c7-2244-a218e8ab65b1} {Suffrage:Voter ID:server-f83b9a4d-5666-f038-68bc-8d477737c76e Address:f83b9a4d-5666-f038-68bc-8d477737c76e} {Suffrage:Voter ID:server-d341e058-e483-b2d3-0509-4356f6738cd5 Address:d341e058-e483-b2d3-0509-4356f6738cd5}]" -2026-04-22T10:10:56.516Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:56.516Z [INFO] server-d341e058-e483-b2d3-0509-4356f6738cd5: entering follower state: follower="Node at d341e058-e483-b2d3-0509-4356f6738cd5 [Follower]" leader-address= leader-id= -2026-04-22T10:10:56.569Z [WARN] server-f83b9a4d-5666-f038-68bc-8d477737c76e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:56.569Z [INFO] server-f83b9a4d-5666-f038-68bc-8d477737c76e: entering candidate state: node="Node at f83b9a4d-5666-f038-68bc-8d477737c76e [Candidate]" term=2 -2026-04-22T10:10:56.569Z [INFO] server-f83b9a4d-5666-f038-68bc-8d477737c76e: election won: term=2 tally=2 -2026-04-22T10:10:56.569Z [INFO] server-f83b9a4d-5666-f038-68bc-8d477737c76e: entering leader state: leader="Node at f83b9a4d-5666-f038-68bc-8d477737c76e [Leader]" -2026-04-22T10:10:56.569Z [INFO] server-f83b9a4d-5666-f038-68bc-8d477737c76e: added peer, starting replication: peer=server-45631c7f-dc65-64c7-2244-a218e8ab65b1 -2026-04-22T10:10:56.569Z [INFO] server-f83b9a4d-5666-f038-68bc-8d477737c76e: added peer, starting replication: peer=server-d341e058-e483-b2d3-0509-4356f6738cd5 -2026-04-22T10:10:56.569Z [INFO] server-f83b9a4d-5666-f038-68bc-8d477737c76e: pipelining replication: peer="{Voter server-45631c7f-dc65-64c7-2244-a218e8ab65b1 45631c7f-dc65-64c7-2244-a218e8ab65b1}" -2026-04-22T10:10:56.569Z [INFO] server-f83b9a4d-5666-f038-68bc-8d477737c76e: pipelining replication: peer="{Voter server-d341e058-e483-b2d3-0509-4356f6738cd5 d341e058-e483-b2d3-0509-4356f6738cd5}" -2026-04-22T10:10:56.675Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:56.569602878 +0000 UTC m=+102.593398999 (1 nodes), 53.212325ms from start of poll, 53.424252ms from cluster start. Timeout at 2026-04-22 10:10:56.674606918 +0000 UTC m=+102.698402999, 105.004ms after stability -2026-04-22T10:10:56.675Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:56.781Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:56.675325758 +0000 UTC m=+102.699121845 (1 nodes), 428ns from start of poll, 159.147098ms from cluster start. Timeout at 2026-04-22 10:10:56.780333064 +0000 UTC m=+102.804129194, 105.007349ms after stability -2026-04-22T10:10:56.781Z [INFO] server-45631c7f-dc65-64c7-2244-a218e8ab65b1: entering candidate state: node="Node at 45631c7f-dc65-64c7-2244-a218e8ab65b1 [Candidate]" term=3 -2026-04-22T10:10:56.781Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:56.781Z [INFO] server-45631c7f-dc65-64c7-2244-a218e8ab65b1: election won: term=3 tally=2 -2026-04-22T10:10:56.781Z [INFO] server-45631c7f-dc65-64c7-2244-a218e8ab65b1: entering leader state: leader="Node at 45631c7f-dc65-64c7-2244-a218e8ab65b1 [Leader]" -2026-04-22T10:10:56.781Z [INFO] server-45631c7f-dc65-64c7-2244-a218e8ab65b1: added peer, starting replication: peer=server-f83b9a4d-5666-f038-68bc-8d477737c76e -2026-04-22T10:10:56.781Z [INFO] server-45631c7f-dc65-64c7-2244-a218e8ab65b1: added peer, starting replication: peer=server-d341e058-e483-b2d3-0509-4356f6738cd5 -2026-04-22T10:10:56.781Z [INFO] server-f83b9a4d-5666-f038-68bc-8d477737c76e: entering follower state: follower="Node at f83b9a4d-5666-f038-68bc-8d477737c76e [Follower]" leader-address= leader-id= -2026-04-22T10:10:56.781Z [INFO] server-f83b9a4d-5666-f038-68bc-8d477737c76e: aborting pipeline replication: peer="{Voter server-45631c7f-dc65-64c7-2244-a218e8ab65b1 45631c7f-dc65-64c7-2244-a218e8ab65b1}" -2026-04-22T10:10:56.781Z [INFO] server-f83b9a4d-5666-f038-68bc-8d477737c76e: aborting pipeline replication: peer="{Voter server-d341e058-e483-b2d3-0509-4356f6738cd5 d341e058-e483-b2d3-0509-4356f6738cd5}" -2026-04-22T10:10:56.781Z [INFO] server-45631c7f-dc65-64c7-2244-a218e8ab65b1: pipelining replication: peer="{Voter server-d341e058-e483-b2d3-0509-4356f6738cd5 d341e058-e483-b2d3-0509-4356f6738cd5}" -2026-04-22T10:10:56.781Z [INFO] server-45631c7f-dc65-64c7-2244-a218e8ab65b1: pipelining replication: peer="{Voter server-f83b9a4d-5666-f038-68bc-8d477737c76e f83b9a4d-5666-f038-68bc-8d477737c76e}" -2026-04-22T10:10:56.887Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:56.781211886 +0000 UTC m=+102.805007979 (1 nodes), 77.045µs from start of poll, 265.033232ms from cluster start. Timeout at 2026-04-22 10:10:56.886226349 +0000 UTC m=+102.910022589, 105.01461ms after stability -2026-04-22T10:10:56.887Z [INFO] server-45631c7f-dc65-64c7-2244-a218e8ab65b1: aborting pipeline replication: peer="{Voter server-d341e058-e483-b2d3-0509-4356f6738cd5 d341e058-e483-b2d3-0509-4356f6738cd5}" -2026-04-22T10:10:56.887Z [INFO] server-45631c7f-dc65-64c7-2244-a218e8ab65b1: aborting pipeline replication: peer="{Voter server-f83b9a4d-5666-f038-68bc-8d477737c76e f83b9a4d-5666-f038-68bc-8d477737c76e}" ---- PASS: TestRaft_LeadershipTransfer (0.37s) +2025-03-20T03:51:14.209Z [INFO] server-5d0501f9-e172-066c-1e7c-44aa734a3047: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5d0501f9-e172-066c-1e7c-44aa734a3047 Address:5d0501f9-e172-066c-1e7c-44aa734a3047} {Suffrage:Voter ID:server-3589d223-22a6-6640-da5f-38ad530e83e8 Address:3589d223-22a6-6640-da5f-38ad530e83e8} {Suffrage:Voter ID:server-6edb12e8-6c18-0344-d0f0-13ecdea61c87 Address:6edb12e8-6c18-0344-d0f0-13ecdea61c87}]" +2025-03-20T03:51:14.209Z [INFO] server-3589d223-22a6-6640-da5f-38ad530e83e8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5d0501f9-e172-066c-1e7c-44aa734a3047 Address:5d0501f9-e172-066c-1e7c-44aa734a3047} {Suffrage:Voter ID:server-3589d223-22a6-6640-da5f-38ad530e83e8 Address:3589d223-22a6-6640-da5f-38ad530e83e8} {Suffrage:Voter ID:server-6edb12e8-6c18-0344-d0f0-13ecdea61c87 Address:6edb12e8-6c18-0344-d0f0-13ecdea61c87}]" +2025-03-20T03:51:14.209Z [INFO] server-5d0501f9-e172-066c-1e7c-44aa734a3047: entering follower state: follower="Node at 5d0501f9-e172-066c-1e7c-44aa734a3047 [Follower]" leader-address= leader-id= +2025-03-20T03:51:14.209Z [INFO] server-3589d223-22a6-6640-da5f-38ad530e83e8: entering follower state: follower="Node at 3589d223-22a6-6640-da5f-38ad530e83e8 [Follower]" leader-address= leader-id= +2025-03-20T03:51:14.209Z [INFO] server-6edb12e8-6c18-0344-d0f0-13ecdea61c87: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5d0501f9-e172-066c-1e7c-44aa734a3047 Address:5d0501f9-e172-066c-1e7c-44aa734a3047} {Suffrage:Voter ID:server-3589d223-22a6-6640-da5f-38ad530e83e8 Address:3589d223-22a6-6640-da5f-38ad530e83e8} {Suffrage:Voter ID:server-6edb12e8-6c18-0344-d0f0-13ecdea61c87 Address:6edb12e8-6c18-0344-d0f0-13ecdea61c87}]" +2025-03-20T03:51:14.209Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:14.209Z [INFO] server-6edb12e8-6c18-0344-d0f0-13ecdea61c87: entering follower state: follower="Node at 6edb12e8-6c18-0344-d0f0-13ecdea61c87 [Follower]" leader-address= leader-id= +2025-03-20T03:51:14.264Z [WARN] server-6edb12e8-6c18-0344-d0f0-13ecdea61c87: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:14.264Z [INFO] server-6edb12e8-6c18-0344-d0f0-13ecdea61c87: entering candidate state: node="Node at 6edb12e8-6c18-0344-d0f0-13ecdea61c87 [Candidate]" term=2 +2025-03-20T03:51:14.264Z [INFO] server-6edb12e8-6c18-0344-d0f0-13ecdea61c87: election won: term=2 tally=2 +2025-03-20T03:51:14.264Z [INFO] server-6edb12e8-6c18-0344-d0f0-13ecdea61c87: entering leader state: leader="Node at 6edb12e8-6c18-0344-d0f0-13ecdea61c87 [Leader]" +2025-03-20T03:51:14.265Z [INFO] server-6edb12e8-6c18-0344-d0f0-13ecdea61c87: added peer, starting replication: peer=server-5d0501f9-e172-066c-1e7c-44aa734a3047 +2025-03-20T03:51:14.265Z [INFO] server-6edb12e8-6c18-0344-d0f0-13ecdea61c87: added peer, starting replication: peer=server-3589d223-22a6-6640-da5f-38ad530e83e8 +2025-03-20T03:51:14.265Z [INFO] server-6edb12e8-6c18-0344-d0f0-13ecdea61c87: pipelining replication: peer="{Voter server-3589d223-22a6-6640-da5f-38ad530e83e8 3589d223-22a6-6640-da5f-38ad530e83e8}" +2025-03-20T03:51:14.265Z [INFO] server-6edb12e8-6c18-0344-d0f0-13ecdea61c87: pipelining replication: peer="{Voter server-5d0501f9-e172-066c-1e7c-44aa734a3047 5d0501f9-e172-066c-1e7c-44aa734a3047}" +2025-03-20T03:51:14.373Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:14.265328117 +0000 UTC m=+105.939781133 (1 nodes), 55.684258ms from start of poll, 56.166059ms from cluster start. Timeout at 2025-03-20 03:51:14.370353059 +0000 UTC m=+106.044806001, 105.024868ms after stability +2025-03-20T03:51:14.373Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:14.479Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:14.373382508 +0000 UTC m=+106.047835455 (1 nodes), 1.945µs from start of poll, 164.220381ms from cluster start. Timeout at 2025-03-20 03:51:14.47845173 +0000 UTC m=+106.152904589, 105.069134ms after stability +2025-03-20T03:51:14.479Z [INFO] server-5d0501f9-e172-066c-1e7c-44aa734a3047: entering candidate state: node="Node at 5d0501f9-e172-066c-1e7c-44aa734a3047 [Candidate]" term=3 +2025-03-20T03:51:14.479Z [INFO] server-6edb12e8-6c18-0344-d0f0-13ecdea61c87: entering follower state: follower="Node at 6edb12e8-6c18-0344-d0f0-13ecdea61c87 [Follower]" leader-address= leader-id= +2025-03-20T03:51:14.479Z [INFO] server-6edb12e8-6c18-0344-d0f0-13ecdea61c87: aborting pipeline replication: peer="{Voter server-3589d223-22a6-6640-da5f-38ad530e83e8 3589d223-22a6-6640-da5f-38ad530e83e8}" +2025-03-20T03:51:14.479Z [INFO] server-5d0501f9-e172-066c-1e7c-44aa734a3047: election won: term=3 tally=2 +2025-03-20T03:51:14.479Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:14.479Z [INFO] server-5d0501f9-e172-066c-1e7c-44aa734a3047: entering leader state: leader="Node at 5d0501f9-e172-066c-1e7c-44aa734a3047 [Leader]" +2025-03-20T03:51:14.479Z [INFO] server-5d0501f9-e172-066c-1e7c-44aa734a3047: added peer, starting replication: peer=server-3589d223-22a6-6640-da5f-38ad530e83e8 +2025-03-20T03:51:14.479Z [INFO] server-5d0501f9-e172-066c-1e7c-44aa734a3047: added peer, starting replication: peer=server-6edb12e8-6c18-0344-d0f0-13ecdea61c87 +2025-03-20T03:51:14.479Z [INFO] server-6edb12e8-6c18-0344-d0f0-13ecdea61c87: aborting pipeline replication: peer="{Voter server-5d0501f9-e172-066c-1e7c-44aa734a3047 5d0501f9-e172-066c-1e7c-44aa734a3047}" +2025-03-20T03:51:14.479Z [INFO] server-5d0501f9-e172-066c-1e7c-44aa734a3047: pipelining replication: peer="{Voter server-6edb12e8-6c18-0344-d0f0-13ecdea61c87 6edb12e8-6c18-0344-d0f0-13ecdea61c87}" +2025-03-20T03:51:14.479Z [INFO] server-5d0501f9-e172-066c-1e7c-44aa734a3047: pipelining replication: peer="{Voter server-3589d223-22a6-6640-da5f-38ad530e83e8 3589d223-22a6-6640-da5f-38ad530e83e8}" +2025-03-20T03:51:14.585Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:14.479587543 +0000 UTC m=+106.154040406 (1 nodes), 155.197µs from start of poll, 270.425332ms from cluster start. Timeout at 2025-03-20 03:51:14.584766843 +0000 UTC m=+106.259219829, 105.179423ms after stability +2025-03-20T03:51:14.586Z [INFO] server-5d0501f9-e172-066c-1e7c-44aa734a3047: aborting pipeline replication: peer="{Voter server-3589d223-22a6-6640-da5f-38ad530e83e8 3589d223-22a6-6640-da5f-38ad530e83e8}" +2025-03-20T03:51:14.587Z [INFO] server-5d0501f9-e172-066c-1e7c-44aa734a3047: aborting pipeline replication: peer="{Voter server-6edb12e8-6c18-0344-d0f0-13ecdea61c87 6edb12e8-6c18-0344-d0f0-13ecdea61c87}" +2025-03-20T03:51:15.090Z [ERROR] server-5d0501f9-e172-066c-1e7c-44aa734a3047: failed to heartbeat to: peer=6edb12e8-6c18-0344-d0f0-13ecdea61c87 backoff time=10ms error="command timed out" +--- PASS: TestRaft_LeadershipTransfer (0.88s) === RUN TestRaft_LeadershipTransferWithOneNode -2026-04-22T10:10:56.888Z [INFO] server-bcd41fa6-8ef6-1348-3a49-c92dcba959d7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bcd41fa6-8ef6-1348-3a49-c92dcba959d7 Address:bcd41fa6-8ef6-1348-3a49-c92dcba959d7}]" -2026-04-22T10:10:56.888Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:56.888Z [INFO] server-bcd41fa6-8ef6-1348-3a49-c92dcba959d7: entering follower state: follower="Node at bcd41fa6-8ef6-1348-3a49-c92dcba959d7 [Follower]" leader-address= leader-id= -2026-04-22T10:10:56.942Z [WARN] server-bcd41fa6-8ef6-1348-3a49-c92dcba959d7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:56.942Z [INFO] server-bcd41fa6-8ef6-1348-3a49-c92dcba959d7: entering candidate state: node="Node at bcd41fa6-8ef6-1348-3a49-c92dcba959d7 [Candidate]" term=2 -2026-04-22T10:10:56.942Z [INFO] server-bcd41fa6-8ef6-1348-3a49-c92dcba959d7: election won: term=2 tally=1 -2026-04-22T10:10:56.942Z [INFO] server-bcd41fa6-8ef6-1348-3a49-c92dcba959d7: entering leader state: leader="Node at bcd41fa6-8ef6-1348-3a49-c92dcba959d7 [Leader]" -2026-04-22T10:10:57.048Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:56.94295473 +0000 UTC m=+102.966750843 (1 nodes), 54.698009ms from start of poll, 54.820359ms from cluster start. Timeout at 2026-04-22 10:10:57.047965547 +0000 UTC m=+103.071761683, 105.01084ms after stability ---- PASS: TestRaft_LeadershipTransferWithOneNode (0.16s) +2025-03-20T03:51:15.092Z [INFO] server-469123d4-d64a-7718-b63e-1b5e6ae071d3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-469123d4-d64a-7718-b63e-1b5e6ae071d3 Address:469123d4-d64a-7718-b63e-1b5e6ae071d3}]" +2025-03-20T03:51:15.092Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:15.092Z [INFO] server-469123d4-d64a-7718-b63e-1b5e6ae071d3: entering follower state: follower="Node at 469123d4-d64a-7718-b63e-1b5e6ae071d3 [Follower]" leader-address= leader-id= +2025-03-20T03:51:15.150Z [WARN] server-469123d4-d64a-7718-b63e-1b5e6ae071d3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:15.150Z [INFO] server-469123d4-d64a-7718-b63e-1b5e6ae071d3: entering candidate state: node="Node at 469123d4-d64a-7718-b63e-1b5e6ae071d3 [Candidate]" term=2 +2025-03-20T03:51:15.150Z [INFO] server-469123d4-d64a-7718-b63e-1b5e6ae071d3: election won: term=2 tally=1 +2025-03-20T03:51:15.150Z [INFO] server-469123d4-d64a-7718-b63e-1b5e6ae071d3: entering leader state: leader="Node at 469123d4-d64a-7718-b63e-1b5e6ae071d3 [Leader]" +2025-03-20T03:51:15.256Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:15.15087085 +0000 UTC m=+106.825323745 (1 nodes), 58.61986ms from start of poll, 58.767089ms from cluster start. Timeout at 2025-03-20 03:51:15.255882202 +0000 UTC m=+106.930335731, 105.011986ms after stability +--- PASS: TestRaft_LeadershipTransferWithOneNode (0.17s) === RUN TestRaft_LeadershipTransferWithSevenNodes -2026-04-22T10:10:57.050Z [INFO] server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a Address:77ca3a41-b7c2-8c2f-f14d-79c56d6d764a} {Suffrage:Voter ID:server-bf574c04-07df-880c-460e-efc428cd5991 Address:bf574c04-07df-880c-460e-efc428cd5991} {Suffrage:Voter ID:server-6b370faa-80e7-008d-a526-dc6a356b2adf Address:6b370faa-80e7-008d-a526-dc6a356b2adf} {Suffrage:Voter ID:server-6fdcffea-780c-54c9-5b52-66b8d441d70e Address:6fdcffea-780c-54c9-5b52-66b8d441d70e} {Suffrage:Voter ID:server-d1294074-ce41-48b4-1359-1206d3c13bbb Address:d1294074-ce41-48b4-1359-1206d3c13bbb} {Suffrage:Voter ID:server-dadab481-6273-c354-6558-8456d105bfa9 Address:dadab481-6273-c354-6558-8456d105bfa9} {Suffrage:Voter ID:server-34b8cf1b-1b4e-ca4d-fc64-1f1fb7f222c3 Address:34b8cf1b-1b4e-ca4d-fc64-1f1fb7f222c3}]" -2026-04-22T10:10:57.050Z [INFO] server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a: entering follower state: follower="Node at 77ca3a41-b7c2-8c2f-f14d-79c56d6d764a [Follower]" leader-address= leader-id= -2026-04-22T10:10:57.050Z [INFO] server-bf574c04-07df-880c-460e-efc428cd5991: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a Address:77ca3a41-b7c2-8c2f-f14d-79c56d6d764a} {Suffrage:Voter ID:server-bf574c04-07df-880c-460e-efc428cd5991 Address:bf574c04-07df-880c-460e-efc428cd5991} {Suffrage:Voter ID:server-6b370faa-80e7-008d-a526-dc6a356b2adf Address:6b370faa-80e7-008d-a526-dc6a356b2adf} {Suffrage:Voter ID:server-6fdcffea-780c-54c9-5b52-66b8d441d70e Address:6fdcffea-780c-54c9-5b52-66b8d441d70e} {Suffrage:Voter ID:server-d1294074-ce41-48b4-1359-1206d3c13bbb Address:d1294074-ce41-48b4-1359-1206d3c13bbb} {Suffrage:Voter ID:server-dadab481-6273-c354-6558-8456d105bfa9 Address:dadab481-6273-c354-6558-8456d105bfa9} {Suffrage:Voter ID:server-34b8cf1b-1b4e-ca4d-fc64-1f1fb7f222c3 Address:34b8cf1b-1b4e-ca4d-fc64-1f1fb7f222c3}]" -2026-04-22T10:10:57.050Z [INFO] server-bf574c04-07df-880c-460e-efc428cd5991: entering follower state: follower="Node at bf574c04-07df-880c-460e-efc428cd5991 [Follower]" leader-address= leader-id= -2026-04-22T10:10:57.050Z [INFO] server-6b370faa-80e7-008d-a526-dc6a356b2adf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a Address:77ca3a41-b7c2-8c2f-f14d-79c56d6d764a} {Suffrage:Voter ID:server-bf574c04-07df-880c-460e-efc428cd5991 Address:bf574c04-07df-880c-460e-efc428cd5991} {Suffrage:Voter ID:server-6b370faa-80e7-008d-a526-dc6a356b2adf Address:6b370faa-80e7-008d-a526-dc6a356b2adf} {Suffrage:Voter ID:server-6fdcffea-780c-54c9-5b52-66b8d441d70e Address:6fdcffea-780c-54c9-5b52-66b8d441d70e} {Suffrage:Voter ID:server-d1294074-ce41-48b4-1359-1206d3c13bbb Address:d1294074-ce41-48b4-1359-1206d3c13bbb} {Suffrage:Voter ID:server-dadab481-6273-c354-6558-8456d105bfa9 Address:dadab481-6273-c354-6558-8456d105bfa9} {Suffrage:Voter ID:server-34b8cf1b-1b4e-ca4d-fc64-1f1fb7f222c3 Address:34b8cf1b-1b4e-ca4d-fc64-1f1fb7f222c3}]" -2026-04-22T10:10:57.050Z [INFO] server-6b370faa-80e7-008d-a526-dc6a356b2adf: entering follower state: follower="Node at 6b370faa-80e7-008d-a526-dc6a356b2adf [Follower]" leader-address= leader-id= -2026-04-22T10:10:57.050Z [INFO] server-6fdcffea-780c-54c9-5b52-66b8d441d70e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a Address:77ca3a41-b7c2-8c2f-f14d-79c56d6d764a} {Suffrage:Voter ID:server-bf574c04-07df-880c-460e-efc428cd5991 Address:bf574c04-07df-880c-460e-efc428cd5991} {Suffrage:Voter ID:server-6b370faa-80e7-008d-a526-dc6a356b2adf Address:6b370faa-80e7-008d-a526-dc6a356b2adf} {Suffrage:Voter ID:server-6fdcffea-780c-54c9-5b52-66b8d441d70e Address:6fdcffea-780c-54c9-5b52-66b8d441d70e} {Suffrage:Voter ID:server-d1294074-ce41-48b4-1359-1206d3c13bbb Address:d1294074-ce41-48b4-1359-1206d3c13bbb} {Suffrage:Voter ID:server-dadab481-6273-c354-6558-8456d105bfa9 Address:dadab481-6273-c354-6558-8456d105bfa9} {Suffrage:Voter ID:server-34b8cf1b-1b4e-ca4d-fc64-1f1fb7f222c3 Address:34b8cf1b-1b4e-ca4d-fc64-1f1fb7f222c3}]" -2026-04-22T10:10:57.050Z [INFO] server-6fdcffea-780c-54c9-5b52-66b8d441d70e: entering follower state: follower="Node at 6fdcffea-780c-54c9-5b52-66b8d441d70e [Follower]" leader-address= leader-id= -2026-04-22T10:10:57.050Z [INFO] server-d1294074-ce41-48b4-1359-1206d3c13bbb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a Address:77ca3a41-b7c2-8c2f-f14d-79c56d6d764a} {Suffrage:Voter ID:server-bf574c04-07df-880c-460e-efc428cd5991 Address:bf574c04-07df-880c-460e-efc428cd5991} {Suffrage:Voter ID:server-6b370faa-80e7-008d-a526-dc6a356b2adf Address:6b370faa-80e7-008d-a526-dc6a356b2adf} {Suffrage:Voter ID:server-6fdcffea-780c-54c9-5b52-66b8d441d70e Address:6fdcffea-780c-54c9-5b52-66b8d441d70e} {Suffrage:Voter ID:server-d1294074-ce41-48b4-1359-1206d3c13bbb Address:d1294074-ce41-48b4-1359-1206d3c13bbb} {Suffrage:Voter ID:server-dadab481-6273-c354-6558-8456d105bfa9 Address:dadab481-6273-c354-6558-8456d105bfa9} {Suffrage:Voter ID:server-34b8cf1b-1b4e-ca4d-fc64-1f1fb7f222c3 Address:34b8cf1b-1b4e-ca4d-fc64-1f1fb7f222c3}]" -2026-04-22T10:10:57.051Z [INFO] server-d1294074-ce41-48b4-1359-1206d3c13bbb: entering follower state: follower="Node at d1294074-ce41-48b4-1359-1206d3c13bbb [Follower]" leader-address= leader-id= -2026-04-22T10:10:57.051Z [INFO] server-dadab481-6273-c354-6558-8456d105bfa9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a Address:77ca3a41-b7c2-8c2f-f14d-79c56d6d764a} {Suffrage:Voter ID:server-bf574c04-07df-880c-460e-efc428cd5991 Address:bf574c04-07df-880c-460e-efc428cd5991} {Suffrage:Voter ID:server-6b370faa-80e7-008d-a526-dc6a356b2adf Address:6b370faa-80e7-008d-a526-dc6a356b2adf} {Suffrage:Voter ID:server-6fdcffea-780c-54c9-5b52-66b8d441d70e Address:6fdcffea-780c-54c9-5b52-66b8d441d70e} {Suffrage:Voter ID:server-d1294074-ce41-48b4-1359-1206d3c13bbb Address:d1294074-ce41-48b4-1359-1206d3c13bbb} {Suffrage:Voter ID:server-dadab481-6273-c354-6558-8456d105bfa9 Address:dadab481-6273-c354-6558-8456d105bfa9} {Suffrage:Voter ID:server-34b8cf1b-1b4e-ca4d-fc64-1f1fb7f222c3 Address:34b8cf1b-1b4e-ca4d-fc64-1f1fb7f222c3}]" -2026-04-22T10:10:57.051Z [INFO] server-dadab481-6273-c354-6558-8456d105bfa9: entering follower state: follower="Node at dadab481-6273-c354-6558-8456d105bfa9 [Follower]" leader-address= leader-id= -2026-04-22T10:10:57.051Z [INFO] server-34b8cf1b-1b4e-ca4d-fc64-1f1fb7f222c3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a Address:77ca3a41-b7c2-8c2f-f14d-79c56d6d764a} {Suffrage:Voter ID:server-bf574c04-07df-880c-460e-efc428cd5991 Address:bf574c04-07df-880c-460e-efc428cd5991} {Suffrage:Voter ID:server-6b370faa-80e7-008d-a526-dc6a356b2adf Address:6b370faa-80e7-008d-a526-dc6a356b2adf} {Suffrage:Voter ID:server-6fdcffea-780c-54c9-5b52-66b8d441d70e Address:6fdcffea-780c-54c9-5b52-66b8d441d70e} {Suffrage:Voter ID:server-d1294074-ce41-48b4-1359-1206d3c13bbb Address:d1294074-ce41-48b4-1359-1206d3c13bbb} {Suffrage:Voter ID:server-dadab481-6273-c354-6558-8456d105bfa9 Address:dadab481-6273-c354-6558-8456d105bfa9} {Suffrage:Voter ID:server-34b8cf1b-1b4e-ca4d-fc64-1f1fb7f222c3 Address:34b8cf1b-1b4e-ca4d-fc64-1f1fb7f222c3}]" -2026-04-22T10:10:57.051Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:57.051Z [INFO] server-34b8cf1b-1b4e-ca4d-fc64-1f1fb7f222c3: entering follower state: follower="Node at 34b8cf1b-1b4e-ca4d-fc64-1f1fb7f222c3 [Follower]" leader-address= leader-id= -2026-04-22T10:10:57.115Z [WARN] server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:57.115Z [INFO] server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a: entering candidate state: node="Node at 77ca3a41-b7c2-8c2f-f14d-79c56d6d764a [Candidate]" term=2 -2026-04-22T10:10:57.115Z [INFO] server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a: election won: term=2 tally=4 -2026-04-22T10:10:57.115Z [INFO] server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a: entering leader state: leader="Node at 77ca3a41-b7c2-8c2f-f14d-79c56d6d764a [Leader]" -2026-04-22T10:10:57.115Z [INFO] server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a: added peer, starting replication: peer=server-bf574c04-07df-880c-460e-efc428cd5991 -2026-04-22T10:10:57.115Z [INFO] server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a: added peer, starting replication: peer=server-6b370faa-80e7-008d-a526-dc6a356b2adf -2026-04-22T10:10:57.115Z [INFO] server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a: added peer, starting replication: peer=server-6fdcffea-780c-54c9-5b52-66b8d441d70e -2026-04-22T10:10:57.115Z [INFO] server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a: added peer, starting replication: peer=server-d1294074-ce41-48b4-1359-1206d3c13bbb -2026-04-22T10:10:57.115Z [INFO] server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a: added peer, starting replication: peer=server-dadab481-6273-c354-6558-8456d105bfa9 -2026-04-22T10:10:57.115Z [INFO] server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a: added peer, starting replication: peer=server-34b8cf1b-1b4e-ca4d-fc64-1f1fb7f222c3 -2026-04-22T10:10:57.116Z [INFO] server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a: pipelining replication: peer="{Voter server-bf574c04-07df-880c-460e-efc428cd5991 bf574c04-07df-880c-460e-efc428cd5991}" -2026-04-22T10:10:57.116Z [INFO] server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a: pipelining replication: peer="{Voter server-6b370faa-80e7-008d-a526-dc6a356b2adf 6b370faa-80e7-008d-a526-dc6a356b2adf}" -2026-04-22T10:10:57.116Z [INFO] server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a: pipelining replication: peer="{Voter server-6fdcffea-780c-54c9-5b52-66b8d441d70e 6fdcffea-780c-54c9-5b52-66b8d441d70e}" -2026-04-22T10:10:57.116Z [INFO] server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a: pipelining replication: peer="{Voter server-d1294074-ce41-48b4-1359-1206d3c13bbb d1294074-ce41-48b4-1359-1206d3c13bbb}" -2026-04-22T10:10:57.116Z [INFO] server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a: pipelining replication: peer="{Voter server-34b8cf1b-1b4e-ca4d-fc64-1f1fb7f222c3 34b8cf1b-1b4e-ca4d-fc64-1f1fb7f222c3}" -2026-04-22T10:10:57.116Z [INFO] server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a: pipelining replication: peer="{Voter server-dadab481-6273-c354-6558-8456d105bfa9 dadab481-6273-c354-6558-8456d105bfa9}" -2026-04-22T10:10:57.221Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:57.115914252 +0000 UTC m=+103.139710366 (1 nodes), 64.635982ms from start of poll, 65.41921ms from cluster start. Timeout at 2026-04-22 10:10:57.220920828 +0000 UTC m=+103.244716961, 105.006595ms after stability -2026-04-22T10:10:57.221Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:10:57.327Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:10:57.221220771 +0000 UTC m=+103.245016978 (6 nodes), 3.181µs from start of poll, 170.725822ms from cluster start. Timeout at 2026-04-22 10:10:57.326245397 +0000 UTC m=+103.350041594, 105.024616ms after stability -2026-04-22T10:10:57.327Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:57.433Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:57.327249184 +0000 UTC m=+103.351045387 (1 nodes), 1.83µs from start of poll, 276.754231ms from cluster start. Timeout at 2026-04-22 10:10:57.432263477 +0000 UTC m=+103.456059575, 105.014188ms after stability -2026-04-22T10:10:57.433Z [INFO] server-bf574c04-07df-880c-460e-efc428cd5991: entering candidate state: node="Node at bf574c04-07df-880c-460e-efc428cd5991 [Candidate]" term=3 -2026-04-22T10:10:57.433Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:57.433Z [INFO] server-bf574c04-07df-880c-460e-efc428cd5991: election won: term=3 tally=4 -2026-04-22T10:10:57.433Z [INFO] server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a: entering follower state: follower="Node at 77ca3a41-b7c2-8c2f-f14d-79c56d6d764a [Follower]" leader-address= leader-id= -2026-04-22T10:10:57.433Z [INFO] server-bf574c04-07df-880c-460e-efc428cd5991: entering leader state: leader="Node at bf574c04-07df-880c-460e-efc428cd5991 [Leader]" -2026-04-22T10:10:57.433Z [INFO] server-bf574c04-07df-880c-460e-efc428cd5991: added peer, starting replication: peer=server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a -2026-04-22T10:10:57.433Z [INFO] server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a: aborting pipeline replication: peer="{Voter server-6b370faa-80e7-008d-a526-dc6a356b2adf 6b370faa-80e7-008d-a526-dc6a356b2adf}" -2026-04-22T10:10:57.433Z [INFO] server-bf574c04-07df-880c-460e-efc428cd5991: added peer, starting replication: peer=server-6b370faa-80e7-008d-a526-dc6a356b2adf -2026-04-22T10:10:57.433Z [INFO] server-bf574c04-07df-880c-460e-efc428cd5991: added peer, starting replication: peer=server-6fdcffea-780c-54c9-5b52-66b8d441d70e -2026-04-22T10:10:57.433Z [INFO] server-bf574c04-07df-880c-460e-efc428cd5991: added peer, starting replication: peer=server-d1294074-ce41-48b4-1359-1206d3c13bbb -2026-04-22T10:10:57.433Z [INFO] server-bf574c04-07df-880c-460e-efc428cd5991: added peer, starting replication: peer=server-dadab481-6273-c354-6558-8456d105bfa9 -2026-04-22T10:10:57.433Z [INFO] server-bf574c04-07df-880c-460e-efc428cd5991: added peer, starting replication: peer=server-34b8cf1b-1b4e-ca4d-fc64-1f1fb7f222c3 -2026-04-22T10:10:57.433Z [INFO] server-bf574c04-07df-880c-460e-efc428cd5991: pipelining replication: peer="{Voter server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a 77ca3a41-b7c2-8c2f-f14d-79c56d6d764a}" -2026-04-22T10:10:57.433Z [INFO] server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a: aborting pipeline replication: peer="{Voter server-34b8cf1b-1b4e-ca4d-fc64-1f1fb7f222c3 34b8cf1b-1b4e-ca4d-fc64-1f1fb7f222c3}" -2026-04-22T10:10:57.433Z [INFO] server-bf574c04-07df-880c-460e-efc428cd5991: pipelining replication: peer="{Voter server-34b8cf1b-1b4e-ca4d-fc64-1f1fb7f222c3 34b8cf1b-1b4e-ca4d-fc64-1f1fb7f222c3}" -2026-04-22T10:10:57.433Z [INFO] server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a: aborting pipeline replication: peer="{Voter server-dadab481-6273-c354-6558-8456d105bfa9 dadab481-6273-c354-6558-8456d105bfa9}" -2026-04-22T10:10:57.433Z [INFO] server-bf574c04-07df-880c-460e-efc428cd5991: pipelining replication: peer="{Voter server-6b370faa-80e7-008d-a526-dc6a356b2adf 6b370faa-80e7-008d-a526-dc6a356b2adf}" -2026-04-22T10:10:57.433Z [INFO] server-bf574c04-07df-880c-460e-efc428cd5991: pipelining replication: peer="{Voter server-d1294074-ce41-48b4-1359-1206d3c13bbb d1294074-ce41-48b4-1359-1206d3c13bbb}" -2026-04-22T10:10:57.433Z [INFO] server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a: aborting pipeline replication: peer="{Voter server-6fdcffea-780c-54c9-5b52-66b8d441d70e 6fdcffea-780c-54c9-5b52-66b8d441d70e}" -2026-04-22T10:10:57.433Z [INFO] server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a: aborting pipeline replication: peer="{Voter server-bf574c04-07df-880c-460e-efc428cd5991 bf574c04-07df-880c-460e-efc428cd5991}" -2026-04-22T10:10:57.433Z [INFO] server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a: aborting pipeline replication: peer="{Voter server-d1294074-ce41-48b4-1359-1206d3c13bbb d1294074-ce41-48b4-1359-1206d3c13bbb}" -2026-04-22T10:10:57.433Z [INFO] server-bf574c04-07df-880c-460e-efc428cd5991: pipelining replication: peer="{Voter server-dadab481-6273-c354-6558-8456d105bfa9 dadab481-6273-c354-6558-8456d105bfa9}" -2026-04-22T10:10:57.433Z [INFO] server-bf574c04-07df-880c-460e-efc428cd5991: pipelining replication: peer="{Voter server-6fdcffea-780c-54c9-5b52-66b8d441d70e 6fdcffea-780c-54c9-5b52-66b8d441d70e}" -2026-04-22T10:10:57.538Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:57.433214217 +0000 UTC m=+103.457010321 (1 nodes), 48.602µs from start of poll, 382.719165ms from cluster start. Timeout at 2026-04-22 10:10:57.53821492 +0000 UTC m=+103.562011001, 105.00068ms after stability -2026-04-22T10:10:57.538Z [INFO] server-bf574c04-07df-880c-460e-efc428cd5991: aborting pipeline replication: peer="{Voter server-d1294074-ce41-48b4-1359-1206d3c13bbb d1294074-ce41-48b4-1359-1206d3c13bbb}" -2026-04-22T10:10:57.538Z [INFO] server-bf574c04-07df-880c-460e-efc428cd5991: aborting pipeline replication: peer="{Voter server-34b8cf1b-1b4e-ca4d-fc64-1f1fb7f222c3 34b8cf1b-1b4e-ca4d-fc64-1f1fb7f222c3}" -2026-04-22T10:10:57.538Z [INFO] server-bf574c04-07df-880c-460e-efc428cd5991: aborting pipeline replication: peer="{Voter server-6b370faa-80e7-008d-a526-dc6a356b2adf 6b370faa-80e7-008d-a526-dc6a356b2adf}" -2026-04-22T10:10:57.538Z [INFO] server-bf574c04-07df-880c-460e-efc428cd5991: aborting pipeline replication: peer="{Voter server-dadab481-6273-c354-6558-8456d105bfa9 dadab481-6273-c354-6558-8456d105bfa9}" -2026-04-22T10:10:57.538Z [INFO] server-bf574c04-07df-880c-460e-efc428cd5991: aborting pipeline replication: peer="{Voter server-77ca3a41-b7c2-8c2f-f14d-79c56d6d764a 77ca3a41-b7c2-8c2f-f14d-79c56d6d764a}" -2026-04-22T10:10:57.538Z [INFO] server-bf574c04-07df-880c-460e-efc428cd5991: aborting pipeline replication: peer="{Voter server-6fdcffea-780c-54c9-5b52-66b8d441d70e 6fdcffea-780c-54c9-5b52-66b8d441d70e}" ---- PASS: TestRaft_LeadershipTransferWithSevenNodes (0.49s) +2025-03-20T03:51:15.259Z [INFO] server-6f4de5a4-abad-2e62-5971-98f4b32882f4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6f4de5a4-abad-2e62-5971-98f4b32882f4 Address:6f4de5a4-abad-2e62-5971-98f4b32882f4} {Suffrage:Voter ID:server-1e5d22e0-c9de-dd8f-20fa-6138f439c93d Address:1e5d22e0-c9de-dd8f-20fa-6138f439c93d} {Suffrage:Voter ID:server-b4cd5ed2-06ce-5443-ce21-3a351fa1eb4e Address:b4cd5ed2-06ce-5443-ce21-3a351fa1eb4e} {Suffrage:Voter ID:server-7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376 Address:7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376} {Suffrage:Voter ID:server-26efd242-aa35-ee29-06ce-813332b2e4f4 Address:26efd242-aa35-ee29-06ce-813332b2e4f4} {Suffrage:Voter ID:server-e43d9f64-536e-d0d3-1916-e29e9bf2b4e4 Address:e43d9f64-536e-d0d3-1916-e29e9bf2b4e4} {Suffrage:Voter ID:server-f2879551-1b71-1ce5-9a54-3e1a651e6633 Address:f2879551-1b71-1ce5-9a54-3e1a651e6633}]" +2025-03-20T03:51:15.259Z [INFO] server-1e5d22e0-c9de-dd8f-20fa-6138f439c93d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6f4de5a4-abad-2e62-5971-98f4b32882f4 Address:6f4de5a4-abad-2e62-5971-98f4b32882f4} {Suffrage:Voter ID:server-1e5d22e0-c9de-dd8f-20fa-6138f439c93d Address:1e5d22e0-c9de-dd8f-20fa-6138f439c93d} {Suffrage:Voter ID:server-b4cd5ed2-06ce-5443-ce21-3a351fa1eb4e Address:b4cd5ed2-06ce-5443-ce21-3a351fa1eb4e} {Suffrage:Voter ID:server-7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376 Address:7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376} {Suffrage:Voter ID:server-26efd242-aa35-ee29-06ce-813332b2e4f4 Address:26efd242-aa35-ee29-06ce-813332b2e4f4} {Suffrage:Voter ID:server-e43d9f64-536e-d0d3-1916-e29e9bf2b4e4 Address:e43d9f64-536e-d0d3-1916-e29e9bf2b4e4} {Suffrage:Voter ID:server-f2879551-1b71-1ce5-9a54-3e1a651e6633 Address:f2879551-1b71-1ce5-9a54-3e1a651e6633}]" +2025-03-20T03:51:15.259Z [INFO] server-6f4de5a4-abad-2e62-5971-98f4b32882f4: entering follower state: follower="Node at 6f4de5a4-abad-2e62-5971-98f4b32882f4 [Follower]" leader-address= leader-id= +2025-03-20T03:51:15.259Z [INFO] server-b4cd5ed2-06ce-5443-ce21-3a351fa1eb4e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6f4de5a4-abad-2e62-5971-98f4b32882f4 Address:6f4de5a4-abad-2e62-5971-98f4b32882f4} {Suffrage:Voter ID:server-1e5d22e0-c9de-dd8f-20fa-6138f439c93d Address:1e5d22e0-c9de-dd8f-20fa-6138f439c93d} {Suffrage:Voter ID:server-b4cd5ed2-06ce-5443-ce21-3a351fa1eb4e Address:b4cd5ed2-06ce-5443-ce21-3a351fa1eb4e} {Suffrage:Voter ID:server-7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376 Address:7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376} {Suffrage:Voter ID:server-26efd242-aa35-ee29-06ce-813332b2e4f4 Address:26efd242-aa35-ee29-06ce-813332b2e4f4} {Suffrage:Voter ID:server-e43d9f64-536e-d0d3-1916-e29e9bf2b4e4 Address:e43d9f64-536e-d0d3-1916-e29e9bf2b4e4} {Suffrage:Voter ID:server-f2879551-1b71-1ce5-9a54-3e1a651e6633 Address:f2879551-1b71-1ce5-9a54-3e1a651e6633}]" +2025-03-20T03:51:15.259Z [INFO] server-1e5d22e0-c9de-dd8f-20fa-6138f439c93d: entering follower state: follower="Node at 1e5d22e0-c9de-dd8f-20fa-6138f439c93d [Follower]" leader-address= leader-id= +2025-03-20T03:51:15.259Z [INFO] server-b4cd5ed2-06ce-5443-ce21-3a351fa1eb4e: entering follower state: follower="Node at b4cd5ed2-06ce-5443-ce21-3a351fa1eb4e [Follower]" leader-address= leader-id= +2025-03-20T03:51:15.261Z [INFO] server-7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6f4de5a4-abad-2e62-5971-98f4b32882f4 Address:6f4de5a4-abad-2e62-5971-98f4b32882f4} {Suffrage:Voter ID:server-1e5d22e0-c9de-dd8f-20fa-6138f439c93d Address:1e5d22e0-c9de-dd8f-20fa-6138f439c93d} {Suffrage:Voter ID:server-b4cd5ed2-06ce-5443-ce21-3a351fa1eb4e Address:b4cd5ed2-06ce-5443-ce21-3a351fa1eb4e} {Suffrage:Voter ID:server-7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376 Address:7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376} {Suffrage:Voter ID:server-26efd242-aa35-ee29-06ce-813332b2e4f4 Address:26efd242-aa35-ee29-06ce-813332b2e4f4} {Suffrage:Voter ID:server-e43d9f64-536e-d0d3-1916-e29e9bf2b4e4 Address:e43d9f64-536e-d0d3-1916-e29e9bf2b4e4} {Suffrage:Voter ID:server-f2879551-1b71-1ce5-9a54-3e1a651e6633 Address:f2879551-1b71-1ce5-9a54-3e1a651e6633}]" +2025-03-20T03:51:15.261Z [INFO] server-26efd242-aa35-ee29-06ce-813332b2e4f4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6f4de5a4-abad-2e62-5971-98f4b32882f4 Address:6f4de5a4-abad-2e62-5971-98f4b32882f4} {Suffrage:Voter ID:server-1e5d22e0-c9de-dd8f-20fa-6138f439c93d Address:1e5d22e0-c9de-dd8f-20fa-6138f439c93d} {Suffrage:Voter ID:server-b4cd5ed2-06ce-5443-ce21-3a351fa1eb4e Address:b4cd5ed2-06ce-5443-ce21-3a351fa1eb4e} {Suffrage:Voter ID:server-7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376 Address:7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376} {Suffrage:Voter ID:server-26efd242-aa35-ee29-06ce-813332b2e4f4 Address:26efd242-aa35-ee29-06ce-813332b2e4f4} {Suffrage:Voter ID:server-e43d9f64-536e-d0d3-1916-e29e9bf2b4e4 Address:e43d9f64-536e-d0d3-1916-e29e9bf2b4e4} {Suffrage:Voter ID:server-f2879551-1b71-1ce5-9a54-3e1a651e6633 Address:f2879551-1b71-1ce5-9a54-3e1a651e6633}]" +2025-03-20T03:51:15.261Z [INFO] server-e43d9f64-536e-d0d3-1916-e29e9bf2b4e4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6f4de5a4-abad-2e62-5971-98f4b32882f4 Address:6f4de5a4-abad-2e62-5971-98f4b32882f4} {Suffrage:Voter ID:server-1e5d22e0-c9de-dd8f-20fa-6138f439c93d Address:1e5d22e0-c9de-dd8f-20fa-6138f439c93d} {Suffrage:Voter ID:server-b4cd5ed2-06ce-5443-ce21-3a351fa1eb4e Address:b4cd5ed2-06ce-5443-ce21-3a351fa1eb4e} {Suffrage:Voter ID:server-7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376 Address:7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376} {Suffrage:Voter ID:server-26efd242-aa35-ee29-06ce-813332b2e4f4 Address:26efd242-aa35-ee29-06ce-813332b2e4f4} {Suffrage:Voter ID:server-e43d9f64-536e-d0d3-1916-e29e9bf2b4e4 Address:e43d9f64-536e-d0d3-1916-e29e9bf2b4e4} {Suffrage:Voter ID:server-f2879551-1b71-1ce5-9a54-3e1a651e6633 Address:f2879551-1b71-1ce5-9a54-3e1a651e6633}]" +2025-03-20T03:51:15.261Z [INFO] server-f2879551-1b71-1ce5-9a54-3e1a651e6633: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6f4de5a4-abad-2e62-5971-98f4b32882f4 Address:6f4de5a4-abad-2e62-5971-98f4b32882f4} {Suffrage:Voter ID:server-1e5d22e0-c9de-dd8f-20fa-6138f439c93d Address:1e5d22e0-c9de-dd8f-20fa-6138f439c93d} {Suffrage:Voter ID:server-b4cd5ed2-06ce-5443-ce21-3a351fa1eb4e Address:b4cd5ed2-06ce-5443-ce21-3a351fa1eb4e} {Suffrage:Voter ID:server-7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376 Address:7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376} {Suffrage:Voter ID:server-26efd242-aa35-ee29-06ce-813332b2e4f4 Address:26efd242-aa35-ee29-06ce-813332b2e4f4} {Suffrage:Voter ID:server-e43d9f64-536e-d0d3-1916-e29e9bf2b4e4 Address:e43d9f64-536e-d0d3-1916-e29e9bf2b4e4} {Suffrage:Voter ID:server-f2879551-1b71-1ce5-9a54-3e1a651e6633 Address:f2879551-1b71-1ce5-9a54-3e1a651e6633}]" +2025-03-20T03:51:15.261Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:15.261Z [INFO] server-7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376: entering follower state: follower="Node at 7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376 [Follower]" leader-address= leader-id= +2025-03-20T03:51:15.261Z [INFO] server-26efd242-aa35-ee29-06ce-813332b2e4f4: entering follower state: follower="Node at 26efd242-aa35-ee29-06ce-813332b2e4f4 [Follower]" leader-address= leader-id= +2025-03-20T03:51:15.261Z [INFO] server-e43d9f64-536e-d0d3-1916-e29e9bf2b4e4: entering follower state: follower="Node at e43d9f64-536e-d0d3-1916-e29e9bf2b4e4 [Follower]" leader-address= leader-id= +2025-03-20T03:51:15.261Z [INFO] server-f2879551-1b71-1ce5-9a54-3e1a651e6633: entering follower state: follower="Node at f2879551-1b71-1ce5-9a54-3e1a651e6633 [Follower]" leader-address= leader-id= +2025-03-20T03:51:15.328Z [WARN] server-7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:15.328Z [INFO] server-7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376: entering candidate state: node="Node at 7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376 [Candidate]" term=2 +2025-03-20T03:51:15.329Z [WARN] server-f2879551-1b71-1ce5-9a54-3e1a651e6633: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:15.329Z [INFO] server-f2879551-1b71-1ce5-9a54-3e1a651e6633: entering candidate state: node="Node at f2879551-1b71-1ce5-9a54-3e1a651e6633 [Candidate]" term=2 +2025-03-20T03:51:15.329Z [INFO] server-6f4de5a4-abad-2e62-5971-98f4b32882f4: duplicate requestVote for same term: term=2 +2025-03-20T03:51:15.329Z [INFO] server-1e5d22e0-c9de-dd8f-20fa-6138f439c93d: duplicate requestVote for same term: term=2 +2025-03-20T03:51:15.329Z [INFO] server-f2879551-1b71-1ce5-9a54-3e1a651e6633: election won: term=2 tally=4 +2025-03-20T03:51:15.329Z [INFO] server-f2879551-1b71-1ce5-9a54-3e1a651e6633: entering leader state: leader="Node at f2879551-1b71-1ce5-9a54-3e1a651e6633 [Leader]" +2025-03-20T03:51:15.329Z [INFO] server-f2879551-1b71-1ce5-9a54-3e1a651e6633: added peer, starting replication: peer=server-6f4de5a4-abad-2e62-5971-98f4b32882f4 +2025-03-20T03:51:15.329Z [INFO] server-f2879551-1b71-1ce5-9a54-3e1a651e6633: added peer, starting replication: peer=server-1e5d22e0-c9de-dd8f-20fa-6138f439c93d +2025-03-20T03:51:15.329Z [INFO] server-f2879551-1b71-1ce5-9a54-3e1a651e6633: added peer, starting replication: peer=server-b4cd5ed2-06ce-5443-ce21-3a351fa1eb4e +2025-03-20T03:51:15.329Z [INFO] server-f2879551-1b71-1ce5-9a54-3e1a651e6633: added peer, starting replication: peer=server-7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376 +2025-03-20T03:51:15.329Z [INFO] server-f2879551-1b71-1ce5-9a54-3e1a651e6633: added peer, starting replication: peer=server-26efd242-aa35-ee29-06ce-813332b2e4f4 +2025-03-20T03:51:15.329Z [INFO] server-f2879551-1b71-1ce5-9a54-3e1a651e6633: added peer, starting replication: peer=server-e43d9f64-536e-d0d3-1916-e29e9bf2b4e4 +2025-03-20T03:51:15.330Z [INFO] server-f2879551-1b71-1ce5-9a54-3e1a651e6633: pipelining replication: peer="{Voter server-e43d9f64-536e-d0d3-1916-e29e9bf2b4e4 e43d9f64-536e-d0d3-1916-e29e9bf2b4e4}" +2025-03-20T03:51:15.330Z [INFO] server-7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376: duplicate requestVote for same term: term=2 +2025-03-20T03:51:15.330Z [INFO] server-b4cd5ed2-06ce-5443-ce21-3a351fa1eb4e: duplicate requestVote for same term: term=2 +2025-03-20T03:51:15.330Z [INFO] server-26efd242-aa35-ee29-06ce-813332b2e4f4: duplicate requestVote for same term: term=2 +2025-03-20T03:51:15.330Z [WARN] server-e43d9f64-536e-d0d3-1916-e29e9bf2b4e4: rejecting vote request since we have a leader: from=7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376 leader=f2879551-1b71-1ce5-9a54-3e1a651e6633 leader-id=server-f2879551-1b71-1ce5-9a54-3e1a651e6633 +2025-03-20T03:51:15.330Z [WARN] server-f2879551-1b71-1ce5-9a54-3e1a651e6633: rejecting vote request since we have a leader: from=7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376 leader=f2879551-1b71-1ce5-9a54-3e1a651e6633 leader-id=server-f2879551-1b71-1ce5-9a54-3e1a651e6633 +2025-03-20T03:51:15.330Z [INFO] server-7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376: entering follower state: follower="Node at 7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376 [Follower]" leader-address=f2879551-1b71-1ce5-9a54-3e1a651e6633 leader-id=server-f2879551-1b71-1ce5-9a54-3e1a651e6633 +2025-03-20T03:51:15.331Z [INFO] server-f2879551-1b71-1ce5-9a54-3e1a651e6633: pipelining replication: peer="{Voter server-7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376 7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376}" +2025-03-20T03:51:15.331Z [INFO] server-f2879551-1b71-1ce5-9a54-3e1a651e6633: pipelining replication: peer="{Voter server-b4cd5ed2-06ce-5443-ce21-3a351fa1eb4e b4cd5ed2-06ce-5443-ce21-3a351fa1eb4e}" +2025-03-20T03:51:15.331Z [INFO] server-f2879551-1b71-1ce5-9a54-3e1a651e6633: pipelining replication: peer="{Voter server-1e5d22e0-c9de-dd8f-20fa-6138f439c93d 1e5d22e0-c9de-dd8f-20fa-6138f439c93d}" +2025-03-20T03:51:15.331Z [INFO] server-f2879551-1b71-1ce5-9a54-3e1a651e6633: pipelining replication: peer="{Voter server-6f4de5a4-abad-2e62-5971-98f4b32882f4 6f4de5a4-abad-2e62-5971-98f4b32882f4}" +2025-03-20T03:51:15.331Z [INFO] server-f2879551-1b71-1ce5-9a54-3e1a651e6633: pipelining replication: peer="{Voter server-26efd242-aa35-ee29-06ce-813332b2e4f4 26efd242-aa35-ee29-06ce-813332b2e4f4}" +2025-03-20T03:51:15.442Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:15.330937762 +0000 UTC m=+107.005390858 (1 nodes), 69.194082ms from start of poll, 71.878024ms from cluster start. Timeout at 2025-03-20 03:51:15.435947705 +0000 UTC m=+107.110400876, 105.010018ms after stability +2025-03-20T03:51:15.442Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:51:15.522Z [WARN] server-f2879551-1b71-1ce5-9a54-3e1a651e6633: failed to contact: server-id=server-b4cd5ed2-06ce-5443-ce21-3a351fa1eb4e time=51.672598ms +2025-03-20T03:51:15.548Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:51:15.442326252 +0000 UTC m=+107.116779169 (6 nodes), 3.327µs from start of poll, 183.266335ms from cluster start. Timeout at 2025-03-20 03:51:15.547387065 +0000 UTC m=+107.221839938, 105.060769ms after stability +2025-03-20T03:51:15.548Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:15.655Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:15.54893988 +0000 UTC m=+107.223392734 (1 nodes), 2.037µs from start of poll, 289.8799ms from cluster start. Timeout at 2025-03-20 03:51:15.654084912 +0000 UTC m=+107.328537677, 105.144943ms after stability +2025-03-20T03:51:15.655Z [INFO] server-6f4de5a4-abad-2e62-5971-98f4b32882f4: entering candidate state: node="Node at 6f4de5a4-abad-2e62-5971-98f4b32882f4 [Candidate]" term=3 +2025-03-20T03:51:15.655Z [INFO] server-f2879551-1b71-1ce5-9a54-3e1a651e6633: entering follower state: follower="Node at f2879551-1b71-1ce5-9a54-3e1a651e6633 [Follower]" leader-address= leader-id= +2025-03-20T03:51:15.655Z [INFO] server-f2879551-1b71-1ce5-9a54-3e1a651e6633: aborting pipeline replication: peer="{Voter server-e43d9f64-536e-d0d3-1916-e29e9bf2b4e4 e43d9f64-536e-d0d3-1916-e29e9bf2b4e4}" +2025-03-20T03:51:15.655Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:15.655Z [INFO] server-6f4de5a4-abad-2e62-5971-98f4b32882f4: election won: term=3 tally=4 +2025-03-20T03:51:15.655Z [INFO] server-6f4de5a4-abad-2e62-5971-98f4b32882f4: entering leader state: leader="Node at 6f4de5a4-abad-2e62-5971-98f4b32882f4 [Leader]" +2025-03-20T03:51:15.655Z [INFO] server-6f4de5a4-abad-2e62-5971-98f4b32882f4: added peer, starting replication: peer=server-1e5d22e0-c9de-dd8f-20fa-6138f439c93d +2025-03-20T03:51:15.655Z [INFO] server-6f4de5a4-abad-2e62-5971-98f4b32882f4: added peer, starting replication: peer=server-b4cd5ed2-06ce-5443-ce21-3a351fa1eb4e +2025-03-20T03:51:15.655Z [INFO] server-6f4de5a4-abad-2e62-5971-98f4b32882f4: added peer, starting replication: peer=server-7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376 +2025-03-20T03:51:15.655Z [INFO] server-6f4de5a4-abad-2e62-5971-98f4b32882f4: added peer, starting replication: peer=server-26efd242-aa35-ee29-06ce-813332b2e4f4 +2025-03-20T03:51:15.655Z [INFO] server-6f4de5a4-abad-2e62-5971-98f4b32882f4: added peer, starting replication: peer=server-e43d9f64-536e-d0d3-1916-e29e9bf2b4e4 +2025-03-20T03:51:15.655Z [INFO] server-6f4de5a4-abad-2e62-5971-98f4b32882f4: added peer, starting replication: peer=server-f2879551-1b71-1ce5-9a54-3e1a651e6633 +2025-03-20T03:51:15.656Z [INFO] server-6f4de5a4-abad-2e62-5971-98f4b32882f4: pipelining replication: peer="{Voter server-f2879551-1b71-1ce5-9a54-3e1a651e6633 f2879551-1b71-1ce5-9a54-3e1a651e6633}" +2025-03-20T03:51:15.656Z [INFO] server-f2879551-1b71-1ce5-9a54-3e1a651e6633: aborting pipeline replication: peer="{Voter server-6f4de5a4-abad-2e62-5971-98f4b32882f4 6f4de5a4-abad-2e62-5971-98f4b32882f4}" +2025-03-20T03:51:15.656Z [INFO] server-f2879551-1b71-1ce5-9a54-3e1a651e6633: aborting pipeline replication: peer="{Voter server-1e5d22e0-c9de-dd8f-20fa-6138f439c93d 1e5d22e0-c9de-dd8f-20fa-6138f439c93d}" +2025-03-20T03:51:15.656Z [INFO] server-f2879551-1b71-1ce5-9a54-3e1a651e6633: aborting pipeline replication: peer="{Voter server-b4cd5ed2-06ce-5443-ce21-3a351fa1eb4e b4cd5ed2-06ce-5443-ce21-3a351fa1eb4e}" +2025-03-20T03:51:15.656Z [INFO] server-f2879551-1b71-1ce5-9a54-3e1a651e6633: aborting pipeline replication: peer="{Voter server-7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376 7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376}" +2025-03-20T03:51:15.656Z [INFO] server-f2879551-1b71-1ce5-9a54-3e1a651e6633: aborting pipeline replication: peer="{Voter server-26efd242-aa35-ee29-06ce-813332b2e4f4 26efd242-aa35-ee29-06ce-813332b2e4f4}" +2025-03-20T03:51:15.656Z [INFO] server-6f4de5a4-abad-2e62-5971-98f4b32882f4: pipelining replication: peer="{Voter server-1e5d22e0-c9de-dd8f-20fa-6138f439c93d 1e5d22e0-c9de-dd8f-20fa-6138f439c93d}" +2025-03-20T03:51:15.656Z [INFO] server-6f4de5a4-abad-2e62-5971-98f4b32882f4: pipelining replication: peer="{Voter server-26efd242-aa35-ee29-06ce-813332b2e4f4 26efd242-aa35-ee29-06ce-813332b2e4f4}" +2025-03-20T03:51:15.656Z [INFO] server-6f4de5a4-abad-2e62-5971-98f4b32882f4: pipelining replication: peer="{Voter server-e43d9f64-536e-d0d3-1916-e29e9bf2b4e4 e43d9f64-536e-d0d3-1916-e29e9bf2b4e4}" +2025-03-20T03:51:15.656Z [INFO] server-6f4de5a4-abad-2e62-5971-98f4b32882f4: pipelining replication: peer="{Voter server-b4cd5ed2-06ce-5443-ce21-3a351fa1eb4e b4cd5ed2-06ce-5443-ce21-3a351fa1eb4e}" +2025-03-20T03:51:15.656Z [INFO] server-6f4de5a4-abad-2e62-5971-98f4b32882f4: pipelining replication: peer="{Voter server-7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376 7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376}" +2025-03-20T03:51:15.771Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:15.656376967 +0000 UTC m=+107.330829759 (1 nodes), 794.112µs from start of poll, 397.316925ms from cluster start. Timeout at 2025-03-20 03:51:15.761402879 +0000 UTC m=+107.435855896, 105.026137ms after stability +2025-03-20T03:51:15.771Z [INFO] server-6f4de5a4-abad-2e62-5971-98f4b32882f4: aborting pipeline replication: peer="{Voter server-e43d9f64-536e-d0d3-1916-e29e9bf2b4e4 e43d9f64-536e-d0d3-1916-e29e9bf2b4e4}" +2025-03-20T03:51:15.771Z [INFO] server-6f4de5a4-abad-2e62-5971-98f4b32882f4: aborting pipeline replication: peer="{Voter server-f2879551-1b71-1ce5-9a54-3e1a651e6633 f2879551-1b71-1ce5-9a54-3e1a651e6633}" +2025-03-20T03:51:15.771Z [INFO] server-6f4de5a4-abad-2e62-5971-98f4b32882f4: aborting pipeline replication: peer="{Voter server-1e5d22e0-c9de-dd8f-20fa-6138f439c93d 1e5d22e0-c9de-dd8f-20fa-6138f439c93d}" +2025-03-20T03:51:15.771Z [INFO] server-6f4de5a4-abad-2e62-5971-98f4b32882f4: aborting pipeline replication: peer="{Voter server-7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376 7e22c6c9-0bc8-c1da-e4b8-f727ac3f3376}" +2025-03-20T03:51:15.771Z [INFO] server-6f4de5a4-abad-2e62-5971-98f4b32882f4: aborting pipeline replication: peer="{Voter server-b4cd5ed2-06ce-5443-ce21-3a351fa1eb4e b4cd5ed2-06ce-5443-ce21-3a351fa1eb4e}" +2025-03-20T03:51:15.771Z [INFO] server-6f4de5a4-abad-2e62-5971-98f4b32882f4: aborting pipeline replication: peer="{Voter server-26efd242-aa35-ee29-06ce-813332b2e4f4 26efd242-aa35-ee29-06ce-813332b2e4f4}" +--- PASS: TestRaft_LeadershipTransferWithSevenNodes (0.52s) === RUN TestRaft_LeadershipTransferToInvalidID -2026-04-22T10:10:57.539Z [INFO] server-9e383c8f-79cf-4589-8a9e-5878b72eca32: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9e383c8f-79cf-4589-8a9e-5878b72eca32 Address:9e383c8f-79cf-4589-8a9e-5878b72eca32} {Suffrage:Voter ID:server-36370d60-e93b-f730-cd5e-9f2925538194 Address:36370d60-e93b-f730-cd5e-9f2925538194} {Suffrage:Voter ID:server-85314991-febd-66ee-ffad-ae976750197b Address:85314991-febd-66ee-ffad-ae976750197b}]" -2026-04-22T10:10:57.539Z [INFO] server-9e383c8f-79cf-4589-8a9e-5878b72eca32: entering follower state: follower="Node at 9e383c8f-79cf-4589-8a9e-5878b72eca32 [Follower]" leader-address= leader-id= -2026-04-22T10:10:57.539Z [INFO] server-36370d60-e93b-f730-cd5e-9f2925538194: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9e383c8f-79cf-4589-8a9e-5878b72eca32 Address:9e383c8f-79cf-4589-8a9e-5878b72eca32} {Suffrage:Voter ID:server-36370d60-e93b-f730-cd5e-9f2925538194 Address:36370d60-e93b-f730-cd5e-9f2925538194} {Suffrage:Voter ID:server-85314991-febd-66ee-ffad-ae976750197b Address:85314991-febd-66ee-ffad-ae976750197b}]" -2026-04-22T10:10:57.539Z [INFO] server-36370d60-e93b-f730-cd5e-9f2925538194: entering follower state: follower="Node at 36370d60-e93b-f730-cd5e-9f2925538194 [Follower]" leader-address= leader-id= -2026-04-22T10:10:57.539Z [INFO] server-85314991-febd-66ee-ffad-ae976750197b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9e383c8f-79cf-4589-8a9e-5878b72eca32 Address:9e383c8f-79cf-4589-8a9e-5878b72eca32} {Suffrage:Voter ID:server-36370d60-e93b-f730-cd5e-9f2925538194 Address:36370d60-e93b-f730-cd5e-9f2925538194} {Suffrage:Voter ID:server-85314991-febd-66ee-ffad-ae976750197b Address:85314991-febd-66ee-ffad-ae976750197b}]" -2026-04-22T10:10:57.539Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:57.539Z [INFO] server-85314991-febd-66ee-ffad-ae976750197b: entering follower state: follower="Node at 85314991-febd-66ee-ffad-ae976750197b [Follower]" leader-address= leader-id= -2026-04-22T10:10:57.629Z [WARN] server-36370d60-e93b-f730-cd5e-9f2925538194: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:57.629Z [INFO] server-36370d60-e93b-f730-cd5e-9f2925538194: entering candidate state: node="Node at 36370d60-e93b-f730-cd5e-9f2925538194 [Candidate]" term=2 -2026-04-22T10:10:57.629Z [INFO] server-36370d60-e93b-f730-cd5e-9f2925538194: election won: term=2 tally=2 -2026-04-22T10:10:57.629Z [INFO] server-36370d60-e93b-f730-cd5e-9f2925538194: entering leader state: leader="Node at 36370d60-e93b-f730-cd5e-9f2925538194 [Leader]" -2026-04-22T10:10:57.629Z [INFO] server-36370d60-e93b-f730-cd5e-9f2925538194: added peer, starting replication: peer=server-9e383c8f-79cf-4589-8a9e-5878b72eca32 -2026-04-22T10:10:57.629Z [INFO] server-36370d60-e93b-f730-cd5e-9f2925538194: added peer, starting replication: peer=server-85314991-febd-66ee-ffad-ae976750197b -2026-04-22T10:10:57.629Z [INFO] server-36370d60-e93b-f730-cd5e-9f2925538194: pipelining replication: peer="{Voter server-9e383c8f-79cf-4589-8a9e-5878b72eca32 9e383c8f-79cf-4589-8a9e-5878b72eca32}" -2026-04-22T10:10:57.629Z [INFO] server-36370d60-e93b-f730-cd5e-9f2925538194: pipelining replication: peer="{Voter server-85314991-febd-66ee-ffad-ae976750197b 85314991-febd-66ee-ffad-ae976750197b}" -2026-04-22T10:10:57.734Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:57.629668175 +0000 UTC m=+103.653464267 (1 nodes), 89.919727ms from start of poll, 90.222727ms from cluster start. Timeout at 2026-04-22 10:10:57.734670792 +0000 UTC m=+103.758466874, 105.002607ms after stability -2026-04-22T10:10:57.735Z [INFO] server-36370d60-e93b-f730-cd5e-9f2925538194: aborting pipeline replication: peer="{Voter server-85314991-febd-66ee-ffad-ae976750197b 85314991-febd-66ee-ffad-ae976750197b}" -2026-04-22T10:10:57.735Z [INFO] server-36370d60-e93b-f730-cd5e-9f2925538194: aborting pipeline replication: peer="{Voter server-9e383c8f-79cf-4589-8a9e-5878b72eca32 9e383c8f-79cf-4589-8a9e-5878b72eca32}" ---- PASS: TestRaft_LeadershipTransferToInvalidID (0.20s) +2025-03-20T03:51:15.774Z [INFO] server-dc55e2de-4173-d78b-775c-53de4252c533: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dc55e2de-4173-d78b-775c-53de4252c533 Address:dc55e2de-4173-d78b-775c-53de4252c533} {Suffrage:Voter ID:server-7ab4df6c-5abd-d4e5-c765-3630d7ffaa97 Address:7ab4df6c-5abd-d4e5-c765-3630d7ffaa97} {Suffrage:Voter ID:server-a660062d-7c5f-0bed-6c56-0a8990bef125 Address:a660062d-7c5f-0bed-6c56-0a8990bef125}]" +2025-03-20T03:51:15.774Z [INFO] server-7ab4df6c-5abd-d4e5-c765-3630d7ffaa97: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dc55e2de-4173-d78b-775c-53de4252c533 Address:dc55e2de-4173-d78b-775c-53de4252c533} {Suffrage:Voter ID:server-7ab4df6c-5abd-d4e5-c765-3630d7ffaa97 Address:7ab4df6c-5abd-d4e5-c765-3630d7ffaa97} {Suffrage:Voter ID:server-a660062d-7c5f-0bed-6c56-0a8990bef125 Address:a660062d-7c5f-0bed-6c56-0a8990bef125}]" +2025-03-20T03:51:15.774Z [INFO] server-a660062d-7c5f-0bed-6c56-0a8990bef125: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dc55e2de-4173-d78b-775c-53de4252c533 Address:dc55e2de-4173-d78b-775c-53de4252c533} {Suffrage:Voter ID:server-7ab4df6c-5abd-d4e5-c765-3630d7ffaa97 Address:7ab4df6c-5abd-d4e5-c765-3630d7ffaa97} {Suffrage:Voter ID:server-a660062d-7c5f-0bed-6c56-0a8990bef125 Address:a660062d-7c5f-0bed-6c56-0a8990bef125}]" +2025-03-20T03:51:15.774Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:15.774Z [INFO] server-dc55e2de-4173-d78b-775c-53de4252c533: entering follower state: follower="Node at dc55e2de-4173-d78b-775c-53de4252c533 [Follower]" leader-address= leader-id= +2025-03-20T03:51:15.774Z [INFO] server-7ab4df6c-5abd-d4e5-c765-3630d7ffaa97: entering follower state: follower="Node at 7ab4df6c-5abd-d4e5-c765-3630d7ffaa97 [Follower]" leader-address= leader-id= +2025-03-20T03:51:15.775Z [INFO] server-a660062d-7c5f-0bed-6c56-0a8990bef125: entering follower state: follower="Node at a660062d-7c5f-0bed-6c56-0a8990bef125 [Follower]" leader-address= leader-id= +2025-03-20T03:51:15.843Z [WARN] server-7ab4df6c-5abd-d4e5-c765-3630d7ffaa97: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:15.843Z [INFO] server-7ab4df6c-5abd-d4e5-c765-3630d7ffaa97: entering candidate state: node="Node at 7ab4df6c-5abd-d4e5-c765-3630d7ffaa97 [Candidate]" term=2 +2025-03-20T03:51:15.843Z [INFO] server-7ab4df6c-5abd-d4e5-c765-3630d7ffaa97: election won: term=2 tally=2 +2025-03-20T03:51:15.843Z [INFO] server-7ab4df6c-5abd-d4e5-c765-3630d7ffaa97: entering leader state: leader="Node at 7ab4df6c-5abd-d4e5-c765-3630d7ffaa97 [Leader]" +2025-03-20T03:51:15.843Z [INFO] server-7ab4df6c-5abd-d4e5-c765-3630d7ffaa97: added peer, starting replication: peer=server-dc55e2de-4173-d78b-775c-53de4252c533 +2025-03-20T03:51:15.843Z [INFO] server-7ab4df6c-5abd-d4e5-c765-3630d7ffaa97: added peer, starting replication: peer=server-a660062d-7c5f-0bed-6c56-0a8990bef125 +2025-03-20T03:51:15.844Z [INFO] server-7ab4df6c-5abd-d4e5-c765-3630d7ffaa97: pipelining replication: peer="{Voter server-dc55e2de-4173-d78b-775c-53de4252c533 dc55e2de-4173-d78b-775c-53de4252c533}" +2025-03-20T03:51:15.844Z [INFO] server-7ab4df6c-5abd-d4e5-c765-3630d7ffaa97: pipelining replication: peer="{Voter server-a660062d-7c5f-0bed-6c56-0a8990bef125 a660062d-7c5f-0bed-6c56-0a8990bef125}" +2025-03-20T03:51:15.950Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:15.844186375 +0000 UTC m=+107.518639162 (1 nodes), 69.321942ms from start of poll, 69.808412ms from cluster start. Timeout at 2025-03-20 03:51:15.949221302 +0000 UTC m=+107.623674316, 105.035154ms after stability +2025-03-20T03:51:15.951Z [INFO] server-7ab4df6c-5abd-d4e5-c765-3630d7ffaa97: aborting pipeline replication: peer="{Voter server-a660062d-7c5f-0bed-6c56-0a8990bef125 a660062d-7c5f-0bed-6c56-0a8990bef125}" +2025-03-20T03:51:15.951Z [INFO] server-7ab4df6c-5abd-d4e5-c765-3630d7ffaa97: aborting pipeline replication: peer="{Voter server-dc55e2de-4173-d78b-775c-53de4252c533 dc55e2de-4173-d78b-775c-53de4252c533}" +2025-03-20T03:51:16.474Z [ERROR] server-7ab4df6c-5abd-d4e5-c765-3630d7ffaa97: failed to heartbeat to: peer=dc55e2de-4173-d78b-775c-53de4252c533 backoff time=10ms error="command timed out" +--- PASS: TestRaft_LeadershipTransferToInvalidID (0.70s) === RUN TestRaft_LeadershipTransferToInvalidAddress -2026-04-22T10:10:57.735Z [INFO] server-f6fa42a3-cb7e-241b-1f6d-c498de3a52a5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f6fa42a3-cb7e-241b-1f6d-c498de3a52a5 Address:f6fa42a3-cb7e-241b-1f6d-c498de3a52a5} {Suffrage:Voter ID:server-12bdd049-3952-9719-0454-9ba3514ba81e Address:12bdd049-3952-9719-0454-9ba3514ba81e} {Suffrage:Voter ID:server-a5ccda94-ce02-81bd-b863-b0fcbfd2fcef Address:a5ccda94-ce02-81bd-b863-b0fcbfd2fcef}]" -2026-04-22T10:10:57.735Z [INFO] server-f6fa42a3-cb7e-241b-1f6d-c498de3a52a5: entering follower state: follower="Node at f6fa42a3-cb7e-241b-1f6d-c498de3a52a5 [Follower]" leader-address= leader-id= -2026-04-22T10:10:57.736Z [INFO] server-12bdd049-3952-9719-0454-9ba3514ba81e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f6fa42a3-cb7e-241b-1f6d-c498de3a52a5 Address:f6fa42a3-cb7e-241b-1f6d-c498de3a52a5} {Suffrage:Voter ID:server-12bdd049-3952-9719-0454-9ba3514ba81e Address:12bdd049-3952-9719-0454-9ba3514ba81e} {Suffrage:Voter ID:server-a5ccda94-ce02-81bd-b863-b0fcbfd2fcef Address:a5ccda94-ce02-81bd-b863-b0fcbfd2fcef}]" -2026-04-22T10:10:57.736Z [INFO] server-12bdd049-3952-9719-0454-9ba3514ba81e: entering follower state: follower="Node at 12bdd049-3952-9719-0454-9ba3514ba81e [Follower]" leader-address= leader-id= -2026-04-22T10:10:57.736Z [INFO] server-a5ccda94-ce02-81bd-b863-b0fcbfd2fcef: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f6fa42a3-cb7e-241b-1f6d-c498de3a52a5 Address:f6fa42a3-cb7e-241b-1f6d-c498de3a52a5} {Suffrage:Voter ID:server-12bdd049-3952-9719-0454-9ba3514ba81e Address:12bdd049-3952-9719-0454-9ba3514ba81e} {Suffrage:Voter ID:server-a5ccda94-ce02-81bd-b863-b0fcbfd2fcef Address:a5ccda94-ce02-81bd-b863-b0fcbfd2fcef}]" -2026-04-22T10:10:57.736Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:10:57.736Z [INFO] server-a5ccda94-ce02-81bd-b863-b0fcbfd2fcef: entering follower state: follower="Node at a5ccda94-ce02-81bd-b863-b0fcbfd2fcef [Follower]" leader-address= leader-id= -2026-04-22T10:10:57.808Z [WARN] server-f6fa42a3-cb7e-241b-1f6d-c498de3a52a5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:57.808Z [INFO] server-f6fa42a3-cb7e-241b-1f6d-c498de3a52a5: entering candidate state: node="Node at f6fa42a3-cb7e-241b-1f6d-c498de3a52a5 [Candidate]" term=2 -2026-04-22T10:10:57.809Z [INFO] server-f6fa42a3-cb7e-241b-1f6d-c498de3a52a5: election won: term=2 tally=2 -2026-04-22T10:10:57.809Z [INFO] server-f6fa42a3-cb7e-241b-1f6d-c498de3a52a5: entering leader state: leader="Node at f6fa42a3-cb7e-241b-1f6d-c498de3a52a5 [Leader]" -2026-04-22T10:10:57.809Z [INFO] server-f6fa42a3-cb7e-241b-1f6d-c498de3a52a5: added peer, starting replication: peer=server-12bdd049-3952-9719-0454-9ba3514ba81e -2026-04-22T10:10:57.809Z [INFO] server-f6fa42a3-cb7e-241b-1f6d-c498de3a52a5: added peer, starting replication: peer=server-a5ccda94-ce02-81bd-b863-b0fcbfd2fcef -2026-04-22T10:10:57.809Z [INFO] server-f6fa42a3-cb7e-241b-1f6d-c498de3a52a5: pipelining replication: peer="{Voter server-a5ccda94-ce02-81bd-b863-b0fcbfd2fcef a5ccda94-ce02-81bd-b863-b0fcbfd2fcef}" -2026-04-22T10:10:57.809Z [INFO] server-f6fa42a3-cb7e-241b-1f6d-c498de3a52a5: pipelining replication: peer="{Voter server-12bdd049-3952-9719-0454-9ba3514ba81e 12bdd049-3952-9719-0454-9ba3514ba81e}" -2026-04-22T10:10:57.914Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:10:57.809174215 +0000 UTC m=+103.832970315 (2 nodes), 73.052918ms from start of poll, 73.353813ms from cluster start. Timeout at 2026-04-22 10:10:57.914178852 +0000 UTC m=+103.937974946, 105.004631ms after stability -2026-04-22T10:10:57.914Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:58.020Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:57.914395442 +0000 UTC m=+103.938191528 (1 nodes), 464ns from start of poll, 178.575026ms from cluster start. Timeout at 2026-04-22 10:10:58.019408608 +0000 UTC m=+104.043204710, 105.013182ms after stability -2026-04-22T10:10:58.020Z [INFO] server-f6fa42a3-cb7e-241b-1f6d-c498de3a52a5: aborting pipeline replication: peer="{Voter server-12bdd049-3952-9719-0454-9ba3514ba81e 12bdd049-3952-9719-0454-9ba3514ba81e}" -2026-04-22T10:10:58.020Z [INFO] server-f6fa42a3-cb7e-241b-1f6d-c498de3a52a5: aborting pipeline replication: peer="{Voter server-a5ccda94-ce02-81bd-b863-b0fcbfd2fcef a5ccda94-ce02-81bd-b863-b0fcbfd2fcef}" +2025-03-20T03:51:16.483Z [INFO] server-e7704582-ce60-dd9a-23c8-2e4497fa1f7c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e7704582-ce60-dd9a-23c8-2e4497fa1f7c Address:e7704582-ce60-dd9a-23c8-2e4497fa1f7c} {Suffrage:Voter ID:server-a8218304-81eb-4746-4baf-230a0bcd63f3 Address:a8218304-81eb-4746-4baf-230a0bcd63f3} {Suffrage:Voter ID:server-7d67a86d-66b9-cdae-68e5-754b2027fba8 Address:7d67a86d-66b9-cdae-68e5-754b2027fba8}]" +2025-03-20T03:51:16.483Z [INFO] server-a8218304-81eb-4746-4baf-230a0bcd63f3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e7704582-ce60-dd9a-23c8-2e4497fa1f7c Address:e7704582-ce60-dd9a-23c8-2e4497fa1f7c} {Suffrage:Voter ID:server-a8218304-81eb-4746-4baf-230a0bcd63f3 Address:a8218304-81eb-4746-4baf-230a0bcd63f3} {Suffrage:Voter ID:server-7d67a86d-66b9-cdae-68e5-754b2027fba8 Address:7d67a86d-66b9-cdae-68e5-754b2027fba8}]" +2025-03-20T03:51:16.483Z [INFO] server-7d67a86d-66b9-cdae-68e5-754b2027fba8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e7704582-ce60-dd9a-23c8-2e4497fa1f7c Address:e7704582-ce60-dd9a-23c8-2e4497fa1f7c} {Suffrage:Voter ID:server-a8218304-81eb-4746-4baf-230a0bcd63f3 Address:a8218304-81eb-4746-4baf-230a0bcd63f3} {Suffrage:Voter ID:server-7d67a86d-66b9-cdae-68e5-754b2027fba8 Address:7d67a86d-66b9-cdae-68e5-754b2027fba8}]" +2025-03-20T03:51:16.483Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:51:16.483Z [INFO] server-e7704582-ce60-dd9a-23c8-2e4497fa1f7c: entering follower state: follower="Node at e7704582-ce60-dd9a-23c8-2e4497fa1f7c [Follower]" leader-address= leader-id= +2025-03-20T03:51:16.483Z [INFO] server-a8218304-81eb-4746-4baf-230a0bcd63f3: entering follower state: follower="Node at a8218304-81eb-4746-4baf-230a0bcd63f3 [Follower]" leader-address= leader-id= +2025-03-20T03:51:16.483Z [INFO] server-7d67a86d-66b9-cdae-68e5-754b2027fba8: entering follower state: follower="Node at 7d67a86d-66b9-cdae-68e5-754b2027fba8 [Follower]" leader-address= leader-id= +2025-03-20T03:51:16.554Z [WARN] server-7d67a86d-66b9-cdae-68e5-754b2027fba8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:16.554Z [INFO] server-7d67a86d-66b9-cdae-68e5-754b2027fba8: entering candidate state: node="Node at 7d67a86d-66b9-cdae-68e5-754b2027fba8 [Candidate]" term=2 +2025-03-20T03:51:16.554Z [INFO] server-7d67a86d-66b9-cdae-68e5-754b2027fba8: election won: term=2 tally=2 +2025-03-20T03:51:16.554Z [INFO] server-7d67a86d-66b9-cdae-68e5-754b2027fba8: entering leader state: leader="Node at 7d67a86d-66b9-cdae-68e5-754b2027fba8 [Leader]" +2025-03-20T03:51:16.554Z [INFO] server-7d67a86d-66b9-cdae-68e5-754b2027fba8: added peer, starting replication: peer=server-e7704582-ce60-dd9a-23c8-2e4497fa1f7c +2025-03-20T03:51:16.554Z [INFO] server-7d67a86d-66b9-cdae-68e5-754b2027fba8: added peer, starting replication: peer=server-a8218304-81eb-4746-4baf-230a0bcd63f3 +2025-03-20T03:51:16.555Z [INFO] server-7d67a86d-66b9-cdae-68e5-754b2027fba8: pipelining replication: peer="{Voter server-e7704582-ce60-dd9a-23c8-2e4497fa1f7c e7704582-ce60-dd9a-23c8-2e4497fa1f7c}" +2025-03-20T03:51:16.556Z [INFO] server-7d67a86d-66b9-cdae-68e5-754b2027fba8: pipelining replication: peer="{Voter server-a8218304-81eb-4746-4baf-230a0bcd63f3 a8218304-81eb-4746-4baf-230a0bcd63f3}" +2025-03-20T03:51:16.662Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:51:16.556205054 +0000 UTC m=+108.230657985 (2 nodes), 72.779795ms from start of poll, 73.299179ms from cluster start. Timeout at 2025-03-20 03:51:16.661214551 +0000 UTC m=+108.335667450, 105.009465ms after stability +2025-03-20T03:51:16.662Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:16.773Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:16.662741887 +0000 UTC m=+108.337194725 (1 nodes), 2.101µs from start of poll, 179.835919ms from cluster start. Timeout at 2025-03-20 03:51:16.767912967 +0000 UTC m=+108.442365973, 105.171248ms after stability +2025-03-20T03:51:16.773Z [INFO] server-7d67a86d-66b9-cdae-68e5-754b2027fba8: aborting pipeline replication: peer="{Voter server-a8218304-81eb-4746-4baf-230a0bcd63f3 a8218304-81eb-4746-4baf-230a0bcd63f3}" +2025-03-20T03:51:16.773Z [INFO] server-7d67a86d-66b9-cdae-68e5-754b2027fba8: aborting pipeline replication: peer="{Voter server-e7704582-ce60-dd9a-23c8-2e4497fa1f7c e7704582-ce60-dd9a-23c8-2e4497fa1f7c}" --- PASS: TestRaft_LeadershipTransferToInvalidAddress (0.29s) === RUN TestRaft_LeadershipTransferToBehindServer -2026-04-22T10:10:58.021Z [INFO] server-d488d71c-f0a7-d28c-9d8d-3a7d9aae5d11: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d488d71c-f0a7-d28c-9d8d-3a7d9aae5d11 Address:d488d71c-f0a7-d28c-9d8d-3a7d9aae5d11} {Suffrage:Voter ID:server-0b23f749-527c-ef04-5be2-304c03e887bf Address:0b23f749-527c-ef04-5be2-304c03e887bf} {Suffrage:Voter ID:server-b2bbf496-81dc-e7b0-6f8c-69cb9178db25 Address:b2bbf496-81dc-e7b0-6f8c-69cb9178db25}]" -2026-04-22T10:10:58.021Z [INFO] server-d488d71c-f0a7-d28c-9d8d-3a7d9aae5d11: entering follower state: follower="Node at d488d71c-f0a7-d28c-9d8d-3a7d9aae5d11 [Follower]" leader-address= leader-id= -2026-04-22T10:10:58.021Z [INFO] server-0b23f749-527c-ef04-5be2-304c03e887bf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d488d71c-f0a7-d28c-9d8d-3a7d9aae5d11 Address:d488d71c-f0a7-d28c-9d8d-3a7d9aae5d11} {Suffrage:Voter ID:server-0b23f749-527c-ef04-5be2-304c03e887bf Address:0b23f749-527c-ef04-5be2-304c03e887bf} {Suffrage:Voter ID:server-b2bbf496-81dc-e7b0-6f8c-69cb9178db25 Address:b2bbf496-81dc-e7b0-6f8c-69cb9178db25}]" -2026-04-22T10:10:58.021Z [INFO] server-0b23f749-527c-ef04-5be2-304c03e887bf: entering follower state: follower="Node at 0b23f749-527c-ef04-5be2-304c03e887bf [Follower]" leader-address= leader-id= -2026-04-22T10:10:58.021Z [INFO] server-b2bbf496-81dc-e7b0-6f8c-69cb9178db25: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d488d71c-f0a7-d28c-9d8d-3a7d9aae5d11 Address:d488d71c-f0a7-d28c-9d8d-3a7d9aae5d11} {Suffrage:Voter ID:server-0b23f749-527c-ef04-5be2-304c03e887bf Address:0b23f749-527c-ef04-5be2-304c03e887bf} {Suffrage:Voter ID:server-b2bbf496-81dc-e7b0-6f8c-69cb9178db25 Address:b2bbf496-81dc-e7b0-6f8c-69cb9178db25}]" -2026-04-22T10:10:58.021Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:58.021Z [INFO] server-b2bbf496-81dc-e7b0-6f8c-69cb9178db25: entering follower state: follower="Node at b2bbf496-81dc-e7b0-6f8c-69cb9178db25 [Follower]" leader-address= leader-id= -2026-04-22T10:10:58.094Z [WARN] server-b2bbf496-81dc-e7b0-6f8c-69cb9178db25: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:58.094Z [INFO] server-b2bbf496-81dc-e7b0-6f8c-69cb9178db25: entering candidate state: node="Node at b2bbf496-81dc-e7b0-6f8c-69cb9178db25 [Candidate]" term=2 -2026-04-22T10:10:58.094Z [INFO] server-b2bbf496-81dc-e7b0-6f8c-69cb9178db25: election won: term=2 tally=2 -2026-04-22T10:10:58.094Z [INFO] server-b2bbf496-81dc-e7b0-6f8c-69cb9178db25: entering leader state: leader="Node at b2bbf496-81dc-e7b0-6f8c-69cb9178db25 [Leader]" -2026-04-22T10:10:58.095Z [INFO] server-b2bbf496-81dc-e7b0-6f8c-69cb9178db25: added peer, starting replication: peer=server-d488d71c-f0a7-d28c-9d8d-3a7d9aae5d11 -2026-04-22T10:10:58.095Z [INFO] server-b2bbf496-81dc-e7b0-6f8c-69cb9178db25: added peer, starting replication: peer=server-0b23f749-527c-ef04-5be2-304c03e887bf -2026-04-22T10:10:58.095Z [INFO] server-b2bbf496-81dc-e7b0-6f8c-69cb9178db25: pipelining replication: peer="{Voter server-0b23f749-527c-ef04-5be2-304c03e887bf 0b23f749-527c-ef04-5be2-304c03e887bf}" -2026-04-22T10:10:58.095Z [INFO] server-b2bbf496-81dc-e7b0-6f8c-69cb9178db25: pipelining replication: peer="{Voter server-d488d71c-f0a7-d28c-9d8d-3a7d9aae5d11 d488d71c-f0a7-d28c-9d8d-3a7d9aae5d11}" -2026-04-22T10:10:58.200Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:58.095066385 +0000 UTC m=+104.118862535 (1 nodes), 73.558782ms from start of poll, 73.878448ms from cluster start. Timeout at 2026-04-22 10:10:58.200070211 +0000 UTC m=+104.223866289, 105.003754ms after stability -2026-04-22T10:10:58.200Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:10:58.305Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:10:58.200191885 +0000 UTC m=+104.223987974 (2 nodes), 795ns from start of poll, 179.003887ms from cluster start. Timeout at 2026-04-22 10:10:58.305224281 +0000 UTC m=+104.329020417, 105.032443ms after stability -2026-04-22T10:10:58.349Z [INFO] server-d488d71c-f0a7-d28c-9d8d-3a7d9aae5d11: entering candidate state: node="Node at d488d71c-f0a7-d28c-9d8d-3a7d9aae5d11 [Candidate]" term=3 -2026-04-22T10:10:58.349Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:58.349Z [INFO] server-d488d71c-f0a7-d28c-9d8d-3a7d9aae5d11: election won: term=3 tally=2 -2026-04-22T10:10:58.349Z [INFO] server-b2bbf496-81dc-e7b0-6f8c-69cb9178db25: entering follower state: follower="Node at b2bbf496-81dc-e7b0-6f8c-69cb9178db25 [Follower]" leader-address= leader-id= -2026-04-22T10:10:58.349Z [INFO] server-d488d71c-f0a7-d28c-9d8d-3a7d9aae5d11: entering leader state: leader="Node at d488d71c-f0a7-d28c-9d8d-3a7d9aae5d11 [Leader]" -2026-04-22T10:10:58.349Z [INFO] server-b2bbf496-81dc-e7b0-6f8c-69cb9178db25: aborting pipeline replication: peer="{Voter server-0b23f749-527c-ef04-5be2-304c03e887bf 0b23f749-527c-ef04-5be2-304c03e887bf}" -2026-04-22T10:10:58.349Z [INFO] server-d488d71c-f0a7-d28c-9d8d-3a7d9aae5d11: added peer, starting replication: peer=server-0b23f749-527c-ef04-5be2-304c03e887bf -2026-04-22T10:10:58.349Z [INFO] server-d488d71c-f0a7-d28c-9d8d-3a7d9aae5d11: added peer, starting replication: peer=server-b2bbf496-81dc-e7b0-6f8c-69cb9178db25 -2026-04-22T10:10:58.349Z [INFO] server-b2bbf496-81dc-e7b0-6f8c-69cb9178db25: aborting pipeline replication: peer="{Voter server-d488d71c-f0a7-d28c-9d8d-3a7d9aae5d11 d488d71c-f0a7-d28c-9d8d-3a7d9aae5d11}" -2026-04-22T10:10:58.349Z [INFO] server-d488d71c-f0a7-d28c-9d8d-3a7d9aae5d11: pipelining replication: peer="{Voter server-0b23f749-527c-ef04-5be2-304c03e887bf 0b23f749-527c-ef04-5be2-304c03e887bf}" -2026-04-22T10:10:58.349Z [INFO] server-d488d71c-f0a7-d28c-9d8d-3a7d9aae5d11: pipelining replication: peer="{Voter server-b2bbf496-81dc-e7b0-6f8c-69cb9178db25 b2bbf496-81dc-e7b0-6f8c-69cb9178db25}" -2026-04-22T10:10:58.455Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:58.349836182 +0000 UTC m=+104.373632275 (1 nodes), 63.235µs from start of poll, 328.648188ms from cluster start. Timeout at 2026-04-22 10:10:58.454857803 +0000 UTC m=+104.478654040, 105.021765ms after stability -2026-04-22T10:10:58.455Z [INFO] server-d488d71c-f0a7-d28c-9d8d-3a7d9aae5d11: aborting pipeline replication: peer="{Voter server-b2bbf496-81dc-e7b0-6f8c-69cb9178db25 b2bbf496-81dc-e7b0-6f8c-69cb9178db25}" -2026-04-22T10:10:58.455Z [INFO] server-d488d71c-f0a7-d28c-9d8d-3a7d9aae5d11: aborting pipeline replication: peer="{Voter server-0b23f749-527c-ef04-5be2-304c03e887bf 0b23f749-527c-ef04-5be2-304c03e887bf}" ---- PASS: TestRaft_LeadershipTransferToBehindServer (0.43s) +2025-03-20T03:51:16.775Z [INFO] server-85a767c8-6099-1453-157d-22ea2dcbddba: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-85a767c8-6099-1453-157d-22ea2dcbddba Address:85a767c8-6099-1453-157d-22ea2dcbddba} {Suffrage:Voter ID:server-70d2b76b-f5b2-22c6-145d-51a070adef09 Address:70d2b76b-f5b2-22c6-145d-51a070adef09} {Suffrage:Voter ID:server-21cbfb70-64b4-85a5-993a-40b9f8a1535a Address:21cbfb70-64b4-85a5-993a-40b9f8a1535a}]" +2025-03-20T03:51:16.775Z [INFO] server-70d2b76b-f5b2-22c6-145d-51a070adef09: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-85a767c8-6099-1453-157d-22ea2dcbddba Address:85a767c8-6099-1453-157d-22ea2dcbddba} {Suffrage:Voter ID:server-70d2b76b-f5b2-22c6-145d-51a070adef09 Address:70d2b76b-f5b2-22c6-145d-51a070adef09} {Suffrage:Voter ID:server-21cbfb70-64b4-85a5-993a-40b9f8a1535a Address:21cbfb70-64b4-85a5-993a-40b9f8a1535a}]" +2025-03-20T03:51:16.775Z [INFO] server-21cbfb70-64b4-85a5-993a-40b9f8a1535a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-85a767c8-6099-1453-157d-22ea2dcbddba Address:85a767c8-6099-1453-157d-22ea2dcbddba} {Suffrage:Voter ID:server-70d2b76b-f5b2-22c6-145d-51a070adef09 Address:70d2b76b-f5b2-22c6-145d-51a070adef09} {Suffrage:Voter ID:server-21cbfb70-64b4-85a5-993a-40b9f8a1535a Address:21cbfb70-64b4-85a5-993a-40b9f8a1535a}]" +2025-03-20T03:51:16.775Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:16.775Z [INFO] server-85a767c8-6099-1453-157d-22ea2dcbddba: entering follower state: follower="Node at 85a767c8-6099-1453-157d-22ea2dcbddba [Follower]" leader-address= leader-id= +2025-03-20T03:51:16.775Z [INFO] server-70d2b76b-f5b2-22c6-145d-51a070adef09: entering follower state: follower="Node at 70d2b76b-f5b2-22c6-145d-51a070adef09 [Follower]" leader-address= leader-id= +2025-03-20T03:51:16.775Z [INFO] server-21cbfb70-64b4-85a5-993a-40b9f8a1535a: entering follower state: follower="Node at 21cbfb70-64b4-85a5-993a-40b9f8a1535a [Follower]" leader-address= leader-id= +2025-03-20T03:51:16.826Z [WARN] server-70d2b76b-f5b2-22c6-145d-51a070adef09: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:16.826Z [INFO] server-70d2b76b-f5b2-22c6-145d-51a070adef09: entering candidate state: node="Node at 70d2b76b-f5b2-22c6-145d-51a070adef09 [Candidate]" term=2 +2025-03-20T03:51:16.829Z [INFO] server-70d2b76b-f5b2-22c6-145d-51a070adef09: election won: term=2 tally=2 +2025-03-20T03:51:16.829Z [INFO] server-70d2b76b-f5b2-22c6-145d-51a070adef09: entering leader state: leader="Node at 70d2b76b-f5b2-22c6-145d-51a070adef09 [Leader]" +2025-03-20T03:51:16.829Z [INFO] server-70d2b76b-f5b2-22c6-145d-51a070adef09: added peer, starting replication: peer=server-85a767c8-6099-1453-157d-22ea2dcbddba +2025-03-20T03:51:16.829Z [INFO] server-70d2b76b-f5b2-22c6-145d-51a070adef09: added peer, starting replication: peer=server-21cbfb70-64b4-85a5-993a-40b9f8a1535a +2025-03-20T03:51:16.830Z [INFO] server-70d2b76b-f5b2-22c6-145d-51a070adef09: pipelining replication: peer="{Voter server-21cbfb70-64b4-85a5-993a-40b9f8a1535a 21cbfb70-64b4-85a5-993a-40b9f8a1535a}" +2025-03-20T03:51:16.830Z [INFO] server-70d2b76b-f5b2-22c6-145d-51a070adef09: pipelining replication: peer="{Voter server-85a767c8-6099-1453-157d-22ea2dcbddba 85a767c8-6099-1453-157d-22ea2dcbddba}" +2025-03-20T03:51:16.935Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:16.830545073 +0000 UTC m=+108.504997885 (1 nodes), 55.02978ms from start of poll, 55.522189ms from cluster start. Timeout at 2025-03-20 03:51:16.93554878 +0000 UTC m=+108.610001758, 105.003873ms after stability +2025-03-20T03:51:16.936Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:51:17.041Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:51:16.936026334 +0000 UTC m=+108.610479237 (2 nodes), 2.896µs from start of poll, 161.003541ms from cluster start. Timeout at 2025-03-20 03:51:17.041082309 +0000 UTC m=+108.715535217, 105.05598ms after stability +2025-03-20T03:51:17.290Z [INFO] server-85a767c8-6099-1453-157d-22ea2dcbddba: entering candidate state: node="Node at 85a767c8-6099-1453-157d-22ea2dcbddba [Candidate]" term=3 +2025-03-20T03:51:17.291Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:17.291Z [INFO] server-70d2b76b-f5b2-22c6-145d-51a070adef09: entering follower state: follower="Node at 70d2b76b-f5b2-22c6-145d-51a070adef09 [Follower]" leader-address= leader-id= +2025-03-20T03:51:17.291Z [INFO] server-70d2b76b-f5b2-22c6-145d-51a070adef09: aborting pipeline replication: peer="{Voter server-85a767c8-6099-1453-157d-22ea2dcbddba 85a767c8-6099-1453-157d-22ea2dcbddba}" +2025-03-20T03:51:17.291Z [INFO] server-85a767c8-6099-1453-157d-22ea2dcbddba: election won: term=3 tally=2 +2025-03-20T03:51:17.291Z [INFO] server-85a767c8-6099-1453-157d-22ea2dcbddba: entering leader state: leader="Node at 85a767c8-6099-1453-157d-22ea2dcbddba [Leader]" +2025-03-20T03:51:17.291Z [INFO] server-85a767c8-6099-1453-157d-22ea2dcbddba: added peer, starting replication: peer=server-70d2b76b-f5b2-22c6-145d-51a070adef09 +2025-03-20T03:51:17.291Z [INFO] server-85a767c8-6099-1453-157d-22ea2dcbddba: added peer, starting replication: peer=server-21cbfb70-64b4-85a5-993a-40b9f8a1535a +2025-03-20T03:51:17.292Z [INFO] server-85a767c8-6099-1453-157d-22ea2dcbddba: pipelining replication: peer="{Voter server-21cbfb70-64b4-85a5-993a-40b9f8a1535a 21cbfb70-64b4-85a5-993a-40b9f8a1535a}" +2025-03-20T03:51:17.292Z [INFO] server-70d2b76b-f5b2-22c6-145d-51a070adef09: aborting pipeline replication: peer="{Voter server-21cbfb70-64b4-85a5-993a-40b9f8a1535a 21cbfb70-64b4-85a5-993a-40b9f8a1535a}" +2025-03-20T03:51:17.292Z [INFO] server-85a767c8-6099-1453-157d-22ea2dcbddba: pipelining replication: peer="{Voter server-70d2b76b-f5b2-22c6-145d-51a070adef09 70d2b76b-f5b2-22c6-145d-51a070adef09}" +2025-03-20T03:51:17.398Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:17.292103971 +0000 UTC m=+108.966556810 (1 nodes), 597.986µs from start of poll, 517.081114ms from cluster start. Timeout at 2025-03-20 03:51:17.397207347 +0000 UTC m=+109.071660351, 105.103541ms after stability +2025-03-20T03:51:17.398Z [INFO] server-85a767c8-6099-1453-157d-22ea2dcbddba: aborting pipeline replication: peer="{Voter server-21cbfb70-64b4-85a5-993a-40b9f8a1535a 21cbfb70-64b4-85a5-993a-40b9f8a1535a}" +2025-03-20T03:51:17.398Z [INFO] server-85a767c8-6099-1453-157d-22ea2dcbddba: aborting pipeline replication: peer="{Voter server-70d2b76b-f5b2-22c6-145d-51a070adef09 70d2b76b-f5b2-22c6-145d-51a070adef09}" +--- PASS: TestRaft_LeadershipTransferToBehindServer (0.63s) === RUN TestRaft_LeadershipTransferToItself -2026-04-22T10:10:58.456Z [INFO] server-d0efee09-c05d-7ba2-c9e8-f4c704a2cd35: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d0efee09-c05d-7ba2-c9e8-f4c704a2cd35 Address:d0efee09-c05d-7ba2-c9e8-f4c704a2cd35} {Suffrage:Voter ID:server-76d5ef58-6f45-5a26-af3d-b6911b83250e Address:76d5ef58-6f45-5a26-af3d-b6911b83250e} {Suffrage:Voter ID:server-b5e7e297-b3c1-2561-7ab5-1a8c858cd19b Address:b5e7e297-b3c1-2561-7ab5-1a8c858cd19b}]" -2026-04-22T10:10:58.456Z [INFO] server-d0efee09-c05d-7ba2-c9e8-f4c704a2cd35: entering follower state: follower="Node at d0efee09-c05d-7ba2-c9e8-f4c704a2cd35 [Follower]" leader-address= leader-id= -2026-04-22T10:10:58.456Z [INFO] server-76d5ef58-6f45-5a26-af3d-b6911b83250e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d0efee09-c05d-7ba2-c9e8-f4c704a2cd35 Address:d0efee09-c05d-7ba2-c9e8-f4c704a2cd35} {Suffrage:Voter ID:server-76d5ef58-6f45-5a26-af3d-b6911b83250e Address:76d5ef58-6f45-5a26-af3d-b6911b83250e} {Suffrage:Voter ID:server-b5e7e297-b3c1-2561-7ab5-1a8c858cd19b Address:b5e7e297-b3c1-2561-7ab5-1a8c858cd19b}]" -2026-04-22T10:10:58.456Z [INFO] server-76d5ef58-6f45-5a26-af3d-b6911b83250e: entering follower state: follower="Node at 76d5ef58-6f45-5a26-af3d-b6911b83250e [Follower]" leader-address= leader-id= -2026-04-22T10:10:58.456Z [INFO] server-b5e7e297-b3c1-2561-7ab5-1a8c858cd19b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d0efee09-c05d-7ba2-c9e8-f4c704a2cd35 Address:d0efee09-c05d-7ba2-c9e8-f4c704a2cd35} {Suffrage:Voter ID:server-76d5ef58-6f45-5a26-af3d-b6911b83250e Address:76d5ef58-6f45-5a26-af3d-b6911b83250e} {Suffrage:Voter ID:server-b5e7e297-b3c1-2561-7ab5-1a8c858cd19b Address:b5e7e297-b3c1-2561-7ab5-1a8c858cd19b}]" -2026-04-22T10:10:58.456Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:58.456Z [INFO] server-b5e7e297-b3c1-2561-7ab5-1a8c858cd19b: entering follower state: follower="Node at b5e7e297-b3c1-2561-7ab5-1a8c858cd19b [Follower]" leader-address= leader-id= -2026-04-22T10:10:58.538Z [WARN] server-b5e7e297-b3c1-2561-7ab5-1a8c858cd19b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:58.538Z [INFO] server-b5e7e297-b3c1-2561-7ab5-1a8c858cd19b: entering candidate state: node="Node at b5e7e297-b3c1-2561-7ab5-1a8c858cd19b [Candidate]" term=2 -2026-04-22T10:10:58.538Z [INFO] server-b5e7e297-b3c1-2561-7ab5-1a8c858cd19b: election won: term=2 tally=2 -2026-04-22T10:10:58.538Z [INFO] server-b5e7e297-b3c1-2561-7ab5-1a8c858cd19b: entering leader state: leader="Node at b5e7e297-b3c1-2561-7ab5-1a8c858cd19b [Leader]" -2026-04-22T10:10:58.538Z [INFO] server-b5e7e297-b3c1-2561-7ab5-1a8c858cd19b: added peer, starting replication: peer=server-d0efee09-c05d-7ba2-c9e8-f4c704a2cd35 -2026-04-22T10:10:58.538Z [INFO] server-b5e7e297-b3c1-2561-7ab5-1a8c858cd19b: added peer, starting replication: peer=server-76d5ef58-6f45-5a26-af3d-b6911b83250e -2026-04-22T10:10:58.538Z [INFO] server-b5e7e297-b3c1-2561-7ab5-1a8c858cd19b: pipelining replication: peer="{Voter server-76d5ef58-6f45-5a26-af3d-b6911b83250e 76d5ef58-6f45-5a26-af3d-b6911b83250e}" -2026-04-22T10:10:58.538Z [INFO] server-b5e7e297-b3c1-2561-7ab5-1a8c858cd19b: pipelining replication: peer="{Voter server-d0efee09-c05d-7ba2-c9e8-f4c704a2cd35 d0efee09-c05d-7ba2-c9e8-f4c704a2cd35}" -2026-04-22T10:10:58.643Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:58.5383452 +0000 UTC m=+104.562141316 (1 nodes), 82.001685ms from start of poll, 82.310348ms from cluster start. Timeout at 2026-04-22 10:10:58.643351785 +0000 UTC m=+104.667147866, 105.00655ms after stability -2026-04-22T10:10:58.643Z [INFO] server-b5e7e297-b3c1-2561-7ab5-1a8c858cd19b: cannot transfer leadership to itself -2026-04-22T10:10:58.643Z [INFO] server-b5e7e297-b3c1-2561-7ab5-1a8c858cd19b: aborting pipeline replication: peer="{Voter server-76d5ef58-6f45-5a26-af3d-b6911b83250e 76d5ef58-6f45-5a26-af3d-b6911b83250e}" -2026-04-22T10:10:58.643Z [INFO] server-b5e7e297-b3c1-2561-7ab5-1a8c858cd19b: aborting pipeline replication: peer="{Voter server-d0efee09-c05d-7ba2-c9e8-f4c704a2cd35 d0efee09-c05d-7ba2-c9e8-f4c704a2cd35}" ---- PASS: TestRaft_LeadershipTransferToItself (0.19s) +2025-03-20T03:51:17.400Z [INFO] server-698453a9-3f71-52f7-7a7d-4850d87fa18b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-698453a9-3f71-52f7-7a7d-4850d87fa18b Address:698453a9-3f71-52f7-7a7d-4850d87fa18b} {Suffrage:Voter ID:server-82ce7816-897d-78e2-6133-6c5ec5996dda Address:82ce7816-897d-78e2-6133-6c5ec5996dda} {Suffrage:Voter ID:server-4788e103-7acc-72d9-674e-bbc2cc286e63 Address:4788e103-7acc-72d9-674e-bbc2cc286e63}]" +2025-03-20T03:51:17.400Z [INFO] server-82ce7816-897d-78e2-6133-6c5ec5996dda: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-698453a9-3f71-52f7-7a7d-4850d87fa18b Address:698453a9-3f71-52f7-7a7d-4850d87fa18b} {Suffrage:Voter ID:server-82ce7816-897d-78e2-6133-6c5ec5996dda Address:82ce7816-897d-78e2-6133-6c5ec5996dda} {Suffrage:Voter ID:server-4788e103-7acc-72d9-674e-bbc2cc286e63 Address:4788e103-7acc-72d9-674e-bbc2cc286e63}]" +2025-03-20T03:51:17.400Z [INFO] server-4788e103-7acc-72d9-674e-bbc2cc286e63: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-698453a9-3f71-52f7-7a7d-4850d87fa18b Address:698453a9-3f71-52f7-7a7d-4850d87fa18b} {Suffrage:Voter ID:server-82ce7816-897d-78e2-6133-6c5ec5996dda Address:82ce7816-897d-78e2-6133-6c5ec5996dda} {Suffrage:Voter ID:server-4788e103-7acc-72d9-674e-bbc2cc286e63 Address:4788e103-7acc-72d9-674e-bbc2cc286e63}]" +2025-03-20T03:51:17.400Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:17.400Z [INFO] server-698453a9-3f71-52f7-7a7d-4850d87fa18b: entering follower state: follower="Node at 698453a9-3f71-52f7-7a7d-4850d87fa18b [Follower]" leader-address= leader-id= +2025-03-20T03:51:17.400Z [INFO] server-82ce7816-897d-78e2-6133-6c5ec5996dda: entering follower state: follower="Node at 82ce7816-897d-78e2-6133-6c5ec5996dda [Follower]" leader-address= leader-id= +2025-03-20T03:51:17.401Z [INFO] server-4788e103-7acc-72d9-674e-bbc2cc286e63: entering follower state: follower="Node at 4788e103-7acc-72d9-674e-bbc2cc286e63 [Follower]" leader-address= leader-id= +2025-03-20T03:51:17.455Z [WARN] server-4788e103-7acc-72d9-674e-bbc2cc286e63: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:17.455Z [INFO] server-4788e103-7acc-72d9-674e-bbc2cc286e63: entering candidate state: node="Node at 4788e103-7acc-72d9-674e-bbc2cc286e63 [Candidate]" term=2 +2025-03-20T03:51:17.455Z [INFO] server-4788e103-7acc-72d9-674e-bbc2cc286e63: election won: term=2 tally=2 +2025-03-20T03:51:17.455Z [INFO] server-4788e103-7acc-72d9-674e-bbc2cc286e63: entering leader state: leader="Node at 4788e103-7acc-72d9-674e-bbc2cc286e63 [Leader]" +2025-03-20T03:51:17.455Z [INFO] server-4788e103-7acc-72d9-674e-bbc2cc286e63: added peer, starting replication: peer=server-698453a9-3f71-52f7-7a7d-4850d87fa18b +2025-03-20T03:51:17.455Z [INFO] server-4788e103-7acc-72d9-674e-bbc2cc286e63: added peer, starting replication: peer=server-82ce7816-897d-78e2-6133-6c5ec5996dda +2025-03-20T03:51:17.455Z [INFO] server-4788e103-7acc-72d9-674e-bbc2cc286e63: pipelining replication: peer="{Voter server-82ce7816-897d-78e2-6133-6c5ec5996dda 82ce7816-897d-78e2-6133-6c5ec5996dda}" +2025-03-20T03:51:17.456Z [INFO] server-4788e103-7acc-72d9-674e-bbc2cc286e63: pipelining replication: peer="{Voter server-698453a9-3f71-52f7-7a7d-4850d87fa18b 698453a9-3f71-52f7-7a7d-4850d87fa18b}" +2025-03-20T03:51:17.562Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:17.456168788 +0000 UTC m=+109.130621645 (1 nodes), 55.40777ms from start of poll, 55.846541ms from cluster start. Timeout at 2025-03-20 03:51:17.561183689 +0000 UTC m=+109.235636552, 105.014907ms after stability +2025-03-20T03:51:17.564Z [INFO] server-4788e103-7acc-72d9-674e-bbc2cc286e63: cannot transfer leadership to itself +2025-03-20T03:51:17.564Z [INFO] server-4788e103-7acc-72d9-674e-bbc2cc286e63: aborting pipeline replication: peer="{Voter server-82ce7816-897d-78e2-6133-6c5ec5996dda 82ce7816-897d-78e2-6133-6c5ec5996dda}" +2025-03-20T03:51:17.564Z [INFO] server-4788e103-7acc-72d9-674e-bbc2cc286e63: aborting pipeline replication: peer="{Voter server-698453a9-3f71-52f7-7a7d-4850d87fa18b 698453a9-3f71-52f7-7a7d-4850d87fa18b}" +--- PASS: TestRaft_LeadershipTransferToItself (0.17s) === RUN TestRaft_LeadershipTransferLeaderRejectsClientRequests -2026-04-22T10:10:58.644Z [INFO] server-3e2578ff-885b-6834-076c-3e753a07ab7b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3e2578ff-885b-6834-076c-3e753a07ab7b Address:3e2578ff-885b-6834-076c-3e753a07ab7b} {Suffrage:Voter ID:server-323eda8a-bdc5-b7d2-3f2e-9fbb0fe1a48a Address:323eda8a-bdc5-b7d2-3f2e-9fbb0fe1a48a} {Suffrage:Voter ID:server-32521446-7b52-e275-8b26-f03cc3e57cc4 Address:32521446-7b52-e275-8b26-f03cc3e57cc4}]" -2026-04-22T10:10:58.644Z [INFO] server-3e2578ff-885b-6834-076c-3e753a07ab7b: entering follower state: follower="Node at 3e2578ff-885b-6834-076c-3e753a07ab7b [Follower]" leader-address= leader-id= -2026-04-22T10:10:58.644Z [INFO] server-323eda8a-bdc5-b7d2-3f2e-9fbb0fe1a48a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3e2578ff-885b-6834-076c-3e753a07ab7b Address:3e2578ff-885b-6834-076c-3e753a07ab7b} {Suffrage:Voter ID:server-323eda8a-bdc5-b7d2-3f2e-9fbb0fe1a48a Address:323eda8a-bdc5-b7d2-3f2e-9fbb0fe1a48a} {Suffrage:Voter ID:server-32521446-7b52-e275-8b26-f03cc3e57cc4 Address:32521446-7b52-e275-8b26-f03cc3e57cc4}]" -2026-04-22T10:10:58.644Z [INFO] server-32521446-7b52-e275-8b26-f03cc3e57cc4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3e2578ff-885b-6834-076c-3e753a07ab7b Address:3e2578ff-885b-6834-076c-3e753a07ab7b} {Suffrage:Voter ID:server-323eda8a-bdc5-b7d2-3f2e-9fbb0fe1a48a Address:323eda8a-bdc5-b7d2-3f2e-9fbb0fe1a48a} {Suffrage:Voter ID:server-32521446-7b52-e275-8b26-f03cc3e57cc4 Address:32521446-7b52-e275-8b26-f03cc3e57cc4}]" -2026-04-22T10:10:58.644Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:58.644Z [INFO] server-323eda8a-bdc5-b7d2-3f2e-9fbb0fe1a48a: entering follower state: follower="Node at 323eda8a-bdc5-b7d2-3f2e-9fbb0fe1a48a [Follower]" leader-address= leader-id= -2026-04-22T10:10:58.644Z [INFO] server-32521446-7b52-e275-8b26-f03cc3e57cc4: entering follower state: follower="Node at 32521446-7b52-e275-8b26-f03cc3e57cc4 [Follower]" leader-address= leader-id= -2026-04-22T10:10:58.696Z [WARN] server-3e2578ff-885b-6834-076c-3e753a07ab7b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:58.696Z [INFO] server-3e2578ff-885b-6834-076c-3e753a07ab7b: entering candidate state: node="Node at 3e2578ff-885b-6834-076c-3e753a07ab7b [Candidate]" term=2 -2026-04-22T10:10:58.696Z [INFO] server-3e2578ff-885b-6834-076c-3e753a07ab7b: election won: term=2 tally=2 -2026-04-22T10:10:58.696Z [INFO] server-3e2578ff-885b-6834-076c-3e753a07ab7b: entering leader state: leader="Node at 3e2578ff-885b-6834-076c-3e753a07ab7b [Leader]" -2026-04-22T10:10:58.696Z [INFO] server-3e2578ff-885b-6834-076c-3e753a07ab7b: added peer, starting replication: peer=server-323eda8a-bdc5-b7d2-3f2e-9fbb0fe1a48a -2026-04-22T10:10:58.696Z [INFO] server-3e2578ff-885b-6834-076c-3e753a07ab7b: added peer, starting replication: peer=server-32521446-7b52-e275-8b26-f03cc3e57cc4 -2026-04-22T10:10:58.696Z [INFO] server-3e2578ff-885b-6834-076c-3e753a07ab7b: pipelining replication: peer="{Voter server-32521446-7b52-e275-8b26-f03cc3e57cc4 32521446-7b52-e275-8b26-f03cc3e57cc4}" -2026-04-22T10:10:58.696Z [INFO] server-3e2578ff-885b-6834-076c-3e753a07ab7b: pipelining replication: peer="{Voter server-323eda8a-bdc5-b7d2-3f2e-9fbb0fe1a48a 323eda8a-bdc5-b7d2-3f2e-9fbb0fe1a48a}" -2026-04-22T10:10:58.801Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:58.696334668 +0000 UTC m=+104.720130922 (1 nodes), 51.539078ms from start of poll, 51.758775ms from cluster start. Timeout at 2026-04-22 10:10:58.801422625 +0000 UTC m=+104.825218785, 105.087863ms after stability +2025-03-20T03:51:17.566Z [INFO] server-5658eb4e-0e98-daeb-e337-eadb3af671d4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5658eb4e-0e98-daeb-e337-eadb3af671d4 Address:5658eb4e-0e98-daeb-e337-eadb3af671d4} {Suffrage:Voter ID:server-5163d754-f311-cef0-7494-2b96235eb20a Address:5163d754-f311-cef0-7494-2b96235eb20a} {Suffrage:Voter ID:server-b3950006-1df3-5cc2-69c6-26f0b226676e Address:b3950006-1df3-5cc2-69c6-26f0b226676e}]" +2025-03-20T03:51:17.566Z [INFO] server-5658eb4e-0e98-daeb-e337-eadb3af671d4: entering follower state: follower="Node at 5658eb4e-0e98-daeb-e337-eadb3af671d4 [Follower]" leader-address= leader-id= +2025-03-20T03:51:17.566Z [INFO] server-5163d754-f311-cef0-7494-2b96235eb20a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5658eb4e-0e98-daeb-e337-eadb3af671d4 Address:5658eb4e-0e98-daeb-e337-eadb3af671d4} {Suffrage:Voter ID:server-5163d754-f311-cef0-7494-2b96235eb20a Address:5163d754-f311-cef0-7494-2b96235eb20a} {Suffrage:Voter ID:server-b3950006-1df3-5cc2-69c6-26f0b226676e Address:b3950006-1df3-5cc2-69c6-26f0b226676e}]" +2025-03-20T03:51:17.567Z [INFO] server-5163d754-f311-cef0-7494-2b96235eb20a: entering follower state: follower="Node at 5163d754-f311-cef0-7494-2b96235eb20a [Follower]" leader-address= leader-id= +2025-03-20T03:51:17.567Z [INFO] server-b3950006-1df3-5cc2-69c6-26f0b226676e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5658eb4e-0e98-daeb-e337-eadb3af671d4 Address:5658eb4e-0e98-daeb-e337-eadb3af671d4} {Suffrage:Voter ID:server-5163d754-f311-cef0-7494-2b96235eb20a Address:5163d754-f311-cef0-7494-2b96235eb20a} {Suffrage:Voter ID:server-b3950006-1df3-5cc2-69c6-26f0b226676e Address:b3950006-1df3-5cc2-69c6-26f0b226676e}]" +2025-03-20T03:51:17.567Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:17.567Z [INFO] server-b3950006-1df3-5cc2-69c6-26f0b226676e: entering follower state: follower="Node at b3950006-1df3-5cc2-69c6-26f0b226676e [Follower]" leader-address= leader-id= +2025-03-20T03:51:17.642Z [WARN] server-5163d754-f311-cef0-7494-2b96235eb20a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:17.642Z [INFO] server-5163d754-f311-cef0-7494-2b96235eb20a: entering candidate state: node="Node at 5163d754-f311-cef0-7494-2b96235eb20a [Candidate]" term=2 +2025-03-20T03:51:17.643Z [INFO] server-5163d754-f311-cef0-7494-2b96235eb20a: election won: term=2 tally=2 +2025-03-20T03:51:17.643Z [INFO] server-5163d754-f311-cef0-7494-2b96235eb20a: entering leader state: leader="Node at 5163d754-f311-cef0-7494-2b96235eb20a [Leader]" +2025-03-20T03:51:17.643Z [INFO] server-5163d754-f311-cef0-7494-2b96235eb20a: added peer, starting replication: peer=server-5658eb4e-0e98-daeb-e337-eadb3af671d4 +2025-03-20T03:51:17.643Z [INFO] server-5163d754-f311-cef0-7494-2b96235eb20a: added peer, starting replication: peer=server-b3950006-1df3-5cc2-69c6-26f0b226676e +2025-03-20T03:51:17.643Z [INFO] server-5163d754-f311-cef0-7494-2b96235eb20a: pipelining replication: peer="{Voter server-b3950006-1df3-5cc2-69c6-26f0b226676e b3950006-1df3-5cc2-69c6-26f0b226676e}" +2025-03-20T03:51:17.643Z [INFO] server-5163d754-f311-cef0-7494-2b96235eb20a: pipelining replication: peer="{Voter server-5658eb4e-0e98-daeb-e337-eadb3af671d4 5658eb4e-0e98-daeb-e337-eadb3af671d4}" +2025-03-20T03:51:17.759Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:17.644071974 +0000 UTC m=+109.318524914 (1 nodes), 76.55078ms from start of poll, 77.474036ms from cluster start. Timeout at 2025-03-20 03:51:17.749139429 +0000 UTC m=+109.423592247, 105.067333ms after stability raft_test.go:2457: waiting on future 0 raft_test.go:2457: waiting on future 1 raft_test.go:2457: waiting on future 2 @@ -3863,348 +3998,391 @@ raft_test.go:2457: waiting on future 4 raft_test.go:2457: waiting on future 5 raft_test.go:2457: waiting on future 6 -2026-04-22T10:10:58.801Z [INFO] server-3e2578ff-885b-6834-076c-3e753a07ab7b: aborting pipeline replication: peer="{Voter server-32521446-7b52-e275-8b26-f03cc3e57cc4 32521446-7b52-e275-8b26-f03cc3e57cc4}" -2026-04-22T10:10:58.801Z [INFO] server-3e2578ff-885b-6834-076c-3e753a07ab7b: aborting pipeline replication: peer="{Voter server-323eda8a-bdc5-b7d2-3f2e-9fbb0fe1a48a 323eda8a-bdc5-b7d2-3f2e-9fbb0fe1a48a}" ---- PASS: TestRaft_LeadershipTransferLeaderRejectsClientRequests (0.16s) +2025-03-20T03:51:17.762Z [INFO] server-5163d754-f311-cef0-7494-2b96235eb20a: aborting pipeline replication: peer="{Voter server-b3950006-1df3-5cc2-69c6-26f0b226676e b3950006-1df3-5cc2-69c6-26f0b226676e}" +2025-03-20T03:51:17.762Z [INFO] server-5163d754-f311-cef0-7494-2b96235eb20a: aborting pipeline replication: peer="{Voter server-5658eb4e-0e98-daeb-e337-eadb3af671d4 5658eb4e-0e98-daeb-e337-eadb3af671d4}" +--- PASS: TestRaft_LeadershipTransferLeaderRejectsClientRequests (0.20s) === RUN TestRaft_LeadershipTransferLeaderReplicationTimeout -2026-04-22T10:10:58.802Z [INFO] server-5c8c2ae5-b485-efb2-23ec-b8bda1a0e0d2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5c8c2ae5-b485-efb2-23ec-b8bda1a0e0d2 Address:5c8c2ae5-b485-efb2-23ec-b8bda1a0e0d2} {Suffrage:Voter ID:server-b4c6452c-bb0b-9735-aa35-12b4fdbe79a1 Address:b4c6452c-bb0b-9735-aa35-12b4fdbe79a1} {Suffrage:Voter ID:server-f16a6078-2c71-6f1f-b942-eea5ead64c67 Address:f16a6078-2c71-6f1f-b942-eea5ead64c67}]" -2026-04-22T10:10:58.802Z [INFO] server-5c8c2ae5-b485-efb2-23ec-b8bda1a0e0d2: entering follower state: follower="Node at 5c8c2ae5-b485-efb2-23ec-b8bda1a0e0d2 [Follower]" leader-address= leader-id= -2026-04-22T10:10:58.802Z [INFO] server-b4c6452c-bb0b-9735-aa35-12b4fdbe79a1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5c8c2ae5-b485-efb2-23ec-b8bda1a0e0d2 Address:5c8c2ae5-b485-efb2-23ec-b8bda1a0e0d2} {Suffrage:Voter ID:server-b4c6452c-bb0b-9735-aa35-12b4fdbe79a1 Address:b4c6452c-bb0b-9735-aa35-12b4fdbe79a1} {Suffrage:Voter ID:server-f16a6078-2c71-6f1f-b942-eea5ead64c67 Address:f16a6078-2c71-6f1f-b942-eea5ead64c67}]" -2026-04-22T10:10:58.802Z [INFO] server-b4c6452c-bb0b-9735-aa35-12b4fdbe79a1: entering follower state: follower="Node at b4c6452c-bb0b-9735-aa35-12b4fdbe79a1 [Follower]" leader-address= leader-id= -2026-04-22T10:10:58.802Z [INFO] server-f16a6078-2c71-6f1f-b942-eea5ead64c67: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5c8c2ae5-b485-efb2-23ec-b8bda1a0e0d2 Address:5c8c2ae5-b485-efb2-23ec-b8bda1a0e0d2} {Suffrage:Voter ID:server-b4c6452c-bb0b-9735-aa35-12b4fdbe79a1 Address:b4c6452c-bb0b-9735-aa35-12b4fdbe79a1} {Suffrage:Voter ID:server-f16a6078-2c71-6f1f-b942-eea5ead64c67 Address:f16a6078-2c71-6f1f-b942-eea5ead64c67}]" -2026-04-22T10:10:58.802Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:58.802Z [INFO] server-f16a6078-2c71-6f1f-b942-eea5ead64c67: entering follower state: follower="Node at f16a6078-2c71-6f1f-b942-eea5ead64c67 [Follower]" leader-address= leader-id= -2026-04-22T10:10:58.854Z [WARN] server-b4c6452c-bb0b-9735-aa35-12b4fdbe79a1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:58.854Z [INFO] server-b4c6452c-bb0b-9735-aa35-12b4fdbe79a1: entering candidate state: node="Node at b4c6452c-bb0b-9735-aa35-12b4fdbe79a1 [Candidate]" term=2 -2026-04-22T10:10:58.854Z [INFO] server-b4c6452c-bb0b-9735-aa35-12b4fdbe79a1: election won: term=2 tally=2 -2026-04-22T10:10:58.854Z [INFO] server-b4c6452c-bb0b-9735-aa35-12b4fdbe79a1: entering leader state: leader="Node at b4c6452c-bb0b-9735-aa35-12b4fdbe79a1 [Leader]" -2026-04-22T10:10:58.854Z [INFO] server-b4c6452c-bb0b-9735-aa35-12b4fdbe79a1: added peer, starting replication: peer=server-5c8c2ae5-b485-efb2-23ec-b8bda1a0e0d2 -2026-04-22T10:10:58.854Z [INFO] server-b4c6452c-bb0b-9735-aa35-12b4fdbe79a1: added peer, starting replication: peer=server-f16a6078-2c71-6f1f-b942-eea5ead64c67 -2026-04-22T10:10:58.854Z [INFO] server-b4c6452c-bb0b-9735-aa35-12b4fdbe79a1: pipelining replication: peer="{Voter server-5c8c2ae5-b485-efb2-23ec-b8bda1a0e0d2 5c8c2ae5-b485-efb2-23ec-b8bda1a0e0d2}" -2026-04-22T10:10:58.854Z [INFO] server-b4c6452c-bb0b-9735-aa35-12b4fdbe79a1: pipelining replication: peer="{Voter server-f16a6078-2c71-6f1f-b942-eea5ead64c67 f16a6078-2c71-6f1f-b942-eea5ead64c67}" -2026-04-22T10:10:58.959Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:58.854376027 +0000 UTC m=+104.878172137 (1 nodes), 51.728931ms from start of poll, 51.944616ms from cluster start. Timeout at 2026-04-22 10:10:58.959388421 +0000 UTC m=+104.983184557, 105.01242ms after stability -2026-04-22T10:10:58.959Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:10:59.065Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:10:58.960009233 +0000 UTC m=+104.983805390 (2 nodes), 1.594µs from start of poll, 157.577869ms from cluster start. Timeout at 2026-04-22 10:10:59.065026257 +0000 UTC m=+105.088822358, 105.016968ms after stability -2026-04-22T10:10:59.494Z [INFO] server-b4c6452c-bb0b-9735-aa35-12b4fdbe79a1: aborting pipeline replication: peer="{Voter server-5c8c2ae5-b485-efb2-23ec-b8bda1a0e0d2 5c8c2ae5-b485-efb2-23ec-b8bda1a0e0d2}" -2026-04-22T10:10:59.494Z [INFO] server-b4c6452c-bb0b-9735-aa35-12b4fdbe79a1: aborting pipeline replication: peer="{Voter server-f16a6078-2c71-6f1f-b942-eea5ead64c67 f16a6078-2c71-6f1f-b942-eea5ead64c67}" ---- PASS: TestRaft_LeadershipTransferLeaderReplicationTimeout (0.69s) +2025-03-20T03:51:17.764Z [INFO] server-b78a439f-3396-d772-4d62-f60787c2bf5d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b78a439f-3396-d772-4d62-f60787c2bf5d Address:b78a439f-3396-d772-4d62-f60787c2bf5d} {Suffrage:Voter ID:server-9ba3e4e3-6a49-5775-b801-7f764df4f2e4 Address:9ba3e4e3-6a49-5775-b801-7f764df4f2e4} {Suffrage:Voter ID:server-339e8eae-6054-f384-d543-edab80158e60 Address:339e8eae-6054-f384-d543-edab80158e60}]" +2025-03-20T03:51:17.764Z [INFO] server-9ba3e4e3-6a49-5775-b801-7f764df4f2e4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b78a439f-3396-d772-4d62-f60787c2bf5d Address:b78a439f-3396-d772-4d62-f60787c2bf5d} {Suffrage:Voter ID:server-9ba3e4e3-6a49-5775-b801-7f764df4f2e4 Address:9ba3e4e3-6a49-5775-b801-7f764df4f2e4} {Suffrage:Voter ID:server-339e8eae-6054-f384-d543-edab80158e60 Address:339e8eae-6054-f384-d543-edab80158e60}]" +2025-03-20T03:51:17.764Z [INFO] server-339e8eae-6054-f384-d543-edab80158e60: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b78a439f-3396-d772-4d62-f60787c2bf5d Address:b78a439f-3396-d772-4d62-f60787c2bf5d} {Suffrage:Voter ID:server-9ba3e4e3-6a49-5775-b801-7f764df4f2e4 Address:9ba3e4e3-6a49-5775-b801-7f764df4f2e4} {Suffrage:Voter ID:server-339e8eae-6054-f384-d543-edab80158e60 Address:339e8eae-6054-f384-d543-edab80158e60}]" +2025-03-20T03:51:17.764Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:17.764Z [INFO] server-b78a439f-3396-d772-4d62-f60787c2bf5d: entering follower state: follower="Node at b78a439f-3396-d772-4d62-f60787c2bf5d [Follower]" leader-address= leader-id= +2025-03-20T03:51:17.764Z [INFO] server-9ba3e4e3-6a49-5775-b801-7f764df4f2e4: entering follower state: follower="Node at 9ba3e4e3-6a49-5775-b801-7f764df4f2e4 [Follower]" leader-address= leader-id= +2025-03-20T03:51:17.764Z [INFO] server-339e8eae-6054-f384-d543-edab80158e60: entering follower state: follower="Node at 339e8eae-6054-f384-d543-edab80158e60 [Follower]" leader-address= leader-id= +2025-03-20T03:51:17.844Z [WARN] server-339e8eae-6054-f384-d543-edab80158e60: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:17.844Z [INFO] server-339e8eae-6054-f384-d543-edab80158e60: entering candidate state: node="Node at 339e8eae-6054-f384-d543-edab80158e60 [Candidate]" term=2 +2025-03-20T03:51:17.844Z [INFO] server-339e8eae-6054-f384-d543-edab80158e60: election won: term=2 tally=2 +2025-03-20T03:51:17.844Z [INFO] server-339e8eae-6054-f384-d543-edab80158e60: entering leader state: leader="Node at 339e8eae-6054-f384-d543-edab80158e60 [Leader]" +2025-03-20T03:51:17.844Z [INFO] server-339e8eae-6054-f384-d543-edab80158e60: added peer, starting replication: peer=server-b78a439f-3396-d772-4d62-f60787c2bf5d +2025-03-20T03:51:17.844Z [INFO] server-339e8eae-6054-f384-d543-edab80158e60: added peer, starting replication: peer=server-9ba3e4e3-6a49-5775-b801-7f764df4f2e4 +2025-03-20T03:51:17.844Z [INFO] server-339e8eae-6054-f384-d543-edab80158e60: pipelining replication: peer="{Voter server-9ba3e4e3-6a49-5775-b801-7f764df4f2e4 9ba3e4e3-6a49-5775-b801-7f764df4f2e4}" +2025-03-20T03:51:17.844Z [WARN] server-b78a439f-3396-d772-4d62-f60787c2bf5d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:17.844Z [INFO] server-b78a439f-3396-d772-4d62-f60787c2bf5d: entering candidate state: node="Node at b78a439f-3396-d772-4d62-f60787c2bf5d [Candidate]" term=2 +2025-03-20T03:51:17.844Z [INFO] server-b78a439f-3396-d772-4d62-f60787c2bf5d: duplicate requestVote for same term: term=2 +2025-03-20T03:51:17.845Z [INFO] server-b78a439f-3396-d772-4d62-f60787c2bf5d: entering follower state: follower="Node at b78a439f-3396-d772-4d62-f60787c2bf5d [Follower]" leader-address=339e8eae-6054-f384-d543-edab80158e60 leader-id=server-339e8eae-6054-f384-d543-edab80158e60 +2025-03-20T03:51:17.845Z [INFO] server-339e8eae-6054-f384-d543-edab80158e60: pipelining replication: peer="{Voter server-b78a439f-3396-d772-4d62-f60787c2bf5d b78a439f-3396-d772-4d62-f60787c2bf5d}" +2025-03-20T03:51:17.845Z [WARN] server-339e8eae-6054-f384-d543-edab80158e60: rejecting vote request since we have a leader: from=b78a439f-3396-d772-4d62-f60787c2bf5d leader=339e8eae-6054-f384-d543-edab80158e60 leader-id=server-339e8eae-6054-f384-d543-edab80158e60 +2025-03-20T03:51:17.845Z [WARN] server-9ba3e4e3-6a49-5775-b801-7f764df4f2e4: rejecting vote request since we have a leader: from=b78a439f-3396-d772-4d62-f60787c2bf5d leader=339e8eae-6054-f384-d543-edab80158e60 leader-id=server-339e8eae-6054-f384-d543-edab80158e60 +2025-03-20T03:51:17.954Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:17.84543023 +0000 UTC m=+109.519883421 (1 nodes), 81.041163ms from start of poll, 81.43928ms from cluster start. Timeout at 2025-03-20 03:51:17.950475932 +0000 UTC m=+109.624928773, 105.045352ms after stability +2025-03-20T03:51:17.954Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:51:18.067Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:51:17.95429399 +0000 UTC m=+109.628746836 (2 nodes), 2.301µs from start of poll, 190.302695ms from cluster start. Timeout at 2025-03-20 03:51:18.059369619 +0000 UTC m=+109.733822709, 105.075873ms after stability +2025-03-20T03:51:18.068Z [WARN] server-b78a439f-3396-d772-4d62-f60787c2bf5d: skipping application of old log: index=2 +2025-03-20T03:51:18.584Z [WARN] server-339e8eae-6054-f384-d543-edab80158e60: failed to contact: server-id=server-b78a439f-3396-d772-4d62-f60787c2bf5d time=53.706461ms +2025-03-20T03:51:18.584Z [WARN] server-339e8eae-6054-f384-d543-edab80158e60: failed to contact: server-id=server-9ba3e4e3-6a49-5775-b801-7f764df4f2e4 time=56.709472ms +2025-03-20T03:51:18.584Z [WARN] server-339e8eae-6054-f384-d543-edab80158e60: failed to contact quorum of nodes, stepping down +2025-03-20T03:51:18.584Z [INFO] server-339e8eae-6054-f384-d543-edab80158e60: entering follower state: follower="Node at 339e8eae-6054-f384-d543-edab80158e60 [Follower]" leader-address= leader-id= +2025-03-20T03:51:18.584Z [INFO] server-339e8eae-6054-f384-d543-edab80158e60: aborting pipeline replication: peer="{Voter server-b78a439f-3396-d772-4d62-f60787c2bf5d b78a439f-3396-d772-4d62-f60787c2bf5d}" +2025-03-20T03:51:18.597Z [INFO] server-339e8eae-6054-f384-d543-edab80158e60: aborting pipeline replication: peer="{Voter server-9ba3e4e3-6a49-5775-b801-7f764df4f2e4 9ba3e4e3-6a49-5775-b801-7f764df4f2e4}" +2025-03-20T03:51:18.646Z [WARN] server-339e8eae-6054-f384-d543-edab80158e60: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:18.646Z [INFO] server-339e8eae-6054-f384-d543-edab80158e60: entering candidate state: node="Node at 339e8eae-6054-f384-d543-edab80158e60 [Candidate]" term=3 +2025-03-20T03:51:18.646Z [WARN] server-9ba3e4e3-6a49-5775-b801-7f764df4f2e4: heartbeat timeout reached, starting election: last-leader-addr=339e8eae-6054-f384-d543-edab80158e60 last-leader-id=server-339e8eae-6054-f384-d543-edab80158e60 +2025-03-20T03:51:18.646Z [INFO] server-9ba3e4e3-6a49-5775-b801-7f764df4f2e4: entering candidate state: node="Node at 9ba3e4e3-6a49-5775-b801-7f764df4f2e4 [Candidate]" term=3 +2025-03-20T03:51:18.646Z [INFO] server-9ba3e4e3-6a49-5775-b801-7f764df4f2e4: duplicate requestVote for same term: term=3 +2025-03-20T03:51:18.646Z [INFO] server-b78a439f-3396-d772-4d62-f60787c2bf5d: duplicate requestVote for same term: term=3 +2025-03-20T03:51:18.646Z [INFO] server-339e8eae-6054-f384-d543-edab80158e60: duplicate requestVote for same term: term=3 +2025-03-20T03:51:18.646Z [INFO] server-339e8eae-6054-f384-d543-edab80158e60: election won: term=3 tally=2 +2025-03-20T03:51:18.646Z [INFO] server-339e8eae-6054-f384-d543-edab80158e60: entering leader state: leader="Node at 339e8eae-6054-f384-d543-edab80158e60 [Leader]" +2025-03-20T03:51:18.646Z [INFO] server-339e8eae-6054-f384-d543-edab80158e60: added peer, starting replication: peer=server-b78a439f-3396-d772-4d62-f60787c2bf5d +2025-03-20T03:51:18.647Z [INFO] server-339e8eae-6054-f384-d543-edab80158e60: added peer, starting replication: peer=server-9ba3e4e3-6a49-5775-b801-7f764df4f2e4 +2025-03-20T03:51:18.647Z [INFO] server-339e8eae-6054-f384-d543-edab80158e60: pipelining replication: peer="{Voter server-b78a439f-3396-d772-4d62-f60787c2bf5d b78a439f-3396-d772-4d62-f60787c2bf5d}" +2025-03-20T03:51:18.647Z [INFO] server-9ba3e4e3-6a49-5775-b801-7f764df4f2e4: entering follower state: follower="Node at 9ba3e4e3-6a49-5775-b801-7f764df4f2e4 [Follower]" leader-address=339e8eae-6054-f384-d543-edab80158e60 leader-id=server-339e8eae-6054-f384-d543-edab80158e60 +2025-03-20T03:51:18.647Z [INFO] server-339e8eae-6054-f384-d543-edab80158e60: pipelining replication: peer="{Voter server-9ba3e4e3-6a49-5775-b801-7f764df4f2e4 9ba3e4e3-6a49-5775-b801-7f764df4f2e4}" +2025-03-20T03:51:20.167Z [INFO] server-339e8eae-6054-f384-d543-edab80158e60: aborting pipeline replication: peer="{Voter server-9ba3e4e3-6a49-5775-b801-7f764df4f2e4 9ba3e4e3-6a49-5775-b801-7f764df4f2e4}" +2025-03-20T03:51:20.167Z [INFO] server-339e8eae-6054-f384-d543-edab80158e60: aborting pipeline replication: peer="{Voter server-b78a439f-3396-d772-4d62-f60787c2bf5d b78a439f-3396-d772-4d62-f60787c2bf5d}" +--- PASS: TestRaft_LeadershipTransferLeaderReplicationTimeout (2.40s) === RUN TestRaft_LeadershipTransferIgnoresNonvoters -2026-04-22T10:10:59.495Z [INFO] server-ad2669b4-ad95-3474-513d-662790621387: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ad2669b4-ad95-3474-513d-662790621387 Address:ad2669b4-ad95-3474-513d-662790621387} {Suffrage:Voter ID:server-fb18f9cf-93b9-85f5-a090-83d23d1bc918 Address:fb18f9cf-93b9-85f5-a090-83d23d1bc918}]" -2026-04-22T10:10:59.495Z [INFO] server-ad2669b4-ad95-3474-513d-662790621387: entering follower state: follower="Node at ad2669b4-ad95-3474-513d-662790621387 [Follower]" leader-address= leader-id= -2026-04-22T10:10:59.495Z [INFO] server-fb18f9cf-93b9-85f5-a090-83d23d1bc918: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ad2669b4-ad95-3474-513d-662790621387 Address:ad2669b4-ad95-3474-513d-662790621387} {Suffrage:Voter ID:server-fb18f9cf-93b9-85f5-a090-83d23d1bc918 Address:fb18f9cf-93b9-85f5-a090-83d23d1bc918}]" -2026-04-22T10:10:59.495Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:10:59.495Z [INFO] server-fb18f9cf-93b9-85f5-a090-83d23d1bc918: entering follower state: follower="Node at fb18f9cf-93b9-85f5-a090-83d23d1bc918 [Follower]" leader-address= leader-id= -2026-04-22T10:10:59.557Z [WARN] server-fb18f9cf-93b9-85f5-a090-83d23d1bc918: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:59.557Z [INFO] server-fb18f9cf-93b9-85f5-a090-83d23d1bc918: entering candidate state: node="Node at fb18f9cf-93b9-85f5-a090-83d23d1bc918 [Candidate]" term=2 -2026-04-22T10:10:59.557Z [INFO] server-fb18f9cf-93b9-85f5-a090-83d23d1bc918: election won: term=2 tally=2 -2026-04-22T10:10:59.557Z [INFO] server-fb18f9cf-93b9-85f5-a090-83d23d1bc918: entering leader state: leader="Node at fb18f9cf-93b9-85f5-a090-83d23d1bc918 [Leader]" -2026-04-22T10:10:59.557Z [INFO] server-fb18f9cf-93b9-85f5-a090-83d23d1bc918: added peer, starting replication: peer=server-ad2669b4-ad95-3474-513d-662790621387 -2026-04-22T10:10:59.557Z [INFO] server-fb18f9cf-93b9-85f5-a090-83d23d1bc918: pipelining replication: peer="{Voter server-ad2669b4-ad95-3474-513d-662790621387 ad2669b4-ad95-3474-513d-662790621387}" -2026-04-22T10:10:59.663Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:10:59.557527007 +0000 UTC m=+105.581323095 (1 nodes), 62.254982ms from start of poll, 62.417796ms from cluster start. Timeout at 2026-04-22 10:10:59.662557858 +0000 UTC m=+105.686353935, 105.03084ms after stability -2026-04-22T10:10:59.663Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:59.769Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:59.663386955 +0000 UTC m=+105.687183044 (1 nodes), 541ns from start of poll, 168.277745ms from cluster start. Timeout at 2026-04-22 10:10:59.768427958 +0000 UTC m=+105.792224079, 105.041035ms after stability -2026-04-22T10:10:59.769Z [INFO] server-fb18f9cf-93b9-85f5-a090-83d23d1bc918: updating configuration: command=DemoteVoter server-id=server-ad2669b4-ad95-3474-513d-662790621387 server-addr= servers="[{Suffrage:Nonvoter ID:server-ad2669b4-ad95-3474-513d-662790621387 Address:ad2669b4-ad95-3474-513d-662790621387} {Suffrage:Voter ID:server-fb18f9cf-93b9-85f5-a090-83d23d1bc918 Address:fb18f9cf-93b9-85f5-a090-83d23d1bc918}]" -2026-04-22T10:10:59.769Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:10:59.874Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:10:59.769652172 +0000 UTC m=+105.793448260 (1 nodes), 527ns from start of poll, 274.542961ms from cluster start. Timeout at 2026-04-22 10:10:59.874662447 +0000 UTC m=+105.898458556, 105.010296ms after stability -2026-04-22T10:10:59.874Z [INFO] server-fb18f9cf-93b9-85f5-a090-83d23d1bc918: aborting pipeline replication: peer="{Voter server-ad2669b4-ad95-3474-513d-662790621387 ad2669b4-ad95-3474-513d-662790621387}" ---- PASS: TestRaft_LeadershipTransferIgnoresNonvoters (0.38s) +2025-03-20T03:51:20.168Z [INFO] server-333fb630-f8d8-f37b-52da-4993d2001b8b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-333fb630-f8d8-f37b-52da-4993d2001b8b Address:333fb630-f8d8-f37b-52da-4993d2001b8b} {Suffrage:Voter ID:server-57178a57-856b-b35f-693d-cdd5cc8f14a7 Address:57178a57-856b-b35f-693d-cdd5cc8f14a7}]" +2025-03-20T03:51:20.169Z [INFO] server-57178a57-856b-b35f-693d-cdd5cc8f14a7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-333fb630-f8d8-f37b-52da-4993d2001b8b Address:333fb630-f8d8-f37b-52da-4993d2001b8b} {Suffrage:Voter ID:server-57178a57-856b-b35f-693d-cdd5cc8f14a7 Address:57178a57-856b-b35f-693d-cdd5cc8f14a7}]" +2025-03-20T03:51:20.169Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:51:20.169Z [INFO] server-333fb630-f8d8-f37b-52da-4993d2001b8b: entering follower state: follower="Node at 333fb630-f8d8-f37b-52da-4993d2001b8b [Follower]" leader-address= leader-id= +2025-03-20T03:51:20.169Z [INFO] server-57178a57-856b-b35f-693d-cdd5cc8f14a7: entering follower state: follower="Node at 57178a57-856b-b35f-693d-cdd5cc8f14a7 [Follower]" leader-address= leader-id= +2025-03-20T03:51:20.238Z [WARN] server-57178a57-856b-b35f-693d-cdd5cc8f14a7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:20.238Z [INFO] server-57178a57-856b-b35f-693d-cdd5cc8f14a7: entering candidate state: node="Node at 57178a57-856b-b35f-693d-cdd5cc8f14a7 [Candidate]" term=2 +2025-03-20T03:51:20.238Z [INFO] server-57178a57-856b-b35f-693d-cdd5cc8f14a7: election won: term=2 tally=2 +2025-03-20T03:51:20.238Z [INFO] server-57178a57-856b-b35f-693d-cdd5cc8f14a7: entering leader state: leader="Node at 57178a57-856b-b35f-693d-cdd5cc8f14a7 [Leader]" +2025-03-20T03:51:20.238Z [INFO] server-57178a57-856b-b35f-693d-cdd5cc8f14a7: added peer, starting replication: peer=server-333fb630-f8d8-f37b-52da-4993d2001b8b +2025-03-20T03:51:20.238Z [INFO] server-57178a57-856b-b35f-693d-cdd5cc8f14a7: pipelining replication: peer="{Voter server-333fb630-f8d8-f37b-52da-4993d2001b8b 333fb630-f8d8-f37b-52da-4993d2001b8b}" +2025-03-20T03:51:20.343Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:51:20.23860892 +0000 UTC m=+111.913061752 (1 nodes), 69.554222ms from start of poll, 69.873058ms from cluster start. Timeout at 2025-03-20 03:51:20.343617815 +0000 UTC m=+112.018070636, 105.008884ms after stability +2025-03-20T03:51:20.343Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:20.455Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:20.34385797 +0000 UTC m=+112.018310820 (1 nodes), 2.179µs from start of poll, 175.122126ms from cluster start. Timeout at 2025-03-20 03:51:20.44903831 +0000 UTC m=+112.123491532, 105.180712ms after stability +2025-03-20T03:51:20.455Z [INFO] server-57178a57-856b-b35f-693d-cdd5cc8f14a7: updating configuration: command=DemoteVoter server-id=server-333fb630-f8d8-f37b-52da-4993d2001b8b server-addr= servers="[{Suffrage:Nonvoter ID:server-333fb630-f8d8-f37b-52da-4993d2001b8b Address:333fb630-f8d8-f37b-52da-4993d2001b8b} {Suffrage:Voter ID:server-57178a57-856b-b35f-693d-cdd5cc8f14a7 Address:57178a57-856b-b35f-693d-cdd5cc8f14a7}]" +2025-03-20T03:51:20.455Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:20.562Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:20.455682459 +0000 UTC m=+112.130135557 (1 nodes), 2.421µs from start of poll, 286.946863ms from cluster start. Timeout at 2025-03-20 03:51:20.560854565 +0000 UTC m=+112.235307323, 105.171766ms after stability +2025-03-20T03:51:20.563Z [INFO] server-57178a57-856b-b35f-693d-cdd5cc8f14a7: aborting pipeline replication: peer="{Voter server-333fb630-f8d8-f37b-52da-4993d2001b8b 333fb630-f8d8-f37b-52da-4993d2001b8b}" +--- PASS: TestRaft_LeadershipTransferIgnoresNonvoters (0.40s) === RUN TestRaft_LeadershipTransferStopRightAway --- PASS: TestRaft_LeadershipTransferStopRightAway (0.00s) === RUN TestRaft_GetConfigurationNoBootstrap -2026-04-22T10:10:59.875Z [INFO] server-cff6fabb-86d4-b28c-5cc9-fa04da23c815: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cff6fabb-86d4-b28c-5cc9-fa04da23c815 Address:cff6fabb-86d4-b28c-5cc9-fa04da23c815} {Suffrage:Voter ID:server-91e05e7d-5a89-c54b-97ed-6edc9e7b3667 Address:91e05e7d-5a89-c54b-97ed-6edc9e7b3667}]" -2026-04-22T10:10:59.875Z [INFO] server-cff6fabb-86d4-b28c-5cc9-fa04da23c815: entering follower state: follower="Node at cff6fabb-86d4-b28c-5cc9-fa04da23c815 [Follower]" leader-address= leader-id= -2026-04-22T10:10:59.875Z [INFO] server-91e05e7d-5a89-c54b-97ed-6edc9e7b3667: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cff6fabb-86d4-b28c-5cc9-fa04da23c815 Address:cff6fabb-86d4-b28c-5cc9-fa04da23c815} {Suffrage:Voter ID:server-91e05e7d-5a89-c54b-97ed-6edc9e7b3667 Address:91e05e7d-5a89-c54b-97ed-6edc9e7b3667}]" -2026-04-22T10:10:59.875Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:10:59.875Z [INFO] server-91e05e7d-5a89-c54b-97ed-6edc9e7b3667: entering follower state: follower="Node at 91e05e7d-5a89-c54b-97ed-6edc9e7b3667 [Follower]" leader-address= leader-id= -2026-04-22T10:10:59.932Z [WARN] server-cff6fabb-86d4-b28c-5cc9-fa04da23c815: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:10:59.932Z [INFO] server-cff6fabb-86d4-b28c-5cc9-fa04da23c815: entering candidate state: node="Node at cff6fabb-86d4-b28c-5cc9-fa04da23c815 [Candidate]" term=2 -2026-04-22T10:10:59.932Z [INFO] server-cff6fabb-86d4-b28c-5cc9-fa04da23c815: election won: term=2 tally=2 -2026-04-22T10:10:59.932Z [INFO] server-cff6fabb-86d4-b28c-5cc9-fa04da23c815: entering leader state: leader="Node at cff6fabb-86d4-b28c-5cc9-fa04da23c815 [Leader]" -2026-04-22T10:10:59.932Z [INFO] server-cff6fabb-86d4-b28c-5cc9-fa04da23c815: added peer, starting replication: peer=server-91e05e7d-5a89-c54b-97ed-6edc9e7b3667 -2026-04-22T10:10:59.932Z [INFO] server-cff6fabb-86d4-b28c-5cc9-fa04da23c815: pipelining replication: peer="{Voter server-91e05e7d-5a89-c54b-97ed-6edc9e7b3667 91e05e7d-5a89-c54b-97ed-6edc9e7b3667}" -2026-04-22T10:11:00.037Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:10:59.932233109 +0000 UTC m=+105.956029259 (1 nodes), 56.709992ms from start of poll, 56.880935ms from cluster start. Timeout at 2026-04-22 10:11:00.0372395 +0000 UTC m=+106.061035640, 105.006381ms after stability -2026-04-22T10:11:00.037Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:11:00.142Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:11:00.037322588 +0000 UTC m=+106.061118676 (1 nodes), 487ns from start of poll, 161.970352ms from cluster start. Timeout at 2026-04-22 10:11:00.14232888 +0000 UTC m=+106.166124991, 105.006315ms after stability -2026-04-22T10:11:00.154Z [INFO] server-91e05e7d-5a89-c54b-97ed-6edc9e7b3667: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cff6fabb-86d4-b28c-5cc9-fa04da23c815 Address:cff6fabb-86d4-b28c-5cc9-fa04da23c815} {Suffrage:Voter ID:server-91e05e7d-5a89-c54b-97ed-6edc9e7b3667 Address:91e05e7d-5a89-c54b-97ed-6edc9e7b3667}]" -2026-04-22T10:11:00.154Z [INFO] server-cff6fabb-86d4-b28c-5cc9-fa04da23c815: aborting pipeline replication: peer="{Voter server-91e05e7d-5a89-c54b-97ed-6edc9e7b3667 91e05e7d-5a89-c54b-97ed-6edc9e7b3667}" ---- PASS: TestRaft_GetConfigurationNoBootstrap (0.28s) +2025-03-20T03:51:20.564Z [INFO] server-3c6da126-be7d-46ab-b13d-3015726a82ec: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3c6da126-be7d-46ab-b13d-3015726a82ec Address:3c6da126-be7d-46ab-b13d-3015726a82ec} {Suffrage:Voter ID:server-a0324ee7-1db0-a481-12f3-91fe0cef49de Address:a0324ee7-1db0-a481-12f3-91fe0cef49de}]" +2025-03-20T03:51:20.564Z [INFO] server-3c6da126-be7d-46ab-b13d-3015726a82ec: entering follower state: follower="Node at 3c6da126-be7d-46ab-b13d-3015726a82ec [Follower]" leader-address= leader-id= +2025-03-20T03:51:20.564Z [INFO] server-a0324ee7-1db0-a481-12f3-91fe0cef49de: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3c6da126-be7d-46ab-b13d-3015726a82ec Address:3c6da126-be7d-46ab-b13d-3015726a82ec} {Suffrage:Voter ID:server-a0324ee7-1db0-a481-12f3-91fe0cef49de Address:a0324ee7-1db0-a481-12f3-91fe0cef49de}]" +2025-03-20T03:51:20.564Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:51:20.564Z [INFO] server-a0324ee7-1db0-a481-12f3-91fe0cef49de: entering follower state: follower="Node at a0324ee7-1db0-a481-12f3-91fe0cef49de [Follower]" leader-address= leader-id= +2025-03-20T03:51:20.637Z [WARN] server-3c6da126-be7d-46ab-b13d-3015726a82ec: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:20.637Z [INFO] server-3c6da126-be7d-46ab-b13d-3015726a82ec: entering candidate state: node="Node at 3c6da126-be7d-46ab-b13d-3015726a82ec [Candidate]" term=2 +2025-03-20T03:51:20.638Z [INFO] server-3c6da126-be7d-46ab-b13d-3015726a82ec: election won: term=2 tally=2 +2025-03-20T03:51:20.638Z [INFO] server-3c6da126-be7d-46ab-b13d-3015726a82ec: entering leader state: leader="Node at 3c6da126-be7d-46ab-b13d-3015726a82ec [Leader]" +2025-03-20T03:51:20.638Z [INFO] server-3c6da126-be7d-46ab-b13d-3015726a82ec: added peer, starting replication: peer=server-a0324ee7-1db0-a481-12f3-91fe0cef49de +2025-03-20T03:51:20.638Z [INFO] server-3c6da126-be7d-46ab-b13d-3015726a82ec: pipelining replication: peer="{Voter server-a0324ee7-1db0-a481-12f3-91fe0cef49de a0324ee7-1db0-a481-12f3-91fe0cef49de}" +2025-03-20T03:51:20.743Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:51:20.638386696 +0000 UTC m=+112.312839540 (1 nodes), 73.623189ms from start of poll, 73.951321ms from cluster start. Timeout at 2025-03-20 03:51:20.743391441 +0000 UTC m=+112.417844341, 105.004801ms after stability +2025-03-20T03:51:20.743Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:20.854Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:20.74353605 +0000 UTC m=+112.417988962 (1 nodes), 2.37µs from start of poll, 179.100743ms from cluster start. Timeout at 2025-03-20 03:51:20.848607476 +0000 UTC m=+112.523060547, 105.071585ms after stability +2025-03-20T03:51:20.868Z [INFO] server-a0324ee7-1db0-a481-12f3-91fe0cef49de: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3c6da126-be7d-46ab-b13d-3015726a82ec Address:3c6da126-be7d-46ab-b13d-3015726a82ec} {Suffrage:Voter ID:server-a0324ee7-1db0-a481-12f3-91fe0cef49de Address:a0324ee7-1db0-a481-12f3-91fe0cef49de}]" +2025-03-20T03:51:20.868Z [INFO] server-3c6da126-be7d-46ab-b13d-3015726a82ec: aborting pipeline replication: peer="{Voter server-a0324ee7-1db0-a481-12f3-91fe0cef49de a0324ee7-1db0-a481-12f3-91fe0cef49de}" +--- PASS: TestRaft_GetConfigurationNoBootstrap (0.31s) === RUN TestRaft_LogStoreIsMonotonic -2026-04-22T10:11:00.155Z [INFO] server-b7202c2b-609f-3985-3739-b2be37503658: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b7202c2b-609f-3985-3739-b2be37503658 Address:b7202c2b-609f-3985-3739-b2be37503658}]" -2026-04-22T10:11:00.155Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:11:00.155Z [INFO] server-b7202c2b-609f-3985-3739-b2be37503658: entering follower state: follower="Node at b7202c2b-609f-3985-3739-b2be37503658 [Follower]" leader-address= leader-id= -2026-04-22T10:11:00.210Z [WARN] server-b7202c2b-609f-3985-3739-b2be37503658: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:11:00.210Z [INFO] server-b7202c2b-609f-3985-3739-b2be37503658: entering candidate state: node="Node at b7202c2b-609f-3985-3739-b2be37503658 [Candidate]" term=2 -2026-04-22T10:11:00.210Z [INFO] server-b7202c2b-609f-3985-3739-b2be37503658: election won: term=2 tally=1 -2026-04-22T10:11:00.210Z [INFO] server-b7202c2b-609f-3985-3739-b2be37503658: entering leader state: leader="Node at b7202c2b-609f-3985-3739-b2be37503658 [Leader]" -2026-04-22T10:11:00.316Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:11:00.210786492 +0000 UTC m=+106.234582582 (1 nodes), 55.368569ms from start of poll, 55.451864ms from cluster start. Timeout at 2026-04-22 10:11:00.31580487 +0000 UTC m=+106.339601006, 105.018424ms after stability ---- PASS: TestRaft_LogStoreIsMonotonic (0.16s) +2025-03-20T03:51:20.869Z [INFO] server-737e83b8-44fb-bb87-e2c6-cb609060df7b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-737e83b8-44fb-bb87-e2c6-cb609060df7b Address:737e83b8-44fb-bb87-e2c6-cb609060df7b}]" +2025-03-20T03:51:20.869Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:20.870Z [INFO] server-737e83b8-44fb-bb87-e2c6-cb609060df7b: entering follower state: follower="Node at 737e83b8-44fb-bb87-e2c6-cb609060df7b [Follower]" leader-address= leader-id= +2025-03-20T03:51:20.950Z [WARN] server-737e83b8-44fb-bb87-e2c6-cb609060df7b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:20.950Z [INFO] server-737e83b8-44fb-bb87-e2c6-cb609060df7b: entering candidate state: node="Node at 737e83b8-44fb-bb87-e2c6-cb609060df7b [Candidate]" term=2 +2025-03-20T03:51:20.950Z [INFO] server-737e83b8-44fb-bb87-e2c6-cb609060df7b: election won: term=2 tally=1 +2025-03-20T03:51:20.950Z [INFO] server-737e83b8-44fb-bb87-e2c6-cb609060df7b: entering leader state: leader="Node at 737e83b8-44fb-bb87-e2c6-cb609060df7b [Leader]" +2025-03-20T03:51:21.063Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:20.950543327 +0000 UTC m=+112.624996252 (1 nodes), 80.556731ms from start of poll, 80.757826ms from cluster start. Timeout at 2025-03-20 03:51:21.055614034 +0000 UTC m=+112.730066929, 105.070677ms after stability +--- PASS: TestRaft_LogStoreIsMonotonic (0.19s) === RUN TestRaft_CacheLogWithStoreError -2026-04-22T10:11:00.317Z [INFO] server-fdddf404-8788-b29f-a092-9b27117c0647: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fdddf404-8788-b29f-a092-9b27117c0647 Address:fdddf404-8788-b29f-a092-9b27117c0647} {Suffrage:Voter ID:server-70be57f5-d9b6-84d0-0fa0-abe54230de90 Address:70be57f5-d9b6-84d0-0fa0-abe54230de90}]" -2026-04-22T10:11:00.317Z [INFO] server-fdddf404-8788-b29f-a092-9b27117c0647: entering follower state: follower="Node at fdddf404-8788-b29f-a092-9b27117c0647 [Follower]" leader-address= leader-id= -2026-04-22T10:11:00.317Z [INFO] server-70be57f5-d9b6-84d0-0fa0-abe54230de90: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fdddf404-8788-b29f-a092-9b27117c0647 Address:fdddf404-8788-b29f-a092-9b27117c0647} {Suffrage:Voter ID:server-70be57f5-d9b6-84d0-0fa0-abe54230de90 Address:70be57f5-d9b6-84d0-0fa0-abe54230de90}]" -2026-04-22T10:11:00.317Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:11:00.317Z [INFO] server-70be57f5-d9b6-84d0-0fa0-abe54230de90: entering follower state: follower="Node at 70be57f5-d9b6-84d0-0fa0-abe54230de90 [Follower]" leader-address= leader-id= -2026-04-22T10:11:00.384Z [WARN] server-fdddf404-8788-b29f-a092-9b27117c0647: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:11:00.384Z [INFO] server-fdddf404-8788-b29f-a092-9b27117c0647: entering candidate state: node="Node at fdddf404-8788-b29f-a092-9b27117c0647 [Candidate]" term=2 -2026-04-22T10:11:00.384Z [INFO] server-fdddf404-8788-b29f-a092-9b27117c0647: election won: term=2 tally=2 -2026-04-22T10:11:00.384Z [INFO] server-fdddf404-8788-b29f-a092-9b27117c0647: entering leader state: leader="Node at fdddf404-8788-b29f-a092-9b27117c0647 [Leader]" -2026-04-22T10:11:00.384Z [INFO] server-fdddf404-8788-b29f-a092-9b27117c0647: added peer, starting replication: peer=server-70be57f5-d9b6-84d0-0fa0-abe54230de90 -2026-04-22T10:11:00.384Z [INFO] server-fdddf404-8788-b29f-a092-9b27117c0647: pipelining replication: peer="{Voter server-70be57f5-d9b6-84d0-0fa0-abe54230de90 70be57f5-d9b6-84d0-0fa0-abe54230de90}" -2026-04-22T10:11:00.490Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:11:00.384784574 +0000 UTC m=+106.408580666 (1 nodes), 67.015597ms from start of poll, 67.795255ms from cluster start. Timeout at 2026-04-22 10:11:00.489787443 +0000 UTC m=+106.513583526, 105.00286ms after stability -2026-04-22T10:11:00.490Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:11:00.595Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:11:00.490194014 +0000 UTC m=+106.513990101 (1 nodes), 479ns from start of poll, 173.20469ms from cluster start. Timeout at 2026-04-22 10:11:00.595231652 +0000 UTC m=+106.619027763, 105.037662ms after stability -2026-04-22T10:11:00.595Z [ERROR] server-70be57f5-d9b6-84d0-0fa0-abe54230de90: failed to append to logs: error="unable to store logs within log store, err: "some error"" -2026-04-22T10:11:00.595Z [WARN] server-70be57f5-d9b6-84d0-0fa0-abe54230de90: failed to get previous log: previous-index=9 last-index=7 error="log not found" -2026-04-22T10:11:00.595Z [INFO] server-fdddf404-8788-b29f-a092-9b27117c0647: aborting pipeline replication: peer="{Voter server-70be57f5-d9b6-84d0-0fa0-abe54230de90 70be57f5-d9b6-84d0-0fa0-abe54230de90}" -2026-04-22T10:11:00.604Z [INFO] server-fdddf404-8788-b29f-a092-9b27117c0647: pipelining replication: peer="{Voter server-70be57f5-d9b6-84d0-0fa0-abe54230de90 70be57f5-d9b6-84d0-0fa0-abe54230de90}" -2026-04-22T10:11:00.605Z [INFO] server-70be57f5-d9b6-84d0-0fa0-abe54230de90: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fdddf404-8788-b29f-a092-9b27117c0647 Address:fdddf404-8788-b29f-a092-9b27117c0647} {Suffrage:Voter ID:server-70be57f5-d9b6-84d0-0fa0-abe54230de90 Address:70be57f5-d9b6-84d0-0fa0-abe54230de90}]" -2026-04-22T10:11:00.605Z [INFO] server-fdddf404-8788-b29f-a092-9b27117c0647: aborting pipeline replication: peer="{Voter server-70be57f5-d9b6-84d0-0fa0-abe54230de90 70be57f5-d9b6-84d0-0fa0-abe54230de90}" -2026-04-22T10:11:01.106Z [ERROR] server-fdddf404-8788-b29f-a092-9b27117c0647: failed to heartbeat to: peer=70be57f5-d9b6-84d0-0fa0-abe54230de90 backoff time=10ms error="command timed out" ---- PASS: TestRaft_CacheLogWithStoreError (0.79s) +2025-03-20T03:51:21.064Z [INFO] server-f8083864-cb87-faee-85e6-c0051d7a0814: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f8083864-cb87-faee-85e6-c0051d7a0814 Address:f8083864-cb87-faee-85e6-c0051d7a0814} {Suffrage:Voter ID:server-2df0c4d1-6aa4-4e52-b4df-c531a89699ae Address:2df0c4d1-6aa4-4e52-b4df-c531a89699ae}]" +2025-03-20T03:51:21.064Z [INFO] server-2df0c4d1-6aa4-4e52-b4df-c531a89699ae: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f8083864-cb87-faee-85e6-c0051d7a0814 Address:f8083864-cb87-faee-85e6-c0051d7a0814} {Suffrage:Voter ID:server-2df0c4d1-6aa4-4e52-b4df-c531a89699ae Address:2df0c4d1-6aa4-4e52-b4df-c531a89699ae}]" +2025-03-20T03:51:21.064Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:51:21.064Z [INFO] server-f8083864-cb87-faee-85e6-c0051d7a0814: entering follower state: follower="Node at f8083864-cb87-faee-85e6-c0051d7a0814 [Follower]" leader-address= leader-id= +2025-03-20T03:51:21.064Z [INFO] server-2df0c4d1-6aa4-4e52-b4df-c531a89699ae: entering follower state: follower="Node at 2df0c4d1-6aa4-4e52-b4df-c531a89699ae [Follower]" leader-address= leader-id= +2025-03-20T03:51:21.128Z [WARN] server-f8083864-cb87-faee-85e6-c0051d7a0814: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:21.128Z [INFO] server-f8083864-cb87-faee-85e6-c0051d7a0814: entering candidate state: node="Node at f8083864-cb87-faee-85e6-c0051d7a0814 [Candidate]" term=2 +2025-03-20T03:51:21.129Z [INFO] server-f8083864-cb87-faee-85e6-c0051d7a0814: election won: term=2 tally=2 +2025-03-20T03:51:21.129Z [INFO] server-f8083864-cb87-faee-85e6-c0051d7a0814: entering leader state: leader="Node at f8083864-cb87-faee-85e6-c0051d7a0814 [Leader]" +2025-03-20T03:51:21.129Z [INFO] server-f8083864-cb87-faee-85e6-c0051d7a0814: added peer, starting replication: peer=server-2df0c4d1-6aa4-4e52-b4df-c531a89699ae +2025-03-20T03:51:21.129Z [INFO] server-f8083864-cb87-faee-85e6-c0051d7a0814: pipelining replication: peer="{Voter server-2df0c4d1-6aa4-4e52-b4df-c531a89699ae 2df0c4d1-6aa4-4e52-b4df-c531a89699ae}" +2025-03-20T03:51:21.245Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:51:21.129413394 +0000 UTC m=+112.803866246 (1 nodes), 64.727047ms from start of poll, 65.047696ms from cluster start. Timeout at 2025-03-20 03:51:21.234433931 +0000 UTC m=+112.908886863, 105.020617ms after stability +2025-03-20T03:51:21.245Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:21.350Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:21.245472795 +0000 UTC m=+112.919925625 (1 nodes), 2.152µs from start of poll, 181.107075ms from cluster start. Timeout at 2025-03-20 03:51:21.35056757 +0000 UTC m=+113.025020775, 105.09515ms after stability +2025-03-20T03:51:21.359Z [ERROR] server-2df0c4d1-6aa4-4e52-b4df-c531a89699ae: failed to append to logs: error="unable to store logs within log store, err: "some error"" +2025-03-20T03:51:21.359Z [WARN] server-2df0c4d1-6aa4-4e52-b4df-c531a89699ae: failed to get previous log: previous-index=9 last-index=7 error="log not found" +2025-03-20T03:51:21.359Z [INFO] server-f8083864-cb87-faee-85e6-c0051d7a0814: aborting pipeline replication: peer="{Voter server-2df0c4d1-6aa4-4e52-b4df-c531a89699ae 2df0c4d1-6aa4-4e52-b4df-c531a89699ae}" +2025-03-20T03:51:21.370Z [INFO] server-f8083864-cb87-faee-85e6-c0051d7a0814: pipelining replication: peer="{Voter server-2df0c4d1-6aa4-4e52-b4df-c531a89699ae 2df0c4d1-6aa4-4e52-b4df-c531a89699ae}" +2025-03-20T03:51:21.371Z [INFO] server-2df0c4d1-6aa4-4e52-b4df-c531a89699ae: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f8083864-cb87-faee-85e6-c0051d7a0814 Address:f8083864-cb87-faee-85e6-c0051d7a0814} {Suffrage:Voter ID:server-2df0c4d1-6aa4-4e52-b4df-c531a89699ae Address:2df0c4d1-6aa4-4e52-b4df-c531a89699ae}]" +2025-03-20T03:51:21.371Z [INFO] server-f8083864-cb87-faee-85e6-c0051d7a0814: aborting pipeline replication: peer="{Voter server-2df0c4d1-6aa4-4e52-b4df-c531a89699ae 2df0c4d1-6aa4-4e52-b4df-c531a89699ae}" +--- PASS: TestRaft_CacheLogWithStoreError (0.31s) === RUN TestRaft_ReloadConfig -2026-04-22T10:11:01.107Z [INFO] server-14bddd41-5a99-d744-87fe-84d6d56512f8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-14bddd41-5a99-d744-87fe-84d6d56512f8 Address:14bddd41-5a99-d744-87fe-84d6d56512f8}]" -2026-04-22T10:11:01.107Z [INFO] server-14bddd41-5a99-d744-87fe-84d6d56512f8: entering follower state: follower="Node at 14bddd41-5a99-d744-87fe-84d6d56512f8 [Shutdown]" leader-address= leader-id= +2025-03-20T03:51:21.372Z [INFO] server-a8cff29b-26e4-356a-5977-71328d83adfb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a8cff29b-26e4-356a-5977-71328d83adfb Address:a8cff29b-26e4-356a-5977-71328d83adfb}]" --- PASS: TestRaft_ReloadConfig (0.00s) === RUN TestRaft_ReloadConfigValidates -2026-04-22T10:11:01.108Z [INFO] server-c83e52c7-1175-46bd-aadd-78edbb599e90: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c83e52c7-1175-46bd-aadd-78edbb599e90 Address:c83e52c7-1175-46bd-aadd-78edbb599e90}]" +2025-03-20T03:51:21.373Z [INFO] server-ff07e2e7-4c15-b184-fc6c-7f8a33b15e3a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff07e2e7-4c15-b184-fc6c-7f8a33b15e3a Address:ff07e2e7-4c15-b184-fc6c-7f8a33b15e3a}]" --- PASS: TestRaft_ReloadConfigValidates (0.00s) === RUN TestRaft_InstallSnapshot_InvalidPeers -2026-04-22T10:11:01.108Z [ERROR] failed to install snapshot: error="failed to decode peers: codec.decoder: Only encoded map or array can be decoded into a slice. (valueType: 2)" +2025-03-20T03:51:21.373Z [ERROR] failed to install snapshot: error="failed to decode peers: codec.decoder: Only encoded map or array can be decoded into a slice. (valueType: 2)" --- PASS: TestRaft_InstallSnapshot_InvalidPeers (0.00s) === RUN TestRaft_VoteNotGranted_WhenNodeNotInCluster -2026-04-22T10:11:01.109Z [INFO] server-eef27240-9675-43d5-15d8-4da355a5007e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eef27240-9675-43d5-15d8-4da355a5007e Address:eef27240-9675-43d5-15d8-4da355a5007e} {Suffrage:Voter ID:server-c050fba8-5421-1404-bc73-6670e7fae1a4 Address:c050fba8-5421-1404-bc73-6670e7fae1a4} {Suffrage:Voter ID:server-9d694bfd-cdfa-ddf1-d72c-8c6e108652ed Address:9d694bfd-cdfa-ddf1-d72c-8c6e108652ed}]" -2026-04-22T10:11:01.109Z [INFO] server-eef27240-9675-43d5-15d8-4da355a5007e: entering follower state: follower="Node at eef27240-9675-43d5-15d8-4da355a5007e [Follower]" leader-address= leader-id= -2026-04-22T10:11:01.109Z [INFO] server-c050fba8-5421-1404-bc73-6670e7fae1a4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eef27240-9675-43d5-15d8-4da355a5007e Address:eef27240-9675-43d5-15d8-4da355a5007e} {Suffrage:Voter ID:server-c050fba8-5421-1404-bc73-6670e7fae1a4 Address:c050fba8-5421-1404-bc73-6670e7fae1a4} {Suffrage:Voter ID:server-9d694bfd-cdfa-ddf1-d72c-8c6e108652ed Address:9d694bfd-cdfa-ddf1-d72c-8c6e108652ed}]" -2026-04-22T10:11:01.109Z [INFO] server-c050fba8-5421-1404-bc73-6670e7fae1a4: entering follower state: follower="Node at c050fba8-5421-1404-bc73-6670e7fae1a4 [Follower]" leader-address= leader-id= -2026-04-22T10:11:01.109Z [INFO] server-9d694bfd-cdfa-ddf1-d72c-8c6e108652ed: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eef27240-9675-43d5-15d8-4da355a5007e Address:eef27240-9675-43d5-15d8-4da355a5007e} {Suffrage:Voter ID:server-c050fba8-5421-1404-bc73-6670e7fae1a4 Address:c050fba8-5421-1404-bc73-6670e7fae1a4} {Suffrage:Voter ID:server-9d694bfd-cdfa-ddf1-d72c-8c6e108652ed Address:9d694bfd-cdfa-ddf1-d72c-8c6e108652ed}]" -2026-04-22T10:11:01.109Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:11:01.109Z [INFO] server-9d694bfd-cdfa-ddf1-d72c-8c6e108652ed: entering follower state: follower="Node at 9d694bfd-cdfa-ddf1-d72c-8c6e108652ed [Follower]" leader-address= leader-id= -2026-04-22T10:11:01.196Z [WARN] server-c050fba8-5421-1404-bc73-6670e7fae1a4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:11:01.196Z [INFO] server-c050fba8-5421-1404-bc73-6670e7fae1a4: entering candidate state: node="Node at c050fba8-5421-1404-bc73-6670e7fae1a4 [Candidate]" term=2 -2026-04-22T10:11:01.196Z [INFO] server-c050fba8-5421-1404-bc73-6670e7fae1a4: election won: term=2 tally=2 -2026-04-22T10:11:01.196Z [INFO] server-c050fba8-5421-1404-bc73-6670e7fae1a4: entering leader state: leader="Node at c050fba8-5421-1404-bc73-6670e7fae1a4 [Leader]" -2026-04-22T10:11:01.196Z [INFO] server-c050fba8-5421-1404-bc73-6670e7fae1a4: added peer, starting replication: peer=server-eef27240-9675-43d5-15d8-4da355a5007e -2026-04-22T10:11:01.196Z [INFO] server-c050fba8-5421-1404-bc73-6670e7fae1a4: added peer, starting replication: peer=server-9d694bfd-cdfa-ddf1-d72c-8c6e108652ed -2026-04-22T10:11:01.196Z [INFO] server-c050fba8-5421-1404-bc73-6670e7fae1a4: pipelining replication: peer="{Voter server-9d694bfd-cdfa-ddf1-d72c-8c6e108652ed 9d694bfd-cdfa-ddf1-d72c-8c6e108652ed}" -2026-04-22T10:11:01.196Z [INFO] server-c050fba8-5421-1404-bc73-6670e7fae1a4: pipelining replication: peer="{Voter server-eef27240-9675-43d5-15d8-4da355a5007e eef27240-9675-43d5-15d8-4da355a5007e}" -2026-04-22T10:11:01.301Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:11:01.196597136 +0000 UTC m=+107.220393286 (1 nodes), 87.330477ms from start of poll, 87.56987ms from cluster start. Timeout at 2026-04-22 10:11:01.301601032 +0000 UTC m=+107.325397113, 105.003827ms after stability -2026-04-22T10:11:01.307Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:11:01.412Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:11:01.307508817 +0000 UTC m=+107.331304905 (2 nodes), 717ns from start of poll, 198.481489ms from cluster start. Timeout at 2026-04-22 10:11:01.412527564 +0000 UTC m=+107.436323767, 105.018862ms after stability -2026-04-22T10:11:01.412Z [INFO] server-c050fba8-5421-1404-bc73-6670e7fae1a4: updating configuration: command=RemoveServer server-id=server-eef27240-9675-43d5-15d8-4da355a5007e server-addr= servers="[{Suffrage:Voter ID:server-c050fba8-5421-1404-bc73-6670e7fae1a4 Address:c050fba8-5421-1404-bc73-6670e7fae1a4} {Suffrage:Voter ID:server-9d694bfd-cdfa-ddf1-d72c-8c6e108652ed Address:9d694bfd-cdfa-ddf1-d72c-8c6e108652ed}]" -2026-04-22T10:11:01.412Z [INFO] server-c050fba8-5421-1404-bc73-6670e7fae1a4: removed peer, stopping replication: peer=server-eef27240-9675-43d5-15d8-4da355a5007e last-index=3 -2026-04-22T10:11:01.412Z [INFO] server-c050fba8-5421-1404-bc73-6670e7fae1a4: aborting pipeline replication: peer="{Voter server-eef27240-9675-43d5-15d8-4da355a5007e eef27240-9675-43d5-15d8-4da355a5007e}" -2026-04-22T10:11:01.519Z [ERROR] server-c050fba8-5421-1404-bc73-6670e7fae1a4: failed to heartbeat to: peer=9d694bfd-cdfa-ddf1-d72c-8c6e108652ed backoff time=10ms error="failed to connect to peer: 9d694bfd-cdfa-ddf1-d72c-8c6e108652ed" -2026-04-22T10:11:01.520Z [ERROR] server-c050fba8-5421-1404-bc73-6670e7fae1a4: failed to pipeline appendEntries: peer="{Voter server-9d694bfd-cdfa-ddf1-d72c-8c6e108652ed 9d694bfd-cdfa-ddf1-d72c-8c6e108652ed}" error="append pipeline closed" -2026-04-22T10:11:01.520Z [INFO] server-c050fba8-5421-1404-bc73-6670e7fae1a4: aborting pipeline replication: peer="{Voter server-9d694bfd-cdfa-ddf1-d72c-8c6e108652ed 9d694bfd-cdfa-ddf1-d72c-8c6e108652ed}" -2026-04-22T10:11:01.527Z [ERROR] server-c050fba8-5421-1404-bc73-6670e7fae1a4: failed to appendEntries to: peer="{Voter server-9d694bfd-cdfa-ddf1-d72c-8c6e108652ed 9d694bfd-cdfa-ddf1-d72c-8c6e108652ed}" error="failed to connect to peer: 9d694bfd-cdfa-ddf1-d72c-8c6e108652ed" -2026-04-22T10:11:01.528Z [WARN] server-eef27240-9675-43d5-15d8-4da355a5007e: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election -2026-04-22T10:11:01.539Z [ERROR] server-c050fba8-5421-1404-bc73-6670e7fae1a4: failed to heartbeat to: peer=9d694bfd-cdfa-ddf1-d72c-8c6e108652ed backoff time=10ms error="failed to connect to peer: 9d694bfd-cdfa-ddf1-d72c-8c6e108652ed" -2026-04-22T10:11:01.543Z [ERROR] server-c050fba8-5421-1404-bc73-6670e7fae1a4: failed to appendEntries to: peer="{Voter server-9d694bfd-cdfa-ddf1-d72c-8c6e108652ed 9d694bfd-cdfa-ddf1-d72c-8c6e108652ed}" error="failed to connect to peer: 9d694bfd-cdfa-ddf1-d72c-8c6e108652ed" -2026-04-22T10:11:01.560Z [ERROR] server-c050fba8-5421-1404-bc73-6670e7fae1a4: failed to heartbeat to: peer=9d694bfd-cdfa-ddf1-d72c-8c6e108652ed backoff time=10ms error="failed to connect to peer: 9d694bfd-cdfa-ddf1-d72c-8c6e108652ed" -2026-04-22T10:11:01.564Z [ERROR] server-c050fba8-5421-1404-bc73-6670e7fae1a4: failed to appendEntries to: peer="{Voter server-9d694bfd-cdfa-ddf1-d72c-8c6e108652ed 9d694bfd-cdfa-ddf1-d72c-8c6e108652ed}" error="failed to connect to peer: 9d694bfd-cdfa-ddf1-d72c-8c6e108652ed" -2026-04-22T10:11:01.564Z [WARN] server-c050fba8-5421-1404-bc73-6670e7fae1a4: failed to contact: server-id=server-9d694bfd-cdfa-ddf1-d72c-8c6e108652ed time=50.96708ms -2026-04-22T10:11:01.564Z [WARN] server-c050fba8-5421-1404-bc73-6670e7fae1a4: failed to contact quorum of nodes, stepping down -2026-04-22T10:11:01.564Z [INFO] server-c050fba8-5421-1404-bc73-6670e7fae1a4: entering follower state: follower="Node at c050fba8-5421-1404-bc73-6670e7fae1a4 [Follower]" leader-address= leader-id= -2026-04-22T10:11:01.593Z [WARN] server-9d694bfd-cdfa-ddf1-d72c-8c6e108652ed: heartbeat timeout reached, starting election: last-leader-addr=c050fba8-5421-1404-bc73-6670e7fae1a4 last-leader-id=server-c050fba8-5421-1404-bc73-6670e7fae1a4 -2026-04-22T10:11:01.593Z [INFO] server-9d694bfd-cdfa-ddf1-d72c-8c6e108652ed: entering candidate state: node="Node at 9d694bfd-cdfa-ddf1-d72c-8c6e108652ed [Candidate]" term=3 -2026-04-22T10:11:01.593Z [ERROR] server-9d694bfd-cdfa-ddf1-d72c-8c6e108652ed: failed to make requestVote RPC: target="{Voter server-c050fba8-5421-1404-bc73-6670e7fae1a4 c050fba8-5421-1404-bc73-6670e7fae1a4}" error="failed to connect to peer: c050fba8-5421-1404-bc73-6670e7fae1a4" term=3 -2026-04-22T10:11:01.624Z [WARN] server-9d694bfd-cdfa-ddf1-d72c-8c6e108652ed: rejecting vote request since node is not in configuration: from=eef27240-9675-43d5-15d8-4da355a5007e ---- PASS: TestRaft_VoteNotGranted_WhenNodeNotInCluster (0.52s) +2025-03-20T03:51:21.391Z [INFO] server-311f54d7-6cd8-a3ba-cdf2-9d3586aee6a4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-311f54d7-6cd8-a3ba-cdf2-9d3586aee6a4 Address:311f54d7-6cd8-a3ba-cdf2-9d3586aee6a4} {Suffrage:Voter ID:server-5a9d4368-bad9-4efc-5321-a31bc089d400 Address:5a9d4368-bad9-4efc-5321-a31bc089d400} {Suffrage:Voter ID:server-97c358ac-9359-9927-f424-1b569f1f1c77 Address:97c358ac-9359-9927-f424-1b569f1f1c77}]" +2025-03-20T03:51:21.391Z [INFO] server-5a9d4368-bad9-4efc-5321-a31bc089d400: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-311f54d7-6cd8-a3ba-cdf2-9d3586aee6a4 Address:311f54d7-6cd8-a3ba-cdf2-9d3586aee6a4} {Suffrage:Voter ID:server-5a9d4368-bad9-4efc-5321-a31bc089d400 Address:5a9d4368-bad9-4efc-5321-a31bc089d400} {Suffrage:Voter ID:server-97c358ac-9359-9927-f424-1b569f1f1c77 Address:97c358ac-9359-9927-f424-1b569f1f1c77}]" +2025-03-20T03:51:21.392Z [INFO] server-311f54d7-6cd8-a3ba-cdf2-9d3586aee6a4: entering follower state: follower="Node at 311f54d7-6cd8-a3ba-cdf2-9d3586aee6a4 [Follower]" leader-address= leader-id= +2025-03-20T03:51:21.392Z [INFO] server-5a9d4368-bad9-4efc-5321-a31bc089d400: entering follower state: follower="Node at 5a9d4368-bad9-4efc-5321-a31bc089d400 [Follower]" leader-address= leader-id= +2025-03-20T03:51:21.396Z [INFO] server-97c358ac-9359-9927-f424-1b569f1f1c77: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-311f54d7-6cd8-a3ba-cdf2-9d3586aee6a4 Address:311f54d7-6cd8-a3ba-cdf2-9d3586aee6a4} {Suffrage:Voter ID:server-5a9d4368-bad9-4efc-5321-a31bc089d400 Address:5a9d4368-bad9-4efc-5321-a31bc089d400} {Suffrage:Voter ID:server-97c358ac-9359-9927-f424-1b569f1f1c77 Address:97c358ac-9359-9927-f424-1b569f1f1c77}]" +2025-03-20T03:51:21.396Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:21.396Z [INFO] server-97c358ac-9359-9927-f424-1b569f1f1c77: entering follower state: follower="Node at 97c358ac-9359-9927-f424-1b569f1f1c77 [Follower]" leader-address= leader-id= +2025-03-20T03:51:21.474Z [WARN] server-5a9d4368-bad9-4efc-5321-a31bc089d400: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:21.474Z [INFO] server-5a9d4368-bad9-4efc-5321-a31bc089d400: entering candidate state: node="Node at 5a9d4368-bad9-4efc-5321-a31bc089d400 [Candidate]" term=2 +2025-03-20T03:51:21.474Z [INFO] server-5a9d4368-bad9-4efc-5321-a31bc089d400: election won: term=2 tally=2 +2025-03-20T03:51:21.474Z [INFO] server-5a9d4368-bad9-4efc-5321-a31bc089d400: entering leader state: leader="Node at 5a9d4368-bad9-4efc-5321-a31bc089d400 [Leader]" +2025-03-20T03:51:21.474Z [INFO] server-5a9d4368-bad9-4efc-5321-a31bc089d400: added peer, starting replication: peer=server-311f54d7-6cd8-a3ba-cdf2-9d3586aee6a4 +2025-03-20T03:51:21.474Z [INFO] server-5a9d4368-bad9-4efc-5321-a31bc089d400: added peer, starting replication: peer=server-97c358ac-9359-9927-f424-1b569f1f1c77 +2025-03-20T03:51:21.474Z [INFO] server-5a9d4368-bad9-4efc-5321-a31bc089d400: pipelining replication: peer="{Voter server-97c358ac-9359-9927-f424-1b569f1f1c77 97c358ac-9359-9927-f424-1b569f1f1c77}" +2025-03-20T03:51:21.474Z [INFO] server-5a9d4368-bad9-4efc-5321-a31bc089d400: pipelining replication: peer="{Voter server-311f54d7-6cd8-a3ba-cdf2-9d3586aee6a4 311f54d7-6cd8-a3ba-cdf2-9d3586aee6a4}" +2025-03-20T03:51:21.583Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:21.474970398 +0000 UTC m=+113.149423215 (1 nodes), 78.535187ms from start of poll, 84.036522ms from cluster start. Timeout at 2025-03-20 03:51:21.579984733 +0000 UTC m=+113.254437565, 105.01435ms after stability +2025-03-20T03:51:21.588Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:51:21.695Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:51:21.588403396 +0000 UTC m=+113.262856180 (2 nodes), 3.972µs from start of poll, 197.469487ms from cluster start. Timeout at 2025-03-20 03:51:21.693565229 +0000 UTC m=+113.368018064, 105.161884ms after stability +2025-03-20T03:51:21.695Z [INFO] server-5a9d4368-bad9-4efc-5321-a31bc089d400: updating configuration: command=RemoveServer server-id=server-311f54d7-6cd8-a3ba-cdf2-9d3586aee6a4 server-addr= servers="[{Suffrage:Voter ID:server-5a9d4368-bad9-4efc-5321-a31bc089d400 Address:5a9d4368-bad9-4efc-5321-a31bc089d400} {Suffrage:Voter ID:server-97c358ac-9359-9927-f424-1b569f1f1c77 Address:97c358ac-9359-9927-f424-1b569f1f1c77}]" +2025-03-20T03:51:21.695Z [INFO] server-5a9d4368-bad9-4efc-5321-a31bc089d400: removed peer, stopping replication: peer=server-311f54d7-6cd8-a3ba-cdf2-9d3586aee6a4 last-index=3 +2025-03-20T03:51:21.696Z [INFO] server-5a9d4368-bad9-4efc-5321-a31bc089d400: aborting pipeline replication: peer="{Voter server-311f54d7-6cd8-a3ba-cdf2-9d3586aee6a4 311f54d7-6cd8-a3ba-cdf2-9d3586aee6a4}" +2025-03-20T03:51:21.782Z [WARN] server-311f54d7-6cd8-a3ba-cdf2-9d3586aee6a4: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +2025-03-20T03:51:21.808Z [ERROR] server-5a9d4368-bad9-4efc-5321-a31bc089d400: failed to heartbeat to: peer=97c358ac-9359-9927-f424-1b569f1f1c77 backoff time=10ms error="failed to connect to peer: 97c358ac-9359-9927-f424-1b569f1f1c77" +2025-03-20T03:51:21.810Z [ERROR] server-5a9d4368-bad9-4efc-5321-a31bc089d400: failed to pipeline appendEntries: peer="{Voter server-97c358ac-9359-9927-f424-1b569f1f1c77 97c358ac-9359-9927-f424-1b569f1f1c77}" error="append pipeline closed" +2025-03-20T03:51:21.810Z [INFO] server-5a9d4368-bad9-4efc-5321-a31bc089d400: aborting pipeline replication: peer="{Voter server-97c358ac-9359-9927-f424-1b569f1f1c77 97c358ac-9359-9927-f424-1b569f1f1c77}" +2025-03-20T03:51:21.817Z [ERROR] server-5a9d4368-bad9-4efc-5321-a31bc089d400: failed to appendEntries to: peer="{Voter server-97c358ac-9359-9927-f424-1b569f1f1c77 97c358ac-9359-9927-f424-1b569f1f1c77}" error="failed to connect to peer: 97c358ac-9359-9927-f424-1b569f1f1c77" +2025-03-20T03:51:21.827Z [ERROR] server-5a9d4368-bad9-4efc-5321-a31bc089d400: failed to heartbeat to: peer=97c358ac-9359-9927-f424-1b569f1f1c77 backoff time=10ms error="failed to connect to peer: 97c358ac-9359-9927-f424-1b569f1f1c77" +2025-03-20T03:51:21.835Z [ERROR] server-5a9d4368-bad9-4efc-5321-a31bc089d400: failed to appendEntries to: peer="{Voter server-97c358ac-9359-9927-f424-1b569f1f1c77 97c358ac-9359-9927-f424-1b569f1f1c77}" error="failed to connect to peer: 97c358ac-9359-9927-f424-1b569f1f1c77" +2025-03-20T03:51:21.847Z [ERROR] server-5a9d4368-bad9-4efc-5321-a31bc089d400: failed to heartbeat to: peer=97c358ac-9359-9927-f424-1b569f1f1c77 backoff time=10ms error="failed to connect to peer: 97c358ac-9359-9927-f424-1b569f1f1c77" +2025-03-20T03:51:21.852Z [ERROR] server-5a9d4368-bad9-4efc-5321-a31bc089d400: failed to appendEntries to: peer="{Voter server-97c358ac-9359-9927-f424-1b569f1f1c77 97c358ac-9359-9927-f424-1b569f1f1c77}" error="failed to connect to peer: 97c358ac-9359-9927-f424-1b569f1f1c77" +2025-03-20T03:51:21.853Z [WARN] server-5a9d4368-bad9-4efc-5321-a31bc089d400: failed to contact: server-id=server-97c358ac-9359-9927-f424-1b569f1f1c77 time=52.178966ms +2025-03-20T03:51:21.853Z [WARN] server-5a9d4368-bad9-4efc-5321-a31bc089d400: failed to contact quorum of nodes, stepping down +2025-03-20T03:51:21.853Z [INFO] server-5a9d4368-bad9-4efc-5321-a31bc089d400: entering follower state: follower="Node at 5a9d4368-bad9-4efc-5321-a31bc089d400 [Follower]" leader-address= leader-id= +2025-03-20T03:51:21.908Z [WARN] server-97c358ac-9359-9927-f424-1b569f1f1c77: heartbeat timeout reached, starting election: last-leader-addr=5a9d4368-bad9-4efc-5321-a31bc089d400 last-leader-id=server-5a9d4368-bad9-4efc-5321-a31bc089d400 +2025-03-20T03:51:21.908Z [INFO] server-97c358ac-9359-9927-f424-1b569f1f1c77: entering candidate state: node="Node at 97c358ac-9359-9927-f424-1b569f1f1c77 [Candidate]" term=3 +2025-03-20T03:51:21.908Z [ERROR] server-97c358ac-9359-9927-f424-1b569f1f1c77: failed to make requestVote RPC: target="{Voter server-5a9d4368-bad9-4efc-5321-a31bc089d400 5a9d4368-bad9-4efc-5321-a31bc089d400}" error="failed to connect to peer: 5a9d4368-bad9-4efc-5321-a31bc089d400" term=3 +2025-03-20T03:51:21.909Z [WARN] server-97c358ac-9359-9927-f424-1b569f1f1c77: rejecting vote request since node is not in configuration: from=311f54d7-6cd8-a3ba-cdf2-9d3586aee6a4 +--- PASS: TestRaft_VoteNotGranted_WhenNodeNotInCluster (0.54s) === RUN TestRaft_ClusterCanRegainStability_WhenNonVoterWithHigherTermJoin -2026-04-22T10:11:01.625Z [INFO] server-c2714c75-176b-29b5-f842-4eb2faa7e453: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c2714c75-176b-29b5-f842-4eb2faa7e453 Address:c2714c75-176b-29b5-f842-4eb2faa7e453} {Suffrage:Voter ID:server-be5436b9-396e-6576-ac1d-f3fac73e6ffa Address:be5436b9-396e-6576-ac1d-f3fac73e6ffa} {Suffrage:Voter ID:server-69bde12a-d80b-e84c-c422-5a8d3f0063eb Address:69bde12a-d80b-e84c-c422-5a8d3f0063eb}]" -2026-04-22T10:11:01.625Z [INFO] server-c2714c75-176b-29b5-f842-4eb2faa7e453: entering follower state: follower="Node at c2714c75-176b-29b5-f842-4eb2faa7e453 [Follower]" leader-address= leader-id= -2026-04-22T10:11:01.626Z [INFO] server-be5436b9-396e-6576-ac1d-f3fac73e6ffa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c2714c75-176b-29b5-f842-4eb2faa7e453 Address:c2714c75-176b-29b5-f842-4eb2faa7e453} {Suffrage:Voter ID:server-be5436b9-396e-6576-ac1d-f3fac73e6ffa Address:be5436b9-396e-6576-ac1d-f3fac73e6ffa} {Suffrage:Voter ID:server-69bde12a-d80b-e84c-c422-5a8d3f0063eb Address:69bde12a-d80b-e84c-c422-5a8d3f0063eb}]" -2026-04-22T10:11:01.626Z [INFO] server-be5436b9-396e-6576-ac1d-f3fac73e6ffa: entering follower state: follower="Node at be5436b9-396e-6576-ac1d-f3fac73e6ffa [Follower]" leader-address= leader-id= -2026-04-22T10:11:01.626Z [INFO] server-69bde12a-d80b-e84c-c422-5a8d3f0063eb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c2714c75-176b-29b5-f842-4eb2faa7e453 Address:c2714c75-176b-29b5-f842-4eb2faa7e453} {Suffrage:Voter ID:server-be5436b9-396e-6576-ac1d-f3fac73e6ffa Address:be5436b9-396e-6576-ac1d-f3fac73e6ffa} {Suffrage:Voter ID:server-69bde12a-d80b-e84c-c422-5a8d3f0063eb Address:69bde12a-d80b-e84c-c422-5a8d3f0063eb}]" -2026-04-22T10:11:01.626Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:11:01.626Z [INFO] server-69bde12a-d80b-e84c-c422-5a8d3f0063eb: entering follower state: follower="Node at 69bde12a-d80b-e84c-c422-5a8d3f0063eb [Follower]" leader-address= leader-id= -2026-04-22T10:11:01.701Z [WARN] server-c2714c75-176b-29b5-f842-4eb2faa7e453: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:11:01.701Z [INFO] server-c2714c75-176b-29b5-f842-4eb2faa7e453: entering candidate state: node="Node at c2714c75-176b-29b5-f842-4eb2faa7e453 [Candidate]" term=2 -2026-04-22T10:11:01.701Z [INFO] server-c2714c75-176b-29b5-f842-4eb2faa7e453: election won: term=2 tally=2 -2026-04-22T10:11:01.701Z [INFO] server-c2714c75-176b-29b5-f842-4eb2faa7e453: entering leader state: leader="Node at c2714c75-176b-29b5-f842-4eb2faa7e453 [Leader]" -2026-04-22T10:11:01.701Z [INFO] server-c2714c75-176b-29b5-f842-4eb2faa7e453: added peer, starting replication: peer=server-be5436b9-396e-6576-ac1d-f3fac73e6ffa -2026-04-22T10:11:01.701Z [INFO] server-c2714c75-176b-29b5-f842-4eb2faa7e453: added peer, starting replication: peer=server-69bde12a-d80b-e84c-c422-5a8d3f0063eb -2026-04-22T10:11:01.701Z [INFO] server-c2714c75-176b-29b5-f842-4eb2faa7e453: pipelining replication: peer="{Voter server-69bde12a-d80b-e84c-c422-5a8d3f0063eb 69bde12a-d80b-e84c-c422-5a8d3f0063eb}" -2026-04-22T10:11:01.701Z [INFO] server-c2714c75-176b-29b5-f842-4eb2faa7e453: pipelining replication: peer="{Voter server-be5436b9-396e-6576-ac1d-f3fac73e6ffa be5436b9-396e-6576-ac1d-f3fac73e6ffa}" -2026-04-22T10:11:01.806Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:11:01.70125104 +0000 UTC m=+107.725047158 (1 nodes), 75.100766ms from start of poll, 75.417871ms from cluster start. Timeout at 2026-04-22 10:11:01.806256568 +0000 UTC m=+107.830052647, 105.005489ms after stability -2026-04-22T10:11:01.811Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:11:01.917Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:11:01.811992937 +0000 UTC m=+107.835789087 (2 nodes), 1.624µs from start of poll, 186.1598ms from cluster start. Timeout at 2026-04-22 10:11:01.917034535 +0000 UTC m=+107.940830616, 105.041529ms after stability -2026-04-22T10:11:01.918Z [ERROR] server-c2714c75-176b-29b5-f842-4eb2faa7e453: failed to pipeline appendEntries: peer="{Voter server-be5436b9-396e-6576-ac1d-f3fac73e6ffa be5436b9-396e-6576-ac1d-f3fac73e6ffa}" error="append pipeline closed" -2026-04-22T10:11:01.918Z [INFO] server-c2714c75-176b-29b5-f842-4eb2faa7e453: aborting pipeline replication: peer="{Voter server-be5436b9-396e-6576-ac1d-f3fac73e6ffa be5436b9-396e-6576-ac1d-f3fac73e6ffa}" -2026-04-22T10:11:01.919Z [ERROR] server-c2714c75-176b-29b5-f842-4eb2faa7e453: failed to heartbeat to: peer=be5436b9-396e-6576-ac1d-f3fac73e6ffa backoff time=10ms error="failed to connect to peer: be5436b9-396e-6576-ac1d-f3fac73e6ffa" -2026-04-22T10:11:01.924Z [ERROR] server-c2714c75-176b-29b5-f842-4eb2faa7e453: failed to appendEntries to: peer="{Voter server-be5436b9-396e-6576-ac1d-f3fac73e6ffa be5436b9-396e-6576-ac1d-f3fac73e6ffa}" error="failed to connect to peer: be5436b9-396e-6576-ac1d-f3fac73e6ffa" -2026-04-22T10:11:01.935Z [ERROR] server-c2714c75-176b-29b5-f842-4eb2faa7e453: failed to heartbeat to: peer=be5436b9-396e-6576-ac1d-f3fac73e6ffa backoff time=10ms error="failed to connect to peer: be5436b9-396e-6576-ac1d-f3fac73e6ffa" -2026-04-22T10:11:01.941Z [ERROR] server-c2714c75-176b-29b5-f842-4eb2faa7e453: failed to appendEntries to: peer="{Voter server-be5436b9-396e-6576-ac1d-f3fac73e6ffa be5436b9-396e-6576-ac1d-f3fac73e6ffa}" error="failed to connect to peer: be5436b9-396e-6576-ac1d-f3fac73e6ffa" -2026-04-22T10:11:01.954Z [ERROR] server-c2714c75-176b-29b5-f842-4eb2faa7e453: failed to heartbeat to: peer=be5436b9-396e-6576-ac1d-f3fac73e6ffa backoff time=10ms error="failed to connect to peer: be5436b9-396e-6576-ac1d-f3fac73e6ffa" -2026-04-22T10:11:01.959Z [ERROR] server-c2714c75-176b-29b5-f842-4eb2faa7e453: failed to appendEntries to: peer="{Voter server-be5436b9-396e-6576-ac1d-f3fac73e6ffa be5436b9-396e-6576-ac1d-f3fac73e6ffa}" error="failed to connect to peer: be5436b9-396e-6576-ac1d-f3fac73e6ffa" -2026-04-22T10:11:01.963Z [WARN] server-c2714c75-176b-29b5-f842-4eb2faa7e453: failed to contact: server-id=server-be5436b9-396e-6576-ac1d-f3fac73e6ffa time=50.272197ms -2026-04-22T10:11:01.973Z [ERROR] server-c2714c75-176b-29b5-f842-4eb2faa7e453: failed to heartbeat to: peer=be5436b9-396e-6576-ac1d-f3fac73e6ffa backoff time=20ms error="failed to connect to peer: be5436b9-396e-6576-ac1d-f3fac73e6ffa" -2026-04-22T10:11:01.987Z [ERROR] server-c2714c75-176b-29b5-f842-4eb2faa7e453: failed to appendEntries to: peer="{Voter server-be5436b9-396e-6576-ac1d-f3fac73e6ffa be5436b9-396e-6576-ac1d-f3fac73e6ffa}" error="failed to connect to peer: be5436b9-396e-6576-ac1d-f3fac73e6ffa" -2026-04-22T10:11:01.997Z [WARN] server-be5436b9-396e-6576-ac1d-f3fac73e6ffa: heartbeat timeout reached, starting election: last-leader-addr=c2714c75-176b-29b5-f842-4eb2faa7e453 last-leader-id=server-c2714c75-176b-29b5-f842-4eb2faa7e453 -2026-04-22T10:11:01.997Z [INFO] server-be5436b9-396e-6576-ac1d-f3fac73e6ffa: entering candidate state: node="Node at be5436b9-396e-6576-ac1d-f3fac73e6ffa [Candidate]" term=3 -2026-04-22T10:11:01.997Z [ERROR] server-be5436b9-396e-6576-ac1d-f3fac73e6ffa: failed to make requestVote RPC: target="{Voter server-69bde12a-d80b-e84c-c422-5a8d3f0063eb 69bde12a-d80b-e84c-c422-5a8d3f0063eb}" error="failed to connect to peer: 69bde12a-d80b-e84c-c422-5a8d3f0063eb" term=3 -2026-04-22T10:11:01.997Z [ERROR] server-be5436b9-396e-6576-ac1d-f3fac73e6ffa: failed to make requestVote RPC: target="{Voter server-c2714c75-176b-29b5-f842-4eb2faa7e453 c2714c75-176b-29b5-f842-4eb2faa7e453}" error="failed to connect to peer: c2714c75-176b-29b5-f842-4eb2faa7e453" term=3 -2026-04-22T10:11:01.999Z [ERROR] server-c2714c75-176b-29b5-f842-4eb2faa7e453: failed to heartbeat to: peer=be5436b9-396e-6576-ac1d-f3fac73e6ffa backoff time=25ms error="failed to connect to peer: be5436b9-396e-6576-ac1d-f3fac73e6ffa" -2026-04-22T10:11:02.012Z [WARN] server-c2714c75-176b-29b5-f842-4eb2faa7e453: failed to contact: server-id=server-be5436b9-396e-6576-ac1d-f3fac73e6ffa time=98.904262ms -2026-04-22T10:11:02.022Z [INFO] server-c2714c75-176b-29b5-f842-4eb2faa7e453: updating configuration: command=RemoveServer server-id=server-be5436b9-396e-6576-ac1d-f3fac73e6ffa server-addr= servers="[{Suffrage:Voter ID:server-c2714c75-176b-29b5-f842-4eb2faa7e453 Address:c2714c75-176b-29b5-f842-4eb2faa7e453} {Suffrage:Voter ID:server-69bde12a-d80b-e84c-c422-5a8d3f0063eb Address:69bde12a-d80b-e84c-c422-5a8d3f0063eb}]" -2026-04-22T10:11:02.022Z [INFO] server-c2714c75-176b-29b5-f842-4eb2faa7e453: removed peer, stopping replication: peer=server-be5436b9-396e-6576-ac1d-f3fac73e6ffa last-index=3 -2026-04-22T10:11:02.022Z [INFO] server-c2714c75-176b-29b5-f842-4eb2faa7e453: updating configuration: command=AddNonvoter server-id=server-be5436b9-396e-6576-ac1d-f3fac73e6ffa server-addr=be5436b9-396e-6576-ac1d-f3fac73e6ffa servers="[{Suffrage:Voter ID:server-c2714c75-176b-29b5-f842-4eb2faa7e453 Address:c2714c75-176b-29b5-f842-4eb2faa7e453} {Suffrage:Voter ID:server-69bde12a-d80b-e84c-c422-5a8d3f0063eb Address:69bde12a-d80b-e84c-c422-5a8d3f0063eb} {Suffrage:Nonvoter ID:server-be5436b9-396e-6576-ac1d-f3fac73e6ffa Address:be5436b9-396e-6576-ac1d-f3fac73e6ffa}]" -2026-04-22T10:11:02.022Z [INFO] server-c2714c75-176b-29b5-f842-4eb2faa7e453: added peer, starting replication: peer=server-be5436b9-396e-6576-ac1d-f3fac73e6ffa -2026-04-22T10:11:02.022Z [ERROR] server-c2714c75-176b-29b5-f842-4eb2faa7e453: failed to appendEntries to: peer="{Nonvoter server-be5436b9-396e-6576-ac1d-f3fac73e6ffa be5436b9-396e-6576-ac1d-f3fac73e6ffa}" error="failed to connect to peer: be5436b9-396e-6576-ac1d-f3fac73e6ffa" -2026-04-22T10:11:02.035Z [ERROR] server-c2714c75-176b-29b5-f842-4eb2faa7e453: peer has newer term, stopping replication: peer="{Voter server-be5436b9-396e-6576-ac1d-f3fac73e6ffa be5436b9-396e-6576-ac1d-f3fac73e6ffa}" -2026-04-22T10:11:02.035Z [INFO] server-c2714c75-176b-29b5-f842-4eb2faa7e453: entering follower state: follower="Node at c2714c75-176b-29b5-f842-4eb2faa7e453 [Follower]" leader-address= leader-id= -2026-04-22T10:11:02.035Z [INFO] server-c2714c75-176b-29b5-f842-4eb2faa7e453: aborting pipeline replication: peer="{Voter server-69bde12a-d80b-e84c-c422-5a8d3f0063eb 69bde12a-d80b-e84c-c422-5a8d3f0063eb}" -2026-04-22T10:11:02.040Z [ERROR] server-c2714c75-176b-29b5-f842-4eb2faa7e453: peer has newer term, stopping replication: peer="{Nonvoter server-be5436b9-396e-6576-ac1d-f3fac73e6ffa be5436b9-396e-6576-ac1d-f3fac73e6ffa}" -2026-04-22T10:11:02.086Z [WARN] server-69bde12a-d80b-e84c-c422-5a8d3f0063eb: heartbeat timeout reached, starting election: last-leader-addr=c2714c75-176b-29b5-f842-4eb2faa7e453 last-leader-id=server-c2714c75-176b-29b5-f842-4eb2faa7e453 -2026-04-22T10:11:02.086Z [INFO] server-69bde12a-d80b-e84c-c422-5a8d3f0063eb: entering candidate state: node="Node at 69bde12a-d80b-e84c-c422-5a8d3f0063eb [Candidate]" term=3 -2026-04-22T10:11:02.086Z [INFO] server-69bde12a-d80b-e84c-c422-5a8d3f0063eb: election won: term=3 tally=2 -2026-04-22T10:11:02.086Z [INFO] server-69bde12a-d80b-e84c-c422-5a8d3f0063eb: entering leader state: leader="Node at 69bde12a-d80b-e84c-c422-5a8d3f0063eb [Leader]" -2026-04-22T10:11:02.086Z [INFO] server-69bde12a-d80b-e84c-c422-5a8d3f0063eb: added peer, starting replication: peer=server-c2714c75-176b-29b5-f842-4eb2faa7e453 -2026-04-22T10:11:02.086Z [INFO] server-69bde12a-d80b-e84c-c422-5a8d3f0063eb: added peer, starting replication: peer=server-be5436b9-396e-6576-ac1d-f3fac73e6ffa -2026-04-22T10:11:02.086Z [ERROR] server-69bde12a-d80b-e84c-c422-5a8d3f0063eb: peer has newer term, stopping replication: peer="{Nonvoter server-be5436b9-396e-6576-ac1d-f3fac73e6ffa be5436b9-396e-6576-ac1d-f3fac73e6ffa}" -2026-04-22T10:11:02.086Z [INFO] server-69bde12a-d80b-e84c-c422-5a8d3f0063eb: entering follower state: follower="Node at 69bde12a-d80b-e84c-c422-5a8d3f0063eb [Follower]" leader-address= leader-id= -2026-04-22T10:11:02.091Z [WARN] server-be5436b9-396e-6576-ac1d-f3fac73e6ffa: Election timeout reached, restarting election -2026-04-22T10:11:02.091Z [INFO] server-be5436b9-396e-6576-ac1d-f3fac73e6ffa: entering candidate state: node="Node at be5436b9-396e-6576-ac1d-f3fac73e6ffa [Candidate]" term=23 -2026-04-22T10:11:02.091Z [WARN] server-69bde12a-d80b-e84c-c422-5a8d3f0063eb: rejecting vote request since node is not a voter: from=be5436b9-396e-6576-ac1d-f3fac73e6ffa -2026-04-22T10:11:02.092Z [WARN] server-c2714c75-176b-29b5-f842-4eb2faa7e453: rejecting vote request since we have a leader: from=be5436b9-396e-6576-ac1d-f3fac73e6ffa leader=69bde12a-d80b-e84c-c422-5a8d3f0063eb leader-id=server-69bde12a-d80b-e84c-c422-5a8d3f0063eb -2026-04-22T10:11:02.128Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:11:02.176Z [WARN] server-69bde12a-d80b-e84c-c422-5a8d3f0063eb: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:11:02.176Z [WARN] server-be5436b9-396e-6576-ac1d-f3fac73e6ffa: Election timeout reached, restarting election -2026-04-22T10:11:02.176Z [INFO] server-69bde12a-d80b-e84c-c422-5a8d3f0063eb: entering candidate state: node="Node at 69bde12a-d80b-e84c-c422-5a8d3f0063eb [Candidate]" term=24 -2026-04-22T10:11:02.176Z [INFO] server-be5436b9-396e-6576-ac1d-f3fac73e6ffa: entering candidate state: node="Node at be5436b9-396e-6576-ac1d-f3fac73e6ffa [Candidate]" term=24 -2026-04-22T10:11:02.176Z [WARN] server-69bde12a-d80b-e84c-c422-5a8d3f0063eb: rejecting vote request since node is not a voter: from=be5436b9-396e-6576-ac1d-f3fac73e6ffa -2026-04-22T10:11:02.176Z [INFO] server-69bde12a-d80b-e84c-c422-5a8d3f0063eb: election won: term=24 tally=2 -2026-04-22T10:11:02.176Z [INFO] server-69bde12a-d80b-e84c-c422-5a8d3f0063eb: entering leader state: leader="Node at 69bde12a-d80b-e84c-c422-5a8d3f0063eb [Leader]" -2026-04-22T10:11:02.176Z [WARN] server-c2714c75-176b-29b5-f842-4eb2faa7e453: rejecting vote request since node is not a voter: from=be5436b9-396e-6576-ac1d-f3fac73e6ffa -2026-04-22T10:11:02.176Z [INFO] server-69bde12a-d80b-e84c-c422-5a8d3f0063eb: added peer, starting replication: peer=server-c2714c75-176b-29b5-f842-4eb2faa7e453 -2026-04-22T10:11:02.176Z [INFO] server-69bde12a-d80b-e84c-c422-5a8d3f0063eb: added peer, starting replication: peer=server-be5436b9-396e-6576-ac1d-f3fac73e6ffa -2026-04-22T10:11:02.176Z [WARN] server-be5436b9-396e-6576-ac1d-f3fac73e6ffa: failed to get previous log: previous-index=5 last-index=2 error="log not found" -2026-04-22T10:11:02.176Z [INFO] server-be5436b9-396e-6576-ac1d-f3fac73e6ffa: entering follower state: follower="Node at be5436b9-396e-6576-ac1d-f3fac73e6ffa [Follower]" leader-address=69bde12a-d80b-e84c-c422-5a8d3f0063eb leader-id=server-69bde12a-d80b-e84c-c422-5a8d3f0063eb -2026-04-22T10:11:02.176Z [INFO] server-69bde12a-d80b-e84c-c422-5a8d3f0063eb: pipelining replication: peer="{Voter server-c2714c75-176b-29b5-f842-4eb2faa7e453 c2714c75-176b-29b5-f842-4eb2faa7e453}" -2026-04-22T10:11:02.176Z [WARN] server-69bde12a-d80b-e84c-c422-5a8d3f0063eb: appendEntries rejected, sending older logs: peer="{Nonvoter server-be5436b9-396e-6576-ac1d-f3fac73e6ffa be5436b9-396e-6576-ac1d-f3fac73e6ffa}" next=3 -2026-04-22T10:11:02.176Z [INFO] server-69bde12a-d80b-e84c-c422-5a8d3f0063eb: pipelining replication: peer="{Nonvoter server-be5436b9-396e-6576-ac1d-f3fac73e6ffa be5436b9-396e-6576-ac1d-f3fac73e6ffa}" -2026-04-22T10:11:02.282Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:11:02.176492275 +0000 UTC m=+108.200288364 (1 nodes), 48.264256ms from start of poll, 550.659077ms from cluster start. Timeout at 2026-04-22 10:11:02.281493493 +0000 UTC m=+108.305289576, 105.001212ms after stability -2026-04-22T10:11:02.309Z [INFO] server-69bde12a-d80b-e84c-c422-5a8d3f0063eb: updating configuration: command=RemoveServer server-id=server-be5436b9-396e-6576-ac1d-f3fac73e6ffa server-addr= servers="[{Suffrage:Voter ID:server-c2714c75-176b-29b5-f842-4eb2faa7e453 Address:c2714c75-176b-29b5-f842-4eb2faa7e453} {Suffrage:Voter ID:server-69bde12a-d80b-e84c-c422-5a8d3f0063eb Address:69bde12a-d80b-e84c-c422-5a8d3f0063eb}]" -2026-04-22T10:11:02.310Z [INFO] server-69bde12a-d80b-e84c-c422-5a8d3f0063eb: removed peer, stopping replication: peer=server-be5436b9-396e-6576-ac1d-f3fac73e6ffa last-index=107 -2026-04-22T10:11:02.310Z [INFO] server-69bde12a-d80b-e84c-c422-5a8d3f0063eb: aborting pipeline replication: peer="{Nonvoter server-be5436b9-396e-6576-ac1d-f3fac73e6ffa be5436b9-396e-6576-ac1d-f3fac73e6ffa}" -2026-04-22T10:11:02.310Z [INFO] server-69bde12a-d80b-e84c-c422-5a8d3f0063eb: updating configuration: command=AddVoter server-id=server-be5436b9-396e-6576-ac1d-f3fac73e6ffa server-addr=be5436b9-396e-6576-ac1d-f3fac73e6ffa servers="[{Suffrage:Voter ID:server-c2714c75-176b-29b5-f842-4eb2faa7e453 Address:c2714c75-176b-29b5-f842-4eb2faa7e453} {Suffrage:Voter ID:server-69bde12a-d80b-e84c-c422-5a8d3f0063eb Address:69bde12a-d80b-e84c-c422-5a8d3f0063eb} {Suffrage:Voter ID:server-be5436b9-396e-6576-ac1d-f3fac73e6ffa Address:be5436b9-396e-6576-ac1d-f3fac73e6ffa}]" -2026-04-22T10:11:02.310Z [INFO] server-69bde12a-d80b-e84c-c422-5a8d3f0063eb: added peer, starting replication: peer=server-be5436b9-396e-6576-ac1d-f3fac73e6ffa -2026-04-22T10:11:02.310Z [WARN] server-be5436b9-396e-6576-ac1d-f3fac73e6ffa: failed to get previous log: previous-index=108 last-index=107 error="log not found" -2026-04-22T10:11:02.310Z [WARN] server-69bde12a-d80b-e84c-c422-5a8d3f0063eb: appendEntries rejected, sending older logs: peer="{Voter server-be5436b9-396e-6576-ac1d-f3fac73e6ffa be5436b9-396e-6576-ac1d-f3fac73e6ffa}" next=108 -2026-04-22T10:11:02.310Z [INFO] server-69bde12a-d80b-e84c-c422-5a8d3f0063eb: pipelining replication: peer="{Voter server-be5436b9-396e-6576-ac1d-f3fac73e6ffa be5436b9-396e-6576-ac1d-f3fac73e6ffa}" -2026-04-22T10:11:03.381Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:11:03.486Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:11:03.381400079 +0000 UTC m=+109.405196168 (1 nodes), 497ns from start of poll, 1.755566881s from cluster start. Timeout at 2026-04-22 10:11:03.486416745 +0000 UTC m=+109.510212965, 105.016797ms after stability -2026-04-22T10:11:03.486Z [INFO] server-69bde12a-d80b-e84c-c422-5a8d3f0063eb: aborting pipeline replication: peer="{Voter server-c2714c75-176b-29b5-f842-4eb2faa7e453 c2714c75-176b-29b5-f842-4eb2faa7e453}" -2026-04-22T10:11:03.486Z [INFO] server-69bde12a-d80b-e84c-c422-5a8d3f0063eb: aborting pipeline replication: peer="{Voter server-be5436b9-396e-6576-ac1d-f3fac73e6ffa be5436b9-396e-6576-ac1d-f3fac73e6ffa}" -2026-04-22T10:11:03.986Z [ERROR] server-69bde12a-d80b-e84c-c422-5a8d3f0063eb: failed to heartbeat to: peer=c2714c75-176b-29b5-f842-4eb2faa7e453 backoff time=10ms error="command timed out" ---- PASS: TestRaft_ClusterCanRegainStability_WhenNonVoterWithHigherTermJoin (2.36s) +2025-03-20T03:51:21.911Z [INFO] server-6fad7824-77d6-b861-d1f4-67a45b577098: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6fad7824-77d6-b861-d1f4-67a45b577098 Address:6fad7824-77d6-b861-d1f4-67a45b577098} {Suffrage:Voter ID:server-65b09b21-7d00-175d-58b9-325040ee6226 Address:65b09b21-7d00-175d-58b9-325040ee6226} {Suffrage:Voter ID:server-adfa0447-0202-9dfa-3ad8-496f14370671 Address:adfa0447-0202-9dfa-3ad8-496f14370671}]" +2025-03-20T03:51:21.912Z [INFO] server-65b09b21-7d00-175d-58b9-325040ee6226: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6fad7824-77d6-b861-d1f4-67a45b577098 Address:6fad7824-77d6-b861-d1f4-67a45b577098} {Suffrage:Voter ID:server-65b09b21-7d00-175d-58b9-325040ee6226 Address:65b09b21-7d00-175d-58b9-325040ee6226} {Suffrage:Voter ID:server-adfa0447-0202-9dfa-3ad8-496f14370671 Address:adfa0447-0202-9dfa-3ad8-496f14370671}]" +2025-03-20T03:51:21.912Z [INFO] server-adfa0447-0202-9dfa-3ad8-496f14370671: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6fad7824-77d6-b861-d1f4-67a45b577098 Address:6fad7824-77d6-b861-d1f4-67a45b577098} {Suffrage:Voter ID:server-65b09b21-7d00-175d-58b9-325040ee6226 Address:65b09b21-7d00-175d-58b9-325040ee6226} {Suffrage:Voter ID:server-adfa0447-0202-9dfa-3ad8-496f14370671 Address:adfa0447-0202-9dfa-3ad8-496f14370671}]" +2025-03-20T03:51:21.912Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:21.912Z [INFO] server-6fad7824-77d6-b861-d1f4-67a45b577098: entering follower state: follower="Node at 6fad7824-77d6-b861-d1f4-67a45b577098 [Follower]" leader-address= leader-id= +2025-03-20T03:51:21.912Z [INFO] server-65b09b21-7d00-175d-58b9-325040ee6226: entering follower state: follower="Node at 65b09b21-7d00-175d-58b9-325040ee6226 [Follower]" leader-address= leader-id= +2025-03-20T03:51:21.912Z [INFO] server-adfa0447-0202-9dfa-3ad8-496f14370671: entering follower state: follower="Node at adfa0447-0202-9dfa-3ad8-496f14370671 [Follower]" leader-address= leader-id= +2025-03-20T03:51:21.989Z [WARN] server-6fad7824-77d6-b861-d1f4-67a45b577098: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:21.989Z [INFO] server-6fad7824-77d6-b861-d1f4-67a45b577098: entering candidate state: node="Node at 6fad7824-77d6-b861-d1f4-67a45b577098 [Candidate]" term=2 +2025-03-20T03:51:21.990Z [INFO] server-6fad7824-77d6-b861-d1f4-67a45b577098: election won: term=2 tally=2 +2025-03-20T03:51:21.990Z [INFO] server-6fad7824-77d6-b861-d1f4-67a45b577098: entering leader state: leader="Node at 6fad7824-77d6-b861-d1f4-67a45b577098 [Leader]" +2025-03-20T03:51:21.990Z [INFO] server-6fad7824-77d6-b861-d1f4-67a45b577098: added peer, starting replication: peer=server-65b09b21-7d00-175d-58b9-325040ee6226 +2025-03-20T03:51:21.990Z [INFO] server-6fad7824-77d6-b861-d1f4-67a45b577098: added peer, starting replication: peer=server-adfa0447-0202-9dfa-3ad8-496f14370671 +2025-03-20T03:51:21.990Z [INFO] server-6fad7824-77d6-b861-d1f4-67a45b577098: pipelining replication: peer="{Voter server-adfa0447-0202-9dfa-3ad8-496f14370671 adfa0447-0202-9dfa-3ad8-496f14370671}" +2025-03-20T03:51:21.990Z [INFO] server-6fad7824-77d6-b861-d1f4-67a45b577098: pipelining replication: peer="{Voter server-65b09b21-7d00-175d-58b9-325040ee6226 65b09b21-7d00-175d-58b9-325040ee6226}" +2025-03-20T03:51:22.095Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:21.990520317 +0000 UTC m=+113.664973246 (1 nodes), 78.21333ms from start of poll, 79.124997ms from cluster start. Timeout at 2025-03-20 03:51:22.095545543 +0000 UTC m=+113.769998634, 105.025388ms after stability +2025-03-20T03:51:22.106Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:51:22.212Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:51:22.106744253 +0000 UTC m=+113.781197086 (2 nodes), 2.496µs from start of poll, 195.348837ms from cluster start. Timeout at 2025-03-20 03:51:22.211791326 +0000 UTC m=+113.886244234, 105.047148ms after stability +2025-03-20T03:51:22.212Z [ERROR] server-6fad7824-77d6-b861-d1f4-67a45b577098: failed to heartbeat to: peer=65b09b21-7d00-175d-58b9-325040ee6226 backoff time=10ms error="failed to connect to peer: 65b09b21-7d00-175d-58b9-325040ee6226" +2025-03-20T03:51:22.230Z [ERROR] server-6fad7824-77d6-b861-d1f4-67a45b577098: failed to pipeline appendEntries: peer="{Voter server-65b09b21-7d00-175d-58b9-325040ee6226 65b09b21-7d00-175d-58b9-325040ee6226}" error="append pipeline closed" +2025-03-20T03:51:22.230Z [INFO] server-6fad7824-77d6-b861-d1f4-67a45b577098: aborting pipeline replication: peer="{Voter server-65b09b21-7d00-175d-58b9-325040ee6226 65b09b21-7d00-175d-58b9-325040ee6226}" +2025-03-20T03:51:22.237Z [ERROR] server-6fad7824-77d6-b861-d1f4-67a45b577098: failed to heartbeat to: peer=65b09b21-7d00-175d-58b9-325040ee6226 backoff time=10ms error="failed to connect to peer: 65b09b21-7d00-175d-58b9-325040ee6226" +2025-03-20T03:51:22.245Z [ERROR] server-6fad7824-77d6-b861-d1f4-67a45b577098: failed to appendEntries to: peer="{Voter server-65b09b21-7d00-175d-58b9-325040ee6226 65b09b21-7d00-175d-58b9-325040ee6226}" error="failed to connect to peer: 65b09b21-7d00-175d-58b9-325040ee6226" +2025-03-20T03:51:22.259Z [WARN] server-6fad7824-77d6-b861-d1f4-67a45b577098: failed to contact: server-id=server-65b09b21-7d00-175d-58b9-325040ee6226 time=51.095234ms +2025-03-20T03:51:22.259Z [ERROR] server-6fad7824-77d6-b861-d1f4-67a45b577098: failed to heartbeat to: peer=65b09b21-7d00-175d-58b9-325040ee6226 backoff time=10ms error="failed to connect to peer: 65b09b21-7d00-175d-58b9-325040ee6226" +2025-03-20T03:51:22.264Z [ERROR] server-6fad7824-77d6-b861-d1f4-67a45b577098: failed to appendEntries to: peer="{Voter server-65b09b21-7d00-175d-58b9-325040ee6226 65b09b21-7d00-175d-58b9-325040ee6226}" error="failed to connect to peer: 65b09b21-7d00-175d-58b9-325040ee6226" +2025-03-20T03:51:22.275Z [WARN] server-65b09b21-7d00-175d-58b9-325040ee6226: heartbeat timeout reached, starting election: last-leader-addr=6fad7824-77d6-b861-d1f4-67a45b577098 last-leader-id=server-6fad7824-77d6-b861-d1f4-67a45b577098 +2025-03-20T03:51:22.275Z [INFO] server-65b09b21-7d00-175d-58b9-325040ee6226: entering candidate state: node="Node at 65b09b21-7d00-175d-58b9-325040ee6226 [Candidate]" term=3 +2025-03-20T03:51:22.275Z [ERROR] server-65b09b21-7d00-175d-58b9-325040ee6226: failed to make requestVote RPC: target="{Voter server-adfa0447-0202-9dfa-3ad8-496f14370671 adfa0447-0202-9dfa-3ad8-496f14370671}" error="failed to connect to peer: adfa0447-0202-9dfa-3ad8-496f14370671" term=3 +2025-03-20T03:51:22.275Z [ERROR] server-65b09b21-7d00-175d-58b9-325040ee6226: failed to make requestVote RPC: target="{Voter server-6fad7824-77d6-b861-d1f4-67a45b577098 6fad7824-77d6-b861-d1f4-67a45b577098}" error="failed to connect to peer: 6fad7824-77d6-b861-d1f4-67a45b577098" term=3 +2025-03-20T03:51:22.280Z [ERROR] server-6fad7824-77d6-b861-d1f4-67a45b577098: failed to heartbeat to: peer=65b09b21-7d00-175d-58b9-325040ee6226 backoff time=20ms error="failed to connect to peer: 65b09b21-7d00-175d-58b9-325040ee6226" +2025-03-20T03:51:22.285Z [ERROR] server-6fad7824-77d6-b861-d1f4-67a45b577098: failed to appendEntries to: peer="{Voter server-65b09b21-7d00-175d-58b9-325040ee6226 65b09b21-7d00-175d-58b9-325040ee6226}" error="failed to connect to peer: 65b09b21-7d00-175d-58b9-325040ee6226" +2025-03-20T03:51:22.306Z [WARN] server-6fad7824-77d6-b861-d1f4-67a45b577098: failed to contact: server-id=server-65b09b21-7d00-175d-58b9-325040ee6226 time=98.663389ms +2025-03-20T03:51:22.310Z [ERROR] server-6fad7824-77d6-b861-d1f4-67a45b577098: failed to heartbeat to: peer=65b09b21-7d00-175d-58b9-325040ee6226 backoff time=25ms error="failed to connect to peer: 65b09b21-7d00-175d-58b9-325040ee6226" +2025-03-20T03:51:22.311Z [ERROR] server-6fad7824-77d6-b861-d1f4-67a45b577098: failed to appendEntries to: peer="{Voter server-65b09b21-7d00-175d-58b9-325040ee6226 65b09b21-7d00-175d-58b9-325040ee6226}" error="failed to connect to peer: 65b09b21-7d00-175d-58b9-325040ee6226" +2025-03-20T03:51:22.323Z [INFO] server-6fad7824-77d6-b861-d1f4-67a45b577098: updating configuration: command=RemoveServer server-id=server-65b09b21-7d00-175d-58b9-325040ee6226 server-addr= servers="[{Suffrage:Voter ID:server-6fad7824-77d6-b861-d1f4-67a45b577098 Address:6fad7824-77d6-b861-d1f4-67a45b577098} {Suffrage:Voter ID:server-adfa0447-0202-9dfa-3ad8-496f14370671 Address:adfa0447-0202-9dfa-3ad8-496f14370671}]" +2025-03-20T03:51:22.323Z [INFO] server-6fad7824-77d6-b861-d1f4-67a45b577098: removed peer, stopping replication: peer=server-65b09b21-7d00-175d-58b9-325040ee6226 last-index=3 +2025-03-20T03:51:22.323Z [INFO] server-6fad7824-77d6-b861-d1f4-67a45b577098: updating configuration: command=AddNonvoter server-id=server-65b09b21-7d00-175d-58b9-325040ee6226 server-addr=65b09b21-7d00-175d-58b9-325040ee6226 servers="[{Suffrage:Voter ID:server-6fad7824-77d6-b861-d1f4-67a45b577098 Address:6fad7824-77d6-b861-d1f4-67a45b577098} {Suffrage:Voter ID:server-adfa0447-0202-9dfa-3ad8-496f14370671 Address:adfa0447-0202-9dfa-3ad8-496f14370671} {Suffrage:Nonvoter ID:server-65b09b21-7d00-175d-58b9-325040ee6226 Address:65b09b21-7d00-175d-58b9-325040ee6226}]" +2025-03-20T03:51:22.323Z [INFO] server-6fad7824-77d6-b861-d1f4-67a45b577098: added peer, starting replication: peer=server-65b09b21-7d00-175d-58b9-325040ee6226 +2025-03-20T03:51:22.323Z [ERROR] server-6fad7824-77d6-b861-d1f4-67a45b577098: failed to appendEntries to: peer="{Nonvoter server-65b09b21-7d00-175d-58b9-325040ee6226 65b09b21-7d00-175d-58b9-325040ee6226}" error="failed to connect to peer: 65b09b21-7d00-175d-58b9-325040ee6226" +2025-03-20T03:51:22.346Z [ERROR] server-6fad7824-77d6-b861-d1f4-67a45b577098: peer has newer term, stopping replication: peer="{Nonvoter server-65b09b21-7d00-175d-58b9-325040ee6226 65b09b21-7d00-175d-58b9-325040ee6226}" +2025-03-20T03:51:22.347Z [INFO] server-6fad7824-77d6-b861-d1f4-67a45b577098: entering follower state: follower="Node at 6fad7824-77d6-b861-d1f4-67a45b577098 [Follower]" leader-address= leader-id= +2025-03-20T03:51:22.347Z [INFO] server-6fad7824-77d6-b861-d1f4-67a45b577098: aborting pipeline replication: peer="{Voter server-adfa0447-0202-9dfa-3ad8-496f14370671 adfa0447-0202-9dfa-3ad8-496f14370671}" +2025-03-20T03:51:22.366Z [WARN] server-65b09b21-7d00-175d-58b9-325040ee6226: Election timeout reached, restarting election +2025-03-20T03:51:22.366Z [INFO] server-65b09b21-7d00-175d-58b9-325040ee6226: entering candidate state: node="Node at 65b09b21-7d00-175d-58b9-325040ee6226 [Candidate]" term=23 +2025-03-20T03:51:22.366Z [WARN] server-adfa0447-0202-9dfa-3ad8-496f14370671: rejecting vote request since we have a leader: from=65b09b21-7d00-175d-58b9-325040ee6226 leader=6fad7824-77d6-b861-d1f4-67a45b577098 leader-id=server-6fad7824-77d6-b861-d1f4-67a45b577098 +2025-03-20T03:51:22.367Z [ERROR] server-6fad7824-77d6-b861-d1f4-67a45b577098: peer has newer term, stopping replication: peer="{Voter server-65b09b21-7d00-175d-58b9-325040ee6226 65b09b21-7d00-175d-58b9-325040ee6226}" +2025-03-20T03:51:22.367Z [WARN] server-6fad7824-77d6-b861-d1f4-67a45b577098: rejecting vote request since node is not a voter: from=65b09b21-7d00-175d-58b9-325040ee6226 +2025-03-20T03:51:22.406Z [WARN] server-6fad7824-77d6-b861-d1f4-67a45b577098: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:22.406Z [INFO] server-6fad7824-77d6-b861-d1f4-67a45b577098: entering candidate state: node="Node at 6fad7824-77d6-b861-d1f4-67a45b577098 [Candidate]" term=24 +2025-03-20T03:51:22.407Z [INFO] server-6fad7824-77d6-b861-d1f4-67a45b577098: election won: term=24 tally=2 +2025-03-20T03:51:22.407Z [INFO] server-6fad7824-77d6-b861-d1f4-67a45b577098: entering leader state: leader="Node at 6fad7824-77d6-b861-d1f4-67a45b577098 [Leader]" +2025-03-20T03:51:22.407Z [INFO] server-6fad7824-77d6-b861-d1f4-67a45b577098: added peer, starting replication: peer=server-adfa0447-0202-9dfa-3ad8-496f14370671 +2025-03-20T03:51:22.407Z [INFO] server-6fad7824-77d6-b861-d1f4-67a45b577098: added peer, starting replication: peer=server-65b09b21-7d00-175d-58b9-325040ee6226 +2025-03-20T03:51:22.407Z [WARN] server-65b09b21-7d00-175d-58b9-325040ee6226: failed to get previous log: previous-index=4 last-index=2 error="log not found" +2025-03-20T03:51:22.407Z [INFO] server-65b09b21-7d00-175d-58b9-325040ee6226: entering follower state: follower="Node at 65b09b21-7d00-175d-58b9-325040ee6226 [Follower]" leader-address=6fad7824-77d6-b861-d1f4-67a45b577098 leader-id=server-6fad7824-77d6-b861-d1f4-67a45b577098 +2025-03-20T03:51:22.407Z [INFO] server-6fad7824-77d6-b861-d1f4-67a45b577098: pipelining replication: peer="{Voter server-adfa0447-0202-9dfa-3ad8-496f14370671 adfa0447-0202-9dfa-3ad8-496f14370671}" +2025-03-20T03:51:22.407Z [WARN] server-6fad7824-77d6-b861-d1f4-67a45b577098: appendEntries rejected, sending older logs: peer="{Nonvoter server-65b09b21-7d00-175d-58b9-325040ee6226 65b09b21-7d00-175d-58b9-325040ee6226}" next=3 +2025-03-20T03:51:22.408Z [INFO] server-6fad7824-77d6-b861-d1f4-67a45b577098: pipelining replication: peer="{Nonvoter server-65b09b21-7d00-175d-58b9-325040ee6226 65b09b21-7d00-175d-58b9-325040ee6226}" +2025-03-20T03:51:22.431Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:22.546Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:22.431265059 +0000 UTC m=+114.105717888 (1 nodes), 124.707µs from start of poll, 519.869639ms from cluster start. Timeout at 2025-03-20 03:51:22.536353079 +0000 UTC m=+114.210806226, 105.088338ms after stability +2025-03-20T03:51:22.650Z [INFO] server-6fad7824-77d6-b861-d1f4-67a45b577098: updating configuration: command=RemoveServer server-id=server-65b09b21-7d00-175d-58b9-325040ee6226 server-addr= servers="[{Suffrage:Voter ID:server-6fad7824-77d6-b861-d1f4-67a45b577098 Address:6fad7824-77d6-b861-d1f4-67a45b577098} {Suffrage:Voter ID:server-adfa0447-0202-9dfa-3ad8-496f14370671 Address:adfa0447-0202-9dfa-3ad8-496f14370671}]" +2025-03-20T03:51:22.650Z [INFO] server-6fad7824-77d6-b861-d1f4-67a45b577098: removed peer, stopping replication: peer=server-65b09b21-7d00-175d-58b9-325040ee6226 last-index=106 +2025-03-20T03:51:22.650Z [INFO] server-6fad7824-77d6-b861-d1f4-67a45b577098: aborting pipeline replication: peer="{Nonvoter server-65b09b21-7d00-175d-58b9-325040ee6226 65b09b21-7d00-175d-58b9-325040ee6226}" +2025-03-20T03:51:22.651Z [INFO] server-6fad7824-77d6-b861-d1f4-67a45b577098: updating configuration: command=AddVoter server-id=server-65b09b21-7d00-175d-58b9-325040ee6226 server-addr=65b09b21-7d00-175d-58b9-325040ee6226 servers="[{Suffrage:Voter ID:server-6fad7824-77d6-b861-d1f4-67a45b577098 Address:6fad7824-77d6-b861-d1f4-67a45b577098} {Suffrage:Voter ID:server-adfa0447-0202-9dfa-3ad8-496f14370671 Address:adfa0447-0202-9dfa-3ad8-496f14370671} {Suffrage:Voter ID:server-65b09b21-7d00-175d-58b9-325040ee6226 Address:65b09b21-7d00-175d-58b9-325040ee6226}]" +2025-03-20T03:51:22.651Z [INFO] server-6fad7824-77d6-b861-d1f4-67a45b577098: added peer, starting replication: peer=server-65b09b21-7d00-175d-58b9-325040ee6226 +2025-03-20T03:51:22.651Z [WARN] server-65b09b21-7d00-175d-58b9-325040ee6226: failed to get previous log: previous-index=107 last-index=106 error="log not found" +2025-03-20T03:51:22.651Z [WARN] server-6fad7824-77d6-b861-d1f4-67a45b577098: appendEntries rejected, sending older logs: peer="{Voter server-65b09b21-7d00-175d-58b9-325040ee6226 65b09b21-7d00-175d-58b9-325040ee6226}" next=107 +2025-03-20T03:51:22.651Z [INFO] server-6fad7824-77d6-b861-d1f4-67a45b577098: pipelining replication: peer="{Voter server-65b09b21-7d00-175d-58b9-325040ee6226 65b09b21-7d00-175d-58b9-325040ee6226}" +2025-03-20T03:51:22.947Z [WARN] server-6fad7824-77d6-b861-d1f4-67a45b577098: failed to contact: server-id=server-65b09b21-7d00-175d-58b9-325040ee6226 time=54.115894ms +2025-03-20T03:51:23.223Z [WARN] server-65b09b21-7d00-175d-58b9-325040ee6226: heartbeat timeout reached, starting election: last-leader-addr=6fad7824-77d6-b861-d1f4-67a45b577098 last-leader-id=server-6fad7824-77d6-b861-d1f4-67a45b577098 +2025-03-20T03:51:23.224Z [INFO] server-65b09b21-7d00-175d-58b9-325040ee6226: entering candidate state: node="Node at 65b09b21-7d00-175d-58b9-325040ee6226 [Candidate]" term=25 +2025-03-20T03:51:23.224Z [WARN] server-6fad7824-77d6-b861-d1f4-67a45b577098: failed to contact: server-id=server-65b09b21-7d00-175d-58b9-325040ee6226 time=53.839571ms +2025-03-20T03:51:23.224Z [WARN] server-6fad7824-77d6-b861-d1f4-67a45b577098: rejecting vote request since we have a leader: from=65b09b21-7d00-175d-58b9-325040ee6226 leader=6fad7824-77d6-b861-d1f4-67a45b577098 leader-id=server-6fad7824-77d6-b861-d1f4-67a45b577098 +2025-03-20T03:51:23.224Z [WARN] server-adfa0447-0202-9dfa-3ad8-496f14370671: rejecting vote request since we have a leader: from=65b09b21-7d00-175d-58b9-325040ee6226 leader=6fad7824-77d6-b861-d1f4-67a45b577098 leader-id=server-6fad7824-77d6-b861-d1f4-67a45b577098 +2025-03-20T03:51:23.224Z [ERROR] server-6fad7824-77d6-b861-d1f4-67a45b577098: peer has newer term, stopping replication: peer="{Voter server-65b09b21-7d00-175d-58b9-325040ee6226 65b09b21-7d00-175d-58b9-325040ee6226}" +2025-03-20T03:51:23.224Z [INFO] server-6fad7824-77d6-b861-d1f4-67a45b577098: aborting pipeline replication: peer="{Voter server-65b09b21-7d00-175d-58b9-325040ee6226 65b09b21-7d00-175d-58b9-325040ee6226}" +2025-03-20T03:51:23.224Z [INFO] server-6fad7824-77d6-b861-d1f4-67a45b577098: entering follower state: follower="Node at 6fad7824-77d6-b861-d1f4-67a45b577098 [Follower]" leader-address= leader-id= +2025-03-20T03:51:23.224Z [INFO] server-6fad7824-77d6-b861-d1f4-67a45b577098: aborting pipeline replication: peer="{Voter server-adfa0447-0202-9dfa-3ad8-496f14370671 adfa0447-0202-9dfa-3ad8-496f14370671}" +2025-03-20T03:51:23.294Z [WARN] server-adfa0447-0202-9dfa-3ad8-496f14370671: heartbeat timeout reached, starting election: last-leader-addr=6fad7824-77d6-b861-d1f4-67a45b577098 last-leader-id=server-6fad7824-77d6-b861-d1f4-67a45b577098 +2025-03-20T03:51:23.294Z [INFO] server-adfa0447-0202-9dfa-3ad8-496f14370671: entering candidate state: node="Node at adfa0447-0202-9dfa-3ad8-496f14370671 [Candidate]" term=25 +2025-03-20T03:51:23.294Z [INFO] server-65b09b21-7d00-175d-58b9-325040ee6226: duplicate requestVote for same term: term=25 +2025-03-20T03:51:23.294Z [INFO] server-adfa0447-0202-9dfa-3ad8-496f14370671: election won: term=25 tally=2 +2025-03-20T03:51:23.294Z [INFO] server-adfa0447-0202-9dfa-3ad8-496f14370671: entering leader state: leader="Node at adfa0447-0202-9dfa-3ad8-496f14370671 [Leader]" +2025-03-20T03:51:23.294Z [INFO] server-adfa0447-0202-9dfa-3ad8-496f14370671: added peer, starting replication: peer=server-6fad7824-77d6-b861-d1f4-67a45b577098 +2025-03-20T03:51:23.294Z [INFO] server-adfa0447-0202-9dfa-3ad8-496f14370671: added peer, starting replication: peer=server-65b09b21-7d00-175d-58b9-325040ee6226 +2025-03-20T03:51:23.294Z [INFO] server-65b09b21-7d00-175d-58b9-325040ee6226: entering follower state: follower="Node at 65b09b21-7d00-175d-58b9-325040ee6226 [Follower]" leader-address=adfa0447-0202-9dfa-3ad8-496f14370671 leader-id=server-adfa0447-0202-9dfa-3ad8-496f14370671 +2025-03-20T03:51:23.294Z [INFO] server-adfa0447-0202-9dfa-3ad8-496f14370671: pipelining replication: peer="{Voter server-65b09b21-7d00-175d-58b9-325040ee6226 65b09b21-7d00-175d-58b9-325040ee6226}" +2025-03-20T03:51:23.295Z [INFO] server-adfa0447-0202-9dfa-3ad8-496f14370671: pipelining replication: peer="{Voter server-6fad7824-77d6-b861-d1f4-67a45b577098 6fad7824-77d6-b861-d1f4-67a45b577098}" + raft_test.go:2916: [ERR] apply err: node is not the leader +2025-03-20T03:51:23.703Z [INFO] server-adfa0447-0202-9dfa-3ad8-496f14370671: aborting pipeline replication: peer="{Voter server-65b09b21-7d00-175d-58b9-325040ee6226 65b09b21-7d00-175d-58b9-325040ee6226}" +2025-03-20T03:51:23.703Z [INFO] server-adfa0447-0202-9dfa-3ad8-496f14370671: aborting pipeline replication: peer="{Voter server-6fad7824-77d6-b861-d1f4-67a45b577098 6fad7824-77d6-b861-d1f4-67a45b577098}" +--- FAIL: TestRaft_ClusterCanRegainStability_WhenNonVoterWithHigherTermJoin (1.79s) === RUN TestRaft_FollowerRemovalNoElection -2026-04-22T10:11:03.987Z [INFO] server-6b068f9e-aa34-7ff0-eb9b-069a24427909: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6b068f9e-aa34-7ff0-eb9b-069a24427909 Address:6b068f9e-aa34-7ff0-eb9b-069a24427909} {Suffrage:Voter ID:server-f6470077-8ef4-28b5-feda-91d5c5ba115a Address:f6470077-8ef4-28b5-feda-91d5c5ba115a} {Suffrage:Voter ID:server-73d75d2c-f4ba-de09-f33b-c87003cf7fa2 Address:73d75d2c-f4ba-de09-f33b-c87003cf7fa2}]" -2026-04-22T10:11:03.987Z [INFO] server-6b068f9e-aa34-7ff0-eb9b-069a24427909: entering follower state: follower="Node at 6b068f9e-aa34-7ff0-eb9b-069a24427909 [Follower]" leader-address= leader-id= -2026-04-22T10:11:03.987Z [INFO] server-f6470077-8ef4-28b5-feda-91d5c5ba115a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6b068f9e-aa34-7ff0-eb9b-069a24427909 Address:6b068f9e-aa34-7ff0-eb9b-069a24427909} {Suffrage:Voter ID:server-f6470077-8ef4-28b5-feda-91d5c5ba115a Address:f6470077-8ef4-28b5-feda-91d5c5ba115a} {Suffrage:Voter ID:server-73d75d2c-f4ba-de09-f33b-c87003cf7fa2 Address:73d75d2c-f4ba-de09-f33b-c87003cf7fa2}]" -2026-04-22T10:11:03.987Z [INFO] server-f6470077-8ef4-28b5-feda-91d5c5ba115a: entering follower state: follower="Node at f6470077-8ef4-28b5-feda-91d5c5ba115a [Follower]" leader-address= leader-id= -2026-04-22T10:11:03.987Z [INFO] server-73d75d2c-f4ba-de09-f33b-c87003cf7fa2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6b068f9e-aa34-7ff0-eb9b-069a24427909 Address:6b068f9e-aa34-7ff0-eb9b-069a24427909} {Suffrage:Voter ID:server-f6470077-8ef4-28b5-feda-91d5c5ba115a Address:f6470077-8ef4-28b5-feda-91d5c5ba115a} {Suffrage:Voter ID:server-73d75d2c-f4ba-de09-f33b-c87003cf7fa2 Address:73d75d2c-f4ba-de09-f33b-c87003cf7fa2}]" -2026-04-22T10:11:03.987Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:11:03.987Z [INFO] server-73d75d2c-f4ba-de09-f33b-c87003cf7fa2: entering follower state: follower="Node at 73d75d2c-f4ba-de09-f33b-c87003cf7fa2 [Follower]" leader-address= leader-id= -2026-04-22T10:11:04.097Z [WARN] server-6b068f9e-aa34-7ff0-eb9b-069a24427909: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:11:04.097Z [INFO] server-6b068f9e-aa34-7ff0-eb9b-069a24427909: entering candidate state: node="Node at 6b068f9e-aa34-7ff0-eb9b-069a24427909 [Candidate]" term=2 -2026-04-22T10:11:04.097Z [INFO] server-6b068f9e-aa34-7ff0-eb9b-069a24427909: election won: term=2 tally=2 -2026-04-22T10:11:04.097Z [INFO] server-6b068f9e-aa34-7ff0-eb9b-069a24427909: entering leader state: leader="Node at 6b068f9e-aa34-7ff0-eb9b-069a24427909 [Leader]" -2026-04-22T10:11:04.097Z [INFO] server-6b068f9e-aa34-7ff0-eb9b-069a24427909: added peer, starting replication: peer=server-f6470077-8ef4-28b5-feda-91d5c5ba115a -2026-04-22T10:11:04.097Z [INFO] server-6b068f9e-aa34-7ff0-eb9b-069a24427909: added peer, starting replication: peer=server-73d75d2c-f4ba-de09-f33b-c87003cf7fa2 -2026-04-22T10:11:04.097Z [INFO] server-6b068f9e-aa34-7ff0-eb9b-069a24427909: pipelining replication: peer="{Voter server-f6470077-8ef4-28b5-feda-91d5c5ba115a f6470077-8ef4-28b5-feda-91d5c5ba115a}" -2026-04-22T10:11:04.097Z [INFO] server-6b068f9e-aa34-7ff0-eb9b-069a24427909: pipelining replication: peer="{Voter server-73d75d2c-f4ba-de09-f33b-c87003cf7fa2 73d75d2c-f4ba-de09-f33b-c87003cf7fa2}" -2026-04-22T10:11:04.302Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:11:04.097218122 +0000 UTC m=+110.121014216 (1 nodes), 109.290462ms from start of poll, 109.602501ms from cluster start. Timeout at 2026-04-22 10:11:04.302227979 +0000 UTC m=+110.326024060, 205.009844ms after stability -2026-04-22T10:11:04.302Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:11:04.508Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:11:04.302320274 +0000 UTC m=+110.326116378 (1 nodes), 733ns from start of poll, 314.704663ms from cluster start. Timeout at 2026-04-22 10:11:04.507345697 +0000 UTC m=+110.531141957, 205.025579ms after stability -2026-04-22T10:11:04.513Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:11:04.719Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:11:04.513687831 +0000 UTC m=+110.537483931 (2 nodes), 1.065µs from start of poll, 526.072216ms from cluster start. Timeout at 2026-04-22 10:11:04.718717808 +0000 UTC m=+110.742514005, 205.030074ms after stability -2026-04-22T10:11:04.719Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:11:04.925Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:11:04.719466682 +0000 UTC m=+110.743262859 (1 nodes), 1.507µs from start of poll, 731.851144ms from cluster start. Timeout at 2026-04-22 10:11:04.924487493 +0000 UTC m=+110.948283611, 205.020752ms after stability - raft_test.go:2961: [INFO] restarting Node at f6470077-8ef4-28b5-feda-91d5c5ba115a [Follower] -2026-04-22T10:11:04.925Z [INFO] server-f6470077-8ef4-28b5-feda-91d5c5ba115a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6b068f9e-aa34-7ff0-eb9b-069a24427909 Address:6b068f9e-aa34-7ff0-eb9b-069a24427909} {Suffrage:Voter ID:server-f6470077-8ef4-28b5-feda-91d5c5ba115a Address:f6470077-8ef4-28b5-feda-91d5c5ba115a} {Suffrage:Voter ID:server-73d75d2c-f4ba-de09-f33b-c87003cf7fa2 Address:73d75d2c-f4ba-de09-f33b-c87003cf7fa2}]" -2026-04-22T10:11:04.925Z [INFO] server-f6470077-8ef4-28b5-feda-91d5c5ba115a: entering follower state: follower="Node at f6470077-8ef4-28b5-feda-91d5c5ba115a [Follower]" leader-address= leader-id= -2026-04-22T10:11:05.175Z [INFO] server-6b068f9e-aa34-7ff0-eb9b-069a24427909: aborting pipeline replication: peer="{Voter server-73d75d2c-f4ba-de09-f33b-c87003cf7fa2 73d75d2c-f4ba-de09-f33b-c87003cf7fa2}" -2026-04-22T10:11:05.558Z [ERROR] server-6b068f9e-aa34-7ff0-eb9b-069a24427909: failed to pipeline appendEntries: peer="{Voter server-f6470077-8ef4-28b5-feda-91d5c5ba115a f6470077-8ef4-28b5-feda-91d5c5ba115a}" error="command enqueue timeout" -2026-04-22T10:11:05.558Z [INFO] server-6b068f9e-aa34-7ff0-eb9b-069a24427909: aborting pipeline replication: peer="{Voter server-f6470077-8ef4-28b5-feda-91d5c5ba115a f6470077-8ef4-28b5-feda-91d5c5ba115a}" -2026-04-22T10:11:05.683Z [ERROR] server-6b068f9e-aa34-7ff0-eb9b-069a24427909: failed to heartbeat to: peer=f6470077-8ef4-28b5-feda-91d5c5ba115a backoff time=10ms error="command timed out" ---- PASS: TestRaft_FollowerRemovalNoElection (1.70s) +2025-03-20T03:51:23.705Z [INFO] server-1e9409f7-6e3b-7e2b-8dad-62a99bfd2a81: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1e9409f7-6e3b-7e2b-8dad-62a99bfd2a81 Address:1e9409f7-6e3b-7e2b-8dad-62a99bfd2a81} {Suffrage:Voter ID:server-dca17b49-9226-28bc-a150-90310b14da85 Address:dca17b49-9226-28bc-a150-90310b14da85} {Suffrage:Voter ID:server-c80cd0cf-83fa-3eee-d4e2-0495d255389a Address:c80cd0cf-83fa-3eee-d4e2-0495d255389a}]" +2025-03-20T03:51:23.705Z [INFO] server-dca17b49-9226-28bc-a150-90310b14da85: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1e9409f7-6e3b-7e2b-8dad-62a99bfd2a81 Address:1e9409f7-6e3b-7e2b-8dad-62a99bfd2a81} {Suffrage:Voter ID:server-dca17b49-9226-28bc-a150-90310b14da85 Address:dca17b49-9226-28bc-a150-90310b14da85} {Suffrage:Voter ID:server-c80cd0cf-83fa-3eee-d4e2-0495d255389a Address:c80cd0cf-83fa-3eee-d4e2-0495d255389a}]" +2025-03-20T03:51:23.705Z [INFO] server-c80cd0cf-83fa-3eee-d4e2-0495d255389a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1e9409f7-6e3b-7e2b-8dad-62a99bfd2a81 Address:1e9409f7-6e3b-7e2b-8dad-62a99bfd2a81} {Suffrage:Voter ID:server-dca17b49-9226-28bc-a150-90310b14da85 Address:dca17b49-9226-28bc-a150-90310b14da85} {Suffrage:Voter ID:server-c80cd0cf-83fa-3eee-d4e2-0495d255389a Address:c80cd0cf-83fa-3eee-d4e2-0495d255389a}]" +2025-03-20T03:51:23.705Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:23.705Z [INFO] server-1e9409f7-6e3b-7e2b-8dad-62a99bfd2a81: entering follower state: follower="Node at 1e9409f7-6e3b-7e2b-8dad-62a99bfd2a81 [Follower]" leader-address= leader-id= +2025-03-20T03:51:23.705Z [INFO] server-dca17b49-9226-28bc-a150-90310b14da85: entering follower state: follower="Node at dca17b49-9226-28bc-a150-90310b14da85 [Follower]" leader-address= leader-id= +2025-03-20T03:51:23.706Z [INFO] server-c80cd0cf-83fa-3eee-d4e2-0495d255389a: entering follower state: follower="Node at c80cd0cf-83fa-3eee-d4e2-0495d255389a [Follower]" leader-address= leader-id= +2025-03-20T03:51:23.867Z [WARN] server-1e9409f7-6e3b-7e2b-8dad-62a99bfd2a81: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:23.867Z [INFO] server-1e9409f7-6e3b-7e2b-8dad-62a99bfd2a81: entering candidate state: node="Node at 1e9409f7-6e3b-7e2b-8dad-62a99bfd2a81 [Candidate]" term=2 +2025-03-20T03:51:23.867Z [WARN] server-c80cd0cf-83fa-3eee-d4e2-0495d255389a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:23.867Z [INFO] server-c80cd0cf-83fa-3eee-d4e2-0495d255389a: entering candidate state: node="Node at c80cd0cf-83fa-3eee-d4e2-0495d255389a [Candidate]" term=2 +2025-03-20T03:51:23.867Z [INFO] server-c80cd0cf-83fa-3eee-d4e2-0495d255389a: duplicate requestVote for same term: term=2 +2025-03-20T03:51:23.867Z [INFO] server-1e9409f7-6e3b-7e2b-8dad-62a99bfd2a81: election won: term=2 tally=2 +2025-03-20T03:51:23.867Z [INFO] server-1e9409f7-6e3b-7e2b-8dad-62a99bfd2a81: entering leader state: leader="Node at 1e9409f7-6e3b-7e2b-8dad-62a99bfd2a81 [Leader]" +2025-03-20T03:51:23.867Z [INFO] server-1e9409f7-6e3b-7e2b-8dad-62a99bfd2a81: added peer, starting replication: peer=server-dca17b49-9226-28bc-a150-90310b14da85 +2025-03-20T03:51:23.867Z [INFO] server-1e9409f7-6e3b-7e2b-8dad-62a99bfd2a81: added peer, starting replication: peer=server-c80cd0cf-83fa-3eee-d4e2-0495d255389a +2025-03-20T03:51:23.867Z [INFO] server-c80cd0cf-83fa-3eee-d4e2-0495d255389a: entering follower state: follower="Node at c80cd0cf-83fa-3eee-d4e2-0495d255389a [Follower]" leader-address=1e9409f7-6e3b-7e2b-8dad-62a99bfd2a81 leader-id=server-1e9409f7-6e3b-7e2b-8dad-62a99bfd2a81 +2025-03-20T03:51:23.867Z [INFO] server-1e9409f7-6e3b-7e2b-8dad-62a99bfd2a81: pipelining replication: peer="{Voter server-c80cd0cf-83fa-3eee-d4e2-0495d255389a c80cd0cf-83fa-3eee-d4e2-0495d255389a}" +2025-03-20T03:51:23.867Z [INFO] server-dca17b49-9226-28bc-a150-90310b14da85: duplicate requestVote for same term: term=2 +2025-03-20T03:51:23.867Z [INFO] server-1e9409f7-6e3b-7e2b-8dad-62a99bfd2a81: pipelining replication: peer="{Voter server-dca17b49-9226-28bc-a150-90310b14da85 dca17b49-9226-28bc-a150-90310b14da85}" +2025-03-20T03:51:23.868Z [WARN] server-1e9409f7-6e3b-7e2b-8dad-62a99bfd2a81: rejecting vote request since we have a leader: from=c80cd0cf-83fa-3eee-d4e2-0495d255389a leader=1e9409f7-6e3b-7e2b-8dad-62a99bfd2a81 leader-id=server-1e9409f7-6e3b-7e2b-8dad-62a99bfd2a81 +2025-03-20T03:51:24.073Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:23.868119466 +0000 UTC m=+115.542572323 (1 nodes), 162.28412ms from start of poll, 162.689869ms from cluster start. Timeout at 2025-03-20 03:51:24.0731222 +0000 UTC m=+115.747575151, 205.002828ms after stability +2025-03-20T03:51:24.073Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:24.278Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:24.073293103 +0000 UTC m=+115.747745989 (1 nodes), 2.432µs from start of poll, 367.863535ms from cluster start. Timeout at 2025-03-20 03:51:24.278340531 +0000 UTC m=+115.952793854, 205.047865ms after stability +2025-03-20T03:51:24.284Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:51:24.490Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:51:24.284902474 +0000 UTC m=+115.959355369 (2 nodes), 2.894µs from start of poll, 579.472915ms from cluster start. Timeout at 2025-03-20 03:51:24.490018379 +0000 UTC m=+116.164471280, 205.115911ms after stability +2025-03-20T03:51:24.490Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:24.698Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:24.490593507 +0000 UTC m=+116.165046554 (1 nodes), 2.395µs from start of poll, 785.1641ms from cluster start. Timeout at 2025-03-20 03:51:24.695680031 +0000 UTC m=+116.370133159, 205.086605ms after stability + raft_test.go:2961: [INFO] restarting Node at dca17b49-9226-28bc-a150-90310b14da85 [Follower] +2025-03-20T03:51:24.699Z [INFO] server-dca17b49-9226-28bc-a150-90310b14da85: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1e9409f7-6e3b-7e2b-8dad-62a99bfd2a81 Address:1e9409f7-6e3b-7e2b-8dad-62a99bfd2a81} {Suffrage:Voter ID:server-dca17b49-9226-28bc-a150-90310b14da85 Address:dca17b49-9226-28bc-a150-90310b14da85} {Suffrage:Voter ID:server-c80cd0cf-83fa-3eee-d4e2-0495d255389a Address:c80cd0cf-83fa-3eee-d4e2-0495d255389a}]" +2025-03-20T03:51:24.699Z [INFO] server-dca17b49-9226-28bc-a150-90310b14da85: entering follower state: follower="Node at dca17b49-9226-28bc-a150-90310b14da85 [Follower]" leader-address= leader-id= +2025-03-20T03:51:24.950Z [INFO] server-1e9409f7-6e3b-7e2b-8dad-62a99bfd2a81: aborting pipeline replication: peer="{Voter server-c80cd0cf-83fa-3eee-d4e2-0495d255389a c80cd0cf-83fa-3eee-d4e2-0495d255389a}" +2025-03-20T03:51:25.386Z [ERROR] server-1e9409f7-6e3b-7e2b-8dad-62a99bfd2a81: failed to pipeline appendEntries: peer="{Voter server-dca17b49-9226-28bc-a150-90310b14da85 dca17b49-9226-28bc-a150-90310b14da85}" error="command enqueue timeout" +2025-03-20T03:51:25.386Z [INFO] server-1e9409f7-6e3b-7e2b-8dad-62a99bfd2a81: aborting pipeline replication: peer="{Voter server-dca17b49-9226-28bc-a150-90310b14da85 dca17b49-9226-28bc-a150-90310b14da85}" +2025-03-20T03:51:25.462Z [ERROR] server-1e9409f7-6e3b-7e2b-8dad-62a99bfd2a81: failed to heartbeat to: peer=dca17b49-9226-28bc-a150-90310b14da85 backoff time=10ms error="command timed out" +--- PASS: TestRaft_FollowerRemovalNoElection (1.76s) === RUN TestRaft_VoteWithNoIDNoAddr -2026-04-22T10:11:05.684Z [INFO] server-25c3a33e-ad03-e556-5dae-b650ee62bac2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-25c3a33e-ad03-e556-5dae-b650ee62bac2 Address:25c3a33e-ad03-e556-5dae-b650ee62bac2} {Suffrage:Voter ID:server-39658725-95ed-aef1-aca9-1f05548374f2 Address:39658725-95ed-aef1-aca9-1f05548374f2} {Suffrage:Voter ID:server-92afba55-9920-6e14-3906-e747a5f2105c Address:92afba55-9920-6e14-3906-e747a5f2105c}]" -2026-04-22T10:11:05.684Z [INFO] server-25c3a33e-ad03-e556-5dae-b650ee62bac2: entering follower state: follower="Node at 25c3a33e-ad03-e556-5dae-b650ee62bac2 [Follower]" leader-address= leader-id= -2026-04-22T10:11:05.684Z [INFO] server-39658725-95ed-aef1-aca9-1f05548374f2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-25c3a33e-ad03-e556-5dae-b650ee62bac2 Address:25c3a33e-ad03-e556-5dae-b650ee62bac2} {Suffrage:Voter ID:server-39658725-95ed-aef1-aca9-1f05548374f2 Address:39658725-95ed-aef1-aca9-1f05548374f2} {Suffrage:Voter ID:server-92afba55-9920-6e14-3906-e747a5f2105c Address:92afba55-9920-6e14-3906-e747a5f2105c}]" -2026-04-22T10:11:05.684Z [INFO] server-39658725-95ed-aef1-aca9-1f05548374f2: entering follower state: follower="Node at 39658725-95ed-aef1-aca9-1f05548374f2 [Follower]" leader-address= leader-id= -2026-04-22T10:11:05.684Z [INFO] server-92afba55-9920-6e14-3906-e747a5f2105c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-25c3a33e-ad03-e556-5dae-b650ee62bac2 Address:25c3a33e-ad03-e556-5dae-b650ee62bac2} {Suffrage:Voter ID:server-39658725-95ed-aef1-aca9-1f05548374f2 Address:39658725-95ed-aef1-aca9-1f05548374f2} {Suffrage:Voter ID:server-92afba55-9920-6e14-3906-e747a5f2105c Address:92afba55-9920-6e14-3906-e747a5f2105c}]" -2026-04-22T10:11:05.684Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:11:05.684Z [INFO] server-92afba55-9920-6e14-3906-e747a5f2105c: entering follower state: follower="Node at 92afba55-9920-6e14-3906-e747a5f2105c [Follower]" leader-address= leader-id= -2026-04-22T10:11:05.742Z [WARN] server-39658725-95ed-aef1-aca9-1f05548374f2: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-04-22T10:11:05.742Z [INFO] server-39658725-95ed-aef1-aca9-1f05548374f2: entering candidate state: node="Node at 39658725-95ed-aef1-aca9-1f05548374f2 [Candidate]" term=2 -2026-04-22T10:11:05.742Z [INFO] server-39658725-95ed-aef1-aca9-1f05548374f2: election won: term=2 tally=2 -2026-04-22T10:11:05.742Z [INFO] server-39658725-95ed-aef1-aca9-1f05548374f2: entering leader state: leader="Node at 39658725-95ed-aef1-aca9-1f05548374f2 [Leader]" -2026-04-22T10:11:05.742Z [INFO] server-39658725-95ed-aef1-aca9-1f05548374f2: added peer, starting replication: peer=server-25c3a33e-ad03-e556-5dae-b650ee62bac2 -2026-04-22T10:11:05.742Z [INFO] server-39658725-95ed-aef1-aca9-1f05548374f2: added peer, starting replication: peer=server-92afba55-9920-6e14-3906-e747a5f2105c -2026-04-22T10:11:05.742Z [INFO] server-39658725-95ed-aef1-aca9-1f05548374f2: pipelining replication: peer="{Voter server-25c3a33e-ad03-e556-5dae-b650ee62bac2 25c3a33e-ad03-e556-5dae-b650ee62bac2}" -2026-04-22T10:11:05.742Z [INFO] server-39658725-95ed-aef1-aca9-1f05548374f2: pipelining replication: peer="{Voter server-92afba55-9920-6e14-3906-e747a5f2105c 92afba55-9920-6e14-3906-e747a5f2105c}" -2026-04-22T10:11:05.847Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:11:05.742251262 +0000 UTC m=+111.766047355 (1 nodes), 57.419163ms from start of poll, 57.724377ms from cluster start. Timeout at 2026-04-22 10:11:05.8472607 +0000 UTC m=+111.871056837, 105.009482ms after stability -2026-04-22T10:11:05.847Z [INFO] cluster: starting stability test: raft-state=Leader -2026-04-22T10:11:05.953Z [INFO] cluster: stable state for Leader reached at 2026-04-22 10:11:05.847889806 +0000 UTC m=+111.871686003 (1 nodes), 1.704µs from start of poll, 163.363025ms from cluster start. Timeout at 2026-04-22 10:11:05.952917761 +0000 UTC m=+111.976713971, 105.027968ms after stability -2026-04-22T10:11:05.958Z [INFO] cluster: starting stability test: raft-state=Follower -2026-04-22T10:11:06.064Z [INFO] cluster: stable state for Follower reached at 2026-04-22 10:11:05.958857117 +0000 UTC m=+111.982653337 (2 nodes), 1.991µs from start of poll, 274.330359ms from cluster start. Timeout at 2026-04-22 10:11:06.063888056 +0000 UTC m=+112.087684236, 105.030899ms after stability -2026-04-22T10:11:06.067Z [ERROR] server-39658725-95ed-aef1-aca9-1f05548374f2: failed to heartbeat to: peer=92afba55-9920-6e14-3906-e747a5f2105c backoff time=10ms error="failed to connect to peer: 92afba55-9920-6e14-3906-e747a5f2105c" -2026-04-22T10:11:06.068Z [ERROR] server-39658725-95ed-aef1-aca9-1f05548374f2: failed to heartbeat to: peer=25c3a33e-ad03-e556-5dae-b650ee62bac2 backoff time=10ms error="failed to connect to peer: 25c3a33e-ad03-e556-5dae-b650ee62bac2" -2026-04-22T10:11:06.068Z [ERROR] server-39658725-95ed-aef1-aca9-1f05548374f2: failed to pipeline appendEntries: peer="{Voter server-92afba55-9920-6e14-3906-e747a5f2105c 92afba55-9920-6e14-3906-e747a5f2105c}" error="append pipeline closed" -2026-04-22T10:11:06.069Z [INFO] server-39658725-95ed-aef1-aca9-1f05548374f2: aborting pipeline replication: peer="{Voter server-92afba55-9920-6e14-3906-e747a5f2105c 92afba55-9920-6e14-3906-e747a5f2105c}" -2026-04-22T10:11:06.069Z [ERROR] server-39658725-95ed-aef1-aca9-1f05548374f2: failed to pipeline appendEntries: peer="{Voter server-25c3a33e-ad03-e556-5dae-b650ee62bac2 25c3a33e-ad03-e556-5dae-b650ee62bac2}" error="append pipeline closed" -2026-04-22T10:11:06.069Z [INFO] server-39658725-95ed-aef1-aca9-1f05548374f2: aborting pipeline replication: peer="{Voter server-25c3a33e-ad03-e556-5dae-b650ee62bac2 25c3a33e-ad03-e556-5dae-b650ee62bac2}" -2026-04-22T10:11:06.075Z [ERROR] server-39658725-95ed-aef1-aca9-1f05548374f2: failed to appendEntries to: peer="{Voter server-25c3a33e-ad03-e556-5dae-b650ee62bac2 25c3a33e-ad03-e556-5dae-b650ee62bac2}" error="failed to connect to peer: 25c3a33e-ad03-e556-5dae-b650ee62bac2" -2026-04-22T10:11:06.077Z [ERROR] server-39658725-95ed-aef1-aca9-1f05548374f2: failed to appendEntries to: peer="{Voter server-92afba55-9920-6e14-3906-e747a5f2105c 92afba55-9920-6e14-3906-e747a5f2105c}" error="failed to connect to peer: 92afba55-9920-6e14-3906-e747a5f2105c" -2026-04-22T10:11:06.085Z [ERROR] server-39658725-95ed-aef1-aca9-1f05548374f2: failed to heartbeat to: peer=25c3a33e-ad03-e556-5dae-b650ee62bac2 backoff time=10ms error="failed to connect to peer: 25c3a33e-ad03-e556-5dae-b650ee62bac2" -2026-04-22T10:11:06.086Z [ERROR] server-39658725-95ed-aef1-aca9-1f05548374f2: failed to heartbeat to: peer=92afba55-9920-6e14-3906-e747a5f2105c backoff time=10ms error="failed to connect to peer: 92afba55-9920-6e14-3906-e747a5f2105c" -2026-04-22T10:11:06.091Z [ERROR] server-39658725-95ed-aef1-aca9-1f05548374f2: failed to appendEntries to: peer="{Voter server-25c3a33e-ad03-e556-5dae-b650ee62bac2 25c3a33e-ad03-e556-5dae-b650ee62bac2}" error="failed to connect to peer: 25c3a33e-ad03-e556-5dae-b650ee62bac2" -2026-04-22T10:11:06.094Z [ERROR] server-39658725-95ed-aef1-aca9-1f05548374f2: failed to appendEntries to: peer="{Voter server-92afba55-9920-6e14-3906-e747a5f2105c 92afba55-9920-6e14-3906-e747a5f2105c}" error="failed to connect to peer: 92afba55-9920-6e14-3906-e747a5f2105c" -2026-04-22T10:11:06.101Z [ERROR] server-39658725-95ed-aef1-aca9-1f05548374f2: failed to heartbeat to: peer=25c3a33e-ad03-e556-5dae-b650ee62bac2 backoff time=10ms error="failed to connect to peer: 25c3a33e-ad03-e556-5dae-b650ee62bac2" -2026-04-22T10:11:06.104Z [ERROR] server-39658725-95ed-aef1-aca9-1f05548374f2: failed to heartbeat to: peer=92afba55-9920-6e14-3906-e747a5f2105c backoff time=10ms error="failed to connect to peer: 92afba55-9920-6e14-3906-e747a5f2105c" -2026-04-22T10:11:06.111Z [ERROR] server-39658725-95ed-aef1-aca9-1f05548374f2: failed to appendEntries to: peer="{Voter server-92afba55-9920-6e14-3906-e747a5f2105c 92afba55-9920-6e14-3906-e747a5f2105c}" error="failed to connect to peer: 92afba55-9920-6e14-3906-e747a5f2105c" -2026-04-22T10:11:06.112Z [ERROR] server-39658725-95ed-aef1-aca9-1f05548374f2: failed to appendEntries to: peer="{Voter server-25c3a33e-ad03-e556-5dae-b650ee62bac2 25c3a33e-ad03-e556-5dae-b650ee62bac2}" error="failed to connect to peer: 25c3a33e-ad03-e556-5dae-b650ee62bac2" -2026-04-22T10:11:06.112Z [WARN] server-39658725-95ed-aef1-aca9-1f05548374f2: failed to contact: server-id=server-25c3a33e-ad03-e556-5dae-b650ee62bac2 time=50.274672ms -2026-04-22T10:11:06.112Z [WARN] server-39658725-95ed-aef1-aca9-1f05548374f2: failed to contact: server-id=server-92afba55-9920-6e14-3906-e747a5f2105c time=50.137334ms -2026-04-22T10:11:06.112Z [WARN] server-39658725-95ed-aef1-aca9-1f05548374f2: failed to contact quorum of nodes, stepping down -2026-04-22T10:11:06.112Z [INFO] server-39658725-95ed-aef1-aca9-1f05548374f2: entering follower state: follower="Node at 39658725-95ed-aef1-aca9-1f05548374f2 [Follower]" leader-address= leader-id= -2026-04-22T10:11:06.166Z [WARN] server-92afba55-9920-6e14-3906-e747a5f2105c: heartbeat timeout reached, starting election: last-leader-addr=39658725-95ed-aef1-aca9-1f05548374f2 last-leader-id=server-39658725-95ed-aef1-aca9-1f05548374f2 -2026-04-22T10:11:06.166Z [INFO] server-92afba55-9920-6e14-3906-e747a5f2105c: entering candidate state: node="Node at 92afba55-9920-6e14-3906-e747a5f2105c [Candidate]" term=3 -2026-04-22T10:11:06.166Z [ERROR] server-92afba55-9920-6e14-3906-e747a5f2105c: failed to make requestVote RPC: target="{Voter server-39658725-95ed-aef1-aca9-1f05548374f2 39658725-95ed-aef1-aca9-1f05548374f2}" error="failed to connect to peer: 39658725-95ed-aef1-aca9-1f05548374f2" term=3 -2026-04-22T10:11:06.166Z [WARN] server-25c3a33e-ad03-e556-5dae-b650ee62bac2: rejecting vote request since we have a leader: from=92afba55-9920-6e14-3906-e747a5f2105c leader=39658725-95ed-aef1-aca9-1f05548374f2 leader-id=server-39658725-95ed-aef1-aca9-1f05548374f2 -2026-04-22T10:11:06.175Z [WARN] server-25c3a33e-ad03-e556-5dae-b650ee62bac2: heartbeat timeout reached, starting election: last-leader-addr=39658725-95ed-aef1-aca9-1f05548374f2 last-leader-id=server-39658725-95ed-aef1-aca9-1f05548374f2 -2026-04-22T10:11:06.175Z [INFO] server-25c3a33e-ad03-e556-5dae-b650ee62bac2: entering candidate state: node="Node at 25c3a33e-ad03-e556-5dae-b650ee62bac2 [Candidate]" term=3 -2026-04-22T10:11:06.175Z [INFO] server-92afba55-9920-6e14-3906-e747a5f2105c: duplicate requestVote for same term: term=3 -2026-04-22T10:11:06.175Z [ERROR] server-25c3a33e-ad03-e556-5dae-b650ee62bac2: failed to make requestVote RPC: target="{Voter server-39658725-95ed-aef1-aca9-1f05548374f2 39658725-95ed-aef1-aca9-1f05548374f2}" error="failed to connect to peer: 39658725-95ed-aef1-aca9-1f05548374f2" term=3 -2026-04-22T10:11:06.176Z [INFO] server-25c3a33e-ad03-e556-5dae-b650ee62bac2: entering follower state: follower="Node at 25c3a33e-ad03-e556-5dae-b650ee62bac2 [Follower]" leader-address= leader-id= ---- PASS: TestRaft_VoteWithNoIDNoAddr (0.49s) +2025-03-20T03:51:25.464Z [INFO] server-896f80f9-e1cd-7bb3-05f7-47010e4626b8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-896f80f9-e1cd-7bb3-05f7-47010e4626b8 Address:896f80f9-e1cd-7bb3-05f7-47010e4626b8} {Suffrage:Voter ID:server-fa9df93c-ca06-c076-cc6b-aecec2946b88 Address:fa9df93c-ca06-c076-cc6b-aecec2946b88} {Suffrage:Voter ID:server-3022d0b1-da7e-6781-155f-89b61bf3ae28 Address:3022d0b1-da7e-6781-155f-89b61bf3ae28}]" +2025-03-20T03:51:25.464Z [INFO] server-fa9df93c-ca06-c076-cc6b-aecec2946b88: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-896f80f9-e1cd-7bb3-05f7-47010e4626b8 Address:896f80f9-e1cd-7bb3-05f7-47010e4626b8} {Suffrage:Voter ID:server-fa9df93c-ca06-c076-cc6b-aecec2946b88 Address:fa9df93c-ca06-c076-cc6b-aecec2946b88} {Suffrage:Voter ID:server-3022d0b1-da7e-6781-155f-89b61bf3ae28 Address:3022d0b1-da7e-6781-155f-89b61bf3ae28}]" +2025-03-20T03:51:25.464Z [INFO] server-3022d0b1-da7e-6781-155f-89b61bf3ae28: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-896f80f9-e1cd-7bb3-05f7-47010e4626b8 Address:896f80f9-e1cd-7bb3-05f7-47010e4626b8} {Suffrage:Voter ID:server-fa9df93c-ca06-c076-cc6b-aecec2946b88 Address:fa9df93c-ca06-c076-cc6b-aecec2946b88} {Suffrage:Voter ID:server-3022d0b1-da7e-6781-155f-89b61bf3ae28 Address:3022d0b1-da7e-6781-155f-89b61bf3ae28}]" +2025-03-20T03:51:25.464Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:25.464Z [INFO] server-896f80f9-e1cd-7bb3-05f7-47010e4626b8: entering follower state: follower="Node at 896f80f9-e1cd-7bb3-05f7-47010e4626b8 [Follower]" leader-address= leader-id= +2025-03-20T03:51:25.464Z [INFO] server-fa9df93c-ca06-c076-cc6b-aecec2946b88: entering follower state: follower="Node at fa9df93c-ca06-c076-cc6b-aecec2946b88 [Follower]" leader-address= leader-id= +2025-03-20T03:51:25.464Z [INFO] server-3022d0b1-da7e-6781-155f-89b61bf3ae28: entering follower state: follower="Node at 3022d0b1-da7e-6781-155f-89b61bf3ae28 [Follower]" leader-address= leader-id= +2025-03-20T03:51:25.542Z [WARN] server-fa9df93c-ca06-c076-cc6b-aecec2946b88: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:25.542Z [INFO] server-fa9df93c-ca06-c076-cc6b-aecec2946b88: entering candidate state: node="Node at fa9df93c-ca06-c076-cc6b-aecec2946b88 [Candidate]" term=2 +2025-03-20T03:51:25.542Z [INFO] server-fa9df93c-ca06-c076-cc6b-aecec2946b88: election won: term=2 tally=2 +2025-03-20T03:51:25.542Z [INFO] server-fa9df93c-ca06-c076-cc6b-aecec2946b88: entering leader state: leader="Node at fa9df93c-ca06-c076-cc6b-aecec2946b88 [Leader]" +2025-03-20T03:51:25.542Z [INFO] server-fa9df93c-ca06-c076-cc6b-aecec2946b88: added peer, starting replication: peer=server-896f80f9-e1cd-7bb3-05f7-47010e4626b8 +2025-03-20T03:51:25.542Z [INFO] server-fa9df93c-ca06-c076-cc6b-aecec2946b88: added peer, starting replication: peer=server-3022d0b1-da7e-6781-155f-89b61bf3ae28 +2025-03-20T03:51:25.543Z [INFO] server-fa9df93c-ca06-c076-cc6b-aecec2946b88: pipelining replication: peer="{Voter server-896f80f9-e1cd-7bb3-05f7-47010e4626b8 896f80f9-e1cd-7bb3-05f7-47010e4626b8}" +2025-03-20T03:51:25.545Z [WARN] server-3022d0b1-da7e-6781-155f-89b61bf3ae28: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:25.545Z [INFO] server-3022d0b1-da7e-6781-155f-89b61bf3ae28: entering candidate state: node="Node at 3022d0b1-da7e-6781-155f-89b61bf3ae28 [Candidate]" term=2 +2025-03-20T03:51:25.545Z [INFO] server-3022d0b1-da7e-6781-155f-89b61bf3ae28: duplicate requestVote for same term: term=2 +2025-03-20T03:51:25.545Z [INFO] server-3022d0b1-da7e-6781-155f-89b61bf3ae28: entering follower state: follower="Node at 3022d0b1-da7e-6781-155f-89b61bf3ae28 [Follower]" leader-address=fa9df93c-ca06-c076-cc6b-aecec2946b88 leader-id=server-fa9df93c-ca06-c076-cc6b-aecec2946b88 +2025-03-20T03:51:25.545Z [INFO] server-fa9df93c-ca06-c076-cc6b-aecec2946b88: pipelining replication: peer="{Voter server-3022d0b1-da7e-6781-155f-89b61bf3ae28 3022d0b1-da7e-6781-155f-89b61bf3ae28}" +2025-03-20T03:51:25.546Z [WARN] server-fa9df93c-ca06-c076-cc6b-aecec2946b88: rejecting vote request since we have a leader: from=3022d0b1-da7e-6781-155f-89b61bf3ae28 leader=fa9df93c-ca06-c076-cc6b-aecec2946b88 leader-id=server-fa9df93c-ca06-c076-cc6b-aecec2946b88 +2025-03-20T03:51:25.546Z [WARN] server-896f80f9-e1cd-7bb3-05f7-47010e4626b8: rejecting vote request since we have a leader: from=3022d0b1-da7e-6781-155f-89b61bf3ae28 leader=fa9df93c-ca06-c076-cc6b-aecec2946b88 leader-id=server-fa9df93c-ca06-c076-cc6b-aecec2946b88 +2025-03-20T03:51:25.651Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:25.546320071 +0000 UTC m=+117.220773082 (1 nodes), 81.727325ms from start of poll, 82.242184ms from cluster start. Timeout at 2025-03-20 03:51:25.651412917 +0000 UTC m=+117.325865692, 105.09261ms after stability +2025-03-20T03:51:25.651Z [INFO] cluster: starting stability test: raft-state=Leader +2025-03-20T03:51:25.762Z [INFO] cluster: stable state for Leader reached at 2025-03-20 03:51:25.651595673 +0000 UTC m=+117.326048459 (1 nodes), 2.027µs from start of poll, 187.517561ms from cluster start. Timeout at 2025-03-20 03:51:25.756675859 +0000 UTC m=+117.431128965, 105.080506ms after stability +2025-03-20T03:51:25.770Z [INFO] cluster: starting stability test: raft-state=Follower +2025-03-20T03:51:25.882Z [INFO] cluster: stable state for Follower reached at 2025-03-20 03:51:25.770947208 +0000 UTC m=+117.445400241 (2 nodes), 4.295µs from start of poll, 306.869343ms from cluster start. Timeout at 2025-03-20 03:51:25.875992561 +0000 UTC m=+117.550445723, 105.045482ms after stability +2025-03-20T03:51:25.886Z [ERROR] server-fa9df93c-ca06-c076-cc6b-aecec2946b88: failed to pipeline appendEntries: peer="{Voter server-3022d0b1-da7e-6781-155f-89b61bf3ae28 3022d0b1-da7e-6781-155f-89b61bf3ae28}" error="append pipeline closed" +2025-03-20T03:51:25.886Z [INFO] server-fa9df93c-ca06-c076-cc6b-aecec2946b88: aborting pipeline replication: peer="{Voter server-3022d0b1-da7e-6781-155f-89b61bf3ae28 3022d0b1-da7e-6781-155f-89b61bf3ae28}" +2025-03-20T03:51:25.886Z [ERROR] server-fa9df93c-ca06-c076-cc6b-aecec2946b88: failed to pipeline appendEntries: peer="{Voter server-896f80f9-e1cd-7bb3-05f7-47010e4626b8 896f80f9-e1cd-7bb3-05f7-47010e4626b8}" error="append pipeline closed" +2025-03-20T03:51:25.886Z [INFO] server-fa9df93c-ca06-c076-cc6b-aecec2946b88: aborting pipeline replication: peer="{Voter server-896f80f9-e1cd-7bb3-05f7-47010e4626b8 896f80f9-e1cd-7bb3-05f7-47010e4626b8}" +2025-03-20T03:51:25.890Z [ERROR] server-fa9df93c-ca06-c076-cc6b-aecec2946b88: failed to heartbeat to: peer=3022d0b1-da7e-6781-155f-89b61bf3ae28 backoff time=10ms error="failed to connect to peer: 3022d0b1-da7e-6781-155f-89b61bf3ae28" +2025-03-20T03:51:25.894Z [ERROR] server-fa9df93c-ca06-c076-cc6b-aecec2946b88: failed to heartbeat to: peer=896f80f9-e1cd-7bb3-05f7-47010e4626b8 backoff time=10ms error="failed to connect to peer: 896f80f9-e1cd-7bb3-05f7-47010e4626b8" +2025-03-20T03:51:25.894Z [ERROR] server-fa9df93c-ca06-c076-cc6b-aecec2946b88: failed to appendEntries to: peer="{Voter server-3022d0b1-da7e-6781-155f-89b61bf3ae28 3022d0b1-da7e-6781-155f-89b61bf3ae28}" error="failed to connect to peer: 3022d0b1-da7e-6781-155f-89b61bf3ae28" +2025-03-20T03:51:25.894Z [ERROR] server-fa9df93c-ca06-c076-cc6b-aecec2946b88: failed to appendEntries to: peer="{Voter server-896f80f9-e1cd-7bb3-05f7-47010e4626b8 896f80f9-e1cd-7bb3-05f7-47010e4626b8}" error="failed to connect to peer: 896f80f9-e1cd-7bb3-05f7-47010e4626b8" +2025-03-20T03:51:25.914Z [ERROR] server-fa9df93c-ca06-c076-cc6b-aecec2946b88: failed to heartbeat to: peer=896f80f9-e1cd-7bb3-05f7-47010e4626b8 backoff time=10ms error="failed to connect to peer: 896f80f9-e1cd-7bb3-05f7-47010e4626b8" +2025-03-20T03:51:25.914Z [ERROR] server-fa9df93c-ca06-c076-cc6b-aecec2946b88: failed to heartbeat to: peer=3022d0b1-da7e-6781-155f-89b61bf3ae28 backoff time=10ms error="failed to connect to peer: 3022d0b1-da7e-6781-155f-89b61bf3ae28" +2025-03-20T03:51:25.918Z [ERROR] server-fa9df93c-ca06-c076-cc6b-aecec2946b88: failed to appendEntries to: peer="{Voter server-3022d0b1-da7e-6781-155f-89b61bf3ae28 3022d0b1-da7e-6781-155f-89b61bf3ae28}" error="failed to connect to peer: 3022d0b1-da7e-6781-155f-89b61bf3ae28" +2025-03-20T03:51:25.918Z [ERROR] server-fa9df93c-ca06-c076-cc6b-aecec2946b88: failed to appendEntries to: peer="{Voter server-896f80f9-e1cd-7bb3-05f7-47010e4626b8 896f80f9-e1cd-7bb3-05f7-47010e4626b8}" error="failed to connect to peer: 896f80f9-e1cd-7bb3-05f7-47010e4626b8" +2025-03-20T03:51:25.934Z [WARN] server-fa9df93c-ca06-c076-cc6b-aecec2946b88: failed to contact: server-id=server-896f80f9-e1cd-7bb3-05f7-47010e4626b8 time=51.910556ms +2025-03-20T03:51:25.934Z [WARN] server-fa9df93c-ca06-c076-cc6b-aecec2946b88: failed to contact: server-id=server-3022d0b1-da7e-6781-155f-89b61bf3ae28 time=51.690824ms +2025-03-20T03:51:25.934Z [WARN] server-fa9df93c-ca06-c076-cc6b-aecec2946b88: failed to contact quorum of nodes, stepping down +2025-03-20T03:51:25.934Z [INFO] server-fa9df93c-ca06-c076-cc6b-aecec2946b88: entering follower state: follower="Node at fa9df93c-ca06-c076-cc6b-aecec2946b88 [Follower]" leader-address= leader-id= +2025-03-20T03:51:25.942Z [ERROR] server-fa9df93c-ca06-c076-cc6b-aecec2946b88: failed to heartbeat to: peer=896f80f9-e1cd-7bb3-05f7-47010e4626b8 backoff time=10ms error="failed to connect to peer: 896f80f9-e1cd-7bb3-05f7-47010e4626b8" +2025-03-20T03:51:25.942Z [ERROR] server-fa9df93c-ca06-c076-cc6b-aecec2946b88: failed to heartbeat to: peer=3022d0b1-da7e-6781-155f-89b61bf3ae28 backoff time=10ms error="failed to connect to peer: 3022d0b1-da7e-6781-155f-89b61bf3ae28" +2025-03-20T03:51:25.943Z [ERROR] server-fa9df93c-ca06-c076-cc6b-aecec2946b88: failed to appendEntries to: peer="{Voter server-896f80f9-e1cd-7bb3-05f7-47010e4626b8 896f80f9-e1cd-7bb3-05f7-47010e4626b8}" error="failed to connect to peer: 896f80f9-e1cd-7bb3-05f7-47010e4626b8" +2025-03-20T03:51:25.943Z [ERROR] server-fa9df93c-ca06-c076-cc6b-aecec2946b88: failed to appendEntries to: peer="{Voter server-3022d0b1-da7e-6781-155f-89b61bf3ae28 3022d0b1-da7e-6781-155f-89b61bf3ae28}" error="failed to connect to peer: 3022d0b1-da7e-6781-155f-89b61bf3ae28" +2025-03-20T03:51:25.978Z [WARN] server-896f80f9-e1cd-7bb3-05f7-47010e4626b8: heartbeat timeout reached, starting election: last-leader-addr=fa9df93c-ca06-c076-cc6b-aecec2946b88 last-leader-id=server-fa9df93c-ca06-c076-cc6b-aecec2946b88 +2025-03-20T03:51:25.978Z [INFO] server-896f80f9-e1cd-7bb3-05f7-47010e4626b8: entering candidate state: node="Node at 896f80f9-e1cd-7bb3-05f7-47010e4626b8 [Candidate]" term=3 +2025-03-20T03:51:25.978Z [WARN] server-3022d0b1-da7e-6781-155f-89b61bf3ae28: rejecting vote request since we have a leader: from=896f80f9-e1cd-7bb3-05f7-47010e4626b8 leader=fa9df93c-ca06-c076-cc6b-aecec2946b88 leader-id=server-fa9df93c-ca06-c076-cc6b-aecec2946b88 +2025-03-20T03:51:25.978Z [ERROR] server-896f80f9-e1cd-7bb3-05f7-47010e4626b8: failed to make requestVote RPC: target="{Voter server-fa9df93c-ca06-c076-cc6b-aecec2946b88 fa9df93c-ca06-c076-cc6b-aecec2946b88}" error="failed to connect to peer: fa9df93c-ca06-c076-cc6b-aecec2946b88" term=3 +2025-03-20T03:51:25.982Z [WARN] server-3022d0b1-da7e-6781-155f-89b61bf3ae28: heartbeat timeout reached, starting election: last-leader-addr=fa9df93c-ca06-c076-cc6b-aecec2946b88 last-leader-id=server-fa9df93c-ca06-c076-cc6b-aecec2946b88 +2025-03-20T03:51:25.982Z [INFO] server-3022d0b1-da7e-6781-155f-89b61bf3ae28: entering candidate state: node="Node at 3022d0b1-da7e-6781-155f-89b61bf3ae28 [Candidate]" term=3 +2025-03-20T03:51:25.982Z [ERROR] server-3022d0b1-da7e-6781-155f-89b61bf3ae28: failed to make requestVote RPC: target="{Voter server-fa9df93c-ca06-c076-cc6b-aecec2946b88 fa9df93c-ca06-c076-cc6b-aecec2946b88}" error="failed to connect to peer: fa9df93c-ca06-c076-cc6b-aecec2946b88" term=3 +2025-03-20T03:51:25.982Z [INFO] server-896f80f9-e1cd-7bb3-05f7-47010e4626b8: duplicate requestVote for same term: term=3 +2025-03-20T03:51:25.991Z [INFO] server-896f80f9-e1cd-7bb3-05f7-47010e4626b8: entering follower state: follower="Node at 896f80f9-e1cd-7bb3-05f7-47010e4626b8 [Follower]" leader-address= leader-id= +--- PASS: TestRaft_VoteWithNoIDNoAddr (0.53s) === RUN TestRaft_runFollower_State_Transition === RUN TestRaft_runFollower_State_Transition/NonVoter -2026-04-22T10:11:06.177Z [INFO] 127.0.0.1:42657 :: starting node: addr=127.0.0.1:42657 -2026-04-22T10:11:06.177Z [INFO] 127.0.0.1:42657 :: initial configuration: index=0 servers=[] -2026-04-22T10:11:06.177Z [INFO] 127.0.0.1:42657 :: entering follower state: follower="Node at 127.0.0.1:42657 [Follower]" leader-address= leader-id= -2026-04-22T10:11:06.238Z [WARN] 127.0.0.1:42657 :: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +2025-03-20T03:51:26.019Z [INFO] 127.0.0.1:34951 :: starting node: addr=127.0.0.1:34951 +2025-03-20T03:51:26.019Z [INFO] 127.0.0.1:34951 :: initial configuration: index=0 servers=[] +2025-03-20T03:51:26.020Z [INFO] 127.0.0.1:34951 :: entering follower state: follower="Node at 127.0.0.1:34951 [Follower]" leader-address= leader-id= +2025-03-20T03:51:26.086Z [WARN] 127.0.0.1:34951 :: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election === RUN TestRaft_runFollower_State_Transition/Voter -2026-04-22T10:11:06.329Z [INFO] 127.0.0.1:34381 :: starting node: addr=127.0.0.1:34381 -2026-04-22T10:11:06.329Z [INFO] 127.0.0.1:34381 :: initial configuration: index=0 servers=[] -2026-04-22T10:11:06.329Z [INFO] 127.0.0.1:34381 :: entering follower state: follower="Node at 127.0.0.1:34381 [Follower]" leader-address= leader-id= -2026-04-22T10:11:06.400Z [WARN] 127.0.0.1:34381 :: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:26.170Z [INFO] 127.0.0.1:37755 :: starting node: addr=127.0.0.1:37755 +2025-03-20T03:51:26.170Z [INFO] 127.0.0.1:37755 :: initial configuration: index=0 servers=[] +2025-03-20T03:51:26.170Z [INFO] 127.0.0.1:37755 :: entering follower state: follower="Node at 127.0.0.1:37755 [Follower]" leader-address= leader-id= +2025-03-20T03:51:26.234Z [WARN] 127.0.0.1:37755 :: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= === RUN TestRaft_runFollower_State_Transition/Not_in_Config -2026-04-22T10:11:06.480Z [INFO] 127.0.0.1:38633 :: starting node: addr=127.0.0.1:38633 -2026-04-22T10:11:06.480Z [INFO] 127.0.0.1:38633 :: initial configuration: index=0 servers=[] -2026-04-22T10:11:06.480Z [INFO] 127.0.0.1:38633 :: entering follower state: follower="Node at 127.0.0.1:38633 [Follower]" leader-address= leader-id= -2026-04-22T10:11:06.565Z [WARN] 127.0.0.1:38633 :: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election ---- PASS: TestRaft_runFollower_State_Transition (0.45s) - --- PASS: TestRaft_runFollower_State_Transition/NonVoter (0.15s) +2025-03-20T03:51:26.326Z [INFO] 127.0.0.1:33941 :: starting node: addr=127.0.0.1:33941 +2025-03-20T03:51:26.327Z [INFO] 127.0.0.1:33941 :: initial configuration: index=0 servers=[] +2025-03-20T03:51:26.327Z [INFO] 127.0.0.1:33941 :: entering follower state: follower="Node at 127.0.0.1:33941 [Follower]" leader-address= leader-id= +2025-03-20T03:51:26.402Z [WARN] 127.0.0.1:33941 :: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +--- PASS: TestRaft_runFollower_State_Transition (0.49s) + --- PASS: TestRaft_runFollower_State_Transition/NonVoter (0.18s) --- PASS: TestRaft_runFollower_State_Transition/Voter (0.15s) --- PASS: TestRaft_runFollower_State_Transition/Not_in_Config (0.15s) === RUN TestRaft_runFollower_ReloadTimeoutConfigs -2026-04-22T10:11:06.633Z [INFO] 127.0.0.1:35061 :: starting node: addr=127.0.0.1:35061 -2026-04-22T10:11:06.634Z [INFO] 127.0.0.1:35061 :: initial configuration: index=0 servers=[] -2026-04-22T10:11:06.634Z [INFO] 127.0.0.1:35061 :: entering follower state: follower="Node at 127.0.0.1:35061 [Follower]" leader-address= leader-id= -2026-04-22T10:11:06.634Z [WARN] 127.0.0.1:35061 :: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-03-20T03:51:26.478Z [INFO] 127.0.0.1:34919 :: starting node: addr=127.0.0.1:34919 +2025-03-20T03:51:26.478Z [INFO] 127.0.0.1:34919 :: initial configuration: index=0 servers=[] +2025-03-20T03:51:26.479Z [INFO] 127.0.0.1:34919 :: entering follower state: follower="Node at 127.0.0.1:34919 [Follower]" leader-address= leader-id= +2025-03-20T03:51:26.479Z [WARN] 127.0.0.1:34919 :: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= --- PASS: TestRaft_runFollower_ReloadTimeoutConfigs (0.15s) === RUN TestSaturationMetric === RUN TestSaturationMetric/without_smoothing @@ -4247,47 +4425,32 @@ === RUN TestMax --- PASS: TestMax (0.00s) === RUN TestGenerateUUID ---- PASS: TestGenerateUUID (0.00s) +--- PASS: TestGenerateUUID (0.04s) === RUN TestBackoff --- PASS: TestBackoff (0.00s) === RUN TestOverrideNotifyBool --- PASS: TestOverrideNotifyBool (0.00s) -PASS -ok github.com/hashicorp/raft 112.820s +FAIL +FAIL github.com/hashicorp/raft 118.425s ? github.com/hashicorp/raft/bench [no test files] +FAIL +dh_auto_test: error: cd obj-i686-linux-gnu && go test -vet=off -v -p 10 github.com/hashicorp/raft github.com/hashicorp/raft/bench returned exit code 1 +make[1]: *** [debian/rules:17: override_dh_auto_test] Error 25 make[1]: Leaving directory '/build/reproducible-path/golang-github-hashicorp-raft-1.5.0' - create-stamp debian/debhelper-build-stamp - dh_testroot -O--buildsystem=golang - dh_prep -O--buildsystem=golang - dh_auto_install --destdir=debian/golang-github-hashicorp-raft-dev/ -O--buildsystem=golang - dh_installdocs -O--buildsystem=golang - dh_installchangelogs -O--buildsystem=golang - dh_installsystemduser -O--buildsystem=golang - dh_perl -O--buildsystem=golang - dh_link -O--buildsystem=golang - dh_strip_nondeterminism -O--buildsystem=golang - dh_compress -O--buildsystem=golang - dh_fixperms -O--buildsystem=golang - dh_missing -O--buildsystem=golang - dh_installdeb -O--buildsystem=golang - dh_golang -O--buildsystem=golang - dh_gencontrol -O--buildsystem=golang - dh_md5sums -O--buildsystem=golang - dh_builddeb -O--buildsystem=golang -dpkg-deb: building package 'golang-github-hashicorp-raft-dev' in '../golang-github-hashicorp-raft-dev_1.5.0-1_all.deb'. - dpkg-genbuildinfo --build=binary -O../golang-github-hashicorp-raft_1.5.0-1_i386.buildinfo - dpkg-genchanges --build=binary -O../golang-github-hashicorp-raft_1.5.0-1_i386.changes -dpkg-genchanges: info: binary-only upload (no source code included) - dpkg-source --after-build . -dpkg-buildpackage: info: binary-only upload (no source included) -dpkg-genchanges: info: including full source code in upload +make: *** [debian/rules:14: binary] Error 2 +dpkg-buildpackage: error: debian/rules binary subprocess returned exit status 2 I: copying local configuration +E: Failed autobuilding of package +I: user script /srv/workspace/pbuilder/44275/tmp/hooks/C01_cleanup starting +debug output: disk usage on i-capture-the-hostname at Thu Mar 20 03:51:32 UTC 2025 +Filesystem Size Used Avail Use% Mounted on +tmpfs 3.9G 0 3.9G 0% /dev/shm + +I: user script /srv/workspace/pbuilder/44275/tmp/hooks/C01_cleanup finished I: unmounting dev/ptmx filesystem I: unmounting dev/pts filesystem I: unmounting dev/shm filesystem I: unmounting proc filesystem I: unmounting sys filesystem I: cleaning the build env -I: removing directory /srv/workspace/pbuilder/53424 and its subdirectories -I: Current time: Tue Apr 21 22:11:10 -12 2026 -I: pbuilder-time-stamp: 1776852670 +I: removing directory /srv/workspace/pbuilder/44275 and its subdirectories