Diff of the two buildlogs: -- --- b1/build.log 2025-02-25 23:55:12.676184849 +0000 +++ b2/build.log 2025-02-25 23:59:31.836478971 +0000 @@ -1,6 +1,6 @@ I: pbuilder: network access will be disabled during build -I: Current time: Mon Mar 30 18:15:16 -12 2026 -I: pbuilder-time-stamp: 1774937716 +I: Current time: Wed Feb 26 13:55:15 +14 2025 +I: pbuilder-time-stamp: 1740527715 I: Building the build Environment I: extracting base tarball [/var/cache/pbuilder/trixie-reproducible-base.tgz] I: copying local configuration @@ -24,52 +24,84 @@ dpkg-source: info: applying 0001-Skip-unreliable-tests.patch I: Not using root during the build. I: Installing the build-deps -I: user script /srv/workspace/pbuilder/674357/tmp/hooks/D02_print_environment starting +I: user script /srv/workspace/pbuilder/4115557/tmp/hooks/D01_modify_environment starting +debug: Running on codethink04-arm64. +I: Changing host+domainname to test build reproducibility +I: Adding a custom variable just for the fun of it... +I: Changing /bin/sh to bash +'/bin/sh' -> '/bin/bash' +lrwxrwxrwx 1 root root 9 Feb 25 23:55 /bin/sh -> /bin/bash +I: Setting pbuilder2's login shell to /bin/bash +I: Setting pbuilder2's GECOS to second user,second room,second work-phone,second home-phone,second other +I: user script /srv/workspace/pbuilder/4115557/tmp/hooks/D01_modify_environment finished +I: user script /srv/workspace/pbuilder/4115557/tmp/hooks/D02_print_environment starting I: set - BUILDDIR='/build/reproducible-path' - BUILDUSERGECOS='first user,first room,first work-phone,first home-phone,first other' - BUILDUSERNAME='pbuilder1' - BUILD_ARCH='arm64' - DEBIAN_FRONTEND='noninteractive' + BASH=/bin/sh + BASHOPTS=checkwinsize:cmdhist:complete_fullquote:extquote:force_fignore:globasciiranges:globskipdots:hostcomplete:interactive_comments:patsub_replacement:progcomp:promptvars:sourcepath + BASH_ALIASES=() + BASH_ARGC=() + BASH_ARGV=() + BASH_CMDS=() + BASH_LINENO=([0]="12" [1]="0") + BASH_LOADABLES_PATH=/usr/local/lib/bash:/usr/lib/bash:/opt/local/lib/bash:/usr/pkg/lib/bash:/opt/pkg/lib/bash:. + BASH_SOURCE=([0]="/tmp/hooks/D02_print_environment" [1]="/tmp/hooks/D02_print_environment") + BASH_VERSINFO=([0]="5" [1]="2" [2]="37" [3]="1" [4]="release" [5]="aarch64-unknown-linux-gnu") + BASH_VERSION='5.2.37(1)-release' + BUILDDIR=/build/reproducible-path + BUILDUSERGECOS='second user,second room,second work-phone,second home-phone,second other' + BUILDUSERNAME=pbuilder2 + BUILD_ARCH=arm64 + DEBIAN_FRONTEND=noninteractive DEB_BUILD_OPTIONS='buildinfo=+all reproducible=+all parallel=12 ' - DISTRIBUTION='trixie' - HOME='/root' - HOST_ARCH='arm64' + DIRSTACK=() + DISTRIBUTION=trixie + EUID=0 + FUNCNAME=([0]="Echo" [1]="main") + GROUPS=() + HOME=/root + HOSTNAME=i-capture-the-hostname + HOSTTYPE=aarch64 + HOST_ARCH=arm64 IFS=' ' - INVOCATION_ID='c8d09372adf848c383950b010f0e8b77' - LANG='C' - LANGUAGE='en_US:en' - LC_ALL='C' - MAIL='/var/mail/root' - OPTIND='1' - PATH='/usr/sbin:/usr/bin:/sbin:/bin:/usr/games' - PBCURRENTCOMMANDLINEOPERATION='build' - PBUILDER_OPERATION='build' - PBUILDER_PKGDATADIR='/usr/share/pbuilder' - PBUILDER_PKGLIBDIR='/usr/lib/pbuilder' - PBUILDER_SYSCONFDIR='/etc' - PPID='674357' - PS1='# ' - PS2='> ' + INVOCATION_ID=bdf65bce2e074198bf2dc16babb3189b + LANG=C + LANGUAGE=nl_BE:nl + LC_ALL=C + MACHTYPE=aarch64-unknown-linux-gnu + MAIL=/var/mail/root + OPTERR=1 + OPTIND=1 + OSTYPE=linux-gnu + PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path + PBCURRENTCOMMANDLINEOPERATION=build + PBUILDER_OPERATION=build + PBUILDER_PKGDATADIR=/usr/share/pbuilder + PBUILDER_PKGLIBDIR=/usr/lib/pbuilder + PBUILDER_SYSCONFDIR=/etc + PIPESTATUS=([0]="0") + POSIXLY_CORRECT=y + PPID=4115557 PS4='+ ' - PWD='/' - SHELL='/bin/bash' - SHLVL='2' - SUDO_COMMAND='/usr/bin/timeout -k 18.1h 18h /usr/bin/ionice -c 3 /usr/bin/nice /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/r-b-build.xirTGdOv/pbuilderrc_sjg6 --distribution trixie --hookdir /etc/pbuilder/first-build-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/trixie-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/r-b-build.xirTGdOv/b1 --logfile b1/build.log golang-github-hashicorp-raft_1.5.0-1.dsc' - SUDO_GID='109' - SUDO_UID='104' - SUDO_USER='jenkins' - TERM='unknown' - TZ='/usr/share/zoneinfo/Etc/GMT+12' - USER='root' - _='/usr/bin/systemd-run' - http_proxy='http://192.168.101.4:3128' + PWD=/ + SHELL=/bin/bash + SHELLOPTS=braceexpand:errexit:hashall:interactive-comments:posix + SHLVL=3 + SUDO_COMMAND='/usr/bin/timeout -k 24.1h 24h /usr/bin/ionice -c 3 /usr/bin/nice -n 11 /usr/bin/unshare --uts -- /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/r-b-build.xirTGdOv/pbuilderrc_M55A --distribution trixie --hookdir /etc/pbuilder/rebuild-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/trixie-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/r-b-build.xirTGdOv/b2 --logfile b2/build.log golang-github-hashicorp-raft_1.5.0-1.dsc' + SUDO_GID=109 + SUDO_UID=104 + SUDO_USER=jenkins + TERM=unknown + TZ=/usr/share/zoneinfo/Etc/GMT-14 + UID=0 + USER=root + _='I: set' + http_proxy=http://192.168.101.4:3128 I: uname -a - Linux codethink03-arm64 6.1.0-31-cloud-arm64 #1 SMP Debian 6.1.128-1 (2025-02-07) aarch64 GNU/Linux + Linux i-capture-the-hostname 6.1.0-31-cloud-arm64 #1 SMP Debian 6.1.128-1 (2025-02-07) aarch64 GNU/Linux I: ls -l /bin - lrwxrwxrwx 1 root root 7 Nov 22 2024 /bin -> usr/bin -I: user script /srv/workspace/pbuilder/674357/tmp/hooks/D02_print_environment finished + lrwxrwxrwx 1 root root 7 Nov 22 14:40 /bin -> usr/bin +I: user script /srv/workspace/pbuilder/4115557/tmp/hooks/D02_print_environment finished -> Attempting to satisfy build-dependencies -> Creating pbuilder-satisfydepends-dummy package Package: pbuilder-satisfydepends-dummy @@ -226,7 +258,7 @@ Get: 107 http://deb.debian.org/debian trixie/main arm64 golang-gopkg-check.v1-dev all 0.0+git20200902.038fdea-1 [32.1 kB] Get: 108 http://deb.debian.org/debian trixie/main arm64 golang-gopkg-mgo.v2-dev all 2016.08.01-7 [316 kB] Get: 109 http://deb.debian.org/debian trixie/main arm64 golang-github-hashicorp-go-msgpack-dev all 0.5.5-1 [43.3 kB] -Fetched 128 MB in 1s (253 MB/s) +Fetched 128 MB in 1s (91.0 MB/s) Preconfiguring packages ... Selecting previously unselected package sensible-utils. (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 19916 files and directories currently installed.) @@ -677,7 +709,11 @@ Building tag database... -> Finished parsing the build-deps I: Building the package -I: Running cd /build/reproducible-path/golang-github-hashicorp-raft-1.5.0/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" HOME="/nonexistent/first-build" dpkg-buildpackage -us -uc -b && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" HOME="/nonexistent/first-build" dpkg-genchanges -S > ../golang-github-hashicorp-raft_1.5.0-1_source.changes +I: user script /srv/workspace/pbuilder/4115557/tmp/hooks/A99_set_merged_usr starting +Not re-configuring usrmerge for trixie +I: user script /srv/workspace/pbuilder/4115557/tmp/hooks/A99_set_merged_usr finished +hostname: Name or service not known +I: Running cd /build/reproducible-path/golang-github-hashicorp-raft-1.5.0/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" HOME="/nonexistent/second-build" dpkg-buildpackage -us -uc -b && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" HOME="/nonexistent/second-build" dpkg-genchanges -S > ../golang-github-hashicorp-raft_1.5.0-1_source.changes dpkg-buildpackage: info: source package golang-github-hashicorp-raft dpkg-buildpackage: info: source version 1.5.0-1 dpkg-buildpackage: info: source distribution unstable @@ -697,201 +733,201 @@ dh_auto_build -O--buildsystem=golang cd obj-aarch64-linux-gnu && go install -trimpath -v -p 12 github.com/hashicorp/raft github.com/hashicorp/raft/bench internal/goarch -internal/asan -internal/unsafeheader -internal/profilerecord -internal/msan internal/coverage/rtcov +internal/unsafeheader internal/godebugs -internal/goexperiment -internal/byteorder internal/goos -internal/runtime/syscall +internal/byteorder +internal/profilerecord internal/runtime/math -internal/abi -sync/atomic -internal/cpu +internal/msan +internal/goexperiment math/bits +internal/cpu unicode unicode/utf8 +internal/runtime/syscall container/list -internal/itoa +internal/asan +internal/abi internal/chacha8rand +internal/itoa crypto/internal/fips140/alias +sync/atomic crypto/internal/fips140deps/byteorder cmp crypto/internal/fips140/subtle crypto/internal/boring/sig encoding +math unicode/utf16 log/internal internal/nettrace vendor/golang.org/x/crypto/cryptobyte/asn1 vendor/golang.org/x/crypto/internal/alias -math -internal/bytealg -crypto/internal/fips140deps/cpu -internal/runtime/sys internal/runtime/atomic -internal/stringslite +internal/runtime/sys +crypto/internal/fips140deps/cpu +internal/bytealg internal/runtime/exithook +internal/stringslite internal/race -internal/sync internal/runtime/maps +internal/sync runtime -iter internal/reflectlite +sync +iter crypto/subtle weak -sync -maps slices +maps +errors +sort internal/bisect -internal/singleflight internal/testlog +internal/singleflight unique runtime/cgo -errors -sort io -path internal/oserror +strconv +path +math/rand/v2 github.com/hashicorp/golang-lru/simplelru -internal/godebug vendor/golang.org/x/net/dns/dnsmessage -math/rand/v2 -strconv -hash +syscall +internal/godebug bytes strings -syscall +hash crypto/internal/randutil -internal/saferio -hash/crc32 -hash/crc64 -crypto/internal/fips140deps/godebug -math/rand crypto +crypto/internal/fips140deps/godebug +internal/saferio reflect +math/rand +bufio +vendor/golang.org/x/text/transform net/netip +hash/crc32 +hash/crc64 crypto/internal/fips140 -net/http/internal/ascii -regexp/syntax crypto/internal/impl -html -bufio github.com/hashicorp/go-immutable-radix -vendor/golang.org/x/text/transform -crypto/internal/fips140/sha256 +net/http/internal/ascii +regexp/syntax crypto/internal/fips140/sha3 +crypto/internal/fips140/sha256 crypto/internal/fips140/sha512 crypto/tls/internal/fips140tls +html crypto/sha3 -crypto/internal/fips140hash crypto/internal/fips140/hmac +time +internal/syscall/unix +internal/syscall/execenv +crypto/internal/fips140hash crypto/internal/fips140/check crypto/internal/fips140/aes crypto/internal/fips140/nistec/fiat -crypto/internal/fips140/tls12 -crypto/internal/fips140/hkdf crypto/internal/fips140/edwards25519/field crypto/internal/fips140/bigmod -regexp +crypto/internal/fips140/hkdf +crypto/internal/fips140/tls12 crypto/internal/fips140/tls13 crypto/internal/fips140/edwards25519 -time -internal/syscall/unix -internal/syscall/execenv -crypto/internal/fips140/nistec +regexp context io/fs internal/poll +crypto/internal/fips140/nistec internal/filepathlite +os internal/fmtsort encoding/binary -os -encoding/base64 -vendor/golang.org/x/crypto/internal/poly1305 -golang.org/x/sys/unix -encoding/pem crypto/internal/sysrand fmt +path/filepath +os/signal io/ioutil internal/sysinfo -os/signal -path/filepath -net crypto/internal/entropy crypto/internal/fips140/drbg -crypto/internal/fips140only crypto/internal/fips140/aes/gcm -crypto/internal/fips140/ecdh -crypto/internal/fips140/ecdsa +crypto/internal/fips140only crypto/internal/fips140/ed25519 -crypto/internal/fips140/rsa crypto/internal/fips140/mlkem +crypto/internal/fips140/ecdh +crypto/internal/fips140/rsa +crypto/internal/fips140/ecdsa crypto/md5 crypto/rc4 +encoding/base64 +golang.org/x/sys/unix +vendor/golang.org/x/crypto/internal/poly1305 crypto/cipher -math/big -log -vendor/golang.org/x/text/unicode/norm -encoding/json -encoding/hex -mime +net +encoding/pem crypto/internal/boring -net/url -vendor/golang.org/x/net/http2/hpack -compress/flate -crypto/aes crypto/des +vendor/golang.org/x/crypto/chacha20 +crypto/aes crypto/ecdh crypto/sha512 -crypto/hmac -vendor/golang.org/x/crypto/chacha20 crypto/sha1 +crypto/hmac crypto/sha256 +math/big +encoding/json +log +compress/flate +encoding/hex +net/url +vendor/golang.org/x/text/unicode/norm +vendor/golang.org/x/crypto/chacha20poly1305 vendor/golang.org/x/text/unicode/bidi +vendor/golang.org/x/net/http2/hpack +mime mime/quotedprintable net/http/internal encoding/gob go/token text/template/parse -compress/gzip -vendor/golang.org/x/crypto/chacha20poly1305 flag +compress/gzip runtime/debug runtime/trace vendor/golang.org/x/text/secure/bidirule vendor/golang.org/x/net/idna -github.com/mattn/go-isatty testing -github.com/mattn/go-colorable text/template -github.com/fatih/color +html/template crypto/rand -encoding/asn1 -crypto/dsa crypto/elliptic crypto/internal/boring/bbig -github.com/hashicorp/go-hclog +encoding/asn1 +crypto/dsa crypto/ed25519 crypto/internal/hpke crypto/rsa vendor/golang.org/x/crypto/cryptobyte crypto/x509/pkix -html/template +github.com/mattn/go-isatty +github.com/mattn/go-colorable crypto/ecdsa -vendor/golang.org/x/net/http/httpproxy +github.com/fatih/color +github.com/hashicorp/go-hclog net/textproto +vendor/golang.org/x/net/http/httpproxy crypto/x509 vendor/golang.org/x/net/http/httpguts mime/multipart crypto/tls net/http/httptrace net/http -net/rpc github.com/armon/go-metrics +net/rpc github.com/hashicorp/go-msgpack/codec github.com/hashicorp/raft github.com/hashicorp/raft/bench @@ -940,32 +976,32 @@ === RUN TestFileSnapshotSinkImpl --- PASS: TestFileSnapshotSinkImpl (0.00s) === RUN TestFileSS_CreateSnapshotMissingParentDir -2026-03-31T06:16:11.730Z [INFO] creating new snapshot: path=/tmp/raft3140976653/raft2138142903/snapshots/3-10-1774937771730.tmp +2025-02-25T23:57:32.395Z [INFO] creating new snapshot: path=/tmp/raft1596322835/raft4096690972/snapshots/3-10-1740527852395.tmp --- PASS: TestFileSS_CreateSnapshotMissingParentDir (0.00s) === RUN TestFileSS_CreateSnapshot -2026-03-31T06:16:11.731Z [INFO] creating new snapshot: path=/tmp/raft85908770/snapshots/3-10-1774937771731.tmp -2026-03-31T06:16:11.731Z [WARN] found temporary snapshot: name=3-10-1774937771731.tmp +2025-02-25T23:57:32.395Z [INFO] creating new snapshot: path=/tmp/raft2954854270/snapshots/3-10-1740527852395.tmp +2025-02-25T23:57:32.395Z [WARN] found temporary snapshot: name=3-10-1740527852395.tmp --- PASS: TestFileSS_CreateSnapshot (0.00s) === RUN TestFileSS_CancelSnapshot -2026-03-31T06:16:11.731Z [INFO] creating new snapshot: path=/tmp/raft4091427153/snapshots/3-10-1774937771731.tmp +2025-02-25T23:57:32.396Z [INFO] creating new snapshot: path=/tmp/raft800470291/snapshots/3-10-1740527852396.tmp --- PASS: TestFileSS_CancelSnapshot (0.00s) === RUN TestFileSS_Retention -2026-03-31T06:16:11.732Z [INFO] creating new snapshot: path=/tmp/raft463726041/snapshots/3-10-1774937771732.tmp -2026-03-31T06:16:11.732Z [INFO] creating new snapshot: path=/tmp/raft463726041/snapshots/3-11-1774937771732.tmp -2026-03-31T06:16:11.732Z [INFO] creating new snapshot: path=/tmp/raft463726041/snapshots/3-12-1774937771732.tmp -2026-03-31T06:16:11.733Z [INFO] reaping snapshot: path=/tmp/raft463726041/snapshots/3-10-1774937771732 -2026-03-31T06:16:11.733Z [INFO] creating new snapshot: path=/tmp/raft463726041/snapshots/3-13-1774937771733.tmp -2026-03-31T06:16:11.733Z [INFO] reaping snapshot: path=/tmp/raft463726041/snapshots/3-11-1774937771732 -2026-03-31T06:16:11.733Z [INFO] creating new snapshot: path=/tmp/raft463726041/snapshots/3-14-1774937771733.tmp -2026-03-31T06:16:11.733Z [INFO] reaping snapshot: path=/tmp/raft463726041/snapshots/3-12-1774937771732 +2025-02-25T23:57:32.396Z [INFO] creating new snapshot: path=/tmp/raft1784104625/snapshots/3-10-1740527852396.tmp +2025-02-25T23:57:32.396Z [INFO] creating new snapshot: path=/tmp/raft1784104625/snapshots/3-11-1740527852396.tmp +2025-02-25T23:57:32.396Z [INFO] creating new snapshot: path=/tmp/raft1784104625/snapshots/3-12-1740527852396.tmp +2025-02-25T23:57:32.396Z [INFO] reaping snapshot: path=/tmp/raft1784104625/snapshots/3-10-1740527852396 +2025-02-25T23:57:32.396Z [INFO] creating new snapshot: path=/tmp/raft1784104625/snapshots/3-13-1740527852396.tmp +2025-02-25T23:57:32.396Z [INFO] reaping snapshot: path=/tmp/raft1784104625/snapshots/3-11-1740527852396 +2025-02-25T23:57:32.396Z [INFO] creating new snapshot: path=/tmp/raft1784104625/snapshots/3-14-1740527852396.tmp +2025-02-25T23:57:32.396Z [INFO] reaping snapshot: path=/tmp/raft1784104625/snapshots/3-12-1740527852396 --- PASS: TestFileSS_Retention (0.00s) === RUN TestFileSS_BadPerm --- PASS: TestFileSS_BadPerm (0.00s) === RUN TestFileSS_MissingParentDir --- PASS: TestFileSS_MissingParentDir (0.00s) === RUN TestFileSS_Ordering -2026-03-31T06:16:11.734Z [INFO] creating new snapshot: path=/tmp/raft3556000333/snapshots/5-130350-1774937771734.tmp -2026-03-31T06:16:11.735Z [INFO] creating new snapshot: path=/tmp/raft3556000333/snapshots/36-204917-1774937771735.tmp +2025-02-25T23:57:32.397Z [INFO] creating new snapshot: path=/tmp/raft544444181/snapshots/5-130350-1740527852397.tmp +2025-02-25T23:57:32.397Z [INFO] creating new snapshot: path=/tmp/raft544444181/snapshots/36-204917-1740527852397.tmp --- PASS: TestFileSS_Ordering (0.00s) === RUN TestDeferFutureSuccess --- PASS: TestDeferFutureSuccess (0.00s) @@ -1002,62 +1038,62 @@ === RUN TestEmitsLogStoreMetrics --- PASS: TestEmitsLogStoreMetrics (0.01s) === RUN TestNetworkTransport_CloseStreams ---- PASS: TestNetworkTransport_CloseStreams (0.01s) +--- PASS: TestNetworkTransport_CloseStreams (0.02s) === RUN TestNetworkTransport_StartStop --- PASS: TestNetworkTransport_StartStop (0.00s) === RUN TestNetworkTransport_Heartbeat_FastPath --- PASS: TestNetworkTransport_Heartbeat_FastPath (0.00s) === RUN TestNetworkTransport_AppendEntries ---- PASS: TestNetworkTransport_AppendEntries (0.00s) +--- PASS: TestNetworkTransport_AppendEntries (0.02s) === RUN TestNetworkTransport_AppendEntriesPipeline ---- PASS: TestNetworkTransport_AppendEntriesPipeline (0.00s) +--- PASS: TestNetworkTransport_AppendEntriesPipeline (0.01s) === RUN TestNetworkTransport_AppendEntriesPipeline_CloseStreams ---- PASS: TestNetworkTransport_AppendEntriesPipeline_CloseStreams (0.02s) +--- PASS: TestNetworkTransport_AppendEntriesPipeline_CloseStreams (0.03s) === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=0 -2026-03-31T06:16:11.836Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=1 === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=2 -2026-03-31T06:16:11.888Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" +2025-02-25T23:57:32.570Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=10 -2026-03-31T06:16:11.940Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" ---- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight (0.16s) - --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=0 (0.05s) +2025-02-25T23:57:32.622Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" +--- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight (0.18s) + --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=0 (0.07s) --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=1 (0.00s) --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=2 (0.05s) - --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=10 (0.05s) + --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=10 (0.06s) === RUN TestNetworkTransport_RequestVote +2025-02-25T23:57:32.682Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" --- PASS: TestNetworkTransport_RequestVote (0.00s) === RUN TestNetworkTransport_InstallSnapshot ---- PASS: TestNetworkTransport_InstallSnapshot (0.00s) +--- PASS: TestNetworkTransport_InstallSnapshot (0.01s) === RUN TestNetworkTransport_EncodeDecode ---- PASS: TestNetworkTransport_EncodeDecode (0.00s) +--- PASS: TestNetworkTransport_EncodeDecode (0.01s) === RUN TestNetworkTransport_EncodeDecode_AddressProvider --- PASS: TestNetworkTransport_EncodeDecode_AddressProvider (0.00s) === RUN TestNetworkTransport_PooledConn ---- PASS: TestNetworkTransport_PooledConn (0.00s) +--- PASS: TestNetworkTransport_PooledConn (0.03s) === RUN TestNetworkTransport_ListenBackoff - net_transport_test.go:839: countingWriter: 2026-03-31T06:16:11.948Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-02-25T23:57:32.730Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2026-03-31T06:16:11.953Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-02-25T23:57:32.737Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2026-03-31T06:16:11.964Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-02-25T23:57:32.749Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2026-03-31T06:16:11.985Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-02-25T23:57:32.773Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2026-03-31T06:16:12.025Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-02-25T23:57:32.817Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2026-03-31T06:16:12.106Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-02-25T23:57:32.901Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2026-03-31T06:16:12.266Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-02-25T23:57:33.065Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2026-03-31T06:16:12.587Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-02-25T23:57:33.389Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2026-03-31T06:16:13.228Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-02-25T23:57:34.033Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2026-03-31T06:16:14.231Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-02-25T23:57:35.034Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2026-03-31T06:16:15.233Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-02-25T23:57:36.037Z [ERROR] test: failed to accept connection: error="intentional error in test" --- PASS: TestNetworkTransport_ListenBackoff (4.00s) === RUN TestPeersJSON_BadConfiguration @@ -1067,2524 +1103,2569 @@ === RUN TestPeersJSON_ReadConfigJSON --- PASS: TestPeersJSON_ReadConfigJSON (0.00s) === RUN TestRaft_StartStop -2026-03-31T06:16:15.949Z [INFO] server-5f839d57-c59f-7f1f-3c2d-7593330fd8a1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5f839d57-c59f-7f1f-3c2d-7593330fd8a1 Address:5f839d57-c59f-7f1f-3c2d-7593330fd8a1}]" +2025-02-25T23:57:36.734Z [INFO] server-6a56fd6b-634a-96eb-22e1-429a5dc7e104: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6a56fd6b-634a-96eb-22e1-429a5dc7e104 Address:6a56fd6b-634a-96eb-22e1-429a5dc7e104}]" --- PASS: TestRaft_StartStop (0.00s) === RUN TestRaft_AfterShutdown -2026-03-31T06:16:15.950Z [INFO] server-b5e0161f-34a4-0e8d-bf56-a6652278082b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b5e0161f-34a4-0e8d-bf56-a6652278082b Address:b5e0161f-34a4-0e8d-bf56-a6652278082b}]" +2025-02-25T23:57:36.734Z [INFO] server-81b7e430-e6aa-babc-cbe8-2f26546daa13: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-81b7e430-e6aa-babc-cbe8-2f26546daa13 Address:81b7e430-e6aa-babc-cbe8-2f26546daa13}]" --- PASS: TestRaft_AfterShutdown (0.00s) === RUN TestRaft_LiveBootstrap -2026-03-31T06:16:15.950Z [INFO] server-5d49e3d8-4c1d-6c7b-5101-d234c85d6fb3: initial configuration: index=0 servers=[] -2026-03-31T06:16:15.950Z [INFO] server-a8e71736-85fc-d3fe-c4d3-3b4f10c9ee7f: initial configuration: index=0 servers=[] -2026-03-31T06:16:15.950Z [INFO] server-04907084-88b6-833a-049f-c616dac0f8d3: initial configuration: index=0 servers=[] -2026-03-31T06:16:15.950Z [INFO] server-5d49e3d8-4c1d-6c7b-5101-d234c85d6fb3: entering follower state: follower="Node at 5d49e3d8-4c1d-6c7b-5101-d234c85d6fb3 [Follower]" leader-address= leader-id= -2026-03-31T06:16:15.950Z [INFO] server-a8e71736-85fc-d3fe-c4d3-3b4f10c9ee7f: entering follower state: follower="Node at a8e71736-85fc-d3fe-c4d3-3b4f10c9ee7f [Follower]" leader-address= leader-id= -2026-03-31T06:16:15.950Z [INFO] server-04907084-88b6-833a-049f-c616dac0f8d3: entering follower state: follower="Node at 04907084-88b6-833a-049f-c616dac0f8d3 [Follower]" leader-address= leader-id= -2026-03-31T06:16:15.950Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:16:16.003Z [WARN] server-04907084-88b6-833a-049f-c616dac0f8d3: no known peers, aborting election -2026-03-31T06:16:16.008Z [WARN] server-5d49e3d8-4c1d-6c7b-5101-d234c85d6fb3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:16.008Z [INFO] server-5d49e3d8-4c1d-6c7b-5101-d234c85d6fb3: entering candidate state: node="Node at 5d49e3d8-4c1d-6c7b-5101-d234c85d6fb3 [Candidate]" term=2 -2026-03-31T06:16:16.008Z [INFO] server-5d49e3d8-4c1d-6c7b-5101-d234c85d6fb3: election won: term=2 tally=2 -2026-03-31T06:16:16.008Z [INFO] server-5d49e3d8-4c1d-6c7b-5101-d234c85d6fb3: entering leader state: leader="Node at 5d49e3d8-4c1d-6c7b-5101-d234c85d6fb3 [Leader]" -2026-03-31T06:16:16.008Z [INFO] server-5d49e3d8-4c1d-6c7b-5101-d234c85d6fb3: added peer, starting replication: peer=server-a8e71736-85fc-d3fe-c4d3-3b4f10c9ee7f -2026-03-31T06:16:16.008Z [INFO] server-5d49e3d8-4c1d-6c7b-5101-d234c85d6fb3: added peer, starting replication: peer=server-04907084-88b6-833a-049f-c616dac0f8d3 -2026-03-31T06:16:16.008Z [WARN] server-04907084-88b6-833a-049f-c616dac0f8d3: failed to get previous log: previous-index=1 last-index=0 error="log not found" -2026-03-31T06:16:16.008Z [WARN] server-5d49e3d8-4c1d-6c7b-5101-d234c85d6fb3: appendEntries rejected, sending older logs: peer="{Voter server-04907084-88b6-833a-049f-c616dac0f8d3 04907084-88b6-833a-049f-c616dac0f8d3}" next=1 -2026-03-31T06:16:16.008Z [WARN] server-a8e71736-85fc-d3fe-c4d3-3b4f10c9ee7f: failed to get previous log: previous-index=1 last-index=0 error="log not found" -2026-03-31T06:16:16.008Z [WARN] server-5d49e3d8-4c1d-6c7b-5101-d234c85d6fb3: appendEntries rejected, sending older logs: peer="{Voter server-a8e71736-85fc-d3fe-c4d3-3b4f10c9ee7f a8e71736-85fc-d3fe-c4d3-3b4f10c9ee7f}" next=1 -2026-03-31T06:16:16.009Z [INFO] server-5d49e3d8-4c1d-6c7b-5101-d234c85d6fb3: pipelining replication: peer="{Voter server-04907084-88b6-833a-049f-c616dac0f8d3 04907084-88b6-833a-049f-c616dac0f8d3}" -2026-03-31T06:16:16.009Z [INFO] server-5d49e3d8-4c1d-6c7b-5101-d234c85d6fb3: pipelining replication: peer="{Voter server-a8e71736-85fc-d3fe-c4d3-3b4f10c9ee7f a8e71736-85fc-d3fe-c4d3-3b4f10c9ee7f}" -2026-03-31T06:16:16.113Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:16:16.00864786 +0000 UTC m=+4.286255241 (2 nodes), 57.80008ms from start of poll, 58.22624ms from cluster start. Timeout at 2026-03-31 06:16:16.1136497 +0000 UTC m=+4.391257121, 105.00188ms after stability -2026-03-31T06:16:16.114Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:16.219Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:16.11425886 +0000 UTC m=+4.391866241 (1 nodes), 920ns from start of poll, 163.83724ms from cluster start. Timeout at 2026-03-31 06:16:16.2192633 +0000 UTC m=+4.496870681, 105.00444ms after stability -2026-03-31T06:16:16.231Z [INFO] server-5d49e3d8-4c1d-6c7b-5101-d234c85d6fb3: aborting pipeline replication: peer="{Voter server-04907084-88b6-833a-049f-c616dac0f8d3 04907084-88b6-833a-049f-c616dac0f8d3}" -2026-03-31T06:16:16.231Z [INFO] server-5d49e3d8-4c1d-6c7b-5101-d234c85d6fb3: aborting pipeline replication: peer="{Voter server-a8e71736-85fc-d3fe-c4d3-3b4f10c9ee7f a8e71736-85fc-d3fe-c4d3-3b4f10c9ee7f}" ---- PASS: TestRaft_LiveBootstrap (0.28s) +2025-02-25T23:57:36.734Z [INFO] server-1114fb22-432e-fff8-522d-febaa1e48040: initial configuration: index=0 servers=[] +2025-02-25T23:57:36.734Z [INFO] server-34ea9d41-446a-d514-b360-a1e0b374129c: initial configuration: index=0 servers=[] +2025-02-25T23:57:36.734Z [INFO] server-e99416a3-85b4-164f-00b9-f1d5d8b08ed3: initial configuration: index=0 servers=[] +2025-02-25T23:57:36.735Z [INFO] server-1114fb22-432e-fff8-522d-febaa1e48040: entering follower state: follower="Node at 1114fb22-432e-fff8-522d-febaa1e48040 [Follower]" leader-address= leader-id= +2025-02-25T23:57:36.735Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:57:36.735Z [INFO] server-34ea9d41-446a-d514-b360-a1e0b374129c: entering follower state: follower="Node at 34ea9d41-446a-d514-b360-a1e0b374129c [Follower]" leader-address= leader-id= +2025-02-25T23:57:36.735Z [INFO] server-e99416a3-85b4-164f-00b9-f1d5d8b08ed3: entering follower state: follower="Node at e99416a3-85b4-164f-00b9-f1d5d8b08ed3 [Follower]" leader-address= leader-id= +2025-02-25T23:57:36.793Z [WARN] server-34ea9d41-446a-d514-b360-a1e0b374129c: no known peers, aborting election +2025-02-25T23:57:36.809Z [WARN] server-e99416a3-85b4-164f-00b9-f1d5d8b08ed3: no known peers, aborting election +2025-02-25T23:57:36.825Z [WARN] server-1114fb22-432e-fff8-522d-febaa1e48040: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:36.825Z [INFO] server-1114fb22-432e-fff8-522d-febaa1e48040: entering candidate state: node="Node at 1114fb22-432e-fff8-522d-febaa1e48040 [Candidate]" term=2 +2025-02-25T23:57:36.826Z [INFO] server-1114fb22-432e-fff8-522d-febaa1e48040: election won: term=2 tally=2 +2025-02-25T23:57:36.826Z [INFO] server-1114fb22-432e-fff8-522d-febaa1e48040: entering leader state: leader="Node at 1114fb22-432e-fff8-522d-febaa1e48040 [Leader]" +2025-02-25T23:57:36.826Z [INFO] server-1114fb22-432e-fff8-522d-febaa1e48040: added peer, starting replication: peer=server-34ea9d41-446a-d514-b360-a1e0b374129c +2025-02-25T23:57:36.826Z [INFO] server-1114fb22-432e-fff8-522d-febaa1e48040: added peer, starting replication: peer=server-e99416a3-85b4-164f-00b9-f1d5d8b08ed3 +2025-02-25T23:57:36.826Z [WARN] server-e99416a3-85b4-164f-00b9-f1d5d8b08ed3: failed to get previous log: previous-index=1 last-index=0 error="log not found" +2025-02-25T23:57:36.826Z [WARN] server-1114fb22-432e-fff8-522d-febaa1e48040: appendEntries rejected, sending older logs: peer="{Voter server-e99416a3-85b4-164f-00b9-f1d5d8b08ed3 e99416a3-85b4-164f-00b9-f1d5d8b08ed3}" next=1 +2025-02-25T23:57:36.826Z [INFO] server-1114fb22-432e-fff8-522d-febaa1e48040: pipelining replication: peer="{Voter server-e99416a3-85b4-164f-00b9-f1d5d8b08ed3 e99416a3-85b4-164f-00b9-f1d5d8b08ed3}" +2025-02-25T23:57:36.826Z [WARN] server-34ea9d41-446a-d514-b360-a1e0b374129c: failed to get previous log: previous-index=1 last-index=0 error="log not found" +2025-02-25T23:57:36.826Z [WARN] server-1114fb22-432e-fff8-522d-febaa1e48040: appendEntries rejected, sending older logs: peer="{Voter server-34ea9d41-446a-d514-b360-a1e0b374129c 34ea9d41-446a-d514-b360-a1e0b374129c}" next=1 +2025-02-25T23:57:36.826Z [INFO] server-1114fb22-432e-fff8-522d-febaa1e48040: pipelining replication: peer="{Voter server-34ea9d41-446a-d514-b360-a1e0b374129c 34ea9d41-446a-d514-b360-a1e0b374129c}" +2025-02-25T23:57:36.933Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:57:36.826333524 +0000 UTC m=+4.435348931 (2 nodes), 91.214621ms from start of poll, 91.452062ms from cluster start. Timeout at 2025-02-25 23:57:36.931336084 +0000 UTC m=+4.540351491, 105.00256ms after stability +2025-02-25T23:57:36.933Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:37.041Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:36.93400961 +0000 UTC m=+4.543025017 (1 nodes), 1.36µs from start of poll, 199.128148ms from cluster start. Timeout at 2025-02-25 23:57:37.03901493 +0000 UTC m=+4.648030337, 105.00532ms after stability +2025-02-25T23:57:37.057Z [INFO] server-1114fb22-432e-fff8-522d-febaa1e48040: aborting pipeline replication: peer="{Voter server-34ea9d41-446a-d514-b360-a1e0b374129c 34ea9d41-446a-d514-b360-a1e0b374129c}" +2025-02-25T23:57:37.058Z [INFO] server-1114fb22-432e-fff8-522d-febaa1e48040: aborting pipeline replication: peer="{Voter server-e99416a3-85b4-164f-00b9-f1d5d8b08ed3 e99416a3-85b4-164f-00b9-f1d5d8b08ed3}" +--- PASS: TestRaft_LiveBootstrap (0.32s) === RUN TestRaft_LiveBootstrap_From_NonVoter -2026-03-31T06:16:16.231Z [INFO] server-f9e54233-cc69-420a-fc40-502fb2e6b344: initial configuration: index=0 servers=[] -2026-03-31T06:16:16.231Z [INFO] server-e13ba5b7-7243-8898-a04d-0f32510b4ab9: initial configuration: index=0 servers=[] -2026-03-31T06:16:16.231Z [INFO] server-f9e54233-cc69-420a-fc40-502fb2e6b344: entering follower state: follower="Node at f9e54233-cc69-420a-fc40-502fb2e6b344 [Follower]" leader-address= leader-id= -2026-03-31T06:16:16.232Z [INFO] server-e13ba5b7-7243-8898-a04d-0f32510b4ab9: entering follower state: follower="Node at e13ba5b7-7243-8898-a04d-0f32510b4ab9 [Follower]" leader-address= leader-id= +2025-02-25T23:57:37.058Z [INFO] server-c9de8ec9-3ce2-924a-03a8-5612966e6ff6: initial configuration: index=0 servers=[] +2025-02-25T23:57:37.058Z [INFO] server-eca17344-b2db-c6c3-6d76-168776b11b6c: initial configuration: index=0 servers=[] +2025-02-25T23:57:37.058Z [INFO] server-c9de8ec9-3ce2-924a-03a8-5612966e6ff6: entering follower state: follower="Node at c9de8ec9-3ce2-924a-03a8-5612966e6ff6 [Follower]" leader-address= leader-id= --- PASS: TestRaft_LiveBootstrap_From_NonVoter (0.00s) === RUN TestRaft_RecoverCluster_NoState -2026-03-31T06:16:16.232Z [INFO] server-dd601535-ef7b-40b8-61f3-adc489ce6c96: initial configuration: index=0 servers=[] +2025-02-25T23:57:37.058Z [INFO] server-c5028e96-9974-c051-918c-68a33338e9a7: initial configuration: index=0 servers=[] --- PASS: TestRaft_RecoverCluster_NoState (0.00s) === RUN TestRaft_RecoverCluster === RUN TestRaft_RecoverCluster/no_snapshot,_no_trailing_logs -2026-03-31T06:16:16.233Z [INFO] server-0c7a4bd6-4359-4f7e-1acb-723e029d5df4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0c7a4bd6-4359-4f7e-1acb-723e029d5df4 Address:0c7a4bd6-4359-4f7e-1acb-723e029d5df4} {Suffrage:Voter ID:server-06de34ac-9c9f-7038-9e62-39d784c0f039 Address:06de34ac-9c9f-7038-9e62-39d784c0f039} {Suffrage:Voter ID:server-55f8baf6-ef9b-9237-dbf4-58580ca5b245 Address:55f8baf6-ef9b-9237-dbf4-58580ca5b245}]" -2026-03-31T06:16:16.233Z [INFO] server-0c7a4bd6-4359-4f7e-1acb-723e029d5df4: entering follower state: follower="Node at 0c7a4bd6-4359-4f7e-1acb-723e029d5df4 [Follower]" leader-address= leader-id= -2026-03-31T06:16:16.233Z [INFO] server-06de34ac-9c9f-7038-9e62-39d784c0f039: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0c7a4bd6-4359-4f7e-1acb-723e029d5df4 Address:0c7a4bd6-4359-4f7e-1acb-723e029d5df4} {Suffrage:Voter ID:server-06de34ac-9c9f-7038-9e62-39d784c0f039 Address:06de34ac-9c9f-7038-9e62-39d784c0f039} {Suffrage:Voter ID:server-55f8baf6-ef9b-9237-dbf4-58580ca5b245 Address:55f8baf6-ef9b-9237-dbf4-58580ca5b245}]" -2026-03-31T06:16:16.233Z [INFO] server-06de34ac-9c9f-7038-9e62-39d784c0f039: entering follower state: follower="Node at 06de34ac-9c9f-7038-9e62-39d784c0f039 [Follower]" leader-address= leader-id= -2026-03-31T06:16:16.233Z [INFO] server-55f8baf6-ef9b-9237-dbf4-58580ca5b245: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0c7a4bd6-4359-4f7e-1acb-723e029d5df4 Address:0c7a4bd6-4359-4f7e-1acb-723e029d5df4} {Suffrage:Voter ID:server-06de34ac-9c9f-7038-9e62-39d784c0f039 Address:06de34ac-9c9f-7038-9e62-39d784c0f039} {Suffrage:Voter ID:server-55f8baf6-ef9b-9237-dbf4-58580ca5b245 Address:55f8baf6-ef9b-9237-dbf4-58580ca5b245}]" -2026-03-31T06:16:16.233Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:16.233Z [INFO] server-55f8baf6-ef9b-9237-dbf4-58580ca5b245: entering follower state: follower="Node at 55f8baf6-ef9b-9237-dbf4-58580ca5b245 [Follower]" leader-address= leader-id= -2026-03-31T06:16:16.307Z [WARN] server-0c7a4bd6-4359-4f7e-1acb-723e029d5df4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:16.307Z [INFO] server-0c7a4bd6-4359-4f7e-1acb-723e029d5df4: entering candidate state: node="Node at 0c7a4bd6-4359-4f7e-1acb-723e029d5df4 [Candidate]" term=2 -2026-03-31T06:16:16.307Z [INFO] server-0c7a4bd6-4359-4f7e-1acb-723e029d5df4: election won: term=2 tally=2 -2026-03-31T06:16:16.307Z [INFO] server-0c7a4bd6-4359-4f7e-1acb-723e029d5df4: entering leader state: leader="Node at 0c7a4bd6-4359-4f7e-1acb-723e029d5df4 [Leader]" -2026-03-31T06:16:16.307Z [INFO] server-0c7a4bd6-4359-4f7e-1acb-723e029d5df4: added peer, starting replication: peer=server-06de34ac-9c9f-7038-9e62-39d784c0f039 -2026-03-31T06:16:16.307Z [INFO] server-0c7a4bd6-4359-4f7e-1acb-723e029d5df4: added peer, starting replication: peer=server-55f8baf6-ef9b-9237-dbf4-58580ca5b245 -2026-03-31T06:16:16.307Z [INFO] server-0c7a4bd6-4359-4f7e-1acb-723e029d5df4: pipelining replication: peer="{Voter server-55f8baf6-ef9b-9237-dbf4-58580ca5b245 55f8baf6-ef9b-9237-dbf4-58580ca5b245}" -2026-03-31T06:16:16.307Z [INFO] server-0c7a4bd6-4359-4f7e-1acb-723e029d5df4: pipelining replication: peer="{Voter server-06de34ac-9c9f-7038-9e62-39d784c0f039 06de34ac-9c9f-7038-9e62-39d784c0f039}" -2026-03-31T06:16:16.413Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:16.30791278 +0000 UTC m=+4.585520161 (1 nodes), 74.3782ms from start of poll, 74.8186ms from cluster start. Timeout at 2026-03-31 06:16:16.41291862 +0000 UTC m=+4.690526041, 105.00588ms after stability -2026-03-31T06:16:16.413Z [INFO] server-0c7a4bd6-4359-4f7e-1acb-723e029d5df4: aborting pipeline replication: peer="{Voter server-55f8baf6-ef9b-9237-dbf4-58580ca5b245 55f8baf6-ef9b-9237-dbf4-58580ca5b245}" -2026-03-31T06:16:16.413Z [INFO] server-0c7a4bd6-4359-4f7e-1acb-723e029d5df4: aborting pipeline replication: peer="{Voter server-06de34ac-9c9f-7038-9e62-39d784c0f039 06de34ac-9c9f-7038-9e62-39d784c0f039}" -2026-03-31T06:16:16.413Z [INFO] creating new snapshot: path=/tmp/raft9000834/snapshots/2-2-1774937776413.tmp -2026-03-31T06:16:16.414Z [INFO] server-0c7a4bd6-4359-4f7e-1acb-723e029d5df4: starting restore from snapshot: id=2-2-1774937776413 last-index=2 last-term=2 size-in-bytes=1 -2026-03-31T06:16:16.414Z [INFO] server-0c7a4bd6-4359-4f7e-1acb-723e029d5df4: snapshot restore progress: id=2-2-1774937776413 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% -2026-03-31T06:16:16.414Z [INFO] server-0c7a4bd6-4359-4f7e-1acb-723e029d5df4: restored from snapshot: id=2-2-1774937776413 last-index=2 last-term=2 size-in-bytes=1 -2026-03-31T06:16:16.414Z [INFO] server-0c7a4bd6-4359-4f7e-1acb-723e029d5df4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0c7a4bd6-4359-4f7e-1acb-723e029d5df4 Address:0c7a4bd6-4359-4f7e-1acb-723e029d5df4} {Suffrage:Voter ID:server-06de34ac-9c9f-7038-9e62-39d784c0f039 Address:06de34ac-9c9f-7038-9e62-39d784c0f039} {Suffrage:Voter ID:server-55f8baf6-ef9b-9237-dbf4-58580ca5b245 Address:55f8baf6-ef9b-9237-dbf4-58580ca5b245}]" -2026-03-31T06:16:16.414Z [INFO] creating new snapshot: path=/tmp/raft2138275400/snapshots/2-2-1774937776414.tmp -2026-03-31T06:16:16.414Z [INFO] server-0c7a4bd6-4359-4f7e-1acb-723e029d5df4: entering follower state: follower="Node at 0c7a4bd6-4359-4f7e-1acb-723e029d5df4 [Follower]" leader-address= leader-id= -2026-03-31T06:16:16.415Z [INFO] server-06de34ac-9c9f-7038-9e62-39d784c0f039: starting restore from snapshot: id=2-2-1774937776414 last-index=2 last-term=2 size-in-bytes=1 -2026-03-31T06:16:16.415Z [INFO] server-06de34ac-9c9f-7038-9e62-39d784c0f039: snapshot restore progress: id=2-2-1774937776414 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% -2026-03-31T06:16:16.415Z [INFO] server-06de34ac-9c9f-7038-9e62-39d784c0f039: restored from snapshot: id=2-2-1774937776414 last-index=2 last-term=2 size-in-bytes=1 -2026-03-31T06:16:16.415Z [INFO] server-06de34ac-9c9f-7038-9e62-39d784c0f039: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0c7a4bd6-4359-4f7e-1acb-723e029d5df4 Address:0c7a4bd6-4359-4f7e-1acb-723e029d5df4} {Suffrage:Voter ID:server-06de34ac-9c9f-7038-9e62-39d784c0f039 Address:06de34ac-9c9f-7038-9e62-39d784c0f039} {Suffrage:Voter ID:server-55f8baf6-ef9b-9237-dbf4-58580ca5b245 Address:55f8baf6-ef9b-9237-dbf4-58580ca5b245}]" -2026-03-31T06:16:16.415Z [INFO] server-06de34ac-9c9f-7038-9e62-39d784c0f039: entering follower state: follower="Node at 06de34ac-9c9f-7038-9e62-39d784c0f039 [Follower]" leader-address= leader-id= -2026-03-31T06:16:16.415Z [INFO] creating new snapshot: path=/tmp/raft2579886401/snapshots/2-2-1774937776415.tmp -2026-03-31T06:16:16.415Z [INFO] server-55f8baf6-ef9b-9237-dbf4-58580ca5b245: starting restore from snapshot: id=2-2-1774937776415 last-index=2 last-term=2 size-in-bytes=1 -2026-03-31T06:16:16.416Z [INFO] server-55f8baf6-ef9b-9237-dbf4-58580ca5b245: snapshot restore progress: id=2-2-1774937776415 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% -2026-03-31T06:16:16.416Z [INFO] server-55f8baf6-ef9b-9237-dbf4-58580ca5b245: restored from snapshot: id=2-2-1774937776415 last-index=2 last-term=2 size-in-bytes=1 -2026-03-31T06:16:16.416Z [INFO] server-55f8baf6-ef9b-9237-dbf4-58580ca5b245: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0c7a4bd6-4359-4f7e-1acb-723e029d5df4 Address:0c7a4bd6-4359-4f7e-1acb-723e029d5df4} {Suffrage:Voter ID:server-06de34ac-9c9f-7038-9e62-39d784c0f039 Address:06de34ac-9c9f-7038-9e62-39d784c0f039} {Suffrage:Voter ID:server-55f8baf6-ef9b-9237-dbf4-58580ca5b245 Address:55f8baf6-ef9b-9237-dbf4-58580ca5b245}]" -2026-03-31T06:16:16.416Z [INFO] server-55f8baf6-ef9b-9237-dbf4-58580ca5b245: entering follower state: follower="Node at 55f8baf6-ef9b-9237-dbf4-58580ca5b245 [Follower]" leader-address= leader-id= -2026-03-31T06:16:16.470Z [WARN] server-0c7a4bd6-4359-4f7e-1acb-723e029d5df4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:16.470Z [INFO] server-0c7a4bd6-4359-4f7e-1acb-723e029d5df4: entering candidate state: node="Node at 0c7a4bd6-4359-4f7e-1acb-723e029d5df4 [Candidate]" term=3 -2026-03-31T06:16:16.471Z [INFO] server-0c7a4bd6-4359-4f7e-1acb-723e029d5df4: election won: term=3 tally=2 -2026-03-31T06:16:16.471Z [INFO] server-0c7a4bd6-4359-4f7e-1acb-723e029d5df4: entering leader state: leader="Node at 0c7a4bd6-4359-4f7e-1acb-723e029d5df4 [Leader]" -2026-03-31T06:16:16.471Z [INFO] server-0c7a4bd6-4359-4f7e-1acb-723e029d5df4: added peer, starting replication: peer=server-06de34ac-9c9f-7038-9e62-39d784c0f039 -2026-03-31T06:16:16.471Z [INFO] server-0c7a4bd6-4359-4f7e-1acb-723e029d5df4: added peer, starting replication: peer=server-55f8baf6-ef9b-9237-dbf4-58580ca5b245 -2026-03-31T06:16:16.471Z [INFO] server-0c7a4bd6-4359-4f7e-1acb-723e029d5df4: pipelining replication: peer="{Voter server-55f8baf6-ef9b-9237-dbf4-58580ca5b245 55f8baf6-ef9b-9237-dbf4-58580ca5b245}" -2026-03-31T06:16:16.471Z [INFO] server-0c7a4bd6-4359-4f7e-1acb-723e029d5df4: pipelining replication: peer="{Voter server-06de34ac-9c9f-7038-9e62-39d784c0f039 06de34ac-9c9f-7038-9e62-39d784c0f039}" -2026-03-31T06:16:16.732Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:16.837Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:16.73226362 +0000 UTC m=+5.009871001 (1 nodes), 38.92µs from start of poll, 499.16944ms from cluster start. Timeout at 2026-03-31 06:16:16.83727246 +0000 UTC m=+5.114879881, 105.00888ms after stability -2026-03-31T06:16:16.837Z [INFO] server-0c7a4bd6-4359-4f7e-1acb-723e029d5df4: aborting pipeline replication: peer="{Voter server-55f8baf6-ef9b-9237-dbf4-58580ca5b245 55f8baf6-ef9b-9237-dbf4-58580ca5b245}" -2026-03-31T06:16:16.837Z [INFO] server-0c7a4bd6-4359-4f7e-1acb-723e029d5df4: aborting pipeline replication: peer="{Voter server-06de34ac-9c9f-7038-9e62-39d784c0f039 06de34ac-9c9f-7038-9e62-39d784c0f039}" +2025-02-25T23:57:37.058Z [INFO] server-d9006fda-3531-13ba-8237-94a52ef6a4a1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d9006fda-3531-13ba-8237-94a52ef6a4a1 Address:d9006fda-3531-13ba-8237-94a52ef6a4a1} {Suffrage:Voter ID:server-f0c55ced-5dc5-4f4c-e375-369c6a1120fb Address:f0c55ced-5dc5-4f4c-e375-369c6a1120fb} {Suffrage:Voter ID:server-0bf47025-1fdd-729d-7126-2bc3e9731ddf Address:0bf47025-1fdd-729d-7126-2bc3e9731ddf}]" +2025-02-25T23:57:37.058Z [INFO] server-f0c55ced-5dc5-4f4c-e375-369c6a1120fb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d9006fda-3531-13ba-8237-94a52ef6a4a1 Address:d9006fda-3531-13ba-8237-94a52ef6a4a1} {Suffrage:Voter ID:server-f0c55ced-5dc5-4f4c-e375-369c6a1120fb Address:f0c55ced-5dc5-4f4c-e375-369c6a1120fb} {Suffrage:Voter ID:server-0bf47025-1fdd-729d-7126-2bc3e9731ddf Address:0bf47025-1fdd-729d-7126-2bc3e9731ddf}]" +2025-02-25T23:57:37.058Z [INFO] server-0bf47025-1fdd-729d-7126-2bc3e9731ddf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d9006fda-3531-13ba-8237-94a52ef6a4a1 Address:d9006fda-3531-13ba-8237-94a52ef6a4a1} {Suffrage:Voter ID:server-f0c55ced-5dc5-4f4c-e375-369c6a1120fb Address:f0c55ced-5dc5-4f4c-e375-369c6a1120fb} {Suffrage:Voter ID:server-0bf47025-1fdd-729d-7126-2bc3e9731ddf Address:0bf47025-1fdd-729d-7126-2bc3e9731ddf}]" +2025-02-25T23:57:37.058Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:37.058Z [INFO] server-d9006fda-3531-13ba-8237-94a52ef6a4a1: entering follower state: follower="Node at d9006fda-3531-13ba-8237-94a52ef6a4a1 [Follower]" leader-address= leader-id= +2025-02-25T23:57:37.058Z [INFO] server-f0c55ced-5dc5-4f4c-e375-369c6a1120fb: entering follower state: follower="Node at f0c55ced-5dc5-4f4c-e375-369c6a1120fb [Follower]" leader-address= leader-id= +2025-02-25T23:57:37.058Z [INFO] server-0bf47025-1fdd-729d-7126-2bc3e9731ddf: entering follower state: follower="Node at 0bf47025-1fdd-729d-7126-2bc3e9731ddf [Follower]" leader-address= leader-id= +2025-02-25T23:57:37.125Z [WARN] server-0bf47025-1fdd-729d-7126-2bc3e9731ddf: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:37.125Z [INFO] server-0bf47025-1fdd-729d-7126-2bc3e9731ddf: entering candidate state: node="Node at 0bf47025-1fdd-729d-7126-2bc3e9731ddf [Candidate]" term=2 +2025-02-25T23:57:37.125Z [INFO] server-0bf47025-1fdd-729d-7126-2bc3e9731ddf: election won: term=2 tally=2 +2025-02-25T23:57:37.125Z [INFO] server-0bf47025-1fdd-729d-7126-2bc3e9731ddf: entering leader state: leader="Node at 0bf47025-1fdd-729d-7126-2bc3e9731ddf [Leader]" +2025-02-25T23:57:37.126Z [INFO] server-0bf47025-1fdd-729d-7126-2bc3e9731ddf: added peer, starting replication: peer=server-d9006fda-3531-13ba-8237-94a52ef6a4a1 +2025-02-25T23:57:37.126Z [INFO] server-0bf47025-1fdd-729d-7126-2bc3e9731ddf: added peer, starting replication: peer=server-f0c55ced-5dc5-4f4c-e375-369c6a1120fb +2025-02-25T23:57:37.126Z [INFO] server-0bf47025-1fdd-729d-7126-2bc3e9731ddf: pipelining replication: peer="{Voter server-f0c55ced-5dc5-4f4c-e375-369c6a1120fb f0c55ced-5dc5-4f4c-e375-369c6a1120fb}" +2025-02-25T23:57:37.126Z [INFO] server-0bf47025-1fdd-729d-7126-2bc3e9731ddf: pipelining replication: peer="{Voter server-d9006fda-3531-13ba-8237-94a52ef6a4a1 d9006fda-3531-13ba-8237-94a52ef6a4a1}" +2025-02-25T23:57:37.234Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:37.126199074 +0000 UTC m=+4.735214481 (1 nodes), 67.266308ms from start of poll, 67.432229ms from cluster start. Timeout at 2025-02-25 23:57:37.231200034 +0000 UTC m=+4.840215441, 105.00096ms after stability +2025-02-25T23:57:37.234Z [INFO] server-0bf47025-1fdd-729d-7126-2bc3e9731ddf: aborting pipeline replication: peer="{Voter server-f0c55ced-5dc5-4f4c-e375-369c6a1120fb f0c55ced-5dc5-4f4c-e375-369c6a1120fb}" +2025-02-25T23:57:37.234Z [INFO] server-0bf47025-1fdd-729d-7126-2bc3e9731ddf: aborting pipeline replication: peer="{Voter server-d9006fda-3531-13ba-8237-94a52ef6a4a1 d9006fda-3531-13ba-8237-94a52ef6a4a1}" +2025-02-25T23:57:37.234Z [INFO] creating new snapshot: path=/tmp/raft885260856/snapshots/2-2-1740527857234.tmp +2025-02-25T23:57:37.234Z [INFO] server-d9006fda-3531-13ba-8237-94a52ef6a4a1: starting restore from snapshot: id=2-2-1740527857234 last-index=2 last-term=2 size-in-bytes=1 +2025-02-25T23:57:37.234Z [INFO] server-d9006fda-3531-13ba-8237-94a52ef6a4a1: snapshot restore progress: id=2-2-1740527857234 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% +2025-02-25T23:57:37.234Z [INFO] server-d9006fda-3531-13ba-8237-94a52ef6a4a1: restored from snapshot: id=2-2-1740527857234 last-index=2 last-term=2 size-in-bytes=1 +2025-02-25T23:57:37.234Z [INFO] server-d9006fda-3531-13ba-8237-94a52ef6a4a1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d9006fda-3531-13ba-8237-94a52ef6a4a1 Address:d9006fda-3531-13ba-8237-94a52ef6a4a1} {Suffrage:Voter ID:server-f0c55ced-5dc5-4f4c-e375-369c6a1120fb Address:f0c55ced-5dc5-4f4c-e375-369c6a1120fb} {Suffrage:Voter ID:server-0bf47025-1fdd-729d-7126-2bc3e9731ddf Address:0bf47025-1fdd-729d-7126-2bc3e9731ddf}]" +2025-02-25T23:57:37.234Z [INFO] creating new snapshot: path=/tmp/raft1132244904/snapshots/2-2-1740527857234.tmp +2025-02-25T23:57:37.234Z [INFO] server-f0c55ced-5dc5-4f4c-e375-369c6a1120fb: starting restore from snapshot: id=2-2-1740527857234 last-index=2 last-term=2 size-in-bytes=1 +2025-02-25T23:57:37.234Z [INFO] server-f0c55ced-5dc5-4f4c-e375-369c6a1120fb: snapshot restore progress: id=2-2-1740527857234 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% +2025-02-25T23:57:37.234Z [INFO] server-f0c55ced-5dc5-4f4c-e375-369c6a1120fb: restored from snapshot: id=2-2-1740527857234 last-index=2 last-term=2 size-in-bytes=1 +2025-02-25T23:57:37.234Z [INFO] server-f0c55ced-5dc5-4f4c-e375-369c6a1120fb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d9006fda-3531-13ba-8237-94a52ef6a4a1 Address:d9006fda-3531-13ba-8237-94a52ef6a4a1} {Suffrage:Voter ID:server-f0c55ced-5dc5-4f4c-e375-369c6a1120fb Address:f0c55ced-5dc5-4f4c-e375-369c6a1120fb} {Suffrage:Voter ID:server-0bf47025-1fdd-729d-7126-2bc3e9731ddf Address:0bf47025-1fdd-729d-7126-2bc3e9731ddf}]" +2025-02-25T23:57:37.234Z [INFO] creating new snapshot: path=/tmp/raft2344572216/snapshots/2-2-1740527857234.tmp +2025-02-25T23:57:37.235Z [INFO] server-0bf47025-1fdd-729d-7126-2bc3e9731ddf: starting restore from snapshot: id=2-2-1740527857234 last-index=2 last-term=2 size-in-bytes=1 +2025-02-25T23:57:37.235Z [INFO] server-0bf47025-1fdd-729d-7126-2bc3e9731ddf: snapshot restore progress: id=2-2-1740527857234 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% +2025-02-25T23:57:37.235Z [INFO] server-0bf47025-1fdd-729d-7126-2bc3e9731ddf: restored from snapshot: id=2-2-1740527857234 last-index=2 last-term=2 size-in-bytes=1 +2025-02-25T23:57:37.235Z [INFO] server-0bf47025-1fdd-729d-7126-2bc3e9731ddf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d9006fda-3531-13ba-8237-94a52ef6a4a1 Address:d9006fda-3531-13ba-8237-94a52ef6a4a1} {Suffrage:Voter ID:server-f0c55ced-5dc5-4f4c-e375-369c6a1120fb Address:f0c55ced-5dc5-4f4c-e375-369c6a1120fb} {Suffrage:Voter ID:server-0bf47025-1fdd-729d-7126-2bc3e9731ddf Address:0bf47025-1fdd-729d-7126-2bc3e9731ddf}]" +2025-02-25T23:57:37.235Z [INFO] server-d9006fda-3531-13ba-8237-94a52ef6a4a1: entering follower state: follower="Node at d9006fda-3531-13ba-8237-94a52ef6a4a1 [Follower]" leader-address= leader-id= +2025-02-25T23:57:37.235Z [INFO] server-f0c55ced-5dc5-4f4c-e375-369c6a1120fb: entering follower state: follower="Node at f0c55ced-5dc5-4f4c-e375-369c6a1120fb [Follower]" leader-address= leader-id= +2025-02-25T23:57:37.235Z [INFO] server-0bf47025-1fdd-729d-7126-2bc3e9731ddf: entering follower state: follower="Node at 0bf47025-1fdd-729d-7126-2bc3e9731ddf [Follower]" leader-address= leader-id= +2025-02-25T23:57:37.293Z [WARN] server-f0c55ced-5dc5-4f4c-e375-369c6a1120fb: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:37.293Z [INFO] server-f0c55ced-5dc5-4f4c-e375-369c6a1120fb: entering candidate state: node="Node at f0c55ced-5dc5-4f4c-e375-369c6a1120fb [Candidate]" term=3 +2025-02-25T23:57:37.293Z [INFO] server-f0c55ced-5dc5-4f4c-e375-369c6a1120fb: election won: term=3 tally=2 +2025-02-25T23:57:37.293Z [INFO] server-f0c55ced-5dc5-4f4c-e375-369c6a1120fb: entering leader state: leader="Node at f0c55ced-5dc5-4f4c-e375-369c6a1120fb [Leader]" +2025-02-25T23:57:37.294Z [INFO] server-f0c55ced-5dc5-4f4c-e375-369c6a1120fb: added peer, starting replication: peer=server-d9006fda-3531-13ba-8237-94a52ef6a4a1 +2025-02-25T23:57:37.294Z [INFO] server-f0c55ced-5dc5-4f4c-e375-369c6a1120fb: added peer, starting replication: peer=server-0bf47025-1fdd-729d-7126-2bc3e9731ddf +2025-02-25T23:57:37.294Z [INFO] server-f0c55ced-5dc5-4f4c-e375-369c6a1120fb: pipelining replication: peer="{Voter server-0bf47025-1fdd-729d-7126-2bc3e9731ddf 0bf47025-1fdd-729d-7126-2bc3e9731ddf}" +2025-02-25T23:57:37.294Z [INFO] server-f0c55ced-5dc5-4f4c-e375-369c6a1120fb: pipelining replication: peer="{Voter server-d9006fda-3531-13ba-8237-94a52ef6a4a1 d9006fda-3531-13ba-8237-94a52ef6a4a1}" +2025-02-25T23:57:37.553Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:37.662Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:37.553933678 +0000 UTC m=+5.162949085 (1 nodes), 29.96µs from start of poll, 495.166833ms from cluster start. Timeout at 2025-02-25 23:57:37.658937918 +0000 UTC m=+5.267953325, 105.00424ms after stability +2025-02-25T23:57:37.662Z [INFO] server-f0c55ced-5dc5-4f4c-e375-369c6a1120fb: aborting pipeline replication: peer="{Voter server-0bf47025-1fdd-729d-7126-2bc3e9731ddf 0bf47025-1fdd-729d-7126-2bc3e9731ddf}" +2025-02-25T23:57:37.662Z [INFO] server-f0c55ced-5dc5-4f4c-e375-369c6a1120fb: aborting pipeline replication: peer="{Voter server-d9006fda-3531-13ba-8237-94a52ef6a4a1 d9006fda-3531-13ba-8237-94a52ef6a4a1}" === RUN TestRaft_RecoverCluster/no_snapshot,_some_trailing_logs -2026-03-31T06:16:16.838Z [INFO] server-ff76cb51-d726-545c-4f69-56250cbe1509: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff76cb51-d726-545c-4f69-56250cbe1509 Address:ff76cb51-d726-545c-4f69-56250cbe1509} {Suffrage:Voter ID:server-5e99fa44-bf1f-a11a-e177-eed474f3a3d1 Address:5e99fa44-bf1f-a11a-e177-eed474f3a3d1} {Suffrage:Voter ID:server-a4890974-a552-9dd5-5a2f-e5a2e18ac6d7 Address:a4890974-a552-9dd5-5a2f-e5a2e18ac6d7}]" -2026-03-31T06:16:16.838Z [INFO] server-5e99fa44-bf1f-a11a-e177-eed474f3a3d1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff76cb51-d726-545c-4f69-56250cbe1509 Address:ff76cb51-d726-545c-4f69-56250cbe1509} {Suffrage:Voter ID:server-5e99fa44-bf1f-a11a-e177-eed474f3a3d1 Address:5e99fa44-bf1f-a11a-e177-eed474f3a3d1} {Suffrage:Voter ID:server-a4890974-a552-9dd5-5a2f-e5a2e18ac6d7 Address:a4890974-a552-9dd5-5a2f-e5a2e18ac6d7}]" -2026-03-31T06:16:16.838Z [INFO] server-ff76cb51-d726-545c-4f69-56250cbe1509: entering follower state: follower="Node at ff76cb51-d726-545c-4f69-56250cbe1509 [Follower]" leader-address= leader-id= -2026-03-31T06:16:16.838Z [INFO] server-5e99fa44-bf1f-a11a-e177-eed474f3a3d1: entering follower state: follower="Node at 5e99fa44-bf1f-a11a-e177-eed474f3a3d1 [Follower]" leader-address= leader-id= -2026-03-31T06:16:16.838Z [INFO] server-a4890974-a552-9dd5-5a2f-e5a2e18ac6d7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff76cb51-d726-545c-4f69-56250cbe1509 Address:ff76cb51-d726-545c-4f69-56250cbe1509} {Suffrage:Voter ID:server-5e99fa44-bf1f-a11a-e177-eed474f3a3d1 Address:5e99fa44-bf1f-a11a-e177-eed474f3a3d1} {Suffrage:Voter ID:server-a4890974-a552-9dd5-5a2f-e5a2e18ac6d7 Address:a4890974-a552-9dd5-5a2f-e5a2e18ac6d7}]" -2026-03-31T06:16:16.838Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:16.838Z [INFO] server-a4890974-a552-9dd5-5a2f-e5a2e18ac6d7: entering follower state: follower="Node at a4890974-a552-9dd5-5a2f-e5a2e18ac6d7 [Follower]" leader-address= leader-id= -2026-03-31T06:16:16.898Z [WARN] server-ff76cb51-d726-545c-4f69-56250cbe1509: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:16.898Z [INFO] server-ff76cb51-d726-545c-4f69-56250cbe1509: entering candidate state: node="Node at ff76cb51-d726-545c-4f69-56250cbe1509 [Candidate]" term=2 -2026-03-31T06:16:16.898Z [INFO] server-ff76cb51-d726-545c-4f69-56250cbe1509: election won: term=2 tally=2 -2026-03-31T06:16:16.898Z [INFO] server-ff76cb51-d726-545c-4f69-56250cbe1509: entering leader state: leader="Node at ff76cb51-d726-545c-4f69-56250cbe1509 [Leader]" -2026-03-31T06:16:16.898Z [INFO] server-ff76cb51-d726-545c-4f69-56250cbe1509: added peer, starting replication: peer=server-5e99fa44-bf1f-a11a-e177-eed474f3a3d1 -2026-03-31T06:16:16.898Z [INFO] server-ff76cb51-d726-545c-4f69-56250cbe1509: added peer, starting replication: peer=server-a4890974-a552-9dd5-5a2f-e5a2e18ac6d7 -2026-03-31T06:16:16.898Z [INFO] server-ff76cb51-d726-545c-4f69-56250cbe1509: pipelining replication: peer="{Voter server-a4890974-a552-9dd5-5a2f-e5a2e18ac6d7 a4890974-a552-9dd5-5a2f-e5a2e18ac6d7}" -2026-03-31T06:16:16.898Z [INFO] server-ff76cb51-d726-545c-4f69-56250cbe1509: pipelining replication: peer="{Voter server-5e99fa44-bf1f-a11a-e177-eed474f3a3d1 5e99fa44-bf1f-a11a-e177-eed474f3a3d1}" -2026-03-31T06:16:17.004Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:16.8987399 +0000 UTC m=+5.176347281 (1 nodes), 59.96684ms from start of poll, 60.3752ms from cluster start. Timeout at 2026-03-31 06:16:17.00374286 +0000 UTC m=+5.281350281, 105.003ms after stability -2026-03-31T06:16:17.005Z [INFO] server-ff76cb51-d726-545c-4f69-56250cbe1509: aborting pipeline replication: peer="{Voter server-a4890974-a552-9dd5-5a2f-e5a2e18ac6d7 a4890974-a552-9dd5-5a2f-e5a2e18ac6d7}" -2026-03-31T06:16:17.005Z [INFO] server-ff76cb51-d726-545c-4f69-56250cbe1509: aborting pipeline replication: peer="{Voter server-5e99fa44-bf1f-a11a-e177-eed474f3a3d1 5e99fa44-bf1f-a11a-e177-eed474f3a3d1}" -2026-03-31T06:16:17.005Z [INFO] creating new snapshot: path=/tmp/raft1344959734/snapshots/2-6-1774937777005.tmp -2026-03-31T06:16:17.006Z [INFO] server-ff76cb51-d726-545c-4f69-56250cbe1509: starting restore from snapshot: id=2-6-1774937777005 last-index=6 last-term=2 size-in-bytes=25 -2026-03-31T06:16:17.006Z [INFO] server-ff76cb51-d726-545c-4f69-56250cbe1509: snapshot restore progress: id=2-6-1774937777005 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% -2026-03-31T06:16:17.006Z [INFO] server-ff76cb51-d726-545c-4f69-56250cbe1509: restored from snapshot: id=2-6-1774937777005 last-index=6 last-term=2 size-in-bytes=25 -2026-03-31T06:16:17.006Z [INFO] server-ff76cb51-d726-545c-4f69-56250cbe1509: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff76cb51-d726-545c-4f69-56250cbe1509 Address:ff76cb51-d726-545c-4f69-56250cbe1509} {Suffrage:Voter ID:server-5e99fa44-bf1f-a11a-e177-eed474f3a3d1 Address:5e99fa44-bf1f-a11a-e177-eed474f3a3d1} {Suffrage:Voter ID:server-a4890974-a552-9dd5-5a2f-e5a2e18ac6d7 Address:a4890974-a552-9dd5-5a2f-e5a2e18ac6d7}]" -2026-03-31T06:16:17.006Z [INFO] creating new snapshot: path=/tmp/raft3239712273/snapshots/2-6-1774937777006.tmp -2026-03-31T06:16:17.006Z [INFO] server-ff76cb51-d726-545c-4f69-56250cbe1509: entering follower state: follower="Node at ff76cb51-d726-545c-4f69-56250cbe1509 [Follower]" leader-address= leader-id= -2026-03-31T06:16:17.006Z [INFO] server-5e99fa44-bf1f-a11a-e177-eed474f3a3d1: starting restore from snapshot: id=2-6-1774937777006 last-index=6 last-term=2 size-in-bytes=25 -2026-03-31T06:16:17.007Z [INFO] server-5e99fa44-bf1f-a11a-e177-eed474f3a3d1: snapshot restore progress: id=2-6-1774937777006 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% -2026-03-31T06:16:17.007Z [INFO] server-5e99fa44-bf1f-a11a-e177-eed474f3a3d1: restored from snapshot: id=2-6-1774937777006 last-index=6 last-term=2 size-in-bytes=25 -2026-03-31T06:16:17.007Z [INFO] server-5e99fa44-bf1f-a11a-e177-eed474f3a3d1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff76cb51-d726-545c-4f69-56250cbe1509 Address:ff76cb51-d726-545c-4f69-56250cbe1509} {Suffrage:Voter ID:server-5e99fa44-bf1f-a11a-e177-eed474f3a3d1 Address:5e99fa44-bf1f-a11a-e177-eed474f3a3d1} {Suffrage:Voter ID:server-a4890974-a552-9dd5-5a2f-e5a2e18ac6d7 Address:a4890974-a552-9dd5-5a2f-e5a2e18ac6d7}]" -2026-03-31T06:16:17.007Z [INFO] server-5e99fa44-bf1f-a11a-e177-eed474f3a3d1: entering follower state: follower="Node at 5e99fa44-bf1f-a11a-e177-eed474f3a3d1 [Follower]" leader-address= leader-id= -2026-03-31T06:16:17.007Z [INFO] creating new snapshot: path=/tmp/raft3948468735/snapshots/2-6-1774937777007.tmp -2026-03-31T06:16:17.007Z [INFO] server-a4890974-a552-9dd5-5a2f-e5a2e18ac6d7: starting restore from snapshot: id=2-6-1774937777007 last-index=6 last-term=2 size-in-bytes=25 -2026-03-31T06:16:17.007Z [INFO] server-a4890974-a552-9dd5-5a2f-e5a2e18ac6d7: snapshot restore progress: id=2-6-1774937777007 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% -2026-03-31T06:16:17.007Z [INFO] server-a4890974-a552-9dd5-5a2f-e5a2e18ac6d7: restored from snapshot: id=2-6-1774937777007 last-index=6 last-term=2 size-in-bytes=25 -2026-03-31T06:16:17.007Z [INFO] server-a4890974-a552-9dd5-5a2f-e5a2e18ac6d7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff76cb51-d726-545c-4f69-56250cbe1509 Address:ff76cb51-d726-545c-4f69-56250cbe1509} {Suffrage:Voter ID:server-5e99fa44-bf1f-a11a-e177-eed474f3a3d1 Address:5e99fa44-bf1f-a11a-e177-eed474f3a3d1} {Suffrage:Voter ID:server-a4890974-a552-9dd5-5a2f-e5a2e18ac6d7 Address:a4890974-a552-9dd5-5a2f-e5a2e18ac6d7}]" -2026-03-31T06:16:17.008Z [INFO] server-a4890974-a552-9dd5-5a2f-e5a2e18ac6d7: entering follower state: follower="Node at a4890974-a552-9dd5-5a2f-e5a2e18ac6d7 [Follower]" leader-address= leader-id= -2026-03-31T06:16:17.063Z [WARN] server-ff76cb51-d726-545c-4f69-56250cbe1509: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:17.063Z [INFO] server-ff76cb51-d726-545c-4f69-56250cbe1509: entering candidate state: node="Node at ff76cb51-d726-545c-4f69-56250cbe1509 [Candidate]" term=3 -2026-03-31T06:16:17.063Z [INFO] server-ff76cb51-d726-545c-4f69-56250cbe1509: election won: term=3 tally=2 -2026-03-31T06:16:17.063Z [INFO] server-ff76cb51-d726-545c-4f69-56250cbe1509: entering leader state: leader="Node at ff76cb51-d726-545c-4f69-56250cbe1509 [Leader]" -2026-03-31T06:16:17.064Z [INFO] server-ff76cb51-d726-545c-4f69-56250cbe1509: added peer, starting replication: peer=server-5e99fa44-bf1f-a11a-e177-eed474f3a3d1 -2026-03-31T06:16:17.064Z [INFO] server-ff76cb51-d726-545c-4f69-56250cbe1509: added peer, starting replication: peer=server-a4890974-a552-9dd5-5a2f-e5a2e18ac6d7 -2026-03-31T06:16:17.064Z [INFO] server-ff76cb51-d726-545c-4f69-56250cbe1509: pipelining replication: peer="{Voter server-a4890974-a552-9dd5-5a2f-e5a2e18ac6d7 a4890974-a552-9dd5-5a2f-e5a2e18ac6d7}" -2026-03-31T06:16:17.064Z [INFO] server-ff76cb51-d726-545c-4f69-56250cbe1509: pipelining replication: peer="{Voter server-5e99fa44-bf1f-a11a-e177-eed474f3a3d1 5e99fa44-bf1f-a11a-e177-eed474f3a3d1}" -2026-03-31T06:16:17.323Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:17.429Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:17.32368618 +0000 UTC m=+5.601293561 (1 nodes), 34.36µs from start of poll, 485.32148ms from cluster start. Timeout at 2026-03-31 06:16:17.4286941 +0000 UTC m=+5.706301481, 105.00792ms after stability -2026-03-31T06:16:17.429Z [INFO] server-ff76cb51-d726-545c-4f69-56250cbe1509: aborting pipeline replication: peer="{Voter server-a4890974-a552-9dd5-5a2f-e5a2e18ac6d7 a4890974-a552-9dd5-5a2f-e5a2e18ac6d7}" -2026-03-31T06:16:17.429Z [INFO] server-ff76cb51-d726-545c-4f69-56250cbe1509: aborting pipeline replication: peer="{Voter server-5e99fa44-bf1f-a11a-e177-eed474f3a3d1 5e99fa44-bf1f-a11a-e177-eed474f3a3d1}" +2025-02-25T23:57:37.662Z [INFO] server-31f19bdc-7187-8327-4f89-42b3ad3dc916: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31f19bdc-7187-8327-4f89-42b3ad3dc916 Address:31f19bdc-7187-8327-4f89-42b3ad3dc916} {Suffrage:Voter ID:server-6a3755b0-9937-7065-81d2-f3956ead5037 Address:6a3755b0-9937-7065-81d2-f3956ead5037} {Suffrage:Voter ID:server-9b2cbfe5-7e03-706f-91fa-3c71f0ef18fd Address:9b2cbfe5-7e03-706f-91fa-3c71f0ef18fd}]" +2025-02-25T23:57:37.662Z [INFO] server-6a3755b0-9937-7065-81d2-f3956ead5037: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31f19bdc-7187-8327-4f89-42b3ad3dc916 Address:31f19bdc-7187-8327-4f89-42b3ad3dc916} {Suffrage:Voter ID:server-6a3755b0-9937-7065-81d2-f3956ead5037 Address:6a3755b0-9937-7065-81d2-f3956ead5037} {Suffrage:Voter ID:server-9b2cbfe5-7e03-706f-91fa-3c71f0ef18fd Address:9b2cbfe5-7e03-706f-91fa-3c71f0ef18fd}]" +2025-02-25T23:57:37.662Z [INFO] server-9b2cbfe5-7e03-706f-91fa-3c71f0ef18fd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31f19bdc-7187-8327-4f89-42b3ad3dc916 Address:31f19bdc-7187-8327-4f89-42b3ad3dc916} {Suffrage:Voter ID:server-6a3755b0-9937-7065-81d2-f3956ead5037 Address:6a3755b0-9937-7065-81d2-f3956ead5037} {Suffrage:Voter ID:server-9b2cbfe5-7e03-706f-91fa-3c71f0ef18fd Address:9b2cbfe5-7e03-706f-91fa-3c71f0ef18fd}]" +2025-02-25T23:57:37.662Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:37.662Z [INFO] server-31f19bdc-7187-8327-4f89-42b3ad3dc916: entering follower state: follower="Node at 31f19bdc-7187-8327-4f89-42b3ad3dc916 [Follower]" leader-address= leader-id= +2025-02-25T23:57:37.662Z [INFO] server-6a3755b0-9937-7065-81d2-f3956ead5037: entering follower state: follower="Node at 6a3755b0-9937-7065-81d2-f3956ead5037 [Follower]" leader-address= leader-id= +2025-02-25T23:57:37.662Z [INFO] server-9b2cbfe5-7e03-706f-91fa-3c71f0ef18fd: entering follower state: follower="Node at 9b2cbfe5-7e03-706f-91fa-3c71f0ef18fd [Follower]" leader-address= leader-id= +2025-02-25T23:57:37.741Z [WARN] server-6a3755b0-9937-7065-81d2-f3956ead5037: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:37.741Z [INFO] server-6a3755b0-9937-7065-81d2-f3956ead5037: entering candidate state: node="Node at 6a3755b0-9937-7065-81d2-f3956ead5037 [Candidate]" term=2 +2025-02-25T23:57:37.742Z [INFO] server-6a3755b0-9937-7065-81d2-f3956ead5037: election won: term=2 tally=2 +2025-02-25T23:57:37.742Z [INFO] server-6a3755b0-9937-7065-81d2-f3956ead5037: entering leader state: leader="Node at 6a3755b0-9937-7065-81d2-f3956ead5037 [Leader]" +2025-02-25T23:57:37.742Z [INFO] server-6a3755b0-9937-7065-81d2-f3956ead5037: added peer, starting replication: peer=server-31f19bdc-7187-8327-4f89-42b3ad3dc916 +2025-02-25T23:57:37.742Z [INFO] server-6a3755b0-9937-7065-81d2-f3956ead5037: added peer, starting replication: peer=server-9b2cbfe5-7e03-706f-91fa-3c71f0ef18fd +2025-02-25T23:57:37.742Z [INFO] server-6a3755b0-9937-7065-81d2-f3956ead5037: pipelining replication: peer="{Voter server-9b2cbfe5-7e03-706f-91fa-3c71f0ef18fd 9b2cbfe5-7e03-706f-91fa-3c71f0ef18fd}" +2025-02-25T23:57:37.742Z [WARN] server-31f19bdc-7187-8327-4f89-42b3ad3dc916: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:37.742Z [INFO] server-31f19bdc-7187-8327-4f89-42b3ad3dc916: entering candidate state: node="Node at 31f19bdc-7187-8327-4f89-42b3ad3dc916 [Candidate]" term=2 +2025-02-25T23:57:37.742Z [WARN] server-9b2cbfe5-7e03-706f-91fa-3c71f0ef18fd: rejecting vote request since we have a leader: from=31f19bdc-7187-8327-4f89-42b3ad3dc916 leader=6a3755b0-9937-7065-81d2-f3956ead5037 leader-id=server-6a3755b0-9937-7065-81d2-f3956ead5037 +2025-02-25T23:57:37.742Z [INFO] server-31f19bdc-7187-8327-4f89-42b3ad3dc916: duplicate requestVote for same term: term=2 +2025-02-25T23:57:37.742Z [INFO] server-31f19bdc-7187-8327-4f89-42b3ad3dc916: entering follower state: follower="Node at 31f19bdc-7187-8327-4f89-42b3ad3dc916 [Follower]" leader-address=6a3755b0-9937-7065-81d2-f3956ead5037 leader-id=server-6a3755b0-9937-7065-81d2-f3956ead5037 +2025-02-25T23:57:37.742Z [INFO] server-6a3755b0-9937-7065-81d2-f3956ead5037: pipelining replication: peer="{Voter server-31f19bdc-7187-8327-4f89-42b3ad3dc916 31f19bdc-7187-8327-4f89-42b3ad3dc916}" +2025-02-25T23:57:37.742Z [WARN] server-6a3755b0-9937-7065-81d2-f3956ead5037: rejecting vote request since we have a leader: from=31f19bdc-7187-8327-4f89-42b3ad3dc916 leader=6a3755b0-9937-7065-81d2-f3956ead5037 leader-id=server-6a3755b0-9937-7065-81d2-f3956ead5037 +2025-02-25T23:57:37.849Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:37.742270958 +0000 UTC m=+5.351286365 (1 nodes), 79.402306ms from start of poll, 79.584347ms from cluster start. Timeout at 2025-02-25 23:57:37.847280878 +0000 UTC m=+5.456296245, 105.00988ms after stability +2025-02-25T23:57:37.850Z [INFO] server-6a3755b0-9937-7065-81d2-f3956ead5037: aborting pipeline replication: peer="{Voter server-9b2cbfe5-7e03-706f-91fa-3c71f0ef18fd 9b2cbfe5-7e03-706f-91fa-3c71f0ef18fd}" +2025-02-25T23:57:37.850Z [INFO] server-6a3755b0-9937-7065-81d2-f3956ead5037: aborting pipeline replication: peer="{Voter server-31f19bdc-7187-8327-4f89-42b3ad3dc916 31f19bdc-7187-8327-4f89-42b3ad3dc916}" +2025-02-25T23:57:37.850Z [INFO] creating new snapshot: path=/tmp/raft1440743987/snapshots/2-6-1740527857850.tmp +2025-02-25T23:57:37.850Z [INFO] server-31f19bdc-7187-8327-4f89-42b3ad3dc916: starting restore from snapshot: id=2-6-1740527857850 last-index=6 last-term=2 size-in-bytes=25 +2025-02-25T23:57:37.850Z [INFO] server-31f19bdc-7187-8327-4f89-42b3ad3dc916: snapshot restore progress: id=2-6-1740527857850 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% +2025-02-25T23:57:37.850Z [INFO] server-31f19bdc-7187-8327-4f89-42b3ad3dc916: restored from snapshot: id=2-6-1740527857850 last-index=6 last-term=2 size-in-bytes=25 +2025-02-25T23:57:37.850Z [INFO] server-31f19bdc-7187-8327-4f89-42b3ad3dc916: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31f19bdc-7187-8327-4f89-42b3ad3dc916 Address:31f19bdc-7187-8327-4f89-42b3ad3dc916} {Suffrage:Voter ID:server-6a3755b0-9937-7065-81d2-f3956ead5037 Address:6a3755b0-9937-7065-81d2-f3956ead5037} {Suffrage:Voter ID:server-9b2cbfe5-7e03-706f-91fa-3c71f0ef18fd Address:9b2cbfe5-7e03-706f-91fa-3c71f0ef18fd}]" +2025-02-25T23:57:37.850Z [INFO] creating new snapshot: path=/tmp/raft1688231414/snapshots/2-6-1740527857850.tmp +2025-02-25T23:57:37.851Z [INFO] server-6a3755b0-9937-7065-81d2-f3956ead5037: starting restore from snapshot: id=2-6-1740527857850 last-index=6 last-term=2 size-in-bytes=25 +2025-02-25T23:57:37.851Z [INFO] server-6a3755b0-9937-7065-81d2-f3956ead5037: snapshot restore progress: id=2-6-1740527857850 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% +2025-02-25T23:57:37.851Z [INFO] server-6a3755b0-9937-7065-81d2-f3956ead5037: restored from snapshot: id=2-6-1740527857850 last-index=6 last-term=2 size-in-bytes=25 +2025-02-25T23:57:37.851Z [INFO] server-6a3755b0-9937-7065-81d2-f3956ead5037: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31f19bdc-7187-8327-4f89-42b3ad3dc916 Address:31f19bdc-7187-8327-4f89-42b3ad3dc916} {Suffrage:Voter ID:server-6a3755b0-9937-7065-81d2-f3956ead5037 Address:6a3755b0-9937-7065-81d2-f3956ead5037} {Suffrage:Voter ID:server-9b2cbfe5-7e03-706f-91fa-3c71f0ef18fd Address:9b2cbfe5-7e03-706f-91fa-3c71f0ef18fd}]" +2025-02-25T23:57:37.851Z [INFO] creating new snapshot: path=/tmp/raft1463247826/snapshots/2-6-1740527857851.tmp +2025-02-25T23:57:37.851Z [INFO] server-9b2cbfe5-7e03-706f-91fa-3c71f0ef18fd: starting restore from snapshot: id=2-6-1740527857851 last-index=6 last-term=2 size-in-bytes=25 +2025-02-25T23:57:37.851Z [INFO] server-9b2cbfe5-7e03-706f-91fa-3c71f0ef18fd: snapshot restore progress: id=2-6-1740527857851 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% +2025-02-25T23:57:37.851Z [INFO] server-9b2cbfe5-7e03-706f-91fa-3c71f0ef18fd: restored from snapshot: id=2-6-1740527857851 last-index=6 last-term=2 size-in-bytes=25 +2025-02-25T23:57:37.851Z [INFO] server-9b2cbfe5-7e03-706f-91fa-3c71f0ef18fd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31f19bdc-7187-8327-4f89-42b3ad3dc916 Address:31f19bdc-7187-8327-4f89-42b3ad3dc916} {Suffrage:Voter ID:server-6a3755b0-9937-7065-81d2-f3956ead5037 Address:6a3755b0-9937-7065-81d2-f3956ead5037} {Suffrage:Voter ID:server-9b2cbfe5-7e03-706f-91fa-3c71f0ef18fd Address:9b2cbfe5-7e03-706f-91fa-3c71f0ef18fd}]" +2025-02-25T23:57:37.851Z [INFO] server-31f19bdc-7187-8327-4f89-42b3ad3dc916: entering follower state: follower="Node at 31f19bdc-7187-8327-4f89-42b3ad3dc916 [Follower]" leader-address= leader-id= +2025-02-25T23:57:37.851Z [INFO] server-6a3755b0-9937-7065-81d2-f3956ead5037: entering follower state: follower="Node at 6a3755b0-9937-7065-81d2-f3956ead5037 [Follower]" leader-address= leader-id= +2025-02-25T23:57:37.851Z [INFO] server-9b2cbfe5-7e03-706f-91fa-3c71f0ef18fd: entering follower state: follower="Node at 9b2cbfe5-7e03-706f-91fa-3c71f0ef18fd [Follower]" leader-address= leader-id= +2025-02-25T23:57:37.913Z [WARN] server-31f19bdc-7187-8327-4f89-42b3ad3dc916: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:37.913Z [INFO] server-31f19bdc-7187-8327-4f89-42b3ad3dc916: entering candidate state: node="Node at 31f19bdc-7187-8327-4f89-42b3ad3dc916 [Candidate]" term=3 +2025-02-25T23:57:37.914Z [INFO] server-31f19bdc-7187-8327-4f89-42b3ad3dc916: election won: term=3 tally=2 +2025-02-25T23:57:37.914Z [INFO] server-31f19bdc-7187-8327-4f89-42b3ad3dc916: entering leader state: leader="Node at 31f19bdc-7187-8327-4f89-42b3ad3dc916 [Leader]" +2025-02-25T23:57:37.914Z [INFO] server-31f19bdc-7187-8327-4f89-42b3ad3dc916: added peer, starting replication: peer=server-6a3755b0-9937-7065-81d2-f3956ead5037 +2025-02-25T23:57:37.914Z [INFO] server-31f19bdc-7187-8327-4f89-42b3ad3dc916: added peer, starting replication: peer=server-9b2cbfe5-7e03-706f-91fa-3c71f0ef18fd +2025-02-25T23:57:37.914Z [INFO] server-31f19bdc-7187-8327-4f89-42b3ad3dc916: pipelining replication: peer="{Voter server-9b2cbfe5-7e03-706f-91fa-3c71f0ef18fd 9b2cbfe5-7e03-706f-91fa-3c71f0ef18fd}" +2025-02-25T23:57:37.914Z [INFO] server-31f19bdc-7187-8327-4f89-42b3ad3dc916: pipelining replication: peer="{Voter server-6a3755b0-9937-7065-81d2-f3956ead5037 6a3755b0-9937-7065-81d2-f3956ead5037}" +2025-02-25T23:57:38.169Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:38.281Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:38.169918882 +0000 UTC m=+5.778934289 (1 nodes), 21.96µs from start of poll, 507.232271ms from cluster start. Timeout at 2025-02-25 23:57:38.274933882 +0000 UTC m=+5.883949289, 105.015ms after stability +2025-02-25T23:57:38.281Z [INFO] server-31f19bdc-7187-8327-4f89-42b3ad3dc916: aborting pipeline replication: peer="{Voter server-9b2cbfe5-7e03-706f-91fa-3c71f0ef18fd 9b2cbfe5-7e03-706f-91fa-3c71f0ef18fd}" +2025-02-25T23:57:38.282Z [INFO] server-31f19bdc-7187-8327-4f89-42b3ad3dc916: aborting pipeline replication: peer="{Voter server-6a3755b0-9937-7065-81d2-f3956ead5037 6a3755b0-9937-7065-81d2-f3956ead5037}" === RUN TestRaft_RecoverCluster/snapshot,_with_trailing_logs -2026-03-31T06:16:17.430Z [INFO] server-ef1867ae-6e71-dd2e-b0ac-aa540f6df96c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ef1867ae-6e71-dd2e-b0ac-aa540f6df96c Address:ef1867ae-6e71-dd2e-b0ac-aa540f6df96c} {Suffrage:Voter ID:server-ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a Address:ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a} {Suffrage:Voter ID:server-137cbdce-bc9a-ea6b-bfda-a7be81de0367 Address:137cbdce-bc9a-ea6b-bfda-a7be81de0367}]" -2026-03-31T06:16:17.430Z [INFO] server-ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ef1867ae-6e71-dd2e-b0ac-aa540f6df96c Address:ef1867ae-6e71-dd2e-b0ac-aa540f6df96c} {Suffrage:Voter ID:server-ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a Address:ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a} {Suffrage:Voter ID:server-137cbdce-bc9a-ea6b-bfda-a7be81de0367 Address:137cbdce-bc9a-ea6b-bfda-a7be81de0367}]" -2026-03-31T06:16:17.430Z [INFO] server-ef1867ae-6e71-dd2e-b0ac-aa540f6df96c: entering follower state: follower="Node at ef1867ae-6e71-dd2e-b0ac-aa540f6df96c [Follower]" leader-address= leader-id= -2026-03-31T06:16:17.430Z [INFO] server-137cbdce-bc9a-ea6b-bfda-a7be81de0367: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ef1867ae-6e71-dd2e-b0ac-aa540f6df96c Address:ef1867ae-6e71-dd2e-b0ac-aa540f6df96c} {Suffrage:Voter ID:server-ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a Address:ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a} {Suffrage:Voter ID:server-137cbdce-bc9a-ea6b-bfda-a7be81de0367 Address:137cbdce-bc9a-ea6b-bfda-a7be81de0367}]" -2026-03-31T06:16:17.430Z [INFO] server-ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a: entering follower state: follower="Node at ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a [Follower]" leader-address= leader-id= -2026-03-31T06:16:17.430Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:17.430Z [INFO] server-137cbdce-bc9a-ea6b-bfda-a7be81de0367: entering follower state: follower="Node at 137cbdce-bc9a-ea6b-bfda-a7be81de0367 [Follower]" leader-address= leader-id= -2026-03-31T06:16:17.485Z [WARN] server-ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:17.486Z [INFO] server-ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a: entering candidate state: node="Node at ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a [Candidate]" term=2 -2026-03-31T06:16:17.486Z [INFO] server-ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a: election won: term=2 tally=2 -2026-03-31T06:16:17.486Z [INFO] server-ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a: entering leader state: leader="Node at ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a [Leader]" -2026-03-31T06:16:17.486Z [INFO] server-ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a: added peer, starting replication: peer=server-ef1867ae-6e71-dd2e-b0ac-aa540f6df96c -2026-03-31T06:16:17.486Z [INFO] server-ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a: added peer, starting replication: peer=server-137cbdce-bc9a-ea6b-bfda-a7be81de0367 -2026-03-31T06:16:17.486Z [INFO] server-ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a: pipelining replication: peer="{Voter server-137cbdce-bc9a-ea6b-bfda-a7be81de0367 137cbdce-bc9a-ea6b-bfda-a7be81de0367}" -2026-03-31T06:16:17.486Z [INFO] server-ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a: pipelining replication: peer="{Voter server-ef1867ae-6e71-dd2e-b0ac-aa540f6df96c ef1867ae-6e71-dd2e-b0ac-aa540f6df96c}" -2026-03-31T06:16:17.591Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:17.48619346 +0000 UTC m=+5.763800841 (1 nodes), 55.43368ms from start of poll, 55.86948ms from cluster start. Timeout at 2026-03-31 06:16:17.59119518 +0000 UTC m=+5.868802561, 105.00172ms after stability -2026-03-31T06:16:17.594Z [INFO] server-ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a: aborting pipeline replication: peer="{Voter server-137cbdce-bc9a-ea6b-bfda-a7be81de0367 137cbdce-bc9a-ea6b-bfda-a7be81de0367}" -2026-03-31T06:16:17.594Z [INFO] server-ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a: aborting pipeline replication: peer="{Voter server-ef1867ae-6e71-dd2e-b0ac-aa540f6df96c ef1867ae-6e71-dd2e-b0ac-aa540f6df96c}" -2026-03-31T06:16:17.594Z [INFO] creating new snapshot: path=/tmp/raft3767806645/snapshots/2-27-1774937777594.tmp -2026-03-31T06:16:17.595Z [INFO] server-ef1867ae-6e71-dd2e-b0ac-aa540f6df96c: starting restore from snapshot: id=2-27-1774937777594 last-index=27 last-term=2 size-in-bytes=168 -2026-03-31T06:16:17.595Z [INFO] server-ef1867ae-6e71-dd2e-b0ac-aa540f6df96c: snapshot restore progress: id=2-27-1774937777594 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% -2026-03-31T06:16:17.596Z [INFO] server-ef1867ae-6e71-dd2e-b0ac-aa540f6df96c: restored from snapshot: id=2-27-1774937777594 last-index=27 last-term=2 size-in-bytes=168 -2026-03-31T06:16:17.596Z [INFO] server-ef1867ae-6e71-dd2e-b0ac-aa540f6df96c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ef1867ae-6e71-dd2e-b0ac-aa540f6df96c Address:ef1867ae-6e71-dd2e-b0ac-aa540f6df96c} {Suffrage:Voter ID:server-ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a Address:ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a} {Suffrage:Voter ID:server-137cbdce-bc9a-ea6b-bfda-a7be81de0367 Address:137cbdce-bc9a-ea6b-bfda-a7be81de0367}]" -2026-03-31T06:16:17.596Z [INFO] creating new snapshot: path=/tmp/raft2126451450/snapshots/2-27-1774937777596.tmp -2026-03-31T06:16:17.596Z [INFO] server-ef1867ae-6e71-dd2e-b0ac-aa540f6df96c: entering follower state: follower="Node at ef1867ae-6e71-dd2e-b0ac-aa540f6df96c [Follower]" leader-address= leader-id= -2026-03-31T06:16:17.596Z [INFO] server-ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a: starting restore from snapshot: id=2-27-1774937777596 last-index=27 last-term=2 size-in-bytes=168 -2026-03-31T06:16:17.596Z [INFO] server-ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a: snapshot restore progress: id=2-27-1774937777596 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% -2026-03-31T06:16:17.596Z [INFO] server-ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a: restored from snapshot: id=2-27-1774937777596 last-index=27 last-term=2 size-in-bytes=168 -2026-03-31T06:16:17.596Z [INFO] server-ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ef1867ae-6e71-dd2e-b0ac-aa540f6df96c Address:ef1867ae-6e71-dd2e-b0ac-aa540f6df96c} {Suffrage:Voter ID:server-ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a Address:ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a} {Suffrage:Voter ID:server-137cbdce-bc9a-ea6b-bfda-a7be81de0367 Address:137cbdce-bc9a-ea6b-bfda-a7be81de0367}]" -2026-03-31T06:16:17.596Z [INFO] creating new snapshot: path=/tmp/raft2729932002/snapshots/2-27-1774937777596.tmp -2026-03-31T06:16:17.596Z [INFO] server-ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a: entering follower state: follower="Node at ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a [Follower]" leader-address= leader-id= -2026-03-31T06:16:17.597Z [INFO] server-137cbdce-bc9a-ea6b-bfda-a7be81de0367: starting restore from snapshot: id=2-27-1774937777596 last-index=27 last-term=2 size-in-bytes=168 -2026-03-31T06:16:17.598Z [INFO] server-137cbdce-bc9a-ea6b-bfda-a7be81de0367: snapshot restore progress: id=2-27-1774937777596 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% -2026-03-31T06:16:17.598Z [INFO] server-137cbdce-bc9a-ea6b-bfda-a7be81de0367: restored from snapshot: id=2-27-1774937777596 last-index=27 last-term=2 size-in-bytes=168 -2026-03-31T06:16:17.598Z [INFO] server-137cbdce-bc9a-ea6b-bfda-a7be81de0367: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ef1867ae-6e71-dd2e-b0ac-aa540f6df96c Address:ef1867ae-6e71-dd2e-b0ac-aa540f6df96c} {Suffrage:Voter ID:server-ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a Address:ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a} {Suffrage:Voter ID:server-137cbdce-bc9a-ea6b-bfda-a7be81de0367 Address:137cbdce-bc9a-ea6b-bfda-a7be81de0367}]" -2026-03-31T06:16:17.598Z [INFO] server-137cbdce-bc9a-ea6b-bfda-a7be81de0367: entering follower state: follower="Node at 137cbdce-bc9a-ea6b-bfda-a7be81de0367 [Follower]" leader-address= leader-id= -2026-03-31T06:16:17.668Z [WARN] server-ef1867ae-6e71-dd2e-b0ac-aa540f6df96c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:17.668Z [INFO] server-ef1867ae-6e71-dd2e-b0ac-aa540f6df96c: entering candidate state: node="Node at ef1867ae-6e71-dd2e-b0ac-aa540f6df96c [Candidate]" term=3 -2026-03-31T06:16:17.669Z [INFO] server-ef1867ae-6e71-dd2e-b0ac-aa540f6df96c: election won: term=3 tally=2 -2026-03-31T06:16:17.669Z [INFO] server-ef1867ae-6e71-dd2e-b0ac-aa540f6df96c: entering leader state: leader="Node at ef1867ae-6e71-dd2e-b0ac-aa540f6df96c [Leader]" -2026-03-31T06:16:17.669Z [INFO] server-ef1867ae-6e71-dd2e-b0ac-aa540f6df96c: added peer, starting replication: peer=server-ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a -2026-03-31T06:16:17.669Z [INFO] server-ef1867ae-6e71-dd2e-b0ac-aa540f6df96c: added peer, starting replication: peer=server-137cbdce-bc9a-ea6b-bfda-a7be81de0367 -2026-03-31T06:16:17.669Z [INFO] server-ef1867ae-6e71-dd2e-b0ac-aa540f6df96c: pipelining replication: peer="{Voter server-137cbdce-bc9a-ea6b-bfda-a7be81de0367 137cbdce-bc9a-ea6b-bfda-a7be81de0367}" -2026-03-31T06:16:17.669Z [INFO] server-ef1867ae-6e71-dd2e-b0ac-aa540f6df96c: pipelining replication: peer="{Voter server-ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a}" -2026-03-31T06:16:17.914Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:18.020Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:17.91450278 +0000 UTC m=+6.192110161 (1 nodes), 30.28µs from start of poll, 484.1788ms from cluster start. Timeout at 2026-03-31 06:16:18.01950978 +0000 UTC m=+6.297117161, 105.007ms after stability -2026-03-31T06:16:18.020Z [INFO] server-ef1867ae-6e71-dd2e-b0ac-aa540f6df96c: aborting pipeline replication: peer="{Voter server-ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a ecbb11df-9dd4-f0c4-fc3a-183b65e64d1a}" -2026-03-31T06:16:18.020Z [INFO] server-ef1867ae-6e71-dd2e-b0ac-aa540f6df96c: aborting pipeline replication: peer="{Voter server-137cbdce-bc9a-ea6b-bfda-a7be81de0367 137cbdce-bc9a-ea6b-bfda-a7be81de0367}" ---- PASS: TestRaft_RecoverCluster (1.79s) - --- PASS: TestRaft_RecoverCluster/no_snapshot,_no_trailing_logs (0.61s) - --- PASS: TestRaft_RecoverCluster/no_snapshot,_some_trailing_logs (0.59s) - --- PASS: TestRaft_RecoverCluster/snapshot,_with_trailing_logs (0.59s) +2025-02-25T23:57:38.282Z [INFO] server-cbdba05e-c886-3a10-9874-35546226a8f0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cbdba05e-c886-3a10-9874-35546226a8f0 Address:cbdba05e-c886-3a10-9874-35546226a8f0} {Suffrage:Voter ID:server-ef9c96fc-a8cd-423b-989c-78e7a8d84d8c Address:ef9c96fc-a8cd-423b-989c-78e7a8d84d8c} {Suffrage:Voter ID:server-4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a Address:4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a}]" +2025-02-25T23:57:38.282Z [INFO] server-ef9c96fc-a8cd-423b-989c-78e7a8d84d8c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cbdba05e-c886-3a10-9874-35546226a8f0 Address:cbdba05e-c886-3a10-9874-35546226a8f0} {Suffrage:Voter ID:server-ef9c96fc-a8cd-423b-989c-78e7a8d84d8c Address:ef9c96fc-a8cd-423b-989c-78e7a8d84d8c} {Suffrage:Voter ID:server-4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a Address:4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a}]" +2025-02-25T23:57:38.282Z [INFO] server-4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cbdba05e-c886-3a10-9874-35546226a8f0 Address:cbdba05e-c886-3a10-9874-35546226a8f0} {Suffrage:Voter ID:server-ef9c96fc-a8cd-423b-989c-78e7a8d84d8c Address:ef9c96fc-a8cd-423b-989c-78e7a8d84d8c} {Suffrage:Voter ID:server-4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a Address:4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a}]" +2025-02-25T23:57:38.282Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:38.282Z [INFO] server-cbdba05e-c886-3a10-9874-35546226a8f0: entering follower state: follower="Node at cbdba05e-c886-3a10-9874-35546226a8f0 [Follower]" leader-address= leader-id= +2025-02-25T23:57:38.282Z [INFO] server-ef9c96fc-a8cd-423b-989c-78e7a8d84d8c: entering follower state: follower="Node at ef9c96fc-a8cd-423b-989c-78e7a8d84d8c [Follower]" leader-address= leader-id= +2025-02-25T23:57:38.282Z [INFO] server-4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a: entering follower state: follower="Node at 4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a [Follower]" leader-address= leader-id= +2025-02-25T23:57:38.349Z [WARN] server-4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:38.349Z [INFO] server-4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a: entering candidate state: node="Node at 4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a [Candidate]" term=2 +2025-02-25T23:57:38.350Z [INFO] server-4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a: election won: term=2 tally=2 +2025-02-25T23:57:38.350Z [INFO] server-4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a: entering leader state: leader="Node at 4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a [Leader]" +2025-02-25T23:57:38.350Z [INFO] server-4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a: added peer, starting replication: peer=server-cbdba05e-c886-3a10-9874-35546226a8f0 +2025-02-25T23:57:38.350Z [INFO] server-4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a: added peer, starting replication: peer=server-ef9c96fc-a8cd-423b-989c-78e7a8d84d8c +2025-02-25T23:57:38.350Z [INFO] server-4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a: pipelining replication: peer="{Voter server-ef9c96fc-a8cd-423b-989c-78e7a8d84d8c ef9c96fc-a8cd-423b-989c-78e7a8d84d8c}" +2025-02-25T23:57:38.350Z [INFO] server-4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a: pipelining replication: peer="{Voter server-cbdba05e-c886-3a10-9874-35546226a8f0 cbdba05e-c886-3a10-9874-35546226a8f0}" +2025-02-25T23:57:38.457Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:38.35020947 +0000 UTC m=+5.959224877 (1 nodes), 67.59539ms from start of poll, 67.789032ms from cluster start. Timeout at 2025-02-25 23:57:38.45521519 +0000 UTC m=+6.064230597, 105.00572ms after stability +2025-02-25T23:57:38.459Z [INFO] server-4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a: aborting pipeline replication: peer="{Voter server-ef9c96fc-a8cd-423b-989c-78e7a8d84d8c ef9c96fc-a8cd-423b-989c-78e7a8d84d8c}" +2025-02-25T23:57:38.459Z [INFO] server-4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a: aborting pipeline replication: peer="{Voter server-cbdba05e-c886-3a10-9874-35546226a8f0 cbdba05e-c886-3a10-9874-35546226a8f0}" +2025-02-25T23:57:38.459Z [INFO] creating new snapshot: path=/tmp/raft4065036488/snapshots/2-27-1740527858459.tmp +2025-02-25T23:57:38.460Z [INFO] server-cbdba05e-c886-3a10-9874-35546226a8f0: starting restore from snapshot: id=2-27-1740527858459 last-index=27 last-term=2 size-in-bytes=168 +2025-02-25T23:57:38.460Z [INFO] server-cbdba05e-c886-3a10-9874-35546226a8f0: snapshot restore progress: id=2-27-1740527858459 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% +2025-02-25T23:57:38.460Z [INFO] server-cbdba05e-c886-3a10-9874-35546226a8f0: restored from snapshot: id=2-27-1740527858459 last-index=27 last-term=2 size-in-bytes=168 +2025-02-25T23:57:38.460Z [INFO] server-cbdba05e-c886-3a10-9874-35546226a8f0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cbdba05e-c886-3a10-9874-35546226a8f0 Address:cbdba05e-c886-3a10-9874-35546226a8f0} {Suffrage:Voter ID:server-ef9c96fc-a8cd-423b-989c-78e7a8d84d8c Address:ef9c96fc-a8cd-423b-989c-78e7a8d84d8c} {Suffrage:Voter ID:server-4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a Address:4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a}]" +2025-02-25T23:57:38.460Z [INFO] creating new snapshot: path=/tmp/raft928141167/snapshots/2-27-1740527858460.tmp +2025-02-25T23:57:38.460Z [INFO] server-ef9c96fc-a8cd-423b-989c-78e7a8d84d8c: starting restore from snapshot: id=2-27-1740527858460 last-index=27 last-term=2 size-in-bytes=168 +2025-02-25T23:57:38.460Z [INFO] server-ef9c96fc-a8cd-423b-989c-78e7a8d84d8c: snapshot restore progress: id=2-27-1740527858460 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% +2025-02-25T23:57:38.460Z [INFO] server-ef9c96fc-a8cd-423b-989c-78e7a8d84d8c: restored from snapshot: id=2-27-1740527858460 last-index=27 last-term=2 size-in-bytes=168 +2025-02-25T23:57:38.460Z [INFO] server-ef9c96fc-a8cd-423b-989c-78e7a8d84d8c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cbdba05e-c886-3a10-9874-35546226a8f0 Address:cbdba05e-c886-3a10-9874-35546226a8f0} {Suffrage:Voter ID:server-ef9c96fc-a8cd-423b-989c-78e7a8d84d8c Address:ef9c96fc-a8cd-423b-989c-78e7a8d84d8c} {Suffrage:Voter ID:server-4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a Address:4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a}]" +2025-02-25T23:57:38.460Z [INFO] creating new snapshot: path=/tmp/raft3783603125/snapshots/2-27-1740527858460.tmp +2025-02-25T23:57:38.460Z [INFO] server-4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a: starting restore from snapshot: id=2-27-1740527858460 last-index=27 last-term=2 size-in-bytes=168 +2025-02-25T23:57:38.460Z [INFO] server-4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a: snapshot restore progress: id=2-27-1740527858460 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% +2025-02-25T23:57:38.460Z [INFO] server-4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a: restored from snapshot: id=2-27-1740527858460 last-index=27 last-term=2 size-in-bytes=168 +2025-02-25T23:57:38.460Z [INFO] server-4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cbdba05e-c886-3a10-9874-35546226a8f0 Address:cbdba05e-c886-3a10-9874-35546226a8f0} {Suffrage:Voter ID:server-ef9c96fc-a8cd-423b-989c-78e7a8d84d8c Address:ef9c96fc-a8cd-423b-989c-78e7a8d84d8c} {Suffrage:Voter ID:server-4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a Address:4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a}]" +2025-02-25T23:57:38.460Z [INFO] server-cbdba05e-c886-3a10-9874-35546226a8f0: entering follower state: follower="Node at cbdba05e-c886-3a10-9874-35546226a8f0 [Follower]" leader-address= leader-id= +2025-02-25T23:57:38.460Z [INFO] server-ef9c96fc-a8cd-423b-989c-78e7a8d84d8c: entering follower state: follower="Node at ef9c96fc-a8cd-423b-989c-78e7a8d84d8c [Follower]" leader-address= leader-id= +2025-02-25T23:57:38.460Z [INFO] server-4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a: entering follower state: follower="Node at 4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a [Follower]" leader-address= leader-id= +2025-02-25T23:57:38.541Z [WARN] server-cbdba05e-c886-3a10-9874-35546226a8f0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:38.541Z [INFO] server-cbdba05e-c886-3a10-9874-35546226a8f0: entering candidate state: node="Node at cbdba05e-c886-3a10-9874-35546226a8f0 [Candidate]" term=3 +2025-02-25T23:57:38.542Z [INFO] server-cbdba05e-c886-3a10-9874-35546226a8f0: election won: term=3 tally=2 +2025-02-25T23:57:38.542Z [INFO] server-cbdba05e-c886-3a10-9874-35546226a8f0: entering leader state: leader="Node at cbdba05e-c886-3a10-9874-35546226a8f0 [Leader]" +2025-02-25T23:57:38.542Z [INFO] server-cbdba05e-c886-3a10-9874-35546226a8f0: added peer, starting replication: peer=server-ef9c96fc-a8cd-423b-989c-78e7a8d84d8c +2025-02-25T23:57:38.542Z [INFO] server-cbdba05e-c886-3a10-9874-35546226a8f0: added peer, starting replication: peer=server-4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a +2025-02-25T23:57:38.542Z [INFO] server-cbdba05e-c886-3a10-9874-35546226a8f0: pipelining replication: peer="{Voter server-4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a 4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a}" +2025-02-25T23:57:38.542Z [INFO] server-cbdba05e-c886-3a10-9874-35546226a8f0: pipelining replication: peer="{Voter server-ef9c96fc-a8cd-423b-989c-78e7a8d84d8c ef9c96fc-a8cd-423b-989c-78e7a8d84d8c}" +2025-02-25T23:57:38.777Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:38.885Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:38.777992035 +0000 UTC m=+6.387007442 (1 nodes), 23.24µs from start of poll, 495.571597ms from cluster start. Timeout at 2025-02-25 23:57:38.883001635 +0000 UTC m=+6.492017042, 105.0096ms after stability +2025-02-25T23:57:38.885Z [INFO] server-cbdba05e-c886-3a10-9874-35546226a8f0: aborting pipeline replication: peer="{Voter server-ef9c96fc-a8cd-423b-989c-78e7a8d84d8c ef9c96fc-a8cd-423b-989c-78e7a8d84d8c}" +2025-02-25T23:57:38.886Z [INFO] server-cbdba05e-c886-3a10-9874-35546226a8f0: aborting pipeline replication: peer="{Voter server-4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a 4a7f0ef4-7869-97e7-d0d0-a4d4c0c9d72a}" +--- PASS: TestRaft_RecoverCluster (1.83s) + --- PASS: TestRaft_RecoverCluster/no_snapshot,_no_trailing_logs (0.60s) + --- PASS: TestRaft_RecoverCluster/no_snapshot,_some_trailing_logs (0.62s) + --- PASS: TestRaft_RecoverCluster/snapshot,_with_trailing_logs (0.60s) === RUN TestRaft_HasExistingState raft_test.go:258: Skipping unreliable test --- SKIP: TestRaft_HasExistingState (0.00s) === RUN TestRaft_SingleNode -2026-03-31T06:16:18.021Z [INFO] server-a6f4bad6-772a-627f-495e-96474a1b896a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a6f4bad6-772a-627f-495e-96474a1b896a Address:a6f4bad6-772a-627f-495e-96474a1b896a}]" -2026-03-31T06:16:18.021Z [INFO] server-a6f4bad6-772a-627f-495e-96474a1b896a: entering follower state: follower="Node at a6f4bad6-772a-627f-495e-96474a1b896a [Follower]" leader-address= leader-id= -2026-03-31T06:16:18.118Z [WARN] server-a6f4bad6-772a-627f-495e-96474a1b896a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:18.118Z [INFO] server-a6f4bad6-772a-627f-495e-96474a1b896a: entering candidate state: node="Node at a6f4bad6-772a-627f-495e-96474a1b896a [Candidate]" term=2 -2026-03-31T06:16:18.119Z [INFO] server-a6f4bad6-772a-627f-495e-96474a1b896a: election won: term=2 tally=1 -2026-03-31T06:16:18.119Z [INFO] server-a6f4bad6-772a-627f-495e-96474a1b896a: entering leader state: leader="Node at a6f4bad6-772a-627f-495e-96474a1b896a [Leader]" +2025-02-25T23:57:38.886Z [INFO] server-3f8f0968-e2b0-b25d-40e3-9f61415cf9a2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3f8f0968-e2b0-b25d-40e3-9f61415cf9a2 Address:3f8f0968-e2b0-b25d-40e3-9f61415cf9a2}]" +2025-02-25T23:57:38.886Z [INFO] server-3f8f0968-e2b0-b25d-40e3-9f61415cf9a2: entering follower state: follower="Node at 3f8f0968-e2b0-b25d-40e3-9f61415cf9a2 [Follower]" leader-address= leader-id= +2025-02-25T23:57:38.989Z [WARN] server-3f8f0968-e2b0-b25d-40e3-9f61415cf9a2: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:38.989Z [INFO] server-3f8f0968-e2b0-b25d-40e3-9f61415cf9a2: entering candidate state: node="Node at 3f8f0968-e2b0-b25d-40e3-9f61415cf9a2 [Candidate]" term=2 +2025-02-25T23:57:38.989Z [INFO] server-3f8f0968-e2b0-b25d-40e3-9f61415cf9a2: election won: term=2 tally=1 +2025-02-25T23:57:38.989Z [INFO] server-3f8f0968-e2b0-b25d-40e3-9f61415cf9a2: entering leader state: leader="Node at 3f8f0968-e2b0-b25d-40e3-9f61415cf9a2 [Leader]" --- PASS: TestRaft_SingleNode (0.10s) === RUN TestRaft_TripleNode -2026-03-31T06:16:18.119Z [INFO] server-0ab41155-c586-3942-8c55-ca0728f19689: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0ab41155-c586-3942-8c55-ca0728f19689 Address:0ab41155-c586-3942-8c55-ca0728f19689} {Suffrage:Voter ID:server-c34f092d-6a17-8044-4298-3c065a931b0d Address:c34f092d-6a17-8044-4298-3c065a931b0d} {Suffrage:Voter ID:server-d2629d42-c97d-bf78-b1a3-90202d2397f2 Address:d2629d42-c97d-bf78-b1a3-90202d2397f2}]" -2026-03-31T06:16:18.120Z [INFO] server-c34f092d-6a17-8044-4298-3c065a931b0d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0ab41155-c586-3942-8c55-ca0728f19689 Address:0ab41155-c586-3942-8c55-ca0728f19689} {Suffrage:Voter ID:server-c34f092d-6a17-8044-4298-3c065a931b0d Address:c34f092d-6a17-8044-4298-3c065a931b0d} {Suffrage:Voter ID:server-d2629d42-c97d-bf78-b1a3-90202d2397f2 Address:d2629d42-c97d-bf78-b1a3-90202d2397f2}]" -2026-03-31T06:16:18.120Z [INFO] server-0ab41155-c586-3942-8c55-ca0728f19689: entering follower state: follower="Node at 0ab41155-c586-3942-8c55-ca0728f19689 [Follower]" leader-address= leader-id= -2026-03-31T06:16:18.120Z [INFO] server-d2629d42-c97d-bf78-b1a3-90202d2397f2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0ab41155-c586-3942-8c55-ca0728f19689 Address:0ab41155-c586-3942-8c55-ca0728f19689} {Suffrage:Voter ID:server-c34f092d-6a17-8044-4298-3c065a931b0d Address:c34f092d-6a17-8044-4298-3c065a931b0d} {Suffrage:Voter ID:server-d2629d42-c97d-bf78-b1a3-90202d2397f2 Address:d2629d42-c97d-bf78-b1a3-90202d2397f2}]" -2026-03-31T06:16:18.120Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:16:18.120Z [INFO] server-c34f092d-6a17-8044-4298-3c065a931b0d: entering follower state: follower="Node at c34f092d-6a17-8044-4298-3c065a931b0d [Follower]" leader-address= leader-id= -2026-03-31T06:16:18.120Z [INFO] server-d2629d42-c97d-bf78-b1a3-90202d2397f2: entering follower state: follower="Node at d2629d42-c97d-bf78-b1a3-90202d2397f2 [Follower]" leader-address= leader-id= -2026-03-31T06:16:18.189Z [WARN] server-0ab41155-c586-3942-8c55-ca0728f19689: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:18.189Z [INFO] server-0ab41155-c586-3942-8c55-ca0728f19689: entering candidate state: node="Node at 0ab41155-c586-3942-8c55-ca0728f19689 [Candidate]" term=2 -2026-03-31T06:16:18.189Z [INFO] server-0ab41155-c586-3942-8c55-ca0728f19689: election won: term=2 tally=2 -2026-03-31T06:16:18.189Z [INFO] server-0ab41155-c586-3942-8c55-ca0728f19689: entering leader state: leader="Node at 0ab41155-c586-3942-8c55-ca0728f19689 [Leader]" -2026-03-31T06:16:18.189Z [INFO] server-0ab41155-c586-3942-8c55-ca0728f19689: added peer, starting replication: peer=server-c34f092d-6a17-8044-4298-3c065a931b0d -2026-03-31T06:16:18.189Z [INFO] server-0ab41155-c586-3942-8c55-ca0728f19689: added peer, starting replication: peer=server-d2629d42-c97d-bf78-b1a3-90202d2397f2 -2026-03-31T06:16:18.189Z [INFO] server-0ab41155-c586-3942-8c55-ca0728f19689: pipelining replication: peer="{Voter server-c34f092d-6a17-8044-4298-3c065a931b0d c34f092d-6a17-8044-4298-3c065a931b0d}" -2026-03-31T06:16:18.189Z [INFO] server-0ab41155-c586-3942-8c55-ca0728f19689: pipelining replication: peer="{Voter server-d2629d42-c97d-bf78-b1a3-90202d2397f2 d2629d42-c97d-bf78-b1a3-90202d2397f2}" -2026-03-31T06:16:18.294Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:16:18.1896047 +0000 UTC m=+6.467212081 (2 nodes), 69.4228ms from start of poll, 69.82196ms from cluster start. Timeout at 2026-03-31 06:16:18.2946061 +0000 UTC m=+6.572213481, 105.0014ms after stability -2026-03-31T06:16:18.294Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:18.400Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:18.29493138 +0000 UTC m=+6.572538761 (1 nodes), 560ns from start of poll, 175.14864ms from cluster start. Timeout at 2026-03-31 06:16:18.39993922 +0000 UTC m=+6.677546641, 105.00788ms after stability -2026-03-31T06:16:18.411Z [INFO] server-0ab41155-c586-3942-8c55-ca0728f19689: aborting pipeline replication: peer="{Voter server-d2629d42-c97d-bf78-b1a3-90202d2397f2 d2629d42-c97d-bf78-b1a3-90202d2397f2}" -2026-03-31T06:16:18.411Z [INFO] server-0ab41155-c586-3942-8c55-ca0728f19689: aborting pipeline replication: peer="{Voter server-c34f092d-6a17-8044-4298-3c065a931b0d c34f092d-6a17-8044-4298-3c065a931b0d}" -2026-03-31T06:16:18.912Z [ERROR] server-0ab41155-c586-3942-8c55-ca0728f19689: failed to heartbeat to: peer=d2629d42-c97d-bf78-b1a3-90202d2397f2 backoff time=10ms error="command timed out" ---- PASS: TestRaft_TripleNode (0.79s) +2025-02-25T23:57:38.990Z [INFO] server-a5dd9044-c75c-6954-bac8-e40c369a757a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a5dd9044-c75c-6954-bac8-e40c369a757a Address:a5dd9044-c75c-6954-bac8-e40c369a757a} {Suffrage:Voter ID:server-5776c4fb-c034-efea-f1ab-86de8e8c1fac Address:5776c4fb-c034-efea-f1ab-86de8e8c1fac} {Suffrage:Voter ID:server-3a668444-626d-e452-0a38-2e4c2a47933f Address:3a668444-626d-e452-0a38-2e4c2a47933f}]" +2025-02-25T23:57:38.990Z [INFO] server-5776c4fb-c034-efea-f1ab-86de8e8c1fac: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a5dd9044-c75c-6954-bac8-e40c369a757a Address:a5dd9044-c75c-6954-bac8-e40c369a757a} {Suffrage:Voter ID:server-5776c4fb-c034-efea-f1ab-86de8e8c1fac Address:5776c4fb-c034-efea-f1ab-86de8e8c1fac} {Suffrage:Voter ID:server-3a668444-626d-e452-0a38-2e4c2a47933f Address:3a668444-626d-e452-0a38-2e4c2a47933f}]" +2025-02-25T23:57:38.990Z [INFO] server-3a668444-626d-e452-0a38-2e4c2a47933f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a5dd9044-c75c-6954-bac8-e40c369a757a Address:a5dd9044-c75c-6954-bac8-e40c369a757a} {Suffrage:Voter ID:server-5776c4fb-c034-efea-f1ab-86de8e8c1fac Address:5776c4fb-c034-efea-f1ab-86de8e8c1fac} {Suffrage:Voter ID:server-3a668444-626d-e452-0a38-2e4c2a47933f Address:3a668444-626d-e452-0a38-2e4c2a47933f}]" +2025-02-25T23:57:38.990Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:57:38.990Z [INFO] server-a5dd9044-c75c-6954-bac8-e40c369a757a: entering follower state: follower="Node at a5dd9044-c75c-6954-bac8-e40c369a757a [Follower]" leader-address= leader-id= +2025-02-25T23:57:38.990Z [INFO] server-5776c4fb-c034-efea-f1ab-86de8e8c1fac: entering follower state: follower="Node at 5776c4fb-c034-efea-f1ab-86de8e8c1fac [Follower]" leader-address= leader-id= +2025-02-25T23:57:38.990Z [INFO] server-3a668444-626d-e452-0a38-2e4c2a47933f: entering follower state: follower="Node at 3a668444-626d-e452-0a38-2e4c2a47933f [Follower]" leader-address= leader-id= +2025-02-25T23:57:39.053Z [WARN] server-5776c4fb-c034-efea-f1ab-86de8e8c1fac: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:39.053Z [INFO] server-5776c4fb-c034-efea-f1ab-86de8e8c1fac: entering candidate state: node="Node at 5776c4fb-c034-efea-f1ab-86de8e8c1fac [Candidate]" term=2 +2025-02-25T23:57:39.053Z [INFO] server-5776c4fb-c034-efea-f1ab-86de8e8c1fac: election won: term=2 tally=2 +2025-02-25T23:57:39.053Z [INFO] server-5776c4fb-c034-efea-f1ab-86de8e8c1fac: entering leader state: leader="Node at 5776c4fb-c034-efea-f1ab-86de8e8c1fac [Leader]" +2025-02-25T23:57:39.054Z [INFO] server-5776c4fb-c034-efea-f1ab-86de8e8c1fac: added peer, starting replication: peer=server-a5dd9044-c75c-6954-bac8-e40c369a757a +2025-02-25T23:57:39.054Z [INFO] server-5776c4fb-c034-efea-f1ab-86de8e8c1fac: added peer, starting replication: peer=server-3a668444-626d-e452-0a38-2e4c2a47933f +2025-02-25T23:57:39.054Z [INFO] server-5776c4fb-c034-efea-f1ab-86de8e8c1fac: pipelining replication: peer="{Voter server-3a668444-626d-e452-0a38-2e4c2a47933f 3a668444-626d-e452-0a38-2e4c2a47933f}" +2025-02-25T23:57:39.054Z [INFO] server-5776c4fb-c034-efea-f1ab-86de8e8c1fac: pipelining replication: peer="{Voter server-a5dd9044-c75c-6954-bac8-e40c369a757a a5dd9044-c75c-6954-bac8-e40c369a757a}" +2025-02-25T23:57:39.161Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:57:39.054196994 +0000 UTC m=+6.663212401 (2 nodes), 63.727886ms from start of poll, 63.896647ms from cluster start. Timeout at 2025-02-25 23:57:39.159200594 +0000 UTC m=+6.768216001, 105.0036ms after stability +2025-02-25T23:57:39.161Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:39.269Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:39.1619346 +0000 UTC m=+6.770950007 (1 nodes), 1.2µs from start of poll, 171.634253ms from cluster start. Timeout at 2025-02-25 23:57:39.26694184 +0000 UTC m=+6.875957247, 105.00724ms after stability +2025-02-25T23:57:39.285Z [INFO] server-5776c4fb-c034-efea-f1ab-86de8e8c1fac: aborting pipeline replication: peer="{Voter server-3a668444-626d-e452-0a38-2e4c2a47933f 3a668444-626d-e452-0a38-2e4c2a47933f}" +2025-02-25T23:57:39.286Z [INFO] server-5776c4fb-c034-efea-f1ab-86de8e8c1fac: aborting pipeline replication: peer="{Voter server-a5dd9044-c75c-6954-bac8-e40c369a757a a5dd9044-c75c-6954-bac8-e40c369a757a}" +--- PASS: TestRaft_TripleNode (0.30s) === RUN TestRaft_LeaderFail -2026-03-31T06:16:18.913Z [INFO] server-7b65268c-9b33-f65a-0ffa-685844701264: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b65268c-9b33-f65a-0ffa-685844701264 Address:7b65268c-9b33-f65a-0ffa-685844701264} {Suffrage:Voter ID:server-18df5993-7078-67ed-d008-2b4a05b6a459 Address:18df5993-7078-67ed-d008-2b4a05b6a459} {Suffrage:Voter ID:server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5 Address:896e7a91-d6dc-d4f3-7117-3c7d045b95a5}]" -2026-03-31T06:16:18.913Z [INFO] server-18df5993-7078-67ed-d008-2b4a05b6a459: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b65268c-9b33-f65a-0ffa-685844701264 Address:7b65268c-9b33-f65a-0ffa-685844701264} {Suffrage:Voter ID:server-18df5993-7078-67ed-d008-2b4a05b6a459 Address:18df5993-7078-67ed-d008-2b4a05b6a459} {Suffrage:Voter ID:server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5 Address:896e7a91-d6dc-d4f3-7117-3c7d045b95a5}]" -2026-03-31T06:16:18.913Z [INFO] server-7b65268c-9b33-f65a-0ffa-685844701264: entering follower state: follower="Node at 7b65268c-9b33-f65a-0ffa-685844701264 [Follower]" leader-address= leader-id= -2026-03-31T06:16:18.913Z [INFO] server-18df5993-7078-67ed-d008-2b4a05b6a459: entering follower state: follower="Node at 18df5993-7078-67ed-d008-2b4a05b6a459 [Follower]" leader-address= leader-id= -2026-03-31T06:16:18.913Z [INFO] server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b65268c-9b33-f65a-0ffa-685844701264 Address:7b65268c-9b33-f65a-0ffa-685844701264} {Suffrage:Voter ID:server-18df5993-7078-67ed-d008-2b4a05b6a459 Address:18df5993-7078-67ed-d008-2b4a05b6a459} {Suffrage:Voter ID:server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5 Address:896e7a91-d6dc-d4f3-7117-3c7d045b95a5}]" -2026-03-31T06:16:18.913Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:16:18.913Z [INFO] server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5: entering follower state: follower="Node at 896e7a91-d6dc-d4f3-7117-3c7d045b95a5 [Follower]" leader-address= leader-id= -2026-03-31T06:16:18.980Z [WARN] server-7b65268c-9b33-f65a-0ffa-685844701264: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:18.981Z [INFO] server-7b65268c-9b33-f65a-0ffa-685844701264: entering candidate state: node="Node at 7b65268c-9b33-f65a-0ffa-685844701264 [Candidate]" term=2 -2026-03-31T06:16:18.981Z [INFO] server-7b65268c-9b33-f65a-0ffa-685844701264: election won: term=2 tally=2 -2026-03-31T06:16:18.981Z [INFO] server-7b65268c-9b33-f65a-0ffa-685844701264: entering leader state: leader="Node at 7b65268c-9b33-f65a-0ffa-685844701264 [Leader]" -2026-03-31T06:16:18.981Z [INFO] server-7b65268c-9b33-f65a-0ffa-685844701264: added peer, starting replication: peer=server-18df5993-7078-67ed-d008-2b4a05b6a459 -2026-03-31T06:16:18.981Z [INFO] server-7b65268c-9b33-f65a-0ffa-685844701264: added peer, starting replication: peer=server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5 -2026-03-31T06:16:18.981Z [INFO] server-7b65268c-9b33-f65a-0ffa-685844701264: pipelining replication: peer="{Voter server-18df5993-7078-67ed-d008-2b4a05b6a459 18df5993-7078-67ed-d008-2b4a05b6a459}" -2026-03-31T06:16:18.981Z [INFO] server-7b65268c-9b33-f65a-0ffa-685844701264: pipelining replication: peer="{Voter server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5 896e7a91-d6dc-d4f3-7117-3c7d045b95a5}" -2026-03-31T06:16:19.086Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:16:18.98124702 +0000 UTC m=+7.258854401 (2 nodes), 67.81ms from start of poll, 68.20088ms from cluster start. Timeout at 2026-03-31 06:16:19.0862491 +0000 UTC m=+7.363856481, 105.00208ms after stability -2026-03-31T06:16:19.086Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:19.192Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:19.0868199 +0000 UTC m=+7.364427281 (1 nodes), 2.76µs from start of poll, 173.77376ms from cluster start. Timeout at 2026-03-31 06:16:19.19182466 +0000 UTC m=+7.469432041, 105.00476ms after stability - raft_test.go:378: [INFO] Disconnecting Node at 7b65268c-9b33-f65a-0ffa-685844701264 [Leader] -2026-03-31T06:16:19.207Z [ERROR] server-7b65268c-9b33-f65a-0ffa-685844701264: failed to heartbeat to: peer=18df5993-7078-67ed-d008-2b4a05b6a459 backoff time=10ms error="failed to connect to peer: 18df5993-7078-67ed-d008-2b4a05b6a459" -2026-03-31T06:16:19.207Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:19.208Z [ERROR] server-7b65268c-9b33-f65a-0ffa-685844701264: failed to pipeline appendEntries: peer="{Voter server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5 896e7a91-d6dc-d4f3-7117-3c7d045b95a5}" error="append pipeline closed" -2026-03-31T06:16:19.208Z [INFO] server-7b65268c-9b33-f65a-0ffa-685844701264: aborting pipeline replication: peer="{Voter server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5 896e7a91-d6dc-d4f3-7117-3c7d045b95a5}" -2026-03-31T06:16:19.208Z [ERROR] server-7b65268c-9b33-f65a-0ffa-685844701264: failed to heartbeat to: peer=896e7a91-d6dc-d4f3-7117-3c7d045b95a5 backoff time=10ms error="failed to connect to peer: 896e7a91-d6dc-d4f3-7117-3c7d045b95a5" -2026-03-31T06:16:19.209Z [ERROR] server-7b65268c-9b33-f65a-0ffa-685844701264: failed to pipeline appendEntries: peer="{Voter server-18df5993-7078-67ed-d008-2b4a05b6a459 18df5993-7078-67ed-d008-2b4a05b6a459}" error="append pipeline closed" -2026-03-31T06:16:19.209Z [INFO] server-7b65268c-9b33-f65a-0ffa-685844701264: aborting pipeline replication: peer="{Voter server-18df5993-7078-67ed-d008-2b4a05b6a459 18df5993-7078-67ed-d008-2b4a05b6a459}" -2026-03-31T06:16:19.218Z [ERROR] server-7b65268c-9b33-f65a-0ffa-685844701264: failed to appendEntries to: peer="{Voter server-18df5993-7078-67ed-d008-2b4a05b6a459 18df5993-7078-67ed-d008-2b4a05b6a459}" error="failed to connect to peer: 18df5993-7078-67ed-d008-2b4a05b6a459" -2026-03-31T06:16:19.218Z [ERROR] server-7b65268c-9b33-f65a-0ffa-685844701264: failed to appendEntries to: peer="{Voter server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5 896e7a91-d6dc-d4f3-7117-3c7d045b95a5}" error="failed to connect to peer: 896e7a91-d6dc-d4f3-7117-3c7d045b95a5" -2026-03-31T06:16:19.223Z [ERROR] server-7b65268c-9b33-f65a-0ffa-685844701264: failed to heartbeat to: peer=18df5993-7078-67ed-d008-2b4a05b6a459 backoff time=10ms error="failed to connect to peer: 18df5993-7078-67ed-d008-2b4a05b6a459" -2026-03-31T06:16:19.226Z [ERROR] server-7b65268c-9b33-f65a-0ffa-685844701264: failed to heartbeat to: peer=896e7a91-d6dc-d4f3-7117-3c7d045b95a5 backoff time=10ms error="failed to connect to peer: 896e7a91-d6dc-d4f3-7117-3c7d045b95a5" -2026-03-31T06:16:19.234Z [ERROR] server-7b65268c-9b33-f65a-0ffa-685844701264: failed to appendEntries to: peer="{Voter server-18df5993-7078-67ed-d008-2b4a05b6a459 18df5993-7078-67ed-d008-2b4a05b6a459}" error="failed to connect to peer: 18df5993-7078-67ed-d008-2b4a05b6a459" -2026-03-31T06:16:19.235Z [ERROR] server-7b65268c-9b33-f65a-0ffa-685844701264: failed to appendEntries to: peer="{Voter server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5 896e7a91-d6dc-d4f3-7117-3c7d045b95a5}" error="failed to connect to peer: 896e7a91-d6dc-d4f3-7117-3c7d045b95a5" -2026-03-31T06:16:19.242Z [ERROR] server-7b65268c-9b33-f65a-0ffa-685844701264: failed to heartbeat to: peer=18df5993-7078-67ed-d008-2b4a05b6a459 backoff time=10ms error="failed to connect to peer: 18df5993-7078-67ed-d008-2b4a05b6a459" -2026-03-31T06:16:19.244Z [ERROR] server-7b65268c-9b33-f65a-0ffa-685844701264: failed to heartbeat to: peer=896e7a91-d6dc-d4f3-7117-3c7d045b95a5 backoff time=10ms error="failed to connect to peer: 896e7a91-d6dc-d4f3-7117-3c7d045b95a5" -2026-03-31T06:16:19.250Z [WARN] server-7b65268c-9b33-f65a-0ffa-685844701264: failed to contact: server-id=server-18df5993-7078-67ed-d008-2b4a05b6a459 time=50.37584ms -2026-03-31T06:16:19.251Z [ERROR] server-7b65268c-9b33-f65a-0ffa-685844701264: failed to appendEntries to: peer="{Voter server-18df5993-7078-67ed-d008-2b4a05b6a459 18df5993-7078-67ed-d008-2b4a05b6a459}" error="failed to connect to peer: 18df5993-7078-67ed-d008-2b4a05b6a459" -2026-03-31T06:16:19.255Z [ERROR] server-7b65268c-9b33-f65a-0ffa-685844701264: failed to appendEntries to: peer="{Voter server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5 896e7a91-d6dc-d4f3-7117-3c7d045b95a5}" error="failed to connect to peer: 896e7a91-d6dc-d4f3-7117-3c7d045b95a5" -2026-03-31T06:16:19.260Z [ERROR] server-7b65268c-9b33-f65a-0ffa-685844701264: failed to heartbeat to: peer=896e7a91-d6dc-d4f3-7117-3c7d045b95a5 backoff time=20ms error="failed to connect to peer: 896e7a91-d6dc-d4f3-7117-3c7d045b95a5" -2026-03-31T06:16:19.260Z [ERROR] server-7b65268c-9b33-f65a-0ffa-685844701264: failed to heartbeat to: peer=18df5993-7078-67ed-d008-2b4a05b6a459 backoff time=20ms error="failed to connect to peer: 18df5993-7078-67ed-d008-2b4a05b6a459" -2026-03-31T06:16:19.261Z [WARN] server-7b65268c-9b33-f65a-0ffa-685844701264: failed to contact: server-id=server-18df5993-7078-67ed-d008-2b4a05b6a459 time=61.29244ms -2026-03-31T06:16:19.261Z [WARN] server-7b65268c-9b33-f65a-0ffa-685844701264: failed to contact: server-id=server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5 time=59.07168ms -2026-03-31T06:16:19.261Z [WARN] server-7b65268c-9b33-f65a-0ffa-685844701264: failed to contact quorum of nodes, stepping down -2026-03-31T06:16:19.261Z [INFO] server-7b65268c-9b33-f65a-0ffa-685844701264: entering follower state: follower="Node at 7b65268c-9b33-f65a-0ffa-685844701264 [Follower]" leader-address= leader-id= -2026-03-31T06:16:19.281Z [ERROR] server-7b65268c-9b33-f65a-0ffa-685844701264: failed to appendEntries to: peer="{Voter server-18df5993-7078-67ed-d008-2b4a05b6a459 18df5993-7078-67ed-d008-2b4a05b6a459}" error="failed to connect to peer: 18df5993-7078-67ed-d008-2b4a05b6a459" -2026-03-31T06:16:19.283Z [WARN] server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5: heartbeat timeout reached, starting election: last-leader-addr=7b65268c-9b33-f65a-0ffa-685844701264 last-leader-id=server-7b65268c-9b33-f65a-0ffa-685844701264 -2026-03-31T06:16:19.283Z [INFO] server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5: entering candidate state: node="Node at 896e7a91-d6dc-d4f3-7117-3c7d045b95a5 [Candidate]" term=3 -2026-03-31T06:16:19.283Z [WARN] server-18df5993-7078-67ed-d008-2b4a05b6a459: rejecting vote request since we have a leader: from=896e7a91-d6dc-d4f3-7117-3c7d045b95a5 leader=7b65268c-9b33-f65a-0ffa-685844701264 leader-id=server-7b65268c-9b33-f65a-0ffa-685844701264 -2026-03-31T06:16:19.283Z [ERROR] server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5: failed to make requestVote RPC: target="{Voter server-7b65268c-9b33-f65a-0ffa-685844701264 7b65268c-9b33-f65a-0ffa-685844701264}" error="failed to connect to peer: 7b65268c-9b33-f65a-0ffa-685844701264" term=3 -2026-03-31T06:16:19.322Z [WARN] server-18df5993-7078-67ed-d008-2b4a05b6a459: heartbeat timeout reached, starting election: last-leader-addr=7b65268c-9b33-f65a-0ffa-685844701264 last-leader-id=server-7b65268c-9b33-f65a-0ffa-685844701264 -2026-03-31T06:16:19.322Z [INFO] server-18df5993-7078-67ed-d008-2b4a05b6a459: entering candidate state: node="Node at 18df5993-7078-67ed-d008-2b4a05b6a459 [Candidate]" term=3 -2026-03-31T06:16:19.322Z [INFO] server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5: duplicate requestVote for same term: term=3 -2026-03-31T06:16:19.322Z [ERROR] server-18df5993-7078-67ed-d008-2b4a05b6a459: failed to make requestVote RPC: target="{Voter server-7b65268c-9b33-f65a-0ffa-685844701264 7b65268c-9b33-f65a-0ffa-685844701264}" error="failed to connect to peer: 7b65268c-9b33-f65a-0ffa-685844701264" term=3 -2026-03-31T06:16:19.338Z [WARN] server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5: Election timeout reached, restarting election -2026-03-31T06:16:19.338Z [INFO] server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5: entering candidate state: node="Node at 896e7a91-d6dc-d4f3-7117-3c7d045b95a5 [Candidate]" term=4 -2026-03-31T06:16:19.338Z [INFO] server-18df5993-7078-67ed-d008-2b4a05b6a459: entering follower state: follower="Node at 18df5993-7078-67ed-d008-2b4a05b6a459 [Follower]" leader-address= leader-id= -2026-03-31T06:16:19.338Z [ERROR] server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5: failed to make requestVote RPC: target="{Voter server-7b65268c-9b33-f65a-0ffa-685844701264 7b65268c-9b33-f65a-0ffa-685844701264}" error="failed to connect to peer: 7b65268c-9b33-f65a-0ffa-685844701264" term=4 -2026-03-31T06:16:19.338Z [INFO] server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5: election won: term=4 tally=2 -2026-03-31T06:16:19.338Z [INFO] server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5: entering leader state: leader="Node at 896e7a91-d6dc-d4f3-7117-3c7d045b95a5 [Leader]" -2026-03-31T06:16:19.338Z [INFO] server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5: added peer, starting replication: peer=server-7b65268c-9b33-f65a-0ffa-685844701264 -2026-03-31T06:16:19.338Z [INFO] server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5: added peer, starting replication: peer=server-18df5993-7078-67ed-d008-2b4a05b6a459 -2026-03-31T06:16:19.338Z [ERROR] server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5: failed to appendEntries to: peer="{Voter server-7b65268c-9b33-f65a-0ffa-685844701264 7b65268c-9b33-f65a-0ffa-685844701264}" error="failed to connect to peer: 7b65268c-9b33-f65a-0ffa-685844701264" -2026-03-31T06:16:19.338Z [INFO] server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5: pipelining replication: peer="{Voter server-18df5993-7078-67ed-d008-2b4a05b6a459 18df5993-7078-67ed-d008-2b4a05b6a459}" -2026-03-31T06:16:19.344Z [ERROR] server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5: failed to heartbeat to: peer=7b65268c-9b33-f65a-0ffa-685844701264 backoff time=10ms error="failed to connect to peer: 7b65268c-9b33-f65a-0ffa-685844701264" -2026-03-31T06:16:19.353Z [WARN] server-7b65268c-9b33-f65a-0ffa-685844701264: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:19.353Z [INFO] server-7b65268c-9b33-f65a-0ffa-685844701264: entering candidate state: node="Node at 7b65268c-9b33-f65a-0ffa-685844701264 [Candidate]" term=3 -2026-03-31T06:16:19.353Z [ERROR] server-7b65268c-9b33-f65a-0ffa-685844701264: failed to make requestVote RPC: target="{Voter server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5 896e7a91-d6dc-d4f3-7117-3c7d045b95a5}" error="failed to connect to peer: 896e7a91-d6dc-d4f3-7117-3c7d045b95a5" term=3 -2026-03-31T06:16:19.353Z [ERROR] server-7b65268c-9b33-f65a-0ffa-685844701264: failed to make requestVote RPC: target="{Voter server-18df5993-7078-67ed-d008-2b4a05b6a459 18df5993-7078-67ed-d008-2b4a05b6a459}" error="failed to connect to peer: 18df5993-7078-67ed-d008-2b4a05b6a459" term=3 -2026-03-31T06:16:19.358Z [ERROR] server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5: failed to appendEntries to: peer="{Voter server-7b65268c-9b33-f65a-0ffa-685844701264 7b65268c-9b33-f65a-0ffa-685844701264}" error="failed to connect to peer: 7b65268c-9b33-f65a-0ffa-685844701264" -2026-03-31T06:16:19.361Z [ERROR] server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5: failed to heartbeat to: peer=7b65268c-9b33-f65a-0ffa-685844701264 backoff time=10ms error="failed to connect to peer: 7b65268c-9b33-f65a-0ffa-685844701264" -2026-03-31T06:16:19.379Z [ERROR] server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5: failed to appendEntries to: peer="{Voter server-7b65268c-9b33-f65a-0ffa-685844701264 7b65268c-9b33-f65a-0ffa-685844701264}" error="failed to connect to peer: 7b65268c-9b33-f65a-0ffa-685844701264" -2026-03-31T06:16:19.381Z [ERROR] server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5: failed to heartbeat to: peer=7b65268c-9b33-f65a-0ffa-685844701264 backoff time=10ms error="failed to connect to peer: 7b65268c-9b33-f65a-0ffa-685844701264" -2026-03-31T06:16:19.389Z [WARN] server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5: failed to contact: server-id=server-7b65268c-9b33-f65a-0ffa-685844701264 time=50.46716ms -2026-03-31T06:16:19.397Z [ERROR] server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5: failed to heartbeat to: peer=7b65268c-9b33-f65a-0ffa-685844701264 backoff time=20ms error="failed to connect to peer: 7b65268c-9b33-f65a-0ffa-685844701264" -2026-03-31T06:16:19.406Z [ERROR] server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5: failed to appendEntries to: peer="{Voter server-7b65268c-9b33-f65a-0ffa-685844701264 7b65268c-9b33-f65a-0ffa-685844701264}" error="failed to connect to peer: 7b65268c-9b33-f65a-0ffa-685844701264" -2026-03-31T06:16:19.427Z [ERROR] server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5: failed to heartbeat to: peer=7b65268c-9b33-f65a-0ffa-685844701264 backoff time=25ms error="failed to connect to peer: 7b65268c-9b33-f65a-0ffa-685844701264" -2026-03-31T06:16:19.432Z [WARN] server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5: failed to contact: server-id=server-7b65268c-9b33-f65a-0ffa-685844701264 time=94.0816ms -2026-03-31T06:16:19.443Z [WARN] server-7b65268c-9b33-f65a-0ffa-685844701264: Election timeout reached, restarting election -2026-03-31T06:16:19.443Z [INFO] server-7b65268c-9b33-f65a-0ffa-685844701264: entering candidate state: node="Node at 7b65268c-9b33-f65a-0ffa-685844701264 [Candidate]" term=4 -2026-03-31T06:16:19.443Z [ERROR] server-7b65268c-9b33-f65a-0ffa-685844701264: failed to make requestVote RPC: target="{Voter server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5 896e7a91-d6dc-d4f3-7117-3c7d045b95a5}" error="failed to connect to peer: 896e7a91-d6dc-d4f3-7117-3c7d045b95a5" term=4 -2026-03-31T06:16:19.443Z [ERROR] server-7b65268c-9b33-f65a-0ffa-685844701264: failed to make requestVote RPC: target="{Voter server-18df5993-7078-67ed-d008-2b4a05b6a459 18df5993-7078-67ed-d008-2b4a05b6a459}" error="failed to connect to peer: 18df5993-7078-67ed-d008-2b4a05b6a459" term=4 -2026-03-31T06:16:19.453Z [ERROR] server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5: failed to appendEntries to: peer="{Voter server-7b65268c-9b33-f65a-0ffa-685844701264 7b65268c-9b33-f65a-0ffa-685844701264}" error="failed to connect to peer: 7b65268c-9b33-f65a-0ffa-685844701264" -2026-03-31T06:16:19.458Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:19.3533147 +0000 UTC m=+7.630922081 (1 nodes), 146.19356ms from start of poll, 440.26856ms from cluster start. Timeout at 2026-03-31 06:16:19.45831746 +0000 UTC m=+7.735924841, 105.00276ms after stability - raft_test.go:413: [INFO] Reconnecting Node at 7b65268c-9b33-f65a-0ffa-685844701264 [Candidate] -2026-03-31T06:16:19.462Z [INFO] server-7b65268c-9b33-f65a-0ffa-685844701264: entering follower state: follower="Node at 7b65268c-9b33-f65a-0ffa-685844701264 [Follower]" leader-address=896e7a91-d6dc-d4f3-7117-3c7d045b95a5 leader-id=server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5 -2026-03-31T06:16:19.539Z [INFO] server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5: pipelining replication: peer="{Voter server-7b65268c-9b33-f65a-0ffa-685844701264 7b65268c-9b33-f65a-0ffa-685844701264}" -2026-03-31T06:16:19.540Z [INFO] server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5: aborting pipeline replication: peer="{Voter server-18df5993-7078-67ed-d008-2b4a05b6a459 18df5993-7078-67ed-d008-2b4a05b6a459}" -2026-03-31T06:16:19.540Z [INFO] server-896e7a91-d6dc-d4f3-7117-3c7d045b95a5: aborting pipeline replication: peer="{Voter server-7b65268c-9b33-f65a-0ffa-685844701264 7b65268c-9b33-f65a-0ffa-685844701264}" ---- PASS: TestRaft_LeaderFail (0.63s) +2025-02-25T23:57:39.286Z [INFO] server-9a00e020-6d2a-aea2-9978-84b49132a696: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9a00e020-6d2a-aea2-9978-84b49132a696 Address:9a00e020-6d2a-aea2-9978-84b49132a696} {Suffrage:Voter ID:server-5b3264be-25bf-d477-3830-7ca1813f941e Address:5b3264be-25bf-d477-3830-7ca1813f941e} {Suffrage:Voter ID:server-4d5d5da9-52cc-b63c-7574-8efb4fbdc991 Address:4d5d5da9-52cc-b63c-7574-8efb4fbdc991}]" +2025-02-25T23:57:39.286Z [INFO] server-5b3264be-25bf-d477-3830-7ca1813f941e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9a00e020-6d2a-aea2-9978-84b49132a696 Address:9a00e020-6d2a-aea2-9978-84b49132a696} {Suffrage:Voter ID:server-5b3264be-25bf-d477-3830-7ca1813f941e Address:5b3264be-25bf-d477-3830-7ca1813f941e} {Suffrage:Voter ID:server-4d5d5da9-52cc-b63c-7574-8efb4fbdc991 Address:4d5d5da9-52cc-b63c-7574-8efb4fbdc991}]" +2025-02-25T23:57:39.286Z [INFO] server-4d5d5da9-52cc-b63c-7574-8efb4fbdc991: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9a00e020-6d2a-aea2-9978-84b49132a696 Address:9a00e020-6d2a-aea2-9978-84b49132a696} {Suffrage:Voter ID:server-5b3264be-25bf-d477-3830-7ca1813f941e Address:5b3264be-25bf-d477-3830-7ca1813f941e} {Suffrage:Voter ID:server-4d5d5da9-52cc-b63c-7574-8efb4fbdc991 Address:4d5d5da9-52cc-b63c-7574-8efb4fbdc991}]" +2025-02-25T23:57:39.286Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:57:39.286Z [INFO] server-9a00e020-6d2a-aea2-9978-84b49132a696: entering follower state: follower="Node at 9a00e020-6d2a-aea2-9978-84b49132a696 [Follower]" leader-address= leader-id= +2025-02-25T23:57:39.286Z [INFO] server-5b3264be-25bf-d477-3830-7ca1813f941e: entering follower state: follower="Node at 5b3264be-25bf-d477-3830-7ca1813f941e [Follower]" leader-address= leader-id= +2025-02-25T23:57:39.286Z [INFO] server-4d5d5da9-52cc-b63c-7574-8efb4fbdc991: entering follower state: follower="Node at 4d5d5da9-52cc-b63c-7574-8efb4fbdc991 [Follower]" leader-address= leader-id= +2025-02-25T23:57:39.345Z [WARN] server-5b3264be-25bf-d477-3830-7ca1813f941e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:39.345Z [INFO] server-5b3264be-25bf-d477-3830-7ca1813f941e: entering candidate state: node="Node at 5b3264be-25bf-d477-3830-7ca1813f941e [Candidate]" term=2 +2025-02-25T23:57:39.346Z [INFO] server-5b3264be-25bf-d477-3830-7ca1813f941e: election won: term=2 tally=2 +2025-02-25T23:57:39.346Z [INFO] server-5b3264be-25bf-d477-3830-7ca1813f941e: entering leader state: leader="Node at 5b3264be-25bf-d477-3830-7ca1813f941e [Leader]" +2025-02-25T23:57:39.346Z [INFO] server-5b3264be-25bf-d477-3830-7ca1813f941e: added peer, starting replication: peer=server-9a00e020-6d2a-aea2-9978-84b49132a696 +2025-02-25T23:57:39.346Z [INFO] server-5b3264be-25bf-d477-3830-7ca1813f941e: added peer, starting replication: peer=server-4d5d5da9-52cc-b63c-7574-8efb4fbdc991 +2025-02-25T23:57:39.346Z [INFO] server-5b3264be-25bf-d477-3830-7ca1813f941e: pipelining replication: peer="{Voter server-4d5d5da9-52cc-b63c-7574-8efb4fbdc991 4d5d5da9-52cc-b63c-7574-8efb4fbdc991}" +2025-02-25T23:57:39.346Z [INFO] server-5b3264be-25bf-d477-3830-7ca1813f941e: pipelining replication: peer="{Voter server-9a00e020-6d2a-aea2-9978-84b49132a696 9a00e020-6d2a-aea2-9978-84b49132a696}" +2025-02-25T23:57:39.453Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:57:39.346240854 +0000 UTC m=+6.955256221 (2 nodes), 59.7177ms from start of poll, 59.897421ms from cluster start. Timeout at 2025-02-25 23:57:39.451242174 +0000 UTC m=+7.060257581, 105.00136ms after stability +2025-02-25T23:57:39.454Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:39.561Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:39.454022221 +0000 UTC m=+7.063037588 (1 nodes), 1.44µs from start of poll, 167.678788ms from cluster start. Timeout at 2025-02-25 23:57:39.559027701 +0000 UTC m=+7.168043108, 105.00552ms after stability + raft_test.go:378: [INFO] Disconnecting Node at 5b3264be-25bf-d477-3830-7ca1813f941e [Leader] +2025-02-25T23:57:39.581Z [ERROR] server-5b3264be-25bf-d477-3830-7ca1813f941e: failed to heartbeat to: peer=9a00e020-6d2a-aea2-9978-84b49132a696 backoff time=10ms error="failed to connect to peer: 9a00e020-6d2a-aea2-9978-84b49132a696" +2025-02-25T23:57:39.581Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:39.581Z [ERROR] server-5b3264be-25bf-d477-3830-7ca1813f941e: failed to pipeline appendEntries: peer="{Voter server-4d5d5da9-52cc-b63c-7574-8efb4fbdc991 4d5d5da9-52cc-b63c-7574-8efb4fbdc991}" error="append pipeline closed" +2025-02-25T23:57:39.581Z [INFO] server-5b3264be-25bf-d477-3830-7ca1813f941e: aborting pipeline replication: peer="{Voter server-4d5d5da9-52cc-b63c-7574-8efb4fbdc991 4d5d5da9-52cc-b63c-7574-8efb4fbdc991}" +2025-02-25T23:57:39.581Z [ERROR] server-5b3264be-25bf-d477-3830-7ca1813f941e: failed to pipeline appendEntries: peer="{Voter server-9a00e020-6d2a-aea2-9978-84b49132a696 9a00e020-6d2a-aea2-9978-84b49132a696}" error="append pipeline closed" +2025-02-25T23:57:39.581Z [INFO] server-5b3264be-25bf-d477-3830-7ca1813f941e: aborting pipeline replication: peer="{Voter server-9a00e020-6d2a-aea2-9978-84b49132a696 9a00e020-6d2a-aea2-9978-84b49132a696}" +2025-02-25T23:57:39.582Z [ERROR] server-5b3264be-25bf-d477-3830-7ca1813f941e: failed to heartbeat to: peer=4d5d5da9-52cc-b63c-7574-8efb4fbdc991 backoff time=10ms error="failed to connect to peer: 4d5d5da9-52cc-b63c-7574-8efb4fbdc991" +2025-02-25T23:57:39.589Z [ERROR] server-5b3264be-25bf-d477-3830-7ca1813f941e: failed to appendEntries to: peer="{Voter server-9a00e020-6d2a-aea2-9978-84b49132a696 9a00e020-6d2a-aea2-9978-84b49132a696}" error="failed to connect to peer: 9a00e020-6d2a-aea2-9978-84b49132a696" +2025-02-25T23:57:39.593Z [ERROR] server-5b3264be-25bf-d477-3830-7ca1813f941e: failed to appendEntries to: peer="{Voter server-4d5d5da9-52cc-b63c-7574-8efb4fbdc991 4d5d5da9-52cc-b63c-7574-8efb4fbdc991}" error="failed to connect to peer: 4d5d5da9-52cc-b63c-7574-8efb4fbdc991" +2025-02-25T23:57:39.601Z [ERROR] server-5b3264be-25bf-d477-3830-7ca1813f941e: failed to heartbeat to: peer=9a00e020-6d2a-aea2-9978-84b49132a696 backoff time=10ms error="failed to connect to peer: 9a00e020-6d2a-aea2-9978-84b49132a696" +2025-02-25T23:57:39.601Z [ERROR] server-5b3264be-25bf-d477-3830-7ca1813f941e: failed to heartbeat to: peer=4d5d5da9-52cc-b63c-7574-8efb4fbdc991 backoff time=10ms error="failed to connect to peer: 4d5d5da9-52cc-b63c-7574-8efb4fbdc991" +2025-02-25T23:57:39.609Z [ERROR] server-5b3264be-25bf-d477-3830-7ca1813f941e: failed to appendEntries to: peer="{Voter server-9a00e020-6d2a-aea2-9978-84b49132a696 9a00e020-6d2a-aea2-9978-84b49132a696}" error="failed to connect to peer: 9a00e020-6d2a-aea2-9978-84b49132a696" +2025-02-25T23:57:39.613Z [ERROR] server-5b3264be-25bf-d477-3830-7ca1813f941e: failed to appendEntries to: peer="{Voter server-4d5d5da9-52cc-b63c-7574-8efb4fbdc991 4d5d5da9-52cc-b63c-7574-8efb4fbdc991}" error="failed to connect to peer: 4d5d5da9-52cc-b63c-7574-8efb4fbdc991" +2025-02-25T23:57:39.621Z [ERROR] server-5b3264be-25bf-d477-3830-7ca1813f941e: failed to heartbeat to: peer=4d5d5da9-52cc-b63c-7574-8efb4fbdc991 backoff time=10ms error="failed to connect to peer: 4d5d5da9-52cc-b63c-7574-8efb4fbdc991" +2025-02-25T23:57:39.621Z [ERROR] server-5b3264be-25bf-d477-3830-7ca1813f941e: failed to heartbeat to: peer=9a00e020-6d2a-aea2-9978-84b49132a696 backoff time=10ms error="failed to connect to peer: 9a00e020-6d2a-aea2-9978-84b49132a696" +2025-02-25T23:57:39.625Z [WARN] server-9a00e020-6d2a-aea2-9978-84b49132a696: heartbeat timeout reached, starting election: last-leader-addr=5b3264be-25bf-d477-3830-7ca1813f941e last-leader-id=server-5b3264be-25bf-d477-3830-7ca1813f941e +2025-02-25T23:57:39.625Z [INFO] server-9a00e020-6d2a-aea2-9978-84b49132a696: entering candidate state: node="Node at 9a00e020-6d2a-aea2-9978-84b49132a696 [Candidate]" term=3 +2025-02-25T23:57:39.626Z [WARN] server-4d5d5da9-52cc-b63c-7574-8efb4fbdc991: rejecting vote request since we have a leader: from=9a00e020-6d2a-aea2-9978-84b49132a696 leader=5b3264be-25bf-d477-3830-7ca1813f941e leader-id=server-5b3264be-25bf-d477-3830-7ca1813f941e +2025-02-25T23:57:39.626Z [ERROR] server-9a00e020-6d2a-aea2-9978-84b49132a696: failed to make requestVote RPC: target="{Voter server-5b3264be-25bf-d477-3830-7ca1813f941e 5b3264be-25bf-d477-3830-7ca1813f941e}" error="failed to connect to peer: 5b3264be-25bf-d477-3830-7ca1813f941e" term=3 +2025-02-25T23:57:39.626Z [WARN] server-5b3264be-25bf-d477-3830-7ca1813f941e: failed to contact: server-id=server-9a00e020-6d2a-aea2-9978-84b49132a696 time=52.135932ms +2025-02-25T23:57:39.626Z [WARN] server-5b3264be-25bf-d477-3830-7ca1813f941e: failed to contact: server-id=server-4d5d5da9-52cc-b63c-7574-8efb4fbdc991 time=52.182292ms +2025-02-25T23:57:39.626Z [WARN] server-5b3264be-25bf-d477-3830-7ca1813f941e: failed to contact quorum of nodes, stepping down +2025-02-25T23:57:39.626Z [INFO] server-5b3264be-25bf-d477-3830-7ca1813f941e: entering follower state: follower="Node at 5b3264be-25bf-d477-3830-7ca1813f941e [Follower]" leader-address= leader-id= +2025-02-25T23:57:39.629Z [ERROR] server-5b3264be-25bf-d477-3830-7ca1813f941e: failed to appendEntries to: peer="{Voter server-9a00e020-6d2a-aea2-9978-84b49132a696 9a00e020-6d2a-aea2-9978-84b49132a696}" error="failed to connect to peer: 9a00e020-6d2a-aea2-9978-84b49132a696" +2025-02-25T23:57:39.633Z [ERROR] server-5b3264be-25bf-d477-3830-7ca1813f941e: failed to appendEntries to: peer="{Voter server-4d5d5da9-52cc-b63c-7574-8efb4fbdc991 4d5d5da9-52cc-b63c-7574-8efb4fbdc991}" error="failed to connect to peer: 4d5d5da9-52cc-b63c-7574-8efb4fbdc991" +2025-02-25T23:57:39.677Z [WARN] server-4d5d5da9-52cc-b63c-7574-8efb4fbdc991: heartbeat timeout reached, starting election: last-leader-addr=5b3264be-25bf-d477-3830-7ca1813f941e last-leader-id=server-5b3264be-25bf-d477-3830-7ca1813f941e +2025-02-25T23:57:39.677Z [INFO] server-4d5d5da9-52cc-b63c-7574-8efb4fbdc991: entering candidate state: node="Node at 4d5d5da9-52cc-b63c-7574-8efb4fbdc991 [Candidate]" term=3 +2025-02-25T23:57:39.677Z [ERROR] server-4d5d5da9-52cc-b63c-7574-8efb4fbdc991: failed to make requestVote RPC: target="{Voter server-5b3264be-25bf-d477-3830-7ca1813f941e 5b3264be-25bf-d477-3830-7ca1813f941e}" error="failed to connect to peer: 5b3264be-25bf-d477-3830-7ca1813f941e" term=3 +2025-02-25T23:57:39.678Z [INFO] server-9a00e020-6d2a-aea2-9978-84b49132a696: duplicate requestVote for same term: term=3 +2025-02-25T23:57:39.705Z [WARN] server-5b3264be-25bf-d477-3830-7ca1813f941e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:39.705Z [INFO] server-5b3264be-25bf-d477-3830-7ca1813f941e: entering candidate state: node="Node at 5b3264be-25bf-d477-3830-7ca1813f941e [Candidate]" term=3 +2025-02-25T23:57:39.705Z [ERROR] server-5b3264be-25bf-d477-3830-7ca1813f941e: failed to make requestVote RPC: target="{Voter server-4d5d5da9-52cc-b63c-7574-8efb4fbdc991 4d5d5da9-52cc-b63c-7574-8efb4fbdc991}" error="failed to connect to peer: 4d5d5da9-52cc-b63c-7574-8efb4fbdc991" term=3 +2025-02-25T23:57:39.706Z [ERROR] server-5b3264be-25bf-d477-3830-7ca1813f941e: failed to make requestVote RPC: target="{Voter server-9a00e020-6d2a-aea2-9978-84b49132a696 9a00e020-6d2a-aea2-9978-84b49132a696}" error="failed to connect to peer: 9a00e020-6d2a-aea2-9978-84b49132a696" term=3 +2025-02-25T23:57:39.713Z [WARN] server-9a00e020-6d2a-aea2-9978-84b49132a696: Election timeout reached, restarting election +2025-02-25T23:57:39.713Z [INFO] server-9a00e020-6d2a-aea2-9978-84b49132a696: entering candidate state: node="Node at 9a00e020-6d2a-aea2-9978-84b49132a696 [Candidate]" term=4 +2025-02-25T23:57:39.713Z [INFO] server-4d5d5da9-52cc-b63c-7574-8efb4fbdc991: entering follower state: follower="Node at 4d5d5da9-52cc-b63c-7574-8efb4fbdc991 [Follower]" leader-address= leader-id= +2025-02-25T23:57:39.713Z [INFO] server-9a00e020-6d2a-aea2-9978-84b49132a696: election won: term=4 tally=2 +2025-02-25T23:57:39.713Z [INFO] server-9a00e020-6d2a-aea2-9978-84b49132a696: entering leader state: leader="Node at 9a00e020-6d2a-aea2-9978-84b49132a696 [Leader]" +2025-02-25T23:57:39.713Z [INFO] server-9a00e020-6d2a-aea2-9978-84b49132a696: added peer, starting replication: peer=server-5b3264be-25bf-d477-3830-7ca1813f941e +2025-02-25T23:57:39.714Z [INFO] server-9a00e020-6d2a-aea2-9978-84b49132a696: added peer, starting replication: peer=server-4d5d5da9-52cc-b63c-7574-8efb4fbdc991 +2025-02-25T23:57:39.714Z [INFO] server-9a00e020-6d2a-aea2-9978-84b49132a696: pipelining replication: peer="{Voter server-4d5d5da9-52cc-b63c-7574-8efb4fbdc991 4d5d5da9-52cc-b63c-7574-8efb4fbdc991}" +2025-02-25T23:57:39.714Z [ERROR] server-9a00e020-6d2a-aea2-9978-84b49132a696: failed to make requestVote RPC: target="{Voter server-5b3264be-25bf-d477-3830-7ca1813f941e 5b3264be-25bf-d477-3830-7ca1813f941e}" error="failed to connect to peer: 5b3264be-25bf-d477-3830-7ca1813f941e" term=4 +2025-02-25T23:57:39.714Z [ERROR] server-9a00e020-6d2a-aea2-9978-84b49132a696: failed to appendEntries to: peer="{Voter server-5b3264be-25bf-d477-3830-7ca1813f941e 5b3264be-25bf-d477-3830-7ca1813f941e}" error="failed to connect to peer: 5b3264be-25bf-d477-3830-7ca1813f941e" +2025-02-25T23:57:39.725Z [ERROR] server-9a00e020-6d2a-aea2-9978-84b49132a696: failed to heartbeat to: peer=5b3264be-25bf-d477-3830-7ca1813f941e backoff time=10ms error="failed to connect to peer: 5b3264be-25bf-d477-3830-7ca1813f941e" +2025-02-25T23:57:39.737Z [ERROR] server-9a00e020-6d2a-aea2-9978-84b49132a696: failed to appendEntries to: peer="{Voter server-5b3264be-25bf-d477-3830-7ca1813f941e 5b3264be-25bf-d477-3830-7ca1813f941e}" error="failed to connect to peer: 5b3264be-25bf-d477-3830-7ca1813f941e" +2025-02-25T23:57:39.745Z [ERROR] server-9a00e020-6d2a-aea2-9978-84b49132a696: failed to heartbeat to: peer=5b3264be-25bf-d477-3830-7ca1813f941e backoff time=10ms error="failed to connect to peer: 5b3264be-25bf-d477-3830-7ca1813f941e" +2025-02-25T23:57:39.761Z [ERROR] server-9a00e020-6d2a-aea2-9978-84b49132a696: failed to appendEntries to: peer="{Voter server-5b3264be-25bf-d477-3830-7ca1813f941e 5b3264be-25bf-d477-3830-7ca1813f941e}" error="failed to connect to peer: 5b3264be-25bf-d477-3830-7ca1813f941e" +2025-02-25T23:57:39.765Z [WARN] server-9a00e020-6d2a-aea2-9978-84b49132a696: failed to contact: server-id=server-5b3264be-25bf-d477-3830-7ca1813f941e time=51.879931ms +2025-02-25T23:57:39.765Z [ERROR] server-9a00e020-6d2a-aea2-9978-84b49132a696: failed to heartbeat to: peer=5b3264be-25bf-d477-3830-7ca1813f941e backoff time=10ms error="failed to connect to peer: 5b3264be-25bf-d477-3830-7ca1813f941e" +2025-02-25T23:57:39.789Z [ERROR] server-9a00e020-6d2a-aea2-9978-84b49132a696: failed to appendEntries to: peer="{Voter server-5b3264be-25bf-d477-3830-7ca1813f941e 5b3264be-25bf-d477-3830-7ca1813f941e}" error="failed to connect to peer: 5b3264be-25bf-d477-3830-7ca1813f941e" +2025-02-25T23:57:39.789Z [ERROR] server-9a00e020-6d2a-aea2-9978-84b49132a696: failed to heartbeat to: peer=5b3264be-25bf-d477-3830-7ca1813f941e backoff time=20ms error="failed to connect to peer: 5b3264be-25bf-d477-3830-7ca1813f941e" +2025-02-25T23:57:39.793Z [WARN] server-5b3264be-25bf-d477-3830-7ca1813f941e: Election timeout reached, restarting election +2025-02-25T23:57:39.793Z [INFO] server-5b3264be-25bf-d477-3830-7ca1813f941e: entering candidate state: node="Node at 5b3264be-25bf-d477-3830-7ca1813f941e [Candidate]" term=4 +2025-02-25T23:57:39.794Z [ERROR] server-5b3264be-25bf-d477-3830-7ca1813f941e: failed to make requestVote RPC: target="{Voter server-4d5d5da9-52cc-b63c-7574-8efb4fbdc991 4d5d5da9-52cc-b63c-7574-8efb4fbdc991}" error="failed to connect to peer: 4d5d5da9-52cc-b63c-7574-8efb4fbdc991" term=4 +2025-02-25T23:57:39.794Z [ERROR] server-5b3264be-25bf-d477-3830-7ca1813f941e: failed to make requestVote RPC: target="{Voter server-9a00e020-6d2a-aea2-9978-84b49132a696 9a00e020-6d2a-aea2-9978-84b49132a696}" error="failed to connect to peer: 9a00e020-6d2a-aea2-9978-84b49132a696" term=4 +2025-02-25T23:57:39.813Z [WARN] server-9a00e020-6d2a-aea2-9978-84b49132a696: failed to contact: server-id=server-5b3264be-25bf-d477-3830-7ca1813f941e time=99.954077ms +2025-02-25T23:57:39.821Z [ERROR] server-9a00e020-6d2a-aea2-9978-84b49132a696: failed to heartbeat to: peer=5b3264be-25bf-d477-3830-7ca1813f941e backoff time=25ms error="failed to connect to peer: 5b3264be-25bf-d477-3830-7ca1813f941e" +2025-02-25T23:57:39.821Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:39.714120157 +0000 UTC m=+7.323135564 (1 nodes), 132.182522ms from start of poll, 427.776764ms from cluster start. Timeout at 2025-02-25 23:57:39.819121357 +0000 UTC m=+7.428136764, 105.0012ms after stability + raft_test.go:413: [INFO] Reconnecting Node at 5b3264be-25bf-d477-3830-7ca1813f941e [Candidate] +2025-02-25T23:57:39.841Z [INFO] server-5b3264be-25bf-d477-3830-7ca1813f941e: entering follower state: follower="Node at 5b3264be-25bf-d477-3830-7ca1813f941e [Follower]" leader-address=9a00e020-6d2a-aea2-9978-84b49132a696 leader-id=server-9a00e020-6d2a-aea2-9978-84b49132a696 +2025-02-25T23:57:39.842Z [INFO] server-9a00e020-6d2a-aea2-9978-84b49132a696: pipelining replication: peer="{Voter server-5b3264be-25bf-d477-3830-7ca1813f941e 5b3264be-25bf-d477-3830-7ca1813f941e}" +2025-02-25T23:57:39.842Z [INFO] server-9a00e020-6d2a-aea2-9978-84b49132a696: aborting pipeline replication: peer="{Voter server-4d5d5da9-52cc-b63c-7574-8efb4fbdc991 4d5d5da9-52cc-b63c-7574-8efb4fbdc991}" +2025-02-25T23:57:39.842Z [INFO] server-9a00e020-6d2a-aea2-9978-84b49132a696: aborting pipeline replication: peer="{Voter server-5b3264be-25bf-d477-3830-7ca1813f941e 5b3264be-25bf-d477-3830-7ca1813f941e}" +--- PASS: TestRaft_LeaderFail (0.56s) === RUN TestRaft_BehindFollower -2026-03-31T06:16:19.541Z [INFO] server-805abb5a-1de8-3992-f75d-6d8c19c77bf3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-805abb5a-1de8-3992-f75d-6d8c19c77bf3 Address:805abb5a-1de8-3992-f75d-6d8c19c77bf3} {Suffrage:Voter ID:server-8ceff989-0d1c-2139-eb40-50a62095d20d Address:8ceff989-0d1c-2139-eb40-50a62095d20d} {Suffrage:Voter ID:server-48d5acbd-b164-5174-fe61-9c7dab4485b0 Address:48d5acbd-b164-5174-fe61-9c7dab4485b0}]" -2026-03-31T06:16:19.541Z [INFO] server-805abb5a-1de8-3992-f75d-6d8c19c77bf3: entering follower state: follower="Node at 805abb5a-1de8-3992-f75d-6d8c19c77bf3 [Follower]" leader-address= leader-id= -2026-03-31T06:16:19.541Z [INFO] server-8ceff989-0d1c-2139-eb40-50a62095d20d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-805abb5a-1de8-3992-f75d-6d8c19c77bf3 Address:805abb5a-1de8-3992-f75d-6d8c19c77bf3} {Suffrage:Voter ID:server-8ceff989-0d1c-2139-eb40-50a62095d20d Address:8ceff989-0d1c-2139-eb40-50a62095d20d} {Suffrage:Voter ID:server-48d5acbd-b164-5174-fe61-9c7dab4485b0 Address:48d5acbd-b164-5174-fe61-9c7dab4485b0}]" -2026-03-31T06:16:19.541Z [INFO] server-8ceff989-0d1c-2139-eb40-50a62095d20d: entering follower state: follower="Node at 8ceff989-0d1c-2139-eb40-50a62095d20d [Follower]" leader-address= leader-id= -2026-03-31T06:16:19.541Z [INFO] server-48d5acbd-b164-5174-fe61-9c7dab4485b0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-805abb5a-1de8-3992-f75d-6d8c19c77bf3 Address:805abb5a-1de8-3992-f75d-6d8c19c77bf3} {Suffrage:Voter ID:server-8ceff989-0d1c-2139-eb40-50a62095d20d Address:8ceff989-0d1c-2139-eb40-50a62095d20d} {Suffrage:Voter ID:server-48d5acbd-b164-5174-fe61-9c7dab4485b0 Address:48d5acbd-b164-5174-fe61-9c7dab4485b0}]" -2026-03-31T06:16:19.541Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:19.541Z [INFO] server-48d5acbd-b164-5174-fe61-9c7dab4485b0: entering follower state: follower="Node at 48d5acbd-b164-5174-fe61-9c7dab4485b0 [Follower]" leader-address= leader-id= -2026-03-31T06:16:19.607Z [WARN] server-48d5acbd-b164-5174-fe61-9c7dab4485b0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:19.607Z [INFO] server-48d5acbd-b164-5174-fe61-9c7dab4485b0: entering candidate state: node="Node at 48d5acbd-b164-5174-fe61-9c7dab4485b0 [Candidate]" term=2 -2026-03-31T06:16:19.607Z [INFO] server-48d5acbd-b164-5174-fe61-9c7dab4485b0: election won: term=2 tally=2 -2026-03-31T06:16:19.607Z [INFO] server-48d5acbd-b164-5174-fe61-9c7dab4485b0: entering leader state: leader="Node at 48d5acbd-b164-5174-fe61-9c7dab4485b0 [Leader]" -2026-03-31T06:16:19.607Z [INFO] server-48d5acbd-b164-5174-fe61-9c7dab4485b0: added peer, starting replication: peer=server-805abb5a-1de8-3992-f75d-6d8c19c77bf3 -2026-03-31T06:16:19.607Z [INFO] server-48d5acbd-b164-5174-fe61-9c7dab4485b0: added peer, starting replication: peer=server-8ceff989-0d1c-2139-eb40-50a62095d20d -2026-03-31T06:16:19.607Z [INFO] server-48d5acbd-b164-5174-fe61-9c7dab4485b0: pipelining replication: peer="{Voter server-805abb5a-1de8-3992-f75d-6d8c19c77bf3 805abb5a-1de8-3992-f75d-6d8c19c77bf3}" -2026-03-31T06:16:19.607Z [INFO] server-48d5acbd-b164-5174-fe61-9c7dab4485b0: pipelining replication: peer="{Voter server-8ceff989-0d1c-2139-eb40-50a62095d20d 8ceff989-0d1c-2139-eb40-50a62095d20d}" -2026-03-31T06:16:19.712Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:19.60759606 +0000 UTC m=+7.885203441 (1 nodes), 65.7132ms from start of poll, 66.16972ms from cluster start. Timeout at 2026-03-31 06:16:19.7125991 +0000 UTC m=+7.990206481, 105.00304ms after stability -2026-03-31T06:16:19.712Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:16:19.818Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:16:19.71271698 +0000 UTC m=+7.990324361 (2 nodes), 1.04µs from start of poll, 171.29064ms from cluster start. Timeout at 2026-03-31 06:16:19.81772174 +0000 UTC m=+8.095329121, 105.00476ms after stability -2026-03-31T06:16:19.818Z [ERROR] server-48d5acbd-b164-5174-fe61-9c7dab4485b0: failed to pipeline appendEntries: peer="{Voter server-805abb5a-1de8-3992-f75d-6d8c19c77bf3 805abb5a-1de8-3992-f75d-6d8c19c77bf3}" error="append pipeline closed" -2026-03-31T06:16:19.818Z [INFO] server-48d5acbd-b164-5174-fe61-9c7dab4485b0: aborting pipeline replication: peer="{Voter server-805abb5a-1de8-3992-f75d-6d8c19c77bf3 805abb5a-1de8-3992-f75d-6d8c19c77bf3}" -2026-03-31T06:16:19.818Z [ERROR] server-48d5acbd-b164-5174-fe61-9c7dab4485b0: failed to appendEntries to: peer="{Voter server-805abb5a-1de8-3992-f75d-6d8c19c77bf3 805abb5a-1de8-3992-f75d-6d8c19c77bf3}" error="failed to connect to peer: 805abb5a-1de8-3992-f75d-6d8c19c77bf3" +2025-02-25T23:57:39.842Z [INFO] server-9a44064f-65b3-960e-ecaa-e45919bb1f81: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9a44064f-65b3-960e-ecaa-e45919bb1f81 Address:9a44064f-65b3-960e-ecaa-e45919bb1f81} {Suffrage:Voter ID:server-1ace86c0-7b7c-b901-9334-97e32cfac72b Address:1ace86c0-7b7c-b901-9334-97e32cfac72b} {Suffrage:Voter ID:server-5275726a-3367-f3e9-fc26-2478a9b957b5 Address:5275726a-3367-f3e9-fc26-2478a9b957b5}]" +2025-02-25T23:57:39.842Z [INFO] server-1ace86c0-7b7c-b901-9334-97e32cfac72b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9a44064f-65b3-960e-ecaa-e45919bb1f81 Address:9a44064f-65b3-960e-ecaa-e45919bb1f81} {Suffrage:Voter ID:server-1ace86c0-7b7c-b901-9334-97e32cfac72b Address:1ace86c0-7b7c-b901-9334-97e32cfac72b} {Suffrage:Voter ID:server-5275726a-3367-f3e9-fc26-2478a9b957b5 Address:5275726a-3367-f3e9-fc26-2478a9b957b5}]" +2025-02-25T23:57:39.842Z [INFO] server-5275726a-3367-f3e9-fc26-2478a9b957b5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9a44064f-65b3-960e-ecaa-e45919bb1f81 Address:9a44064f-65b3-960e-ecaa-e45919bb1f81} {Suffrage:Voter ID:server-1ace86c0-7b7c-b901-9334-97e32cfac72b Address:1ace86c0-7b7c-b901-9334-97e32cfac72b} {Suffrage:Voter ID:server-5275726a-3367-f3e9-fc26-2478a9b957b5 Address:5275726a-3367-f3e9-fc26-2478a9b957b5}]" +2025-02-25T23:57:39.842Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:39.842Z [INFO] server-9a44064f-65b3-960e-ecaa-e45919bb1f81: entering follower state: follower="Node at 9a44064f-65b3-960e-ecaa-e45919bb1f81 [Follower]" leader-address= leader-id= +2025-02-25T23:57:39.842Z [INFO] server-1ace86c0-7b7c-b901-9334-97e32cfac72b: entering follower state: follower="Node at 1ace86c0-7b7c-b901-9334-97e32cfac72b [Follower]" leader-address= leader-id= +2025-02-25T23:57:39.842Z [INFO] server-5275726a-3367-f3e9-fc26-2478a9b957b5: entering follower state: follower="Node at 5275726a-3367-f3e9-fc26-2478a9b957b5 [Follower]" leader-address= leader-id= +2025-02-25T23:57:39.909Z [WARN] server-9a44064f-65b3-960e-ecaa-e45919bb1f81: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:39.909Z [INFO] server-9a44064f-65b3-960e-ecaa-e45919bb1f81: entering candidate state: node="Node at 9a44064f-65b3-960e-ecaa-e45919bb1f81 [Candidate]" term=2 +2025-02-25T23:57:39.910Z [INFO] server-9a44064f-65b3-960e-ecaa-e45919bb1f81: election won: term=2 tally=2 +2025-02-25T23:57:39.910Z [INFO] server-9a44064f-65b3-960e-ecaa-e45919bb1f81: entering leader state: leader="Node at 9a44064f-65b3-960e-ecaa-e45919bb1f81 [Leader]" +2025-02-25T23:57:39.910Z [INFO] server-9a44064f-65b3-960e-ecaa-e45919bb1f81: added peer, starting replication: peer=server-1ace86c0-7b7c-b901-9334-97e32cfac72b +2025-02-25T23:57:39.910Z [INFO] server-9a44064f-65b3-960e-ecaa-e45919bb1f81: added peer, starting replication: peer=server-5275726a-3367-f3e9-fc26-2478a9b957b5 +2025-02-25T23:57:39.910Z [INFO] server-9a44064f-65b3-960e-ecaa-e45919bb1f81: pipelining replication: peer="{Voter server-5275726a-3367-f3e9-fc26-2478a9b957b5 5275726a-3367-f3e9-fc26-2478a9b957b5}" +2025-02-25T23:57:39.910Z [INFO] server-9a44064f-65b3-960e-ecaa-e45919bb1f81: pipelining replication: peer="{Voter server-1ace86c0-7b7c-b901-9334-97e32cfac72b 1ace86c0-7b7c-b901-9334-97e32cfac72b}" +2025-02-25T23:57:40.017Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:39.910228166 +0000 UTC m=+7.519243573 (1 nodes), 67.55735ms from start of poll, 67.726671ms from cluster start. Timeout at 2025-02-25 23:57:40.015235526 +0000 UTC m=+7.624250933, 105.00736ms after stability +2025-02-25T23:57:40.017Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:57:40.125Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:57:40.017979092 +0000 UTC m=+7.626994499 (2 nodes), 600ns from start of poll, 175.477597ms from cluster start. Timeout at 2025-02-25 23:57:40.122987132 +0000 UTC m=+7.732002499, 105.008ms after stability +2025-02-25T23:57:40.126Z [ERROR] server-9a44064f-65b3-960e-ecaa-e45919bb1f81: failed to pipeline appendEntries: peer="{Voter server-1ace86c0-7b7c-b901-9334-97e32cfac72b 1ace86c0-7b7c-b901-9334-97e32cfac72b}" error="append pipeline closed" +2025-02-25T23:57:40.126Z [INFO] server-9a44064f-65b3-960e-ecaa-e45919bb1f81: aborting pipeline replication: peer="{Voter server-1ace86c0-7b7c-b901-9334-97e32cfac72b 1ace86c0-7b7c-b901-9334-97e32cfac72b}" +2025-02-25T23:57:40.126Z [ERROR] server-9a44064f-65b3-960e-ecaa-e45919bb1f81: failed to appendEntries to: peer="{Voter server-1ace86c0-7b7c-b901-9334-97e32cfac72b 1ace86c0-7b7c-b901-9334-97e32cfac72b}" error="failed to connect to peer: 1ace86c0-7b7c-b901-9334-97e32cfac72b" raft_test.go:459: [INFO] Finished apply without behind follower -2026-03-31T06:16:19.829Z [INFO] server-48d5acbd-b164-5174-fe61-9c7dab4485b0: pipelining replication: peer="{Voter server-805abb5a-1de8-3992-f75d-6d8c19c77bf3 805abb5a-1de8-3992-f75d-6d8c19c77bf3}" -2026-03-31T06:16:19.840Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:19.946Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:19.84047854 +0000 UTC m=+8.118085921 (1 nodes), 880ns from start of poll, 299.0522ms from cluster start. Timeout at 2026-03-31 06:16:19.94548218 +0000 UTC m=+8.223089561, 105.00364ms after stability -2026-03-31T06:16:19.946Z [INFO] server-48d5acbd-b164-5174-fe61-9c7dab4485b0: aborting pipeline replication: peer="{Voter server-8ceff989-0d1c-2139-eb40-50a62095d20d 8ceff989-0d1c-2139-eb40-50a62095d20d}" -2026-03-31T06:16:19.946Z [INFO] server-48d5acbd-b164-5174-fe61-9c7dab4485b0: aborting pipeline replication: peer="{Voter server-805abb5a-1de8-3992-f75d-6d8c19c77bf3 805abb5a-1de8-3992-f75d-6d8c19c77bf3}" ---- PASS: TestRaft_BehindFollower (0.41s) +2025-02-25T23:57:40.137Z [INFO] server-9a44064f-65b3-960e-ecaa-e45919bb1f81: pipelining replication: peer="{Voter server-1ace86c0-7b7c-b901-9334-97e32cfac72b 1ace86c0-7b7c-b901-9334-97e32cfac72b}" +2025-02-25T23:57:40.153Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:40.261Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:40.153977119 +0000 UTC m=+7.762992486 (1 nodes), 1µs from start of poll, 311.475584ms from cluster start. Timeout at 2025-02-25 23:57:40.258982719 +0000 UTC m=+7.867998126, 105.00564ms after stability +2025-02-25T23:57:40.261Z [INFO] server-9a44064f-65b3-960e-ecaa-e45919bb1f81: aborting pipeline replication: peer="{Voter server-5275726a-3367-f3e9-fc26-2478a9b957b5 5275726a-3367-f3e9-fc26-2478a9b957b5}" +2025-02-25T23:57:40.262Z [INFO] server-9a44064f-65b3-960e-ecaa-e45919bb1f81: aborting pipeline replication: peer="{Voter server-1ace86c0-7b7c-b901-9334-97e32cfac72b 1ace86c0-7b7c-b901-9334-97e32cfac72b}" +2025-02-25T23:57:40.765Z [ERROR] server-9a44064f-65b3-960e-ecaa-e45919bb1f81: failed to heartbeat to: peer=1ace86c0-7b7c-b901-9334-97e32cfac72b backoff time=10ms error="command timed out" +--- PASS: TestRaft_BehindFollower (0.92s) === RUN TestRaft_ApplyNonLeader -2026-03-31T06:16:19.947Z [INFO] server-b519d033-d31b-60de-1b83-4925c20d146e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b519d033-d31b-60de-1b83-4925c20d146e Address:b519d033-d31b-60de-1b83-4925c20d146e} {Suffrage:Voter ID:server-b4e122ef-a4ae-7c9b-687f-72e54d6e3652 Address:b4e122ef-a4ae-7c9b-687f-72e54d6e3652} {Suffrage:Voter ID:server-04dc1cf0-b759-2d77-6f98-e070e55ec3b0 Address:04dc1cf0-b759-2d77-6f98-e070e55ec3b0}]" -2026-03-31T06:16:19.947Z [INFO] server-b519d033-d31b-60de-1b83-4925c20d146e: entering follower state: follower="Node at b519d033-d31b-60de-1b83-4925c20d146e [Follower]" leader-address= leader-id= -2026-03-31T06:16:19.947Z [INFO] server-b4e122ef-a4ae-7c9b-687f-72e54d6e3652: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b519d033-d31b-60de-1b83-4925c20d146e Address:b519d033-d31b-60de-1b83-4925c20d146e} {Suffrage:Voter ID:server-b4e122ef-a4ae-7c9b-687f-72e54d6e3652 Address:b4e122ef-a4ae-7c9b-687f-72e54d6e3652} {Suffrage:Voter ID:server-04dc1cf0-b759-2d77-6f98-e070e55ec3b0 Address:04dc1cf0-b759-2d77-6f98-e070e55ec3b0}]" -2026-03-31T06:16:19.947Z [INFO] server-b4e122ef-a4ae-7c9b-687f-72e54d6e3652: entering follower state: follower="Node at b4e122ef-a4ae-7c9b-687f-72e54d6e3652 [Follower]" leader-address= leader-id= -2026-03-31T06:16:19.947Z [INFO] server-04dc1cf0-b759-2d77-6f98-e070e55ec3b0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b519d033-d31b-60de-1b83-4925c20d146e Address:b519d033-d31b-60de-1b83-4925c20d146e} {Suffrage:Voter ID:server-b4e122ef-a4ae-7c9b-687f-72e54d6e3652 Address:b4e122ef-a4ae-7c9b-687f-72e54d6e3652} {Suffrage:Voter ID:server-04dc1cf0-b759-2d77-6f98-e070e55ec3b0 Address:04dc1cf0-b759-2d77-6f98-e070e55ec3b0}]" -2026-03-31T06:16:19.947Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:19.947Z [INFO] server-04dc1cf0-b759-2d77-6f98-e070e55ec3b0: entering follower state: follower="Node at 04dc1cf0-b759-2d77-6f98-e070e55ec3b0 [Follower]" leader-address= leader-id= -2026-03-31T06:16:20.006Z [WARN] server-b519d033-d31b-60de-1b83-4925c20d146e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:20.006Z [INFO] server-b519d033-d31b-60de-1b83-4925c20d146e: entering candidate state: node="Node at b519d033-d31b-60de-1b83-4925c20d146e [Candidate]" term=2 -2026-03-31T06:16:20.006Z [INFO] server-b519d033-d31b-60de-1b83-4925c20d146e: election won: term=2 tally=2 -2026-03-31T06:16:20.006Z [INFO] server-b519d033-d31b-60de-1b83-4925c20d146e: entering leader state: leader="Node at b519d033-d31b-60de-1b83-4925c20d146e [Leader]" -2026-03-31T06:16:20.006Z [INFO] server-b519d033-d31b-60de-1b83-4925c20d146e: added peer, starting replication: peer=server-b4e122ef-a4ae-7c9b-687f-72e54d6e3652 -2026-03-31T06:16:20.006Z [INFO] server-b519d033-d31b-60de-1b83-4925c20d146e: added peer, starting replication: peer=server-04dc1cf0-b759-2d77-6f98-e070e55ec3b0 -2026-03-31T06:16:20.006Z [INFO] server-b519d033-d31b-60de-1b83-4925c20d146e: pipelining replication: peer="{Voter server-b4e122ef-a4ae-7c9b-687f-72e54d6e3652 b4e122ef-a4ae-7c9b-687f-72e54d6e3652}" -2026-03-31T06:16:20.006Z [INFO] server-b519d033-d31b-60de-1b83-4925c20d146e: pipelining replication: peer="{Voter server-04dc1cf0-b759-2d77-6f98-e070e55ec3b0 04dc1cf0-b759-2d77-6f98-e070e55ec3b0}" -2026-03-31T06:16:20.112Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:20.0069299 +0000 UTC m=+8.284537281 (1 nodes), 59.37ms from start of poll, 59.89936ms from cluster start. Timeout at 2026-03-31 06:16:20.11193394 +0000 UTC m=+8.389541321, 105.00404ms after stability -2026-03-31T06:16:20.112Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:16:20.217Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:16:20.11223634 +0000 UTC m=+8.389843761 (2 nodes), 1.16µs from start of poll, 165.20584ms from cluster start. Timeout at 2026-03-31 06:16:20.2172403 +0000 UTC m=+8.494847681, 105.00392ms after stability -2026-03-31T06:16:20.217Z [INFO] server-b519d033-d31b-60de-1b83-4925c20d146e: aborting pipeline replication: peer="{Voter server-04dc1cf0-b759-2d77-6f98-e070e55ec3b0 04dc1cf0-b759-2d77-6f98-e070e55ec3b0}" -2026-03-31T06:16:20.217Z [INFO] server-b519d033-d31b-60de-1b83-4925c20d146e: aborting pipeline replication: peer="{Voter server-b4e122ef-a4ae-7c9b-687f-72e54d6e3652 b4e122ef-a4ae-7c9b-687f-72e54d6e3652}" +2025-02-25T23:57:40.766Z [INFO] server-a325cca9-441a-5b5d-01c7-190c99477c34: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a325cca9-441a-5b5d-01c7-190c99477c34 Address:a325cca9-441a-5b5d-01c7-190c99477c34} {Suffrage:Voter ID:server-3310aeb5-d8a1-65ac-07a5-a896c477aa06 Address:3310aeb5-d8a1-65ac-07a5-a896c477aa06} {Suffrage:Voter ID:server-9046951d-ad78-882e-4146-b45c997ff8a0 Address:9046951d-ad78-882e-4146-b45c997ff8a0}]" +2025-02-25T23:57:40.766Z [INFO] server-3310aeb5-d8a1-65ac-07a5-a896c477aa06: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a325cca9-441a-5b5d-01c7-190c99477c34 Address:a325cca9-441a-5b5d-01c7-190c99477c34} {Suffrage:Voter ID:server-3310aeb5-d8a1-65ac-07a5-a896c477aa06 Address:3310aeb5-d8a1-65ac-07a5-a896c477aa06} {Suffrage:Voter ID:server-9046951d-ad78-882e-4146-b45c997ff8a0 Address:9046951d-ad78-882e-4146-b45c997ff8a0}]" +2025-02-25T23:57:40.766Z [INFO] server-9046951d-ad78-882e-4146-b45c997ff8a0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a325cca9-441a-5b5d-01c7-190c99477c34 Address:a325cca9-441a-5b5d-01c7-190c99477c34} {Suffrage:Voter ID:server-3310aeb5-d8a1-65ac-07a5-a896c477aa06 Address:3310aeb5-d8a1-65ac-07a5-a896c477aa06} {Suffrage:Voter ID:server-9046951d-ad78-882e-4146-b45c997ff8a0 Address:9046951d-ad78-882e-4146-b45c997ff8a0}]" +2025-02-25T23:57:40.766Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:40.766Z [INFO] server-a325cca9-441a-5b5d-01c7-190c99477c34: entering follower state: follower="Node at a325cca9-441a-5b5d-01c7-190c99477c34 [Follower]" leader-address= leader-id= +2025-02-25T23:57:40.766Z [INFO] server-3310aeb5-d8a1-65ac-07a5-a896c477aa06: entering follower state: follower="Node at 3310aeb5-d8a1-65ac-07a5-a896c477aa06 [Follower]" leader-address= leader-id= +2025-02-25T23:57:40.766Z [INFO] server-9046951d-ad78-882e-4146-b45c997ff8a0: entering follower state: follower="Node at 9046951d-ad78-882e-4146-b45c997ff8a0 [Follower]" leader-address= leader-id= +2025-02-25T23:57:40.821Z [WARN] server-9046951d-ad78-882e-4146-b45c997ff8a0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:40.822Z [INFO] server-9046951d-ad78-882e-4146-b45c997ff8a0: entering candidate state: node="Node at 9046951d-ad78-882e-4146-b45c997ff8a0 [Candidate]" term=2 +2025-02-25T23:57:40.822Z [INFO] server-9046951d-ad78-882e-4146-b45c997ff8a0: election won: term=2 tally=2 +2025-02-25T23:57:40.822Z [INFO] server-9046951d-ad78-882e-4146-b45c997ff8a0: entering leader state: leader="Node at 9046951d-ad78-882e-4146-b45c997ff8a0 [Leader]" +2025-02-25T23:57:40.822Z [INFO] server-9046951d-ad78-882e-4146-b45c997ff8a0: added peer, starting replication: peer=server-a325cca9-441a-5b5d-01c7-190c99477c34 +2025-02-25T23:57:40.822Z [INFO] server-9046951d-ad78-882e-4146-b45c997ff8a0: added peer, starting replication: peer=server-3310aeb5-d8a1-65ac-07a5-a896c477aa06 +2025-02-25T23:57:40.822Z [INFO] server-9046951d-ad78-882e-4146-b45c997ff8a0: pipelining replication: peer="{Voter server-3310aeb5-d8a1-65ac-07a5-a896c477aa06 3310aeb5-d8a1-65ac-07a5-a896c477aa06}" +2025-02-25T23:57:40.822Z [INFO] server-9046951d-ad78-882e-4146-b45c997ff8a0: pipelining replication: peer="{Voter server-a325cca9-441a-5b5d-01c7-190c99477c34 a325cca9-441a-5b5d-01c7-190c99477c34}" +2025-02-25T23:57:40.929Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:40.822277855 +0000 UTC m=+8.431293262 (1 nodes), 55.840316ms from start of poll, 56.010357ms from cluster start. Timeout at 2025-02-25 23:57:40.927282335 +0000 UTC m=+8.536297742, 105.00448ms after stability +2025-02-25T23:57:40.929Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:57:41.035Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:57:40.929926821 +0000 UTC m=+8.538942228 (2 nodes), 1.12µs from start of poll, 163.659323ms from cluster start. Timeout at 2025-02-25 23:57:41.034932661 +0000 UTC m=+8.643948068, 105.00584ms after stability +2025-02-25T23:57:41.035Z [INFO] server-9046951d-ad78-882e-4146-b45c997ff8a0: aborting pipeline replication: peer="{Voter server-3310aeb5-d8a1-65ac-07a5-a896c477aa06 3310aeb5-d8a1-65ac-07a5-a896c477aa06}" +2025-02-25T23:57:41.035Z [INFO] server-9046951d-ad78-882e-4146-b45c997ff8a0: aborting pipeline replication: peer="{Voter server-a325cca9-441a-5b5d-01c7-190c99477c34 a325cca9-441a-5b5d-01c7-190c99477c34}" --- PASS: TestRaft_ApplyNonLeader (0.27s) === RUN TestRaft_ApplyConcurrent -2026-03-31T06:16:20.218Z [INFO] server-6a431bab-9d17-b7d7-84bb-fa7083f86350: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6a431bab-9d17-b7d7-84bb-fa7083f86350 Address:6a431bab-9d17-b7d7-84bb-fa7083f86350} {Suffrage:Voter ID:server-6bce5666-e071-9fd6-6b49-96d1eefbf07e Address:6bce5666-e071-9fd6-6b49-96d1eefbf07e} {Suffrage:Voter ID:server-7910871e-c42a-1b27-1be7-f93678240163 Address:7910871e-c42a-1b27-1be7-f93678240163}]" -2026-03-31T06:16:20.218Z [INFO] server-6bce5666-e071-9fd6-6b49-96d1eefbf07e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6a431bab-9d17-b7d7-84bb-fa7083f86350 Address:6a431bab-9d17-b7d7-84bb-fa7083f86350} {Suffrage:Voter ID:server-6bce5666-e071-9fd6-6b49-96d1eefbf07e Address:6bce5666-e071-9fd6-6b49-96d1eefbf07e} {Suffrage:Voter ID:server-7910871e-c42a-1b27-1be7-f93678240163 Address:7910871e-c42a-1b27-1be7-f93678240163}]" -2026-03-31T06:16:20.218Z [INFO] server-6a431bab-9d17-b7d7-84bb-fa7083f86350: entering follower state: follower="Node at 6a431bab-9d17-b7d7-84bb-fa7083f86350 [Follower]" leader-address= leader-id= -2026-03-31T06:16:20.218Z [INFO] server-6bce5666-e071-9fd6-6b49-96d1eefbf07e: entering follower state: follower="Node at 6bce5666-e071-9fd6-6b49-96d1eefbf07e [Follower]" leader-address= leader-id= -2026-03-31T06:16:20.218Z [INFO] server-7910871e-c42a-1b27-1be7-f93678240163: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6a431bab-9d17-b7d7-84bb-fa7083f86350 Address:6a431bab-9d17-b7d7-84bb-fa7083f86350} {Suffrage:Voter ID:server-6bce5666-e071-9fd6-6b49-96d1eefbf07e Address:6bce5666-e071-9fd6-6b49-96d1eefbf07e} {Suffrage:Voter ID:server-7910871e-c42a-1b27-1be7-f93678240163 Address:7910871e-c42a-1b27-1be7-f93678240163}]" -2026-03-31T06:16:20.218Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:20.218Z [INFO] server-7910871e-c42a-1b27-1be7-f93678240163: entering follower state: follower="Node at 7910871e-c42a-1b27-1be7-f93678240163 [Follower]" leader-address= leader-id= -2026-03-31T06:16:20.357Z [WARN] server-7910871e-c42a-1b27-1be7-f93678240163: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:20.357Z [INFO] server-7910871e-c42a-1b27-1be7-f93678240163: entering candidate state: node="Node at 7910871e-c42a-1b27-1be7-f93678240163 [Candidate]" term=2 -2026-03-31T06:16:20.357Z [INFO] server-7910871e-c42a-1b27-1be7-f93678240163: election won: term=2 tally=2 -2026-03-31T06:16:20.357Z [INFO] server-7910871e-c42a-1b27-1be7-f93678240163: entering leader state: leader="Node at 7910871e-c42a-1b27-1be7-f93678240163 [Leader]" -2026-03-31T06:16:20.357Z [INFO] server-7910871e-c42a-1b27-1be7-f93678240163: added peer, starting replication: peer=server-6a431bab-9d17-b7d7-84bb-fa7083f86350 -2026-03-31T06:16:20.357Z [INFO] server-7910871e-c42a-1b27-1be7-f93678240163: added peer, starting replication: peer=server-6bce5666-e071-9fd6-6b49-96d1eefbf07e -2026-03-31T06:16:20.357Z [INFO] server-7910871e-c42a-1b27-1be7-f93678240163: pipelining replication: peer="{Voter server-6a431bab-9d17-b7d7-84bb-fa7083f86350 6a431bab-9d17-b7d7-84bb-fa7083f86350}" -2026-03-31T06:16:20.357Z [INFO] server-7910871e-c42a-1b27-1be7-f93678240163: pipelining replication: peer="{Voter server-6bce5666-e071-9fd6-6b49-96d1eefbf07e 6bce5666-e071-9fd6-6b49-96d1eefbf07e}" -2026-03-31T06:16:20.563Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:20.35778102 +0000 UTC m=+8.635388401 (1 nodes), 139.25108ms from start of poll, 139.63992ms from cluster start. Timeout at 2026-03-31 06:16:20.56278706 +0000 UTC m=+8.840394441, 205.00604ms after stability -2026-03-31T06:16:20.578Z [INFO] server-7910871e-c42a-1b27-1be7-f93678240163: aborting pipeline replication: peer="{Voter server-6bce5666-e071-9fd6-6b49-96d1eefbf07e 6bce5666-e071-9fd6-6b49-96d1eefbf07e}" -2026-03-31T06:16:20.578Z [INFO] server-7910871e-c42a-1b27-1be7-f93678240163: aborting pipeline replication: peer="{Voter server-6a431bab-9d17-b7d7-84bb-fa7083f86350 6a431bab-9d17-b7d7-84bb-fa7083f86350}" ---- PASS: TestRaft_ApplyConcurrent (0.36s) +2025-02-25T23:57:41.035Z [INFO] server-74991e49-ad77-573c-cc81-2361657eec46: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-74991e49-ad77-573c-cc81-2361657eec46 Address:74991e49-ad77-573c-cc81-2361657eec46} {Suffrage:Voter ID:server-3d4d3003-f654-e216-5900-d8fd5849728b Address:3d4d3003-f654-e216-5900-d8fd5849728b} {Suffrage:Voter ID:server-675fb079-ff39-6c5b-7ff5-52c2a60a5313 Address:675fb079-ff39-6c5b-7ff5-52c2a60a5313}]" +2025-02-25T23:57:41.035Z [INFO] server-3d4d3003-f654-e216-5900-d8fd5849728b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-74991e49-ad77-573c-cc81-2361657eec46 Address:74991e49-ad77-573c-cc81-2361657eec46} {Suffrage:Voter ID:server-3d4d3003-f654-e216-5900-d8fd5849728b Address:3d4d3003-f654-e216-5900-d8fd5849728b} {Suffrage:Voter ID:server-675fb079-ff39-6c5b-7ff5-52c2a60a5313 Address:675fb079-ff39-6c5b-7ff5-52c2a60a5313}]" +2025-02-25T23:57:41.036Z [INFO] server-675fb079-ff39-6c5b-7ff5-52c2a60a5313: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-74991e49-ad77-573c-cc81-2361657eec46 Address:74991e49-ad77-573c-cc81-2361657eec46} {Suffrage:Voter ID:server-3d4d3003-f654-e216-5900-d8fd5849728b Address:3d4d3003-f654-e216-5900-d8fd5849728b} {Suffrage:Voter ID:server-675fb079-ff39-6c5b-7ff5-52c2a60a5313 Address:675fb079-ff39-6c5b-7ff5-52c2a60a5313}]" +2025-02-25T23:57:41.036Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:41.036Z [INFO] server-74991e49-ad77-573c-cc81-2361657eec46: entering follower state: follower="Node at 74991e49-ad77-573c-cc81-2361657eec46 [Follower]" leader-address= leader-id= +2025-02-25T23:57:41.036Z [INFO] server-3d4d3003-f654-e216-5900-d8fd5849728b: entering follower state: follower="Node at 3d4d3003-f654-e216-5900-d8fd5849728b [Follower]" leader-address= leader-id= +2025-02-25T23:57:41.036Z [INFO] server-675fb079-ff39-6c5b-7ff5-52c2a60a5313: entering follower state: follower="Node at 675fb079-ff39-6c5b-7ff5-52c2a60a5313 [Follower]" leader-address= leader-id= +2025-02-25T23:57:41.141Z [WARN] server-675fb079-ff39-6c5b-7ff5-52c2a60a5313: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:41.141Z [INFO] server-675fb079-ff39-6c5b-7ff5-52c2a60a5313: entering candidate state: node="Node at 675fb079-ff39-6c5b-7ff5-52c2a60a5313 [Candidate]" term=2 +2025-02-25T23:57:41.141Z [INFO] server-675fb079-ff39-6c5b-7ff5-52c2a60a5313: election won: term=2 tally=2 +2025-02-25T23:57:41.141Z [INFO] server-675fb079-ff39-6c5b-7ff5-52c2a60a5313: entering leader state: leader="Node at 675fb079-ff39-6c5b-7ff5-52c2a60a5313 [Leader]" +2025-02-25T23:57:41.141Z [INFO] server-675fb079-ff39-6c5b-7ff5-52c2a60a5313: added peer, starting replication: peer=server-74991e49-ad77-573c-cc81-2361657eec46 +2025-02-25T23:57:41.142Z [INFO] server-675fb079-ff39-6c5b-7ff5-52c2a60a5313: added peer, starting replication: peer=server-3d4d3003-f654-e216-5900-d8fd5849728b +2025-02-25T23:57:41.142Z [INFO] server-675fb079-ff39-6c5b-7ff5-52c2a60a5313: pipelining replication: peer="{Voter server-3d4d3003-f654-e216-5900-d8fd5849728b 3d4d3003-f654-e216-5900-d8fd5849728b}" +2025-02-25T23:57:41.142Z [INFO] server-675fb079-ff39-6c5b-7ff5-52c2a60a5313: pipelining replication: peer="{Voter server-74991e49-ad77-573c-cc81-2361657eec46 74991e49-ad77-573c-cc81-2361657eec46}" +2025-02-25T23:57:41.349Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:41.142218973 +0000 UTC m=+8.751234340 (1 nodes), 106.188596ms from start of poll, 106.404238ms from cluster start. Timeout at 2025-02-25 23:57:41.347224133 +0000 UTC m=+8.956239540, 205.0052ms after stability +2025-02-25T23:57:41.378Z [INFO] server-675fb079-ff39-6c5b-7ff5-52c2a60a5313: aborting pipeline replication: peer="{Voter server-3d4d3003-f654-e216-5900-d8fd5849728b 3d4d3003-f654-e216-5900-d8fd5849728b}" +2025-02-25T23:57:41.378Z [INFO] server-675fb079-ff39-6c5b-7ff5-52c2a60a5313: aborting pipeline replication: peer="{Voter server-74991e49-ad77-573c-cc81-2361657eec46 74991e49-ad77-573c-cc81-2361657eec46}" +2025-02-25T23:57:41.881Z [ERROR] server-675fb079-ff39-6c5b-7ff5-52c2a60a5313: failed to heartbeat to: peer=3d4d3003-f654-e216-5900-d8fd5849728b backoff time=10ms error="command timed out" +--- PASS: TestRaft_ApplyConcurrent (0.85s) === RUN TestRaft_ApplyConcurrent_Timeout -2026-03-31T06:16:20.578Z [INFO] server-b51f4794-0c81-1d25-de20-6c2d54d1d61f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b51f4794-0c81-1d25-de20-6c2d54d1d61f Address:b51f4794-0c81-1d25-de20-6c2d54d1d61f}]" -2026-03-31T06:16:20.578Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:20.578Z [INFO] server-b51f4794-0c81-1d25-de20-6c2d54d1d61f: entering follower state: follower="Node at b51f4794-0c81-1d25-de20-6c2d54d1d61f [Follower]" leader-address= leader-id= -2026-03-31T06:16:20.739Z [WARN] server-b51f4794-0c81-1d25-de20-6c2d54d1d61f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:20.739Z [INFO] server-b51f4794-0c81-1d25-de20-6c2d54d1d61f: entering candidate state: node="Node at b51f4794-0c81-1d25-de20-6c2d54d1d61f [Candidate]" term=2 -2026-03-31T06:16:20.739Z [INFO] server-b51f4794-0c81-1d25-de20-6c2d54d1d61f: election won: term=2 tally=1 -2026-03-31T06:16:20.739Z [INFO] server-b51f4794-0c81-1d25-de20-6c2d54d1d61f: entering leader state: leader="Node at b51f4794-0c81-1d25-de20-6c2d54d1d61f [Leader]" -2026-03-31T06:16:20.941Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:20.73965426 +0000 UTC m=+9.017261641 (1 nodes), 160.75504ms from start of poll, 160.93788ms from cluster start. Timeout at 2026-03-31 06:16:20.94065734 +0000 UTC m=+9.218264721, 201.00308ms after stability ---- PASS: TestRaft_ApplyConcurrent_Timeout (0.36s) +2025-02-25T23:57:41.882Z [INFO] server-6aecafeb-0807-7cea-860f-d575cf10b740: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6aecafeb-0807-7cea-860f-d575cf10b740 Address:6aecafeb-0807-7cea-860f-d575cf10b740}]" +2025-02-25T23:57:41.882Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:41.882Z [INFO] server-6aecafeb-0807-7cea-860f-d575cf10b740: entering follower state: follower="Node at 6aecafeb-0807-7cea-860f-d575cf10b740 [Follower]" leader-address= leader-id= +2025-02-25T23:57:42.049Z [WARN] server-6aecafeb-0807-7cea-860f-d575cf10b740: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:42.049Z [INFO] server-6aecafeb-0807-7cea-860f-d575cf10b740: entering candidate state: node="Node at 6aecafeb-0807-7cea-860f-d575cf10b740 [Candidate]" term=2 +2025-02-25T23:57:42.050Z [INFO] server-6aecafeb-0807-7cea-860f-d575cf10b740: election won: term=2 tally=1 +2025-02-25T23:57:42.050Z [INFO] server-6aecafeb-0807-7cea-860f-d575cf10b740: entering leader state: leader="Node at 6aecafeb-0807-7cea-860f-d575cf10b740 [Leader]" +2025-02-25T23:57:42.253Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:42.050057635 +0000 UTC m=+9.659073042 (1 nodes), 167.699988ms from start of poll, 167.795949ms from cluster start. Timeout at 2025-02-25 23:57:42.251062875 +0000 UTC m=+9.860078242, 201.0052ms after stability +--- PASS: TestRaft_ApplyConcurrent_Timeout (0.37s) === RUN TestRaft_JoinNode -2026-03-31T06:16:20.942Z [INFO] server-9335ae3d-a47c-730f-dda1-bb94a4c94c3f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9335ae3d-a47c-730f-dda1-bb94a4c94c3f Address:9335ae3d-a47c-730f-dda1-bb94a4c94c3f} {Suffrage:Voter ID:server-6163e78d-d2c6-0207-021c-580c19332fd3 Address:6163e78d-d2c6-0207-021c-580c19332fd3}]" -2026-03-31T06:16:20.942Z [INFO] server-6163e78d-d2c6-0207-021c-580c19332fd3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9335ae3d-a47c-730f-dda1-bb94a4c94c3f Address:9335ae3d-a47c-730f-dda1-bb94a4c94c3f} {Suffrage:Voter ID:server-6163e78d-d2c6-0207-021c-580c19332fd3 Address:6163e78d-d2c6-0207-021c-580c19332fd3}]" -2026-03-31T06:16:20.942Z [INFO] server-9335ae3d-a47c-730f-dda1-bb94a4c94c3f: entering follower state: follower="Node at 9335ae3d-a47c-730f-dda1-bb94a4c94c3f [Follower]" leader-address= leader-id= -2026-03-31T06:16:20.942Z [INFO] server-6163e78d-d2c6-0207-021c-580c19332fd3: entering follower state: follower="Node at 6163e78d-d2c6-0207-021c-580c19332fd3 [Follower]" leader-address= leader-id= -2026-03-31T06:16:20.942Z [INFO] server-6e984533-1245-9731-be03-46cba4baba10: initial configuration: index=0 servers=[] -2026-03-31T06:16:20.942Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:20.942Z [INFO] server-6e984533-1245-9731-be03-46cba4baba10: entering follower state: follower="Node at 6e984533-1245-9731-be03-46cba4baba10 [Follower]" leader-address= leader-id= -2026-03-31T06:16:20.998Z [WARN] server-6163e78d-d2c6-0207-021c-580c19332fd3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:20.998Z [INFO] server-6163e78d-d2c6-0207-021c-580c19332fd3: entering candidate state: node="Node at 6163e78d-d2c6-0207-021c-580c19332fd3 [Candidate]" term=2 -2026-03-31T06:16:20.998Z [INFO] server-6163e78d-d2c6-0207-021c-580c19332fd3: election won: term=2 tally=2 -2026-03-31T06:16:20.998Z [INFO] server-6163e78d-d2c6-0207-021c-580c19332fd3: entering leader state: leader="Node at 6163e78d-d2c6-0207-021c-580c19332fd3 [Leader]" -2026-03-31T06:16:20.998Z [INFO] server-6163e78d-d2c6-0207-021c-580c19332fd3: added peer, starting replication: peer=server-9335ae3d-a47c-730f-dda1-bb94a4c94c3f -2026-03-31T06:16:20.998Z [INFO] server-6163e78d-d2c6-0207-021c-580c19332fd3: pipelining replication: peer="{Voter server-9335ae3d-a47c-730f-dda1-bb94a4c94c3f 9335ae3d-a47c-730f-dda1-bb94a4c94c3f}" -2026-03-31T06:16:21.034Z [WARN] server-6e984533-1245-9731-be03-46cba4baba10: no known peers, aborting election -2026-03-31T06:16:21.104Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:20.99881058 +0000 UTC m=+9.276417961 (1 nodes), 56.06148ms from start of poll, 56.52536ms from cluster start. Timeout at 2026-03-31 06:16:21.10381398 +0000 UTC m=+9.381421361, 105.0034ms after stability -2026-03-31T06:16:21.104Z [INFO] server-6163e78d-d2c6-0207-021c-580c19332fd3: updating configuration: command=AddVoter server-id=server-6e984533-1245-9731-be03-46cba4baba10 server-addr=6e984533-1245-9731-be03-46cba4baba10 servers="[{Suffrage:Voter ID:server-9335ae3d-a47c-730f-dda1-bb94a4c94c3f Address:9335ae3d-a47c-730f-dda1-bb94a4c94c3f} {Suffrage:Voter ID:server-6163e78d-d2c6-0207-021c-580c19332fd3 Address:6163e78d-d2c6-0207-021c-580c19332fd3} {Suffrage:Voter ID:server-6e984533-1245-9731-be03-46cba4baba10 Address:6e984533-1245-9731-be03-46cba4baba10}]" -2026-03-31T06:16:21.104Z [INFO] server-6163e78d-d2c6-0207-021c-580c19332fd3: added peer, starting replication: peer=server-6e984533-1245-9731-be03-46cba4baba10 -2026-03-31T06:16:21.104Z [WARN] server-6e984533-1245-9731-be03-46cba4baba10: failed to get previous log: previous-index=3 last-index=0 error="log not found" -2026-03-31T06:16:21.104Z [WARN] server-6163e78d-d2c6-0207-021c-580c19332fd3: appendEntries rejected, sending older logs: peer="{Voter server-6e984533-1245-9731-be03-46cba4baba10 6e984533-1245-9731-be03-46cba4baba10}" next=1 -2026-03-31T06:16:21.105Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:21.105Z [INFO] server-6163e78d-d2c6-0207-021c-580c19332fd3: pipelining replication: peer="{Voter server-6e984533-1245-9731-be03-46cba4baba10 6e984533-1245-9731-be03-46cba4baba10}" -2026-03-31T06:16:21.210Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:21.10510234 +0000 UTC m=+9.382709721 (1 nodes), 1.08µs from start of poll, 162.81712ms from cluster start. Timeout at 2026-03-31 06:16:21.21010614 +0000 UTC m=+9.487713521, 105.0038ms after stability -2026-03-31T06:16:21.210Z [INFO] server-6163e78d-d2c6-0207-021c-580c19332fd3: aborting pipeline replication: peer="{Voter server-9335ae3d-a47c-730f-dda1-bb94a4c94c3f 9335ae3d-a47c-730f-dda1-bb94a4c94c3f}" -2026-03-31T06:16:21.210Z [INFO] server-6163e78d-d2c6-0207-021c-580c19332fd3: aborting pipeline replication: peer="{Voter server-6e984533-1245-9731-be03-46cba4baba10 6e984533-1245-9731-be03-46cba4baba10}" ---- PASS: TestRaft_JoinNode (0.27s) +2025-02-25T23:57:42.256Z [INFO] server-d1a08c1a-d4b9-38e5-2ab1-c73d72a6e2fd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d1a08c1a-d4b9-38e5-2ab1-c73d72a6e2fd Address:d1a08c1a-d4b9-38e5-2ab1-c73d72a6e2fd} {Suffrage:Voter ID:server-0fe2fff9-70fd-1d63-ba28-80c9b47cd065 Address:0fe2fff9-70fd-1d63-ba28-80c9b47cd065}]" +2025-02-25T23:57:42.256Z [INFO] server-0fe2fff9-70fd-1d63-ba28-80c9b47cd065: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d1a08c1a-d4b9-38e5-2ab1-c73d72a6e2fd Address:d1a08c1a-d4b9-38e5-2ab1-c73d72a6e2fd} {Suffrage:Voter ID:server-0fe2fff9-70fd-1d63-ba28-80c9b47cd065 Address:0fe2fff9-70fd-1d63-ba28-80c9b47cd065}]" +2025-02-25T23:57:42.256Z [INFO] server-d796507a-62ce-6993-c53f-3e7af52c63ab: initial configuration: index=0 servers=[] +2025-02-25T23:57:42.256Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:42.256Z [INFO] server-d1a08c1a-d4b9-38e5-2ab1-c73d72a6e2fd: entering follower state: follower="Node at d1a08c1a-d4b9-38e5-2ab1-c73d72a6e2fd [Follower]" leader-address= leader-id= +2025-02-25T23:57:42.256Z [INFO] server-0fe2fff9-70fd-1d63-ba28-80c9b47cd065: entering follower state: follower="Node at 0fe2fff9-70fd-1d63-ba28-80c9b47cd065 [Follower]" leader-address= leader-id= +2025-02-25T23:57:42.256Z [INFO] server-d796507a-62ce-6993-c53f-3e7af52c63ab: entering follower state: follower="Node at d796507a-62ce-6993-c53f-3e7af52c63ab [Follower]" leader-address= leader-id= +2025-02-25T23:57:42.322Z [WARN] server-d796507a-62ce-6993-c53f-3e7af52c63ab: no known peers, aborting election +2025-02-25T23:57:42.333Z [WARN] server-d1a08c1a-d4b9-38e5-2ab1-c73d72a6e2fd: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:42.333Z [INFO] server-d1a08c1a-d4b9-38e5-2ab1-c73d72a6e2fd: entering candidate state: node="Node at d1a08c1a-d4b9-38e5-2ab1-c73d72a6e2fd [Candidate]" term=2 +2025-02-25T23:57:42.333Z [WARN] server-0fe2fff9-70fd-1d63-ba28-80c9b47cd065: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:42.333Z [INFO] server-0fe2fff9-70fd-1d63-ba28-80c9b47cd065: entering candidate state: node="Node at 0fe2fff9-70fd-1d63-ba28-80c9b47cd065 [Candidate]" term=2 +2025-02-25T23:57:42.333Z [INFO] server-0fe2fff9-70fd-1d63-ba28-80c9b47cd065: duplicate requestVote for same term: term=2 +2025-02-25T23:57:42.334Z [INFO] server-d1a08c1a-d4b9-38e5-2ab1-c73d72a6e2fd: duplicate requestVote for same term: term=2 +2025-02-25T23:57:42.389Z [WARN] server-d1a08c1a-d4b9-38e5-2ab1-c73d72a6e2fd: Election timeout reached, restarting election +2025-02-25T23:57:42.389Z [INFO] server-d1a08c1a-d4b9-38e5-2ab1-c73d72a6e2fd: entering candidate state: node="Node at d1a08c1a-d4b9-38e5-2ab1-c73d72a6e2fd [Candidate]" term=3 +2025-02-25T23:57:42.389Z [INFO] server-0fe2fff9-70fd-1d63-ba28-80c9b47cd065: entering follower state: follower="Node at 0fe2fff9-70fd-1d63-ba28-80c9b47cd065 [Follower]" leader-address= leader-id= +2025-02-25T23:57:42.389Z [INFO] server-d1a08c1a-d4b9-38e5-2ab1-c73d72a6e2fd: election won: term=3 tally=2 +2025-02-25T23:57:42.389Z [INFO] server-d1a08c1a-d4b9-38e5-2ab1-c73d72a6e2fd: entering leader state: leader="Node at d1a08c1a-d4b9-38e5-2ab1-c73d72a6e2fd [Leader]" +2025-02-25T23:57:42.389Z [INFO] server-d1a08c1a-d4b9-38e5-2ab1-c73d72a6e2fd: added peer, starting replication: peer=server-0fe2fff9-70fd-1d63-ba28-80c9b47cd065 +2025-02-25T23:57:42.390Z [INFO] server-d1a08c1a-d4b9-38e5-2ab1-c73d72a6e2fd: pipelining replication: peer="{Voter server-0fe2fff9-70fd-1d63-ba28-80c9b47cd065 0fe2fff9-70fd-1d63-ba28-80c9b47cd065}" +2025-02-25T23:57:42.497Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:42.390091201 +0000 UTC m=+9.999106608 (1 nodes), 133.39317ms from start of poll, 133.581371ms from cluster start. Timeout at 2025-02-25 23:57:42.495097121 +0000 UTC m=+10.104112488, 105.00588ms after stability +2025-02-25T23:57:42.497Z [INFO] server-d1a08c1a-d4b9-38e5-2ab1-c73d72a6e2fd: updating configuration: command=AddVoter server-id=server-d796507a-62ce-6993-c53f-3e7af52c63ab server-addr=d796507a-62ce-6993-c53f-3e7af52c63ab servers="[{Suffrage:Voter ID:server-d1a08c1a-d4b9-38e5-2ab1-c73d72a6e2fd Address:d1a08c1a-d4b9-38e5-2ab1-c73d72a6e2fd} {Suffrage:Voter ID:server-0fe2fff9-70fd-1d63-ba28-80c9b47cd065 Address:0fe2fff9-70fd-1d63-ba28-80c9b47cd065} {Suffrage:Voter ID:server-d796507a-62ce-6993-c53f-3e7af52c63ab Address:d796507a-62ce-6993-c53f-3e7af52c63ab}]" +2025-02-25T23:57:42.497Z [INFO] server-d1a08c1a-d4b9-38e5-2ab1-c73d72a6e2fd: added peer, starting replication: peer=server-d796507a-62ce-6993-c53f-3e7af52c63ab +2025-02-25T23:57:42.498Z [WARN] server-d796507a-62ce-6993-c53f-3e7af52c63ab: failed to get previous log: previous-index=3 last-index=0 error="log not found" +2025-02-25T23:57:42.498Z [WARN] server-d1a08c1a-d4b9-38e5-2ab1-c73d72a6e2fd: appendEntries rejected, sending older logs: peer="{Voter server-d796507a-62ce-6993-c53f-3e7af52c63ab d796507a-62ce-6993-c53f-3e7af52c63ab}" next=1 +2025-02-25T23:57:42.498Z [INFO] server-d1a08c1a-d4b9-38e5-2ab1-c73d72a6e2fd: pipelining replication: peer="{Voter server-d796507a-62ce-6993-c53f-3e7af52c63ab d796507a-62ce-6993-c53f-3e7af52c63ab}" +2025-02-25T23:57:42.498Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:42.605Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:42.498185289 +0000 UTC m=+10.107200656 (1 nodes), 760ns from start of poll, 241.675419ms from cluster start. Timeout at 2025-02-25 23:57:42.603189369 +0000 UTC m=+10.212204776, 105.00412ms after stability +2025-02-25T23:57:42.606Z [INFO] server-d1a08c1a-d4b9-38e5-2ab1-c73d72a6e2fd: aborting pipeline replication: peer="{Voter server-0fe2fff9-70fd-1d63-ba28-80c9b47cd065 0fe2fff9-70fd-1d63-ba28-80c9b47cd065}" +2025-02-25T23:57:42.606Z [INFO] server-d1a08c1a-d4b9-38e5-2ab1-c73d72a6e2fd: aborting pipeline replication: peer="{Voter server-d796507a-62ce-6993-c53f-3e7af52c63ab d796507a-62ce-6993-c53f-3e7af52c63ab}" +--- PASS: TestRaft_JoinNode (0.35s) === RUN TestRaft_JoinNode_ConfigStore -2026-03-31T06:16:21.211Z [INFO] server-58fdd232-3ed7-6d1d-fb99-4a69719cef94: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-58fdd232-3ed7-6d1d-fb99-4a69719cef94 Address:58fdd232-3ed7-6d1d-fb99-4a69719cef94}]" -2026-03-31T06:16:21.211Z [INFO] server-58fdd232-3ed7-6d1d-fb99-4a69719cef94: entering follower state: follower="Node at 58fdd232-3ed7-6d1d-fb99-4a69719cef94 [Follower]" leader-address= leader-id= -2026-03-31T06:16:21.211Z [INFO] server-b88643c9-13dd-bac9-6875-4f9622c41007: initial configuration: index=0 servers=[] -2026-03-31T06:16:21.211Z [INFO] server-b88643c9-13dd-bac9-6875-4f9622c41007: entering follower state: follower="Node at b88643c9-13dd-bac9-6875-4f9622c41007 [Follower]" leader-address= leader-id= -2026-03-31T06:16:21.211Z [INFO] server-f27d6332-7234-8e36-704a-aae9f4b32b16: initial configuration: index=0 servers=[] -2026-03-31T06:16:21.211Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:21.211Z [INFO] server-f27d6332-7234-8e36-704a-aae9f4b32b16: entering follower state: follower="Node at f27d6332-7234-8e36-704a-aae9f4b32b16 [Follower]" leader-address= leader-id= -2026-03-31T06:16:21.274Z [WARN] server-b88643c9-13dd-bac9-6875-4f9622c41007: no known peers, aborting election -2026-03-31T06:16:21.295Z [WARN] server-f27d6332-7234-8e36-704a-aae9f4b32b16: no known peers, aborting election -2026-03-31T06:16:21.296Z [WARN] server-58fdd232-3ed7-6d1d-fb99-4a69719cef94: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:21.296Z [INFO] server-58fdd232-3ed7-6d1d-fb99-4a69719cef94: entering candidate state: node="Node at 58fdd232-3ed7-6d1d-fb99-4a69719cef94 [Candidate]" term=2 -2026-03-31T06:16:21.296Z [INFO] server-58fdd232-3ed7-6d1d-fb99-4a69719cef94: election won: term=2 tally=1 -2026-03-31T06:16:21.296Z [INFO] server-58fdd232-3ed7-6d1d-fb99-4a69719cef94: entering leader state: leader="Node at 58fdd232-3ed7-6d1d-fb99-4a69719cef94 [Leader]" -2026-03-31T06:16:21.402Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:21.29681234 +0000 UTC m=+9.574419721 (1 nodes), 85.0566ms from start of poll, 85.56656ms from cluster start. Timeout at 2026-03-31 06:16:21.4018137 +0000 UTC m=+9.679421081, 105.00136ms after stability -2026-03-31T06:16:21.402Z [INFO] server-58fdd232-3ed7-6d1d-fb99-4a69719cef94: updating configuration: command=AddVoter server-id=server-b88643c9-13dd-bac9-6875-4f9622c41007 server-addr=b88643c9-13dd-bac9-6875-4f9622c41007 servers="[{Suffrage:Voter ID:server-58fdd232-3ed7-6d1d-fb99-4a69719cef94 Address:58fdd232-3ed7-6d1d-fb99-4a69719cef94} {Suffrage:Voter ID:server-b88643c9-13dd-bac9-6875-4f9622c41007 Address:b88643c9-13dd-bac9-6875-4f9622c41007}]" -2026-03-31T06:16:21.402Z [INFO] server-58fdd232-3ed7-6d1d-fb99-4a69719cef94: added peer, starting replication: peer=server-b88643c9-13dd-bac9-6875-4f9622c41007 -2026-03-31T06:16:21.402Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:21.402Z [WARN] server-b88643c9-13dd-bac9-6875-4f9622c41007: failed to get previous log: previous-index=3 last-index=0 error="log not found" -2026-03-31T06:16:21.402Z [WARN] server-58fdd232-3ed7-6d1d-fb99-4a69719cef94: appendEntries rejected, sending older logs: peer="{Voter server-b88643c9-13dd-bac9-6875-4f9622c41007 b88643c9-13dd-bac9-6875-4f9622c41007}" next=1 -2026-03-31T06:16:21.402Z [INFO] server-58fdd232-3ed7-6d1d-fb99-4a69719cef94: pipelining replication: peer="{Voter server-b88643c9-13dd-bac9-6875-4f9622c41007 b88643c9-13dd-bac9-6875-4f9622c41007}" -2026-03-31T06:16:21.508Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:21.4026179 +0000 UTC m=+9.680225281 (1 nodes), 520ns from start of poll, 191.37212ms from cluster start. Timeout at 2026-03-31 06:16:21.5076233 +0000 UTC m=+9.785230681, 105.0054ms after stability -2026-03-31T06:16:21.508Z [INFO] server-58fdd232-3ed7-6d1d-fb99-4a69719cef94: updating configuration: command=AddVoter server-id=server-f27d6332-7234-8e36-704a-aae9f4b32b16 server-addr=f27d6332-7234-8e36-704a-aae9f4b32b16 servers="[{Suffrage:Voter ID:server-58fdd232-3ed7-6d1d-fb99-4a69719cef94 Address:58fdd232-3ed7-6d1d-fb99-4a69719cef94} {Suffrage:Voter ID:server-b88643c9-13dd-bac9-6875-4f9622c41007 Address:b88643c9-13dd-bac9-6875-4f9622c41007} {Suffrage:Voter ID:server-f27d6332-7234-8e36-704a-aae9f4b32b16 Address:f27d6332-7234-8e36-704a-aae9f4b32b16}]" -2026-03-31T06:16:21.508Z [INFO] server-58fdd232-3ed7-6d1d-fb99-4a69719cef94: added peer, starting replication: peer=server-f27d6332-7234-8e36-704a-aae9f4b32b16 -2026-03-31T06:16:21.508Z [WARN] server-f27d6332-7234-8e36-704a-aae9f4b32b16: failed to get previous log: previous-index=4 last-index=0 error="log not found" -2026-03-31T06:16:21.509Z [WARN] server-58fdd232-3ed7-6d1d-fb99-4a69719cef94: appendEntries rejected, sending older logs: peer="{Voter server-f27d6332-7234-8e36-704a-aae9f4b32b16 f27d6332-7234-8e36-704a-aae9f4b32b16}" next=1 -2026-03-31T06:16:21.509Z [INFO] server-58fdd232-3ed7-6d1d-fb99-4a69719cef94: pipelining replication: peer="{Voter server-f27d6332-7234-8e36-704a-aae9f4b32b16 f27d6332-7234-8e36-704a-aae9f4b32b16}" -2026-03-31T06:16:21.509Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:21.614Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:21.50921238 +0000 UTC m=+9.786819761 (1 nodes), 720ns from start of poll, 297.9666ms from cluster start. Timeout at 2026-03-31 06:16:21.61421682 +0000 UTC m=+9.891824201, 105.00444ms after stability -2026-03-31T06:16:21.614Z [INFO] server-58fdd232-3ed7-6d1d-fb99-4a69719cef94: aborting pipeline replication: peer="{Voter server-f27d6332-7234-8e36-704a-aae9f4b32b16 f27d6332-7234-8e36-704a-aae9f4b32b16}" -2026-03-31T06:16:21.614Z [INFO] server-58fdd232-3ed7-6d1d-fb99-4a69719cef94: aborting pipeline replication: peer="{Voter server-b88643c9-13dd-bac9-6875-4f9622c41007 b88643c9-13dd-bac9-6875-4f9622c41007}" ---- PASS: TestRaft_JoinNode_ConfigStore (0.40s) +2025-02-25T23:57:42.606Z [INFO] server-e56709e6-d7ae-1526-430d-dece2125a867: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e56709e6-d7ae-1526-430d-dece2125a867 Address:e56709e6-d7ae-1526-430d-dece2125a867}]" +2025-02-25T23:57:42.606Z [INFO] server-e0ac9a7c-21b9-669d-b02e-46cb6ef2be0d: initial configuration: index=0 servers=[] +2025-02-25T23:57:42.606Z [INFO] server-e83fbd61-411d-7ffb-5154-8b8d74e39104: initial configuration: index=0 servers=[] +2025-02-25T23:57:42.606Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:42.606Z [INFO] server-e56709e6-d7ae-1526-430d-dece2125a867: entering follower state: follower="Node at e56709e6-d7ae-1526-430d-dece2125a867 [Follower]" leader-address= leader-id= +2025-02-25T23:57:42.606Z [INFO] server-e0ac9a7c-21b9-669d-b02e-46cb6ef2be0d: entering follower state: follower="Node at e0ac9a7c-21b9-669d-b02e-46cb6ef2be0d [Follower]" leader-address= leader-id= +2025-02-25T23:57:42.606Z [INFO] server-e83fbd61-411d-7ffb-5154-8b8d74e39104: entering follower state: follower="Node at e83fbd61-411d-7ffb-5154-8b8d74e39104 [Follower]" leader-address= leader-id= +2025-02-25T23:57:42.677Z [WARN] server-e83fbd61-411d-7ffb-5154-8b8d74e39104: no known peers, aborting election +2025-02-25T23:57:42.690Z [WARN] server-e0ac9a7c-21b9-669d-b02e-46cb6ef2be0d: no known peers, aborting election +2025-02-25T23:57:42.705Z [WARN] server-e56709e6-d7ae-1526-430d-dece2125a867: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:42.705Z [INFO] server-e56709e6-d7ae-1526-430d-dece2125a867: entering candidate state: node="Node at e56709e6-d7ae-1526-430d-dece2125a867 [Candidate]" term=2 +2025-02-25T23:57:42.705Z [INFO] server-e56709e6-d7ae-1526-430d-dece2125a867: election won: term=2 tally=1 +2025-02-25T23:57:42.705Z [INFO] server-e56709e6-d7ae-1526-430d-dece2125a867: entering leader state: leader="Node at e56709e6-d7ae-1526-430d-dece2125a867 [Leader]" +2025-02-25T23:57:42.811Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:42.706052053 +0000 UTC m=+10.315067420 (1 nodes), 99.535954ms from start of poll, 99.767395ms from cluster start. Timeout at 2025-02-25 23:57:42.811054533 +0000 UTC m=+10.420069980, 105.00256ms after stability +2025-02-25T23:57:42.811Z [INFO] server-e56709e6-d7ae-1526-430d-dece2125a867: updating configuration: command=AddVoter server-id=server-e0ac9a7c-21b9-669d-b02e-46cb6ef2be0d server-addr=e0ac9a7c-21b9-669d-b02e-46cb6ef2be0d servers="[{Suffrage:Voter ID:server-e56709e6-d7ae-1526-430d-dece2125a867 Address:e56709e6-d7ae-1526-430d-dece2125a867} {Suffrage:Voter ID:server-e0ac9a7c-21b9-669d-b02e-46cb6ef2be0d Address:e0ac9a7c-21b9-669d-b02e-46cb6ef2be0d}]" +2025-02-25T23:57:42.811Z [INFO] server-e56709e6-d7ae-1526-430d-dece2125a867: added peer, starting replication: peer=server-e0ac9a7c-21b9-669d-b02e-46cb6ef2be0d +2025-02-25T23:57:42.811Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:42.811Z [WARN] server-e0ac9a7c-21b9-669d-b02e-46cb6ef2be0d: failed to get previous log: previous-index=3 last-index=0 error="log not found" +2025-02-25T23:57:42.811Z [WARN] server-e56709e6-d7ae-1526-430d-dece2125a867: appendEntries rejected, sending older logs: peer="{Voter server-e0ac9a7c-21b9-669d-b02e-46cb6ef2be0d e0ac9a7c-21b9-669d-b02e-46cb6ef2be0d}" next=1 +2025-02-25T23:57:42.811Z [INFO] server-e56709e6-d7ae-1526-430d-dece2125a867: pipelining replication: peer="{Voter server-e0ac9a7c-21b9-669d-b02e-46cb6ef2be0d e0ac9a7c-21b9-669d-b02e-46cb6ef2be0d}" +2025-02-25T23:57:42.917Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:42.811608645 +0000 UTC m=+10.420624012 (1 nodes), 440ns from start of poll, 205.323987ms from cluster start. Timeout at 2025-02-25 23:57:42.916616965 +0000 UTC m=+10.525632372, 105.00836ms after stability +2025-02-25T23:57:42.917Z [INFO] server-e56709e6-d7ae-1526-430d-dece2125a867: updating configuration: command=AddVoter server-id=server-e83fbd61-411d-7ffb-5154-8b8d74e39104 server-addr=e83fbd61-411d-7ffb-5154-8b8d74e39104 servers="[{Suffrage:Voter ID:server-e56709e6-d7ae-1526-430d-dece2125a867 Address:e56709e6-d7ae-1526-430d-dece2125a867} {Suffrage:Voter ID:server-e0ac9a7c-21b9-669d-b02e-46cb6ef2be0d Address:e0ac9a7c-21b9-669d-b02e-46cb6ef2be0d} {Suffrage:Voter ID:server-e83fbd61-411d-7ffb-5154-8b8d74e39104 Address:e83fbd61-411d-7ffb-5154-8b8d74e39104}]" +2025-02-25T23:57:42.917Z [INFO] server-e56709e6-d7ae-1526-430d-dece2125a867: added peer, starting replication: peer=server-e83fbd61-411d-7ffb-5154-8b8d74e39104 +2025-02-25T23:57:42.918Z [WARN] server-e83fbd61-411d-7ffb-5154-8b8d74e39104: failed to get previous log: previous-index=4 last-index=0 error="log not found" +2025-02-25T23:57:42.918Z [WARN] server-e56709e6-d7ae-1526-430d-dece2125a867: appendEntries rejected, sending older logs: peer="{Voter server-e83fbd61-411d-7ffb-5154-8b8d74e39104 e83fbd61-411d-7ffb-5154-8b8d74e39104}" next=1 +2025-02-25T23:57:42.918Z [INFO] server-e56709e6-d7ae-1526-430d-dece2125a867: pipelining replication: peer="{Voter server-e83fbd61-411d-7ffb-5154-8b8d74e39104 e83fbd61-411d-7ffb-5154-8b8d74e39104}" +2025-02-25T23:57:42.918Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:43.025Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:42.918207844 +0000 UTC m=+10.527223251 (1 nodes), 480ns from start of poll, 311.923226ms from cluster start. Timeout at 2025-02-25 23:57:43.023214204 +0000 UTC m=+10.632229611, 105.00636ms after stability +2025-02-25T23:57:43.026Z [INFO] server-e56709e6-d7ae-1526-430d-dece2125a867: aborting pipeline replication: peer="{Voter server-e83fbd61-411d-7ffb-5154-8b8d74e39104 e83fbd61-411d-7ffb-5154-8b8d74e39104}" +2025-02-25T23:57:43.026Z [INFO] server-e56709e6-d7ae-1526-430d-dece2125a867: aborting pipeline replication: peer="{Voter server-e0ac9a7c-21b9-669d-b02e-46cb6ef2be0d e0ac9a7c-21b9-669d-b02e-46cb6ef2be0d}" +--- PASS: TestRaft_JoinNode_ConfigStore (0.42s) === RUN TestRaft_RemoveFollower -2026-03-31T06:16:21.615Z [INFO] server-80f53b23-acd0-3cc7-7ae2-87ba2a93a672: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-80f53b23-acd0-3cc7-7ae2-87ba2a93a672 Address:80f53b23-acd0-3cc7-7ae2-87ba2a93a672} {Suffrage:Voter ID:server-ae99ad6e-813e-bb0a-abb8-e2b73d7148a9 Address:ae99ad6e-813e-bb0a-abb8-e2b73d7148a9} {Suffrage:Voter ID:server-744f63e4-fb77-a40c-c79d-75a8ff36341e Address:744f63e4-fb77-a40c-c79d-75a8ff36341e}]" -2026-03-31T06:16:21.615Z [INFO] server-80f53b23-acd0-3cc7-7ae2-87ba2a93a672: entering follower state: follower="Node at 80f53b23-acd0-3cc7-7ae2-87ba2a93a672 [Follower]" leader-address= leader-id= -2026-03-31T06:16:21.615Z [INFO] server-ae99ad6e-813e-bb0a-abb8-e2b73d7148a9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-80f53b23-acd0-3cc7-7ae2-87ba2a93a672 Address:80f53b23-acd0-3cc7-7ae2-87ba2a93a672} {Suffrage:Voter ID:server-ae99ad6e-813e-bb0a-abb8-e2b73d7148a9 Address:ae99ad6e-813e-bb0a-abb8-e2b73d7148a9} {Suffrage:Voter ID:server-744f63e4-fb77-a40c-c79d-75a8ff36341e Address:744f63e4-fb77-a40c-c79d-75a8ff36341e}]" -2026-03-31T06:16:21.615Z [INFO] server-744f63e4-fb77-a40c-c79d-75a8ff36341e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-80f53b23-acd0-3cc7-7ae2-87ba2a93a672 Address:80f53b23-acd0-3cc7-7ae2-87ba2a93a672} {Suffrage:Voter ID:server-ae99ad6e-813e-bb0a-abb8-e2b73d7148a9 Address:ae99ad6e-813e-bb0a-abb8-e2b73d7148a9} {Suffrage:Voter ID:server-744f63e4-fb77-a40c-c79d-75a8ff36341e Address:744f63e4-fb77-a40c-c79d-75a8ff36341e}]" -2026-03-31T06:16:21.615Z [INFO] server-ae99ad6e-813e-bb0a-abb8-e2b73d7148a9: entering follower state: follower="Node at ae99ad6e-813e-bb0a-abb8-e2b73d7148a9 [Follower]" leader-address= leader-id= -2026-03-31T06:16:21.615Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:21.615Z [INFO] server-744f63e4-fb77-a40c-c79d-75a8ff36341e: entering follower state: follower="Node at 744f63e4-fb77-a40c-c79d-75a8ff36341e [Follower]" leader-address= leader-id= -2026-03-31T06:16:21.666Z [WARN] server-ae99ad6e-813e-bb0a-abb8-e2b73d7148a9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:21.666Z [INFO] server-ae99ad6e-813e-bb0a-abb8-e2b73d7148a9: entering candidate state: node="Node at ae99ad6e-813e-bb0a-abb8-e2b73d7148a9 [Candidate]" term=2 -2026-03-31T06:16:21.666Z [INFO] server-ae99ad6e-813e-bb0a-abb8-e2b73d7148a9: election won: term=2 tally=2 -2026-03-31T06:16:21.666Z [INFO] server-ae99ad6e-813e-bb0a-abb8-e2b73d7148a9: entering leader state: leader="Node at ae99ad6e-813e-bb0a-abb8-e2b73d7148a9 [Leader]" -2026-03-31T06:16:21.666Z [INFO] server-ae99ad6e-813e-bb0a-abb8-e2b73d7148a9: added peer, starting replication: peer=server-80f53b23-acd0-3cc7-7ae2-87ba2a93a672 -2026-03-31T06:16:21.666Z [INFO] server-ae99ad6e-813e-bb0a-abb8-e2b73d7148a9: added peer, starting replication: peer=server-744f63e4-fb77-a40c-c79d-75a8ff36341e -2026-03-31T06:16:21.667Z [INFO] server-ae99ad6e-813e-bb0a-abb8-e2b73d7148a9: pipelining replication: peer="{Voter server-80f53b23-acd0-3cc7-7ae2-87ba2a93a672 80f53b23-acd0-3cc7-7ae2-87ba2a93a672}" -2026-03-31T06:16:21.667Z [INFO] server-ae99ad6e-813e-bb0a-abb8-e2b73d7148a9: pipelining replication: peer="{Voter server-744f63e4-fb77-a40c-c79d-75a8ff36341e 744f63e4-fb77-a40c-c79d-75a8ff36341e}" -2026-03-31T06:16:21.772Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:21.66741822 +0000 UTC m=+9.945025601 (1 nodes), 51.6744ms from start of poll, 52.12216ms from cluster start. Timeout at 2026-03-31 06:16:21.7724201 +0000 UTC m=+10.050027481, 105.00188ms after stability -2026-03-31T06:16:21.778Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:16:21.884Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:16:21.7786595 +0000 UTC m=+10.056266881 (2 nodes), 2.32µs from start of poll, 163.36344ms from cluster start. Timeout at 2026-03-31 06:16:21.88366522 +0000 UTC m=+10.161272601, 105.00572ms after stability -2026-03-31T06:16:21.884Z [INFO] server-ae99ad6e-813e-bb0a-abb8-e2b73d7148a9: updating configuration: command=RemoveServer server-id=server-80f53b23-acd0-3cc7-7ae2-87ba2a93a672 server-addr= servers="[{Suffrage:Voter ID:server-ae99ad6e-813e-bb0a-abb8-e2b73d7148a9 Address:ae99ad6e-813e-bb0a-abb8-e2b73d7148a9} {Suffrage:Voter ID:server-744f63e4-fb77-a40c-c79d-75a8ff36341e Address:744f63e4-fb77-a40c-c79d-75a8ff36341e}]" -2026-03-31T06:16:21.884Z [INFO] server-ae99ad6e-813e-bb0a-abb8-e2b73d7148a9: removed peer, stopping replication: peer=server-80f53b23-acd0-3cc7-7ae2-87ba2a93a672 last-index=3 -2026-03-31T06:16:21.884Z [INFO] server-ae99ad6e-813e-bb0a-abb8-e2b73d7148a9: aborting pipeline replication: peer="{Voter server-80f53b23-acd0-3cc7-7ae2-87ba2a93a672 80f53b23-acd0-3cc7-7ae2-87ba2a93a672}" -2026-03-31T06:16:21.977Z [WARN] server-80f53b23-acd0-3cc7-7ae2-87ba2a93a672: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election -2026-03-31T06:16:21.990Z [INFO] server-ae99ad6e-813e-bb0a-abb8-e2b73d7148a9: aborting pipeline replication: peer="{Voter server-744f63e4-fb77-a40c-c79d-75a8ff36341e 744f63e4-fb77-a40c-c79d-75a8ff36341e}" -2026-03-31T06:16:22.491Z [ERROR] server-ae99ad6e-813e-bb0a-abb8-e2b73d7148a9: failed to heartbeat to: peer=744f63e4-fb77-a40c-c79d-75a8ff36341e backoff time=10ms error="command timed out" ---- PASS: TestRaft_RemoveFollower (0.88s) +2025-02-25T23:57:43.026Z [INFO] server-63c2b480-ec7a-eaab-995e-23023d54d8dd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-63c2b480-ec7a-eaab-995e-23023d54d8dd Address:63c2b480-ec7a-eaab-995e-23023d54d8dd} {Suffrage:Voter ID:server-e72cd380-2139-c201-40ab-7d4fcb3de510 Address:e72cd380-2139-c201-40ab-7d4fcb3de510} {Suffrage:Voter ID:server-26a658ec-ad64-4df7-e11e-55e586a66852 Address:26a658ec-ad64-4df7-e11e-55e586a66852}]" +2025-02-25T23:57:43.026Z [INFO] server-e72cd380-2139-c201-40ab-7d4fcb3de510: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-63c2b480-ec7a-eaab-995e-23023d54d8dd Address:63c2b480-ec7a-eaab-995e-23023d54d8dd} {Suffrage:Voter ID:server-e72cd380-2139-c201-40ab-7d4fcb3de510 Address:e72cd380-2139-c201-40ab-7d4fcb3de510} {Suffrage:Voter ID:server-26a658ec-ad64-4df7-e11e-55e586a66852 Address:26a658ec-ad64-4df7-e11e-55e586a66852}]" +2025-02-25T23:57:43.026Z [INFO] server-26a658ec-ad64-4df7-e11e-55e586a66852: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-63c2b480-ec7a-eaab-995e-23023d54d8dd Address:63c2b480-ec7a-eaab-995e-23023d54d8dd} {Suffrage:Voter ID:server-e72cd380-2139-c201-40ab-7d4fcb3de510 Address:e72cd380-2139-c201-40ab-7d4fcb3de510} {Suffrage:Voter ID:server-26a658ec-ad64-4df7-e11e-55e586a66852 Address:26a658ec-ad64-4df7-e11e-55e586a66852}]" +2025-02-25T23:57:43.026Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:43.026Z [INFO] server-63c2b480-ec7a-eaab-995e-23023d54d8dd: entering follower state: follower="Node at 63c2b480-ec7a-eaab-995e-23023d54d8dd [Follower]" leader-address= leader-id= +2025-02-25T23:57:43.026Z [INFO] server-e72cd380-2139-c201-40ab-7d4fcb3de510: entering follower state: follower="Node at e72cd380-2139-c201-40ab-7d4fcb3de510 [Follower]" leader-address= leader-id= +2025-02-25T23:57:43.026Z [INFO] server-26a658ec-ad64-4df7-e11e-55e586a66852: entering follower state: follower="Node at 26a658ec-ad64-4df7-e11e-55e586a66852 [Follower]" leader-address= leader-id= +2025-02-25T23:57:43.089Z [WARN] server-63c2b480-ec7a-eaab-995e-23023d54d8dd: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:43.089Z [INFO] server-63c2b480-ec7a-eaab-995e-23023d54d8dd: entering candidate state: node="Node at 63c2b480-ec7a-eaab-995e-23023d54d8dd [Candidate]" term=2 +2025-02-25T23:57:43.090Z [INFO] server-63c2b480-ec7a-eaab-995e-23023d54d8dd: election won: term=2 tally=2 +2025-02-25T23:57:43.090Z [INFO] server-63c2b480-ec7a-eaab-995e-23023d54d8dd: entering leader state: leader="Node at 63c2b480-ec7a-eaab-995e-23023d54d8dd [Leader]" +2025-02-25T23:57:43.090Z [INFO] server-63c2b480-ec7a-eaab-995e-23023d54d8dd: added peer, starting replication: peer=server-e72cd380-2139-c201-40ab-7d4fcb3de510 +2025-02-25T23:57:43.090Z [INFO] server-63c2b480-ec7a-eaab-995e-23023d54d8dd: added peer, starting replication: peer=server-26a658ec-ad64-4df7-e11e-55e586a66852 +2025-02-25T23:57:43.090Z [INFO] server-63c2b480-ec7a-eaab-995e-23023d54d8dd: pipelining replication: peer="{Voter server-26a658ec-ad64-4df7-e11e-55e586a66852 26a658ec-ad64-4df7-e11e-55e586a66852}" +2025-02-25T23:57:43.090Z [INFO] server-63c2b480-ec7a-eaab-995e-23023d54d8dd: pipelining replication: peer="{Voter server-e72cd380-2139-c201-40ab-7d4fcb3de510 e72cd380-2139-c201-40ab-7d4fcb3de510}" +2025-02-25T23:57:43.197Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:43.09020582 +0000 UTC m=+10.699221227 (1 nodes), 63.523765ms from start of poll, 63.738526ms from cluster start. Timeout at 2025-02-25 23:57:43.19520758 +0000 UTC m=+10.804222947, 105.00172ms after stability +2025-02-25T23:57:43.205Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:57:43.313Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:57:43.205974037 +0000 UTC m=+10.814989444 (2 nodes), 3.2µs from start of poll, 179.506743ms from cluster start. Timeout at 2025-02-25 23:57:43.310980117 +0000 UTC m=+10.919995524, 105.00608ms after stability +2025-02-25T23:57:43.313Z [INFO] server-63c2b480-ec7a-eaab-995e-23023d54d8dd: updating configuration: command=RemoveServer server-id=server-e72cd380-2139-c201-40ab-7d4fcb3de510 server-addr= servers="[{Suffrage:Voter ID:server-63c2b480-ec7a-eaab-995e-23023d54d8dd Address:63c2b480-ec7a-eaab-995e-23023d54d8dd} {Suffrage:Voter ID:server-26a658ec-ad64-4df7-e11e-55e586a66852 Address:26a658ec-ad64-4df7-e11e-55e586a66852}]" +2025-02-25T23:57:43.314Z [INFO] server-63c2b480-ec7a-eaab-995e-23023d54d8dd: removed peer, stopping replication: peer=server-e72cd380-2139-c201-40ab-7d4fcb3de510 last-index=3 +2025-02-25T23:57:43.314Z [INFO] server-63c2b480-ec7a-eaab-995e-23023d54d8dd: aborting pipeline replication: peer="{Voter server-e72cd380-2139-c201-40ab-7d4fcb3de510 e72cd380-2139-c201-40ab-7d4fcb3de510}" +2025-02-25T23:57:43.393Z [WARN] server-e72cd380-2139-c201-40ab-7d4fcb3de510: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +2025-02-25T23:57:43.421Z [INFO] server-63c2b480-ec7a-eaab-995e-23023d54d8dd: aborting pipeline replication: peer="{Voter server-26a658ec-ad64-4df7-e11e-55e586a66852 26a658ec-ad64-4df7-e11e-55e586a66852}" +2025-02-25T23:57:43.925Z [ERROR] server-63c2b480-ec7a-eaab-995e-23023d54d8dd: failed to heartbeat to: peer=26a658ec-ad64-4df7-e11e-55e586a66852 backoff time=10ms error="command timed out" +--- PASS: TestRaft_RemoveFollower (0.90s) === RUN TestRaft_RemoveLeader -2026-03-31T06:16:22.492Z [INFO] server-1eabc591-ab00-eea7-12bd-f3853f88487f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1eabc591-ab00-eea7-12bd-f3853f88487f Address:1eabc591-ab00-eea7-12bd-f3853f88487f} {Suffrage:Voter ID:server-817ad916-9f5d-19f3-3db9-d139a195b3f6 Address:817ad916-9f5d-19f3-3db9-d139a195b3f6} {Suffrage:Voter ID:server-151608f6-3f1c-c194-5bc1-77aac3d79581 Address:151608f6-3f1c-c194-5bc1-77aac3d79581}]" -2026-03-31T06:16:22.492Z [INFO] server-817ad916-9f5d-19f3-3db9-d139a195b3f6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1eabc591-ab00-eea7-12bd-f3853f88487f Address:1eabc591-ab00-eea7-12bd-f3853f88487f} {Suffrage:Voter ID:server-817ad916-9f5d-19f3-3db9-d139a195b3f6 Address:817ad916-9f5d-19f3-3db9-d139a195b3f6} {Suffrage:Voter ID:server-151608f6-3f1c-c194-5bc1-77aac3d79581 Address:151608f6-3f1c-c194-5bc1-77aac3d79581}]" -2026-03-31T06:16:22.492Z [INFO] server-1eabc591-ab00-eea7-12bd-f3853f88487f: entering follower state: follower="Node at 1eabc591-ab00-eea7-12bd-f3853f88487f [Follower]" leader-address= leader-id= -2026-03-31T06:16:22.492Z [INFO] server-151608f6-3f1c-c194-5bc1-77aac3d79581: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1eabc591-ab00-eea7-12bd-f3853f88487f Address:1eabc591-ab00-eea7-12bd-f3853f88487f} {Suffrage:Voter ID:server-817ad916-9f5d-19f3-3db9-d139a195b3f6 Address:817ad916-9f5d-19f3-3db9-d139a195b3f6} {Suffrage:Voter ID:server-151608f6-3f1c-c194-5bc1-77aac3d79581 Address:151608f6-3f1c-c194-5bc1-77aac3d79581}]" -2026-03-31T06:16:22.492Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:22.492Z [INFO] server-817ad916-9f5d-19f3-3db9-d139a195b3f6: entering follower state: follower="Node at 817ad916-9f5d-19f3-3db9-d139a195b3f6 [Follower]" leader-address= leader-id= -2026-03-31T06:16:22.492Z [INFO] server-151608f6-3f1c-c194-5bc1-77aac3d79581: entering follower state: follower="Node at 151608f6-3f1c-c194-5bc1-77aac3d79581 [Follower]" leader-address= leader-id= -2026-03-31T06:16:22.558Z [WARN] server-817ad916-9f5d-19f3-3db9-d139a195b3f6: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:22.558Z [INFO] server-817ad916-9f5d-19f3-3db9-d139a195b3f6: entering candidate state: node="Node at 817ad916-9f5d-19f3-3db9-d139a195b3f6 [Candidate]" term=2 -2026-03-31T06:16:22.558Z [INFO] server-817ad916-9f5d-19f3-3db9-d139a195b3f6: election won: term=2 tally=2 -2026-03-31T06:16:22.558Z [INFO] server-817ad916-9f5d-19f3-3db9-d139a195b3f6: entering leader state: leader="Node at 817ad916-9f5d-19f3-3db9-d139a195b3f6 [Leader]" -2026-03-31T06:16:22.558Z [INFO] server-817ad916-9f5d-19f3-3db9-d139a195b3f6: added peer, starting replication: peer=server-1eabc591-ab00-eea7-12bd-f3853f88487f -2026-03-31T06:16:22.558Z [INFO] server-817ad916-9f5d-19f3-3db9-d139a195b3f6: added peer, starting replication: peer=server-151608f6-3f1c-c194-5bc1-77aac3d79581 -2026-03-31T06:16:22.558Z [INFO] server-817ad916-9f5d-19f3-3db9-d139a195b3f6: pipelining replication: peer="{Voter server-151608f6-3f1c-c194-5bc1-77aac3d79581 151608f6-3f1c-c194-5bc1-77aac3d79581}" -2026-03-31T06:16:22.558Z [INFO] server-817ad916-9f5d-19f3-3db9-d139a195b3f6: pipelining replication: peer="{Voter server-1eabc591-ab00-eea7-12bd-f3853f88487f 1eabc591-ab00-eea7-12bd-f3853f88487f}" -2026-03-31T06:16:22.663Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:22.55820138 +0000 UTC m=+10.835808801 (1 nodes), 65.68236ms from start of poll, 66.078ms from cluster start. Timeout at 2026-03-31 06:16:22.6632033 +0000 UTC m=+10.940810721, 105.00192ms after stability -2026-03-31T06:16:22.669Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:16:22.774Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:16:22.66937326 +0000 UTC m=+10.946980681 (2 nodes), 1.2µs from start of poll, 177.24988ms from cluster start. Timeout at 2026-03-31 06:16:22.77437758 +0000 UTC m=+11.051984961, 105.00428ms after stability -2026-03-31T06:16:22.774Z [INFO] server-817ad916-9f5d-19f3-3db9-d139a195b3f6: updating configuration: command=RemoveServer server-id=server-817ad916-9f5d-19f3-3db9-d139a195b3f6 server-addr= servers="[{Suffrage:Voter ID:server-1eabc591-ab00-eea7-12bd-f3853f88487f Address:1eabc591-ab00-eea7-12bd-f3853f88487f} {Suffrage:Voter ID:server-151608f6-3f1c-c194-5bc1-77aac3d79581 Address:151608f6-3f1c-c194-5bc1-77aac3d79581}]" -2026-03-31T06:16:22.774Z [INFO] server-817ad916-9f5d-19f3-3db9-d139a195b3f6: removed ourself, shutting down -2026-03-31T06:16:22.774Z [INFO] server-817ad916-9f5d-19f3-3db9-d139a195b3f6: aborting pipeline replication: peer="{Voter server-151608f6-3f1c-c194-5bc1-77aac3d79581 151608f6-3f1c-c194-5bc1-77aac3d79581}" -2026-03-31T06:16:22.775Z [INFO] server-817ad916-9f5d-19f3-3db9-d139a195b3f6: aborting pipeline replication: peer="{Voter server-1eabc591-ab00-eea7-12bd-f3853f88487f 1eabc591-ab00-eea7-12bd-f3853f88487f}" -2026-03-31T06:16:22.846Z [WARN] server-1eabc591-ab00-eea7-12bd-f3853f88487f: heartbeat timeout reached, starting election: last-leader-addr=817ad916-9f5d-19f3-3db9-d139a195b3f6 last-leader-id=server-817ad916-9f5d-19f3-3db9-d139a195b3f6 -2026-03-31T06:16:22.846Z [INFO] server-1eabc591-ab00-eea7-12bd-f3853f88487f: entering candidate state: node="Node at 1eabc591-ab00-eea7-12bd-f3853f88487f [Candidate]" term=3 -2026-03-31T06:16:22.846Z [WARN] server-151608f6-3f1c-c194-5bc1-77aac3d79581: rejecting vote request since we have a leader: from=1eabc591-ab00-eea7-12bd-f3853f88487f leader=817ad916-9f5d-19f3-3db9-d139a195b3f6 leader-id=server-817ad916-9f5d-19f3-3db9-d139a195b3f6 -2026-03-31T06:16:22.847Z [WARN] server-151608f6-3f1c-c194-5bc1-77aac3d79581: heartbeat timeout reached, starting election: last-leader-addr=817ad916-9f5d-19f3-3db9-d139a195b3f6 last-leader-id=server-817ad916-9f5d-19f3-3db9-d139a195b3f6 -2026-03-31T06:16:22.847Z [INFO] server-151608f6-3f1c-c194-5bc1-77aac3d79581: entering candidate state: node="Node at 151608f6-3f1c-c194-5bc1-77aac3d79581 [Candidate]" term=3 -2026-03-31T06:16:22.847Z [INFO] server-1eabc591-ab00-eea7-12bd-f3853f88487f: duplicate requestVote for same term: term=3 -2026-03-31T06:16:22.908Z [WARN] server-151608f6-3f1c-c194-5bc1-77aac3d79581: Election timeout reached, restarting election -2026-03-31T06:16:22.908Z [INFO] server-151608f6-3f1c-c194-5bc1-77aac3d79581: entering candidate state: node="Node at 151608f6-3f1c-c194-5bc1-77aac3d79581 [Candidate]" term=4 -2026-03-31T06:16:22.909Z [INFO] server-1eabc591-ab00-eea7-12bd-f3853f88487f: entering follower state: follower="Node at 1eabc591-ab00-eea7-12bd-f3853f88487f [Follower]" leader-address= leader-id= -2026-03-31T06:16:22.909Z [INFO] server-151608f6-3f1c-c194-5bc1-77aac3d79581: election won: term=4 tally=2 -2026-03-31T06:16:22.909Z [INFO] server-151608f6-3f1c-c194-5bc1-77aac3d79581: entering leader state: leader="Node at 151608f6-3f1c-c194-5bc1-77aac3d79581 [Leader]" -2026-03-31T06:16:22.909Z [INFO] server-151608f6-3f1c-c194-5bc1-77aac3d79581: added peer, starting replication: peer=server-1eabc591-ab00-eea7-12bd-f3853f88487f -2026-03-31T06:16:22.909Z [INFO] server-151608f6-3f1c-c194-5bc1-77aac3d79581: pipelining replication: peer="{Voter server-1eabc591-ab00-eea7-12bd-f3853f88487f 1eabc591-ab00-eea7-12bd-f3853f88487f}" -2026-03-31T06:16:22.986Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:23.091Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:22.98633582 +0000 UTC m=+11.263943201 (1 nodes), 71.44µs from start of poll, 494.2124ms from cluster start. Timeout at 2026-03-31 06:16:23.09134618 +0000 UTC m=+11.368953561, 105.01036ms after stability -2026-03-31T06:16:23.091Z [INFO] server-151608f6-3f1c-c194-5bc1-77aac3d79581: aborting pipeline replication: peer="{Voter server-1eabc591-ab00-eea7-12bd-f3853f88487f 1eabc591-ab00-eea7-12bd-f3853f88487f}" ---- PASS: TestRaft_RemoveLeader (0.60s) +2025-02-25T23:57:43.926Z [INFO] server-b894b9a4-7026-bcc9-3f59-dadc72e074c8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b894b9a4-7026-bcc9-3f59-dadc72e074c8 Address:b894b9a4-7026-bcc9-3f59-dadc72e074c8} {Suffrage:Voter ID:server-2f4b9705-48f7-ff5a-e5a2-a7123c85a3f0 Address:2f4b9705-48f7-ff5a-e5a2-a7123c85a3f0} {Suffrage:Voter ID:server-a2409fb2-2c83-eb7f-102d-09e9cab17ce3 Address:a2409fb2-2c83-eb7f-102d-09e9cab17ce3}]" +2025-02-25T23:57:43.926Z [INFO] server-2f4b9705-48f7-ff5a-e5a2-a7123c85a3f0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b894b9a4-7026-bcc9-3f59-dadc72e074c8 Address:b894b9a4-7026-bcc9-3f59-dadc72e074c8} {Suffrage:Voter ID:server-2f4b9705-48f7-ff5a-e5a2-a7123c85a3f0 Address:2f4b9705-48f7-ff5a-e5a2-a7123c85a3f0} {Suffrage:Voter ID:server-a2409fb2-2c83-eb7f-102d-09e9cab17ce3 Address:a2409fb2-2c83-eb7f-102d-09e9cab17ce3}]" +2025-02-25T23:57:43.926Z [INFO] server-a2409fb2-2c83-eb7f-102d-09e9cab17ce3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b894b9a4-7026-bcc9-3f59-dadc72e074c8 Address:b894b9a4-7026-bcc9-3f59-dadc72e074c8} {Suffrage:Voter ID:server-2f4b9705-48f7-ff5a-e5a2-a7123c85a3f0 Address:2f4b9705-48f7-ff5a-e5a2-a7123c85a3f0} {Suffrage:Voter ID:server-a2409fb2-2c83-eb7f-102d-09e9cab17ce3 Address:a2409fb2-2c83-eb7f-102d-09e9cab17ce3}]" +2025-02-25T23:57:43.926Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:43.926Z [INFO] server-b894b9a4-7026-bcc9-3f59-dadc72e074c8: entering follower state: follower="Node at b894b9a4-7026-bcc9-3f59-dadc72e074c8 [Follower]" leader-address= leader-id= +2025-02-25T23:57:43.926Z [INFO] server-2f4b9705-48f7-ff5a-e5a2-a7123c85a3f0: entering follower state: follower="Node at 2f4b9705-48f7-ff5a-e5a2-a7123c85a3f0 [Follower]" leader-address= leader-id= +2025-02-25T23:57:43.926Z [INFO] server-a2409fb2-2c83-eb7f-102d-09e9cab17ce3: entering follower state: follower="Node at a2409fb2-2c83-eb7f-102d-09e9cab17ce3 [Follower]" leader-address= leader-id= +2025-02-25T23:57:43.993Z [WARN] server-2f4b9705-48f7-ff5a-e5a2-a7123c85a3f0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:43.993Z [INFO] server-2f4b9705-48f7-ff5a-e5a2-a7123c85a3f0: entering candidate state: node="Node at 2f4b9705-48f7-ff5a-e5a2-a7123c85a3f0 [Candidate]" term=2 +2025-02-25T23:57:43.993Z [INFO] server-2f4b9705-48f7-ff5a-e5a2-a7123c85a3f0: election won: term=2 tally=2 +2025-02-25T23:57:43.994Z [INFO] server-2f4b9705-48f7-ff5a-e5a2-a7123c85a3f0: entering leader state: leader="Node at 2f4b9705-48f7-ff5a-e5a2-a7123c85a3f0 [Leader]" +2025-02-25T23:57:43.994Z [INFO] server-2f4b9705-48f7-ff5a-e5a2-a7123c85a3f0: added peer, starting replication: peer=server-b894b9a4-7026-bcc9-3f59-dadc72e074c8 +2025-02-25T23:57:43.994Z [INFO] server-2f4b9705-48f7-ff5a-e5a2-a7123c85a3f0: added peer, starting replication: peer=server-a2409fb2-2c83-eb7f-102d-09e9cab17ce3 +2025-02-25T23:57:43.994Z [INFO] server-2f4b9705-48f7-ff5a-e5a2-a7123c85a3f0: pipelining replication: peer="{Voter server-a2409fb2-2c83-eb7f-102d-09e9cab17ce3 a2409fb2-2c83-eb7f-102d-09e9cab17ce3}" +2025-02-25T23:57:43.994Z [INFO] server-2f4b9705-48f7-ff5a-e5a2-a7123c85a3f0: pipelining replication: peer="{Voter server-b894b9a4-7026-bcc9-3f59-dadc72e074c8 b894b9a4-7026-bcc9-3f59-dadc72e074c8}" +2025-02-25T23:57:44.101Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:43.994193097 +0000 UTC m=+11.603208504 (1 nodes), 67.751031ms from start of poll, 67.926432ms from cluster start. Timeout at 2025-02-25 23:57:44.099194057 +0000 UTC m=+11.708209464, 105.00096ms after stability +2025-02-25T23:57:44.107Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:57:44.213Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:57:44.107528459 +0000 UTC m=+11.716543826 (2 nodes), 840ns from start of poll, 181.261754ms from cluster start. Timeout at 2025-02-25 23:57:44.212539859 +0000 UTC m=+11.821555266, 105.01144ms after stability +2025-02-25T23:57:44.213Z [INFO] server-2f4b9705-48f7-ff5a-e5a2-a7123c85a3f0: updating configuration: command=RemoveServer server-id=server-2f4b9705-48f7-ff5a-e5a2-a7123c85a3f0 server-addr= servers="[{Suffrage:Voter ID:server-b894b9a4-7026-bcc9-3f59-dadc72e074c8 Address:b894b9a4-7026-bcc9-3f59-dadc72e074c8} {Suffrage:Voter ID:server-a2409fb2-2c83-eb7f-102d-09e9cab17ce3 Address:a2409fb2-2c83-eb7f-102d-09e9cab17ce3}]" +2025-02-25T23:57:44.214Z [INFO] server-2f4b9705-48f7-ff5a-e5a2-a7123c85a3f0: removed ourself, shutting down +2025-02-25T23:57:44.214Z [INFO] server-2f4b9705-48f7-ff5a-e5a2-a7123c85a3f0: aborting pipeline replication: peer="{Voter server-a2409fb2-2c83-eb7f-102d-09e9cab17ce3 a2409fb2-2c83-eb7f-102d-09e9cab17ce3}" +2025-02-25T23:57:44.214Z [INFO] server-2f4b9705-48f7-ff5a-e5a2-a7123c85a3f0: aborting pipeline replication: peer="{Voter server-b894b9a4-7026-bcc9-3f59-dadc72e074c8 b894b9a4-7026-bcc9-3f59-dadc72e074c8}" +2025-02-25T23:57:44.321Z [WARN] server-b894b9a4-7026-bcc9-3f59-dadc72e074c8: heartbeat timeout reached, starting election: last-leader-addr=2f4b9705-48f7-ff5a-e5a2-a7123c85a3f0 last-leader-id=server-2f4b9705-48f7-ff5a-e5a2-a7123c85a3f0 +2025-02-25T23:57:44.321Z [INFO] server-b894b9a4-7026-bcc9-3f59-dadc72e074c8: entering candidate state: node="Node at b894b9a4-7026-bcc9-3f59-dadc72e074c8 [Candidate]" term=3 +2025-02-25T23:57:44.321Z [WARN] server-a2409fb2-2c83-eb7f-102d-09e9cab17ce3: rejecting vote request since we have a leader: from=b894b9a4-7026-bcc9-3f59-dadc72e074c8 leader=2f4b9705-48f7-ff5a-e5a2-a7123c85a3f0 leader-id=server-2f4b9705-48f7-ff5a-e5a2-a7123c85a3f0 +2025-02-25T23:57:44.341Z [WARN] server-a2409fb2-2c83-eb7f-102d-09e9cab17ce3: heartbeat timeout reached, starting election: last-leader-addr=2f4b9705-48f7-ff5a-e5a2-a7123c85a3f0 last-leader-id=server-2f4b9705-48f7-ff5a-e5a2-a7123c85a3f0 +2025-02-25T23:57:44.341Z [INFO] server-a2409fb2-2c83-eb7f-102d-09e9cab17ce3: entering candidate state: node="Node at a2409fb2-2c83-eb7f-102d-09e9cab17ce3 [Candidate]" term=3 +2025-02-25T23:57:44.341Z [INFO] server-b894b9a4-7026-bcc9-3f59-dadc72e074c8: duplicate requestVote for same term: term=3 +2025-02-25T23:57:44.381Z [WARN] server-b894b9a4-7026-bcc9-3f59-dadc72e074c8: Election timeout reached, restarting election +2025-02-25T23:57:44.381Z [INFO] server-b894b9a4-7026-bcc9-3f59-dadc72e074c8: entering candidate state: node="Node at b894b9a4-7026-bcc9-3f59-dadc72e074c8 [Candidate]" term=4 +2025-02-25T23:57:44.381Z [INFO] server-a2409fb2-2c83-eb7f-102d-09e9cab17ce3: entering follower state: follower="Node at a2409fb2-2c83-eb7f-102d-09e9cab17ce3 [Follower]" leader-address= leader-id= +2025-02-25T23:57:44.382Z [INFO] server-b894b9a4-7026-bcc9-3f59-dadc72e074c8: election won: term=4 tally=2 +2025-02-25T23:57:44.382Z [INFO] server-b894b9a4-7026-bcc9-3f59-dadc72e074c8: entering leader state: leader="Node at b894b9a4-7026-bcc9-3f59-dadc72e074c8 [Leader]" +2025-02-25T23:57:44.382Z [INFO] server-b894b9a4-7026-bcc9-3f59-dadc72e074c8: added peer, starting replication: peer=server-a2409fb2-2c83-eb7f-102d-09e9cab17ce3 +2025-02-25T23:57:44.382Z [INFO] server-b894b9a4-7026-bcc9-3f59-dadc72e074c8: pipelining replication: peer="{Voter server-a2409fb2-2c83-eb7f-102d-09e9cab17ce3 a2409fb2-2c83-eb7f-102d-09e9cab17ce3}" +2025-02-25T23:57:44.429Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:44.537Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:44.429935673 +0000 UTC m=+12.038951080 (1 nodes), 30.321µs from start of poll, 503.669008ms from cluster start. Timeout at 2025-02-25 23:57:44.534941873 +0000 UTC m=+12.143957280, 105.0062ms after stability +2025-02-25T23:57:44.538Z [INFO] server-b894b9a4-7026-bcc9-3f59-dadc72e074c8: aborting pipeline replication: peer="{Voter server-a2409fb2-2c83-eb7f-102d-09e9cab17ce3 a2409fb2-2c83-eb7f-102d-09e9cab17ce3}" +--- PASS: TestRaft_RemoveLeader (0.61s) === RUN TestRaft_RemoveLeader_NoShutdown -2026-03-31T06:16:23.092Z [INFO] server-206e8035-06fc-d2ab-83c8-804b52c0b508: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-206e8035-06fc-d2ab-83c8-804b52c0b508 Address:206e8035-06fc-d2ab-83c8-804b52c0b508} {Suffrage:Voter ID:server-733a912c-14fc-08d5-3e8e-a79353ae5d05 Address:733a912c-14fc-08d5-3e8e-a79353ae5d05} {Suffrage:Voter ID:server-093a8f37-29dd-6357-7fc7-f9ef53ae6bc1 Address:093a8f37-29dd-6357-7fc7-f9ef53ae6bc1}]" -2026-03-31T06:16:23.092Z [INFO] server-206e8035-06fc-d2ab-83c8-804b52c0b508: entering follower state: follower="Node at 206e8035-06fc-d2ab-83c8-804b52c0b508 [Follower]" leader-address= leader-id= -2026-03-31T06:16:23.092Z [INFO] server-733a912c-14fc-08d5-3e8e-a79353ae5d05: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-206e8035-06fc-d2ab-83c8-804b52c0b508 Address:206e8035-06fc-d2ab-83c8-804b52c0b508} {Suffrage:Voter ID:server-733a912c-14fc-08d5-3e8e-a79353ae5d05 Address:733a912c-14fc-08d5-3e8e-a79353ae5d05} {Suffrage:Voter ID:server-093a8f37-29dd-6357-7fc7-f9ef53ae6bc1 Address:093a8f37-29dd-6357-7fc7-f9ef53ae6bc1}]" -2026-03-31T06:16:23.092Z [INFO] server-733a912c-14fc-08d5-3e8e-a79353ae5d05: entering follower state: follower="Node at 733a912c-14fc-08d5-3e8e-a79353ae5d05 [Follower]" leader-address= leader-id= -2026-03-31T06:16:23.092Z [INFO] server-093a8f37-29dd-6357-7fc7-f9ef53ae6bc1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-206e8035-06fc-d2ab-83c8-804b52c0b508 Address:206e8035-06fc-d2ab-83c8-804b52c0b508} {Suffrage:Voter ID:server-733a912c-14fc-08d5-3e8e-a79353ae5d05 Address:733a912c-14fc-08d5-3e8e-a79353ae5d05} {Suffrage:Voter ID:server-093a8f37-29dd-6357-7fc7-f9ef53ae6bc1 Address:093a8f37-29dd-6357-7fc7-f9ef53ae6bc1}]" -2026-03-31T06:16:23.092Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:16:23.092Z [INFO] server-093a8f37-29dd-6357-7fc7-f9ef53ae6bc1: entering follower state: follower="Node at 093a8f37-29dd-6357-7fc7-f9ef53ae6bc1 [Follower]" leader-address= leader-id= -2026-03-31T06:16:23.158Z [WARN] server-093a8f37-29dd-6357-7fc7-f9ef53ae6bc1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:23.158Z [INFO] server-093a8f37-29dd-6357-7fc7-f9ef53ae6bc1: entering candidate state: node="Node at 093a8f37-29dd-6357-7fc7-f9ef53ae6bc1 [Candidate]" term=2 -2026-03-31T06:16:23.158Z [INFO] server-093a8f37-29dd-6357-7fc7-f9ef53ae6bc1: election won: term=2 tally=2 -2026-03-31T06:16:23.158Z [INFO] server-093a8f37-29dd-6357-7fc7-f9ef53ae6bc1: entering leader state: leader="Node at 093a8f37-29dd-6357-7fc7-f9ef53ae6bc1 [Leader]" -2026-03-31T06:16:23.158Z [INFO] server-093a8f37-29dd-6357-7fc7-f9ef53ae6bc1: added peer, starting replication: peer=server-206e8035-06fc-d2ab-83c8-804b52c0b508 -2026-03-31T06:16:23.158Z [INFO] server-093a8f37-29dd-6357-7fc7-f9ef53ae6bc1: added peer, starting replication: peer=server-733a912c-14fc-08d5-3e8e-a79353ae5d05 -2026-03-31T06:16:23.158Z [INFO] server-093a8f37-29dd-6357-7fc7-f9ef53ae6bc1: pipelining replication: peer="{Voter server-733a912c-14fc-08d5-3e8e-a79353ae5d05 733a912c-14fc-08d5-3e8e-a79353ae5d05}" -2026-03-31T06:16:23.158Z [INFO] server-093a8f37-29dd-6357-7fc7-f9ef53ae6bc1: pipelining replication: peer="{Voter server-206e8035-06fc-d2ab-83c8-804b52c0b508 206e8035-06fc-d2ab-83c8-804b52c0b508}" -2026-03-31T06:16:23.264Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:16:23.15883102 +0000 UTC m=+11.436438401 (2 nodes), 65.88564ms from start of poll, 66.43092ms from cluster start. Timeout at 2026-03-31 06:16:23.26383362 +0000 UTC m=+11.541441001, 105.0026ms after stability -2026-03-31T06:16:23.264Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:23.371Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:23.26496006 +0000 UTC m=+11.542567441 (1 nodes), 1.92µs from start of poll, 172.55996ms from cluster start. Timeout at 2026-03-31 06:16:23.36996422 +0000 UTC m=+11.647571561, 105.00412ms after stability -2026-03-31T06:16:23.375Z [INFO] server-093a8f37-29dd-6357-7fc7-f9ef53ae6bc1: updating configuration: command=RemoveServer server-id=server-093a8f37-29dd-6357-7fc7-f9ef53ae6bc1 server-addr= servers="[{Suffrage:Voter ID:server-206e8035-06fc-d2ab-83c8-804b52c0b508 Address:206e8035-06fc-d2ab-83c8-804b52c0b508} {Suffrage:Voter ID:server-733a912c-14fc-08d5-3e8e-a79353ae5d05 Address:733a912c-14fc-08d5-3e8e-a79353ae5d05}]" -2026-03-31T06:16:23.376Z [INFO] server-093a8f37-29dd-6357-7fc7-f9ef53ae6bc1: removed ourself, transitioning to follower -2026-03-31T06:16:23.376Z [INFO] server-093a8f37-29dd-6357-7fc7-f9ef53ae6bc1: entering follower state: follower="Node at 093a8f37-29dd-6357-7fc7-f9ef53ae6bc1 [Follower]" leader-address= leader-id= -2026-03-31T06:16:23.376Z [INFO] server-093a8f37-29dd-6357-7fc7-f9ef53ae6bc1: aborting pipeline replication: peer="{Voter server-733a912c-14fc-08d5-3e8e-a79353ae5d05 733a912c-14fc-08d5-3e8e-a79353ae5d05}" -2026-03-31T06:16:23.376Z [INFO] server-093a8f37-29dd-6357-7fc7-f9ef53ae6bc1: aborting pipeline replication: peer="{Voter server-206e8035-06fc-d2ab-83c8-804b52c0b508 206e8035-06fc-d2ab-83c8-804b52c0b508}" -2026-03-31T06:16:23.441Z [WARN] server-733a912c-14fc-08d5-3e8e-a79353ae5d05: heartbeat timeout reached, starting election: last-leader-addr=093a8f37-29dd-6357-7fc7-f9ef53ae6bc1 last-leader-id=server-093a8f37-29dd-6357-7fc7-f9ef53ae6bc1 -2026-03-31T06:16:23.441Z [INFO] server-733a912c-14fc-08d5-3e8e-a79353ae5d05: entering candidate state: node="Node at 733a912c-14fc-08d5-3e8e-a79353ae5d05 [Candidate]" term=3 -2026-03-31T06:16:23.441Z [WARN] server-206e8035-06fc-d2ab-83c8-804b52c0b508: rejecting vote request since we have a leader: from=733a912c-14fc-08d5-3e8e-a79353ae5d05 leader=093a8f37-29dd-6357-7fc7-f9ef53ae6bc1 leader-id=server-093a8f37-29dd-6357-7fc7-f9ef53ae6bc1 -2026-03-31T06:16:23.451Z [WARN] server-093a8f37-29dd-6357-7fc7-f9ef53ae6bc1: not part of stable configuration, aborting election -2026-03-31T06:16:23.482Z [WARN] server-206e8035-06fc-d2ab-83c8-804b52c0b508: heartbeat timeout reached, starting election: last-leader-addr=093a8f37-29dd-6357-7fc7-f9ef53ae6bc1 last-leader-id=server-093a8f37-29dd-6357-7fc7-f9ef53ae6bc1 -2026-03-31T06:16:23.482Z [INFO] server-206e8035-06fc-d2ab-83c8-804b52c0b508: entering candidate state: node="Node at 206e8035-06fc-d2ab-83c8-804b52c0b508 [Candidate]" term=3 -2026-03-31T06:16:23.482Z [INFO] server-733a912c-14fc-08d5-3e8e-a79353ae5d05: duplicate requestVote for same term: term=3 -2026-03-31T06:16:23.482Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:23.530Z [WARN] server-733a912c-14fc-08d5-3e8e-a79353ae5d05: Election timeout reached, restarting election -2026-03-31T06:16:23.530Z [INFO] server-733a912c-14fc-08d5-3e8e-a79353ae5d05: entering candidate state: node="Node at 733a912c-14fc-08d5-3e8e-a79353ae5d05 [Candidate]" term=4 -2026-03-31T06:16:23.530Z [INFO] server-206e8035-06fc-d2ab-83c8-804b52c0b508: entering follower state: follower="Node at 206e8035-06fc-d2ab-83c8-804b52c0b508 [Follower]" leader-address= leader-id= -2026-03-31T06:16:23.530Z [INFO] server-733a912c-14fc-08d5-3e8e-a79353ae5d05: election won: term=4 tally=2 -2026-03-31T06:16:23.530Z [INFO] server-733a912c-14fc-08d5-3e8e-a79353ae5d05: entering leader state: leader="Node at 733a912c-14fc-08d5-3e8e-a79353ae5d05 [Leader]" -2026-03-31T06:16:23.530Z [INFO] server-733a912c-14fc-08d5-3e8e-a79353ae5d05: added peer, starting replication: peer=server-206e8035-06fc-d2ab-83c8-804b52c0b508 -2026-03-31T06:16:23.530Z [INFO] server-733a912c-14fc-08d5-3e8e-a79353ae5d05: pipelining replication: peer="{Voter server-206e8035-06fc-d2ab-83c8-804b52c0b508 206e8035-06fc-d2ab-83c8-804b52c0b508}" -2026-03-31T06:16:23.636Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:23.53061138 +0000 UTC m=+11.808218801 (1 nodes), 48.19668ms from start of poll, 438.21132ms from cluster start. Timeout at 2026-03-31 06:16:23.63561314 +0000 UTC m=+11.913220521, 105.00172ms after stability -2026-03-31T06:16:23.742Z [INFO] server-733a912c-14fc-08d5-3e8e-a79353ae5d05: aborting pipeline replication: peer="{Voter server-206e8035-06fc-d2ab-83c8-804b52c0b508 206e8035-06fc-d2ab-83c8-804b52c0b508}" ---- PASS: TestRaft_RemoveLeader_NoShutdown (0.65s) +2025-02-25T23:57:44.538Z [INFO] server-7d1eea76-43ae-ea6a-0282-3df6a0c63e49: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7d1eea76-43ae-ea6a-0282-3df6a0c63e49 Address:7d1eea76-43ae-ea6a-0282-3df6a0c63e49} {Suffrage:Voter ID:server-dc48c15e-7707-20b8-8362-e4600141d154 Address:dc48c15e-7707-20b8-8362-e4600141d154} {Suffrage:Voter ID:server-f9e5fe7c-459f-fe0c-b135-d4103d238ce8 Address:f9e5fe7c-459f-fe0c-b135-d4103d238ce8}]" +2025-02-25T23:57:44.538Z [INFO] server-dc48c15e-7707-20b8-8362-e4600141d154: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7d1eea76-43ae-ea6a-0282-3df6a0c63e49 Address:7d1eea76-43ae-ea6a-0282-3df6a0c63e49} {Suffrage:Voter ID:server-dc48c15e-7707-20b8-8362-e4600141d154 Address:dc48c15e-7707-20b8-8362-e4600141d154} {Suffrage:Voter ID:server-f9e5fe7c-459f-fe0c-b135-d4103d238ce8 Address:f9e5fe7c-459f-fe0c-b135-d4103d238ce8}]" +2025-02-25T23:57:44.538Z [INFO] server-f9e5fe7c-459f-fe0c-b135-d4103d238ce8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7d1eea76-43ae-ea6a-0282-3df6a0c63e49 Address:7d1eea76-43ae-ea6a-0282-3df6a0c63e49} {Suffrage:Voter ID:server-dc48c15e-7707-20b8-8362-e4600141d154 Address:dc48c15e-7707-20b8-8362-e4600141d154} {Suffrage:Voter ID:server-f9e5fe7c-459f-fe0c-b135-d4103d238ce8 Address:f9e5fe7c-459f-fe0c-b135-d4103d238ce8}]" +2025-02-25T23:57:44.538Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:57:44.538Z [INFO] server-7d1eea76-43ae-ea6a-0282-3df6a0c63e49: entering follower state: follower="Node at 7d1eea76-43ae-ea6a-0282-3df6a0c63e49 [Follower]" leader-address= leader-id= +2025-02-25T23:57:44.538Z [INFO] server-dc48c15e-7707-20b8-8362-e4600141d154: entering follower state: follower="Node at dc48c15e-7707-20b8-8362-e4600141d154 [Follower]" leader-address= leader-id= +2025-02-25T23:57:44.538Z [INFO] server-f9e5fe7c-459f-fe0c-b135-d4103d238ce8: entering follower state: follower="Node at f9e5fe7c-459f-fe0c-b135-d4103d238ce8 [Follower]" leader-address= leader-id= +2025-02-25T23:57:44.609Z [WARN] server-f9e5fe7c-459f-fe0c-b135-d4103d238ce8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:44.609Z [INFO] server-f9e5fe7c-459f-fe0c-b135-d4103d238ce8: entering candidate state: node="Node at f9e5fe7c-459f-fe0c-b135-d4103d238ce8 [Candidate]" term=2 +2025-02-25T23:57:44.610Z [INFO] server-f9e5fe7c-459f-fe0c-b135-d4103d238ce8: election won: term=2 tally=2 +2025-02-25T23:57:44.610Z [INFO] server-f9e5fe7c-459f-fe0c-b135-d4103d238ce8: entering leader state: leader="Node at f9e5fe7c-459f-fe0c-b135-d4103d238ce8 [Leader]" +2025-02-25T23:57:44.610Z [INFO] server-f9e5fe7c-459f-fe0c-b135-d4103d238ce8: added peer, starting replication: peer=server-7d1eea76-43ae-ea6a-0282-3df6a0c63e49 +2025-02-25T23:57:44.610Z [INFO] server-f9e5fe7c-459f-fe0c-b135-d4103d238ce8: added peer, starting replication: peer=server-dc48c15e-7707-20b8-8362-e4600141d154 +2025-02-25T23:57:44.610Z [INFO] server-f9e5fe7c-459f-fe0c-b135-d4103d238ce8: pipelining replication: peer="{Voter server-dc48c15e-7707-20b8-8362-e4600141d154 dc48c15e-7707-20b8-8362-e4600141d154}" +2025-02-25T23:57:44.610Z [INFO] server-f9e5fe7c-459f-fe0c-b135-d4103d238ce8: pipelining replication: peer="{Voter server-7d1eea76-43ae-ea6a-0282-3df6a0c63e49 7d1eea76-43ae-ea6a-0282-3df6a0c63e49}" +2025-02-25T23:57:44.717Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:57:44.610226621 +0000 UTC m=+12.219242028 (2 nodes), 71.690577ms from start of poll, 71.856698ms from cluster start. Timeout at 2025-02-25 23:57:44.715229821 +0000 UTC m=+12.324245228, 105.0032ms after stability +2025-02-25T23:57:44.718Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:44.825Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:44.718030267 +0000 UTC m=+12.327045634 (1 nodes), 400ns from start of poll, 179.660304ms from cluster start. Timeout at 2025-02-25 23:57:44.823040027 +0000 UTC m=+12.432055394, 105.00976ms after stability +2025-02-25T23:57:44.828Z [INFO] server-f9e5fe7c-459f-fe0c-b135-d4103d238ce8: updating configuration: command=RemoveServer server-id=server-f9e5fe7c-459f-fe0c-b135-d4103d238ce8 server-addr= servers="[{Suffrage:Voter ID:server-7d1eea76-43ae-ea6a-0282-3df6a0c63e49 Address:7d1eea76-43ae-ea6a-0282-3df6a0c63e49} {Suffrage:Voter ID:server-dc48c15e-7707-20b8-8362-e4600141d154 Address:dc48c15e-7707-20b8-8362-e4600141d154}]" +2025-02-25T23:57:44.828Z [INFO] server-f9e5fe7c-459f-fe0c-b135-d4103d238ce8: removed ourself, transitioning to follower +2025-02-25T23:57:44.828Z [INFO] server-f9e5fe7c-459f-fe0c-b135-d4103d238ce8: entering follower state: follower="Node at f9e5fe7c-459f-fe0c-b135-d4103d238ce8 [Follower]" leader-address= leader-id= +2025-02-25T23:57:44.828Z [INFO] server-f9e5fe7c-459f-fe0c-b135-d4103d238ce8: aborting pipeline replication: peer="{Voter server-dc48c15e-7707-20b8-8362-e4600141d154 dc48c15e-7707-20b8-8362-e4600141d154}" +2025-02-25T23:57:44.828Z [INFO] server-f9e5fe7c-459f-fe0c-b135-d4103d238ce8: aborting pipeline replication: peer="{Voter server-7d1eea76-43ae-ea6a-0282-3df6a0c63e49 7d1eea76-43ae-ea6a-0282-3df6a0c63e49}" +2025-02-25T23:57:44.913Z [WARN] server-f9e5fe7c-459f-fe0c-b135-d4103d238ce8: not part of stable configuration, aborting election +2025-02-25T23:57:44.933Z [WARN] server-7d1eea76-43ae-ea6a-0282-3df6a0c63e49: heartbeat timeout reached, starting election: last-leader-addr=f9e5fe7c-459f-fe0c-b135-d4103d238ce8 last-leader-id=server-f9e5fe7c-459f-fe0c-b135-d4103d238ce8 +2025-02-25T23:57:44.933Z [INFO] server-7d1eea76-43ae-ea6a-0282-3df6a0c63e49: entering candidate state: node="Node at 7d1eea76-43ae-ea6a-0282-3df6a0c63e49 [Candidate]" term=3 +2025-02-25T23:57:44.933Z [WARN] server-dc48c15e-7707-20b8-8362-e4600141d154: rejecting vote request since we have a leader: from=7d1eea76-43ae-ea6a-0282-3df6a0c63e49 leader=f9e5fe7c-459f-fe0c-b135-d4103d238ce8 leader-id=server-f9e5fe7c-459f-fe0c-b135-d4103d238ce8 +2025-02-25T23:57:44.933Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:44.945Z [WARN] server-dc48c15e-7707-20b8-8362-e4600141d154: heartbeat timeout reached, starting election: last-leader-addr=f9e5fe7c-459f-fe0c-b135-d4103d238ce8 last-leader-id=server-f9e5fe7c-459f-fe0c-b135-d4103d238ce8 +2025-02-25T23:57:44.945Z [INFO] server-dc48c15e-7707-20b8-8362-e4600141d154: entering candidate state: node="Node at dc48c15e-7707-20b8-8362-e4600141d154 [Candidate]" term=3 +2025-02-25T23:57:44.945Z [INFO] server-7d1eea76-43ae-ea6a-0282-3df6a0c63e49: duplicate requestVote for same term: term=3 +2025-02-25T23:57:45.033Z [WARN] server-7d1eea76-43ae-ea6a-0282-3df6a0c63e49: Election timeout reached, restarting election +2025-02-25T23:57:45.033Z [INFO] server-7d1eea76-43ae-ea6a-0282-3df6a0c63e49: entering candidate state: node="Node at 7d1eea76-43ae-ea6a-0282-3df6a0c63e49 [Candidate]" term=4 +2025-02-25T23:57:45.033Z [INFO] server-dc48c15e-7707-20b8-8362-e4600141d154: entering follower state: follower="Node at dc48c15e-7707-20b8-8362-e4600141d154 [Follower]" leader-address= leader-id= +2025-02-25T23:57:45.033Z [INFO] server-7d1eea76-43ae-ea6a-0282-3df6a0c63e49: election won: term=4 tally=2 +2025-02-25T23:57:45.033Z [INFO] server-7d1eea76-43ae-ea6a-0282-3df6a0c63e49: entering leader state: leader="Node at 7d1eea76-43ae-ea6a-0282-3df6a0c63e49 [Leader]" +2025-02-25T23:57:45.034Z [INFO] server-7d1eea76-43ae-ea6a-0282-3df6a0c63e49: added peer, starting replication: peer=server-dc48c15e-7707-20b8-8362-e4600141d154 +2025-02-25T23:57:45.034Z [INFO] server-7d1eea76-43ae-ea6a-0282-3df6a0c63e49: pipelining replication: peer="{Voter server-dc48c15e-7707-20b8-8362-e4600141d154 dc48c15e-7707-20b8-8362-e4600141d154}" +2025-02-25T23:57:45.141Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:45.03410392 +0000 UTC m=+12.643119327 (1 nodes), 100.113037ms from start of poll, 495.733997ms from cluster start. Timeout at 2025-02-25 23:57:45.13910716 +0000 UTC m=+12.748122527, 105.0032ms after stability +2025-02-25T23:57:45.249Z [INFO] server-7d1eea76-43ae-ea6a-0282-3df6a0c63e49: aborting pipeline replication: peer="{Voter server-dc48c15e-7707-20b8-8362-e4600141d154 dc48c15e-7707-20b8-8362-e4600141d154}" +--- PASS: TestRaft_RemoveLeader_NoShutdown (0.71s) === RUN TestRaft_RemoveFollower_SplitCluster -2026-03-31T06:16:23.743Z [INFO] server-2d50918c-128e-a400-4c46-c2c4410f26c9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2d50918c-128e-a400-4c46-c2c4410f26c9 Address:2d50918c-128e-a400-4c46-c2c4410f26c9} {Suffrage:Voter ID:server-8ab3fd73-750c-5061-ffdc-b419fcbc9a81 Address:8ab3fd73-750c-5061-ffdc-b419fcbc9a81} {Suffrage:Voter ID:server-3d1b2d19-c112-cbdd-d59f-a1c10ba297e4 Address:3d1b2d19-c112-cbdd-d59f-a1c10ba297e4} {Suffrage:Voter ID:server-6fdca903-eef1-a757-3da1-a30a9fe3095e Address:6fdca903-eef1-a757-3da1-a30a9fe3095e}]" -2026-03-31T06:16:23.743Z [INFO] server-2d50918c-128e-a400-4c46-c2c4410f26c9: entering follower state: follower="Node at 2d50918c-128e-a400-4c46-c2c4410f26c9 [Follower]" leader-address= leader-id= -2026-03-31T06:16:23.743Z [INFO] server-8ab3fd73-750c-5061-ffdc-b419fcbc9a81: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2d50918c-128e-a400-4c46-c2c4410f26c9 Address:2d50918c-128e-a400-4c46-c2c4410f26c9} {Suffrage:Voter ID:server-8ab3fd73-750c-5061-ffdc-b419fcbc9a81 Address:8ab3fd73-750c-5061-ffdc-b419fcbc9a81} {Suffrage:Voter ID:server-3d1b2d19-c112-cbdd-d59f-a1c10ba297e4 Address:3d1b2d19-c112-cbdd-d59f-a1c10ba297e4} {Suffrage:Voter ID:server-6fdca903-eef1-a757-3da1-a30a9fe3095e Address:6fdca903-eef1-a757-3da1-a30a9fe3095e}]" -2026-03-31T06:16:23.743Z [INFO] server-3d1b2d19-c112-cbdd-d59f-a1c10ba297e4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2d50918c-128e-a400-4c46-c2c4410f26c9 Address:2d50918c-128e-a400-4c46-c2c4410f26c9} {Suffrage:Voter ID:server-8ab3fd73-750c-5061-ffdc-b419fcbc9a81 Address:8ab3fd73-750c-5061-ffdc-b419fcbc9a81} {Suffrage:Voter ID:server-3d1b2d19-c112-cbdd-d59f-a1c10ba297e4 Address:3d1b2d19-c112-cbdd-d59f-a1c10ba297e4} {Suffrage:Voter ID:server-6fdca903-eef1-a757-3da1-a30a9fe3095e Address:6fdca903-eef1-a757-3da1-a30a9fe3095e}]" -2026-03-31T06:16:23.743Z [INFO] server-8ab3fd73-750c-5061-ffdc-b419fcbc9a81: entering follower state: follower="Node at 8ab3fd73-750c-5061-ffdc-b419fcbc9a81 [Follower]" leader-address= leader-id= -2026-03-31T06:16:23.743Z [INFO] server-3d1b2d19-c112-cbdd-d59f-a1c10ba297e4: entering follower state: follower="Node at 3d1b2d19-c112-cbdd-d59f-a1c10ba297e4 [Follower]" leader-address= leader-id= -2026-03-31T06:16:23.743Z [INFO] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2d50918c-128e-a400-4c46-c2c4410f26c9 Address:2d50918c-128e-a400-4c46-c2c4410f26c9} {Suffrage:Voter ID:server-8ab3fd73-750c-5061-ffdc-b419fcbc9a81 Address:8ab3fd73-750c-5061-ffdc-b419fcbc9a81} {Suffrage:Voter ID:server-3d1b2d19-c112-cbdd-d59f-a1c10ba297e4 Address:3d1b2d19-c112-cbdd-d59f-a1c10ba297e4} {Suffrage:Voter ID:server-6fdca903-eef1-a757-3da1-a30a9fe3095e Address:6fdca903-eef1-a757-3da1-a30a9fe3095e}]" -2026-03-31T06:16:23.743Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:23.743Z [INFO] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: entering follower state: follower="Node at 6fdca903-eef1-a757-3da1-a30a9fe3095e [Follower]" leader-address= leader-id= -2026-03-31T06:16:23.806Z [WARN] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:23.806Z [INFO] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: entering candidate state: node="Node at 6fdca903-eef1-a757-3da1-a30a9fe3095e [Candidate]" term=2 -2026-03-31T06:16:23.806Z [INFO] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: election won: term=2 tally=3 -2026-03-31T06:16:23.806Z [INFO] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: entering leader state: leader="Node at 6fdca903-eef1-a757-3da1-a30a9fe3095e [Leader]" -2026-03-31T06:16:23.807Z [INFO] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: added peer, starting replication: peer=server-2d50918c-128e-a400-4c46-c2c4410f26c9 -2026-03-31T06:16:23.807Z [INFO] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: added peer, starting replication: peer=server-8ab3fd73-750c-5061-ffdc-b419fcbc9a81 -2026-03-31T06:16:23.807Z [INFO] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: added peer, starting replication: peer=server-3d1b2d19-c112-cbdd-d59f-a1c10ba297e4 -2026-03-31T06:16:23.807Z [INFO] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: pipelining replication: peer="{Voter server-2d50918c-128e-a400-4c46-c2c4410f26c9 2d50918c-128e-a400-4c46-c2c4410f26c9}" -2026-03-31T06:16:23.807Z [INFO] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: pipelining replication: peer="{Voter server-3d1b2d19-c112-cbdd-d59f-a1c10ba297e4 3d1b2d19-c112-cbdd-d59f-a1c10ba297e4}" -2026-03-31T06:16:23.807Z [INFO] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: pipelining replication: peer="{Voter server-8ab3fd73-750c-5061-ffdc-b419fcbc9a81 8ab3fd73-750c-5061-ffdc-b419fcbc9a81}" -2026-03-31T06:16:23.912Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:23.80703418 +0000 UTC m=+12.084641561 (1 nodes), 63.46944ms from start of poll, 64.04212ms from cluster start. Timeout at 2026-03-31 06:16:23.91211286 +0000 UTC m=+12.189720241, 105.07868ms after stability -2026-03-31T06:16:24.018Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:16:24.123Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:16:24.01814518 +0000 UTC m=+12.295752561 (3 nodes), 2.6µs from start of poll, 275.15312ms from cluster start. Timeout at 2026-03-31 06:16:24.12315246 +0000 UTC m=+12.400759841, 105.00728ms after stability -2026-03-31T06:16:24.123Z [INFO] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: updating configuration: command=RemoveServer server-id=server-3d1b2d19-c112-cbdd-d59f-a1c10ba297e4 server-addr= servers="[{Suffrage:Voter ID:server-2d50918c-128e-a400-4c46-c2c4410f26c9 Address:2d50918c-128e-a400-4c46-c2c4410f26c9} {Suffrage:Voter ID:server-8ab3fd73-750c-5061-ffdc-b419fcbc9a81 Address:8ab3fd73-750c-5061-ffdc-b419fcbc9a81} {Suffrage:Voter ID:server-6fdca903-eef1-a757-3da1-a30a9fe3095e Address:6fdca903-eef1-a757-3da1-a30a9fe3095e}]" -2026-03-31T06:16:24.123Z [INFO] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: removed peer, stopping replication: peer=server-3d1b2d19-c112-cbdd-d59f-a1c10ba297e4 last-index=3 -2026-03-31T06:16:24.123Z [ERROR] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: failed to pipeline appendEntries: peer="{Voter server-2d50918c-128e-a400-4c46-c2c4410f26c9 2d50918c-128e-a400-4c46-c2c4410f26c9}" error="append pipeline closed" -2026-03-31T06:16:24.123Z [INFO] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: aborting pipeline replication: peer="{Voter server-2d50918c-128e-a400-4c46-c2c4410f26c9 2d50918c-128e-a400-4c46-c2c4410f26c9}" -2026-03-31T06:16:24.123Z [INFO] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: aborting pipeline replication: peer="{Voter server-3d1b2d19-c112-cbdd-d59f-a1c10ba297e4 3d1b2d19-c112-cbdd-d59f-a1c10ba297e4}" -2026-03-31T06:16:24.123Z [ERROR] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: failed to pipeline appendEntries: peer="{Voter server-8ab3fd73-750c-5061-ffdc-b419fcbc9a81 8ab3fd73-750c-5061-ffdc-b419fcbc9a81}" error="append pipeline closed" -2026-03-31T06:16:24.124Z [INFO] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: aborting pipeline replication: peer="{Voter server-8ab3fd73-750c-5061-ffdc-b419fcbc9a81 8ab3fd73-750c-5061-ffdc-b419fcbc9a81}" -2026-03-31T06:16:24.128Z [ERROR] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: failed to heartbeat to: peer=2d50918c-128e-a400-4c46-c2c4410f26c9 backoff time=10ms error="failed to connect to peer: 2d50918c-128e-a400-4c46-c2c4410f26c9" -2026-03-31T06:16:24.128Z [ERROR] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: failed to heartbeat to: peer=8ab3fd73-750c-5061-ffdc-b419fcbc9a81 backoff time=10ms error="failed to connect to peer: 8ab3fd73-750c-5061-ffdc-b419fcbc9a81" -2026-03-31T06:16:24.129Z [ERROR] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: failed to appendEntries to: peer="{Voter server-8ab3fd73-750c-5061-ffdc-b419fcbc9a81 8ab3fd73-750c-5061-ffdc-b419fcbc9a81}" error="failed to connect to peer: 8ab3fd73-750c-5061-ffdc-b419fcbc9a81" -2026-03-31T06:16:24.134Z [ERROR] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: failed to appendEntries to: peer="{Voter server-2d50918c-128e-a400-4c46-c2c4410f26c9 2d50918c-128e-a400-4c46-c2c4410f26c9}" error="failed to connect to peer: 2d50918c-128e-a400-4c46-c2c4410f26c9" -2026-03-31T06:16:24.146Z [ERROR] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: failed to heartbeat to: peer=2d50918c-128e-a400-4c46-c2c4410f26c9 backoff time=10ms error="failed to connect to peer: 2d50918c-128e-a400-4c46-c2c4410f26c9" -2026-03-31T06:16:24.149Z [ERROR] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: failed to heartbeat to: peer=8ab3fd73-750c-5061-ffdc-b419fcbc9a81 backoff time=10ms error="failed to connect to peer: 8ab3fd73-750c-5061-ffdc-b419fcbc9a81" -2026-03-31T06:16:24.149Z [ERROR] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: failed to appendEntries to: peer="{Voter server-8ab3fd73-750c-5061-ffdc-b419fcbc9a81 8ab3fd73-750c-5061-ffdc-b419fcbc9a81}" error="failed to connect to peer: 8ab3fd73-750c-5061-ffdc-b419fcbc9a81" -2026-03-31T06:16:24.154Z [ERROR] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: failed to appendEntries to: peer="{Voter server-2d50918c-128e-a400-4c46-c2c4410f26c9 2d50918c-128e-a400-4c46-c2c4410f26c9}" error="failed to connect to peer: 2d50918c-128e-a400-4c46-c2c4410f26c9" -2026-03-31T06:16:24.163Z [ERROR] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: failed to heartbeat to: peer=2d50918c-128e-a400-4c46-c2c4410f26c9 backoff time=10ms error="failed to connect to peer: 2d50918c-128e-a400-4c46-c2c4410f26c9" -2026-03-31T06:16:24.167Z [ERROR] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: failed to heartbeat to: peer=8ab3fd73-750c-5061-ffdc-b419fcbc9a81 backoff time=10ms error="failed to connect to peer: 8ab3fd73-750c-5061-ffdc-b419fcbc9a81" -2026-03-31T06:16:24.169Z [ERROR] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: failed to appendEntries to: peer="{Voter server-8ab3fd73-750c-5061-ffdc-b419fcbc9a81 8ab3fd73-750c-5061-ffdc-b419fcbc9a81}" error="failed to connect to peer: 8ab3fd73-750c-5061-ffdc-b419fcbc9a81" -2026-03-31T06:16:24.172Z [WARN] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: failed to contact: server-id=server-2d50918c-128e-a400-4c46-c2c4410f26c9 time=50.67476ms -2026-03-31T06:16:24.172Z [WARN] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: failed to contact: server-id=server-8ab3fd73-750c-5061-ffdc-b419fcbc9a81 time=50.61688ms -2026-03-31T06:16:24.172Z [WARN] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: failed to contact quorum of nodes, stepping down -2026-03-31T06:16:24.172Z [INFO] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: entering follower state: follower="Node at 6fdca903-eef1-a757-3da1-a30a9fe3095e [Follower]" leader-address= leader-id= -2026-03-31T06:16:24.172Z [ERROR] server-6fdca903-eef1-a757-3da1-a30a9fe3095e: failed to appendEntries to: peer="{Voter server-2d50918c-128e-a400-4c46-c2c4410f26c9 2d50918c-128e-a400-4c46-c2c4410f26c9}" error="failed to connect to peer: 2d50918c-128e-a400-4c46-c2c4410f26c9" ---- PASS: TestRaft_RemoveFollower_SplitCluster (0.43s) +2025-02-25T23:57:45.250Z [INFO] server-2d5be957-3ba5-a191-69b8-149e5fdd85c7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2d5be957-3ba5-a191-69b8-149e5fdd85c7 Address:2d5be957-3ba5-a191-69b8-149e5fdd85c7} {Suffrage:Voter ID:server-a0c45385-4cf3-c246-2aeb-888119d612cc Address:a0c45385-4cf3-c246-2aeb-888119d612cc} {Suffrage:Voter ID:server-00b2833b-dd57-90de-3742-0a8365b5da74 Address:00b2833b-dd57-90de-3742-0a8365b5da74} {Suffrage:Voter ID:server-05655d7d-1934-c96a-28b3-ae6fb55f7fc1 Address:05655d7d-1934-c96a-28b3-ae6fb55f7fc1}]" +2025-02-25T23:57:45.250Z [INFO] server-a0c45385-4cf3-c246-2aeb-888119d612cc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2d5be957-3ba5-a191-69b8-149e5fdd85c7 Address:2d5be957-3ba5-a191-69b8-149e5fdd85c7} {Suffrage:Voter ID:server-a0c45385-4cf3-c246-2aeb-888119d612cc Address:a0c45385-4cf3-c246-2aeb-888119d612cc} {Suffrage:Voter ID:server-00b2833b-dd57-90de-3742-0a8365b5da74 Address:00b2833b-dd57-90de-3742-0a8365b5da74} {Suffrage:Voter ID:server-05655d7d-1934-c96a-28b3-ae6fb55f7fc1 Address:05655d7d-1934-c96a-28b3-ae6fb55f7fc1}]" +2025-02-25T23:57:45.250Z [INFO] server-00b2833b-dd57-90de-3742-0a8365b5da74: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2d5be957-3ba5-a191-69b8-149e5fdd85c7 Address:2d5be957-3ba5-a191-69b8-149e5fdd85c7} {Suffrage:Voter ID:server-a0c45385-4cf3-c246-2aeb-888119d612cc Address:a0c45385-4cf3-c246-2aeb-888119d612cc} {Suffrage:Voter ID:server-00b2833b-dd57-90de-3742-0a8365b5da74 Address:00b2833b-dd57-90de-3742-0a8365b5da74} {Suffrage:Voter ID:server-05655d7d-1934-c96a-28b3-ae6fb55f7fc1 Address:05655d7d-1934-c96a-28b3-ae6fb55f7fc1}]" +2025-02-25T23:57:45.250Z [INFO] server-05655d7d-1934-c96a-28b3-ae6fb55f7fc1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2d5be957-3ba5-a191-69b8-149e5fdd85c7 Address:2d5be957-3ba5-a191-69b8-149e5fdd85c7} {Suffrage:Voter ID:server-a0c45385-4cf3-c246-2aeb-888119d612cc Address:a0c45385-4cf3-c246-2aeb-888119d612cc} {Suffrage:Voter ID:server-00b2833b-dd57-90de-3742-0a8365b5da74 Address:00b2833b-dd57-90de-3742-0a8365b5da74} {Suffrage:Voter ID:server-05655d7d-1934-c96a-28b3-ae6fb55f7fc1 Address:05655d7d-1934-c96a-28b3-ae6fb55f7fc1}]" +2025-02-25T23:57:45.250Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:45.250Z [INFO] server-2d5be957-3ba5-a191-69b8-149e5fdd85c7: entering follower state: follower="Node at 2d5be957-3ba5-a191-69b8-149e5fdd85c7 [Follower]" leader-address= leader-id= +2025-02-25T23:57:45.250Z [INFO] server-a0c45385-4cf3-c246-2aeb-888119d612cc: entering follower state: follower="Node at a0c45385-4cf3-c246-2aeb-888119d612cc [Follower]" leader-address= leader-id= +2025-02-25T23:57:45.250Z [INFO] server-00b2833b-dd57-90de-3742-0a8365b5da74: entering follower state: follower="Node at 00b2833b-dd57-90de-3742-0a8365b5da74 [Follower]" leader-address= leader-id= +2025-02-25T23:57:45.250Z [INFO] server-05655d7d-1934-c96a-28b3-ae6fb55f7fc1: entering follower state: follower="Node at 05655d7d-1934-c96a-28b3-ae6fb55f7fc1 [Follower]" leader-address= leader-id= +2025-02-25T23:57:45.317Z [WARN] server-a0c45385-4cf3-c246-2aeb-888119d612cc: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:45.317Z [INFO] server-a0c45385-4cf3-c246-2aeb-888119d612cc: entering candidate state: node="Node at a0c45385-4cf3-c246-2aeb-888119d612cc [Candidate]" term=2 +2025-02-25T23:57:45.317Z [WARN] server-05655d7d-1934-c96a-28b3-ae6fb55f7fc1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:45.317Z [INFO] server-05655d7d-1934-c96a-28b3-ae6fb55f7fc1: entering candidate state: node="Node at 05655d7d-1934-c96a-28b3-ae6fb55f7fc1 [Candidate]" term=2 +2025-02-25T23:57:45.317Z [INFO] server-05655d7d-1934-c96a-28b3-ae6fb55f7fc1: duplicate requestVote for same term: term=2 +2025-02-25T23:57:45.318Z [INFO] server-a0c45385-4cf3-c246-2aeb-888119d612cc: election won: term=2 tally=3 +2025-02-25T23:57:45.318Z [INFO] server-a0c45385-4cf3-c246-2aeb-888119d612cc: entering leader state: leader="Node at a0c45385-4cf3-c246-2aeb-888119d612cc [Leader]" +2025-02-25T23:57:45.318Z [INFO] server-a0c45385-4cf3-c246-2aeb-888119d612cc: added peer, starting replication: peer=server-2d5be957-3ba5-a191-69b8-149e5fdd85c7 +2025-02-25T23:57:45.318Z [INFO] server-a0c45385-4cf3-c246-2aeb-888119d612cc: added peer, starting replication: peer=server-00b2833b-dd57-90de-3742-0a8365b5da74 +2025-02-25T23:57:45.318Z [INFO] server-a0c45385-4cf3-c246-2aeb-888119d612cc: added peer, starting replication: peer=server-05655d7d-1934-c96a-28b3-ae6fb55f7fc1 +2025-02-25T23:57:45.318Z [INFO] server-05655d7d-1934-c96a-28b3-ae6fb55f7fc1: entering follower state: follower="Node at 05655d7d-1934-c96a-28b3-ae6fb55f7fc1 [Follower]" leader-address=a0c45385-4cf3-c246-2aeb-888119d612cc leader-id=server-a0c45385-4cf3-c246-2aeb-888119d612cc +2025-02-25T23:57:45.318Z [INFO] server-a0c45385-4cf3-c246-2aeb-888119d612cc: pipelining replication: peer="{Voter server-05655d7d-1934-c96a-28b3-ae6fb55f7fc1 05655d7d-1934-c96a-28b3-ae6fb55f7fc1}" +2025-02-25T23:57:45.318Z [INFO] server-2d5be957-3ba5-a191-69b8-149e5fdd85c7: duplicate requestVote for same term: term=2 +2025-02-25T23:57:45.318Z [WARN] server-a0c45385-4cf3-c246-2aeb-888119d612cc: rejecting vote request since we have a leader: from=05655d7d-1934-c96a-28b3-ae6fb55f7fc1 leader=a0c45385-4cf3-c246-2aeb-888119d612cc leader-id=server-a0c45385-4cf3-c246-2aeb-888119d612cc +2025-02-25T23:57:45.318Z [INFO] server-00b2833b-dd57-90de-3742-0a8365b5da74: duplicate requestVote for same term: term=2 +2025-02-25T23:57:45.318Z [INFO] server-a0c45385-4cf3-c246-2aeb-888119d612cc: pipelining replication: peer="{Voter server-2d5be957-3ba5-a191-69b8-149e5fdd85c7 2d5be957-3ba5-a191-69b8-149e5fdd85c7}" +2025-02-25T23:57:45.318Z [INFO] server-a0c45385-4cf3-c246-2aeb-888119d612cc: pipelining replication: peer="{Voter server-00b2833b-dd57-90de-3742-0a8365b5da74 00b2833b-dd57-90de-3742-0a8365b5da74}" +2025-02-25T23:57:45.425Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:45.31824469 +0000 UTC m=+12.927260097 (1 nodes), 67.682991ms from start of poll, 67.926432ms from cluster start. Timeout at 2025-02-25 23:57:45.42324577 +0000 UTC m=+13.032261177, 105.00108ms after stability +2025-02-25T23:57:45.533Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:57:45.641Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:57:45.533991624 +0000 UTC m=+13.143007031 (3 nodes), 1.96µs from start of poll, 283.673366ms from cluster start. Timeout at 2025-02-25 23:57:45.639000864 +0000 UTC m=+13.248016271, 105.00924ms after stability +2025-02-25T23:57:45.641Z [INFO] server-a0c45385-4cf3-c246-2aeb-888119d612cc: updating configuration: command=RemoveServer server-id=server-05655d7d-1934-c96a-28b3-ae6fb55f7fc1 server-addr= servers="[{Suffrage:Voter ID:server-2d5be957-3ba5-a191-69b8-149e5fdd85c7 Address:2d5be957-3ba5-a191-69b8-149e5fdd85c7} {Suffrage:Voter ID:server-a0c45385-4cf3-c246-2aeb-888119d612cc Address:a0c45385-4cf3-c246-2aeb-888119d612cc} {Suffrage:Voter ID:server-00b2833b-dd57-90de-3742-0a8365b5da74 Address:00b2833b-dd57-90de-3742-0a8365b5da74}]" +2025-02-25T23:57:45.642Z [INFO] server-a0c45385-4cf3-c246-2aeb-888119d612cc: removed peer, stopping replication: peer=server-05655d7d-1934-c96a-28b3-ae6fb55f7fc1 last-index=3 +2025-02-25T23:57:45.642Z [INFO] server-a0c45385-4cf3-c246-2aeb-888119d612cc: aborting pipeline replication: peer="{Voter server-05655d7d-1934-c96a-28b3-ae6fb55f7fc1 05655d7d-1934-c96a-28b3-ae6fb55f7fc1}" +2025-02-25T23:57:45.642Z [ERROR] server-a0c45385-4cf3-c246-2aeb-888119d612cc: failed to pipeline appendEntries: peer="{Voter server-2d5be957-3ba5-a191-69b8-149e5fdd85c7 2d5be957-3ba5-a191-69b8-149e5fdd85c7}" error="append pipeline closed" +2025-02-25T23:57:45.642Z [INFO] server-a0c45385-4cf3-c246-2aeb-888119d612cc: aborting pipeline replication: peer="{Voter server-2d5be957-3ba5-a191-69b8-149e5fdd85c7 2d5be957-3ba5-a191-69b8-149e5fdd85c7}" +2025-02-25T23:57:45.642Z [ERROR] server-a0c45385-4cf3-c246-2aeb-888119d612cc: failed to pipeline appendEntries: peer="{Voter server-00b2833b-dd57-90de-3742-0a8365b5da74 00b2833b-dd57-90de-3742-0a8365b5da74}" error="append pipeline closed" +2025-02-25T23:57:45.642Z [INFO] server-a0c45385-4cf3-c246-2aeb-888119d612cc: aborting pipeline replication: peer="{Voter server-00b2833b-dd57-90de-3742-0a8365b5da74 00b2833b-dd57-90de-3742-0a8365b5da74}" +2025-02-25T23:57:45.645Z [ERROR] server-a0c45385-4cf3-c246-2aeb-888119d612cc: failed to heartbeat to: peer=00b2833b-dd57-90de-3742-0a8365b5da74 backoff time=10ms error="failed to connect to peer: 00b2833b-dd57-90de-3742-0a8365b5da74" +2025-02-25T23:57:45.649Z [ERROR] server-a0c45385-4cf3-c246-2aeb-888119d612cc: failed to heartbeat to: peer=2d5be957-3ba5-a191-69b8-149e5fdd85c7 backoff time=10ms error="failed to connect to peer: 2d5be957-3ba5-a191-69b8-149e5fdd85c7" +2025-02-25T23:57:45.649Z [ERROR] server-a0c45385-4cf3-c246-2aeb-888119d612cc: failed to appendEntries to: peer="{Voter server-2d5be957-3ba5-a191-69b8-149e5fdd85c7 2d5be957-3ba5-a191-69b8-149e5fdd85c7}" error="failed to connect to peer: 2d5be957-3ba5-a191-69b8-149e5fdd85c7" +2025-02-25T23:57:45.653Z [ERROR] server-a0c45385-4cf3-c246-2aeb-888119d612cc: failed to appendEntries to: peer="{Voter server-00b2833b-dd57-90de-3742-0a8365b5da74 00b2833b-dd57-90de-3742-0a8365b5da74}" error="failed to connect to peer: 00b2833b-dd57-90de-3742-0a8365b5da74" +2025-02-25T23:57:45.669Z [ERROR] server-a0c45385-4cf3-c246-2aeb-888119d612cc: failed to heartbeat to: peer=00b2833b-dd57-90de-3742-0a8365b5da74 backoff time=10ms error="failed to connect to peer: 00b2833b-dd57-90de-3742-0a8365b5da74" +2025-02-25T23:57:45.669Z [ERROR] server-a0c45385-4cf3-c246-2aeb-888119d612cc: failed to heartbeat to: peer=2d5be957-3ba5-a191-69b8-149e5fdd85c7 backoff time=10ms error="failed to connect to peer: 2d5be957-3ba5-a191-69b8-149e5fdd85c7" +2025-02-25T23:57:45.669Z [ERROR] server-a0c45385-4cf3-c246-2aeb-888119d612cc: failed to appendEntries to: peer="{Voter server-2d5be957-3ba5-a191-69b8-149e5fdd85c7 2d5be957-3ba5-a191-69b8-149e5fdd85c7}" error="failed to connect to peer: 2d5be957-3ba5-a191-69b8-149e5fdd85c7" +2025-02-25T23:57:45.673Z [ERROR] server-a0c45385-4cf3-c246-2aeb-888119d612cc: failed to appendEntries to: peer="{Voter server-00b2833b-dd57-90de-3742-0a8365b5da74 00b2833b-dd57-90de-3742-0a8365b5da74}" error="failed to connect to peer: 00b2833b-dd57-90de-3742-0a8365b5da74" +2025-02-25T23:57:45.689Z [WARN] server-a0c45385-4cf3-c246-2aeb-888119d612cc: failed to contact: server-id=server-2d5be957-3ba5-a191-69b8-149e5fdd85c7 time=51.84001ms +2025-02-25T23:57:45.689Z [WARN] server-a0c45385-4cf3-c246-2aeb-888119d612cc: failed to contact: server-id=server-00b2833b-dd57-90de-3742-0a8365b5da74 time=51.81153ms +2025-02-25T23:57:45.689Z [WARN] server-a0c45385-4cf3-c246-2aeb-888119d612cc: failed to contact quorum of nodes, stepping down +2025-02-25T23:57:45.689Z [INFO] server-a0c45385-4cf3-c246-2aeb-888119d612cc: entering follower state: follower="Node at a0c45385-4cf3-c246-2aeb-888119d612cc [Follower]" leader-address= leader-id= +2025-02-25T23:57:45.689Z [ERROR] server-a0c45385-4cf3-c246-2aeb-888119d612cc: failed to appendEntries to: peer="{Voter server-2d5be957-3ba5-a191-69b8-149e5fdd85c7 2d5be957-3ba5-a191-69b8-149e5fdd85c7}" error="failed to connect to peer: 2d5be957-3ba5-a191-69b8-149e5fdd85c7" +--- PASS: TestRaft_RemoveFollower_SplitCluster (0.44s) === RUN TestRaft_AddKnownPeer -2026-03-31T06:16:24.173Z [INFO] server-281c9aa4-aeff-a825-cf72-0fed0ef320ca: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-281c9aa4-aeff-a825-cf72-0fed0ef320ca Address:281c9aa4-aeff-a825-cf72-0fed0ef320ca} {Suffrage:Voter ID:server-ae1c07e2-5520-78d7-ed95-96dbabebd5bd Address:ae1c07e2-5520-78d7-ed95-96dbabebd5bd} {Suffrage:Voter ID:server-bbd93742-ca21-d7d6-2c60-fd7d66c55b3c Address:bbd93742-ca21-d7d6-2c60-fd7d66c55b3c}]" -2026-03-31T06:16:24.173Z [INFO] server-281c9aa4-aeff-a825-cf72-0fed0ef320ca: entering follower state: follower="Node at 281c9aa4-aeff-a825-cf72-0fed0ef320ca [Follower]" leader-address= leader-id= -2026-03-31T06:16:24.173Z [INFO] server-ae1c07e2-5520-78d7-ed95-96dbabebd5bd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-281c9aa4-aeff-a825-cf72-0fed0ef320ca Address:281c9aa4-aeff-a825-cf72-0fed0ef320ca} {Suffrage:Voter ID:server-ae1c07e2-5520-78d7-ed95-96dbabebd5bd Address:ae1c07e2-5520-78d7-ed95-96dbabebd5bd} {Suffrage:Voter ID:server-bbd93742-ca21-d7d6-2c60-fd7d66c55b3c Address:bbd93742-ca21-d7d6-2c60-fd7d66c55b3c}]" -2026-03-31T06:16:24.173Z [INFO] server-bbd93742-ca21-d7d6-2c60-fd7d66c55b3c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-281c9aa4-aeff-a825-cf72-0fed0ef320ca Address:281c9aa4-aeff-a825-cf72-0fed0ef320ca} {Suffrage:Voter ID:server-ae1c07e2-5520-78d7-ed95-96dbabebd5bd Address:ae1c07e2-5520-78d7-ed95-96dbabebd5bd} {Suffrage:Voter ID:server-bbd93742-ca21-d7d6-2c60-fd7d66c55b3c Address:bbd93742-ca21-d7d6-2c60-fd7d66c55b3c}]" -2026-03-31T06:16:24.173Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:24.173Z [INFO] server-ae1c07e2-5520-78d7-ed95-96dbabebd5bd: entering follower state: follower="Node at ae1c07e2-5520-78d7-ed95-96dbabebd5bd [Follower]" leader-address= leader-id= -2026-03-31T06:16:24.173Z [INFO] server-bbd93742-ca21-d7d6-2c60-fd7d66c55b3c: entering follower state: follower="Node at bbd93742-ca21-d7d6-2c60-fd7d66c55b3c [Follower]" leader-address= leader-id= -2026-03-31T06:16:24.226Z [WARN] server-281c9aa4-aeff-a825-cf72-0fed0ef320ca: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:24.226Z [INFO] server-281c9aa4-aeff-a825-cf72-0fed0ef320ca: entering candidate state: node="Node at 281c9aa4-aeff-a825-cf72-0fed0ef320ca [Candidate]" term=2 -2026-03-31T06:16:24.226Z [INFO] server-281c9aa4-aeff-a825-cf72-0fed0ef320ca: election won: term=2 tally=2 -2026-03-31T06:16:24.226Z [INFO] server-281c9aa4-aeff-a825-cf72-0fed0ef320ca: entering leader state: leader="Node at 281c9aa4-aeff-a825-cf72-0fed0ef320ca [Leader]" -2026-03-31T06:16:24.226Z [INFO] server-281c9aa4-aeff-a825-cf72-0fed0ef320ca: added peer, starting replication: peer=server-ae1c07e2-5520-78d7-ed95-96dbabebd5bd -2026-03-31T06:16:24.226Z [INFO] server-281c9aa4-aeff-a825-cf72-0fed0ef320ca: added peer, starting replication: peer=server-bbd93742-ca21-d7d6-2c60-fd7d66c55b3c -2026-03-31T06:16:24.226Z [INFO] server-281c9aa4-aeff-a825-cf72-0fed0ef320ca: pipelining replication: peer="{Voter server-ae1c07e2-5520-78d7-ed95-96dbabebd5bd ae1c07e2-5520-78d7-ed95-96dbabebd5bd}" -2026-03-31T06:16:24.226Z [INFO] server-281c9aa4-aeff-a825-cf72-0fed0ef320ca: pipelining replication: peer="{Voter server-bbd93742-ca21-d7d6-2c60-fd7d66c55b3c bbd93742-ca21-d7d6-2c60-fd7d66c55b3c}" -2026-03-31T06:16:24.332Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:24.22622198 +0000 UTC m=+12.503829401 (1 nodes), 52.79972ms from start of poll, 53.1958ms from cluster start. Timeout at 2026-03-31 06:16:24.33122378 +0000 UTC m=+12.608831201, 105.0018ms after stability -2026-03-31T06:16:24.332Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:16:24.437Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:16:24.33244118 +0000 UTC m=+12.610048601 (2 nodes), 1.4µs from start of poll, 159.415ms from cluster start. Timeout at 2026-03-31 06:16:24.43744918 +0000 UTC m=+12.715056561, 105.00796ms after stability -2026-03-31T06:16:24.438Z [INFO] server-281c9aa4-aeff-a825-cf72-0fed0ef320ca: updating configuration: command=AddVoter server-id=server-ae1c07e2-5520-78d7-ed95-96dbabebd5bd server-addr=ae1c07e2-5520-78d7-ed95-96dbabebd5bd servers="[{Suffrage:Voter ID:server-281c9aa4-aeff-a825-cf72-0fed0ef320ca Address:281c9aa4-aeff-a825-cf72-0fed0ef320ca} {Suffrage:Voter ID:server-ae1c07e2-5520-78d7-ed95-96dbabebd5bd Address:ae1c07e2-5520-78d7-ed95-96dbabebd5bd} {Suffrage:Voter ID:server-bbd93742-ca21-d7d6-2c60-fd7d66c55b3c Address:bbd93742-ca21-d7d6-2c60-fd7d66c55b3c}]" -2026-03-31T06:16:24.438Z [INFO] server-281c9aa4-aeff-a825-cf72-0fed0ef320ca: aborting pipeline replication: peer="{Voter server-bbd93742-ca21-d7d6-2c60-fd7d66c55b3c bbd93742-ca21-d7d6-2c60-fd7d66c55b3c}" -2026-03-31T06:16:24.438Z [INFO] server-281c9aa4-aeff-a825-cf72-0fed0ef320ca: aborting pipeline replication: peer="{Voter server-ae1c07e2-5520-78d7-ed95-96dbabebd5bd ae1c07e2-5520-78d7-ed95-96dbabebd5bd}" +2025-02-25T23:57:45.690Z [INFO] server-7cb88397-958c-b5a9-4d44-8f9066d039f4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7cb88397-958c-b5a9-4d44-8f9066d039f4 Address:7cb88397-958c-b5a9-4d44-8f9066d039f4} {Suffrage:Voter ID:server-6695c3ac-086e-48c8-925b-f7cba9c4f9f5 Address:6695c3ac-086e-48c8-925b-f7cba9c4f9f5} {Suffrage:Voter ID:server-9c036fb7-7fad-6caa-e30c-704cc6565ed0 Address:9c036fb7-7fad-6caa-e30c-704cc6565ed0}]" +2025-02-25T23:57:45.690Z [INFO] server-6695c3ac-086e-48c8-925b-f7cba9c4f9f5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7cb88397-958c-b5a9-4d44-8f9066d039f4 Address:7cb88397-958c-b5a9-4d44-8f9066d039f4} {Suffrage:Voter ID:server-6695c3ac-086e-48c8-925b-f7cba9c4f9f5 Address:6695c3ac-086e-48c8-925b-f7cba9c4f9f5} {Suffrage:Voter ID:server-9c036fb7-7fad-6caa-e30c-704cc6565ed0 Address:9c036fb7-7fad-6caa-e30c-704cc6565ed0}]" +2025-02-25T23:57:45.690Z [INFO] server-9c036fb7-7fad-6caa-e30c-704cc6565ed0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7cb88397-958c-b5a9-4d44-8f9066d039f4 Address:7cb88397-958c-b5a9-4d44-8f9066d039f4} {Suffrage:Voter ID:server-6695c3ac-086e-48c8-925b-f7cba9c4f9f5 Address:6695c3ac-086e-48c8-925b-f7cba9c4f9f5} {Suffrage:Voter ID:server-9c036fb7-7fad-6caa-e30c-704cc6565ed0 Address:9c036fb7-7fad-6caa-e30c-704cc6565ed0}]" +2025-02-25T23:57:45.690Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:45.690Z [INFO] server-7cb88397-958c-b5a9-4d44-8f9066d039f4: entering follower state: follower="Node at 7cb88397-958c-b5a9-4d44-8f9066d039f4 [Follower]" leader-address= leader-id= +2025-02-25T23:57:45.690Z [INFO] server-6695c3ac-086e-48c8-925b-f7cba9c4f9f5: entering follower state: follower="Node at 6695c3ac-086e-48c8-925b-f7cba9c4f9f5 [Follower]" leader-address= leader-id= +2025-02-25T23:57:45.690Z [INFO] server-9c036fb7-7fad-6caa-e30c-704cc6565ed0: entering follower state: follower="Node at 9c036fb7-7fad-6caa-e30c-704cc6565ed0 [Follower]" leader-address= leader-id= +2025-02-25T23:57:45.745Z [WARN] server-6695c3ac-086e-48c8-925b-f7cba9c4f9f5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:45.745Z [INFO] server-6695c3ac-086e-48c8-925b-f7cba9c4f9f5: entering candidate state: node="Node at 6695c3ac-086e-48c8-925b-f7cba9c4f9f5 [Candidate]" term=2 +2025-02-25T23:57:45.745Z [INFO] server-6695c3ac-086e-48c8-925b-f7cba9c4f9f5: election won: term=2 tally=2 +2025-02-25T23:57:45.745Z [INFO] server-6695c3ac-086e-48c8-925b-f7cba9c4f9f5: entering leader state: leader="Node at 6695c3ac-086e-48c8-925b-f7cba9c4f9f5 [Leader]" +2025-02-25T23:57:45.746Z [INFO] server-6695c3ac-086e-48c8-925b-f7cba9c4f9f5: added peer, starting replication: peer=server-7cb88397-958c-b5a9-4d44-8f9066d039f4 +2025-02-25T23:57:45.746Z [INFO] server-6695c3ac-086e-48c8-925b-f7cba9c4f9f5: added peer, starting replication: peer=server-9c036fb7-7fad-6caa-e30c-704cc6565ed0 +2025-02-25T23:57:45.746Z [INFO] server-6695c3ac-086e-48c8-925b-f7cba9c4f9f5: pipelining replication: peer="{Voter server-9c036fb7-7fad-6caa-e30c-704cc6565ed0 9c036fb7-7fad-6caa-e30c-704cc6565ed0}" +2025-02-25T23:57:45.746Z [INFO] server-6695c3ac-086e-48c8-925b-f7cba9c4f9f5: pipelining replication: peer="{Voter server-7cb88397-958c-b5a9-4d44-8f9066d039f4 7cb88397-958c-b5a9-4d44-8f9066d039f4}" +2025-02-25T23:57:45.853Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:45.746203696 +0000 UTC m=+13.355219103 (1 nodes), 55.659555ms from start of poll, 55.827756ms from cluster start. Timeout at 2025-02-25 23:57:45.851211976 +0000 UTC m=+13.460227383, 105.00828ms after stability +2025-02-25T23:57:45.853Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:57:45.961Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:57:45.853932102 +0000 UTC m=+13.462947509 (2 nodes), 1.12µs from start of poll, 163.556162ms from cluster start. Timeout at 2025-02-25 23:57:45.958941862 +0000 UTC m=+13.567957269, 105.00976ms after stability +2025-02-25T23:57:45.962Z [INFO] server-6695c3ac-086e-48c8-925b-f7cba9c4f9f5: updating configuration: command=AddVoter server-id=server-7cb88397-958c-b5a9-4d44-8f9066d039f4 server-addr=7cb88397-958c-b5a9-4d44-8f9066d039f4 servers="[{Suffrage:Voter ID:server-7cb88397-958c-b5a9-4d44-8f9066d039f4 Address:7cb88397-958c-b5a9-4d44-8f9066d039f4} {Suffrage:Voter ID:server-6695c3ac-086e-48c8-925b-f7cba9c4f9f5 Address:6695c3ac-086e-48c8-925b-f7cba9c4f9f5} {Suffrage:Voter ID:server-9c036fb7-7fad-6caa-e30c-704cc6565ed0 Address:9c036fb7-7fad-6caa-e30c-704cc6565ed0}]" +2025-02-25T23:57:45.962Z [INFO] server-6695c3ac-086e-48c8-925b-f7cba9c4f9f5: aborting pipeline replication: peer="{Voter server-9c036fb7-7fad-6caa-e30c-704cc6565ed0 9c036fb7-7fad-6caa-e30c-704cc6565ed0}" +2025-02-25T23:57:45.962Z [INFO] server-6695c3ac-086e-48c8-925b-f7cba9c4f9f5: aborting pipeline replication: peer="{Voter server-7cb88397-958c-b5a9-4d44-8f9066d039f4 7cb88397-958c-b5a9-4d44-8f9066d039f4}" --- PASS: TestRaft_AddKnownPeer (0.27s) === RUN TestRaft_RemoveUnknownPeer -2026-03-31T06:16:24.439Z [INFO] server-863f4971-e881-9ecd-9f62-dcb87bccd606: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-863f4971-e881-9ecd-9f62-dcb87bccd606 Address:863f4971-e881-9ecd-9f62-dcb87bccd606} {Suffrage:Voter ID:server-4e45e5d6-cc5e-9090-8713-d0e61ffeff2a Address:4e45e5d6-cc5e-9090-8713-d0e61ffeff2a} {Suffrage:Voter ID:server-a06c8f26-5165-2a8e-d209-6efc70600249 Address:a06c8f26-5165-2a8e-d209-6efc70600249}]" -2026-03-31T06:16:24.439Z [INFO] server-4e45e5d6-cc5e-9090-8713-d0e61ffeff2a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-863f4971-e881-9ecd-9f62-dcb87bccd606 Address:863f4971-e881-9ecd-9f62-dcb87bccd606} {Suffrage:Voter ID:server-4e45e5d6-cc5e-9090-8713-d0e61ffeff2a Address:4e45e5d6-cc5e-9090-8713-d0e61ffeff2a} {Suffrage:Voter ID:server-a06c8f26-5165-2a8e-d209-6efc70600249 Address:a06c8f26-5165-2a8e-d209-6efc70600249}]" -2026-03-31T06:16:24.439Z [INFO] server-863f4971-e881-9ecd-9f62-dcb87bccd606: entering follower state: follower="Node at 863f4971-e881-9ecd-9f62-dcb87bccd606 [Follower]" leader-address= leader-id= -2026-03-31T06:16:24.439Z [INFO] server-a06c8f26-5165-2a8e-d209-6efc70600249: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-863f4971-e881-9ecd-9f62-dcb87bccd606 Address:863f4971-e881-9ecd-9f62-dcb87bccd606} {Suffrage:Voter ID:server-4e45e5d6-cc5e-9090-8713-d0e61ffeff2a Address:4e45e5d6-cc5e-9090-8713-d0e61ffeff2a} {Suffrage:Voter ID:server-a06c8f26-5165-2a8e-d209-6efc70600249 Address:a06c8f26-5165-2a8e-d209-6efc70600249}]" -2026-03-31T06:16:24.439Z [INFO] server-4e45e5d6-cc5e-9090-8713-d0e61ffeff2a: entering follower state: follower="Node at 4e45e5d6-cc5e-9090-8713-d0e61ffeff2a [Follower]" leader-address= leader-id= -2026-03-31T06:16:24.439Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:24.439Z [INFO] server-a06c8f26-5165-2a8e-d209-6efc70600249: entering follower state: follower="Node at a06c8f26-5165-2a8e-d209-6efc70600249 [Follower]" leader-address= leader-id= -2026-03-31T06:16:24.495Z [WARN] server-a06c8f26-5165-2a8e-d209-6efc70600249: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:24.495Z [INFO] server-a06c8f26-5165-2a8e-d209-6efc70600249: entering candidate state: node="Node at a06c8f26-5165-2a8e-d209-6efc70600249 [Candidate]" term=2 -2026-03-31T06:16:24.495Z [INFO] server-a06c8f26-5165-2a8e-d209-6efc70600249: election won: term=2 tally=2 -2026-03-31T06:16:24.495Z [INFO] server-a06c8f26-5165-2a8e-d209-6efc70600249: entering leader state: leader="Node at a06c8f26-5165-2a8e-d209-6efc70600249 [Leader]" -2026-03-31T06:16:24.495Z [INFO] server-a06c8f26-5165-2a8e-d209-6efc70600249: added peer, starting replication: peer=server-863f4971-e881-9ecd-9f62-dcb87bccd606 -2026-03-31T06:16:24.495Z [INFO] server-a06c8f26-5165-2a8e-d209-6efc70600249: added peer, starting replication: peer=server-4e45e5d6-cc5e-9090-8713-d0e61ffeff2a -2026-03-31T06:16:24.495Z [INFO] server-a06c8f26-5165-2a8e-d209-6efc70600249: pipelining replication: peer="{Voter server-863f4971-e881-9ecd-9f62-dcb87bccd606 863f4971-e881-9ecd-9f62-dcb87bccd606}" -2026-03-31T06:16:24.495Z [INFO] server-a06c8f26-5165-2a8e-d209-6efc70600249: pipelining replication: peer="{Voter server-4e45e5d6-cc5e-9090-8713-d0e61ffeff2a 4e45e5d6-cc5e-9090-8713-d0e61ffeff2a}" -2026-03-31T06:16:24.602Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:24.49579774 +0000 UTC m=+12.773405161 (1 nodes), 56.30644ms from start of poll, 56.75132ms from cluster start. Timeout at 2026-03-31 06:16:24.6008809 +0000 UTC m=+12.878488281, 105.08312ms after stability -2026-03-31T06:16:24.602Z [INFO] server-a06c8f26-5165-2a8e-d209-6efc70600249: updating configuration: command=RemoveServer server-id=bc9fff21-40a9-8492-124e-aabf4ffea9e9 server-addr= servers="[{Suffrage:Voter ID:server-863f4971-e881-9ecd-9f62-dcb87bccd606 Address:863f4971-e881-9ecd-9f62-dcb87bccd606} {Suffrage:Voter ID:server-4e45e5d6-cc5e-9090-8713-d0e61ffeff2a Address:4e45e5d6-cc5e-9090-8713-d0e61ffeff2a} {Suffrage:Voter ID:server-a06c8f26-5165-2a8e-d209-6efc70600249 Address:a06c8f26-5165-2a8e-d209-6efc70600249}]" -2026-03-31T06:16:24.602Z [INFO] server-a06c8f26-5165-2a8e-d209-6efc70600249: aborting pipeline replication: peer="{Voter server-4e45e5d6-cc5e-9090-8713-d0e61ffeff2a 4e45e5d6-cc5e-9090-8713-d0e61ffeff2a}" -2026-03-31T06:16:24.602Z [INFO] server-a06c8f26-5165-2a8e-d209-6efc70600249: aborting pipeline replication: peer="{Voter server-863f4971-e881-9ecd-9f62-dcb87bccd606 863f4971-e881-9ecd-9f62-dcb87bccd606}" +2025-02-25T23:57:45.962Z [INFO] server-80a4ab6e-1832-86cd-7815-f1b0959917a1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-80a4ab6e-1832-86cd-7815-f1b0959917a1 Address:80a4ab6e-1832-86cd-7815-f1b0959917a1} {Suffrage:Voter ID:server-924b4a00-8ae3-0acf-ecaa-9fbbc1334db7 Address:924b4a00-8ae3-0acf-ecaa-9fbbc1334db7} {Suffrage:Voter ID:server-a3ad1109-5805-729a-3782-706505010dc1 Address:a3ad1109-5805-729a-3782-706505010dc1}]" +2025-02-25T23:57:45.962Z [INFO] server-924b4a00-8ae3-0acf-ecaa-9fbbc1334db7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-80a4ab6e-1832-86cd-7815-f1b0959917a1 Address:80a4ab6e-1832-86cd-7815-f1b0959917a1} {Suffrage:Voter ID:server-924b4a00-8ae3-0acf-ecaa-9fbbc1334db7 Address:924b4a00-8ae3-0acf-ecaa-9fbbc1334db7} {Suffrage:Voter ID:server-a3ad1109-5805-729a-3782-706505010dc1 Address:a3ad1109-5805-729a-3782-706505010dc1}]" +2025-02-25T23:57:45.962Z [INFO] server-a3ad1109-5805-729a-3782-706505010dc1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-80a4ab6e-1832-86cd-7815-f1b0959917a1 Address:80a4ab6e-1832-86cd-7815-f1b0959917a1} {Suffrage:Voter ID:server-924b4a00-8ae3-0acf-ecaa-9fbbc1334db7 Address:924b4a00-8ae3-0acf-ecaa-9fbbc1334db7} {Suffrage:Voter ID:server-a3ad1109-5805-729a-3782-706505010dc1 Address:a3ad1109-5805-729a-3782-706505010dc1}]" +2025-02-25T23:57:45.962Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:45.962Z [INFO] server-80a4ab6e-1832-86cd-7815-f1b0959917a1: entering follower state: follower="Node at 80a4ab6e-1832-86cd-7815-f1b0959917a1 [Follower]" leader-address= leader-id= +2025-02-25T23:57:45.962Z [INFO] server-924b4a00-8ae3-0acf-ecaa-9fbbc1334db7: entering follower state: follower="Node at 924b4a00-8ae3-0acf-ecaa-9fbbc1334db7 [Follower]" leader-address= leader-id= +2025-02-25T23:57:45.962Z [INFO] server-a3ad1109-5805-729a-3782-706505010dc1: entering follower state: follower="Node at a3ad1109-5805-729a-3782-706505010dc1 [Follower]" leader-address= leader-id= +2025-02-25T23:57:46.017Z [WARN] server-a3ad1109-5805-729a-3782-706505010dc1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:46.017Z [INFO] server-a3ad1109-5805-729a-3782-706505010dc1: entering candidate state: node="Node at a3ad1109-5805-729a-3782-706505010dc1 [Candidate]" term=2 +2025-02-25T23:57:46.017Z [INFO] server-a3ad1109-5805-729a-3782-706505010dc1: election won: term=2 tally=2 +2025-02-25T23:57:46.018Z [INFO] server-a3ad1109-5805-729a-3782-706505010dc1: entering leader state: leader="Node at a3ad1109-5805-729a-3782-706505010dc1 [Leader]" +2025-02-25T23:57:46.018Z [INFO] server-a3ad1109-5805-729a-3782-706505010dc1: added peer, starting replication: peer=server-80a4ab6e-1832-86cd-7815-f1b0959917a1 +2025-02-25T23:57:46.018Z [INFO] server-a3ad1109-5805-729a-3782-706505010dc1: added peer, starting replication: peer=server-924b4a00-8ae3-0acf-ecaa-9fbbc1334db7 +2025-02-25T23:57:46.018Z [INFO] server-a3ad1109-5805-729a-3782-706505010dc1: pipelining replication: peer="{Voter server-924b4a00-8ae3-0acf-ecaa-9fbbc1334db7 924b4a00-8ae3-0acf-ecaa-9fbbc1334db7}" +2025-02-25T23:57:46.018Z [INFO] server-a3ad1109-5805-729a-3782-706505010dc1: pipelining replication: peer="{Voter server-80a4ab6e-1832-86cd-7815-f1b0959917a1 80a4ab6e-1832-86cd-7815-f1b0959917a1}" +2025-02-25T23:57:46.125Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:46.018199788 +0000 UTC m=+13.627215195 (1 nodes), 55.470673ms from start of poll, 55.625794ms from cluster start. Timeout at 2025-02-25 23:57:46.123216068 +0000 UTC m=+13.732231475, 105.01628ms after stability +2025-02-25T23:57:46.125Z [INFO] server-a3ad1109-5805-729a-3782-706505010dc1: updating configuration: command=RemoveServer server-id=1727bb11-5ef8-2ca8-199f-e606fd0be00f server-addr= servers="[{Suffrage:Voter ID:server-80a4ab6e-1832-86cd-7815-f1b0959917a1 Address:80a4ab6e-1832-86cd-7815-f1b0959917a1} {Suffrage:Voter ID:server-924b4a00-8ae3-0acf-ecaa-9fbbc1334db7 Address:924b4a00-8ae3-0acf-ecaa-9fbbc1334db7} {Suffrage:Voter ID:server-a3ad1109-5805-729a-3782-706505010dc1 Address:a3ad1109-5805-729a-3782-706505010dc1}]" +2025-02-25T23:57:46.126Z [INFO] server-a3ad1109-5805-729a-3782-706505010dc1: aborting pipeline replication: peer="{Voter server-924b4a00-8ae3-0acf-ecaa-9fbbc1334db7 924b4a00-8ae3-0acf-ecaa-9fbbc1334db7}" +2025-02-25T23:57:46.126Z [INFO] server-a3ad1109-5805-729a-3782-706505010dc1: aborting pipeline replication: peer="{Voter server-80a4ab6e-1832-86cd-7815-f1b0959917a1 80a4ab6e-1832-86cd-7815-f1b0959917a1}" --- PASS: TestRaft_RemoveUnknownPeer (0.16s) === RUN TestRaft_SnapshotRestore -2026-03-31T06:16:24.602Z [INFO] server-d3ae96dc-33f3-446c-dd49-cd98c916bb77: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d3ae96dc-33f3-446c-dd49-cd98c916bb77 Address:d3ae96dc-33f3-446c-dd49-cd98c916bb77}]" -2026-03-31T06:16:24.602Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:24.602Z [INFO] server-d3ae96dc-33f3-446c-dd49-cd98c916bb77: entering follower state: follower="Node at d3ae96dc-33f3-446c-dd49-cd98c916bb77 [Follower]" leader-address= leader-id= -2026-03-31T06:16:24.671Z [WARN] server-d3ae96dc-33f3-446c-dd49-cd98c916bb77: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:24.671Z [INFO] server-d3ae96dc-33f3-446c-dd49-cd98c916bb77: entering candidate state: node="Node at d3ae96dc-33f3-446c-dd49-cd98c916bb77 [Candidate]" term=2 -2026-03-31T06:16:24.671Z [INFO] server-d3ae96dc-33f3-446c-dd49-cd98c916bb77: election won: term=2 tally=1 -2026-03-31T06:16:24.671Z [INFO] server-d3ae96dc-33f3-446c-dd49-cd98c916bb77: entering leader state: leader="Node at d3ae96dc-33f3-446c-dd49-cd98c916bb77 [Leader]" -2026-03-31T06:16:24.777Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:24.67143702 +0000 UTC m=+12.949044401 (1 nodes), 68.51212ms from start of poll, 68.6144ms from cluster start. Timeout at 2026-03-31 06:16:24.77643994 +0000 UTC m=+13.054047361, 105.00296ms after stability -2026-03-31T06:16:24.780Z [INFO] server-d3ae96dc-33f3-446c-dd49-cd98c916bb77: starting snapshot up to: index=102 -2026-03-31T06:16:24.780Z [INFO] creating new snapshot: path=/tmp/raft135916015/snapshots/2-102-1774937784780.tmp -2026-03-31T06:16:24.780Z [INFO] server-d3ae96dc-33f3-446c-dd49-cd98c916bb77: compacting logs: from=1 to=92 -2026-03-31T06:16:24.780Z [INFO] server-d3ae96dc-33f3-446c-dd49-cd98c916bb77: snapshot complete up to: index=102 -2026-03-31T06:16:24.780Z [INFO] server-d3ae96dc-33f3-446c-dd49-cd98c916bb77: starting restore from snapshot: id=2-102-1774937784780 last-index=102 last-term=2 size-in-bytes=693 -2026-03-31T06:16:24.781Z [INFO] server-d3ae96dc-33f3-446c-dd49-cd98c916bb77: snapshot restore progress: id=2-102-1774937784780 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% -2026-03-31T06:16:24.781Z [INFO] server-d3ae96dc-33f3-446c-dd49-cd98c916bb77: restored from snapshot: id=2-102-1774937784780 last-index=102 last-term=2 size-in-bytes=693 -2026-03-31T06:16:24.781Z [INFO] server-d3ae96dc-33f3-446c-dd49-cd98c916bb77: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d3ae96dc-33f3-446c-dd49-cd98c916bb77 Address:d3ae96dc-33f3-446c-dd49-cd98c916bb77}]" ---- PASS: TestRaft_SnapshotRestore (0.18s) +2025-02-25T23:57:46.126Z [INFO] server-47e3d1fb-6e2c-77f7-c47d-a413e61b1d59: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-47e3d1fb-6e2c-77f7-c47d-a413e61b1d59 Address:47e3d1fb-6e2c-77f7-c47d-a413e61b1d59}]" +2025-02-25T23:57:46.126Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:46.126Z [INFO] server-47e3d1fb-6e2c-77f7-c47d-a413e61b1d59: entering follower state: follower="Node at 47e3d1fb-6e2c-77f7-c47d-a413e61b1d59 [Follower]" leader-address= leader-id= +2025-02-25T23:57:46.181Z [WARN] server-47e3d1fb-6e2c-77f7-c47d-a413e61b1d59: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:46.181Z [INFO] server-47e3d1fb-6e2c-77f7-c47d-a413e61b1d59: entering candidate state: node="Node at 47e3d1fb-6e2c-77f7-c47d-a413e61b1d59 [Candidate]" term=2 +2025-02-25T23:57:46.181Z [INFO] server-47e3d1fb-6e2c-77f7-c47d-a413e61b1d59: election won: term=2 tally=1 +2025-02-25T23:57:46.181Z [INFO] server-47e3d1fb-6e2c-77f7-c47d-a413e61b1d59: entering leader state: leader="Node at 47e3d1fb-6e2c-77f7-c47d-a413e61b1d59 [Leader]" +2025-02-25T23:57:46.289Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:46.181994631 +0000 UTC m=+13.791010038 (1 nodes), 55.481113ms from start of poll, 55.552273ms from cluster start. Timeout at 2025-02-25 23:57:46.286997511 +0000 UTC m=+13.896012918, 105.00288ms after stability +2025-02-25T23:57:46.292Z [INFO] server-47e3d1fb-6e2c-77f7-c47d-a413e61b1d59: starting snapshot up to: index=102 +2025-02-25T23:57:46.292Z [INFO] creating new snapshot: path=/tmp/raft3619486460/snapshots/2-102-1740527866292.tmp +2025-02-25T23:57:46.298Z [INFO] server-47e3d1fb-6e2c-77f7-c47d-a413e61b1d59: compacting logs: from=1 to=92 +2025-02-25T23:57:46.298Z [INFO] server-47e3d1fb-6e2c-77f7-c47d-a413e61b1d59: snapshot complete up to: index=102 +2025-02-25T23:57:46.298Z [INFO] server-47e3d1fb-6e2c-77f7-c47d-a413e61b1d59: starting restore from snapshot: id=2-102-1740527866292 last-index=102 last-term=2 size-in-bytes=693 +2025-02-25T23:57:46.298Z [INFO] server-47e3d1fb-6e2c-77f7-c47d-a413e61b1d59: snapshot restore progress: id=2-102-1740527866292 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% +2025-02-25T23:57:46.298Z [INFO] server-47e3d1fb-6e2c-77f7-c47d-a413e61b1d59: restored from snapshot: id=2-102-1740527866292 last-index=102 last-term=2 size-in-bytes=693 +2025-02-25T23:57:46.298Z [INFO] server-47e3d1fb-6e2c-77f7-c47d-a413e61b1d59: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-47e3d1fb-6e2c-77f7-c47d-a413e61b1d59 Address:47e3d1fb-6e2c-77f7-c47d-a413e61b1d59}]" +--- PASS: TestRaft_SnapshotRestore (0.17s) === RUN TestRaft_RestoreSnapshotOnStartup_Monotonic -2026-03-31T06:16:24.781Z [INFO] server-4312a246-f720-0798-e053-91a1e5c82917: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4312a246-f720-0798-e053-91a1e5c82917 Address:4312a246-f720-0798-e053-91a1e5c82917}]" -2026-03-31T06:16:24.781Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:24.781Z [INFO] server-4312a246-f720-0798-e053-91a1e5c82917: entering follower state: follower="Node at 4312a246-f720-0798-e053-91a1e5c82917 [Follower]" leader-address= leader-id= -2026-03-31T06:16:24.870Z [WARN] server-4312a246-f720-0798-e053-91a1e5c82917: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:24.870Z [INFO] server-4312a246-f720-0798-e053-91a1e5c82917: entering candidate state: node="Node at 4312a246-f720-0798-e053-91a1e5c82917 [Candidate]" term=2 -2026-03-31T06:16:24.870Z [INFO] server-4312a246-f720-0798-e053-91a1e5c82917: election won: term=2 tally=1 -2026-03-31T06:16:24.870Z [INFO] server-4312a246-f720-0798-e053-91a1e5c82917: entering leader state: leader="Node at 4312a246-f720-0798-e053-91a1e5c82917 [Leader]" -2026-03-31T06:16:24.975Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:24.87036626 +0000 UTC m=+13.147973641 (1 nodes), 88.46244ms from start of poll, 88.65188ms from cluster start. Timeout at 2026-03-31 06:16:24.97536866 +0000 UTC m=+13.252976081, 105.00244ms after stability -2026-03-31T06:16:24.979Z [INFO] server-4312a246-f720-0798-e053-91a1e5c82917: starting snapshot up to: index=102 -2026-03-31T06:16:24.980Z [INFO] creating new snapshot: path=/tmp/raft2946128651/snapshots/2-102-1774937784980.tmp -2026-03-31T06:16:24.980Z [INFO] server-4312a246-f720-0798-e053-91a1e5c82917: compacting logs: from=1 to=92 -2026-03-31T06:16:24.980Z [INFO] server-4312a246-f720-0798-e053-91a1e5c82917: snapshot complete up to: index=102 -2026-03-31T06:16:24.980Z [INFO] server-4312a246-f720-0798-e053-91a1e5c82917: starting restore from snapshot: id=2-102-1774937784980 last-index=102 last-term=2 size-in-bytes=693 -2026-03-31T06:16:24.981Z [INFO] server-4312a246-f720-0798-e053-91a1e5c82917: snapshot restore progress: id=2-102-1774937784980 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% -2026-03-31T06:16:24.981Z [INFO] server-4312a246-f720-0798-e053-91a1e5c82917: restored from snapshot: id=2-102-1774937784980 last-index=102 last-term=2 size-in-bytes=693 -2026-03-31T06:16:24.981Z [INFO] server-4312a246-f720-0798-e053-91a1e5c82917: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4312a246-f720-0798-e053-91a1e5c82917 Address:4312a246-f720-0798-e053-91a1e5c82917}]" ---- PASS: TestRaft_RestoreSnapshotOnStartup_Monotonic (0.20s) +2025-02-25T23:57:46.298Z [INFO] server-e76e91eb-5238-5d3d-24f4-1a3dffbe3ce8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e76e91eb-5238-5d3d-24f4-1a3dffbe3ce8 Address:e76e91eb-5238-5d3d-24f4-1a3dffbe3ce8}]" +2025-02-25T23:57:46.298Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:46.298Z [INFO] server-e76e91eb-5238-5d3d-24f4-1a3dffbe3ce8: entering follower state: follower="Node at e76e91eb-5238-5d3d-24f4-1a3dffbe3ce8 [Follower]" leader-address= leader-id= +2025-02-25T23:57:46.365Z [WARN] server-e76e91eb-5238-5d3d-24f4-1a3dffbe3ce8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:46.365Z [INFO] server-e76e91eb-5238-5d3d-24f4-1a3dffbe3ce8: entering candidate state: node="Node at e76e91eb-5238-5d3d-24f4-1a3dffbe3ce8 [Candidate]" term=2 +2025-02-25T23:57:46.365Z [INFO] server-e76e91eb-5238-5d3d-24f4-1a3dffbe3ce8: election won: term=2 tally=1 +2025-02-25T23:57:46.365Z [INFO] server-e76e91eb-5238-5d3d-24f4-1a3dffbe3ce8: entering leader state: leader="Node at e76e91eb-5238-5d3d-24f4-1a3dffbe3ce8 [Leader]" +2025-02-25T23:57:46.473Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:46.366010003 +0000 UTC m=+13.975025370 (1 nodes), 67.191268ms from start of poll, 67.256788ms from cluster start. Timeout at 2025-02-25 23:57:46.471015203 +0000 UTC m=+14.080030610, 105.00524ms after stability +2025-02-25T23:57:46.474Z [INFO] server-e76e91eb-5238-5d3d-24f4-1a3dffbe3ce8: starting snapshot up to: index=102 +2025-02-25T23:57:46.474Z [INFO] creating new snapshot: path=/tmp/raft3162704262/snapshots/2-102-1740527866474.tmp +2025-02-25T23:57:46.475Z [INFO] server-e76e91eb-5238-5d3d-24f4-1a3dffbe3ce8: compacting logs: from=1 to=92 +2025-02-25T23:57:46.475Z [INFO] server-e76e91eb-5238-5d3d-24f4-1a3dffbe3ce8: snapshot complete up to: index=102 +2025-02-25T23:57:46.475Z [INFO] server-e76e91eb-5238-5d3d-24f4-1a3dffbe3ce8: starting restore from snapshot: id=2-102-1740527866474 last-index=102 last-term=2 size-in-bytes=693 +2025-02-25T23:57:46.475Z [INFO] server-e76e91eb-5238-5d3d-24f4-1a3dffbe3ce8: snapshot restore progress: id=2-102-1740527866474 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% +2025-02-25T23:57:46.475Z [INFO] server-e76e91eb-5238-5d3d-24f4-1a3dffbe3ce8: restored from snapshot: id=2-102-1740527866474 last-index=102 last-term=2 size-in-bytes=693 +2025-02-25T23:57:46.475Z [INFO] server-e76e91eb-5238-5d3d-24f4-1a3dffbe3ce8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e76e91eb-5238-5d3d-24f4-1a3dffbe3ce8 Address:e76e91eb-5238-5d3d-24f4-1a3dffbe3ce8}]" +--- PASS: TestRaft_RestoreSnapshotOnStartup_Monotonic (0.18s) === RUN TestRaft_SnapshotRestore_Progress -2026-03-31T06:16:24.981Z [INFO] server-ebbba981-e6f3-1e13-993f-c1cebd3fe2f9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ebbba981-e6f3-1e13-993f-c1cebd3fe2f9 Address:ebbba981-e6f3-1e13-993f-c1cebd3fe2f9}]" -2026-03-31T06:16:24.981Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:24.981Z [INFO] server-ebbba981-e6f3-1e13-993f-c1cebd3fe2f9: entering follower state: follower="Node at ebbba981-e6f3-1e13-993f-c1cebd3fe2f9 [Follower]" leader-address= leader-id= -2026-03-31T06:16:25.034Z [WARN] server-ebbba981-e6f3-1e13-993f-c1cebd3fe2f9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:25.034Z [INFO] server-ebbba981-e6f3-1e13-993f-c1cebd3fe2f9: entering candidate state: node="Node at ebbba981-e6f3-1e13-993f-c1cebd3fe2f9 [Candidate]" term=2 -2026-03-31T06:16:25.034Z [INFO] server-ebbba981-e6f3-1e13-993f-c1cebd3fe2f9: election won: term=2 tally=1 -2026-03-31T06:16:25.034Z [INFO] server-ebbba981-e6f3-1e13-993f-c1cebd3fe2f9: entering leader state: leader="Node at ebbba981-e6f3-1e13-993f-c1cebd3fe2f9 [Leader]" -2026-03-31T06:16:25.140Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:25.03433718 +0000 UTC m=+13.311944561 (1 nodes), 52.5816ms from start of poll, 52.7604ms from cluster start. Timeout at 2026-03-31 06:16:25.1393385 +0000 UTC m=+13.416945921, 105.00136ms after stability -2026-03-31T06:16:25.143Z [INFO] server-ebbba981-e6f3-1e13-993f-c1cebd3fe2f9: starting snapshot up to: index=102 -2026-03-31T06:16:25.143Z [INFO] creating new snapshot: path=/tmp/raft3753111462/snapshots/2-102-1774937785143.tmp -2026-03-31T06:16:25.143Z [INFO] server-ebbba981-e6f3-1e13-993f-c1cebd3fe2f9: compacting logs: from=1 to=92 -2026-03-31T06:16:25.143Z [INFO] server-ebbba981-e6f3-1e13-993f-c1cebd3fe2f9: snapshot complete up to: index=102 ---- PASS: TestRaft_SnapshotRestore_Progress (0.16s) +2025-02-25T23:57:46.475Z [INFO] server-c108e57e-0a17-d70f-cf07-76a877d6008e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c108e57e-0a17-d70f-cf07-76a877d6008e Address:c108e57e-0a17-d70f-cf07-76a877d6008e}]" +2025-02-25T23:57:46.475Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:46.475Z [INFO] server-c108e57e-0a17-d70f-cf07-76a877d6008e: entering follower state: follower="Node at c108e57e-0a17-d70f-cf07-76a877d6008e [Follower]" leader-address= leader-id= +2025-02-25T23:57:46.557Z [WARN] server-c108e57e-0a17-d70f-cf07-76a877d6008e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:46.557Z [INFO] server-c108e57e-0a17-d70f-cf07-76a877d6008e: entering candidate state: node="Node at c108e57e-0a17-d70f-cf07-76a877d6008e [Candidate]" term=2 +2025-02-25T23:57:46.557Z [INFO] server-c108e57e-0a17-d70f-cf07-76a877d6008e: election won: term=2 tally=1 +2025-02-25T23:57:46.557Z [INFO] server-c108e57e-0a17-d70f-cf07-76a877d6008e: entering leader state: leader="Node at c108e57e-0a17-d70f-cf07-76a877d6008e [Leader]" +2025-02-25T23:57:46.665Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:46.558035626 +0000 UTC m=+14.167051033 (1 nodes), 82.320244ms from start of poll, 82.389964ms from cluster start. Timeout at 2025-02-25 23:57:46.663040226 +0000 UTC m=+14.272055633, 105.0046ms after stability +2025-02-25T23:57:46.666Z [INFO] server-c108e57e-0a17-d70f-cf07-76a877d6008e: starting snapshot up to: index=102 +2025-02-25T23:57:46.666Z [INFO] creating new snapshot: path=/tmp/raft2258873942/snapshots/2-102-1740527866666.tmp +2025-02-25T23:57:46.667Z [INFO] server-c108e57e-0a17-d70f-cf07-76a877d6008e: compacting logs: from=1 to=92 +2025-02-25T23:57:46.667Z [INFO] server-c108e57e-0a17-d70f-cf07-76a877d6008e: snapshot complete up to: index=102 +--- PASS: TestRaft_SnapshotRestore_Progress (0.19s) === RUN TestRaft_NoRestoreOnStart -2026-03-31T06:16:25.144Z [INFO] server-f0491838-8406-a74f-5b62-2309b8312d19: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f0491838-8406-a74f-5b62-2309b8312d19 Address:f0491838-8406-a74f-5b62-2309b8312d19}]" -2026-03-31T06:16:25.144Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:25.145Z [INFO] server-f0491838-8406-a74f-5b62-2309b8312d19: entering follower state: follower="Node at f0491838-8406-a74f-5b62-2309b8312d19 [Follower]" leader-address= leader-id= -2026-03-31T06:16:25.213Z [WARN] server-f0491838-8406-a74f-5b62-2309b8312d19: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:25.213Z [INFO] server-f0491838-8406-a74f-5b62-2309b8312d19: entering candidate state: node="Node at f0491838-8406-a74f-5b62-2309b8312d19 [Candidate]" term=2 -2026-03-31T06:16:25.213Z [INFO] server-f0491838-8406-a74f-5b62-2309b8312d19: election won: term=2 tally=1 -2026-03-31T06:16:25.213Z [INFO] server-f0491838-8406-a74f-5b62-2309b8312d19: entering leader state: leader="Node at f0491838-8406-a74f-5b62-2309b8312d19 [Leader]" -2026-03-31T06:16:25.318Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:25.21321686 +0000 UTC m=+13.490824241 (1 nodes), 68.20564ms from start of poll, 68.38264ms from cluster start. Timeout at 2026-03-31 06:16:25.31821918 +0000 UTC m=+13.595826561, 105.00232ms after stability -2026-03-31T06:16:25.321Z [INFO] server-f0491838-8406-a74f-5b62-2309b8312d19: starting snapshot up to: index=102 -2026-03-31T06:16:25.321Z [INFO] creating new snapshot: path=/tmp/raft3388937895/snapshots/2-102-1774937785321.tmp -2026-03-31T06:16:25.321Z [INFO] server-f0491838-8406-a74f-5b62-2309b8312d19: compacting logs: from=1 to=92 -2026-03-31T06:16:25.322Z [INFO] server-f0491838-8406-a74f-5b62-2309b8312d19: snapshot complete up to: index=102 -2026-03-31T06:16:25.322Z [INFO] server-f0491838-8406-a74f-5b62-2309b8312d19: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f0491838-8406-a74f-5b62-2309b8312d19 Address:f0491838-8406-a74f-5b62-2309b8312d19}]" ---- PASS: TestRaft_NoRestoreOnStart (0.18s) +2025-02-25T23:57:46.667Z [INFO] server-8cbf7f2e-a43d-00d2-7375-a3d52c27cf32: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8cbf7f2e-a43d-00d2-7375-a3d52c27cf32 Address:8cbf7f2e-a43d-00d2-7375-a3d52c27cf32}]" +2025-02-25T23:57:46.667Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:46.667Z [INFO] server-8cbf7f2e-a43d-00d2-7375-a3d52c27cf32: entering follower state: follower="Node at 8cbf7f2e-a43d-00d2-7375-a3d52c27cf32 [Follower]" leader-address= leader-id= +2025-02-25T23:57:46.729Z [WARN] server-8cbf7f2e-a43d-00d2-7375-a3d52c27cf32: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:46.729Z [INFO] server-8cbf7f2e-a43d-00d2-7375-a3d52c27cf32: entering candidate state: node="Node at 8cbf7f2e-a43d-00d2-7375-a3d52c27cf32 [Candidate]" term=2 +2025-02-25T23:57:46.729Z [INFO] server-8cbf7f2e-a43d-00d2-7375-a3d52c27cf32: election won: term=2 tally=1 +2025-02-25T23:57:46.729Z [INFO] server-8cbf7f2e-a43d-00d2-7375-a3d52c27cf32: entering leader state: leader="Node at 8cbf7f2e-a43d-00d2-7375-a3d52c27cf32 [Leader]" +2025-02-25T23:57:46.837Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:46.730017882 +0000 UTC m=+14.339033289 (1 nodes), 62.244117ms from start of poll, 62.312117ms from cluster start. Timeout at 2025-02-25 23:57:46.835021042 +0000 UTC m=+14.444036489, 105.0032ms after stability +2025-02-25T23:57:46.838Z [INFO] server-8cbf7f2e-a43d-00d2-7375-a3d52c27cf32: starting snapshot up to: index=102 +2025-02-25T23:57:46.838Z [INFO] creating new snapshot: path=/tmp/raft3704095598/snapshots/2-102-1740527866838.tmp +2025-02-25T23:57:46.839Z [INFO] server-8cbf7f2e-a43d-00d2-7375-a3d52c27cf32: compacting logs: from=1 to=92 +2025-02-25T23:57:46.839Z [INFO] server-8cbf7f2e-a43d-00d2-7375-a3d52c27cf32: snapshot complete up to: index=102 +2025-02-25T23:57:46.839Z [INFO] server-8cbf7f2e-a43d-00d2-7375-a3d52c27cf32: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8cbf7f2e-a43d-00d2-7375-a3d52c27cf32 Address:8cbf7f2e-a43d-00d2-7375-a3d52c27cf32}]" +--- PASS: TestRaft_NoRestoreOnStart (0.17s) === RUN TestRaft_SnapshotRestore_PeerChange -2026-03-31T06:16:25.322Z [INFO] server-f0491838-8406-a74f-5b62-2309b8312d19: entering follower state: follower="Node at f0491838-8406-a74f-5b62-2309b8312d19 [Follower]" leader-address= leader-id= raft_test.go:1266: Skipping unreliable test --- SKIP: TestRaft_SnapshotRestore_PeerChange (0.00s) === RUN TestRaft_AutoSnapshot -2026-03-31T06:16:25.322Z [INFO] server-367ee835-7411-704f-23d2-e5c6dcc02ab4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-367ee835-7411-704f-23d2-e5c6dcc02ab4 Address:367ee835-7411-704f-23d2-e5c6dcc02ab4}]" -2026-03-31T06:16:25.322Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:25.322Z [INFO] server-367ee835-7411-704f-23d2-e5c6dcc02ab4: entering follower state: follower="Node at 367ee835-7411-704f-23d2-e5c6dcc02ab4 [Follower]" leader-address= leader-id= -2026-03-31T06:16:25.390Z [WARN] server-f0491838-8406-a74f-5b62-2309b8312d19: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:25.390Z [INFO] server-f0491838-8406-a74f-5b62-2309b8312d19: entering candidate state: node="Node at f0491838-8406-a74f-5b62-2309b8312d19 [Candidate]" term=3 -2026-03-31T06:16:25.390Z [INFO] server-f0491838-8406-a74f-5b62-2309b8312d19: election won: term=3 tally=1 -2026-03-31T06:16:25.390Z [INFO] server-f0491838-8406-a74f-5b62-2309b8312d19: entering leader state: leader="Node at f0491838-8406-a74f-5b62-2309b8312d19 [Leader]" -2026-03-31T06:16:25.414Z [WARN] server-367ee835-7411-704f-23d2-e5c6dcc02ab4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:25.414Z [INFO] server-367ee835-7411-704f-23d2-e5c6dcc02ab4: entering candidate state: node="Node at 367ee835-7411-704f-23d2-e5c6dcc02ab4 [Candidate]" term=2 -2026-03-31T06:16:25.414Z [INFO] server-367ee835-7411-704f-23d2-e5c6dcc02ab4: election won: term=2 tally=1 -2026-03-31T06:16:25.414Z [INFO] server-367ee835-7411-704f-23d2-e5c6dcc02ab4: entering leader state: leader="Node at 367ee835-7411-704f-23d2-e5c6dcc02ab4 [Leader]" -2026-03-31T06:16:25.520Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:25.41473554 +0000 UTC m=+13.692342921 (1 nodes), 92.07676ms from start of poll, 92.24068ms from cluster start. Timeout at 2026-03-31 06:16:25.51973674 +0000 UTC m=+13.797344121, 105.0012ms after stability -2026-03-31T06:16:25.524Z [INFO] server-367ee835-7411-704f-23d2-e5c6dcc02ab4: starting snapshot up to: index=92 -2026-03-31T06:16:25.524Z [INFO] creating new snapshot: path=/tmp/raft1774242072/snapshots/2-92-1774937785524.tmp -2026-03-31T06:16:25.524Z [INFO] server-367ee835-7411-704f-23d2-e5c6dcc02ab4: compacting logs: from=1 to=92 -2026-03-31T06:16:25.524Z [INFO] server-367ee835-7411-704f-23d2-e5c6dcc02ab4: snapshot complete up to: index=92 ---- PASS: TestRaft_AutoSnapshot (0.31s) +2025-02-25T23:57:46.839Z [INFO] server-a45d765d-5d79-fa86-035b-f97572bfef07: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a45d765d-5d79-fa86-035b-f97572bfef07 Address:a45d765d-5d79-fa86-035b-f97572bfef07}]" +2025-02-25T23:57:46.839Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:46.839Z [INFO] server-8cbf7f2e-a43d-00d2-7375-a3d52c27cf32: entering follower state: follower="Node at 8cbf7f2e-a43d-00d2-7375-a3d52c27cf32 [Follower]" leader-address= leader-id= +2025-02-25T23:57:46.839Z [INFO] server-a45d765d-5d79-fa86-035b-f97572bfef07: entering follower state: follower="Node at a45d765d-5d79-fa86-035b-f97572bfef07 [Follower]" leader-address= leader-id= +2025-02-25T23:57:46.901Z [WARN] server-8cbf7f2e-a43d-00d2-7375-a3d52c27cf32: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:46.901Z [INFO] server-8cbf7f2e-a43d-00d2-7375-a3d52c27cf32: entering candidate state: node="Node at 8cbf7f2e-a43d-00d2-7375-a3d52c27cf32 [Candidate]" term=3 +2025-02-25T23:57:46.901Z [INFO] server-8cbf7f2e-a43d-00d2-7375-a3d52c27cf32: election won: term=3 tally=1 +2025-02-25T23:57:46.901Z [INFO] server-8cbf7f2e-a43d-00d2-7375-a3d52c27cf32: entering leader state: leader="Node at 8cbf7f2e-a43d-00d2-7375-a3d52c27cf32 [Leader]" +2025-02-25T23:57:46.917Z [WARN] server-a45d765d-5d79-fa86-035b-f97572bfef07: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:46.917Z [INFO] server-a45d765d-5d79-fa86-035b-f97572bfef07: entering candidate state: node="Node at a45d765d-5d79-fa86-035b-f97572bfef07 [Candidate]" term=2 +2025-02-25T23:57:46.917Z [INFO] server-a45d765d-5d79-fa86-035b-f97572bfef07: election won: term=2 tally=1 +2025-02-25T23:57:46.917Z [INFO] server-a45d765d-5d79-fa86-035b-f97572bfef07: entering leader state: leader="Node at a45d765d-5d79-fa86-035b-f97572bfef07 [Leader]" +2025-02-25T23:57:47.025Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:46.917984359 +0000 UTC m=+14.526999726 (1 nodes), 78.48994ms from start of poll, 78.55794ms from cluster start. Timeout at 2025-02-25 23:57:47.022989279 +0000 UTC m=+14.632004686, 105.00496ms after stability +2025-02-25T23:57:47.037Z [INFO] server-a45d765d-5d79-fa86-035b-f97572bfef07: starting snapshot up to: index=102 +2025-02-25T23:57:47.037Z [INFO] creating new snapshot: path=/tmp/raft1853313342/snapshots/2-102-1740527867037.tmp +2025-02-25T23:57:47.038Z [INFO] server-a45d765d-5d79-fa86-035b-f97572bfef07: compacting logs: from=1 to=92 +2025-02-25T23:57:47.038Z [INFO] server-a45d765d-5d79-fa86-035b-f97572bfef07: snapshot complete up to: index=102 +--- PASS: TestRaft_AutoSnapshot (0.29s) === RUN TestRaft_UserSnapshot -2026-03-31T06:16:25.631Z [INFO] server-1c0a7c0d-5f14-71a9-1b59-06b1b6572756: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1c0a7c0d-5f14-71a9-1b59-06b1b6572756 Address:1c0a7c0d-5f14-71a9-1b59-06b1b6572756}]" -2026-03-31T06:16:25.631Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:25.631Z [INFO] server-1c0a7c0d-5f14-71a9-1b59-06b1b6572756: entering follower state: follower="Node at 1c0a7c0d-5f14-71a9-1b59-06b1b6572756 [Follower]" leader-address= leader-id= -2026-03-31T06:16:25.724Z [WARN] server-1c0a7c0d-5f14-71a9-1b59-06b1b6572756: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:25.724Z [INFO] server-1c0a7c0d-5f14-71a9-1b59-06b1b6572756: entering candidate state: node="Node at 1c0a7c0d-5f14-71a9-1b59-06b1b6572756 [Candidate]" term=2 -2026-03-31T06:16:25.724Z [INFO] server-1c0a7c0d-5f14-71a9-1b59-06b1b6572756: election won: term=2 tally=1 -2026-03-31T06:16:25.724Z [INFO] server-1c0a7c0d-5f14-71a9-1b59-06b1b6572756: entering leader state: leader="Node at 1c0a7c0d-5f14-71a9-1b59-06b1b6572756 [Leader]" -2026-03-31T06:16:25.831Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:25.7250139 +0000 UTC m=+14.002621281 (1 nodes), 93.61656ms from start of poll, 93.80816ms from cluster start. Timeout at 2026-03-31 06:16:25.8300161 +0000 UTC m=+14.107623481, 105.0022ms after stability -2026-03-31T06:16:25.831Z [ERROR] server-1c0a7c0d-5f14-71a9-1b59-06b1b6572756: failed to take snapshot: error="nothing new to snapshot" -2026-03-31T06:16:25.831Z [INFO] server-1c0a7c0d-5f14-71a9-1b59-06b1b6572756: starting snapshot up to: index=12 -2026-03-31T06:16:25.831Z [INFO] creating new snapshot: path=/tmp/raft2787383667/snapshots/2-12-1774937785831.tmp -2026-03-31T06:16:25.831Z [INFO] server-1c0a7c0d-5f14-71a9-1b59-06b1b6572756: compacting logs: from=1 to=2 -2026-03-31T06:16:25.832Z [INFO] server-1c0a7c0d-5f14-71a9-1b59-06b1b6572756: snapshot complete up to: index=12 +2025-02-25T23:57:47.134Z [INFO] server-fd38a7ef-d06c-cb4e-0339-52520f296dc1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fd38a7ef-d06c-cb4e-0339-52520f296dc1 Address:fd38a7ef-d06c-cb4e-0339-52520f296dc1}]" +2025-02-25T23:57:47.134Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:47.134Z [INFO] server-fd38a7ef-d06c-cb4e-0339-52520f296dc1: entering follower state: follower="Node at fd38a7ef-d06c-cb4e-0339-52520f296dc1 [Follower]" leader-address= leader-id= +2025-02-25T23:57:47.225Z [WARN] server-fd38a7ef-d06c-cb4e-0339-52520f296dc1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:47.225Z [INFO] server-fd38a7ef-d06c-cb4e-0339-52520f296dc1: entering candidate state: node="Node at fd38a7ef-d06c-cb4e-0339-52520f296dc1 [Candidate]" term=2 +2025-02-25T23:57:47.225Z [INFO] server-fd38a7ef-d06c-cb4e-0339-52520f296dc1: election won: term=2 tally=1 +2025-02-25T23:57:47.225Z [INFO] server-fd38a7ef-d06c-cb4e-0339-52520f296dc1: entering leader state: leader="Node at fd38a7ef-d06c-cb4e-0339-52520f296dc1 [Leader]" +2025-02-25T23:57:47.333Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:47.226023441 +0000 UTC m=+14.835038848 (1 nodes), 91.656224ms from start of poll, 91.753625ms from cluster start. Timeout at 2025-02-25 23:57:47.331024361 +0000 UTC m=+14.940039768, 105.00092ms after stability +2025-02-25T23:57:47.333Z [ERROR] server-fd38a7ef-d06c-cb4e-0339-52520f296dc1: failed to take snapshot: error="nothing new to snapshot" +2025-02-25T23:57:47.334Z [INFO] server-fd38a7ef-d06c-cb4e-0339-52520f296dc1: starting snapshot up to: index=12 +2025-02-25T23:57:47.334Z [INFO] creating new snapshot: path=/tmp/raft1078857429/snapshots/2-12-1740527867334.tmp +2025-02-25T23:57:47.334Z [INFO] server-fd38a7ef-d06c-cb4e-0339-52520f296dc1: compacting logs: from=1 to=2 +2025-02-25T23:57:47.334Z [INFO] server-fd38a7ef-d06c-cb4e-0339-52520f296dc1: snapshot complete up to: index=12 --- PASS: TestRaft_UserSnapshot (0.20s) === RUN TestRaft_UserRestore === RUN TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_false -2026-03-31T06:16:25.832Z [INFO] server-38472f22-a93f-becd-8296-5595571dca61: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-38472f22-a93f-becd-8296-5595571dca61 Address:38472f22-a93f-becd-8296-5595571dca61} {Suffrage:Voter ID:server-6ffcb4fd-b7d3-8fa2-db69-0a913145ac11 Address:6ffcb4fd-b7d3-8fa2-db69-0a913145ac11} {Suffrage:Voter ID:server-134d61c4-dd66-ff95-9cb7-36b477acc453 Address:134d61c4-dd66-ff95-9cb7-36b477acc453}]" -2026-03-31T06:16:25.832Z [INFO] server-6ffcb4fd-b7d3-8fa2-db69-0a913145ac11: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-38472f22-a93f-becd-8296-5595571dca61 Address:38472f22-a93f-becd-8296-5595571dca61} {Suffrage:Voter ID:server-6ffcb4fd-b7d3-8fa2-db69-0a913145ac11 Address:6ffcb4fd-b7d3-8fa2-db69-0a913145ac11} {Suffrage:Voter ID:server-134d61c4-dd66-ff95-9cb7-36b477acc453 Address:134d61c4-dd66-ff95-9cb7-36b477acc453}]" -2026-03-31T06:16:25.833Z [INFO] server-6ffcb4fd-b7d3-8fa2-db69-0a913145ac11: entering follower state: follower="Node at 6ffcb4fd-b7d3-8fa2-db69-0a913145ac11 [Follower]" leader-address= leader-id= -2026-03-31T06:16:25.832Z [INFO] server-38472f22-a93f-becd-8296-5595571dca61: entering follower state: follower="Node at 38472f22-a93f-becd-8296-5595571dca61 [Follower]" leader-address= leader-id= -2026-03-31T06:16:25.833Z [INFO] server-134d61c4-dd66-ff95-9cb7-36b477acc453: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-38472f22-a93f-becd-8296-5595571dca61 Address:38472f22-a93f-becd-8296-5595571dca61} {Suffrage:Voter ID:server-6ffcb4fd-b7d3-8fa2-db69-0a913145ac11 Address:6ffcb4fd-b7d3-8fa2-db69-0a913145ac11} {Suffrage:Voter ID:server-134d61c4-dd66-ff95-9cb7-36b477acc453 Address:134d61c4-dd66-ff95-9cb7-36b477acc453}]" -2026-03-31T06:16:25.833Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:25.833Z [INFO] server-134d61c4-dd66-ff95-9cb7-36b477acc453: entering follower state: follower="Node at 134d61c4-dd66-ff95-9cb7-36b477acc453 [Follower]" leader-address= leader-id= -2026-03-31T06:16:26.348Z [WARN] server-6ffcb4fd-b7d3-8fa2-db69-0a913145ac11: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:26.348Z [INFO] server-6ffcb4fd-b7d3-8fa2-db69-0a913145ac11: entering candidate state: node="Node at 6ffcb4fd-b7d3-8fa2-db69-0a913145ac11 [Candidate]" term=2 -2026-03-31T06:16:26.348Z [INFO] server-6ffcb4fd-b7d3-8fa2-db69-0a913145ac11: election won: term=2 tally=2 -2026-03-31T06:16:26.348Z [INFO] server-6ffcb4fd-b7d3-8fa2-db69-0a913145ac11: entering leader state: leader="Node at 6ffcb4fd-b7d3-8fa2-db69-0a913145ac11 [Leader]" -2026-03-31T06:16:26.348Z [INFO] server-6ffcb4fd-b7d3-8fa2-db69-0a913145ac11: added peer, starting replication: peer=server-38472f22-a93f-becd-8296-5595571dca61 -2026-03-31T06:16:26.348Z [INFO] server-6ffcb4fd-b7d3-8fa2-db69-0a913145ac11: added peer, starting replication: peer=server-134d61c4-dd66-ff95-9cb7-36b477acc453 -2026-03-31T06:16:26.348Z [INFO] server-6ffcb4fd-b7d3-8fa2-db69-0a913145ac11: pipelining replication: peer="{Voter server-134d61c4-dd66-ff95-9cb7-36b477acc453 134d61c4-dd66-ff95-9cb7-36b477acc453}" -2026-03-31T06:16:26.348Z [INFO] server-6ffcb4fd-b7d3-8fa2-db69-0a913145ac11: pipelining replication: peer="{Voter server-38472f22-a93f-becd-8296-5595571dca61 38472f22-a93f-becd-8296-5595571dca61}" -2026-03-31T06:16:27.354Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:26.34824774 +0000 UTC m=+14.625855121 (1 nodes), 515.11136ms from start of poll, 515.56212ms from cluster start. Timeout at 2026-03-31 06:16:27.35325126 +0000 UTC m=+15.630858641, 1.00500352s after stability -2026-03-31T06:16:27.354Z [INFO] server-6ffcb4fd-b7d3-8fa2-db69-0a913145ac11: starting snapshot up to: index=12 -2026-03-31T06:16:27.354Z [INFO] creating new snapshot: path=/tmp/raft3355499100/snapshots/2-12-1774937787354.tmp -2026-03-31T06:16:27.355Z [INFO] server-6ffcb4fd-b7d3-8fa2-db69-0a913145ac11: snapshot complete up to: index=12 -2026-03-31T06:16:27.356Z [INFO] creating new snapshot: path=/tmp/raft3355499100/snapshots/2-23-1774937787356.tmp -2026-03-31T06:16:27.356Z [INFO] server-6ffcb4fd-b7d3-8fa2-db69-0a913145ac11: copied to local snapshot: bytes=71 -2026-03-31T06:16:27.356Z [INFO] server-6ffcb4fd-b7d3-8fa2-db69-0a913145ac11: snapshot restore progress: id=2-23-1774937787356 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:27.357Z [INFO] server-6ffcb4fd-b7d3-8fa2-db69-0a913145ac11: restored user snapshot: index=23 -2026-03-31T06:16:27.357Z [ERROR] server-6ffcb4fd-b7d3-8fa2-db69-0a913145ac11: failed to get log: index=23 error="log not found" -2026-03-31T06:16:27.357Z [INFO] server-6ffcb4fd-b7d3-8fa2-db69-0a913145ac11: aborting pipeline replication: peer="{Voter server-134d61c4-dd66-ff95-9cb7-36b477acc453 134d61c4-dd66-ff95-9cb7-36b477acc453}" -2026-03-31T06:16:27.357Z [ERROR] server-6ffcb4fd-b7d3-8fa2-db69-0a913145ac11: failed to get log: index=23 error="log not found" -2026-03-31T06:16:27.357Z [INFO] server-6ffcb4fd-b7d3-8fa2-db69-0a913145ac11: aborting pipeline replication: peer="{Voter server-38472f22-a93f-becd-8296-5595571dca61 38472f22-a93f-becd-8296-5595571dca61}" -2026-03-31T06:16:27.364Z [ERROR] server-6ffcb4fd-b7d3-8fa2-db69-0a913145ac11: failed to get log: index=23 error="log not found" -2026-03-31T06:16:27.364Z [INFO] creating new snapshot: path=/tmp/raft3383489412/snapshots/2-23-1774937787364.tmp -2026-03-31T06:16:27.364Z [INFO] server-134d61c4-dd66-ff95-9cb7-36b477acc453: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:27.365Z [INFO] server-134d61c4-dd66-ff95-9cb7-36b477acc453: copied to local snapshot: bytes=71 -2026-03-31T06:16:27.365Z [INFO] server-134d61c4-dd66-ff95-9cb7-36b477acc453: snapshot restore progress: id=2-23-1774937787364 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:27.365Z [INFO] server-134d61c4-dd66-ff95-9cb7-36b477acc453: Installed remote snapshot -2026-03-31T06:16:27.366Z [WARN] server-134d61c4-dd66-ff95-9cb7-36b477acc453: skipping application of old log: index=22 -2026-03-31T06:16:27.366Z [ERROR] server-6ffcb4fd-b7d3-8fa2-db69-0a913145ac11: failed to get log: index=23 error="log not found" -2026-03-31T06:16:27.366Z [INFO] server-6ffcb4fd-b7d3-8fa2-db69-0a913145ac11: pipelining replication: peer="{Voter server-134d61c4-dd66-ff95-9cb7-36b477acc453 134d61c4-dd66-ff95-9cb7-36b477acc453}" -2026-03-31T06:16:27.366Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:16:27.367Z [INFO] creating new snapshot: path=/tmp/raft1961861786/snapshots/2-23-1774937787367.tmp -2026-03-31T06:16:27.367Z [INFO] server-38472f22-a93f-becd-8296-5595571dca61: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:27.367Z [INFO] server-38472f22-a93f-becd-8296-5595571dca61: copied to local snapshot: bytes=71 -2026-03-31T06:16:27.368Z [INFO] server-38472f22-a93f-becd-8296-5595571dca61: snapshot restore progress: id=2-23-1774937787367 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:27.368Z [INFO] server-38472f22-a93f-becd-8296-5595571dca61: Installed remote snapshot -2026-03-31T06:16:27.368Z [INFO] server-6ffcb4fd-b7d3-8fa2-db69-0a913145ac11: pipelining replication: peer="{Voter server-38472f22-a93f-becd-8296-5595571dca61 38472f22-a93f-becd-8296-5595571dca61}" -2026-03-31T06:16:28.371Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:16:27.36664662 +0000 UTC m=+15.644254041 (2 nodes), 13.96µs from start of poll, 1.53396104s from cluster start. Timeout at 2026-03-31 06:16:28.37165286 +0000 UTC m=+16.649260201, 1.00500616s after stability -2026-03-31T06:16:28.383Z [INFO] server-6ffcb4fd-b7d3-8fa2-db69-0a913145ac11: aborting pipeline replication: peer="{Voter server-134d61c4-dd66-ff95-9cb7-36b477acc453 134d61c4-dd66-ff95-9cb7-36b477acc453}" -2026-03-31T06:16:28.383Z [INFO] server-6ffcb4fd-b7d3-8fa2-db69-0a913145ac11: aborting pipeline replication: peer="{Voter server-38472f22-a93f-becd-8296-5595571dca61 38472f22-a93f-becd-8296-5595571dca61}" +2025-02-25T23:57:47.334Z [INFO] server-48002dd6-f0fa-37b3-6517-f58cb7988614: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-48002dd6-f0fa-37b3-6517-f58cb7988614 Address:48002dd6-f0fa-37b3-6517-f58cb7988614} {Suffrage:Voter ID:server-e6d7a580-0805-f9da-eb2b-f2da06c13bec Address:e6d7a580-0805-f9da-eb2b-f2da06c13bec} {Suffrage:Voter ID:server-031005e0-129a-f813-4096-39023414ad64 Address:031005e0-129a-f813-4096-39023414ad64}]" +2025-02-25T23:57:47.334Z [INFO] server-e6d7a580-0805-f9da-eb2b-f2da06c13bec: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-48002dd6-f0fa-37b3-6517-f58cb7988614 Address:48002dd6-f0fa-37b3-6517-f58cb7988614} {Suffrage:Voter ID:server-e6d7a580-0805-f9da-eb2b-f2da06c13bec Address:e6d7a580-0805-f9da-eb2b-f2da06c13bec} {Suffrage:Voter ID:server-031005e0-129a-f813-4096-39023414ad64 Address:031005e0-129a-f813-4096-39023414ad64}]" +2025-02-25T23:57:47.334Z [INFO] server-031005e0-129a-f813-4096-39023414ad64: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-48002dd6-f0fa-37b3-6517-f58cb7988614 Address:48002dd6-f0fa-37b3-6517-f58cb7988614} {Suffrage:Voter ID:server-e6d7a580-0805-f9da-eb2b-f2da06c13bec Address:e6d7a580-0805-f9da-eb2b-f2da06c13bec} {Suffrage:Voter ID:server-031005e0-129a-f813-4096-39023414ad64 Address:031005e0-129a-f813-4096-39023414ad64}]" +2025-02-25T23:57:47.334Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:47.334Z [INFO] server-48002dd6-f0fa-37b3-6517-f58cb7988614: entering follower state: follower="Node at 48002dd6-f0fa-37b3-6517-f58cb7988614 [Follower]" leader-address= leader-id= +2025-02-25T23:57:47.334Z [INFO] server-e6d7a580-0805-f9da-eb2b-f2da06c13bec: entering follower state: follower="Node at e6d7a580-0805-f9da-eb2b-f2da06c13bec [Follower]" leader-address= leader-id= +2025-02-25T23:57:47.334Z [INFO] server-031005e0-129a-f813-4096-39023414ad64: entering follower state: follower="Node at 031005e0-129a-f813-4096-39023414ad64 [Follower]" leader-address= leader-id= +2025-02-25T23:57:48.205Z [WARN] server-e6d7a580-0805-f9da-eb2b-f2da06c13bec: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:48.205Z [INFO] server-e6d7a580-0805-f9da-eb2b-f2da06c13bec: entering candidate state: node="Node at e6d7a580-0805-f9da-eb2b-f2da06c13bec [Candidate]" term=2 +2025-02-25T23:57:48.205Z [INFO] server-e6d7a580-0805-f9da-eb2b-f2da06c13bec: election won: term=2 tally=2 +2025-02-25T23:57:48.205Z [INFO] server-e6d7a580-0805-f9da-eb2b-f2da06c13bec: entering leader state: leader="Node at e6d7a580-0805-f9da-eb2b-f2da06c13bec [Leader]" +2025-02-25T23:57:48.206Z [INFO] server-e6d7a580-0805-f9da-eb2b-f2da06c13bec: added peer, starting replication: peer=server-48002dd6-f0fa-37b3-6517-f58cb7988614 +2025-02-25T23:57:48.206Z [INFO] server-e6d7a580-0805-f9da-eb2b-f2da06c13bec: added peer, starting replication: peer=server-031005e0-129a-f813-4096-39023414ad64 +2025-02-25T23:57:48.206Z [INFO] server-e6d7a580-0805-f9da-eb2b-f2da06c13bec: pipelining replication: peer="{Voter server-031005e0-129a-f813-4096-39023414ad64 031005e0-129a-f813-4096-39023414ad64}" +2025-02-25T23:57:48.206Z [INFO] server-e6d7a580-0805-f9da-eb2b-f2da06c13bec: pipelining replication: peer="{Voter server-48002dd6-f0fa-37b3-6517-f58cb7988614 48002dd6-f0fa-37b3-6517-f58cb7988614}" +2025-02-25T23:57:49.213Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:48.206224084 +0000 UTC m=+15.815239491 (1 nodes), 871.38363ms from start of poll, 871.552511ms from cluster start. Timeout at 2025-02-25 23:57:49.211227244 +0000 UTC m=+16.820242691, 1.0050032s after stability +2025-02-25T23:57:49.214Z [INFO] server-e6d7a580-0805-f9da-eb2b-f2da06c13bec: starting snapshot up to: index=12 +2025-02-25T23:57:49.214Z [INFO] creating new snapshot: path=/tmp/raft1822487351/snapshots/2-12-1740527869214.tmp +2025-02-25T23:57:49.214Z [INFO] server-e6d7a580-0805-f9da-eb2b-f2da06c13bec: snapshot complete up to: index=12 +2025-02-25T23:57:49.215Z [INFO] creating new snapshot: path=/tmp/raft1822487351/snapshots/2-23-1740527869215.tmp +2025-02-25T23:57:49.215Z [INFO] server-e6d7a580-0805-f9da-eb2b-f2da06c13bec: copied to local snapshot: bytes=71 +2025-02-25T23:57:49.215Z [INFO] server-e6d7a580-0805-f9da-eb2b-f2da06c13bec: snapshot restore progress: id=2-23-1740527869215 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:57:49.215Z [INFO] server-e6d7a580-0805-f9da-eb2b-f2da06c13bec: restored user snapshot: index=23 +2025-02-25T23:57:49.215Z [ERROR] server-e6d7a580-0805-f9da-eb2b-f2da06c13bec: failed to get log: index=23 error="log not found" +2025-02-25T23:57:49.215Z [INFO] server-e6d7a580-0805-f9da-eb2b-f2da06c13bec: aborting pipeline replication: peer="{Voter server-48002dd6-f0fa-37b3-6517-f58cb7988614 48002dd6-f0fa-37b3-6517-f58cb7988614}" +2025-02-25T23:57:49.216Z [ERROR] server-e6d7a580-0805-f9da-eb2b-f2da06c13bec: failed to get log: index=23 error="log not found" +2025-02-25T23:57:49.216Z [INFO] server-e6d7a580-0805-f9da-eb2b-f2da06c13bec: aborting pipeline replication: peer="{Voter server-031005e0-129a-f813-4096-39023414ad64 031005e0-129a-f813-4096-39023414ad64}" +2025-02-25T23:57:49.225Z [ERROR] server-e6d7a580-0805-f9da-eb2b-f2da06c13bec: failed to get log: index=23 error="log not found" +2025-02-25T23:57:49.226Z [INFO] creating new snapshot: path=/tmp/raft3766925477/snapshots/2-23-1740527869226.tmp +2025-02-25T23:57:49.226Z [INFO] server-031005e0-129a-f813-4096-39023414ad64: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:57:49.226Z [INFO] server-031005e0-129a-f813-4096-39023414ad64: copied to local snapshot: bytes=71 +2025-02-25T23:57:49.226Z [INFO] server-031005e0-129a-f813-4096-39023414ad64: snapshot restore progress: id=2-23-1740527869226 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:57:49.226Z [INFO] server-031005e0-129a-f813-4096-39023414ad64: Installed remote snapshot +2025-02-25T23:57:49.226Z [WARN] server-031005e0-129a-f813-4096-39023414ad64: skipping application of old log: index=22 +2025-02-25T23:57:49.226Z [INFO] server-e6d7a580-0805-f9da-eb2b-f2da06c13bec: pipelining replication: peer="{Voter server-031005e0-129a-f813-4096-39023414ad64 031005e0-129a-f813-4096-39023414ad64}" +2025-02-25T23:57:49.226Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:57:49.229Z [ERROR] server-e6d7a580-0805-f9da-eb2b-f2da06c13bec: failed to get log: index=23 error="log not found" +2025-02-25T23:57:49.230Z [INFO] creating new snapshot: path=/tmp/raft245105348/snapshots/2-23-1740527869230.tmp +2025-02-25T23:57:49.230Z [INFO] server-48002dd6-f0fa-37b3-6517-f58cb7988614: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:57:49.230Z [INFO] server-48002dd6-f0fa-37b3-6517-f58cb7988614: copied to local snapshot: bytes=71 +2025-02-25T23:57:49.230Z [INFO] server-48002dd6-f0fa-37b3-6517-f58cb7988614: snapshot restore progress: id=2-23-1740527869230 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:57:49.230Z [INFO] server-48002dd6-f0fa-37b3-6517-f58cb7988614: Installed remote snapshot +2025-02-25T23:57:49.230Z [INFO] server-e6d7a580-0805-f9da-eb2b-f2da06c13bec: pipelining replication: peer="{Voter server-48002dd6-f0fa-37b3-6517-f58cb7988614 48002dd6-f0fa-37b3-6517-f58cb7988614}" +2025-02-25T23:57:50.233Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:57:49.226750183 +0000 UTC m=+16.835765590 (2 nodes), 520ns from start of poll, 1.89207861s from cluster start. Timeout at 2025-02-25 23:57:50.231755383 +0000 UTC m=+17.840770790, 1.0050052s after stability +2025-02-25T23:57:50.249Z [INFO] server-e6d7a580-0805-f9da-eb2b-f2da06c13bec: aborting pipeline replication: peer="{Voter server-031005e0-129a-f813-4096-39023414ad64 031005e0-129a-f813-4096-39023414ad64}" +2025-02-25T23:57:50.249Z [INFO] server-e6d7a580-0805-f9da-eb2b-f2da06c13bec: aborting pipeline replication: peer="{Voter server-48002dd6-f0fa-37b3-6517-f58cb7988614 48002dd6-f0fa-37b3-6517-f58cb7988614}" === RUN TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_false -2026-03-31T06:16:28.384Z [INFO] server-f01d8e74-4617-1aef-3fc2-9dc5ae844d12: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f01d8e74-4617-1aef-3fc2-9dc5ae844d12 Address:f01d8e74-4617-1aef-3fc2-9dc5ae844d12} {Suffrage:Voter ID:server-91a6f5ef-66b0-e68d-6cc8-2b0e5f3a1deb Address:91a6f5ef-66b0-e68d-6cc8-2b0e5f3a1deb} {Suffrage:Voter ID:server-740bd9a0-3fba-a645-be76-72b5c9b5d0b1 Address:740bd9a0-3fba-a645-be76-72b5c9b5d0b1}]" -2026-03-31T06:16:28.385Z [INFO] server-91a6f5ef-66b0-e68d-6cc8-2b0e5f3a1deb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f01d8e74-4617-1aef-3fc2-9dc5ae844d12 Address:f01d8e74-4617-1aef-3fc2-9dc5ae844d12} {Suffrage:Voter ID:server-91a6f5ef-66b0-e68d-6cc8-2b0e5f3a1deb Address:91a6f5ef-66b0-e68d-6cc8-2b0e5f3a1deb} {Suffrage:Voter ID:server-740bd9a0-3fba-a645-be76-72b5c9b5d0b1 Address:740bd9a0-3fba-a645-be76-72b5c9b5d0b1}]" -2026-03-31T06:16:28.385Z [INFO] server-f01d8e74-4617-1aef-3fc2-9dc5ae844d12: entering follower state: follower="Node at f01d8e74-4617-1aef-3fc2-9dc5ae844d12 [Follower]" leader-address= leader-id= -2026-03-31T06:16:28.385Z [INFO] server-91a6f5ef-66b0-e68d-6cc8-2b0e5f3a1deb: entering follower state: follower="Node at 91a6f5ef-66b0-e68d-6cc8-2b0e5f3a1deb [Follower]" leader-address= leader-id= -2026-03-31T06:16:28.385Z [INFO] server-740bd9a0-3fba-a645-be76-72b5c9b5d0b1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f01d8e74-4617-1aef-3fc2-9dc5ae844d12 Address:f01d8e74-4617-1aef-3fc2-9dc5ae844d12} {Suffrage:Voter ID:server-91a6f5ef-66b0-e68d-6cc8-2b0e5f3a1deb Address:91a6f5ef-66b0-e68d-6cc8-2b0e5f3a1deb} {Suffrage:Voter ID:server-740bd9a0-3fba-a645-be76-72b5c9b5d0b1 Address:740bd9a0-3fba-a645-be76-72b5c9b5d0b1}]" -2026-03-31T06:16:28.385Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:28.385Z [INFO] server-740bd9a0-3fba-a645-be76-72b5c9b5d0b1: entering follower state: follower="Node at 740bd9a0-3fba-a645-be76-72b5c9b5d0b1 [Follower]" leader-address= leader-id= -2026-03-31T06:16:28.905Z [WARN] server-f01d8e74-4617-1aef-3fc2-9dc5ae844d12: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:28.905Z [INFO] server-f01d8e74-4617-1aef-3fc2-9dc5ae844d12: entering candidate state: node="Node at f01d8e74-4617-1aef-3fc2-9dc5ae844d12 [Candidate]" term=2 -2026-03-31T06:16:28.905Z [INFO] server-f01d8e74-4617-1aef-3fc2-9dc5ae844d12: election won: term=2 tally=2 -2026-03-31T06:16:28.905Z [INFO] server-f01d8e74-4617-1aef-3fc2-9dc5ae844d12: entering leader state: leader="Node at f01d8e74-4617-1aef-3fc2-9dc5ae844d12 [Leader]" -2026-03-31T06:16:28.905Z [INFO] server-f01d8e74-4617-1aef-3fc2-9dc5ae844d12: added peer, starting replication: peer=server-91a6f5ef-66b0-e68d-6cc8-2b0e5f3a1deb -2026-03-31T06:16:28.905Z [INFO] server-f01d8e74-4617-1aef-3fc2-9dc5ae844d12: added peer, starting replication: peer=server-740bd9a0-3fba-a645-be76-72b5c9b5d0b1 -2026-03-31T06:16:28.906Z [INFO] server-f01d8e74-4617-1aef-3fc2-9dc5ae844d12: pipelining replication: peer="{Voter server-740bd9a0-3fba-a645-be76-72b5c9b5d0b1 740bd9a0-3fba-a645-be76-72b5c9b5d0b1}" -2026-03-31T06:16:28.906Z [INFO] server-f01d8e74-4617-1aef-3fc2-9dc5ae844d12: pipelining replication: peer="{Voter server-91a6f5ef-66b0-e68d-6cc8-2b0e5f3a1deb 91a6f5ef-66b0-e68d-6cc8-2b0e5f3a1deb}" -2026-03-31T06:16:29.910Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:28.90590426 +0000 UTC m=+17.183511641 (1 nodes), 520.65728ms from start of poll, 521.05852ms from cluster start. Timeout at 2026-03-31 06:16:29.91090626 +0000 UTC m=+18.188513561, 1.00500192s after stability -2026-03-31T06:16:29.911Z [INFO] server-f01d8e74-4617-1aef-3fc2-9dc5ae844d12: starting snapshot up to: index=12 -2026-03-31T06:16:29.911Z [INFO] creating new snapshot: path=/tmp/raft437249075/snapshots/2-12-1774937789911.tmp -2026-03-31T06:16:29.912Z [INFO] server-f01d8e74-4617-1aef-3fc2-9dc5ae844d12: snapshot complete up to: index=12 -2026-03-31T06:16:29.913Z [INFO] creating new snapshot: path=/tmp/raft437249075/snapshots/2-23-1774937789913.tmp -2026-03-31T06:16:29.914Z [INFO] server-f01d8e74-4617-1aef-3fc2-9dc5ae844d12: copied to local snapshot: bytes=71 -2026-03-31T06:16:29.915Z [INFO] server-f01d8e74-4617-1aef-3fc2-9dc5ae844d12: snapshot restore progress: id=2-23-1774937789913 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:29.915Z [INFO] server-f01d8e74-4617-1aef-3fc2-9dc5ae844d12: removing all old logs from log store -2026-03-31T06:16:29.915Z [INFO] server-f01d8e74-4617-1aef-3fc2-9dc5ae844d12: compacting logs: from=1 to=22 -2026-03-31T06:16:29.915Z [INFO] server-f01d8e74-4617-1aef-3fc2-9dc5ae844d12: restored user snapshot: index=23 -2026-03-31T06:16:29.915Z [ERROR] server-f01d8e74-4617-1aef-3fc2-9dc5ae844d12: failed to get log: index=22 error="log not found" -2026-03-31T06:16:29.915Z [ERROR] server-f01d8e74-4617-1aef-3fc2-9dc5ae844d12: failed to get log: index=22 error="log not found" -2026-03-31T06:16:29.915Z [INFO] server-f01d8e74-4617-1aef-3fc2-9dc5ae844d12: aborting pipeline replication: peer="{Voter server-91a6f5ef-66b0-e68d-6cc8-2b0e5f3a1deb 91a6f5ef-66b0-e68d-6cc8-2b0e5f3a1deb}" -2026-03-31T06:16:29.915Z [INFO] server-f01d8e74-4617-1aef-3fc2-9dc5ae844d12: aborting pipeline replication: peer="{Voter server-740bd9a0-3fba-a645-be76-72b5c9b5d0b1 740bd9a0-3fba-a645-be76-72b5c9b5d0b1}" -2026-03-31T06:16:29.924Z [ERROR] server-f01d8e74-4617-1aef-3fc2-9dc5ae844d12: failed to get log: index=22 error="log not found" -2026-03-31T06:16:29.924Z [INFO] creating new snapshot: path=/tmp/raft3353438048/snapshots/2-23-1774937789924.tmp -2026-03-31T06:16:29.924Z [INFO] server-740bd9a0-3fba-a645-be76-72b5c9b5d0b1: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:29.925Z [INFO] server-740bd9a0-3fba-a645-be76-72b5c9b5d0b1: copied to local snapshot: bytes=71 -2026-03-31T06:16:29.925Z [ERROR] server-f01d8e74-4617-1aef-3fc2-9dc5ae844d12: failed to get log: index=22 error="log not found" -2026-03-31T06:16:29.925Z [INFO] server-740bd9a0-3fba-a645-be76-72b5c9b5d0b1: snapshot restore progress: id=2-23-1774937789924 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:29.925Z [INFO] server-740bd9a0-3fba-a645-be76-72b5c9b5d0b1: removing all old logs from log store -2026-03-31T06:16:29.925Z [INFO] server-740bd9a0-3fba-a645-be76-72b5c9b5d0b1: compacting logs: from=1 to=22 -2026-03-31T06:16:29.925Z [INFO] server-740bd9a0-3fba-a645-be76-72b5c9b5d0b1: Installed remote snapshot -2026-03-31T06:16:29.925Z [WARN] server-740bd9a0-3fba-a645-be76-72b5c9b5d0b1: skipping application of old log: index=22 -2026-03-31T06:16:29.925Z [INFO] server-f01d8e74-4617-1aef-3fc2-9dc5ae844d12: pipelining replication: peer="{Voter server-740bd9a0-3fba-a645-be76-72b5c9b5d0b1 740bd9a0-3fba-a645-be76-72b5c9b5d0b1}" -2026-03-31T06:16:29.925Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:16:29.925Z [INFO] creating new snapshot: path=/tmp/raft2910877366/snapshots/2-23-1774937789925.tmp -2026-03-31T06:16:29.926Z [INFO] server-91a6f5ef-66b0-e68d-6cc8-2b0e5f3a1deb: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:29.926Z [INFO] server-91a6f5ef-66b0-e68d-6cc8-2b0e5f3a1deb: copied to local snapshot: bytes=71 -2026-03-31T06:16:29.926Z [INFO] server-91a6f5ef-66b0-e68d-6cc8-2b0e5f3a1deb: snapshot restore progress: id=2-23-1774937789925 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:29.926Z [INFO] server-91a6f5ef-66b0-e68d-6cc8-2b0e5f3a1deb: removing all old logs from log store -2026-03-31T06:16:29.926Z [INFO] server-91a6f5ef-66b0-e68d-6cc8-2b0e5f3a1deb: compacting logs: from=1 to=22 -2026-03-31T06:16:29.926Z [INFO] server-91a6f5ef-66b0-e68d-6cc8-2b0e5f3a1deb: Installed remote snapshot -2026-03-31T06:16:29.926Z [INFO] server-f01d8e74-4617-1aef-3fc2-9dc5ae844d12: pipelining replication: peer="{Voter server-91a6f5ef-66b0-e68d-6cc8-2b0e5f3a1deb 91a6f5ef-66b0-e68d-6cc8-2b0e5f3a1deb}" -2026-03-31T06:16:30.931Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:16:29.92570502 +0000 UTC m=+18.203312401 (2 nodes), 2.16µs from start of poll, 1.54085928s from cluster start. Timeout at 2026-03-31 06:16:30.93071042 +0000 UTC m=+19.208317801, 1.0050054s after stability -2026-03-31T06:16:30.943Z [INFO] server-f01d8e74-4617-1aef-3fc2-9dc5ae844d12: aborting pipeline replication: peer="{Voter server-740bd9a0-3fba-a645-be76-72b5c9b5d0b1 740bd9a0-3fba-a645-be76-72b5c9b5d0b1}" -2026-03-31T06:16:30.943Z [INFO] server-f01d8e74-4617-1aef-3fc2-9dc5ae844d12: aborting pipeline replication: peer="{Voter server-91a6f5ef-66b0-e68d-6cc8-2b0e5f3a1deb 91a6f5ef-66b0-e68d-6cc8-2b0e5f3a1deb}" +2025-02-25T23:57:50.250Z [INFO] server-a032edb3-d9f0-7cf8-91d7-5883f94efc93: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a032edb3-d9f0-7cf8-91d7-5883f94efc93 Address:a032edb3-d9f0-7cf8-91d7-5883f94efc93} {Suffrage:Voter ID:server-e9baa5ed-e2d8-281d-5837-4e6fd799d622 Address:e9baa5ed-e2d8-281d-5837-4e6fd799d622} {Suffrage:Voter ID:server-144a6270-e1f6-f786-5a6d-3e192f167de0 Address:144a6270-e1f6-f786-5a6d-3e192f167de0}]" +2025-02-25T23:57:50.250Z [INFO] server-e9baa5ed-e2d8-281d-5837-4e6fd799d622: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a032edb3-d9f0-7cf8-91d7-5883f94efc93 Address:a032edb3-d9f0-7cf8-91d7-5883f94efc93} {Suffrage:Voter ID:server-e9baa5ed-e2d8-281d-5837-4e6fd799d622 Address:e9baa5ed-e2d8-281d-5837-4e6fd799d622} {Suffrage:Voter ID:server-144a6270-e1f6-f786-5a6d-3e192f167de0 Address:144a6270-e1f6-f786-5a6d-3e192f167de0}]" +2025-02-25T23:57:50.251Z [INFO] server-144a6270-e1f6-f786-5a6d-3e192f167de0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a032edb3-d9f0-7cf8-91d7-5883f94efc93 Address:a032edb3-d9f0-7cf8-91d7-5883f94efc93} {Suffrage:Voter ID:server-e9baa5ed-e2d8-281d-5837-4e6fd799d622 Address:e9baa5ed-e2d8-281d-5837-4e6fd799d622} {Suffrage:Voter ID:server-144a6270-e1f6-f786-5a6d-3e192f167de0 Address:144a6270-e1f6-f786-5a6d-3e192f167de0}]" +2025-02-25T23:57:50.251Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:50.251Z [INFO] server-a032edb3-d9f0-7cf8-91d7-5883f94efc93: entering follower state: follower="Node at a032edb3-d9f0-7cf8-91d7-5883f94efc93 [Follower]" leader-address= leader-id= +2025-02-25T23:57:50.251Z [INFO] server-e9baa5ed-e2d8-281d-5837-4e6fd799d622: entering follower state: follower="Node at e9baa5ed-e2d8-281d-5837-4e6fd799d622 [Follower]" leader-address= leader-id= +2025-02-25T23:57:50.251Z [INFO] server-144a6270-e1f6-f786-5a6d-3e192f167de0: entering follower state: follower="Node at 144a6270-e1f6-f786-5a6d-3e192f167de0 [Follower]" leader-address= leader-id= +2025-02-25T23:57:50.889Z [WARN] server-144a6270-e1f6-f786-5a6d-3e192f167de0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:50.889Z [INFO] server-144a6270-e1f6-f786-5a6d-3e192f167de0: entering candidate state: node="Node at 144a6270-e1f6-f786-5a6d-3e192f167de0 [Candidate]" term=2 +2025-02-25T23:57:50.890Z [INFO] server-144a6270-e1f6-f786-5a6d-3e192f167de0: election won: term=2 tally=2 +2025-02-25T23:57:50.890Z [INFO] server-144a6270-e1f6-f786-5a6d-3e192f167de0: entering leader state: leader="Node at 144a6270-e1f6-f786-5a6d-3e192f167de0 [Leader]" +2025-02-25T23:57:50.890Z [INFO] server-144a6270-e1f6-f786-5a6d-3e192f167de0: added peer, starting replication: peer=server-a032edb3-d9f0-7cf8-91d7-5883f94efc93 +2025-02-25T23:57:50.890Z [INFO] server-144a6270-e1f6-f786-5a6d-3e192f167de0: added peer, starting replication: peer=server-e9baa5ed-e2d8-281d-5837-4e6fd799d622 +2025-02-25T23:57:50.890Z [INFO] server-144a6270-e1f6-f786-5a6d-3e192f167de0: pipelining replication: peer="{Voter server-e9baa5ed-e2d8-281d-5837-4e6fd799d622 e9baa5ed-e2d8-281d-5837-4e6fd799d622}" +2025-02-25T23:57:50.890Z [INFO] server-144a6270-e1f6-f786-5a6d-3e192f167de0: pipelining replication: peer="{Voter server-a032edb3-d9f0-7cf8-91d7-5883f94efc93 a032edb3-d9f0-7cf8-91d7-5883f94efc93}" +2025-02-25T23:57:51.897Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:50.890236098 +0000 UTC m=+18.499251505 (1 nodes), 639.187991ms from start of poll, 639.827755ms from cluster start. Timeout at 2025-02-25 23:57:51.895240938 +0000 UTC m=+19.504256345, 1.00500484s after stability +2025-02-25T23:57:51.898Z [INFO] server-144a6270-e1f6-f786-5a6d-3e192f167de0: starting snapshot up to: index=12 +2025-02-25T23:57:51.898Z [INFO] creating new snapshot: path=/tmp/raft3910745328/snapshots/2-12-1740527871898.tmp +2025-02-25T23:57:51.898Z [INFO] server-144a6270-e1f6-f786-5a6d-3e192f167de0: snapshot complete up to: index=12 +2025-02-25T23:57:51.899Z [INFO] creating new snapshot: path=/tmp/raft3910745328/snapshots/2-23-1740527871899.tmp +2025-02-25T23:57:51.899Z [INFO] server-144a6270-e1f6-f786-5a6d-3e192f167de0: copied to local snapshot: bytes=71 +2025-02-25T23:57:51.899Z [INFO] server-144a6270-e1f6-f786-5a6d-3e192f167de0: snapshot restore progress: id=2-23-1740527871899 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:57:51.899Z [INFO] server-144a6270-e1f6-f786-5a6d-3e192f167de0: removing all old logs from log store +2025-02-25T23:57:51.899Z [INFO] server-144a6270-e1f6-f786-5a6d-3e192f167de0: compacting logs: from=1 to=22 +2025-02-25T23:57:51.899Z [INFO] server-144a6270-e1f6-f786-5a6d-3e192f167de0: restored user snapshot: index=23 +2025-02-25T23:57:51.899Z [ERROR] server-144a6270-e1f6-f786-5a6d-3e192f167de0: failed to get log: index=22 error="log not found" +2025-02-25T23:57:51.899Z [INFO] server-144a6270-e1f6-f786-5a6d-3e192f167de0: aborting pipeline replication: peer="{Voter server-a032edb3-d9f0-7cf8-91d7-5883f94efc93 a032edb3-d9f0-7cf8-91d7-5883f94efc93}" +2025-02-25T23:57:51.899Z [ERROR] server-144a6270-e1f6-f786-5a6d-3e192f167de0: failed to get log: index=20 error="log not found" +2025-02-25T23:57:51.899Z [INFO] server-144a6270-e1f6-f786-5a6d-3e192f167de0: aborting pipeline replication: peer="{Voter server-e9baa5ed-e2d8-281d-5837-4e6fd799d622 e9baa5ed-e2d8-281d-5837-4e6fd799d622}" +2025-02-25T23:57:51.899Z [ERROR] server-144a6270-e1f6-f786-5a6d-3e192f167de0: failed to get log: index=20 error="log not found" +2025-02-25T23:57:51.899Z [INFO] creating new snapshot: path=/tmp/raft2554380764/snapshots/2-23-1740527871899.tmp +2025-02-25T23:57:51.899Z [INFO] server-e9baa5ed-e2d8-281d-5837-4e6fd799d622: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:57:51.899Z [INFO] server-e9baa5ed-e2d8-281d-5837-4e6fd799d622: copied to local snapshot: bytes=71 +2025-02-25T23:57:51.899Z [INFO] server-e9baa5ed-e2d8-281d-5837-4e6fd799d622: snapshot restore progress: id=2-23-1740527871899 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:57:51.899Z [INFO] server-e9baa5ed-e2d8-281d-5837-4e6fd799d622: removing all old logs from log store +2025-02-25T23:57:51.899Z [INFO] server-e9baa5ed-e2d8-281d-5837-4e6fd799d622: compacting logs: from=1 to=20 +2025-02-25T23:57:51.899Z [INFO] server-e9baa5ed-e2d8-281d-5837-4e6fd799d622: Installed remote snapshot +2025-02-25T23:57:51.899Z [WARN] server-e9baa5ed-e2d8-281d-5837-4e6fd799d622: skipping application of old log: index=22 +2025-02-25T23:57:51.899Z [INFO] server-144a6270-e1f6-f786-5a6d-3e192f167de0: pipelining replication: peer="{Voter server-e9baa5ed-e2d8-281d-5837-4e6fd799d622 e9baa5ed-e2d8-281d-5837-4e6fd799d622}" +2025-02-25T23:57:51.899Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:57:51.909Z [ERROR] server-144a6270-e1f6-f786-5a6d-3e192f167de0: failed to get log: index=22 error="log not found" +2025-02-25T23:57:51.917Z [INFO] creating new snapshot: path=/tmp/raft322994066/snapshots/2-23-1740527871917.tmp +2025-02-25T23:57:51.918Z [INFO] server-a032edb3-d9f0-7cf8-91d7-5883f94efc93: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:57:51.918Z [INFO] server-a032edb3-d9f0-7cf8-91d7-5883f94efc93: copied to local snapshot: bytes=71 +2025-02-25T23:57:51.918Z [INFO] server-a032edb3-d9f0-7cf8-91d7-5883f94efc93: snapshot restore progress: id=2-23-1740527871917 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:57:51.918Z [INFO] server-a032edb3-d9f0-7cf8-91d7-5883f94efc93: removing all old logs from log store +2025-02-25T23:57:51.918Z [INFO] server-a032edb3-d9f0-7cf8-91d7-5883f94efc93: compacting logs: from=1 to=22 +2025-02-25T23:57:51.918Z [INFO] server-a032edb3-d9f0-7cf8-91d7-5883f94efc93: Installed remote snapshot +2025-02-25T23:57:51.918Z [INFO] server-144a6270-e1f6-f786-5a6d-3e192f167de0: pipelining replication: peer="{Voter server-a032edb3-d9f0-7cf8-91d7-5883f94efc93 a032edb3-d9f0-7cf8-91d7-5883f94efc93}" +2025-02-25T23:57:52.905Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:57:51.899884768 +0000 UTC m=+19.508900175 (2 nodes), 400ns from start of poll, 1.649476425s from cluster start. Timeout at 2025-02-25 23:57:52.904889968 +0000 UTC m=+20.513905415, 1.00500524s after stability +2025-02-25T23:57:52.921Z [INFO] server-144a6270-e1f6-f786-5a6d-3e192f167de0: aborting pipeline replication: peer="{Voter server-e9baa5ed-e2d8-281d-5837-4e6fd799d622 e9baa5ed-e2d8-281d-5837-4e6fd799d622}" +2025-02-25T23:57:52.921Z [INFO] server-144a6270-e1f6-f786-5a6d-3e192f167de0: aborting pipeline replication: peer="{Voter server-a032edb3-d9f0-7cf8-91d7-5883f94efc93 a032edb3-d9f0-7cf8-91d7-5883f94efc93}" === RUN TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_true -2026-03-31T06:16:30.945Z [INFO] server-81f2cd18-c224-6e16-c95a-72edfb8f7b43: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-81f2cd18-c224-6e16-c95a-72edfb8f7b43 Address:81f2cd18-c224-6e16-c95a-72edfb8f7b43} {Suffrage:Voter ID:server-6888f31a-0f01-ab3b-c81c-bc5b93c62a94 Address:6888f31a-0f01-ab3b-c81c-bc5b93c62a94} {Suffrage:Voter ID:server-1290471b-0fd0-ca61-9b8e-50c783c5f8a9 Address:1290471b-0fd0-ca61-9b8e-50c783c5f8a9}]" -2026-03-31T06:16:30.945Z [INFO] server-6888f31a-0f01-ab3b-c81c-bc5b93c62a94: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-81f2cd18-c224-6e16-c95a-72edfb8f7b43 Address:81f2cd18-c224-6e16-c95a-72edfb8f7b43} {Suffrage:Voter ID:server-6888f31a-0f01-ab3b-c81c-bc5b93c62a94 Address:6888f31a-0f01-ab3b-c81c-bc5b93c62a94} {Suffrage:Voter ID:server-1290471b-0fd0-ca61-9b8e-50c783c5f8a9 Address:1290471b-0fd0-ca61-9b8e-50c783c5f8a9}]" -2026-03-31T06:16:30.945Z [INFO] server-81f2cd18-c224-6e16-c95a-72edfb8f7b43: entering follower state: follower="Node at 81f2cd18-c224-6e16-c95a-72edfb8f7b43 [Follower]" leader-address= leader-id= -2026-03-31T06:16:30.945Z [INFO] server-6888f31a-0f01-ab3b-c81c-bc5b93c62a94: entering follower state: follower="Node at 6888f31a-0f01-ab3b-c81c-bc5b93c62a94 [Follower]" leader-address= leader-id= -2026-03-31T06:16:30.945Z [INFO] server-1290471b-0fd0-ca61-9b8e-50c783c5f8a9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-81f2cd18-c224-6e16-c95a-72edfb8f7b43 Address:81f2cd18-c224-6e16-c95a-72edfb8f7b43} {Suffrage:Voter ID:server-6888f31a-0f01-ab3b-c81c-bc5b93c62a94 Address:6888f31a-0f01-ab3b-c81c-bc5b93c62a94} {Suffrage:Voter ID:server-1290471b-0fd0-ca61-9b8e-50c783c5f8a9 Address:1290471b-0fd0-ca61-9b8e-50c783c5f8a9}]" -2026-03-31T06:16:30.945Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:30.945Z [INFO] server-1290471b-0fd0-ca61-9b8e-50c783c5f8a9: entering follower state: follower="Node at 1290471b-0fd0-ca61-9b8e-50c783c5f8a9 [Follower]" leader-address= leader-id= -2026-03-31T06:16:31.559Z [WARN] server-6888f31a-0f01-ab3b-c81c-bc5b93c62a94: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:31.559Z [INFO] server-6888f31a-0f01-ab3b-c81c-bc5b93c62a94: entering candidate state: node="Node at 6888f31a-0f01-ab3b-c81c-bc5b93c62a94 [Candidate]" term=2 -2026-03-31T06:16:31.559Z [INFO] server-6888f31a-0f01-ab3b-c81c-bc5b93c62a94: election won: term=2 tally=2 -2026-03-31T06:16:31.559Z [INFO] server-6888f31a-0f01-ab3b-c81c-bc5b93c62a94: entering leader state: leader="Node at 6888f31a-0f01-ab3b-c81c-bc5b93c62a94 [Leader]" -2026-03-31T06:16:31.559Z [INFO] server-6888f31a-0f01-ab3b-c81c-bc5b93c62a94: added peer, starting replication: peer=server-81f2cd18-c224-6e16-c95a-72edfb8f7b43 -2026-03-31T06:16:31.559Z [INFO] server-6888f31a-0f01-ab3b-c81c-bc5b93c62a94: added peer, starting replication: peer=server-1290471b-0fd0-ca61-9b8e-50c783c5f8a9 -2026-03-31T06:16:31.559Z [INFO] server-6888f31a-0f01-ab3b-c81c-bc5b93c62a94: pipelining replication: peer="{Voter server-81f2cd18-c224-6e16-c95a-72edfb8f7b43 81f2cd18-c224-6e16-c95a-72edfb8f7b43}" -2026-03-31T06:16:31.559Z [INFO] server-6888f31a-0f01-ab3b-c81c-bc5b93c62a94: pipelining replication: peer="{Voter server-1290471b-0fd0-ca61-9b8e-50c783c5f8a9 1290471b-0fd0-ca61-9b8e-50c783c5f8a9}" -2026-03-31T06:16:32.565Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:31.5596427 +0000 UTC m=+19.837250081 (1 nodes), 614.33152ms from start of poll, 614.7414ms from cluster start. Timeout at 2026-03-31 06:16:32.5646489 +0000 UTC m=+20.842256281, 1.0050062s after stability -2026-03-31T06:16:32.566Z [INFO] server-6888f31a-0f01-ab3b-c81c-bc5b93c62a94: starting snapshot up to: index=12 -2026-03-31T06:16:32.566Z [INFO] creating new snapshot: path=/tmp/raft3747166050/snapshots/2-12-1774937792566.tmp -2026-03-31T06:16:32.566Z [INFO] server-6888f31a-0f01-ab3b-c81c-bc5b93c62a94: snapshot complete up to: index=12 -2026-03-31T06:16:32.569Z [INFO] server-dbbb8175-4c5d-7bb6-d372-646abd205236: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dbbb8175-4c5d-7bb6-d372-646abd205236 Address:dbbb8175-4c5d-7bb6-d372-646abd205236} {Suffrage:Voter ID:server-4029e8e5-63e2-6291-fb9d-1fd11ba6063a Address:4029e8e5-63e2-6291-fb9d-1fd11ba6063a} {Suffrage:Voter ID:server-2dbd3c89-4372-d301-5a7c-4d5eb25141fd Address:2dbd3c89-4372-d301-5a7c-4d5eb25141fd}]" -2026-03-31T06:16:32.569Z [INFO] server-dbbb8175-4c5d-7bb6-d372-646abd205236: entering follower state: follower="Node at dbbb8175-4c5d-7bb6-d372-646abd205236 [Follower]" leader-address= leader-id= -2026-03-31T06:16:32.569Z [INFO] server-4029e8e5-63e2-6291-fb9d-1fd11ba6063a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dbbb8175-4c5d-7bb6-d372-646abd205236 Address:dbbb8175-4c5d-7bb6-d372-646abd205236} {Suffrage:Voter ID:server-4029e8e5-63e2-6291-fb9d-1fd11ba6063a Address:4029e8e5-63e2-6291-fb9d-1fd11ba6063a} {Suffrage:Voter ID:server-2dbd3c89-4372-d301-5a7c-4d5eb25141fd Address:2dbd3c89-4372-d301-5a7c-4d5eb25141fd}]" -2026-03-31T06:16:32.569Z [INFO] server-4029e8e5-63e2-6291-fb9d-1fd11ba6063a: entering follower state: follower="Node at 4029e8e5-63e2-6291-fb9d-1fd11ba6063a [Follower]" leader-address= leader-id= -2026-03-31T06:16:32.570Z [INFO] server-2dbd3c89-4372-d301-5a7c-4d5eb25141fd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dbbb8175-4c5d-7bb6-d372-646abd205236 Address:dbbb8175-4c5d-7bb6-d372-646abd205236} {Suffrage:Voter ID:server-4029e8e5-63e2-6291-fb9d-1fd11ba6063a Address:4029e8e5-63e2-6291-fb9d-1fd11ba6063a} {Suffrage:Voter ID:server-2dbd3c89-4372-d301-5a7c-4d5eb25141fd Address:2dbd3c89-4372-d301-5a7c-4d5eb25141fd}]" -2026-03-31T06:16:32.570Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:32.570Z [INFO] server-2dbd3c89-4372-d301-5a7c-4d5eb25141fd: entering follower state: follower="Node at 2dbd3c89-4372-d301-5a7c-4d5eb25141fd [Follower]" leader-address= leader-id= -2026-03-31T06:16:33.127Z [WARN] server-dbbb8175-4c5d-7bb6-d372-646abd205236: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:33.127Z [INFO] server-dbbb8175-4c5d-7bb6-d372-646abd205236: entering candidate state: node="Node at dbbb8175-4c5d-7bb6-d372-646abd205236 [Candidate]" term=2 -2026-03-31T06:16:33.127Z [INFO] server-dbbb8175-4c5d-7bb6-d372-646abd205236: election won: term=2 tally=2 -2026-03-31T06:16:33.127Z [INFO] server-dbbb8175-4c5d-7bb6-d372-646abd205236: entering leader state: leader="Node at dbbb8175-4c5d-7bb6-d372-646abd205236 [Leader]" -2026-03-31T06:16:33.127Z [INFO] server-dbbb8175-4c5d-7bb6-d372-646abd205236: added peer, starting replication: peer=server-4029e8e5-63e2-6291-fb9d-1fd11ba6063a -2026-03-31T06:16:33.127Z [INFO] server-dbbb8175-4c5d-7bb6-d372-646abd205236: added peer, starting replication: peer=server-2dbd3c89-4372-d301-5a7c-4d5eb25141fd -2026-03-31T06:16:33.127Z [INFO] server-dbbb8175-4c5d-7bb6-d372-646abd205236: pipelining replication: peer="{Voter server-2dbd3c89-4372-d301-5a7c-4d5eb25141fd 2dbd3c89-4372-d301-5a7c-4d5eb25141fd}" -2026-03-31T06:16:33.127Z [INFO] server-dbbb8175-4c5d-7bb6-d372-646abd205236: pipelining replication: peer="{Voter server-4029e8e5-63e2-6291-fb9d-1fd11ba6063a 4029e8e5-63e2-6291-fb9d-1fd11ba6063a}" -2026-03-31T06:16:34.132Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:33.12779546 +0000 UTC m=+21.405402841 (1 nodes), 557.629ms from start of poll, 558.58036ms from cluster start. Timeout at 2026-03-31 06:16:34.13280114 +0000 UTC m=+22.410408521, 1.00500568s after stability -2026-03-31T06:16:34.133Z [INFO] creating new snapshot: path=/tmp/raft2998966099/snapshots/2-13-1774937794133.tmp -2026-03-31T06:16:34.133Z [INFO] server-dbbb8175-4c5d-7bb6-d372-646abd205236: copied to local snapshot: bytes=71 -2026-03-31T06:16:34.133Z [INFO] server-dbbb8175-4c5d-7bb6-d372-646abd205236: snapshot restore progress: id=2-13-1774937794133 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:34.133Z [INFO] server-dbbb8175-4c5d-7bb6-d372-646abd205236: restored user snapshot: index=13 -2026-03-31T06:16:34.134Z [ERROR] server-dbbb8175-4c5d-7bb6-d372-646abd205236: failed to get log: index=3 error="log not found" -2026-03-31T06:16:34.134Z [INFO] server-dbbb8175-4c5d-7bb6-d372-646abd205236: aborting pipeline replication: peer="{Voter server-2dbd3c89-4372-d301-5a7c-4d5eb25141fd 2dbd3c89-4372-d301-5a7c-4d5eb25141fd}" -2026-03-31T06:16:34.134Z [ERROR] server-dbbb8175-4c5d-7bb6-d372-646abd205236: failed to get log: index=3 error="log not found" -2026-03-31T06:16:34.134Z [INFO] server-dbbb8175-4c5d-7bb6-d372-646abd205236: aborting pipeline replication: peer="{Voter server-4029e8e5-63e2-6291-fb9d-1fd11ba6063a 4029e8e5-63e2-6291-fb9d-1fd11ba6063a}" -2026-03-31T06:16:34.139Z [ERROR] server-dbbb8175-4c5d-7bb6-d372-646abd205236: failed to get log: index=3 error="log not found" -2026-03-31T06:16:34.140Z [INFO] creating new snapshot: path=/tmp/raft3811405727/snapshots/2-13-1774937794140.tmp -2026-03-31T06:16:34.140Z [INFO] server-4029e8e5-63e2-6291-fb9d-1fd11ba6063a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:34.140Z [INFO] server-4029e8e5-63e2-6291-fb9d-1fd11ba6063a: copied to local snapshot: bytes=71 -2026-03-31T06:16:34.140Z [INFO] server-4029e8e5-63e2-6291-fb9d-1fd11ba6063a: snapshot restore progress: id=2-13-1774937794140 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:34.141Z [INFO] server-4029e8e5-63e2-6291-fb9d-1fd11ba6063a: Installed remote snapshot -2026-03-31T06:16:34.141Z [INFO] server-dbbb8175-4c5d-7bb6-d372-646abd205236: pipelining replication: peer="{Voter server-4029e8e5-63e2-6291-fb9d-1fd11ba6063a 4029e8e5-63e2-6291-fb9d-1fd11ba6063a}" -2026-03-31T06:16:34.141Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:16:34.141Z [ERROR] server-dbbb8175-4c5d-7bb6-d372-646abd205236: failed to get log: index=3 error="log not found" -2026-03-31T06:16:34.142Z [INFO] creating new snapshot: path=/tmp/raft237358390/snapshots/2-13-1774937794142.tmp -2026-03-31T06:16:34.142Z [INFO] server-2dbd3c89-4372-d301-5a7c-4d5eb25141fd: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:34.142Z [INFO] server-2dbd3c89-4372-d301-5a7c-4d5eb25141fd: copied to local snapshot: bytes=71 -2026-03-31T06:16:34.143Z [INFO] server-2dbd3c89-4372-d301-5a7c-4d5eb25141fd: snapshot restore progress: id=2-13-1774937794142 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:34.143Z [INFO] server-2dbd3c89-4372-d301-5a7c-4d5eb25141fd: Installed remote snapshot -2026-03-31T06:16:34.143Z [INFO] server-dbbb8175-4c5d-7bb6-d372-646abd205236: pipelining replication: peer="{Voter server-2dbd3c89-4372-d301-5a7c-4d5eb25141fd 2dbd3c89-4372-d301-5a7c-4d5eb25141fd}" -2026-03-31T06:16:35.146Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:16:34.14120318 +0000 UTC m=+22.418810561 (2 nodes), 1.28µs from start of poll, 1.57198808s from cluster start. Timeout at 2026-03-31 06:16:35.14620674 +0000 UTC m=+23.423814121, 1.00500356s after stability -2026-03-31T06:16:35.159Z [INFO] server-6888f31a-0f01-ab3b-c81c-bc5b93c62a94: aborting pipeline replication: peer="{Voter server-81f2cd18-c224-6e16-c95a-72edfb8f7b43 81f2cd18-c224-6e16-c95a-72edfb8f7b43}" -2026-03-31T06:16:35.159Z [INFO] server-6888f31a-0f01-ab3b-c81c-bc5b93c62a94: aborting pipeline replication: peer="{Voter server-1290471b-0fd0-ca61-9b8e-50c783c5f8a9 1290471b-0fd0-ca61-9b8e-50c783c5f8a9}" +2025-02-25T23:57:52.922Z [INFO] server-a2e1305f-3ac5-4c3f-31f2-adcbf4696f6c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a2e1305f-3ac5-4c3f-31f2-adcbf4696f6c Address:a2e1305f-3ac5-4c3f-31f2-adcbf4696f6c} {Suffrage:Voter ID:server-8d657a58-4f4f-5cd6-9392-06688657d68a Address:8d657a58-4f4f-5cd6-9392-06688657d68a} {Suffrage:Voter ID:server-5cd162e4-a3f8-0150-99f9-832a8fa6591c Address:5cd162e4-a3f8-0150-99f9-832a8fa6591c}]" +2025-02-25T23:57:52.922Z [INFO] server-8d657a58-4f4f-5cd6-9392-06688657d68a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a2e1305f-3ac5-4c3f-31f2-adcbf4696f6c Address:a2e1305f-3ac5-4c3f-31f2-adcbf4696f6c} {Suffrage:Voter ID:server-8d657a58-4f4f-5cd6-9392-06688657d68a Address:8d657a58-4f4f-5cd6-9392-06688657d68a} {Suffrage:Voter ID:server-5cd162e4-a3f8-0150-99f9-832a8fa6591c Address:5cd162e4-a3f8-0150-99f9-832a8fa6591c}]" +2025-02-25T23:57:52.922Z [INFO] server-5cd162e4-a3f8-0150-99f9-832a8fa6591c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a2e1305f-3ac5-4c3f-31f2-adcbf4696f6c Address:a2e1305f-3ac5-4c3f-31f2-adcbf4696f6c} {Suffrage:Voter ID:server-8d657a58-4f4f-5cd6-9392-06688657d68a Address:8d657a58-4f4f-5cd6-9392-06688657d68a} {Suffrage:Voter ID:server-5cd162e4-a3f8-0150-99f9-832a8fa6591c Address:5cd162e4-a3f8-0150-99f9-832a8fa6591c}]" +2025-02-25T23:57:52.922Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:52.922Z [INFO] server-a2e1305f-3ac5-4c3f-31f2-adcbf4696f6c: entering follower state: follower="Node at a2e1305f-3ac5-4c3f-31f2-adcbf4696f6c [Follower]" leader-address= leader-id= +2025-02-25T23:57:52.922Z [INFO] server-8d657a58-4f4f-5cd6-9392-06688657d68a: entering follower state: follower="Node at 8d657a58-4f4f-5cd6-9392-06688657d68a [Follower]" leader-address= leader-id= +2025-02-25T23:57:52.922Z [INFO] server-5cd162e4-a3f8-0150-99f9-832a8fa6591c: entering follower state: follower="Node at 5cd162e4-a3f8-0150-99f9-832a8fa6591c [Follower]" leader-address= leader-id= +2025-02-25T23:57:53.669Z [WARN] server-a2e1305f-3ac5-4c3f-31f2-adcbf4696f6c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:53.669Z [INFO] server-a2e1305f-3ac5-4c3f-31f2-adcbf4696f6c: entering candidate state: node="Node at a2e1305f-3ac5-4c3f-31f2-adcbf4696f6c [Candidate]" term=2 +2025-02-25T23:57:53.669Z [INFO] server-a2e1305f-3ac5-4c3f-31f2-adcbf4696f6c: election won: term=2 tally=2 +2025-02-25T23:57:53.670Z [INFO] server-a2e1305f-3ac5-4c3f-31f2-adcbf4696f6c: entering leader state: leader="Node at a2e1305f-3ac5-4c3f-31f2-adcbf4696f6c [Leader]" +2025-02-25T23:57:53.670Z [INFO] server-a2e1305f-3ac5-4c3f-31f2-adcbf4696f6c: added peer, starting replication: peer=server-8d657a58-4f4f-5cd6-9392-06688657d68a +2025-02-25T23:57:53.670Z [INFO] server-a2e1305f-3ac5-4c3f-31f2-adcbf4696f6c: added peer, starting replication: peer=server-5cd162e4-a3f8-0150-99f9-832a8fa6591c +2025-02-25T23:57:53.670Z [INFO] server-a2e1305f-3ac5-4c3f-31f2-adcbf4696f6c: pipelining replication: peer="{Voter server-5cd162e4-a3f8-0150-99f9-832a8fa6591c 5cd162e4-a3f8-0150-99f9-832a8fa6591c}" +2025-02-25T23:57:53.670Z [INFO] server-a2e1305f-3ac5-4c3f-31f2-adcbf4696f6c: pipelining replication: peer="{Voter server-8d657a58-4f4f-5cd6-9392-06688657d68a 8d657a58-4f4f-5cd6-9392-06688657d68a}" +2025-02-25T23:57:54.677Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:53.670196283 +0000 UTC m=+21.279211690 (1 nodes), 747.640922ms from start of poll, 747.814803ms from cluster start. Timeout at 2025-02-25 23:57:54.675197643 +0000 UTC m=+22.284213050, 1.00500136s after stability +2025-02-25T23:57:54.678Z [INFO] server-a2e1305f-3ac5-4c3f-31f2-adcbf4696f6c: starting snapshot up to: index=12 +2025-02-25T23:57:54.678Z [INFO] creating new snapshot: path=/tmp/raft1444446834/snapshots/2-12-1740527874678.tmp +2025-02-25T23:57:54.678Z [INFO] server-a2e1305f-3ac5-4c3f-31f2-adcbf4696f6c: snapshot complete up to: index=12 +2025-02-25T23:57:54.679Z [INFO] server-e54369d6-76c7-768e-2dab-d3398cb333be: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e54369d6-76c7-768e-2dab-d3398cb333be Address:e54369d6-76c7-768e-2dab-d3398cb333be} {Suffrage:Voter ID:server-b5591081-e1f2-cc4e-4b95-edbdd6bb4533 Address:b5591081-e1f2-cc4e-4b95-edbdd6bb4533} {Suffrage:Voter ID:server-9f7f14c7-875d-d1ea-bd4f-09c45ef1ede6 Address:9f7f14c7-875d-d1ea-bd4f-09c45ef1ede6}]" +2025-02-25T23:57:54.679Z [INFO] server-b5591081-e1f2-cc4e-4b95-edbdd6bb4533: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e54369d6-76c7-768e-2dab-d3398cb333be Address:e54369d6-76c7-768e-2dab-d3398cb333be} {Suffrage:Voter ID:server-b5591081-e1f2-cc4e-4b95-edbdd6bb4533 Address:b5591081-e1f2-cc4e-4b95-edbdd6bb4533} {Suffrage:Voter ID:server-9f7f14c7-875d-d1ea-bd4f-09c45ef1ede6 Address:9f7f14c7-875d-d1ea-bd4f-09c45ef1ede6}]" +2025-02-25T23:57:54.679Z [INFO] server-9f7f14c7-875d-d1ea-bd4f-09c45ef1ede6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e54369d6-76c7-768e-2dab-d3398cb333be Address:e54369d6-76c7-768e-2dab-d3398cb333be} {Suffrage:Voter ID:server-b5591081-e1f2-cc4e-4b95-edbdd6bb4533 Address:b5591081-e1f2-cc4e-4b95-edbdd6bb4533} {Suffrage:Voter ID:server-9f7f14c7-875d-d1ea-bd4f-09c45ef1ede6 Address:9f7f14c7-875d-d1ea-bd4f-09c45ef1ede6}]" +2025-02-25T23:57:54.679Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:54.679Z [INFO] server-e54369d6-76c7-768e-2dab-d3398cb333be: entering follower state: follower="Node at e54369d6-76c7-768e-2dab-d3398cb333be [Follower]" leader-address= leader-id= +2025-02-25T23:57:54.679Z [INFO] server-b5591081-e1f2-cc4e-4b95-edbdd6bb4533: entering follower state: follower="Node at b5591081-e1f2-cc4e-4b95-edbdd6bb4533 [Follower]" leader-address= leader-id= +2025-02-25T23:57:54.679Z [INFO] server-9f7f14c7-875d-d1ea-bd4f-09c45ef1ede6: entering follower state: follower="Node at 9f7f14c7-875d-d1ea-bd4f-09c45ef1ede6 [Follower]" leader-address= leader-id= +2025-02-25T23:57:55.282Z [WARN] server-9f7f14c7-875d-d1ea-bd4f-09c45ef1ede6: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:55.282Z [INFO] server-9f7f14c7-875d-d1ea-bd4f-09c45ef1ede6: entering candidate state: node="Node at 9f7f14c7-875d-d1ea-bd4f-09c45ef1ede6 [Candidate]" term=2 +2025-02-25T23:57:55.282Z [INFO] server-9f7f14c7-875d-d1ea-bd4f-09c45ef1ede6: election won: term=2 tally=2 +2025-02-25T23:57:55.282Z [INFO] server-9f7f14c7-875d-d1ea-bd4f-09c45ef1ede6: entering leader state: leader="Node at 9f7f14c7-875d-d1ea-bd4f-09c45ef1ede6 [Leader]" +2025-02-25T23:57:55.282Z [INFO] server-9f7f14c7-875d-d1ea-bd4f-09c45ef1ede6: added peer, starting replication: peer=server-e54369d6-76c7-768e-2dab-d3398cb333be +2025-02-25T23:57:55.282Z [INFO] server-9f7f14c7-875d-d1ea-bd4f-09c45ef1ede6: added peer, starting replication: peer=server-b5591081-e1f2-cc4e-4b95-edbdd6bb4533 +2025-02-25T23:57:55.282Z [INFO] server-9f7f14c7-875d-d1ea-bd4f-09c45ef1ede6: pipelining replication: peer="{Voter server-b5591081-e1f2-cc4e-4b95-edbdd6bb4533 b5591081-e1f2-cc4e-4b95-edbdd6bb4533}" +2025-02-25T23:57:55.282Z [INFO] server-9f7f14c7-875d-d1ea-bd4f-09c45ef1ede6: pipelining replication: peer="{Voter server-e54369d6-76c7-768e-2dab-d3398cb333be e54369d6-76c7-768e-2dab-d3398cb333be}" +2025-02-25T23:57:56.290Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:55.282939794 +0000 UTC m=+22.891955201 (1 nodes), 603.625685ms from start of poll, 603.777926ms from cluster start. Timeout at 2025-02-25 23:57:56.287943394 +0000 UTC m=+23.896958801, 1.0050036s after stability +2025-02-25T23:57:56.290Z [INFO] creating new snapshot: path=/tmp/raft3629176156/snapshots/2-13-1740527876290.tmp +2025-02-25T23:57:56.290Z [INFO] server-9f7f14c7-875d-d1ea-bd4f-09c45ef1ede6: copied to local snapshot: bytes=71 +2025-02-25T23:57:56.290Z [INFO] server-9f7f14c7-875d-d1ea-bd4f-09c45ef1ede6: snapshot restore progress: id=2-13-1740527876290 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:57:56.290Z [INFO] server-9f7f14c7-875d-d1ea-bd4f-09c45ef1ede6: restored user snapshot: index=13 +2025-02-25T23:57:56.290Z [ERROR] server-9f7f14c7-875d-d1ea-bd4f-09c45ef1ede6: failed to get log: index=3 error="log not found" +2025-02-25T23:57:56.290Z [INFO] server-9f7f14c7-875d-d1ea-bd4f-09c45ef1ede6: aborting pipeline replication: peer="{Voter server-b5591081-e1f2-cc4e-4b95-edbdd6bb4533 b5591081-e1f2-cc4e-4b95-edbdd6bb4533}" +2025-02-25T23:57:56.290Z [ERROR] server-9f7f14c7-875d-d1ea-bd4f-09c45ef1ede6: failed to get log: index=3 error="log not found" +2025-02-25T23:57:56.290Z [INFO] server-9f7f14c7-875d-d1ea-bd4f-09c45ef1ede6: aborting pipeline replication: peer="{Voter server-e54369d6-76c7-768e-2dab-d3398cb333be e54369d6-76c7-768e-2dab-d3398cb333be}" +2025-02-25T23:57:56.297Z [ERROR] server-9f7f14c7-875d-d1ea-bd4f-09c45ef1ede6: failed to get log: index=3 error="log not found" +2025-02-25T23:57:56.298Z [INFO] creating new snapshot: path=/tmp/raft1636282740/snapshots/2-13-1740527876298.tmp +2025-02-25T23:57:56.299Z [INFO] server-b5591081-e1f2-cc4e-4b95-edbdd6bb4533: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:57:56.299Z [INFO] server-b5591081-e1f2-cc4e-4b95-edbdd6bb4533: copied to local snapshot: bytes=71 +2025-02-25T23:57:56.299Z [INFO] server-b5591081-e1f2-cc4e-4b95-edbdd6bb4533: snapshot restore progress: id=2-13-1740527876298 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:57:56.299Z [INFO] server-b5591081-e1f2-cc4e-4b95-edbdd6bb4533: Installed remote snapshot +2025-02-25T23:57:56.300Z [INFO] server-9f7f14c7-875d-d1ea-bd4f-09c45ef1ede6: pipelining replication: peer="{Voter server-b5591081-e1f2-cc4e-4b95-edbdd6bb4533 b5591081-e1f2-cc4e-4b95-edbdd6bb4533}" +2025-02-25T23:57:56.300Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:57:56.300Z [ERROR] server-9f7f14c7-875d-d1ea-bd4f-09c45ef1ede6: failed to get log: index=3 error="log not found" +2025-02-25T23:57:56.300Z [INFO] creating new snapshot: path=/tmp/raft1746505073/snapshots/2-13-1740527876300.tmp +2025-02-25T23:57:56.300Z [INFO] server-e54369d6-76c7-768e-2dab-d3398cb333be: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:57:56.300Z [INFO] server-e54369d6-76c7-768e-2dab-d3398cb333be: copied to local snapshot: bytes=71 +2025-02-25T23:57:56.306Z [INFO] server-e54369d6-76c7-768e-2dab-d3398cb333be: snapshot restore progress: id=2-13-1740527876300 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:57:56.306Z [INFO] server-e54369d6-76c7-768e-2dab-d3398cb333be: Installed remote snapshot +2025-02-25T23:57:56.306Z [INFO] server-9f7f14c7-875d-d1ea-bd4f-09c45ef1ede6: pipelining replication: peer="{Voter server-e54369d6-76c7-768e-2dab-d3398cb333be e54369d6-76c7-768e-2dab-d3398cb333be}" +2025-02-25T23:57:57.305Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:57:56.300097792 +0000 UTC m=+23.909113199 (2 nodes), 440ns from start of poll, 1.620935924s from cluster start. Timeout at 2025-02-25 23:57:57.305109472 +0000 UTC m=+24.914124839, 1.00501164s after stability +2025-02-25T23:57:57.321Z [INFO] server-a2e1305f-3ac5-4c3f-31f2-adcbf4696f6c: aborting pipeline replication: peer="{Voter server-5cd162e4-a3f8-0150-99f9-832a8fa6591c 5cd162e4-a3f8-0150-99f9-832a8fa6591c}" +2025-02-25T23:57:57.321Z [INFO] server-a2e1305f-3ac5-4c3f-31f2-adcbf4696f6c: aborting pipeline replication: peer="{Voter server-8d657a58-4f4f-5cd6-9392-06688657d68a 8d657a58-4f4f-5cd6-9392-06688657d68a}" === RUN TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_true -2026-03-31T06:16:35.160Z [INFO] server-e59c77f4-52fb-a5b4-e2ab-7614508a7ad7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e59c77f4-52fb-a5b4-e2ab-7614508a7ad7 Address:e59c77f4-52fb-a5b4-e2ab-7614508a7ad7} {Suffrage:Voter ID:server-d1bf814d-82db-7f77-9976-78a3241bcb93 Address:d1bf814d-82db-7f77-9976-78a3241bcb93} {Suffrage:Voter ID:server-4c12f43f-dd8d-f129-e09f-06cc29032f6e Address:4c12f43f-dd8d-f129-e09f-06cc29032f6e}]" -2026-03-31T06:16:35.160Z [INFO] server-d1bf814d-82db-7f77-9976-78a3241bcb93: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e59c77f4-52fb-a5b4-e2ab-7614508a7ad7 Address:e59c77f4-52fb-a5b4-e2ab-7614508a7ad7} {Suffrage:Voter ID:server-d1bf814d-82db-7f77-9976-78a3241bcb93 Address:d1bf814d-82db-7f77-9976-78a3241bcb93} {Suffrage:Voter ID:server-4c12f43f-dd8d-f129-e09f-06cc29032f6e Address:4c12f43f-dd8d-f129-e09f-06cc29032f6e}]" -2026-03-31T06:16:35.160Z [INFO] server-e59c77f4-52fb-a5b4-e2ab-7614508a7ad7: entering follower state: follower="Node at e59c77f4-52fb-a5b4-e2ab-7614508a7ad7 [Follower]" leader-address= leader-id= -2026-03-31T06:16:35.160Z [INFO] server-d1bf814d-82db-7f77-9976-78a3241bcb93: entering follower state: follower="Node at d1bf814d-82db-7f77-9976-78a3241bcb93 [Follower]" leader-address= leader-id= -2026-03-31T06:16:35.160Z [INFO] server-4c12f43f-dd8d-f129-e09f-06cc29032f6e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e59c77f4-52fb-a5b4-e2ab-7614508a7ad7 Address:e59c77f4-52fb-a5b4-e2ab-7614508a7ad7} {Suffrage:Voter ID:server-d1bf814d-82db-7f77-9976-78a3241bcb93 Address:d1bf814d-82db-7f77-9976-78a3241bcb93} {Suffrage:Voter ID:server-4c12f43f-dd8d-f129-e09f-06cc29032f6e Address:4c12f43f-dd8d-f129-e09f-06cc29032f6e}]" -2026-03-31T06:16:35.160Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:35.160Z [INFO] server-4c12f43f-dd8d-f129-e09f-06cc29032f6e: entering follower state: follower="Node at 4c12f43f-dd8d-f129-e09f-06cc29032f6e [Follower]" leader-address= leader-id= -2026-03-31T06:16:36.029Z [WARN] server-d1bf814d-82db-7f77-9976-78a3241bcb93: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:36.029Z [INFO] server-d1bf814d-82db-7f77-9976-78a3241bcb93: entering candidate state: node="Node at d1bf814d-82db-7f77-9976-78a3241bcb93 [Candidate]" term=2 -2026-03-31T06:16:36.029Z [INFO] server-d1bf814d-82db-7f77-9976-78a3241bcb93: election won: term=2 tally=2 -2026-03-31T06:16:36.029Z [INFO] server-d1bf814d-82db-7f77-9976-78a3241bcb93: entering leader state: leader="Node at d1bf814d-82db-7f77-9976-78a3241bcb93 [Leader]" -2026-03-31T06:16:36.029Z [INFO] server-d1bf814d-82db-7f77-9976-78a3241bcb93: added peer, starting replication: peer=server-e59c77f4-52fb-a5b4-e2ab-7614508a7ad7 -2026-03-31T06:16:36.029Z [INFO] server-d1bf814d-82db-7f77-9976-78a3241bcb93: added peer, starting replication: peer=server-4c12f43f-dd8d-f129-e09f-06cc29032f6e -2026-03-31T06:16:36.029Z [INFO] server-d1bf814d-82db-7f77-9976-78a3241bcb93: pipelining replication: peer="{Voter server-e59c77f4-52fb-a5b4-e2ab-7614508a7ad7 e59c77f4-52fb-a5b4-e2ab-7614508a7ad7}" -2026-03-31T06:16:36.029Z [INFO] server-d1bf814d-82db-7f77-9976-78a3241bcb93: pipelining replication: peer="{Voter server-4c12f43f-dd8d-f129-e09f-06cc29032f6e 4c12f43f-dd8d-f129-e09f-06cc29032f6e}" -2026-03-31T06:16:37.035Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:36.02953062 +0000 UTC m=+24.307138001 (1 nodes), 868.5464ms from start of poll, 868.95944ms from cluster start. Timeout at 2026-03-31 06:16:37.03453626 +0000 UTC m=+25.312143641, 1.00500564s after stability -2026-03-31T06:16:37.035Z [INFO] server-d1bf814d-82db-7f77-9976-78a3241bcb93: starting snapshot up to: index=12 -2026-03-31T06:16:37.035Z [INFO] creating new snapshot: path=/tmp/raft27049692/snapshots/2-12-1774937797035.tmp -2026-03-31T06:16:37.036Z [INFO] server-d1bf814d-82db-7f77-9976-78a3241bcb93: snapshot complete up to: index=12 -2026-03-31T06:16:37.038Z [INFO] server-f990da9c-3bb4-1284-fe64-6b865bb760c8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f990da9c-3bb4-1284-fe64-6b865bb760c8 Address:f990da9c-3bb4-1284-fe64-6b865bb760c8} {Suffrage:Voter ID:server-c5786428-a93a-d807-cfdf-81d45b7caf1e Address:c5786428-a93a-d807-cfdf-81d45b7caf1e} {Suffrage:Voter ID:server-5cfa8c24-ae0c-7b71-bd67-d713e599aef5 Address:5cfa8c24-ae0c-7b71-bd67-d713e599aef5}]" -2026-03-31T06:16:37.038Z [INFO] server-c5786428-a93a-d807-cfdf-81d45b7caf1e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f990da9c-3bb4-1284-fe64-6b865bb760c8 Address:f990da9c-3bb4-1284-fe64-6b865bb760c8} {Suffrage:Voter ID:server-c5786428-a93a-d807-cfdf-81d45b7caf1e Address:c5786428-a93a-d807-cfdf-81d45b7caf1e} {Suffrage:Voter ID:server-5cfa8c24-ae0c-7b71-bd67-d713e599aef5 Address:5cfa8c24-ae0c-7b71-bd67-d713e599aef5}]" -2026-03-31T06:16:37.038Z [INFO] server-f990da9c-3bb4-1284-fe64-6b865bb760c8: entering follower state: follower="Node at f990da9c-3bb4-1284-fe64-6b865bb760c8 [Follower]" leader-address= leader-id= -2026-03-31T06:16:37.038Z [INFO] server-5cfa8c24-ae0c-7b71-bd67-d713e599aef5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f990da9c-3bb4-1284-fe64-6b865bb760c8 Address:f990da9c-3bb4-1284-fe64-6b865bb760c8} {Suffrage:Voter ID:server-c5786428-a93a-d807-cfdf-81d45b7caf1e Address:c5786428-a93a-d807-cfdf-81d45b7caf1e} {Suffrage:Voter ID:server-5cfa8c24-ae0c-7b71-bd67-d713e599aef5 Address:5cfa8c24-ae0c-7b71-bd67-d713e599aef5}]" -2026-03-31T06:16:37.038Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:37.038Z [INFO] server-c5786428-a93a-d807-cfdf-81d45b7caf1e: entering follower state: follower="Node at c5786428-a93a-d807-cfdf-81d45b7caf1e [Follower]" leader-address= leader-id= -2026-03-31T06:16:37.038Z [INFO] server-5cfa8c24-ae0c-7b71-bd67-d713e599aef5: entering follower state: follower="Node at 5cfa8c24-ae0c-7b71-bd67-d713e599aef5 [Follower]" leader-address= leader-id= -2026-03-31T06:16:37.789Z [WARN] server-5cfa8c24-ae0c-7b71-bd67-d713e599aef5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:37.789Z [INFO] server-5cfa8c24-ae0c-7b71-bd67-d713e599aef5: entering candidate state: node="Node at 5cfa8c24-ae0c-7b71-bd67-d713e599aef5 [Candidate]" term=2 -2026-03-31T06:16:37.789Z [INFO] server-5cfa8c24-ae0c-7b71-bd67-d713e599aef5: election won: term=2 tally=2 -2026-03-31T06:16:37.789Z [INFO] server-5cfa8c24-ae0c-7b71-bd67-d713e599aef5: entering leader state: leader="Node at 5cfa8c24-ae0c-7b71-bd67-d713e599aef5 [Leader]" -2026-03-31T06:16:37.789Z [INFO] server-5cfa8c24-ae0c-7b71-bd67-d713e599aef5: added peer, starting replication: peer=server-f990da9c-3bb4-1284-fe64-6b865bb760c8 -2026-03-31T06:16:37.789Z [INFO] server-5cfa8c24-ae0c-7b71-bd67-d713e599aef5: added peer, starting replication: peer=server-c5786428-a93a-d807-cfdf-81d45b7caf1e -2026-03-31T06:16:37.789Z [INFO] server-5cfa8c24-ae0c-7b71-bd67-d713e599aef5: pipelining replication: peer="{Voter server-f990da9c-3bb4-1284-fe64-6b865bb760c8 f990da9c-3bb4-1284-fe64-6b865bb760c8}" -2026-03-31T06:16:37.789Z [INFO] server-5cfa8c24-ae0c-7b71-bd67-d713e599aef5: pipelining replication: peer="{Voter server-c5786428-a93a-d807-cfdf-81d45b7caf1e c5786428-a93a-d807-cfdf-81d45b7caf1e}" -2026-03-31T06:16:38.794Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:37.78958926 +0000 UTC m=+26.067196641 (1 nodes), 751.0266ms from start of poll, 751.44592ms from cluster start. Timeout at 2026-03-31 06:16:38.79459046 +0000 UTC m=+27.072197881, 1.00500124s after stability -2026-03-31T06:16:38.795Z [INFO] creating new snapshot: path=/tmp/raft1541100578/snapshots/2-13-1774937798795.tmp -2026-03-31T06:16:38.795Z [INFO] server-5cfa8c24-ae0c-7b71-bd67-d713e599aef5: copied to local snapshot: bytes=71 -2026-03-31T06:16:38.795Z [INFO] server-5cfa8c24-ae0c-7b71-bd67-d713e599aef5: snapshot restore progress: id=2-13-1774937798795 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:38.795Z [INFO] server-5cfa8c24-ae0c-7b71-bd67-d713e599aef5: removing all old logs from log store -2026-03-31T06:16:38.795Z [INFO] server-5cfa8c24-ae0c-7b71-bd67-d713e599aef5: compacting logs: from=1 to=2 -2026-03-31T06:16:38.795Z [INFO] server-5cfa8c24-ae0c-7b71-bd67-d713e599aef5: restored user snapshot: index=13 -2026-03-31T06:16:38.796Z [ERROR] server-5cfa8c24-ae0c-7b71-bd67-d713e599aef5: failed to get log: index=2 error="log not found" -2026-03-31T06:16:38.796Z [INFO] server-5cfa8c24-ae0c-7b71-bd67-d713e599aef5: aborting pipeline replication: peer="{Voter server-f990da9c-3bb4-1284-fe64-6b865bb760c8 f990da9c-3bb4-1284-fe64-6b865bb760c8}" -2026-03-31T06:16:38.796Z [ERROR] server-5cfa8c24-ae0c-7b71-bd67-d713e599aef5: failed to get log: index=2 error="log not found" -2026-03-31T06:16:38.796Z [INFO] server-5cfa8c24-ae0c-7b71-bd67-d713e599aef5: aborting pipeline replication: peer="{Voter server-c5786428-a93a-d807-cfdf-81d45b7caf1e c5786428-a93a-d807-cfdf-81d45b7caf1e}" -2026-03-31T06:16:38.802Z [ERROR] server-5cfa8c24-ae0c-7b71-bd67-d713e599aef5: failed to get log: index=2 error="log not found" -2026-03-31T06:16:38.802Z [INFO] creating new snapshot: path=/tmp/raft255633866/snapshots/2-13-1774937798802.tmp -2026-03-31T06:16:38.802Z [INFO] server-c5786428-a93a-d807-cfdf-81d45b7caf1e: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:38.803Z [INFO] server-c5786428-a93a-d807-cfdf-81d45b7caf1e: copied to local snapshot: bytes=71 -2026-03-31T06:16:38.803Z [INFO] server-c5786428-a93a-d807-cfdf-81d45b7caf1e: snapshot restore progress: id=2-13-1774937798802 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:38.803Z [INFO] server-c5786428-a93a-d807-cfdf-81d45b7caf1e: removing all old logs from log store -2026-03-31T06:16:38.803Z [INFO] server-c5786428-a93a-d807-cfdf-81d45b7caf1e: compacting logs: from=1 to=2 -2026-03-31T06:16:38.803Z [INFO] server-c5786428-a93a-d807-cfdf-81d45b7caf1e: Installed remote snapshot -2026-03-31T06:16:38.803Z [INFO] server-5cfa8c24-ae0c-7b71-bd67-d713e599aef5: pipelining replication: peer="{Voter server-c5786428-a93a-d807-cfdf-81d45b7caf1e c5786428-a93a-d807-cfdf-81d45b7caf1e}" -2026-03-31T06:16:38.803Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:16:38.803Z [ERROR] server-5cfa8c24-ae0c-7b71-bd67-d713e599aef5: failed to get log: index=2 error="log not found" -2026-03-31T06:16:38.804Z [INFO] creating new snapshot: path=/tmp/raft3581378896/snapshots/2-13-1774937798804.tmp -2026-03-31T06:16:38.804Z [INFO] server-f990da9c-3bb4-1284-fe64-6b865bb760c8: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:38.804Z [INFO] server-f990da9c-3bb4-1284-fe64-6b865bb760c8: copied to local snapshot: bytes=71 -2026-03-31T06:16:38.804Z [INFO] server-f990da9c-3bb4-1284-fe64-6b865bb760c8: snapshot restore progress: id=2-13-1774937798804 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:38.805Z [INFO] server-f990da9c-3bb4-1284-fe64-6b865bb760c8: removing all old logs from log store -2026-03-31T06:16:38.805Z [INFO] server-f990da9c-3bb4-1284-fe64-6b865bb760c8: compacting logs: from=1 to=2 -2026-03-31T06:16:38.805Z [INFO] server-f990da9c-3bb4-1284-fe64-6b865bb760c8: Installed remote snapshot -2026-03-31T06:16:38.805Z [INFO] server-5cfa8c24-ae0c-7b71-bd67-d713e599aef5: pipelining replication: peer="{Voter server-f990da9c-3bb4-1284-fe64-6b865bb760c8 f990da9c-3bb4-1284-fe64-6b865bb760c8}" -2026-03-31T06:16:39.809Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:16:38.80367718 +0000 UTC m=+27.081284561 (2 nodes), 720ns from start of poll, 1.76553384s from cluster start. Timeout at 2026-03-31 06:16:39.8086885 +0000 UTC m=+28.086295881, 1.00501132s after stability -2026-03-31T06:16:39.821Z [INFO] server-d1bf814d-82db-7f77-9976-78a3241bcb93: aborting pipeline replication: peer="{Voter server-4c12f43f-dd8d-f129-e09f-06cc29032f6e 4c12f43f-dd8d-f129-e09f-06cc29032f6e}" -2026-03-31T06:16:39.821Z [INFO] server-d1bf814d-82db-7f77-9976-78a3241bcb93: aborting pipeline replication: peer="{Voter server-e59c77f4-52fb-a5b4-e2ab-7614508a7ad7 e59c77f4-52fb-a5b4-e2ab-7614508a7ad7}" +2025-02-25T23:57:57.322Z [INFO] server-a3bd9110-94f5-52a8-4035-41880b86379d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a3bd9110-94f5-52a8-4035-41880b86379d Address:a3bd9110-94f5-52a8-4035-41880b86379d} {Suffrage:Voter ID:server-ec20e5d0-1daa-6272-46ac-7204606c82e5 Address:ec20e5d0-1daa-6272-46ac-7204606c82e5} {Suffrage:Voter ID:server-fabc5617-dc70-4a04-3acd-72b2a059950a Address:fabc5617-dc70-4a04-3acd-72b2a059950a}]" +2025-02-25T23:57:57.322Z [INFO] server-ec20e5d0-1daa-6272-46ac-7204606c82e5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a3bd9110-94f5-52a8-4035-41880b86379d Address:a3bd9110-94f5-52a8-4035-41880b86379d} {Suffrage:Voter ID:server-ec20e5d0-1daa-6272-46ac-7204606c82e5 Address:ec20e5d0-1daa-6272-46ac-7204606c82e5} {Suffrage:Voter ID:server-fabc5617-dc70-4a04-3acd-72b2a059950a Address:fabc5617-dc70-4a04-3acd-72b2a059950a}]" +2025-02-25T23:57:57.322Z [INFO] server-fabc5617-dc70-4a04-3acd-72b2a059950a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a3bd9110-94f5-52a8-4035-41880b86379d Address:a3bd9110-94f5-52a8-4035-41880b86379d} {Suffrage:Voter ID:server-ec20e5d0-1daa-6272-46ac-7204606c82e5 Address:ec20e5d0-1daa-6272-46ac-7204606c82e5} {Suffrage:Voter ID:server-fabc5617-dc70-4a04-3acd-72b2a059950a Address:fabc5617-dc70-4a04-3acd-72b2a059950a}]" +2025-02-25T23:57:57.322Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:57.322Z [INFO] server-a3bd9110-94f5-52a8-4035-41880b86379d: entering follower state: follower="Node at a3bd9110-94f5-52a8-4035-41880b86379d [Follower]" leader-address= leader-id= +2025-02-25T23:57:57.322Z [INFO] server-ec20e5d0-1daa-6272-46ac-7204606c82e5: entering follower state: follower="Node at ec20e5d0-1daa-6272-46ac-7204606c82e5 [Follower]" leader-address= leader-id= +2025-02-25T23:57:57.322Z [INFO] server-fabc5617-dc70-4a04-3acd-72b2a059950a: entering follower state: follower="Node at fabc5617-dc70-4a04-3acd-72b2a059950a [Follower]" leader-address= leader-id= +2025-02-25T23:57:57.905Z [WARN] server-a3bd9110-94f5-52a8-4035-41880b86379d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:57.905Z [INFO] server-a3bd9110-94f5-52a8-4035-41880b86379d: entering candidate state: node="Node at a3bd9110-94f5-52a8-4035-41880b86379d [Candidate]" term=2 +2025-02-25T23:57:57.905Z [INFO] server-a3bd9110-94f5-52a8-4035-41880b86379d: election won: term=2 tally=2 +2025-02-25T23:57:57.905Z [INFO] server-a3bd9110-94f5-52a8-4035-41880b86379d: entering leader state: leader="Node at a3bd9110-94f5-52a8-4035-41880b86379d [Leader]" +2025-02-25T23:57:57.906Z [INFO] server-a3bd9110-94f5-52a8-4035-41880b86379d: added peer, starting replication: peer=server-ec20e5d0-1daa-6272-46ac-7204606c82e5 +2025-02-25T23:57:57.906Z [INFO] server-a3bd9110-94f5-52a8-4035-41880b86379d: added peer, starting replication: peer=server-fabc5617-dc70-4a04-3acd-72b2a059950a +2025-02-25T23:57:57.906Z [INFO] server-a3bd9110-94f5-52a8-4035-41880b86379d: pipelining replication: peer="{Voter server-fabc5617-dc70-4a04-3acd-72b2a059950a fabc5617-dc70-4a04-3acd-72b2a059950a}" +2025-02-25T23:57:57.906Z [INFO] server-a3bd9110-94f5-52a8-4035-41880b86379d: pipelining replication: peer="{Voter server-ec20e5d0-1daa-6272-46ac-7204606c82e5 ec20e5d0-1daa-6272-46ac-7204606c82e5}" +2025-02-25T23:57:58.913Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:57.90619886 +0000 UTC m=+25.515214267 (1 nodes), 583.679677ms from start of poll, 583.851478ms from cluster start. Timeout at 2025-02-25 23:57:58.91120186 +0000 UTC m=+26.520217267, 1.005003s after stability +2025-02-25T23:57:58.914Z [INFO] server-a3bd9110-94f5-52a8-4035-41880b86379d: starting snapshot up to: index=12 +2025-02-25T23:57:58.914Z [INFO] creating new snapshot: path=/tmp/raft2292562646/snapshots/2-12-1740527878914.tmp +2025-02-25T23:57:58.914Z [INFO] server-a3bd9110-94f5-52a8-4035-41880b86379d: snapshot complete up to: index=12 +2025-02-25T23:57:58.915Z [INFO] server-c39da356-161b-7f15-06f8-4bd84cc3a52f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c39da356-161b-7f15-06f8-4bd84cc3a52f Address:c39da356-161b-7f15-06f8-4bd84cc3a52f} {Suffrage:Voter ID:server-e5bdf986-c20c-d599-8489-ac274394150d Address:e5bdf986-c20c-d599-8489-ac274394150d} {Suffrage:Voter ID:server-7fce6688-a1a0-1d4b-3426-a6e4c31d44e5 Address:7fce6688-a1a0-1d4b-3426-a6e4c31d44e5}]" +2025-02-25T23:57:58.915Z [INFO] server-e5bdf986-c20c-d599-8489-ac274394150d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c39da356-161b-7f15-06f8-4bd84cc3a52f Address:c39da356-161b-7f15-06f8-4bd84cc3a52f} {Suffrage:Voter ID:server-e5bdf986-c20c-d599-8489-ac274394150d Address:e5bdf986-c20c-d599-8489-ac274394150d} {Suffrage:Voter ID:server-7fce6688-a1a0-1d4b-3426-a6e4c31d44e5 Address:7fce6688-a1a0-1d4b-3426-a6e4c31d44e5}]" +2025-02-25T23:57:58.915Z [INFO] server-7fce6688-a1a0-1d4b-3426-a6e4c31d44e5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c39da356-161b-7f15-06f8-4bd84cc3a52f Address:c39da356-161b-7f15-06f8-4bd84cc3a52f} {Suffrage:Voter ID:server-e5bdf986-c20c-d599-8489-ac274394150d Address:e5bdf986-c20c-d599-8489-ac274394150d} {Suffrage:Voter ID:server-7fce6688-a1a0-1d4b-3426-a6e4c31d44e5 Address:7fce6688-a1a0-1d4b-3426-a6e4c31d44e5}]" +2025-02-25T23:57:58.915Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:57:58.915Z [INFO] server-c39da356-161b-7f15-06f8-4bd84cc3a52f: entering follower state: follower="Node at c39da356-161b-7f15-06f8-4bd84cc3a52f [Follower]" leader-address= leader-id= +2025-02-25T23:57:58.915Z [INFO] server-e5bdf986-c20c-d599-8489-ac274394150d: entering follower state: follower="Node at e5bdf986-c20c-d599-8489-ac274394150d [Follower]" leader-address= leader-id= +2025-02-25T23:57:58.915Z [INFO] server-7fce6688-a1a0-1d4b-3426-a6e4c31d44e5: entering follower state: follower="Node at 7fce6688-a1a0-1d4b-3426-a6e4c31d44e5 [Follower]" leader-address= leader-id= +2025-02-25T23:57:59.505Z [WARN] server-e5bdf986-c20c-d599-8489-ac274394150d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:57:59.505Z [INFO] server-e5bdf986-c20c-d599-8489-ac274394150d: entering candidate state: node="Node at e5bdf986-c20c-d599-8489-ac274394150d [Candidate]" term=2 +2025-02-25T23:57:59.505Z [INFO] server-e5bdf986-c20c-d599-8489-ac274394150d: election won: term=2 tally=2 +2025-02-25T23:57:59.505Z [INFO] server-e5bdf986-c20c-d599-8489-ac274394150d: entering leader state: leader="Node at e5bdf986-c20c-d599-8489-ac274394150d [Leader]" +2025-02-25T23:57:59.505Z [INFO] server-e5bdf986-c20c-d599-8489-ac274394150d: added peer, starting replication: peer=server-c39da356-161b-7f15-06f8-4bd84cc3a52f +2025-02-25T23:57:59.506Z [INFO] server-e5bdf986-c20c-d599-8489-ac274394150d: added peer, starting replication: peer=server-7fce6688-a1a0-1d4b-3426-a6e4c31d44e5 +2025-02-25T23:57:59.506Z [INFO] server-e5bdf986-c20c-d599-8489-ac274394150d: pipelining replication: peer="{Voter server-c39da356-161b-7f15-06f8-4bd84cc3a52f c39da356-161b-7f15-06f8-4bd84cc3a52f}" +2025-02-25T23:57:59.506Z [INFO] server-e5bdf986-c20c-d599-8489-ac274394150d: pipelining replication: peer="{Voter server-7fce6688-a1a0-1d4b-3426-a6e4c31d44e5 7fce6688-a1a0-1d4b-3426-a6e4c31d44e5}" +2025-02-25T23:58:00.513Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:57:59.50622285 +0000 UTC m=+27.115238217 (1 nodes), 590.994164ms from start of poll, 591.160285ms from cluster start. Timeout at 2025-02-25 23:58:00.51122925 +0000 UTC m=+28.120244617, 1.0050064s after stability +2025-02-25T23:58:00.514Z [INFO] creating new snapshot: path=/tmp/raft945658895/snapshots/2-13-1740527880514.tmp +2025-02-25T23:58:00.514Z [INFO] server-e5bdf986-c20c-d599-8489-ac274394150d: copied to local snapshot: bytes=71 +2025-02-25T23:58:00.514Z [INFO] server-e5bdf986-c20c-d599-8489-ac274394150d: snapshot restore progress: id=2-13-1740527880514 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:00.514Z [INFO] server-e5bdf986-c20c-d599-8489-ac274394150d: removing all old logs from log store +2025-02-25T23:58:00.514Z [INFO] server-e5bdf986-c20c-d599-8489-ac274394150d: compacting logs: from=1 to=2 +2025-02-25T23:58:00.514Z [INFO] server-e5bdf986-c20c-d599-8489-ac274394150d: restored user snapshot: index=13 +2025-02-25T23:58:00.514Z [ERROR] server-e5bdf986-c20c-d599-8489-ac274394150d: failed to get log: index=2 error="log not found" +2025-02-25T23:58:00.514Z [INFO] server-e5bdf986-c20c-d599-8489-ac274394150d: aborting pipeline replication: peer="{Voter server-7fce6688-a1a0-1d4b-3426-a6e4c31d44e5 7fce6688-a1a0-1d4b-3426-a6e4c31d44e5}" +2025-02-25T23:58:00.514Z [ERROR] server-e5bdf986-c20c-d599-8489-ac274394150d: failed to get log: index=2 error="log not found" +2025-02-25T23:58:00.514Z [INFO] server-e5bdf986-c20c-d599-8489-ac274394150d: aborting pipeline replication: peer="{Voter server-c39da356-161b-7f15-06f8-4bd84cc3a52f c39da356-161b-7f15-06f8-4bd84cc3a52f}" +2025-02-25T23:58:00.525Z [ERROR] server-e5bdf986-c20c-d599-8489-ac274394150d: failed to get log: index=2 error="log not found" +2025-02-25T23:58:00.526Z [INFO] creating new snapshot: path=/tmp/raft3982867083/snapshots/2-13-1740527880526.tmp +2025-02-25T23:58:00.526Z [INFO] server-c39da356-161b-7f15-06f8-4bd84cc3a52f: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:00.526Z [INFO] server-c39da356-161b-7f15-06f8-4bd84cc3a52f: copied to local snapshot: bytes=71 +2025-02-25T23:58:00.526Z [INFO] server-c39da356-161b-7f15-06f8-4bd84cc3a52f: snapshot restore progress: id=2-13-1740527880526 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:00.526Z [INFO] server-c39da356-161b-7f15-06f8-4bd84cc3a52f: removing all old logs from log store +2025-02-25T23:58:00.526Z [INFO] server-c39da356-161b-7f15-06f8-4bd84cc3a52f: compacting logs: from=1 to=2 +2025-02-25T23:58:00.526Z [INFO] server-c39da356-161b-7f15-06f8-4bd84cc3a52f: Installed remote snapshot +2025-02-25T23:58:00.526Z [INFO] server-e5bdf986-c20c-d599-8489-ac274394150d: pipelining replication: peer="{Voter server-c39da356-161b-7f15-06f8-4bd84cc3a52f c39da356-161b-7f15-06f8-4bd84cc3a52f}" +2025-02-25T23:58:00.526Z [ERROR] server-e5bdf986-c20c-d599-8489-ac274394150d: failed to get log: index=2 error="log not found" +2025-02-25T23:58:00.526Z [INFO] creating new snapshot: path=/tmp/raft3388394152/snapshots/2-13-1740527880526.tmp +2025-02-25T23:58:00.526Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:58:00.526Z [INFO] server-7fce6688-a1a0-1d4b-3426-a6e4c31d44e5: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:00.526Z [INFO] server-7fce6688-a1a0-1d4b-3426-a6e4c31d44e5: copied to local snapshot: bytes=71 +2025-02-25T23:58:00.527Z [INFO] server-7fce6688-a1a0-1d4b-3426-a6e4c31d44e5: snapshot restore progress: id=2-13-1740527880526 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:00.527Z [INFO] server-7fce6688-a1a0-1d4b-3426-a6e4c31d44e5: removing all old logs from log store +2025-02-25T23:58:00.527Z [INFO] server-7fce6688-a1a0-1d4b-3426-a6e4c31d44e5: compacting logs: from=1 to=2 +2025-02-25T23:58:00.527Z [INFO] server-7fce6688-a1a0-1d4b-3426-a6e4c31d44e5: Installed remote snapshot +2025-02-25T23:58:00.527Z [INFO] server-e5bdf986-c20c-d599-8489-ac274394150d: pipelining replication: peer="{Voter server-7fce6688-a1a0-1d4b-3426-a6e4c31d44e5 7fce6688-a1a0-1d4b-3426-a6e4c31d44e5}" +2025-02-25T23:58:01.533Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:58:00.52685919 +0000 UTC m=+28.135874597 (2 nodes), 400ns from start of poll, 1.611796665s from cluster start. Timeout at 2025-02-25 23:58:01.53186967 +0000 UTC m=+29.140885077, 1.00501048s after stability +2025-02-25T23:58:01.549Z [INFO] server-a3bd9110-94f5-52a8-4035-41880b86379d: aborting pipeline replication: peer="{Voter server-fabc5617-dc70-4a04-3acd-72b2a059950a fabc5617-dc70-4a04-3acd-72b2a059950a}" +2025-02-25T23:58:01.549Z [INFO] server-a3bd9110-94f5-52a8-4035-41880b86379d: aborting pipeline replication: peer="{Voter server-ec20e5d0-1daa-6272-46ac-7204606c82e5 ec20e5d0-1daa-6272-46ac-7204606c82e5}" === RUN TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_false -2026-03-31T06:16:39.823Z [INFO] server-5c34ba5d-1a3d-4d4e-087e-17def3913c15: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5c34ba5d-1a3d-4d4e-087e-17def3913c15 Address:5c34ba5d-1a3d-4d4e-087e-17def3913c15} {Suffrage:Voter ID:server-b2bae6bb-39c9-371e-0aa6-d6cd2d6cac5c Address:b2bae6bb-39c9-371e-0aa6-d6cd2d6cac5c} {Suffrage:Voter ID:server-0be0a92d-d930-68fe-91b9-7a5ab0c0d00a Address:0be0a92d-d930-68fe-91b9-7a5ab0c0d00a}]" -2026-03-31T06:16:39.823Z [INFO] server-b2bae6bb-39c9-371e-0aa6-d6cd2d6cac5c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5c34ba5d-1a3d-4d4e-087e-17def3913c15 Address:5c34ba5d-1a3d-4d4e-087e-17def3913c15} {Suffrage:Voter ID:server-b2bae6bb-39c9-371e-0aa6-d6cd2d6cac5c Address:b2bae6bb-39c9-371e-0aa6-d6cd2d6cac5c} {Suffrage:Voter ID:server-0be0a92d-d930-68fe-91b9-7a5ab0c0d00a Address:0be0a92d-d930-68fe-91b9-7a5ab0c0d00a}]" -2026-03-31T06:16:39.823Z [INFO] server-5c34ba5d-1a3d-4d4e-087e-17def3913c15: entering follower state: follower="Node at 5c34ba5d-1a3d-4d4e-087e-17def3913c15 [Follower]" leader-address= leader-id= -2026-03-31T06:16:39.823Z [INFO] server-b2bae6bb-39c9-371e-0aa6-d6cd2d6cac5c: entering follower state: follower="Node at b2bae6bb-39c9-371e-0aa6-d6cd2d6cac5c [Follower]" leader-address= leader-id= -2026-03-31T06:16:39.823Z [INFO] server-0be0a92d-d930-68fe-91b9-7a5ab0c0d00a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5c34ba5d-1a3d-4d4e-087e-17def3913c15 Address:5c34ba5d-1a3d-4d4e-087e-17def3913c15} {Suffrage:Voter ID:server-b2bae6bb-39c9-371e-0aa6-d6cd2d6cac5c Address:b2bae6bb-39c9-371e-0aa6-d6cd2d6cac5c} {Suffrage:Voter ID:server-0be0a92d-d930-68fe-91b9-7a5ab0c0d00a Address:0be0a92d-d930-68fe-91b9-7a5ab0c0d00a}]" -2026-03-31T06:16:39.823Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:39.823Z [INFO] server-0be0a92d-d930-68fe-91b9-7a5ab0c0d00a: entering follower state: follower="Node at 0be0a92d-d930-68fe-91b9-7a5ab0c0d00a [Follower]" leader-address= leader-id= -2026-03-31T06:16:40.500Z [WARN] server-5c34ba5d-1a3d-4d4e-087e-17def3913c15: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:40.501Z [INFO] server-5c34ba5d-1a3d-4d4e-087e-17def3913c15: entering candidate state: node="Node at 5c34ba5d-1a3d-4d4e-087e-17def3913c15 [Candidate]" term=2 -2026-03-31T06:16:40.501Z [INFO] server-5c34ba5d-1a3d-4d4e-087e-17def3913c15: election won: term=2 tally=2 -2026-03-31T06:16:40.501Z [INFO] server-5c34ba5d-1a3d-4d4e-087e-17def3913c15: entering leader state: leader="Node at 5c34ba5d-1a3d-4d4e-087e-17def3913c15 [Leader]" -2026-03-31T06:16:40.501Z [INFO] server-5c34ba5d-1a3d-4d4e-087e-17def3913c15: added peer, starting replication: peer=server-b2bae6bb-39c9-371e-0aa6-d6cd2d6cac5c -2026-03-31T06:16:40.501Z [INFO] server-5c34ba5d-1a3d-4d4e-087e-17def3913c15: added peer, starting replication: peer=server-0be0a92d-d930-68fe-91b9-7a5ab0c0d00a -2026-03-31T06:16:40.501Z [INFO] server-5c34ba5d-1a3d-4d4e-087e-17def3913c15: pipelining replication: peer="{Voter server-b2bae6bb-39c9-371e-0aa6-d6cd2d6cac5c b2bae6bb-39c9-371e-0aa6-d6cd2d6cac5c}" -2026-03-31T06:16:40.501Z [INFO] server-5c34ba5d-1a3d-4d4e-087e-17def3913c15: pipelining replication: peer="{Voter server-0be0a92d-d930-68fe-91b9-7a5ab0c0d00a 0be0a92d-d930-68fe-91b9-7a5ab0c0d00a}" -2026-03-31T06:16:41.507Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:40.50130974 +0000 UTC m=+28.778917121 (1 nodes), 677.496ms from start of poll, 678.51804ms from cluster start. Timeout at 2026-03-31 06:16:41.50631414 +0000 UTC m=+29.783921521, 1.0050044s after stability -2026-03-31T06:16:41.507Z [INFO] server-5c34ba5d-1a3d-4d4e-087e-17def3913c15: starting snapshot up to: index=12 -2026-03-31T06:16:41.507Z [INFO] creating new snapshot: path=/tmp/raft1896642446/snapshots/2-12-1774937801507.tmp -2026-03-31T06:16:41.508Z [INFO] server-5c34ba5d-1a3d-4d4e-087e-17def3913c15: snapshot complete up to: index=12 -2026-03-31T06:16:41.509Z [INFO] creating new snapshot: path=/tmp/raft1896642446/snapshots/2-23-1774937801509.tmp -2026-03-31T06:16:41.509Z [INFO] server-5c34ba5d-1a3d-4d4e-087e-17def3913c15: copied to local snapshot: bytes=71 -2026-03-31T06:16:41.510Z [INFO] server-5c34ba5d-1a3d-4d4e-087e-17def3913c15: snapshot restore progress: id=2-23-1774937801509 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:41.510Z [INFO] server-5c34ba5d-1a3d-4d4e-087e-17def3913c15: restored user snapshot: index=23 -2026-03-31T06:16:41.511Z [ERROR] server-5c34ba5d-1a3d-4d4e-087e-17def3913c15: failed to get log: index=23 error="log not found" -2026-03-31T06:16:41.511Z [INFO] server-5c34ba5d-1a3d-4d4e-087e-17def3913c15: aborting pipeline replication: peer="{Voter server-0be0a92d-d930-68fe-91b9-7a5ab0c0d00a 0be0a92d-d930-68fe-91b9-7a5ab0c0d00a}" -2026-03-31T06:16:41.511Z [ERROR] server-5c34ba5d-1a3d-4d4e-087e-17def3913c15: failed to get log: index=23 error="log not found" -2026-03-31T06:16:41.511Z [INFO] server-5c34ba5d-1a3d-4d4e-087e-17def3913c15: aborting pipeline replication: peer="{Voter server-b2bae6bb-39c9-371e-0aa6-d6cd2d6cac5c b2bae6bb-39c9-371e-0aa6-d6cd2d6cac5c}" -2026-03-31T06:16:41.517Z [ERROR] server-5c34ba5d-1a3d-4d4e-087e-17def3913c15: failed to get log: index=23 error="log not found" -2026-03-31T06:16:41.517Z [INFO] creating new snapshot: path=/tmp/raft1122762306/snapshots/2-23-1774937801517.tmp -2026-03-31T06:16:41.518Z [INFO] server-0be0a92d-d930-68fe-91b9-7a5ab0c0d00a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:41.518Z [INFO] server-0be0a92d-d930-68fe-91b9-7a5ab0c0d00a: copied to local snapshot: bytes=71 -2026-03-31T06:16:41.518Z [ERROR] server-5c34ba5d-1a3d-4d4e-087e-17def3913c15: failed to get log: index=23 error="log not found" -2026-03-31T06:16:41.518Z [INFO] server-0be0a92d-d930-68fe-91b9-7a5ab0c0d00a: snapshot restore progress: id=2-23-1774937801517 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:41.518Z [INFO] server-0be0a92d-d930-68fe-91b9-7a5ab0c0d00a: Installed remote snapshot -2026-03-31T06:16:41.518Z [WARN] server-0be0a92d-d930-68fe-91b9-7a5ab0c0d00a: skipping application of old log: index=22 -2026-03-31T06:16:41.518Z [INFO] server-5c34ba5d-1a3d-4d4e-087e-17def3913c15: pipelining replication: peer="{Voter server-0be0a92d-d930-68fe-91b9-7a5ab0c0d00a 0be0a92d-d930-68fe-91b9-7a5ab0c0d00a}" -2026-03-31T06:16:41.518Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:16:41.518Z [INFO] creating new snapshot: path=/tmp/raft2636022118/snapshots/2-23-1774937801518.tmp -2026-03-31T06:16:41.519Z [INFO] server-b2bae6bb-39c9-371e-0aa6-d6cd2d6cac5c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:41.519Z [INFO] server-b2bae6bb-39c9-371e-0aa6-d6cd2d6cac5c: copied to local snapshot: bytes=71 -2026-03-31T06:16:41.519Z [INFO] server-b2bae6bb-39c9-371e-0aa6-d6cd2d6cac5c: snapshot restore progress: id=2-23-1774937801518 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:41.519Z [INFO] server-b2bae6bb-39c9-371e-0aa6-d6cd2d6cac5c: Installed remote snapshot -2026-03-31T06:16:41.519Z [INFO] server-5c34ba5d-1a3d-4d4e-087e-17def3913c15: pipelining replication: peer="{Voter server-b2bae6bb-39c9-371e-0aa6-d6cd2d6cac5c b2bae6bb-39c9-371e-0aa6-d6cd2d6cac5c}" -2026-03-31T06:16:42.524Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:16:41.51883926 +0000 UTC m=+29.796446641 (2 nodes), 920ns from start of poll, 1.69604756s from cluster start. Timeout at 2026-03-31 06:16:42.5238473 +0000 UTC m=+30.801454681, 1.00500804s after stability -2026-03-31T06:16:42.538Z [INFO] server-5c34ba5d-1a3d-4d4e-087e-17def3913c15: aborting pipeline replication: peer="{Voter server-0be0a92d-d930-68fe-91b9-7a5ab0c0d00a 0be0a92d-d930-68fe-91b9-7a5ab0c0d00a}" -2026-03-31T06:16:42.538Z [INFO] server-5c34ba5d-1a3d-4d4e-087e-17def3913c15: aborting pipeline replication: peer="{Voter server-b2bae6bb-39c9-371e-0aa6-d6cd2d6cac5c b2bae6bb-39c9-371e-0aa6-d6cd2d6cac5c}" +2025-02-25T23:58:01.550Z [INFO] server-ed8934a7-6884-151a-5ec2-82e70380d25e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ed8934a7-6884-151a-5ec2-82e70380d25e Address:ed8934a7-6884-151a-5ec2-82e70380d25e} {Suffrage:Voter ID:server-9ca884d7-829b-df16-c309-972480951d71 Address:9ca884d7-829b-df16-c309-972480951d71} {Suffrage:Voter ID:server-ce1368fe-626a-4ac1-98bc-b0c6f241c120 Address:ce1368fe-626a-4ac1-98bc-b0c6f241c120}]" +2025-02-25T23:58:01.550Z [INFO] server-9ca884d7-829b-df16-c309-972480951d71: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ed8934a7-6884-151a-5ec2-82e70380d25e Address:ed8934a7-6884-151a-5ec2-82e70380d25e} {Suffrage:Voter ID:server-9ca884d7-829b-df16-c309-972480951d71 Address:9ca884d7-829b-df16-c309-972480951d71} {Suffrage:Voter ID:server-ce1368fe-626a-4ac1-98bc-b0c6f241c120 Address:ce1368fe-626a-4ac1-98bc-b0c6f241c120}]" +2025-02-25T23:58:01.550Z [INFO] server-ce1368fe-626a-4ac1-98bc-b0c6f241c120: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ed8934a7-6884-151a-5ec2-82e70380d25e Address:ed8934a7-6884-151a-5ec2-82e70380d25e} {Suffrage:Voter ID:server-9ca884d7-829b-df16-c309-972480951d71 Address:9ca884d7-829b-df16-c309-972480951d71} {Suffrage:Voter ID:server-ce1368fe-626a-4ac1-98bc-b0c6f241c120 Address:ce1368fe-626a-4ac1-98bc-b0c6f241c120}]" +2025-02-25T23:58:01.550Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:58:01.550Z [INFO] server-ed8934a7-6884-151a-5ec2-82e70380d25e: entering follower state: follower="Node at ed8934a7-6884-151a-5ec2-82e70380d25e [Follower]" leader-address= leader-id= +2025-02-25T23:58:01.550Z [INFO] server-9ca884d7-829b-df16-c309-972480951d71: entering follower state: follower="Node at 9ca884d7-829b-df16-c309-972480951d71 [Follower]" leader-address= leader-id= +2025-02-25T23:58:01.550Z [INFO] server-ce1368fe-626a-4ac1-98bc-b0c6f241c120: entering follower state: follower="Node at ce1368fe-626a-4ac1-98bc-b0c6f241c120 [Follower]" leader-address= leader-id= +2025-02-25T23:58:02.229Z [WARN] server-9ca884d7-829b-df16-c309-972480951d71: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:58:02.230Z [INFO] server-9ca884d7-829b-df16-c309-972480951d71: entering candidate state: node="Node at 9ca884d7-829b-df16-c309-972480951d71 [Candidate]" term=2 +2025-02-25T23:58:02.230Z [INFO] server-9ca884d7-829b-df16-c309-972480951d71: election won: term=2 tally=2 +2025-02-25T23:58:02.230Z [INFO] server-9ca884d7-829b-df16-c309-972480951d71: entering leader state: leader="Node at 9ca884d7-829b-df16-c309-972480951d71 [Leader]" +2025-02-25T23:58:02.230Z [INFO] server-9ca884d7-829b-df16-c309-972480951d71: added peer, starting replication: peer=server-ed8934a7-6884-151a-5ec2-82e70380d25e +2025-02-25T23:58:02.230Z [INFO] server-9ca884d7-829b-df16-c309-972480951d71: added peer, starting replication: peer=server-ce1368fe-626a-4ac1-98bc-b0c6f241c120 +2025-02-25T23:58:02.230Z [INFO] server-9ca884d7-829b-df16-c309-972480951d71: pipelining replication: peer="{Voter server-ce1368fe-626a-4ac1-98bc-b0c6f241c120 ce1368fe-626a-4ac1-98bc-b0c6f241c120}" +2025-02-25T23:58:02.230Z [INFO] server-9ca884d7-829b-df16-c309-972480951d71: pipelining replication: peer="{Voter server-ed8934a7-6884-151a-5ec2-82e70380d25e ed8934a7-6884-151a-5ec2-82e70380d25e}" +2025-02-25T23:58:03.238Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:58:02.230307918 +0000 UTC m=+29.839323285 (1 nodes), 679.765009ms from start of poll, 679.94141ms from cluster start. Timeout at 2025-02-25 23:58:03.235310758 +0000 UTC m=+30.844326125, 1.00500284s after stability +2025-02-25T23:58:03.238Z [INFO] server-9ca884d7-829b-df16-c309-972480951d71: starting snapshot up to: index=12 +2025-02-25T23:58:03.238Z [INFO] creating new snapshot: path=/tmp/raft493182164/snapshots/2-12-1740527883238.tmp +2025-02-25T23:58:03.238Z [INFO] server-9ca884d7-829b-df16-c309-972480951d71: snapshot complete up to: index=12 +2025-02-25T23:58:03.239Z [INFO] creating new snapshot: path=/tmp/raft493182164/snapshots/2-23-1740527883239.tmp +2025-02-25T23:58:03.239Z [INFO] server-9ca884d7-829b-df16-c309-972480951d71: copied to local snapshot: bytes=71 +2025-02-25T23:58:03.239Z [INFO] server-9ca884d7-829b-df16-c309-972480951d71: snapshot restore progress: id=2-23-1740527883239 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:03.239Z [INFO] server-9ca884d7-829b-df16-c309-972480951d71: restored user snapshot: index=23 +2025-02-25T23:58:03.239Z [ERROR] server-9ca884d7-829b-df16-c309-972480951d71: failed to get log: index=23 error="log not found" +2025-02-25T23:58:03.239Z [INFO] server-9ca884d7-829b-df16-c309-972480951d71: aborting pipeline replication: peer="{Voter server-ce1368fe-626a-4ac1-98bc-b0c6f241c120 ce1368fe-626a-4ac1-98bc-b0c6f241c120}" +2025-02-25T23:58:03.239Z [ERROR] server-9ca884d7-829b-df16-c309-972480951d71: failed to get log: index=23 error="log not found" +2025-02-25T23:58:03.239Z [INFO] server-9ca884d7-829b-df16-c309-972480951d71: aborting pipeline replication: peer="{Voter server-ed8934a7-6884-151a-5ec2-82e70380d25e ed8934a7-6884-151a-5ec2-82e70380d25e}" +2025-02-25T23:58:03.249Z [ERROR] server-9ca884d7-829b-df16-c309-972480951d71: failed to get log: index=23 error="log not found" +2025-02-25T23:58:03.250Z [INFO] creating new snapshot: path=/tmp/raft3468133404/snapshots/2-23-1740527883250.tmp +2025-02-25T23:58:03.250Z [INFO] server-ed8934a7-6884-151a-5ec2-82e70380d25e: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:03.250Z [INFO] server-ed8934a7-6884-151a-5ec2-82e70380d25e: copied to local snapshot: bytes=71 +2025-02-25T23:58:03.250Z [INFO] server-ed8934a7-6884-151a-5ec2-82e70380d25e: snapshot restore progress: id=2-23-1740527883250 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:03.250Z [INFO] server-ed8934a7-6884-151a-5ec2-82e70380d25e: Installed remote snapshot +2025-02-25T23:58:03.250Z [INFO] server-9ca884d7-829b-df16-c309-972480951d71: pipelining replication: peer="{Voter server-ed8934a7-6884-151a-5ec2-82e70380d25e ed8934a7-6884-151a-5ec2-82e70380d25e}" +2025-02-25T23:58:03.250Z [ERROR] server-9ca884d7-829b-df16-c309-972480951d71: failed to get log: index=23 error="log not found" +2025-02-25T23:58:03.250Z [INFO] creating new snapshot: path=/tmp/raft1465290360/snapshots/2-23-1740527883250.tmp +2025-02-25T23:58:03.250Z [INFO] server-ce1368fe-626a-4ac1-98bc-b0c6f241c120: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:03.251Z [INFO] server-ce1368fe-626a-4ac1-98bc-b0c6f241c120: copied to local snapshot: bytes=71 +2025-02-25T23:58:03.251Z [INFO] server-ce1368fe-626a-4ac1-98bc-b0c6f241c120: snapshot restore progress: id=2-23-1740527883250 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:03.251Z [INFO] server-ce1368fe-626a-4ac1-98bc-b0c6f241c120: Installed remote snapshot +2025-02-25T23:58:03.251Z [WARN] server-ce1368fe-626a-4ac1-98bc-b0c6f241c120: skipping application of old log: index=22 +2025-02-25T23:58:03.251Z [INFO] server-9ca884d7-829b-df16-c309-972480951d71: pipelining replication: peer="{Voter server-ce1368fe-626a-4ac1-98bc-b0c6f241c120 ce1368fe-626a-4ac1-98bc-b0c6f241c120}" +2025-02-25T23:58:03.251Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:58:04.258Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:58:03.25125514 +0000 UTC m=+30.860270547 (2 nodes), 520ns from start of poll, 1.700888672s from cluster start. Timeout at 2025-02-25 23:58:04.2562635 +0000 UTC m=+31.865278907, 1.00500836s after stability +2025-02-25T23:58:04.282Z [INFO] server-9ca884d7-829b-df16-c309-972480951d71: aborting pipeline replication: peer="{Voter server-ce1368fe-626a-4ac1-98bc-b0c6f241c120 ce1368fe-626a-4ac1-98bc-b0c6f241c120}" +2025-02-25T23:58:04.282Z [INFO] server-9ca884d7-829b-df16-c309-972480951d71: aborting pipeline replication: peer="{Voter server-ed8934a7-6884-151a-5ec2-82e70380d25e ed8934a7-6884-151a-5ec2-82e70380d25e}" === RUN TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_false -2026-03-31T06:16:42.539Z [INFO] server-98dd69e3-084a-8ea0-013e-387f33f8c1bc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-98dd69e3-084a-8ea0-013e-387f33f8c1bc Address:98dd69e3-084a-8ea0-013e-387f33f8c1bc} {Suffrage:Voter ID:server-0af4e891-8cfd-6b4b-79a4-0e68bf25b3c9 Address:0af4e891-8cfd-6b4b-79a4-0e68bf25b3c9} {Suffrage:Voter ID:server-8c1fffcd-5833-5365-3d14-a7327fdfcbed Address:8c1fffcd-5833-5365-3d14-a7327fdfcbed}]" -2026-03-31T06:16:42.539Z [INFO] server-0af4e891-8cfd-6b4b-79a4-0e68bf25b3c9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-98dd69e3-084a-8ea0-013e-387f33f8c1bc Address:98dd69e3-084a-8ea0-013e-387f33f8c1bc} {Suffrage:Voter ID:server-0af4e891-8cfd-6b4b-79a4-0e68bf25b3c9 Address:0af4e891-8cfd-6b4b-79a4-0e68bf25b3c9} {Suffrage:Voter ID:server-8c1fffcd-5833-5365-3d14-a7327fdfcbed Address:8c1fffcd-5833-5365-3d14-a7327fdfcbed}]" -2026-03-31T06:16:42.539Z [INFO] server-98dd69e3-084a-8ea0-013e-387f33f8c1bc: entering follower state: follower="Node at 98dd69e3-084a-8ea0-013e-387f33f8c1bc [Follower]" leader-address= leader-id= -2026-03-31T06:16:42.539Z [INFO] server-0af4e891-8cfd-6b4b-79a4-0e68bf25b3c9: entering follower state: follower="Node at 0af4e891-8cfd-6b4b-79a4-0e68bf25b3c9 [Follower]" leader-address= leader-id= -2026-03-31T06:16:42.539Z [INFO] server-8c1fffcd-5833-5365-3d14-a7327fdfcbed: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-98dd69e3-084a-8ea0-013e-387f33f8c1bc Address:98dd69e3-084a-8ea0-013e-387f33f8c1bc} {Suffrage:Voter ID:server-0af4e891-8cfd-6b4b-79a4-0e68bf25b3c9 Address:0af4e891-8cfd-6b4b-79a4-0e68bf25b3c9} {Suffrage:Voter ID:server-8c1fffcd-5833-5365-3d14-a7327fdfcbed Address:8c1fffcd-5833-5365-3d14-a7327fdfcbed}]" -2026-03-31T06:16:42.539Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:42.539Z [INFO] server-8c1fffcd-5833-5365-3d14-a7327fdfcbed: entering follower state: follower="Node at 8c1fffcd-5833-5365-3d14-a7327fdfcbed [Follower]" leader-address= leader-id= -2026-03-31T06:16:43.128Z [WARN] server-8c1fffcd-5833-5365-3d14-a7327fdfcbed: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:43.128Z [INFO] server-8c1fffcd-5833-5365-3d14-a7327fdfcbed: entering candidate state: node="Node at 8c1fffcd-5833-5365-3d14-a7327fdfcbed [Candidate]" term=2 -2026-03-31T06:16:43.128Z [INFO] server-8c1fffcd-5833-5365-3d14-a7327fdfcbed: election won: term=2 tally=2 -2026-03-31T06:16:43.128Z [INFO] server-8c1fffcd-5833-5365-3d14-a7327fdfcbed: entering leader state: leader="Node at 8c1fffcd-5833-5365-3d14-a7327fdfcbed [Leader]" -2026-03-31T06:16:43.128Z [INFO] server-8c1fffcd-5833-5365-3d14-a7327fdfcbed: added peer, starting replication: peer=server-98dd69e3-084a-8ea0-013e-387f33f8c1bc -2026-03-31T06:16:43.128Z [INFO] server-8c1fffcd-5833-5365-3d14-a7327fdfcbed: added peer, starting replication: peer=server-0af4e891-8cfd-6b4b-79a4-0e68bf25b3c9 -2026-03-31T06:16:43.128Z [INFO] server-8c1fffcd-5833-5365-3d14-a7327fdfcbed: pipelining replication: peer="{Voter server-0af4e891-8cfd-6b4b-79a4-0e68bf25b3c9 0af4e891-8cfd-6b4b-79a4-0e68bf25b3c9}" -2026-03-31T06:16:43.128Z [INFO] server-8c1fffcd-5833-5365-3d14-a7327fdfcbed: pipelining replication: peer="{Voter server-98dd69e3-084a-8ea0-013e-387f33f8c1bc 98dd69e3-084a-8ea0-013e-387f33f8c1bc}" -2026-03-31T06:16:44.133Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:43.12822926 +0000 UTC m=+31.405836641 (1 nodes), 588.70296ms from start of poll, 589.16124ms from cluster start. Timeout at 2026-03-31 06:16:44.13323598 +0000 UTC m=+32.410843361, 1.00500672s after stability -2026-03-31T06:16:44.134Z [INFO] server-8c1fffcd-5833-5365-3d14-a7327fdfcbed: starting snapshot up to: index=12 -2026-03-31T06:16:44.134Z [INFO] creating new snapshot: path=/tmp/raft3710432933/snapshots/2-12-1774937804134.tmp -2026-03-31T06:16:44.135Z [INFO] server-8c1fffcd-5833-5365-3d14-a7327fdfcbed: snapshot complete up to: index=12 -2026-03-31T06:16:44.136Z [INFO] creating new snapshot: path=/tmp/raft3710432933/snapshots/2-23-1774937804136.tmp -2026-03-31T06:16:44.136Z [INFO] server-8c1fffcd-5833-5365-3d14-a7327fdfcbed: copied to local snapshot: bytes=71 -2026-03-31T06:16:44.136Z [INFO] server-8c1fffcd-5833-5365-3d14-a7327fdfcbed: snapshot restore progress: id=2-23-1774937804136 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:44.137Z [INFO] server-8c1fffcd-5833-5365-3d14-a7327fdfcbed: removing all old logs from log store -2026-03-31T06:16:44.137Z [INFO] server-8c1fffcd-5833-5365-3d14-a7327fdfcbed: compacting logs: from=1 to=22 -2026-03-31T06:16:44.137Z [INFO] server-8c1fffcd-5833-5365-3d14-a7327fdfcbed: restored user snapshot: index=23 -2026-03-31T06:16:44.137Z [ERROR] server-8c1fffcd-5833-5365-3d14-a7327fdfcbed: failed to get log: index=22 error="log not found" -2026-03-31T06:16:44.137Z [INFO] server-8c1fffcd-5833-5365-3d14-a7327fdfcbed: aborting pipeline replication: peer="{Voter server-98dd69e3-084a-8ea0-013e-387f33f8c1bc 98dd69e3-084a-8ea0-013e-387f33f8c1bc}" -2026-03-31T06:16:44.137Z [ERROR] server-8c1fffcd-5833-5365-3d14-a7327fdfcbed: failed to get log: index=22 error="log not found" -2026-03-31T06:16:44.137Z [INFO] server-8c1fffcd-5833-5365-3d14-a7327fdfcbed: aborting pipeline replication: peer="{Voter server-0af4e891-8cfd-6b4b-79a4-0e68bf25b3c9 0af4e891-8cfd-6b4b-79a4-0e68bf25b3c9}" -2026-03-31T06:16:44.143Z [ERROR] server-8c1fffcd-5833-5365-3d14-a7327fdfcbed: failed to get log: index=22 error="log not found" -2026-03-31T06:16:44.143Z [ERROR] server-8c1fffcd-5833-5365-3d14-a7327fdfcbed: failed to get log: index=22 error="log not found" -2026-03-31T06:16:44.145Z [INFO] creating new snapshot: path=/tmp/raft3472649605/snapshots/2-23-1774937804145.tmp -2026-03-31T06:16:44.146Z [INFO] creating new snapshot: path=/tmp/raft1077602195/snapshots/2-23-1774937804146.tmp -2026-03-31T06:16:44.146Z [INFO] server-0af4e891-8cfd-6b4b-79a4-0e68bf25b3c9: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:44.146Z [INFO] server-98dd69e3-084a-8ea0-013e-387f33f8c1bc: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:44.146Z [INFO] server-0af4e891-8cfd-6b4b-79a4-0e68bf25b3c9: copied to local snapshot: bytes=71 -2026-03-31T06:16:44.146Z [INFO] server-98dd69e3-084a-8ea0-013e-387f33f8c1bc: copied to local snapshot: bytes=71 -2026-03-31T06:16:44.146Z [INFO] server-0af4e891-8cfd-6b4b-79a4-0e68bf25b3c9: snapshot restore progress: id=2-23-1774937804145 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:44.146Z [INFO] server-0af4e891-8cfd-6b4b-79a4-0e68bf25b3c9: removing all old logs from log store -2026-03-31T06:16:44.146Z [INFO] server-0af4e891-8cfd-6b4b-79a4-0e68bf25b3c9: compacting logs: from=1 to=22 -2026-03-31T06:16:44.146Z [INFO] server-0af4e891-8cfd-6b4b-79a4-0e68bf25b3c9: Installed remote snapshot -2026-03-31T06:16:44.146Z [INFO] server-98dd69e3-084a-8ea0-013e-387f33f8c1bc: snapshot restore progress: id=2-23-1774937804146 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:44.146Z [INFO] server-98dd69e3-084a-8ea0-013e-387f33f8c1bc: removing all old logs from log store -2026-03-31T06:16:44.146Z [INFO] server-98dd69e3-084a-8ea0-013e-387f33f8c1bc: compacting logs: from=1 to=22 -2026-03-31T06:16:44.146Z [WARN] server-0af4e891-8cfd-6b4b-79a4-0e68bf25b3c9: skipping application of old log: index=22 -2026-03-31T06:16:44.146Z [INFO] server-98dd69e3-084a-8ea0-013e-387f33f8c1bc: Installed remote snapshot -2026-03-31T06:16:44.146Z [INFO] server-8c1fffcd-5833-5365-3d14-a7327fdfcbed: pipelining replication: peer="{Voter server-0af4e891-8cfd-6b4b-79a4-0e68bf25b3c9 0af4e891-8cfd-6b4b-79a4-0e68bf25b3c9}" -2026-03-31T06:16:44.146Z [INFO] server-8c1fffcd-5833-5365-3d14-a7327fdfcbed: pipelining replication: peer="{Voter server-98dd69e3-084a-8ea0-013e-387f33f8c1bc 98dd69e3-084a-8ea0-013e-387f33f8c1bc}" -2026-03-31T06:16:44.146Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:16:45.152Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:16:44.14699926 +0000 UTC m=+32.424606641 (2 nodes), 3.88µs from start of poll, 1.60793124s from cluster start. Timeout at 2026-03-31 06:16:45.1520085 +0000 UTC m=+33.429615881, 1.00500924s after stability -2026-03-31T06:16:45.165Z [INFO] server-8c1fffcd-5833-5365-3d14-a7327fdfcbed: aborting pipeline replication: peer="{Voter server-0af4e891-8cfd-6b4b-79a4-0e68bf25b3c9 0af4e891-8cfd-6b4b-79a4-0e68bf25b3c9}" -2026-03-31T06:16:45.165Z [INFO] server-8c1fffcd-5833-5365-3d14-a7327fdfcbed: aborting pipeline replication: peer="{Voter server-98dd69e3-084a-8ea0-013e-387f33f8c1bc 98dd69e3-084a-8ea0-013e-387f33f8c1bc}" +2025-02-25T23:58:04.282Z [INFO] server-1bff658a-e914-afa2-4fb1-4a22bd1bf0bf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1bff658a-e914-afa2-4fb1-4a22bd1bf0bf Address:1bff658a-e914-afa2-4fb1-4a22bd1bf0bf} {Suffrage:Voter ID:server-f25f33fa-6f2b-796e-8851-32556b37d753 Address:f25f33fa-6f2b-796e-8851-32556b37d753} {Suffrage:Voter ID:server-46c6455e-3a47-2a6b-43c0-77e31dbe1b59 Address:46c6455e-3a47-2a6b-43c0-77e31dbe1b59}]" +2025-02-25T23:58:04.282Z [INFO] server-f25f33fa-6f2b-796e-8851-32556b37d753: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1bff658a-e914-afa2-4fb1-4a22bd1bf0bf Address:1bff658a-e914-afa2-4fb1-4a22bd1bf0bf} {Suffrage:Voter ID:server-f25f33fa-6f2b-796e-8851-32556b37d753 Address:f25f33fa-6f2b-796e-8851-32556b37d753} {Suffrage:Voter ID:server-46c6455e-3a47-2a6b-43c0-77e31dbe1b59 Address:46c6455e-3a47-2a6b-43c0-77e31dbe1b59}]" +2025-02-25T23:58:04.282Z [INFO] server-46c6455e-3a47-2a6b-43c0-77e31dbe1b59: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1bff658a-e914-afa2-4fb1-4a22bd1bf0bf Address:1bff658a-e914-afa2-4fb1-4a22bd1bf0bf} {Suffrage:Voter ID:server-f25f33fa-6f2b-796e-8851-32556b37d753 Address:f25f33fa-6f2b-796e-8851-32556b37d753} {Suffrage:Voter ID:server-46c6455e-3a47-2a6b-43c0-77e31dbe1b59 Address:46c6455e-3a47-2a6b-43c0-77e31dbe1b59}]" +2025-02-25T23:58:04.282Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:58:04.282Z [INFO] server-1bff658a-e914-afa2-4fb1-4a22bd1bf0bf: entering follower state: follower="Node at 1bff658a-e914-afa2-4fb1-4a22bd1bf0bf [Follower]" leader-address= leader-id= +2025-02-25T23:58:04.282Z [INFO] server-f25f33fa-6f2b-796e-8851-32556b37d753: entering follower state: follower="Node at f25f33fa-6f2b-796e-8851-32556b37d753 [Follower]" leader-address= leader-id= +2025-02-25T23:58:04.282Z [INFO] server-46c6455e-3a47-2a6b-43c0-77e31dbe1b59: entering follower state: follower="Node at 46c6455e-3a47-2a6b-43c0-77e31dbe1b59 [Follower]" leader-address= leader-id= +2025-02-25T23:58:04.786Z [WARN] server-f25f33fa-6f2b-796e-8851-32556b37d753: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:58:04.786Z [INFO] server-f25f33fa-6f2b-796e-8851-32556b37d753: entering candidate state: node="Node at f25f33fa-6f2b-796e-8851-32556b37d753 [Candidate]" term=2 +2025-02-25T23:58:04.786Z [INFO] server-f25f33fa-6f2b-796e-8851-32556b37d753: election won: term=2 tally=2 +2025-02-25T23:58:04.786Z [INFO] server-f25f33fa-6f2b-796e-8851-32556b37d753: entering leader state: leader="Node at f25f33fa-6f2b-796e-8851-32556b37d753 [Leader]" +2025-02-25T23:58:04.786Z [INFO] server-f25f33fa-6f2b-796e-8851-32556b37d753: added peer, starting replication: peer=server-1bff658a-e914-afa2-4fb1-4a22bd1bf0bf +2025-02-25T23:58:04.786Z [INFO] server-f25f33fa-6f2b-796e-8851-32556b37d753: added peer, starting replication: peer=server-46c6455e-3a47-2a6b-43c0-77e31dbe1b59 +2025-02-25T23:58:04.786Z [INFO] server-f25f33fa-6f2b-796e-8851-32556b37d753: pipelining replication: peer="{Voter server-46c6455e-3a47-2a6b-43c0-77e31dbe1b59 46c6455e-3a47-2a6b-43c0-77e31dbe1b59}" +2025-02-25T23:58:04.786Z [INFO] server-f25f33fa-6f2b-796e-8851-32556b37d753: pipelining replication: peer="{Voter server-1bff658a-e914-afa2-4fb1-4a22bd1bf0bf 1bff658a-e914-afa2-4fb1-4a22bd1bf0bf}" +2025-02-25T23:58:05.794Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:58:04.786331596 +0000 UTC m=+32.395347003 (1 nodes), 503.681647ms from start of poll, 503.862328ms from cluster start. Timeout at 2025-02-25 23:58:05.791333716 +0000 UTC m=+33.400349083, 1.00500208s after stability +2025-02-25T23:58:05.794Z [INFO] server-f25f33fa-6f2b-796e-8851-32556b37d753: starting snapshot up to: index=12 +2025-02-25T23:58:05.794Z [INFO] creating new snapshot: path=/tmp/raft39246602/snapshots/2-12-1740527885794.tmp +2025-02-25T23:58:05.795Z [INFO] server-f25f33fa-6f2b-796e-8851-32556b37d753: snapshot complete up to: index=12 +2025-02-25T23:58:05.797Z [INFO] creating new snapshot: path=/tmp/raft39246602/snapshots/2-23-1740527885797.tmp +2025-02-25T23:58:05.797Z [INFO] server-f25f33fa-6f2b-796e-8851-32556b37d753: copied to local snapshot: bytes=71 +2025-02-25T23:58:05.797Z [INFO] server-f25f33fa-6f2b-796e-8851-32556b37d753: snapshot restore progress: id=2-23-1740527885797 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:05.797Z [INFO] server-f25f33fa-6f2b-796e-8851-32556b37d753: removing all old logs from log store +2025-02-25T23:58:05.797Z [INFO] server-f25f33fa-6f2b-796e-8851-32556b37d753: compacting logs: from=1 to=22 +2025-02-25T23:58:05.797Z [INFO] server-f25f33fa-6f2b-796e-8851-32556b37d753: restored user snapshot: index=23 +2025-02-25T23:58:05.797Z [ERROR] server-f25f33fa-6f2b-796e-8851-32556b37d753: failed to get log: index=22 error="log not found" +2025-02-25T23:58:05.797Z [INFO] server-f25f33fa-6f2b-796e-8851-32556b37d753: aborting pipeline replication: peer="{Voter server-46c6455e-3a47-2a6b-43c0-77e31dbe1b59 46c6455e-3a47-2a6b-43c0-77e31dbe1b59}" +2025-02-25T23:58:05.797Z [ERROR] server-f25f33fa-6f2b-796e-8851-32556b37d753: failed to get log: index=22 error="log not found" +2025-02-25T23:58:05.797Z [INFO] server-f25f33fa-6f2b-796e-8851-32556b37d753: aborting pipeline replication: peer="{Voter server-1bff658a-e914-afa2-4fb1-4a22bd1bf0bf 1bff658a-e914-afa2-4fb1-4a22bd1bf0bf}" +2025-02-25T23:58:05.809Z [ERROR] server-f25f33fa-6f2b-796e-8851-32556b37d753: failed to get log: index=20 error="log not found" +2025-02-25T23:58:05.810Z [INFO] creating new snapshot: path=/tmp/raft2155291545/snapshots/2-23-1740527885810.tmp +2025-02-25T23:58:05.810Z [INFO] server-46c6455e-3a47-2a6b-43c0-77e31dbe1b59: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:05.810Z [INFO] server-46c6455e-3a47-2a6b-43c0-77e31dbe1b59: copied to local snapshot: bytes=71 +2025-02-25T23:58:05.810Z [INFO] server-46c6455e-3a47-2a6b-43c0-77e31dbe1b59: snapshot restore progress: id=2-23-1740527885810 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:05.810Z [INFO] server-46c6455e-3a47-2a6b-43c0-77e31dbe1b59: removing all old logs from log store +2025-02-25T23:58:05.810Z [INFO] server-46c6455e-3a47-2a6b-43c0-77e31dbe1b59: compacting logs: from=1 to=22 +2025-02-25T23:58:05.810Z [INFO] server-46c6455e-3a47-2a6b-43c0-77e31dbe1b59: Installed remote snapshot +2025-02-25T23:58:05.810Z [ERROR] server-f25f33fa-6f2b-796e-8851-32556b37d753: failed to get log: index=22 error="log not found" +2025-02-25T23:58:05.810Z [INFO] creating new snapshot: path=/tmp/raft1753621927/snapshots/2-23-1740527885810.tmp +2025-02-25T23:58:05.810Z [INFO] server-1bff658a-e914-afa2-4fb1-4a22bd1bf0bf: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:05.810Z [INFO] server-1bff658a-e914-afa2-4fb1-4a22bd1bf0bf: copied to local snapshot: bytes=71 +2025-02-25T23:58:05.811Z [INFO] server-1bff658a-e914-afa2-4fb1-4a22bd1bf0bf: snapshot restore progress: id=2-23-1740527885810 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:05.811Z [INFO] server-1bff658a-e914-afa2-4fb1-4a22bd1bf0bf: removing all old logs from log store +2025-02-25T23:58:05.811Z [INFO] server-1bff658a-e914-afa2-4fb1-4a22bd1bf0bf: compacting logs: from=1 to=22 +2025-02-25T23:58:05.811Z [INFO] server-1bff658a-e914-afa2-4fb1-4a22bd1bf0bf: Installed remote snapshot +2025-02-25T23:58:05.811Z [WARN] server-1bff658a-e914-afa2-4fb1-4a22bd1bf0bf: skipping application of old log: index=22 +2025-02-25T23:58:05.811Z [INFO] server-f25f33fa-6f2b-796e-8851-32556b37d753: pipelining replication: peer="{Voter server-1bff658a-e914-afa2-4fb1-4a22bd1bf0bf 1bff658a-e914-afa2-4fb1-4a22bd1bf0bf}" +2025-02-25T23:58:05.811Z [WARN] server-46c6455e-3a47-2a6b-43c0-77e31dbe1b59: skipping application of old log: index=22 +2025-02-25T23:58:05.811Z [INFO] server-f25f33fa-6f2b-796e-8851-32556b37d753: pipelining replication: peer="{Voter server-46c6455e-3a47-2a6b-43c0-77e31dbe1b59 46c6455e-3a47-2a6b-43c0-77e31dbe1b59}" +2025-02-25T23:58:05.811Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:58:06.817Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:58:05.811199963 +0000 UTC m=+33.420215370 (2 nodes), 400ns from start of poll, 1.528730695s from cluster start. Timeout at 2025-02-25 23:58:06.816213603 +0000 UTC m=+34.425229010, 1.00501364s after stability +2025-02-25T23:58:06.834Z [INFO] server-f25f33fa-6f2b-796e-8851-32556b37d753: aborting pipeline replication: peer="{Voter server-46c6455e-3a47-2a6b-43c0-77e31dbe1b59 46c6455e-3a47-2a6b-43c0-77e31dbe1b59}" +2025-02-25T23:58:06.834Z [INFO] server-f25f33fa-6f2b-796e-8851-32556b37d753: aborting pipeline replication: peer="{Voter server-1bff658a-e914-afa2-4fb1-4a22bd1bf0bf 1bff658a-e914-afa2-4fb1-4a22bd1bf0bf}" === RUN TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_true -2026-03-31T06:16:45.167Z [INFO] server-e5b7b0ed-32c8-7309-a352-5645ef13f920: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e5b7b0ed-32c8-7309-a352-5645ef13f920 Address:e5b7b0ed-32c8-7309-a352-5645ef13f920} {Suffrage:Voter ID:server-f78d6045-bd91-9be4-0def-faddaffb7fba Address:f78d6045-bd91-9be4-0def-faddaffb7fba} {Suffrage:Voter ID:server-c24e2188-81de-f73d-f9e4-670caeb5a3b2 Address:c24e2188-81de-f73d-f9e4-670caeb5a3b2}]" -2026-03-31T06:16:45.167Z [INFO] server-e5b7b0ed-32c8-7309-a352-5645ef13f920: entering follower state: follower="Node at e5b7b0ed-32c8-7309-a352-5645ef13f920 [Follower]" leader-address= leader-id= -2026-03-31T06:16:45.167Z [INFO] server-f78d6045-bd91-9be4-0def-faddaffb7fba: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e5b7b0ed-32c8-7309-a352-5645ef13f920 Address:e5b7b0ed-32c8-7309-a352-5645ef13f920} {Suffrage:Voter ID:server-f78d6045-bd91-9be4-0def-faddaffb7fba Address:f78d6045-bd91-9be4-0def-faddaffb7fba} {Suffrage:Voter ID:server-c24e2188-81de-f73d-f9e4-670caeb5a3b2 Address:c24e2188-81de-f73d-f9e4-670caeb5a3b2}]" -2026-03-31T06:16:45.167Z [INFO] server-f78d6045-bd91-9be4-0def-faddaffb7fba: entering follower state: follower="Node at f78d6045-bd91-9be4-0def-faddaffb7fba [Follower]" leader-address= leader-id= -2026-03-31T06:16:45.167Z [INFO] server-c24e2188-81de-f73d-f9e4-670caeb5a3b2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e5b7b0ed-32c8-7309-a352-5645ef13f920 Address:e5b7b0ed-32c8-7309-a352-5645ef13f920} {Suffrage:Voter ID:server-f78d6045-bd91-9be4-0def-faddaffb7fba Address:f78d6045-bd91-9be4-0def-faddaffb7fba} {Suffrage:Voter ID:server-c24e2188-81de-f73d-f9e4-670caeb5a3b2 Address:c24e2188-81de-f73d-f9e4-670caeb5a3b2}]" -2026-03-31T06:16:45.168Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:45.168Z [INFO] server-c24e2188-81de-f73d-f9e4-670caeb5a3b2: entering follower state: follower="Node at c24e2188-81de-f73d-f9e4-670caeb5a3b2 [Follower]" leader-address= leader-id= -2026-03-31T06:16:45.933Z [WARN] server-e5b7b0ed-32c8-7309-a352-5645ef13f920: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:45.933Z [INFO] server-e5b7b0ed-32c8-7309-a352-5645ef13f920: entering candidate state: node="Node at e5b7b0ed-32c8-7309-a352-5645ef13f920 [Candidate]" term=2 -2026-03-31T06:16:45.933Z [INFO] server-e5b7b0ed-32c8-7309-a352-5645ef13f920: election won: term=2 tally=2 -2026-03-31T06:16:45.933Z [INFO] server-e5b7b0ed-32c8-7309-a352-5645ef13f920: entering leader state: leader="Node at e5b7b0ed-32c8-7309-a352-5645ef13f920 [Leader]" -2026-03-31T06:16:45.933Z [INFO] server-e5b7b0ed-32c8-7309-a352-5645ef13f920: added peer, starting replication: peer=server-f78d6045-bd91-9be4-0def-faddaffb7fba -2026-03-31T06:16:45.933Z [INFO] server-e5b7b0ed-32c8-7309-a352-5645ef13f920: added peer, starting replication: peer=server-c24e2188-81de-f73d-f9e4-670caeb5a3b2 -2026-03-31T06:16:45.933Z [INFO] server-e5b7b0ed-32c8-7309-a352-5645ef13f920: pipelining replication: peer="{Voter server-c24e2188-81de-f73d-f9e4-670caeb5a3b2 c24e2188-81de-f73d-f9e4-670caeb5a3b2}" -2026-03-31T06:16:45.933Z [INFO] server-e5b7b0ed-32c8-7309-a352-5645ef13f920: pipelining replication: peer="{Voter server-f78d6045-bd91-9be4-0def-faddaffb7fba f78d6045-bd91-9be4-0def-faddaffb7fba}" -2026-03-31T06:16:46.939Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:45.93366334 +0000 UTC m=+34.211270721 (1 nodes), 765.55312ms from start of poll, 766.55288ms from cluster start. Timeout at 2026-03-31 06:16:46.9386689 +0000 UTC m=+35.216276281, 1.00500556s after stability -2026-03-31T06:16:46.939Z [INFO] server-e5b7b0ed-32c8-7309-a352-5645ef13f920: starting snapshot up to: index=12 -2026-03-31T06:16:46.939Z [INFO] creating new snapshot: path=/tmp/raft933494939/snapshots/2-12-1774937806939.tmp -2026-03-31T06:16:46.940Z [INFO] server-e5b7b0ed-32c8-7309-a352-5645ef13f920: snapshot complete up to: index=12 -2026-03-31T06:16:46.941Z [INFO] server-90eedf0d-3c89-750b-6864-4c3da44b7e91: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-90eedf0d-3c89-750b-6864-4c3da44b7e91 Address:90eedf0d-3c89-750b-6864-4c3da44b7e91} {Suffrage:Voter ID:server-aeee5fc3-30a2-d120-fcdc-983714743ad2 Address:aeee5fc3-30a2-d120-fcdc-983714743ad2} {Suffrage:Voter ID:server-28cbb796-4f9b-647d-2b44-2ff0186643dc Address:28cbb796-4f9b-647d-2b44-2ff0186643dc}]" -2026-03-31T06:16:46.941Z [INFO] server-aeee5fc3-30a2-d120-fcdc-983714743ad2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-90eedf0d-3c89-750b-6864-4c3da44b7e91 Address:90eedf0d-3c89-750b-6864-4c3da44b7e91} {Suffrage:Voter ID:server-aeee5fc3-30a2-d120-fcdc-983714743ad2 Address:aeee5fc3-30a2-d120-fcdc-983714743ad2} {Suffrage:Voter ID:server-28cbb796-4f9b-647d-2b44-2ff0186643dc Address:28cbb796-4f9b-647d-2b44-2ff0186643dc}]" -2026-03-31T06:16:46.941Z [INFO] server-28cbb796-4f9b-647d-2b44-2ff0186643dc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-90eedf0d-3c89-750b-6864-4c3da44b7e91 Address:90eedf0d-3c89-750b-6864-4c3da44b7e91} {Suffrage:Voter ID:server-aeee5fc3-30a2-d120-fcdc-983714743ad2 Address:aeee5fc3-30a2-d120-fcdc-983714743ad2} {Suffrage:Voter ID:server-28cbb796-4f9b-647d-2b44-2ff0186643dc Address:28cbb796-4f9b-647d-2b44-2ff0186643dc}]" -2026-03-31T06:16:46.941Z [INFO] server-90eedf0d-3c89-750b-6864-4c3da44b7e91: entering follower state: follower="Node at 90eedf0d-3c89-750b-6864-4c3da44b7e91 [Follower]" leader-address= leader-id= -2026-03-31T06:16:46.941Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:46.941Z [INFO] server-aeee5fc3-30a2-d120-fcdc-983714743ad2: entering follower state: follower="Node at aeee5fc3-30a2-d120-fcdc-983714743ad2 [Follower]" leader-address= leader-id= -2026-03-31T06:16:46.941Z [INFO] server-28cbb796-4f9b-647d-2b44-2ff0186643dc: entering follower state: follower="Node at 28cbb796-4f9b-647d-2b44-2ff0186643dc [Follower]" leader-address= leader-id= -2026-03-31T06:16:47.582Z [WARN] server-aeee5fc3-30a2-d120-fcdc-983714743ad2: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:47.582Z [INFO] server-aeee5fc3-30a2-d120-fcdc-983714743ad2: entering candidate state: node="Node at aeee5fc3-30a2-d120-fcdc-983714743ad2 [Candidate]" term=2 -2026-03-31T06:16:47.582Z [INFO] server-aeee5fc3-30a2-d120-fcdc-983714743ad2: election won: term=2 tally=2 -2026-03-31T06:16:47.582Z [INFO] server-aeee5fc3-30a2-d120-fcdc-983714743ad2: entering leader state: leader="Node at aeee5fc3-30a2-d120-fcdc-983714743ad2 [Leader]" -2026-03-31T06:16:47.582Z [INFO] server-aeee5fc3-30a2-d120-fcdc-983714743ad2: added peer, starting replication: peer=server-90eedf0d-3c89-750b-6864-4c3da44b7e91 -2026-03-31T06:16:47.582Z [INFO] server-aeee5fc3-30a2-d120-fcdc-983714743ad2: added peer, starting replication: peer=server-28cbb796-4f9b-647d-2b44-2ff0186643dc -2026-03-31T06:16:47.582Z [INFO] server-aeee5fc3-30a2-d120-fcdc-983714743ad2: pipelining replication: peer="{Voter server-28cbb796-4f9b-647d-2b44-2ff0186643dc 28cbb796-4f9b-647d-2b44-2ff0186643dc}" -2026-03-31T06:16:47.582Z [INFO] server-aeee5fc3-30a2-d120-fcdc-983714743ad2: pipelining replication: peer="{Voter server-90eedf0d-3c89-750b-6864-4c3da44b7e91 90eedf0d-3c89-750b-6864-4c3da44b7e91}" -2026-03-31T06:16:48.587Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:47.5823271 +0000 UTC m=+35.859934481 (1 nodes), 640.9116ms from start of poll, 641.334ms from cluster start. Timeout at 2026-03-31 06:16:48.5874123 +0000 UTC m=+36.865019721, 1.00508524s after stability -2026-03-31T06:16:48.587Z [INFO] creating new snapshot: path=/tmp/raft2488077100/snapshots/2-14-1774937808587.tmp -2026-03-31T06:16:48.588Z [INFO] server-aeee5fc3-30a2-d120-fcdc-983714743ad2: copied to local snapshot: bytes=71 -2026-03-31T06:16:48.588Z [INFO] server-aeee5fc3-30a2-d120-fcdc-983714743ad2: snapshot restore progress: id=2-14-1774937808587 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:48.588Z [INFO] server-aeee5fc3-30a2-d120-fcdc-983714743ad2: restored user snapshot: index=14 -2026-03-31T06:16:48.589Z [ERROR] server-aeee5fc3-30a2-d120-fcdc-983714743ad2: failed to get log: index=3 error="log not found" -2026-03-31T06:16:48.589Z [INFO] server-aeee5fc3-30a2-d120-fcdc-983714743ad2: aborting pipeline replication: peer="{Voter server-28cbb796-4f9b-647d-2b44-2ff0186643dc 28cbb796-4f9b-647d-2b44-2ff0186643dc}" -2026-03-31T06:16:48.589Z [ERROR] server-aeee5fc3-30a2-d120-fcdc-983714743ad2: failed to get log: index=3 error="log not found" -2026-03-31T06:16:48.589Z [INFO] server-aeee5fc3-30a2-d120-fcdc-983714743ad2: aborting pipeline replication: peer="{Voter server-90eedf0d-3c89-750b-6864-4c3da44b7e91 90eedf0d-3c89-750b-6864-4c3da44b7e91}" -2026-03-31T06:16:48.598Z [ERROR] server-aeee5fc3-30a2-d120-fcdc-983714743ad2: failed to get log: index=3 error="log not found" -2026-03-31T06:16:48.598Z [ERROR] server-aeee5fc3-30a2-d120-fcdc-983714743ad2: failed to get log: index=3 error="log not found" -2026-03-31T06:16:48.598Z [INFO] creating new snapshot: path=/tmp/raft3736991391/snapshots/2-14-1774937808598.tmp -2026-03-31T06:16:48.598Z [INFO] creating new snapshot: path=/tmp/raft568671114/snapshots/2-14-1774937808598.tmp -2026-03-31T06:16:48.599Z [INFO] server-28cbb796-4f9b-647d-2b44-2ff0186643dc: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:48.599Z [INFO] server-90eedf0d-3c89-750b-6864-4c3da44b7e91: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:48.599Z [INFO] server-28cbb796-4f9b-647d-2b44-2ff0186643dc: copied to local snapshot: bytes=71 -2026-03-31T06:16:48.599Z [INFO] server-90eedf0d-3c89-750b-6864-4c3da44b7e91: copied to local snapshot: bytes=71 -2026-03-31T06:16:48.599Z [INFO] server-90eedf0d-3c89-750b-6864-4c3da44b7e91: snapshot restore progress: id=2-14-1774937808598 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:48.599Z [INFO] server-90eedf0d-3c89-750b-6864-4c3da44b7e91: Installed remote snapshot -2026-03-31T06:16:48.599Z [INFO] server-28cbb796-4f9b-647d-2b44-2ff0186643dc: snapshot restore progress: id=2-14-1774937808598 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:48.599Z [INFO] server-28cbb796-4f9b-647d-2b44-2ff0186643dc: Installed remote snapshot -2026-03-31T06:16:48.599Z [INFO] server-aeee5fc3-30a2-d120-fcdc-983714743ad2: pipelining replication: peer="{Voter server-90eedf0d-3c89-750b-6864-4c3da44b7e91 90eedf0d-3c89-750b-6864-4c3da44b7e91}" -2026-03-31T06:16:48.599Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:16:48.600Z [INFO] server-aeee5fc3-30a2-d120-fcdc-983714743ad2: pipelining replication: peer="{Voter server-28cbb796-4f9b-647d-2b44-2ff0186643dc 28cbb796-4f9b-647d-2b44-2ff0186643dc}" -2026-03-31T06:16:49.605Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:16:48.60000786 +0000 UTC m=+36.877615241 (2 nodes), 1.08µs from start of poll, 1.65901476s from cluster start. Timeout at 2026-03-31 06:16:49.60502398 +0000 UTC m=+37.882631361, 1.00501612s after stability -2026-03-31T06:16:49.619Z [INFO] server-e5b7b0ed-32c8-7309-a352-5645ef13f920: aborting pipeline replication: peer="{Voter server-c24e2188-81de-f73d-f9e4-670caeb5a3b2 c24e2188-81de-f73d-f9e4-670caeb5a3b2}" -2026-03-31T06:16:49.619Z [INFO] server-e5b7b0ed-32c8-7309-a352-5645ef13f920: aborting pipeline replication: peer="{Voter server-f78d6045-bd91-9be4-0def-faddaffb7fba f78d6045-bd91-9be4-0def-faddaffb7fba}" +2025-02-25T23:58:06.834Z [INFO] server-29aa60a8-a0c8-e70f-76bb-6f93cbf413fa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-29aa60a8-a0c8-e70f-76bb-6f93cbf413fa Address:29aa60a8-a0c8-e70f-76bb-6f93cbf413fa} {Suffrage:Voter ID:server-bbc5405b-3c84-1627-730e-4432acd1f826 Address:bbc5405b-3c84-1627-730e-4432acd1f826} {Suffrage:Voter ID:server-22d8f678-81f7-da10-825c-4136a55e8849 Address:22d8f678-81f7-da10-825c-4136a55e8849}]" +2025-02-25T23:58:06.834Z [INFO] server-bbc5405b-3c84-1627-730e-4432acd1f826: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-29aa60a8-a0c8-e70f-76bb-6f93cbf413fa Address:29aa60a8-a0c8-e70f-76bb-6f93cbf413fa} {Suffrage:Voter ID:server-bbc5405b-3c84-1627-730e-4432acd1f826 Address:bbc5405b-3c84-1627-730e-4432acd1f826} {Suffrage:Voter ID:server-22d8f678-81f7-da10-825c-4136a55e8849 Address:22d8f678-81f7-da10-825c-4136a55e8849}]" +2025-02-25T23:58:06.834Z [INFO] server-22d8f678-81f7-da10-825c-4136a55e8849: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-29aa60a8-a0c8-e70f-76bb-6f93cbf413fa Address:29aa60a8-a0c8-e70f-76bb-6f93cbf413fa} {Suffrage:Voter ID:server-bbc5405b-3c84-1627-730e-4432acd1f826 Address:bbc5405b-3c84-1627-730e-4432acd1f826} {Suffrage:Voter ID:server-22d8f678-81f7-da10-825c-4136a55e8849 Address:22d8f678-81f7-da10-825c-4136a55e8849}]" +2025-02-25T23:58:06.834Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:58:06.834Z [INFO] server-29aa60a8-a0c8-e70f-76bb-6f93cbf413fa: entering follower state: follower="Node at 29aa60a8-a0c8-e70f-76bb-6f93cbf413fa [Follower]" leader-address= leader-id= +2025-02-25T23:58:06.834Z [INFO] server-bbc5405b-3c84-1627-730e-4432acd1f826: entering follower state: follower="Node at bbc5405b-3c84-1627-730e-4432acd1f826 [Follower]" leader-address= leader-id= +2025-02-25T23:58:06.834Z [INFO] server-22d8f678-81f7-da10-825c-4136a55e8849: entering follower state: follower="Node at 22d8f678-81f7-da10-825c-4136a55e8849 [Follower]" leader-address= leader-id= +2025-02-25T23:58:07.585Z [WARN] server-22d8f678-81f7-da10-825c-4136a55e8849: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:58:07.585Z [INFO] server-22d8f678-81f7-da10-825c-4136a55e8849: entering candidate state: node="Node at 22d8f678-81f7-da10-825c-4136a55e8849 [Candidate]" term=2 +2025-02-25T23:58:07.586Z [INFO] server-22d8f678-81f7-da10-825c-4136a55e8849: election won: term=2 tally=2 +2025-02-25T23:58:07.586Z [INFO] server-22d8f678-81f7-da10-825c-4136a55e8849: entering leader state: leader="Node at 22d8f678-81f7-da10-825c-4136a55e8849 [Leader]" +2025-02-25T23:58:07.586Z [INFO] server-22d8f678-81f7-da10-825c-4136a55e8849: added peer, starting replication: peer=server-29aa60a8-a0c8-e70f-76bb-6f93cbf413fa +2025-02-25T23:58:07.586Z [INFO] server-22d8f678-81f7-da10-825c-4136a55e8849: added peer, starting replication: peer=server-bbc5405b-3c84-1627-730e-4432acd1f826 +2025-02-25T23:58:07.586Z [INFO] server-22d8f678-81f7-da10-825c-4136a55e8849: pipelining replication: peer="{Voter server-bbc5405b-3c84-1627-730e-4432acd1f826 bbc5405b-3c84-1627-730e-4432acd1f826}" +2025-02-25T23:58:07.586Z [INFO] server-22d8f678-81f7-da10-825c-4136a55e8849: pipelining replication: peer="{Voter server-29aa60a8-a0c8-e70f-76bb-6f93cbf413fa 29aa60a8-a0c8-e70f-76bb-6f93cbf413fa}" +2025-02-25T23:58:08.593Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:58:07.586205107 +0000 UTC m=+35.195220514 (1 nodes), 751.494786ms from start of poll, 751.662707ms from cluster start. Timeout at 2025-02-25 23:58:08.591214667 +0000 UTC m=+36.200230074, 1.00500956s after stability +2025-02-25T23:58:08.594Z [INFO] server-22d8f678-81f7-da10-825c-4136a55e8849: starting snapshot up to: index=12 +2025-02-25T23:58:08.594Z [INFO] creating new snapshot: path=/tmp/raft1476309145/snapshots/2-12-1740527888594.tmp +2025-02-25T23:58:08.594Z [INFO] server-22d8f678-81f7-da10-825c-4136a55e8849: snapshot complete up to: index=12 +2025-02-25T23:58:08.595Z [INFO] server-6df326c5-2961-e90e-3f18-3788b6ca74e8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6df326c5-2961-e90e-3f18-3788b6ca74e8 Address:6df326c5-2961-e90e-3f18-3788b6ca74e8} {Suffrage:Voter ID:server-1572a6b5-406d-26f9-a038-4e508fa08a88 Address:1572a6b5-406d-26f9-a038-4e508fa08a88} {Suffrage:Voter ID:server-1ba7170a-d1e8-3e50-b43e-2e98811f922f Address:1ba7170a-d1e8-3e50-b43e-2e98811f922f}]" +2025-02-25T23:58:08.595Z [INFO] server-1572a6b5-406d-26f9-a038-4e508fa08a88: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6df326c5-2961-e90e-3f18-3788b6ca74e8 Address:6df326c5-2961-e90e-3f18-3788b6ca74e8} {Suffrage:Voter ID:server-1572a6b5-406d-26f9-a038-4e508fa08a88 Address:1572a6b5-406d-26f9-a038-4e508fa08a88} {Suffrage:Voter ID:server-1ba7170a-d1e8-3e50-b43e-2e98811f922f Address:1ba7170a-d1e8-3e50-b43e-2e98811f922f}]" +2025-02-25T23:58:08.595Z [INFO] server-1ba7170a-d1e8-3e50-b43e-2e98811f922f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6df326c5-2961-e90e-3f18-3788b6ca74e8 Address:6df326c5-2961-e90e-3f18-3788b6ca74e8} {Suffrage:Voter ID:server-1572a6b5-406d-26f9-a038-4e508fa08a88 Address:1572a6b5-406d-26f9-a038-4e508fa08a88} {Suffrage:Voter ID:server-1ba7170a-d1e8-3e50-b43e-2e98811f922f Address:1ba7170a-d1e8-3e50-b43e-2e98811f922f}]" +2025-02-25T23:58:08.595Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:58:08.595Z [INFO] server-6df326c5-2961-e90e-3f18-3788b6ca74e8: entering follower state: follower="Node at 6df326c5-2961-e90e-3f18-3788b6ca74e8 [Follower]" leader-address= leader-id= +2025-02-25T23:58:08.595Z [INFO] server-1572a6b5-406d-26f9-a038-4e508fa08a88: entering follower state: follower="Node at 1572a6b5-406d-26f9-a038-4e508fa08a88 [Follower]" leader-address= leader-id= +2025-02-25T23:58:08.595Z [INFO] server-1ba7170a-d1e8-3e50-b43e-2e98811f922f: entering follower state: follower="Node at 1ba7170a-d1e8-3e50-b43e-2e98811f922f [Follower]" leader-address= leader-id= +2025-02-25T23:58:09.110Z [WARN] server-1ba7170a-d1e8-3e50-b43e-2e98811f922f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:58:09.110Z [INFO] server-1ba7170a-d1e8-3e50-b43e-2e98811f922f: entering candidate state: node="Node at 1ba7170a-d1e8-3e50-b43e-2e98811f922f [Candidate]" term=2 +2025-02-25T23:58:09.110Z [INFO] server-1ba7170a-d1e8-3e50-b43e-2e98811f922f: election won: term=2 tally=2 +2025-02-25T23:58:09.110Z [INFO] server-1ba7170a-d1e8-3e50-b43e-2e98811f922f: entering leader state: leader="Node at 1ba7170a-d1e8-3e50-b43e-2e98811f922f [Leader]" +2025-02-25T23:58:09.110Z [INFO] server-1ba7170a-d1e8-3e50-b43e-2e98811f922f: added peer, starting replication: peer=server-6df326c5-2961-e90e-3f18-3788b6ca74e8 +2025-02-25T23:58:09.110Z [INFO] server-1ba7170a-d1e8-3e50-b43e-2e98811f922f: added peer, starting replication: peer=server-1572a6b5-406d-26f9-a038-4e508fa08a88 +2025-02-25T23:58:09.110Z [INFO] server-1ba7170a-d1e8-3e50-b43e-2e98811f922f: pipelining replication: peer="{Voter server-1572a6b5-406d-26f9-a038-4e508fa08a88 1572a6b5-406d-26f9-a038-4e508fa08a88}" +2025-02-25T23:58:09.110Z [INFO] server-1ba7170a-d1e8-3e50-b43e-2e98811f922f: pipelining replication: peer="{Voter server-6df326c5-2961-e90e-3f18-3788b6ca74e8 6df326c5-2961-e90e-3f18-3788b6ca74e8}" +2025-02-25T23:58:10.118Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:58:09.110302413 +0000 UTC m=+36.719317820 (1 nodes), 515.124121ms from start of poll, 515.284562ms from cluster start. Timeout at 2025-02-25 23:58:10.115304693 +0000 UTC m=+37.724320100, 1.00500228s after stability +2025-02-25T23:58:10.118Z [INFO] creating new snapshot: path=/tmp/raft2795884327/snapshots/2-14-1740527890118.tmp +2025-02-25T23:58:10.118Z [INFO] server-1ba7170a-d1e8-3e50-b43e-2e98811f922f: copied to local snapshot: bytes=71 +2025-02-25T23:58:10.118Z [INFO] server-1ba7170a-d1e8-3e50-b43e-2e98811f922f: snapshot restore progress: id=2-14-1740527890118 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:10.118Z [INFO] server-1ba7170a-d1e8-3e50-b43e-2e98811f922f: restored user snapshot: index=14 +2025-02-25T23:58:10.118Z [ERROR] server-1ba7170a-d1e8-3e50-b43e-2e98811f922f: failed to get log: index=3 error="log not found" +2025-02-25T23:58:10.118Z [INFO] server-1ba7170a-d1e8-3e50-b43e-2e98811f922f: aborting pipeline replication: peer="{Voter server-1572a6b5-406d-26f9-a038-4e508fa08a88 1572a6b5-406d-26f9-a038-4e508fa08a88}" +2025-02-25T23:58:10.118Z [ERROR] server-1ba7170a-d1e8-3e50-b43e-2e98811f922f: failed to get log: index=3 error="log not found" +2025-02-25T23:58:10.118Z [INFO] server-1ba7170a-d1e8-3e50-b43e-2e98811f922f: aborting pipeline replication: peer="{Voter server-6df326c5-2961-e90e-3f18-3788b6ca74e8 6df326c5-2961-e90e-3f18-3788b6ca74e8}" +2025-02-25T23:58:10.129Z [ERROR] server-1ba7170a-d1e8-3e50-b43e-2e98811f922f: failed to get log: index=3 error="log not found" +2025-02-25T23:58:10.130Z [INFO] creating new snapshot: path=/tmp/raft1699547693/snapshots/2-14-1740527890130.tmp +2025-02-25T23:58:10.130Z [INFO] server-1572a6b5-406d-26f9-a038-4e508fa08a88: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:10.130Z [INFO] server-1572a6b5-406d-26f9-a038-4e508fa08a88: copied to local snapshot: bytes=71 +2025-02-25T23:58:10.130Z [ERROR] server-1ba7170a-d1e8-3e50-b43e-2e98811f922f: failed to get log: index=3 error="log not found" +2025-02-25T23:58:10.130Z [INFO] creating new snapshot: path=/tmp/raft23391051/snapshots/2-14-1740527890130.tmp +2025-02-25T23:58:10.130Z [INFO] server-6df326c5-2961-e90e-3f18-3788b6ca74e8: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:10.130Z [INFO] server-6df326c5-2961-e90e-3f18-3788b6ca74e8: copied to local snapshot: bytes=71 +2025-02-25T23:58:10.130Z [INFO] server-6df326c5-2961-e90e-3f18-3788b6ca74e8: snapshot restore progress: id=2-14-1740527890130 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:10.131Z [INFO] server-6df326c5-2961-e90e-3f18-3788b6ca74e8: Installed remote snapshot +2025-02-25T23:58:10.131Z [INFO] server-1ba7170a-d1e8-3e50-b43e-2e98811f922f: pipelining replication: peer="{Voter server-6df326c5-2961-e90e-3f18-3788b6ca74e8 6df326c5-2961-e90e-3f18-3788b6ca74e8}" +2025-02-25T23:58:10.131Z [INFO] server-1572a6b5-406d-26f9-a038-4e508fa08a88: snapshot restore progress: id=2-14-1740527890130 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:10.131Z [INFO] server-1572a6b5-406d-26f9-a038-4e508fa08a88: Installed remote snapshot +2025-02-25T23:58:10.131Z [INFO] server-1ba7170a-d1e8-3e50-b43e-2e98811f922f: pipelining replication: peer="{Voter server-1572a6b5-406d-26f9-a038-4e508fa08a88 1572a6b5-406d-26f9-a038-4e508fa08a88}" +2025-02-25T23:58:10.131Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:58:11.138Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:58:10.131272915 +0000 UTC m=+37.740288322 (2 nodes), 440ns from start of poll, 1.536255064s from cluster start. Timeout at 2025-02-25 23:58:11.136279595 +0000 UTC m=+38.745295002, 1.00500668s after stability +2025-02-25T23:58:11.154Z [INFO] server-22d8f678-81f7-da10-825c-4136a55e8849: aborting pipeline replication: peer="{Voter server-bbc5405b-3c84-1627-730e-4432acd1f826 bbc5405b-3c84-1627-730e-4432acd1f826}" +2025-02-25T23:58:11.154Z [INFO] server-22d8f678-81f7-da10-825c-4136a55e8849: aborting pipeline replication: peer="{Voter server-29aa60a8-a0c8-e70f-76bb-6f93cbf413fa 29aa60a8-a0c8-e70f-76bb-6f93cbf413fa}" === RUN TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_true -2026-03-31T06:16:49.620Z [INFO] server-c69cb052-0d27-1ea9-113e-a81a09f0fa93: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c69cb052-0d27-1ea9-113e-a81a09f0fa93 Address:c69cb052-0d27-1ea9-113e-a81a09f0fa93} {Suffrage:Voter ID:server-29f61a21-4097-8f91-9dd8-1aa115d62c1b Address:29f61a21-4097-8f91-9dd8-1aa115d62c1b} {Suffrage:Voter ID:server-b249238b-5ec9-11b5-2fc6-2f7a7a87a2da Address:b249238b-5ec9-11b5-2fc6-2f7a7a87a2da}]" -2026-03-31T06:16:49.620Z [INFO] server-29f61a21-4097-8f91-9dd8-1aa115d62c1b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c69cb052-0d27-1ea9-113e-a81a09f0fa93 Address:c69cb052-0d27-1ea9-113e-a81a09f0fa93} {Suffrage:Voter ID:server-29f61a21-4097-8f91-9dd8-1aa115d62c1b Address:29f61a21-4097-8f91-9dd8-1aa115d62c1b} {Suffrage:Voter ID:server-b249238b-5ec9-11b5-2fc6-2f7a7a87a2da Address:b249238b-5ec9-11b5-2fc6-2f7a7a87a2da}]" -2026-03-31T06:16:49.620Z [INFO] server-c69cb052-0d27-1ea9-113e-a81a09f0fa93: entering follower state: follower="Node at c69cb052-0d27-1ea9-113e-a81a09f0fa93 [Follower]" leader-address= leader-id= -2026-03-31T06:16:49.620Z [INFO] server-29f61a21-4097-8f91-9dd8-1aa115d62c1b: entering follower state: follower="Node at 29f61a21-4097-8f91-9dd8-1aa115d62c1b [Follower]" leader-address= leader-id= -2026-03-31T06:16:49.620Z [INFO] server-b249238b-5ec9-11b5-2fc6-2f7a7a87a2da: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c69cb052-0d27-1ea9-113e-a81a09f0fa93 Address:c69cb052-0d27-1ea9-113e-a81a09f0fa93} {Suffrage:Voter ID:server-29f61a21-4097-8f91-9dd8-1aa115d62c1b Address:29f61a21-4097-8f91-9dd8-1aa115d62c1b} {Suffrage:Voter ID:server-b249238b-5ec9-11b5-2fc6-2f7a7a87a2da Address:b249238b-5ec9-11b5-2fc6-2f7a7a87a2da}]" -2026-03-31T06:16:49.620Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:49.620Z [INFO] server-b249238b-5ec9-11b5-2fc6-2f7a7a87a2da: entering follower state: follower="Node at b249238b-5ec9-11b5-2fc6-2f7a7a87a2da [Follower]" leader-address= leader-id= -2026-03-31T06:16:50.331Z [WARN] server-b249238b-5ec9-11b5-2fc6-2f7a7a87a2da: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:50.331Z [INFO] server-b249238b-5ec9-11b5-2fc6-2f7a7a87a2da: entering candidate state: node="Node at b249238b-5ec9-11b5-2fc6-2f7a7a87a2da [Candidate]" term=2 -2026-03-31T06:16:50.331Z [INFO] server-b249238b-5ec9-11b5-2fc6-2f7a7a87a2da: election won: term=2 tally=2 -2026-03-31T06:16:50.331Z [INFO] server-b249238b-5ec9-11b5-2fc6-2f7a7a87a2da: entering leader state: leader="Node at b249238b-5ec9-11b5-2fc6-2f7a7a87a2da [Leader]" -2026-03-31T06:16:50.331Z [INFO] server-b249238b-5ec9-11b5-2fc6-2f7a7a87a2da: added peer, starting replication: peer=server-c69cb052-0d27-1ea9-113e-a81a09f0fa93 -2026-03-31T06:16:50.331Z [INFO] server-b249238b-5ec9-11b5-2fc6-2f7a7a87a2da: added peer, starting replication: peer=server-29f61a21-4097-8f91-9dd8-1aa115d62c1b -2026-03-31T06:16:50.331Z [INFO] server-b249238b-5ec9-11b5-2fc6-2f7a7a87a2da: pipelining replication: peer="{Voter server-c69cb052-0d27-1ea9-113e-a81a09f0fa93 c69cb052-0d27-1ea9-113e-a81a09f0fa93}" -2026-03-31T06:16:50.331Z [INFO] server-b249238b-5ec9-11b5-2fc6-2f7a7a87a2da: pipelining replication: peer="{Voter server-29f61a21-4097-8f91-9dd8-1aa115d62c1b 29f61a21-4097-8f91-9dd8-1aa115d62c1b}" -2026-03-31T06:16:51.336Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:50.33150638 +0000 UTC m=+38.609113761 (1 nodes), 710.79532ms from start of poll, 711.21264ms from cluster start. Timeout at 2026-03-31 06:16:51.33651662 +0000 UTC m=+39.614124001, 1.00501024s after stability -2026-03-31T06:16:51.337Z [INFO] server-b249238b-5ec9-11b5-2fc6-2f7a7a87a2da: starting snapshot up to: index=12 -2026-03-31T06:16:51.337Z [INFO] creating new snapshot: path=/tmp/raft810639377/snapshots/2-12-1774937811337.tmp -2026-03-31T06:16:51.337Z [INFO] server-b249238b-5ec9-11b5-2fc6-2f7a7a87a2da: snapshot complete up to: index=12 -2026-03-31T06:16:51.338Z [INFO] server-4b659f42-3962-81fd-8af2-c8865b770f62: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4b659f42-3962-81fd-8af2-c8865b770f62 Address:4b659f42-3962-81fd-8af2-c8865b770f62} {Suffrage:Voter ID:server-0f4237d3-95a2-ffe7-11b5-b9b630241781 Address:0f4237d3-95a2-ffe7-11b5-b9b630241781} {Suffrage:Voter ID:server-1f568bcf-75bf-a19a-4ea3-6b75ad05af3f Address:1f568bcf-75bf-a19a-4ea3-6b75ad05af3f}]" -2026-03-31T06:16:51.338Z [INFO] server-0f4237d3-95a2-ffe7-11b5-b9b630241781: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4b659f42-3962-81fd-8af2-c8865b770f62 Address:4b659f42-3962-81fd-8af2-c8865b770f62} {Suffrage:Voter ID:server-0f4237d3-95a2-ffe7-11b5-b9b630241781 Address:0f4237d3-95a2-ffe7-11b5-b9b630241781} {Suffrage:Voter ID:server-1f568bcf-75bf-a19a-4ea3-6b75ad05af3f Address:1f568bcf-75bf-a19a-4ea3-6b75ad05af3f}]" -2026-03-31T06:16:51.338Z [INFO] server-4b659f42-3962-81fd-8af2-c8865b770f62: entering follower state: follower="Node at 4b659f42-3962-81fd-8af2-c8865b770f62 [Follower]" leader-address= leader-id= -2026-03-31T06:16:51.339Z [INFO] server-0f4237d3-95a2-ffe7-11b5-b9b630241781: entering follower state: follower="Node at 0f4237d3-95a2-ffe7-11b5-b9b630241781 [Follower]" leader-address= leader-id= -2026-03-31T06:16:51.339Z [INFO] server-1f568bcf-75bf-a19a-4ea3-6b75ad05af3f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4b659f42-3962-81fd-8af2-c8865b770f62 Address:4b659f42-3962-81fd-8af2-c8865b770f62} {Suffrage:Voter ID:server-0f4237d3-95a2-ffe7-11b5-b9b630241781 Address:0f4237d3-95a2-ffe7-11b5-b9b630241781} {Suffrage:Voter ID:server-1f568bcf-75bf-a19a-4ea3-6b75ad05af3f Address:1f568bcf-75bf-a19a-4ea3-6b75ad05af3f}]" -2026-03-31T06:16:51.339Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:51.339Z [INFO] server-1f568bcf-75bf-a19a-4ea3-6b75ad05af3f: entering follower state: follower="Node at 1f568bcf-75bf-a19a-4ea3-6b75ad05af3f [Follower]" leader-address= leader-id= -2026-03-31T06:16:51.901Z [WARN] server-0f4237d3-95a2-ffe7-11b5-b9b630241781: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:51.901Z [INFO] server-0f4237d3-95a2-ffe7-11b5-b9b630241781: entering candidate state: node="Node at 0f4237d3-95a2-ffe7-11b5-b9b630241781 [Candidate]" term=2 -2026-03-31T06:16:51.901Z [INFO] server-0f4237d3-95a2-ffe7-11b5-b9b630241781: election won: term=2 tally=2 -2026-03-31T06:16:51.901Z [INFO] server-0f4237d3-95a2-ffe7-11b5-b9b630241781: entering leader state: leader="Node at 0f4237d3-95a2-ffe7-11b5-b9b630241781 [Leader]" -2026-03-31T06:16:51.901Z [INFO] server-0f4237d3-95a2-ffe7-11b5-b9b630241781: added peer, starting replication: peer=server-4b659f42-3962-81fd-8af2-c8865b770f62 -2026-03-31T06:16:51.901Z [INFO] server-0f4237d3-95a2-ffe7-11b5-b9b630241781: added peer, starting replication: peer=server-1f568bcf-75bf-a19a-4ea3-6b75ad05af3f -2026-03-31T06:16:51.902Z [INFO] server-0f4237d3-95a2-ffe7-11b5-b9b630241781: pipelining replication: peer="{Voter server-1f568bcf-75bf-a19a-4ea3-6b75ad05af3f 1f568bcf-75bf-a19a-4ea3-6b75ad05af3f}" -2026-03-31T06:16:51.902Z [INFO] server-0f4237d3-95a2-ffe7-11b5-b9b630241781: pipelining replication: peer="{Voter server-4b659f42-3962-81fd-8af2-c8865b770f62 4b659f42-3962-81fd-8af2-c8865b770f62}" -2026-03-31T06:16:52.907Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:51.9019755 +0000 UTC m=+40.179582881 (1 nodes), 562.79412ms from start of poll, 563.27096ms from cluster start. Timeout at 2026-03-31 06:16:52.90697702 +0000 UTC m=+41.184584401, 1.00500152s after stability -2026-03-31T06:16:52.908Z [INFO] creating new snapshot: path=/tmp/raft2866743044/snapshots/2-14-1774937812908.tmp -2026-03-31T06:16:52.909Z [INFO] server-0f4237d3-95a2-ffe7-11b5-b9b630241781: copied to local snapshot: bytes=71 -2026-03-31T06:16:52.909Z [INFO] server-0f4237d3-95a2-ffe7-11b5-b9b630241781: snapshot restore progress: id=2-14-1774937812908 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:52.909Z [INFO] server-0f4237d3-95a2-ffe7-11b5-b9b630241781: removing all old logs from log store -2026-03-31T06:16:52.909Z [INFO] server-0f4237d3-95a2-ffe7-11b5-b9b630241781: compacting logs: from=1 to=2 -2026-03-31T06:16:52.909Z [INFO] server-0f4237d3-95a2-ffe7-11b5-b9b630241781: restored user snapshot: index=14 -2026-03-31T06:16:52.909Z [ERROR] server-0f4237d3-95a2-ffe7-11b5-b9b630241781: failed to get log: index=2 error="log not found" -2026-03-31T06:16:52.909Z [INFO] server-0f4237d3-95a2-ffe7-11b5-b9b630241781: aborting pipeline replication: peer="{Voter server-4b659f42-3962-81fd-8af2-c8865b770f62 4b659f42-3962-81fd-8af2-c8865b770f62}" -2026-03-31T06:16:52.909Z [ERROR] server-0f4237d3-95a2-ffe7-11b5-b9b630241781: failed to get log: index=2 error="log not found" -2026-03-31T06:16:52.909Z [INFO] server-0f4237d3-95a2-ffe7-11b5-b9b630241781: aborting pipeline replication: peer="{Voter server-1f568bcf-75bf-a19a-4ea3-6b75ad05af3f 1f568bcf-75bf-a19a-4ea3-6b75ad05af3f}" -2026-03-31T06:16:52.919Z [ERROR] server-0f4237d3-95a2-ffe7-11b5-b9b630241781: failed to get log: index=2 error="log not found" -2026-03-31T06:16:52.919Z [ERROR] server-0f4237d3-95a2-ffe7-11b5-b9b630241781: failed to get log: index=2 error="log not found" -2026-03-31T06:16:52.920Z [INFO] creating new snapshot: path=/tmp/raft901924995/snapshots/2-14-1774937812920.tmp -2026-03-31T06:16:52.920Z [INFO] creating new snapshot: path=/tmp/raft913811481/snapshots/2-14-1774937812920.tmp -2026-03-31T06:16:52.920Z [INFO] server-1f568bcf-75bf-a19a-4ea3-6b75ad05af3f: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:52.920Z [INFO] server-4b659f42-3962-81fd-8af2-c8865b770f62: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:52.920Z [INFO] server-1f568bcf-75bf-a19a-4ea3-6b75ad05af3f: copied to local snapshot: bytes=71 -2026-03-31T06:16:52.920Z [INFO] server-4b659f42-3962-81fd-8af2-c8865b770f62: copied to local snapshot: bytes=71 -2026-03-31T06:16:52.920Z [INFO] server-1f568bcf-75bf-a19a-4ea3-6b75ad05af3f: snapshot restore progress: id=2-14-1774937812920 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:52.920Z [INFO] server-1f568bcf-75bf-a19a-4ea3-6b75ad05af3f: removing all old logs from log store -2026-03-31T06:16:52.920Z [INFO] server-1f568bcf-75bf-a19a-4ea3-6b75ad05af3f: compacting logs: from=1 to=2 -2026-03-31T06:16:52.921Z [INFO] server-1f568bcf-75bf-a19a-4ea3-6b75ad05af3f: Installed remote snapshot -2026-03-31T06:16:52.920Z [INFO] server-4b659f42-3962-81fd-8af2-c8865b770f62: snapshot restore progress: id=2-14-1774937812920 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:52.921Z [INFO] server-4b659f42-3962-81fd-8af2-c8865b770f62: removing all old logs from log store -2026-03-31T06:16:52.921Z [INFO] server-4b659f42-3962-81fd-8af2-c8865b770f62: compacting logs: from=1 to=2 -2026-03-31T06:16:52.921Z [INFO] server-4b659f42-3962-81fd-8af2-c8865b770f62: Installed remote snapshot -2026-03-31T06:16:52.921Z [INFO] server-0f4237d3-95a2-ffe7-11b5-b9b630241781: pipelining replication: peer="{Voter server-1f568bcf-75bf-a19a-4ea3-6b75ad05af3f 1f568bcf-75bf-a19a-4ea3-6b75ad05af3f}" -2026-03-31T06:16:52.921Z [INFO] server-0f4237d3-95a2-ffe7-11b5-b9b630241781: pipelining replication: peer="{Voter server-4b659f42-3962-81fd-8af2-c8865b770f62 4b659f42-3962-81fd-8af2-c8865b770f62}" -2026-03-31T06:16:52.921Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:16:53.926Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:16:52.92122878 +0000 UTC m=+41.198836161 (2 nodes), 1.04µs from start of poll, 1.58252424s from cluster start. Timeout at 2026-03-31 06:16:53.92623334 +0000 UTC m=+42.203840681, 1.00500452s after stability -2026-03-31T06:16:53.938Z [INFO] server-b249238b-5ec9-11b5-2fc6-2f7a7a87a2da: aborting pipeline replication: peer="{Voter server-29f61a21-4097-8f91-9dd8-1aa115d62c1b 29f61a21-4097-8f91-9dd8-1aa115d62c1b}" -2026-03-31T06:16:53.938Z [INFO] server-b249238b-5ec9-11b5-2fc6-2f7a7a87a2da: aborting pipeline replication: peer="{Voter server-c69cb052-0d27-1ea9-113e-a81a09f0fa93 c69cb052-0d27-1ea9-113e-a81a09f0fa93}" +2025-02-25T23:58:11.154Z [INFO] server-6b540eb5-6f41-076f-3311-60dfd45b3b32: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6b540eb5-6f41-076f-3311-60dfd45b3b32 Address:6b540eb5-6f41-076f-3311-60dfd45b3b32} {Suffrage:Voter ID:server-4a92bbc7-8441-0159-3133-f8d5d408323b Address:4a92bbc7-8441-0159-3133-f8d5d408323b} {Suffrage:Voter ID:server-e20b79de-316f-62e9-e651-37596be0aeb9 Address:e20b79de-316f-62e9-e651-37596be0aeb9}]" +2025-02-25T23:58:11.154Z [INFO] server-4a92bbc7-8441-0159-3133-f8d5d408323b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6b540eb5-6f41-076f-3311-60dfd45b3b32 Address:6b540eb5-6f41-076f-3311-60dfd45b3b32} {Suffrage:Voter ID:server-4a92bbc7-8441-0159-3133-f8d5d408323b Address:4a92bbc7-8441-0159-3133-f8d5d408323b} {Suffrage:Voter ID:server-e20b79de-316f-62e9-e651-37596be0aeb9 Address:e20b79de-316f-62e9-e651-37596be0aeb9}]" +2025-02-25T23:58:11.154Z [INFO] server-e20b79de-316f-62e9-e651-37596be0aeb9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6b540eb5-6f41-076f-3311-60dfd45b3b32 Address:6b540eb5-6f41-076f-3311-60dfd45b3b32} {Suffrage:Voter ID:server-4a92bbc7-8441-0159-3133-f8d5d408323b Address:4a92bbc7-8441-0159-3133-f8d5d408323b} {Suffrage:Voter ID:server-e20b79de-316f-62e9-e651-37596be0aeb9 Address:e20b79de-316f-62e9-e651-37596be0aeb9}]" +2025-02-25T23:58:11.154Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:58:11.154Z [INFO] server-6b540eb5-6f41-076f-3311-60dfd45b3b32: entering follower state: follower="Node at 6b540eb5-6f41-076f-3311-60dfd45b3b32 [Follower]" leader-address= leader-id= +2025-02-25T23:58:11.154Z [INFO] server-4a92bbc7-8441-0159-3133-f8d5d408323b: entering follower state: follower="Node at 4a92bbc7-8441-0159-3133-f8d5d408323b [Follower]" leader-address= leader-id= +2025-02-25T23:58:11.154Z [INFO] server-e20b79de-316f-62e9-e651-37596be0aeb9: entering follower state: follower="Node at e20b79de-316f-62e9-e651-37596be0aeb9 [Follower]" leader-address= leader-id= +2025-02-25T23:58:11.898Z [WARN] server-e20b79de-316f-62e9-e651-37596be0aeb9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:58:11.898Z [INFO] server-e20b79de-316f-62e9-e651-37596be0aeb9: entering candidate state: node="Node at e20b79de-316f-62e9-e651-37596be0aeb9 [Candidate]" term=2 +2025-02-25T23:58:11.898Z [INFO] server-e20b79de-316f-62e9-e651-37596be0aeb9: election won: term=2 tally=2 +2025-02-25T23:58:11.898Z [INFO] server-e20b79de-316f-62e9-e651-37596be0aeb9: entering leader state: leader="Node at e20b79de-316f-62e9-e651-37596be0aeb9 [Leader]" +2025-02-25T23:58:11.898Z [INFO] server-e20b79de-316f-62e9-e651-37596be0aeb9: added peer, starting replication: peer=server-6b540eb5-6f41-076f-3311-60dfd45b3b32 +2025-02-25T23:58:11.898Z [INFO] server-e20b79de-316f-62e9-e651-37596be0aeb9: added peer, starting replication: peer=server-4a92bbc7-8441-0159-3133-f8d5d408323b +2025-02-25T23:58:11.898Z [INFO] server-e20b79de-316f-62e9-e651-37596be0aeb9: pipelining replication: peer="{Voter server-4a92bbc7-8441-0159-3133-f8d5d408323b 4a92bbc7-8441-0159-3133-f8d5d408323b}" +2025-02-25T23:58:11.898Z [INFO] server-e20b79de-316f-62e9-e651-37596be0aeb9: pipelining replication: peer="{Voter server-6b540eb5-6f41-076f-3311-60dfd45b3b32 6b540eb5-6f41-076f-3311-60dfd45b3b32}" +2025-02-25T23:58:12.905Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:58:11.898352848 +0000 UTC m=+39.507368255 (1 nodes), 743.708256ms from start of poll, 743.893738ms from cluster start. Timeout at 2025-02-25 23:58:12.903355088 +0000 UTC m=+40.512370495, 1.00500224s after stability +2025-02-25T23:58:12.906Z [INFO] server-e20b79de-316f-62e9-e651-37596be0aeb9: starting snapshot up to: index=12 +2025-02-25T23:58:12.906Z [INFO] creating new snapshot: path=/tmp/raft3115987250/snapshots/2-12-1740527892906.tmp +2025-02-25T23:58:12.906Z [INFO] server-e20b79de-316f-62e9-e651-37596be0aeb9: snapshot complete up to: index=12 +2025-02-25T23:58:12.907Z [INFO] server-c3dc7a54-109f-6c9c-cd8c-bebbc6d2b2fa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c3dc7a54-109f-6c9c-cd8c-bebbc6d2b2fa Address:c3dc7a54-109f-6c9c-cd8c-bebbc6d2b2fa} {Suffrage:Voter ID:server-7573cadb-52ba-02dd-728b-15e4593e6ee8 Address:7573cadb-52ba-02dd-728b-15e4593e6ee8} {Suffrage:Voter ID:server-a1e124fd-b296-4a82-0346-5068818db6c8 Address:a1e124fd-b296-4a82-0346-5068818db6c8}]" +2025-02-25T23:58:12.907Z [INFO] server-7573cadb-52ba-02dd-728b-15e4593e6ee8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c3dc7a54-109f-6c9c-cd8c-bebbc6d2b2fa Address:c3dc7a54-109f-6c9c-cd8c-bebbc6d2b2fa} {Suffrage:Voter ID:server-7573cadb-52ba-02dd-728b-15e4593e6ee8 Address:7573cadb-52ba-02dd-728b-15e4593e6ee8} {Suffrage:Voter ID:server-a1e124fd-b296-4a82-0346-5068818db6c8 Address:a1e124fd-b296-4a82-0346-5068818db6c8}]" +2025-02-25T23:58:12.907Z [INFO] server-a1e124fd-b296-4a82-0346-5068818db6c8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c3dc7a54-109f-6c9c-cd8c-bebbc6d2b2fa Address:c3dc7a54-109f-6c9c-cd8c-bebbc6d2b2fa} {Suffrage:Voter ID:server-7573cadb-52ba-02dd-728b-15e4593e6ee8 Address:7573cadb-52ba-02dd-728b-15e4593e6ee8} {Suffrage:Voter ID:server-a1e124fd-b296-4a82-0346-5068818db6c8 Address:a1e124fd-b296-4a82-0346-5068818db6c8}]" +2025-02-25T23:58:12.907Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:58:12.907Z [INFO] server-c3dc7a54-109f-6c9c-cd8c-bebbc6d2b2fa: entering follower state: follower="Node at c3dc7a54-109f-6c9c-cd8c-bebbc6d2b2fa [Follower]" leader-address= leader-id= +2025-02-25T23:58:12.907Z [INFO] server-7573cadb-52ba-02dd-728b-15e4593e6ee8: entering follower state: follower="Node at 7573cadb-52ba-02dd-728b-15e4593e6ee8 [Follower]" leader-address= leader-id= +2025-02-25T23:58:12.907Z [INFO] server-a1e124fd-b296-4a82-0346-5068818db6c8: entering follower state: follower="Node at a1e124fd-b296-4a82-0346-5068818db6c8 [Follower]" leader-address= leader-id= +2025-02-25T23:58:13.697Z [WARN] server-c3dc7a54-109f-6c9c-cd8c-bebbc6d2b2fa: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:58:13.697Z [INFO] server-c3dc7a54-109f-6c9c-cd8c-bebbc6d2b2fa: entering candidate state: node="Node at c3dc7a54-109f-6c9c-cd8c-bebbc6d2b2fa [Candidate]" term=2 +2025-02-25T23:58:13.698Z [INFO] server-c3dc7a54-109f-6c9c-cd8c-bebbc6d2b2fa: election won: term=2 tally=2 +2025-02-25T23:58:13.698Z [INFO] server-c3dc7a54-109f-6c9c-cd8c-bebbc6d2b2fa: entering leader state: leader="Node at c3dc7a54-109f-6c9c-cd8c-bebbc6d2b2fa [Leader]" +2025-02-25T23:58:13.698Z [INFO] server-c3dc7a54-109f-6c9c-cd8c-bebbc6d2b2fa: added peer, starting replication: peer=server-7573cadb-52ba-02dd-728b-15e4593e6ee8 +2025-02-25T23:58:13.698Z [INFO] server-c3dc7a54-109f-6c9c-cd8c-bebbc6d2b2fa: added peer, starting replication: peer=server-a1e124fd-b296-4a82-0346-5068818db6c8 +2025-02-25T23:58:13.698Z [INFO] server-c3dc7a54-109f-6c9c-cd8c-bebbc6d2b2fa: pipelining replication: peer="{Voter server-a1e124fd-b296-4a82-0346-5068818db6c8 a1e124fd-b296-4a82-0346-5068818db6c8}" +2025-02-25T23:58:13.698Z [INFO] server-c3dc7a54-109f-6c9c-cd8c-bebbc6d2b2fa: pipelining replication: peer="{Voter server-7573cadb-52ba-02dd-728b-15e4593e6ee8 7573cadb-52ba-02dd-728b-15e4593e6ee8}" +2025-02-25T23:58:14.705Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:58:13.69825495 +0000 UTC m=+41.307270357 (1 nodes), 790.821396ms from start of poll, 790.985077ms from cluster start. Timeout at 2025-02-25 23:58:14.70326179 +0000 UTC m=+42.312277197, 1.00500684s after stability +2025-02-25T23:58:14.706Z [INFO] creating new snapshot: path=/tmp/raft1955199245/snapshots/2-14-1740527894706.tmp +2025-02-25T23:58:14.706Z [INFO] server-c3dc7a54-109f-6c9c-cd8c-bebbc6d2b2fa: copied to local snapshot: bytes=71 +2025-02-25T23:58:14.706Z [INFO] server-c3dc7a54-109f-6c9c-cd8c-bebbc6d2b2fa: snapshot restore progress: id=2-14-1740527894706 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:14.706Z [INFO] server-c3dc7a54-109f-6c9c-cd8c-bebbc6d2b2fa: removing all old logs from log store +2025-02-25T23:58:14.706Z [INFO] server-c3dc7a54-109f-6c9c-cd8c-bebbc6d2b2fa: compacting logs: from=1 to=2 +2025-02-25T23:58:14.706Z [INFO] server-c3dc7a54-109f-6c9c-cd8c-bebbc6d2b2fa: restored user snapshot: index=14 +2025-02-25T23:58:14.706Z [ERROR] server-c3dc7a54-109f-6c9c-cd8c-bebbc6d2b2fa: failed to get log: index=2 error="log not found" +2025-02-25T23:58:14.706Z [INFO] server-c3dc7a54-109f-6c9c-cd8c-bebbc6d2b2fa: aborting pipeline replication: peer="{Voter server-a1e124fd-b296-4a82-0346-5068818db6c8 a1e124fd-b296-4a82-0346-5068818db6c8}" +2025-02-25T23:58:14.706Z [ERROR] server-c3dc7a54-109f-6c9c-cd8c-bebbc6d2b2fa: failed to get log: index=2 error="log not found" +2025-02-25T23:58:14.706Z [INFO] server-c3dc7a54-109f-6c9c-cd8c-bebbc6d2b2fa: aborting pipeline replication: peer="{Voter server-7573cadb-52ba-02dd-728b-15e4593e6ee8 7573cadb-52ba-02dd-728b-15e4593e6ee8}" +2025-02-25T23:58:14.717Z [ERROR] server-c3dc7a54-109f-6c9c-cd8c-bebbc6d2b2fa: failed to get log: index=2 error="log not found" +2025-02-25T23:58:14.718Z [INFO] creating new snapshot: path=/tmp/raft175191100/snapshots/2-14-1740527894718.tmp +2025-02-25T23:58:14.718Z [INFO] server-a1e124fd-b296-4a82-0346-5068818db6c8: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:14.718Z [INFO] server-a1e124fd-b296-4a82-0346-5068818db6c8: copied to local snapshot: bytes=71 +2025-02-25T23:58:14.720Z [ERROR] server-c3dc7a54-109f-6c9c-cd8c-bebbc6d2b2fa: failed to get log: index=2 error="log not found" +2025-02-25T23:58:14.720Z [INFO] creating new snapshot: path=/tmp/raft2915920842/snapshots/2-14-1740527894720.tmp +2025-02-25T23:58:14.721Z [INFO] server-7573cadb-52ba-02dd-728b-15e4593e6ee8: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:14.721Z [INFO] server-7573cadb-52ba-02dd-728b-15e4593e6ee8: copied to local snapshot: bytes=71 +2025-02-25T23:58:14.721Z [INFO] server-7573cadb-52ba-02dd-728b-15e4593e6ee8: snapshot restore progress: id=2-14-1740527894720 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:14.721Z [INFO] server-7573cadb-52ba-02dd-728b-15e4593e6ee8: removing all old logs from log store +2025-02-25T23:58:14.721Z [INFO] server-7573cadb-52ba-02dd-728b-15e4593e6ee8: compacting logs: from=1 to=2 +2025-02-25T23:58:14.721Z [INFO] server-7573cadb-52ba-02dd-728b-15e4593e6ee8: Installed remote snapshot +2025-02-25T23:58:14.721Z [INFO] server-c3dc7a54-109f-6c9c-cd8c-bebbc6d2b2fa: pipelining replication: peer="{Voter server-7573cadb-52ba-02dd-728b-15e4593e6ee8 7573cadb-52ba-02dd-728b-15e4593e6ee8}" +2025-02-25T23:58:14.721Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:58:14.721Z [INFO] server-a1e124fd-b296-4a82-0346-5068818db6c8: snapshot restore progress: id=2-14-1740527894718 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:14.721Z [INFO] server-a1e124fd-b296-4a82-0346-5068818db6c8: removing all old logs from log store +2025-02-25T23:58:14.721Z [INFO] server-a1e124fd-b296-4a82-0346-5068818db6c8: compacting logs: from=1 to=2 +2025-02-25T23:58:14.721Z [INFO] server-a1e124fd-b296-4a82-0346-5068818db6c8: Installed remote snapshot +2025-02-25T23:58:14.721Z [INFO] server-c3dc7a54-109f-6c9c-cd8c-bebbc6d2b2fa: pipelining replication: peer="{Voter server-a1e124fd-b296-4a82-0346-5068818db6c8 a1e124fd-b296-4a82-0346-5068818db6c8}" +2025-02-25T23:58:15.729Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:58:14.721427825 +0000 UTC m=+42.330443232 (2 nodes), 640ns from start of poll, 1.814157952s from cluster start. Timeout at 2025-02-25 23:58:15.726448425 +0000 UTC m=+43.335463832, 1.0050206s after stability +2025-02-25T23:58:15.746Z [INFO] server-e20b79de-316f-62e9-e651-37596be0aeb9: aborting pipeline replication: peer="{Voter server-4a92bbc7-8441-0159-3133-f8d5d408323b 4a92bbc7-8441-0159-3133-f8d5d408323b}" +2025-02-25T23:58:15.746Z [INFO] server-e20b79de-316f-62e9-e651-37596be0aeb9: aborting pipeline replication: peer="{Voter server-6b540eb5-6f41-076f-3311-60dfd45b3b32 6b540eb5-6f41-076f-3311-60dfd45b3b32}" === RUN TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_false -2026-03-31T06:16:53.939Z [INFO] server-8eb8fab8-b3f5-a50a-2cfb-0dc5b7ce71e5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8eb8fab8-b3f5-a50a-2cfb-0dc5b7ce71e5 Address:8eb8fab8-b3f5-a50a-2cfb-0dc5b7ce71e5} {Suffrage:Voter ID:server-c75c71bc-c67b-8099-23cc-d9240b18590b Address:c75c71bc-c67b-8099-23cc-d9240b18590b} {Suffrage:Voter ID:server-c655b3df-3990-aff6-7632-79deda5c51c3 Address:c655b3df-3990-aff6-7632-79deda5c51c3}]" -2026-03-31T06:16:53.939Z [INFO] server-c75c71bc-c67b-8099-23cc-d9240b18590b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8eb8fab8-b3f5-a50a-2cfb-0dc5b7ce71e5 Address:8eb8fab8-b3f5-a50a-2cfb-0dc5b7ce71e5} {Suffrage:Voter ID:server-c75c71bc-c67b-8099-23cc-d9240b18590b Address:c75c71bc-c67b-8099-23cc-d9240b18590b} {Suffrage:Voter ID:server-c655b3df-3990-aff6-7632-79deda5c51c3 Address:c655b3df-3990-aff6-7632-79deda5c51c3}]" -2026-03-31T06:16:53.939Z [INFO] server-8eb8fab8-b3f5-a50a-2cfb-0dc5b7ce71e5: entering follower state: follower="Node at 8eb8fab8-b3f5-a50a-2cfb-0dc5b7ce71e5 [Follower]" leader-address= leader-id= -2026-03-31T06:16:53.939Z [INFO] server-c75c71bc-c67b-8099-23cc-d9240b18590b: entering follower state: follower="Node at c75c71bc-c67b-8099-23cc-d9240b18590b [Follower]" leader-address= leader-id= -2026-03-31T06:16:53.939Z [INFO] server-c655b3df-3990-aff6-7632-79deda5c51c3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8eb8fab8-b3f5-a50a-2cfb-0dc5b7ce71e5 Address:8eb8fab8-b3f5-a50a-2cfb-0dc5b7ce71e5} {Suffrage:Voter ID:server-c75c71bc-c67b-8099-23cc-d9240b18590b Address:c75c71bc-c67b-8099-23cc-d9240b18590b} {Suffrage:Voter ID:server-c655b3df-3990-aff6-7632-79deda5c51c3 Address:c655b3df-3990-aff6-7632-79deda5c51c3}]" -2026-03-31T06:16:53.939Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:53.939Z [INFO] server-c655b3df-3990-aff6-7632-79deda5c51c3: entering follower state: follower="Node at c655b3df-3990-aff6-7632-79deda5c51c3 [Follower]" leader-address= leader-id= -2026-03-31T06:16:54.777Z [WARN] server-c75c71bc-c67b-8099-23cc-d9240b18590b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:54.777Z [INFO] server-c75c71bc-c67b-8099-23cc-d9240b18590b: entering candidate state: node="Node at c75c71bc-c67b-8099-23cc-d9240b18590b [Candidate]" term=2 -2026-03-31T06:16:54.777Z [INFO] server-c75c71bc-c67b-8099-23cc-d9240b18590b: election won: term=2 tally=2 -2026-03-31T06:16:54.777Z [INFO] server-c75c71bc-c67b-8099-23cc-d9240b18590b: entering leader state: leader="Node at c75c71bc-c67b-8099-23cc-d9240b18590b [Leader]" -2026-03-31T06:16:54.778Z [INFO] server-c75c71bc-c67b-8099-23cc-d9240b18590b: added peer, starting replication: peer=server-8eb8fab8-b3f5-a50a-2cfb-0dc5b7ce71e5 -2026-03-31T06:16:54.778Z [INFO] server-c75c71bc-c67b-8099-23cc-d9240b18590b: added peer, starting replication: peer=server-c655b3df-3990-aff6-7632-79deda5c51c3 -2026-03-31T06:16:54.778Z [INFO] server-c75c71bc-c67b-8099-23cc-d9240b18590b: pipelining replication: peer="{Voter server-c655b3df-3990-aff6-7632-79deda5c51c3 c655b3df-3990-aff6-7632-79deda5c51c3}" -2026-03-31T06:16:54.778Z [INFO] server-c75c71bc-c67b-8099-23cc-d9240b18590b: pipelining replication: peer="{Voter server-8eb8fab8-b3f5-a50a-2cfb-0dc5b7ce71e5 8eb8fab8-b3f5-a50a-2cfb-0dc5b7ce71e5}" -2026-03-31T06:16:55.783Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:54.77811018 +0000 UTC m=+43.055717601 (1 nodes), 838.51344ms from start of poll, 839.00888ms from cluster start. Timeout at 2026-03-31 06:16:55.78313346 +0000 UTC m=+44.060740841, 1.00502324s after stability -2026-03-31T06:16:55.784Z [INFO] server-c75c71bc-c67b-8099-23cc-d9240b18590b: starting snapshot up to: index=12 -2026-03-31T06:16:55.784Z [INFO] creating new snapshot: path=/tmp/raft1693134070/snapshots/2-12-1774937815784.tmp -2026-03-31T06:16:55.784Z [INFO] server-c75c71bc-c67b-8099-23cc-d9240b18590b: snapshot complete up to: index=12 -2026-03-31T06:16:55.785Z [INFO] creating new snapshot: path=/tmp/raft1693134070/snapshots/2-23-1774937815785.tmp -2026-03-31T06:16:55.786Z [INFO] server-c75c71bc-c67b-8099-23cc-d9240b18590b: copied to local snapshot: bytes=71 -2026-03-31T06:16:55.786Z [INFO] server-c75c71bc-c67b-8099-23cc-d9240b18590b: snapshot restore progress: id=2-23-1774937815785 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:55.786Z [INFO] server-c75c71bc-c67b-8099-23cc-d9240b18590b: restored user snapshot: index=23 -2026-03-31T06:16:55.786Z [ERROR] server-c75c71bc-c67b-8099-23cc-d9240b18590b: failed to get log: index=23 error="log not found" -2026-03-31T06:16:55.786Z [INFO] server-c75c71bc-c67b-8099-23cc-d9240b18590b: aborting pipeline replication: peer="{Voter server-c655b3df-3990-aff6-7632-79deda5c51c3 c655b3df-3990-aff6-7632-79deda5c51c3}" -2026-03-31T06:16:55.786Z [ERROR] server-c75c71bc-c67b-8099-23cc-d9240b18590b: failed to get log: index=23 error="log not found" -2026-03-31T06:16:55.786Z [INFO] server-c75c71bc-c67b-8099-23cc-d9240b18590b: aborting pipeline replication: peer="{Voter server-8eb8fab8-b3f5-a50a-2cfb-0dc5b7ce71e5 8eb8fab8-b3f5-a50a-2cfb-0dc5b7ce71e5}" -2026-03-31T06:16:55.794Z [ERROR] server-c75c71bc-c67b-8099-23cc-d9240b18590b: failed to get log: index=23 error="log not found" -2026-03-31T06:16:55.794Z [INFO] creating new snapshot: path=/tmp/raft2178799551/snapshots/2-23-1774937815794.tmp -2026-03-31T06:16:55.795Z [INFO] server-8eb8fab8-b3f5-a50a-2cfb-0dc5b7ce71e5: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:55.795Z [INFO] server-8eb8fab8-b3f5-a50a-2cfb-0dc5b7ce71e5: copied to local snapshot: bytes=71 -2026-03-31T06:16:55.795Z [ERROR] server-c75c71bc-c67b-8099-23cc-d9240b18590b: failed to get log: index=23 error="log not found" -2026-03-31T06:16:55.795Z [INFO] server-8eb8fab8-b3f5-a50a-2cfb-0dc5b7ce71e5: snapshot restore progress: id=2-23-1774937815794 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:55.796Z [INFO] server-8eb8fab8-b3f5-a50a-2cfb-0dc5b7ce71e5: Installed remote snapshot -2026-03-31T06:16:55.796Z [WARN] server-8eb8fab8-b3f5-a50a-2cfb-0dc5b7ce71e5: skipping application of old log: index=22 -2026-03-31T06:16:55.796Z [INFO] server-c75c71bc-c67b-8099-23cc-d9240b18590b: pipelining replication: peer="{Voter server-8eb8fab8-b3f5-a50a-2cfb-0dc5b7ce71e5 8eb8fab8-b3f5-a50a-2cfb-0dc5b7ce71e5}" -2026-03-31T06:16:55.796Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:16:55.796Z [INFO] creating new snapshot: path=/tmp/raft3677130111/snapshots/2-23-1774937815796.tmp -2026-03-31T06:16:55.796Z [INFO] server-c655b3df-3990-aff6-7632-79deda5c51c3: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:55.796Z [INFO] server-c655b3df-3990-aff6-7632-79deda5c51c3: copied to local snapshot: bytes=71 -2026-03-31T06:16:55.797Z [INFO] server-c655b3df-3990-aff6-7632-79deda5c51c3: snapshot restore progress: id=2-23-1774937815796 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:55.797Z [INFO] server-c655b3df-3990-aff6-7632-79deda5c51c3: Installed remote snapshot -2026-03-31T06:16:55.797Z [INFO] server-c75c71bc-c67b-8099-23cc-d9240b18590b: pipelining replication: peer="{Voter server-c655b3df-3990-aff6-7632-79deda5c51c3 c655b3df-3990-aff6-7632-79deda5c51c3}" -2026-03-31T06:16:56.802Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:16:55.79619798 +0000 UTC m=+44.073805361 (2 nodes), 1.08µs from start of poll, 1.85709664s from cluster start. Timeout at 2026-03-31 06:16:56.80121314 +0000 UTC m=+45.078820521, 1.00501516s after stability -2026-03-31T06:16:56.814Z [INFO] server-c75c71bc-c67b-8099-23cc-d9240b18590b: aborting pipeline replication: peer="{Voter server-c655b3df-3990-aff6-7632-79deda5c51c3 c655b3df-3990-aff6-7632-79deda5c51c3}" -2026-03-31T06:16:56.814Z [INFO] server-c75c71bc-c67b-8099-23cc-d9240b18590b: aborting pipeline replication: peer="{Voter server-8eb8fab8-b3f5-a50a-2cfb-0dc5b7ce71e5 8eb8fab8-b3f5-a50a-2cfb-0dc5b7ce71e5}" +2025-02-25T23:58:15.746Z [INFO] server-6c90d2c3-b9b1-fabe-2798-a3aadc9f0677: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6c90d2c3-b9b1-fabe-2798-a3aadc9f0677 Address:6c90d2c3-b9b1-fabe-2798-a3aadc9f0677} {Suffrage:Voter ID:server-8271de02-9131-5280-f37c-65bc6223ef37 Address:8271de02-9131-5280-f37c-65bc6223ef37} {Suffrage:Voter ID:server-4614615c-0bd1-fbc1-fff9-4cd3d6771be4 Address:4614615c-0bd1-fbc1-fff9-4cd3d6771be4}]" +2025-02-25T23:58:15.746Z [INFO] server-8271de02-9131-5280-f37c-65bc6223ef37: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6c90d2c3-b9b1-fabe-2798-a3aadc9f0677 Address:6c90d2c3-b9b1-fabe-2798-a3aadc9f0677} {Suffrage:Voter ID:server-8271de02-9131-5280-f37c-65bc6223ef37 Address:8271de02-9131-5280-f37c-65bc6223ef37} {Suffrage:Voter ID:server-4614615c-0bd1-fbc1-fff9-4cd3d6771be4 Address:4614615c-0bd1-fbc1-fff9-4cd3d6771be4}]" +2025-02-25T23:58:15.746Z [INFO] server-4614615c-0bd1-fbc1-fff9-4cd3d6771be4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6c90d2c3-b9b1-fabe-2798-a3aadc9f0677 Address:6c90d2c3-b9b1-fabe-2798-a3aadc9f0677} {Suffrage:Voter ID:server-8271de02-9131-5280-f37c-65bc6223ef37 Address:8271de02-9131-5280-f37c-65bc6223ef37} {Suffrage:Voter ID:server-4614615c-0bd1-fbc1-fff9-4cd3d6771be4 Address:4614615c-0bd1-fbc1-fff9-4cd3d6771be4}]" +2025-02-25T23:58:15.746Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:58:15.746Z [INFO] server-6c90d2c3-b9b1-fabe-2798-a3aadc9f0677: entering follower state: follower="Node at 6c90d2c3-b9b1-fabe-2798-a3aadc9f0677 [Follower]" leader-address= leader-id= +2025-02-25T23:58:15.746Z [INFO] server-8271de02-9131-5280-f37c-65bc6223ef37: entering follower state: follower="Node at 8271de02-9131-5280-f37c-65bc6223ef37 [Follower]" leader-address= leader-id= +2025-02-25T23:58:15.746Z [INFO] server-4614615c-0bd1-fbc1-fff9-4cd3d6771be4: entering follower state: follower="Node at 4614615c-0bd1-fbc1-fff9-4cd3d6771be4 [Follower]" leader-address= leader-id= +2025-02-25T23:58:16.337Z [WARN] server-8271de02-9131-5280-f37c-65bc6223ef37: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:58:16.337Z [INFO] server-8271de02-9131-5280-f37c-65bc6223ef37: entering candidate state: node="Node at 8271de02-9131-5280-f37c-65bc6223ef37 [Candidate]" term=2 +2025-02-25T23:58:16.337Z [INFO] server-8271de02-9131-5280-f37c-65bc6223ef37: election won: term=2 tally=2 +2025-02-25T23:58:16.337Z [INFO] server-8271de02-9131-5280-f37c-65bc6223ef37: entering leader state: leader="Node at 8271de02-9131-5280-f37c-65bc6223ef37 [Leader]" +2025-02-25T23:58:16.338Z [INFO] server-8271de02-9131-5280-f37c-65bc6223ef37: added peer, starting replication: peer=server-6c90d2c3-b9b1-fabe-2798-a3aadc9f0677 +2025-02-25T23:58:16.338Z [INFO] server-8271de02-9131-5280-f37c-65bc6223ef37: added peer, starting replication: peer=server-4614615c-0bd1-fbc1-fff9-4cd3d6771be4 +2025-02-25T23:58:16.338Z [INFO] server-8271de02-9131-5280-f37c-65bc6223ef37: pipelining replication: peer="{Voter server-4614615c-0bd1-fbc1-fff9-4cd3d6771be4 4614615c-0bd1-fbc1-fff9-4cd3d6771be4}" +2025-02-25T23:58:16.338Z [INFO] server-8271de02-9131-5280-f37c-65bc6223ef37: pipelining replication: peer="{Voter server-6c90d2c3-b9b1-fabe-2798-a3aadc9f0677 6c90d2c3-b9b1-fabe-2798-a3aadc9f0677}" +2025-02-25T23:58:17.346Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:58:16.338217041 +0000 UTC m=+43.947232448 (1 nodes), 591.531847ms from start of poll, 591.702048ms from cluster start. Timeout at 2025-02-25 23:58:17.343219361 +0000 UTC m=+44.952234768, 1.00500232s after stability +2025-02-25T23:58:17.346Z [INFO] server-8271de02-9131-5280-f37c-65bc6223ef37: starting snapshot up to: index=12 +2025-02-25T23:58:17.346Z [INFO] creating new snapshot: path=/tmp/raft3576476640/snapshots/2-12-1740527897346.tmp +2025-02-25T23:58:17.346Z [INFO] server-8271de02-9131-5280-f37c-65bc6223ef37: snapshot complete up to: index=12 +2025-02-25T23:58:17.347Z [INFO] creating new snapshot: path=/tmp/raft3576476640/snapshots/2-23-1740527897347.tmp +2025-02-25T23:58:17.347Z [INFO] server-8271de02-9131-5280-f37c-65bc6223ef37: copied to local snapshot: bytes=71 +2025-02-25T23:58:17.347Z [INFO] server-8271de02-9131-5280-f37c-65bc6223ef37: snapshot restore progress: id=2-23-1740527897347 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:17.347Z [INFO] server-8271de02-9131-5280-f37c-65bc6223ef37: restored user snapshot: index=23 +2025-02-25T23:58:17.347Z [ERROR] server-8271de02-9131-5280-f37c-65bc6223ef37: failed to get log: index=23 error="log not found" +2025-02-25T23:58:17.347Z [INFO] server-8271de02-9131-5280-f37c-65bc6223ef37: aborting pipeline replication: peer="{Voter server-4614615c-0bd1-fbc1-fff9-4cd3d6771be4 4614615c-0bd1-fbc1-fff9-4cd3d6771be4}" +2025-02-25T23:58:17.347Z [ERROR] server-8271de02-9131-5280-f37c-65bc6223ef37: failed to get log: index=23 error="log not found" +2025-02-25T23:58:17.347Z [INFO] server-8271de02-9131-5280-f37c-65bc6223ef37: aborting pipeline replication: peer="{Voter server-6c90d2c3-b9b1-fabe-2798-a3aadc9f0677 6c90d2c3-b9b1-fabe-2798-a3aadc9f0677}" +2025-02-25T23:58:17.357Z [ERROR] server-8271de02-9131-5280-f37c-65bc6223ef37: failed to get log: index=23 error="log not found" +2025-02-25T23:58:17.358Z [INFO] creating new snapshot: path=/tmp/raft2365417494/snapshots/2-23-1740527897358.tmp +2025-02-25T23:58:17.358Z [INFO] server-6c90d2c3-b9b1-fabe-2798-a3aadc9f0677: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:17.358Z [INFO] server-6c90d2c3-b9b1-fabe-2798-a3aadc9f0677: copied to local snapshot: bytes=71 +2025-02-25T23:58:17.358Z [INFO] server-6c90d2c3-b9b1-fabe-2798-a3aadc9f0677: snapshot restore progress: id=2-23-1740527897358 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:17.358Z [INFO] server-6c90d2c3-b9b1-fabe-2798-a3aadc9f0677: Installed remote snapshot +2025-02-25T23:58:17.358Z [WARN] server-6c90d2c3-b9b1-fabe-2798-a3aadc9f0677: skipping application of old log: index=22 +2025-02-25T23:58:17.358Z [INFO] server-8271de02-9131-5280-f37c-65bc6223ef37: pipelining replication: peer="{Voter server-6c90d2c3-b9b1-fabe-2798-a3aadc9f0677 6c90d2c3-b9b1-fabe-2798-a3aadc9f0677}" +2025-02-25T23:58:17.358Z [ERROR] server-8271de02-9131-5280-f37c-65bc6223ef37: failed to get log: index=23 error="log not found" +2025-02-25T23:58:17.358Z [INFO] creating new snapshot: path=/tmp/raft4159915609/snapshots/2-23-1740527897358.tmp +2025-02-25T23:58:17.358Z [INFO] server-4614615c-0bd1-fbc1-fff9-4cd3d6771be4: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:17.358Z [INFO] server-4614615c-0bd1-fbc1-fff9-4cd3d6771be4: copied to local snapshot: bytes=71 +2025-02-25T23:58:17.359Z [INFO] server-4614615c-0bd1-fbc1-fff9-4cd3d6771be4: snapshot restore progress: id=2-23-1740527897358 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:17.359Z [INFO] server-4614615c-0bd1-fbc1-fff9-4cd3d6771be4: Installed remote snapshot +2025-02-25T23:58:17.359Z [WARN] server-4614615c-0bd1-fbc1-fff9-4cd3d6771be4: skipping application of old log: index=22 +2025-02-25T23:58:17.359Z [INFO] server-8271de02-9131-5280-f37c-65bc6223ef37: pipelining replication: peer="{Voter server-4614615c-0bd1-fbc1-fff9-4cd3d6771be4 4614615c-0bd1-fbc1-fff9-4cd3d6771be4}" +2025-02-25T23:58:17.359Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:58:18.366Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:58:17.359180623 +0000 UTC m=+44.968196030 (2 nodes), 440ns from start of poll, 1.61266563s from cluster start. Timeout at 2025-02-25 23:58:18.364199984 +0000 UTC m=+45.973215391, 1.005019361s after stability +2025-02-25T23:58:18.382Z [INFO] server-8271de02-9131-5280-f37c-65bc6223ef37: aborting pipeline replication: peer="{Voter server-4614615c-0bd1-fbc1-fff9-4cd3d6771be4 4614615c-0bd1-fbc1-fff9-4cd3d6771be4}" +2025-02-25T23:58:18.382Z [INFO] server-8271de02-9131-5280-f37c-65bc6223ef37: aborting pipeline replication: peer="{Voter server-6c90d2c3-b9b1-fabe-2798-a3aadc9f0677 6c90d2c3-b9b1-fabe-2798-a3aadc9f0677}" === RUN TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_false -2026-03-31T06:16:56.815Z [INFO] server-d4cbca73-380a-507a-3365-ae83b93cdc11: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d4cbca73-380a-507a-3365-ae83b93cdc11 Address:d4cbca73-380a-507a-3365-ae83b93cdc11} {Suffrage:Voter ID:server-b940b8ae-0576-b95d-b87a-7d37ce7cc849 Address:b940b8ae-0576-b95d-b87a-7d37ce7cc849} {Suffrage:Voter ID:server-acbd1506-7795-c321-e1de-11abd27ef796 Address:acbd1506-7795-c321-e1de-11abd27ef796}]" -2026-03-31T06:16:56.816Z [INFO] server-d4cbca73-380a-507a-3365-ae83b93cdc11: entering follower state: follower="Node at d4cbca73-380a-507a-3365-ae83b93cdc11 [Follower]" leader-address= leader-id= -2026-03-31T06:16:56.816Z [INFO] server-b940b8ae-0576-b95d-b87a-7d37ce7cc849: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d4cbca73-380a-507a-3365-ae83b93cdc11 Address:d4cbca73-380a-507a-3365-ae83b93cdc11} {Suffrage:Voter ID:server-b940b8ae-0576-b95d-b87a-7d37ce7cc849 Address:b940b8ae-0576-b95d-b87a-7d37ce7cc849} {Suffrage:Voter ID:server-acbd1506-7795-c321-e1de-11abd27ef796 Address:acbd1506-7795-c321-e1de-11abd27ef796}]" -2026-03-31T06:16:56.816Z [INFO] server-b940b8ae-0576-b95d-b87a-7d37ce7cc849: entering follower state: follower="Node at b940b8ae-0576-b95d-b87a-7d37ce7cc849 [Follower]" leader-address= leader-id= -2026-03-31T06:16:56.816Z [INFO] server-acbd1506-7795-c321-e1de-11abd27ef796: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d4cbca73-380a-507a-3365-ae83b93cdc11 Address:d4cbca73-380a-507a-3365-ae83b93cdc11} {Suffrage:Voter ID:server-b940b8ae-0576-b95d-b87a-7d37ce7cc849 Address:b940b8ae-0576-b95d-b87a-7d37ce7cc849} {Suffrage:Voter ID:server-acbd1506-7795-c321-e1de-11abd27ef796 Address:acbd1506-7795-c321-e1de-11abd27ef796}]" -2026-03-31T06:16:56.816Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:56.816Z [INFO] server-acbd1506-7795-c321-e1de-11abd27ef796: entering follower state: follower="Node at acbd1506-7795-c321-e1de-11abd27ef796 [Follower]" leader-address= leader-id= -2026-03-31T06:16:57.386Z [WARN] server-d4cbca73-380a-507a-3365-ae83b93cdc11: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:57.386Z [INFO] server-d4cbca73-380a-507a-3365-ae83b93cdc11: entering candidate state: node="Node at d4cbca73-380a-507a-3365-ae83b93cdc11 [Candidate]" term=2 -2026-03-31T06:16:57.386Z [INFO] server-d4cbca73-380a-507a-3365-ae83b93cdc11: election won: term=2 tally=2 -2026-03-31T06:16:57.386Z [INFO] server-d4cbca73-380a-507a-3365-ae83b93cdc11: entering leader state: leader="Node at d4cbca73-380a-507a-3365-ae83b93cdc11 [Leader]" -2026-03-31T06:16:57.386Z [INFO] server-d4cbca73-380a-507a-3365-ae83b93cdc11: added peer, starting replication: peer=server-b940b8ae-0576-b95d-b87a-7d37ce7cc849 -2026-03-31T06:16:57.386Z [INFO] server-d4cbca73-380a-507a-3365-ae83b93cdc11: added peer, starting replication: peer=server-acbd1506-7795-c321-e1de-11abd27ef796 -2026-03-31T06:16:57.386Z [INFO] server-d4cbca73-380a-507a-3365-ae83b93cdc11: pipelining replication: peer="{Voter server-acbd1506-7795-c321-e1de-11abd27ef796 acbd1506-7795-c321-e1de-11abd27ef796}" -2026-03-31T06:16:57.386Z [INFO] server-d4cbca73-380a-507a-3365-ae83b93cdc11: pipelining replication: peer="{Voter server-b940b8ae-0576-b95d-b87a-7d37ce7cc849 b940b8ae-0576-b95d-b87a-7d37ce7cc849}" -2026-03-31T06:16:58.392Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:57.3866745 +0000 UTC m=+45.664281881 (1 nodes), 570.25096ms from start of poll, 570.99924ms from cluster start. Timeout at 2026-03-31 06:16:58.39167822 +0000 UTC m=+46.669285601, 1.00500372s after stability -2026-03-31T06:16:58.392Z [INFO] server-d4cbca73-380a-507a-3365-ae83b93cdc11: starting snapshot up to: index=12 -2026-03-31T06:16:58.393Z [INFO] creating new snapshot: path=/tmp/raft3213364505/snapshots/2-12-1774937818392.tmp -2026-03-31T06:16:58.393Z [INFO] server-d4cbca73-380a-507a-3365-ae83b93cdc11: snapshot complete up to: index=12 -2026-03-31T06:16:58.394Z [INFO] creating new snapshot: path=/tmp/raft3213364505/snapshots/2-23-1774937818394.tmp -2026-03-31T06:16:58.394Z [INFO] server-d4cbca73-380a-507a-3365-ae83b93cdc11: copied to local snapshot: bytes=71 -2026-03-31T06:16:58.395Z [INFO] server-d4cbca73-380a-507a-3365-ae83b93cdc11: snapshot restore progress: id=2-23-1774937818394 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:58.395Z [INFO] server-d4cbca73-380a-507a-3365-ae83b93cdc11: removing all old logs from log store -2026-03-31T06:16:58.395Z [INFO] server-d4cbca73-380a-507a-3365-ae83b93cdc11: compacting logs: from=1 to=22 -2026-03-31T06:16:58.395Z [INFO] server-d4cbca73-380a-507a-3365-ae83b93cdc11: restored user snapshot: index=23 -2026-03-31T06:16:58.395Z [ERROR] server-d4cbca73-380a-507a-3365-ae83b93cdc11: failed to get log: index=22 error="log not found" -2026-03-31T06:16:58.395Z [INFO] server-d4cbca73-380a-507a-3365-ae83b93cdc11: aborting pipeline replication: peer="{Voter server-acbd1506-7795-c321-e1de-11abd27ef796 acbd1506-7795-c321-e1de-11abd27ef796}" -2026-03-31T06:16:58.395Z [ERROR] server-d4cbca73-380a-507a-3365-ae83b93cdc11: failed to get log: index=22 error="log not found" -2026-03-31T06:16:58.395Z [INFO] server-d4cbca73-380a-507a-3365-ae83b93cdc11: aborting pipeline replication: peer="{Voter server-b940b8ae-0576-b95d-b87a-7d37ce7cc849 b940b8ae-0576-b95d-b87a-7d37ce7cc849}" -2026-03-31T06:16:58.402Z [ERROR] server-d4cbca73-380a-507a-3365-ae83b93cdc11: failed to get log: index=22 error="log not found" -2026-03-31T06:16:58.402Z [INFO] creating new snapshot: path=/tmp/raft393117621/snapshots/2-23-1774937818402.tmp -2026-03-31T06:16:58.403Z [INFO] server-acbd1506-7795-c321-e1de-11abd27ef796: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:58.403Z [INFO] server-acbd1506-7795-c321-e1de-11abd27ef796: copied to local snapshot: bytes=71 -2026-03-31T06:16:58.403Z [INFO] server-acbd1506-7795-c321-e1de-11abd27ef796: snapshot restore progress: id=2-23-1774937818402 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:58.403Z [ERROR] server-d4cbca73-380a-507a-3365-ae83b93cdc11: failed to get log: index=22 error="log not found" -2026-03-31T06:16:58.403Z [INFO] server-acbd1506-7795-c321-e1de-11abd27ef796: removing all old logs from log store -2026-03-31T06:16:58.403Z [INFO] server-acbd1506-7795-c321-e1de-11abd27ef796: compacting logs: from=1 to=22 -2026-03-31T06:16:58.403Z [INFO] server-acbd1506-7795-c321-e1de-11abd27ef796: Installed remote snapshot -2026-03-31T06:16:58.403Z [WARN] server-acbd1506-7795-c321-e1de-11abd27ef796: skipping application of old log: index=22 -2026-03-31T06:16:58.403Z [INFO] server-d4cbca73-380a-507a-3365-ae83b93cdc11: pipelining replication: peer="{Voter server-acbd1506-7795-c321-e1de-11abd27ef796 acbd1506-7795-c321-e1de-11abd27ef796}" -2026-03-31T06:16:58.403Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:16:58.403Z [INFO] creating new snapshot: path=/tmp/raft4033972450/snapshots/2-23-1774937818403.tmp -2026-03-31T06:16:58.404Z [INFO] server-b940b8ae-0576-b95d-b87a-7d37ce7cc849: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:58.404Z [INFO] server-b940b8ae-0576-b95d-b87a-7d37ce7cc849: copied to local snapshot: bytes=71 -2026-03-31T06:16:58.404Z [INFO] server-b940b8ae-0576-b95d-b87a-7d37ce7cc849: snapshot restore progress: id=2-23-1774937818403 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:16:58.404Z [INFO] server-b940b8ae-0576-b95d-b87a-7d37ce7cc849: removing all old logs from log store -2026-03-31T06:16:58.404Z [INFO] server-b940b8ae-0576-b95d-b87a-7d37ce7cc849: compacting logs: from=1 to=22 -2026-03-31T06:16:58.404Z [INFO] server-b940b8ae-0576-b95d-b87a-7d37ce7cc849: Installed remote snapshot -2026-03-31T06:16:58.404Z [INFO] server-d4cbca73-380a-507a-3365-ae83b93cdc11: pipelining replication: peer="{Voter server-b940b8ae-0576-b95d-b87a-7d37ce7cc849 b940b8ae-0576-b95d-b87a-7d37ce7cc849}" -2026-03-31T06:16:59.409Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:16:58.40377446 +0000 UTC m=+46.681381841 (2 nodes), 960ns from start of poll, 1.5880992s from cluster start. Timeout at 2026-03-31 06:16:59.40879114 +0000 UTC m=+47.686398521, 1.00501668s after stability -2026-03-31T06:16:59.421Z [INFO] server-d4cbca73-380a-507a-3365-ae83b93cdc11: aborting pipeline replication: peer="{Voter server-acbd1506-7795-c321-e1de-11abd27ef796 acbd1506-7795-c321-e1de-11abd27ef796}" -2026-03-31T06:16:59.421Z [INFO] server-d4cbca73-380a-507a-3365-ae83b93cdc11: aborting pipeline replication: peer="{Voter server-b940b8ae-0576-b95d-b87a-7d37ce7cc849 b940b8ae-0576-b95d-b87a-7d37ce7cc849}" +2025-02-25T23:58:18.382Z [INFO] server-762b7343-5875-cbfc-7e55-eb680cc19a16: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-762b7343-5875-cbfc-7e55-eb680cc19a16 Address:762b7343-5875-cbfc-7e55-eb680cc19a16} {Suffrage:Voter ID:server-7d76405a-0fd4-abde-4e53-2a429588b77e Address:7d76405a-0fd4-abde-4e53-2a429588b77e} {Suffrage:Voter ID:server-4a942a91-b2a5-af30-4f5e-d0ca4f69b31b Address:4a942a91-b2a5-af30-4f5e-d0ca4f69b31b}]" +2025-02-25T23:58:18.382Z [INFO] server-7d76405a-0fd4-abde-4e53-2a429588b77e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-762b7343-5875-cbfc-7e55-eb680cc19a16 Address:762b7343-5875-cbfc-7e55-eb680cc19a16} {Suffrage:Voter ID:server-7d76405a-0fd4-abde-4e53-2a429588b77e Address:7d76405a-0fd4-abde-4e53-2a429588b77e} {Suffrage:Voter ID:server-4a942a91-b2a5-af30-4f5e-d0ca4f69b31b Address:4a942a91-b2a5-af30-4f5e-d0ca4f69b31b}]" +2025-02-25T23:58:18.382Z [INFO] server-4a942a91-b2a5-af30-4f5e-d0ca4f69b31b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-762b7343-5875-cbfc-7e55-eb680cc19a16 Address:762b7343-5875-cbfc-7e55-eb680cc19a16} {Suffrage:Voter ID:server-7d76405a-0fd4-abde-4e53-2a429588b77e Address:7d76405a-0fd4-abde-4e53-2a429588b77e} {Suffrage:Voter ID:server-4a942a91-b2a5-af30-4f5e-d0ca4f69b31b Address:4a942a91-b2a5-af30-4f5e-d0ca4f69b31b}]" +2025-02-25T23:58:18.382Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:58:18.382Z [INFO] server-762b7343-5875-cbfc-7e55-eb680cc19a16: entering follower state: follower="Node at 762b7343-5875-cbfc-7e55-eb680cc19a16 [Follower]" leader-address= leader-id= +2025-02-25T23:58:18.382Z [INFO] server-7d76405a-0fd4-abde-4e53-2a429588b77e: entering follower state: follower="Node at 7d76405a-0fd4-abde-4e53-2a429588b77e [Follower]" leader-address= leader-id= +2025-02-25T23:58:18.382Z [INFO] server-4a942a91-b2a5-af30-4f5e-d0ca4f69b31b: entering follower state: follower="Node at 4a942a91-b2a5-af30-4f5e-d0ca4f69b31b [Follower]" leader-address= leader-id= +2025-02-25T23:58:19.097Z [WARN] server-7d76405a-0fd4-abde-4e53-2a429588b77e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:58:19.097Z [INFO] server-7d76405a-0fd4-abde-4e53-2a429588b77e: entering candidate state: node="Node at 7d76405a-0fd4-abde-4e53-2a429588b77e [Candidate]" term=2 +2025-02-25T23:58:19.098Z [INFO] server-7d76405a-0fd4-abde-4e53-2a429588b77e: election won: term=2 tally=2 +2025-02-25T23:58:19.098Z [INFO] server-7d76405a-0fd4-abde-4e53-2a429588b77e: entering leader state: leader="Node at 7d76405a-0fd4-abde-4e53-2a429588b77e [Leader]" +2025-02-25T23:58:19.098Z [INFO] server-7d76405a-0fd4-abde-4e53-2a429588b77e: added peer, starting replication: peer=server-762b7343-5875-cbfc-7e55-eb680cc19a16 +2025-02-25T23:58:19.098Z [INFO] server-7d76405a-0fd4-abde-4e53-2a429588b77e: added peer, starting replication: peer=server-4a942a91-b2a5-af30-4f5e-d0ca4f69b31b +2025-02-25T23:58:19.098Z [INFO] server-7d76405a-0fd4-abde-4e53-2a429588b77e: pipelining replication: peer="{Voter server-4a942a91-b2a5-af30-4f5e-d0ca4f69b31b 4a942a91-b2a5-af30-4f5e-d0ca4f69b31b}" +2025-02-25T23:58:19.098Z [INFO] server-7d76405a-0fd4-abde-4e53-2a429588b77e: pipelining replication: peer="{Voter server-762b7343-5875-cbfc-7e55-eb680cc19a16 762b7343-5875-cbfc-7e55-eb680cc19a16}" +2025-02-25T23:58:20.105Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:58:19.098234017 +0000 UTC m=+46.707249384 (1 nodes), 715.430996ms from start of poll, 715.620357ms from cluster start. Timeout at 2025-02-25 23:58:20.103237577 +0000 UTC m=+47.712253024, 1.00500364s after stability +2025-02-25T23:58:20.106Z [INFO] server-7d76405a-0fd4-abde-4e53-2a429588b77e: starting snapshot up to: index=12 +2025-02-25T23:58:20.106Z [INFO] creating new snapshot: path=/tmp/raft1837748962/snapshots/2-12-1740527900106.tmp +2025-02-25T23:58:20.106Z [INFO] server-7d76405a-0fd4-abde-4e53-2a429588b77e: snapshot complete up to: index=12 +2025-02-25T23:58:20.107Z [INFO] creating new snapshot: path=/tmp/raft1837748962/snapshots/2-23-1740527900107.tmp +2025-02-25T23:58:20.107Z [INFO] server-7d76405a-0fd4-abde-4e53-2a429588b77e: copied to local snapshot: bytes=71 +2025-02-25T23:58:20.107Z [INFO] server-7d76405a-0fd4-abde-4e53-2a429588b77e: snapshot restore progress: id=2-23-1740527900107 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:20.107Z [INFO] server-7d76405a-0fd4-abde-4e53-2a429588b77e: removing all old logs from log store +2025-02-25T23:58:20.107Z [INFO] server-7d76405a-0fd4-abde-4e53-2a429588b77e: compacting logs: from=1 to=22 +2025-02-25T23:58:20.107Z [INFO] server-7d76405a-0fd4-abde-4e53-2a429588b77e: restored user snapshot: index=23 +2025-02-25T23:58:20.107Z [ERROR] server-7d76405a-0fd4-abde-4e53-2a429588b77e: failed to get log: index=22 error="log not found" +2025-02-25T23:58:20.107Z [INFO] server-7d76405a-0fd4-abde-4e53-2a429588b77e: aborting pipeline replication: peer="{Voter server-4a942a91-b2a5-af30-4f5e-d0ca4f69b31b 4a942a91-b2a5-af30-4f5e-d0ca4f69b31b}" +2025-02-25T23:58:20.107Z [ERROR] server-7d76405a-0fd4-abde-4e53-2a429588b77e: failed to get log: index=22 error="log not found" +2025-02-25T23:58:20.107Z [INFO] server-7d76405a-0fd4-abde-4e53-2a429588b77e: aborting pipeline replication: peer="{Voter server-762b7343-5875-cbfc-7e55-eb680cc19a16 762b7343-5875-cbfc-7e55-eb680cc19a16}" +2025-02-25T23:58:20.113Z [ERROR] server-7d76405a-0fd4-abde-4e53-2a429588b77e: failed to get log: index=22 error="log not found" +2025-02-25T23:58:20.114Z [INFO] creating new snapshot: path=/tmp/raft468547615/snapshots/2-23-1740527900114.tmp +2025-02-25T23:58:20.114Z [INFO] server-762b7343-5875-cbfc-7e55-eb680cc19a16: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:20.114Z [INFO] server-762b7343-5875-cbfc-7e55-eb680cc19a16: copied to local snapshot: bytes=71 +2025-02-25T23:58:20.114Z [INFO] server-762b7343-5875-cbfc-7e55-eb680cc19a16: snapshot restore progress: id=2-23-1740527900114 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:20.114Z [INFO] server-762b7343-5875-cbfc-7e55-eb680cc19a16: removing all old logs from log store +2025-02-25T23:58:20.114Z [INFO] server-762b7343-5875-cbfc-7e55-eb680cc19a16: compacting logs: from=1 to=22 +2025-02-25T23:58:20.114Z [INFO] server-762b7343-5875-cbfc-7e55-eb680cc19a16: Installed remote snapshot +2025-02-25T23:58:20.114Z [WARN] server-762b7343-5875-cbfc-7e55-eb680cc19a16: skipping application of old log: index=22 +2025-02-25T23:58:20.114Z [INFO] server-7d76405a-0fd4-abde-4e53-2a429588b77e: pipelining replication: peer="{Voter server-762b7343-5875-cbfc-7e55-eb680cc19a16 762b7343-5875-cbfc-7e55-eb680cc19a16}" +2025-02-25T23:58:20.114Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:58:20.121Z [ERROR] server-7d76405a-0fd4-abde-4e53-2a429588b77e: failed to get log: index=22 error="log not found" +2025-02-25T23:58:20.122Z [INFO] creating new snapshot: path=/tmp/raft2028101101/snapshots/2-23-1740527900122.tmp +2025-02-25T23:58:20.122Z [INFO] server-4a942a91-b2a5-af30-4f5e-d0ca4f69b31b: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:20.122Z [INFO] server-4a942a91-b2a5-af30-4f5e-d0ca4f69b31b: copied to local snapshot: bytes=71 +2025-02-25T23:58:20.122Z [INFO] server-4a942a91-b2a5-af30-4f5e-d0ca4f69b31b: snapshot restore progress: id=2-23-1740527900122 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:20.122Z [INFO] server-4a942a91-b2a5-af30-4f5e-d0ca4f69b31b: removing all old logs from log store +2025-02-25T23:58:20.122Z [INFO] server-4a942a91-b2a5-af30-4f5e-d0ca4f69b31b: compacting logs: from=1 to=22 +2025-02-25T23:58:20.122Z [INFO] server-4a942a91-b2a5-af30-4f5e-d0ca4f69b31b: Installed remote snapshot +2025-02-25T23:58:20.122Z [INFO] server-7d76405a-0fd4-abde-4e53-2a429588b77e: pipelining replication: peer="{Voter server-4a942a91-b2a5-af30-4f5e-d0ca4f69b31b 4a942a91-b2a5-af30-4f5e-d0ca4f69b31b}" +2025-02-25T23:58:21.121Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:58:20.11467149 +0000 UTC m=+47.723686897 (2 nodes), 480ns from start of poll, 1.73205787s from cluster start. Timeout at 2025-02-25 23:58:21.11968505 +0000 UTC m=+48.728700457, 1.00501356s after stability +2025-02-25T23:58:21.138Z [INFO] server-7d76405a-0fd4-abde-4e53-2a429588b77e: aborting pipeline replication: peer="{Voter server-4a942a91-b2a5-af30-4f5e-d0ca4f69b31b 4a942a91-b2a5-af30-4f5e-d0ca4f69b31b}" +2025-02-25T23:58:21.138Z [INFO] server-7d76405a-0fd4-abde-4e53-2a429588b77e: aborting pipeline replication: peer="{Voter server-762b7343-5875-cbfc-7e55-eb680cc19a16 762b7343-5875-cbfc-7e55-eb680cc19a16}" === RUN TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_true -2026-03-31T06:16:59.422Z [INFO] server-d120d7d0-ae87-696c-a3d3-a07c0c87f0f9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d120d7d0-ae87-696c-a3d3-a07c0c87f0f9 Address:d120d7d0-ae87-696c-a3d3-a07c0c87f0f9} {Suffrage:Voter ID:server-4d59777c-b92d-9cf3-d7d1-43fc8ab92bd8 Address:4d59777c-b92d-9cf3-d7d1-43fc8ab92bd8} {Suffrage:Voter ID:server-b864b142-50a9-d4b8-a398-cfefc5fc2e3b Address:b864b142-50a9-d4b8-a398-cfefc5fc2e3b}]" -2026-03-31T06:16:59.422Z [INFO] server-4d59777c-b92d-9cf3-d7d1-43fc8ab92bd8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d120d7d0-ae87-696c-a3d3-a07c0c87f0f9 Address:d120d7d0-ae87-696c-a3d3-a07c0c87f0f9} {Suffrage:Voter ID:server-4d59777c-b92d-9cf3-d7d1-43fc8ab92bd8 Address:4d59777c-b92d-9cf3-d7d1-43fc8ab92bd8} {Suffrage:Voter ID:server-b864b142-50a9-d4b8-a398-cfefc5fc2e3b Address:b864b142-50a9-d4b8-a398-cfefc5fc2e3b}]" -2026-03-31T06:16:59.422Z [INFO] server-d120d7d0-ae87-696c-a3d3-a07c0c87f0f9: entering follower state: follower="Node at d120d7d0-ae87-696c-a3d3-a07c0c87f0f9 [Follower]" leader-address= leader-id= -2026-03-31T06:16:59.422Z [INFO] server-4d59777c-b92d-9cf3-d7d1-43fc8ab92bd8: entering follower state: follower="Node at 4d59777c-b92d-9cf3-d7d1-43fc8ab92bd8 [Follower]" leader-address= leader-id= -2026-03-31T06:16:59.422Z [INFO] server-b864b142-50a9-d4b8-a398-cfefc5fc2e3b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d120d7d0-ae87-696c-a3d3-a07c0c87f0f9 Address:d120d7d0-ae87-696c-a3d3-a07c0c87f0f9} {Suffrage:Voter ID:server-4d59777c-b92d-9cf3-d7d1-43fc8ab92bd8 Address:4d59777c-b92d-9cf3-d7d1-43fc8ab92bd8} {Suffrage:Voter ID:server-b864b142-50a9-d4b8-a398-cfefc5fc2e3b Address:b864b142-50a9-d4b8-a398-cfefc5fc2e3b}]" -2026-03-31T06:16:59.422Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:16:59.422Z [INFO] server-b864b142-50a9-d4b8-a398-cfefc5fc2e3b: entering follower state: follower="Node at b864b142-50a9-d4b8-a398-cfefc5fc2e3b [Follower]" leader-address= leader-id= -2026-03-31T06:16:59.980Z [WARN] server-b864b142-50a9-d4b8-a398-cfefc5fc2e3b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:16:59.980Z [INFO] server-b864b142-50a9-d4b8-a398-cfefc5fc2e3b: entering candidate state: node="Node at b864b142-50a9-d4b8-a398-cfefc5fc2e3b [Candidate]" term=2 -2026-03-31T06:16:59.981Z [INFO] server-b864b142-50a9-d4b8-a398-cfefc5fc2e3b: election won: term=2 tally=2 -2026-03-31T06:16:59.981Z [INFO] server-b864b142-50a9-d4b8-a398-cfefc5fc2e3b: entering leader state: leader="Node at b864b142-50a9-d4b8-a398-cfefc5fc2e3b [Leader]" -2026-03-31T06:16:59.981Z [INFO] server-b864b142-50a9-d4b8-a398-cfefc5fc2e3b: added peer, starting replication: peer=server-d120d7d0-ae87-696c-a3d3-a07c0c87f0f9 -2026-03-31T06:16:59.981Z [INFO] server-b864b142-50a9-d4b8-a398-cfefc5fc2e3b: added peer, starting replication: peer=server-4d59777c-b92d-9cf3-d7d1-43fc8ab92bd8 -2026-03-31T06:16:59.981Z [INFO] server-b864b142-50a9-d4b8-a398-cfefc5fc2e3b: pipelining replication: peer="{Voter server-d120d7d0-ae87-696c-a3d3-a07c0c87f0f9 d120d7d0-ae87-696c-a3d3-a07c0c87f0f9}" -2026-03-31T06:16:59.981Z [INFO] server-b864b142-50a9-d4b8-a398-cfefc5fc2e3b: pipelining replication: peer="{Voter server-4d59777c-b92d-9cf3-d7d1-43fc8ab92bd8 4d59777c-b92d-9cf3-d7d1-43fc8ab92bd8}" -2026-03-31T06:17:00.986Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:16:59.98118638 +0000 UTC m=+48.258793761 (1 nodes), 558.5954ms from start of poll, 559.00568ms from cluster start. Timeout at 2026-03-31 06:17:00.98618818 +0000 UTC m=+49.263795521, 1.00500176s after stability -2026-03-31T06:17:00.987Z [INFO] server-b864b142-50a9-d4b8-a398-cfefc5fc2e3b: starting snapshot up to: index=12 -2026-03-31T06:17:00.987Z [INFO] creating new snapshot: path=/tmp/raft510648591/snapshots/2-12-1774937820987.tmp -2026-03-31T06:17:00.988Z [INFO] server-b864b142-50a9-d4b8-a398-cfefc5fc2e3b: snapshot complete up to: index=12 -2026-03-31T06:17:00.989Z [INFO] server-266bf53f-bab9-bd0b-2137-1dbe55e1d964: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-266bf53f-bab9-bd0b-2137-1dbe55e1d964 Address:266bf53f-bab9-bd0b-2137-1dbe55e1d964} {Suffrage:Voter ID:server-32ca7425-0dbf-7b0b-4a45-09c2c64f6d3c Address:32ca7425-0dbf-7b0b-4a45-09c2c64f6d3c} {Suffrage:Voter ID:server-2c2a2976-cfce-78ce-ffb3-d5a52e7d2882 Address:2c2a2976-cfce-78ce-ffb3-d5a52e7d2882}]" -2026-03-31T06:17:00.989Z [INFO] server-266bf53f-bab9-bd0b-2137-1dbe55e1d964: entering follower state: follower="Node at 266bf53f-bab9-bd0b-2137-1dbe55e1d964 [Follower]" leader-address= leader-id= -2026-03-31T06:17:00.989Z [INFO] server-32ca7425-0dbf-7b0b-4a45-09c2c64f6d3c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-266bf53f-bab9-bd0b-2137-1dbe55e1d964 Address:266bf53f-bab9-bd0b-2137-1dbe55e1d964} {Suffrage:Voter ID:server-32ca7425-0dbf-7b0b-4a45-09c2c64f6d3c Address:32ca7425-0dbf-7b0b-4a45-09c2c64f6d3c} {Suffrage:Voter ID:server-2c2a2976-cfce-78ce-ffb3-d5a52e7d2882 Address:2c2a2976-cfce-78ce-ffb3-d5a52e7d2882}]" -2026-03-31T06:17:00.990Z [INFO] server-32ca7425-0dbf-7b0b-4a45-09c2c64f6d3c: entering follower state: follower="Node at 32ca7425-0dbf-7b0b-4a45-09c2c64f6d3c [Follower]" leader-address= leader-id= -2026-03-31T06:17:00.990Z [INFO] server-2c2a2976-cfce-78ce-ffb3-d5a52e7d2882: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-266bf53f-bab9-bd0b-2137-1dbe55e1d964 Address:266bf53f-bab9-bd0b-2137-1dbe55e1d964} {Suffrage:Voter ID:server-32ca7425-0dbf-7b0b-4a45-09c2c64f6d3c Address:32ca7425-0dbf-7b0b-4a45-09c2c64f6d3c} {Suffrage:Voter ID:server-2c2a2976-cfce-78ce-ffb3-d5a52e7d2882 Address:2c2a2976-cfce-78ce-ffb3-d5a52e7d2882}]" -2026-03-31T06:17:00.990Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:00.990Z [INFO] server-2c2a2976-cfce-78ce-ffb3-d5a52e7d2882: entering follower state: follower="Node at 2c2a2976-cfce-78ce-ffb3-d5a52e7d2882 [Follower]" leader-address= leader-id= -2026-03-31T06:17:01.691Z [WARN] server-266bf53f-bab9-bd0b-2137-1dbe55e1d964: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:01.691Z [INFO] server-266bf53f-bab9-bd0b-2137-1dbe55e1d964: entering candidate state: node="Node at 266bf53f-bab9-bd0b-2137-1dbe55e1d964 [Candidate]" term=2 -2026-03-31T06:17:01.691Z [INFO] server-266bf53f-bab9-bd0b-2137-1dbe55e1d964: election won: term=2 tally=2 -2026-03-31T06:17:01.691Z [INFO] server-266bf53f-bab9-bd0b-2137-1dbe55e1d964: entering leader state: leader="Node at 266bf53f-bab9-bd0b-2137-1dbe55e1d964 [Leader]" -2026-03-31T06:17:01.691Z [INFO] server-266bf53f-bab9-bd0b-2137-1dbe55e1d964: added peer, starting replication: peer=server-32ca7425-0dbf-7b0b-4a45-09c2c64f6d3c -2026-03-31T06:17:01.691Z [INFO] server-266bf53f-bab9-bd0b-2137-1dbe55e1d964: added peer, starting replication: peer=server-2c2a2976-cfce-78ce-ffb3-d5a52e7d2882 -2026-03-31T06:17:01.691Z [INFO] server-266bf53f-bab9-bd0b-2137-1dbe55e1d964: pipelining replication: peer="{Voter server-2c2a2976-cfce-78ce-ffb3-d5a52e7d2882 2c2a2976-cfce-78ce-ffb3-d5a52e7d2882}" -2026-03-31T06:17:01.692Z [INFO] server-266bf53f-bab9-bd0b-2137-1dbe55e1d964: pipelining replication: peer="{Voter server-32ca7425-0dbf-7b0b-4a45-09c2c64f6d3c 32ca7425-0dbf-7b0b-4a45-09c2c64f6d3c}" -2026-03-31T06:17:02.697Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:01.69193606 +0000 UTC m=+49.969543441 (1 nodes), 701.6848ms from start of poll, 702.36716ms from cluster start. Timeout at 2026-03-31 06:17:02.69694422 +0000 UTC m=+50.974551601, 1.00500816s after stability -2026-03-31T06:17:02.697Z [INFO] creating new snapshot: path=/tmp/raft1432414814/snapshots/2-15-1774937822697.tmp -2026-03-31T06:17:02.697Z [INFO] server-266bf53f-bab9-bd0b-2137-1dbe55e1d964: copied to local snapshot: bytes=71 -2026-03-31T06:17:02.698Z [INFO] server-266bf53f-bab9-bd0b-2137-1dbe55e1d964: snapshot restore progress: id=2-15-1774937822697 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:02.698Z [INFO] server-266bf53f-bab9-bd0b-2137-1dbe55e1d964: restored user snapshot: index=15 -2026-03-31T06:17:02.698Z [ERROR] server-266bf53f-bab9-bd0b-2137-1dbe55e1d964: failed to get log: index=3 error="log not found" -2026-03-31T06:17:02.698Z [INFO] server-266bf53f-bab9-bd0b-2137-1dbe55e1d964: aborting pipeline replication: peer="{Voter server-2c2a2976-cfce-78ce-ffb3-d5a52e7d2882 2c2a2976-cfce-78ce-ffb3-d5a52e7d2882}" -2026-03-31T06:17:02.698Z [ERROR] server-266bf53f-bab9-bd0b-2137-1dbe55e1d964: failed to get log: index=3 error="log not found" -2026-03-31T06:17:02.698Z [INFO] server-266bf53f-bab9-bd0b-2137-1dbe55e1d964: aborting pipeline replication: peer="{Voter server-32ca7425-0dbf-7b0b-4a45-09c2c64f6d3c 32ca7425-0dbf-7b0b-4a45-09c2c64f6d3c}" -2026-03-31T06:17:02.707Z [ERROR] server-266bf53f-bab9-bd0b-2137-1dbe55e1d964: failed to get log: index=3 error="log not found" -2026-03-31T06:17:02.708Z [INFO] creating new snapshot: path=/tmp/raft2332189455/snapshots/2-15-1774937822708.tmp -2026-03-31T06:17:02.708Z [ERROR] server-266bf53f-bab9-bd0b-2137-1dbe55e1d964: failed to get log: index=3 error="log not found" -2026-03-31T06:17:02.708Z [INFO] server-32ca7425-0dbf-7b0b-4a45-09c2c64f6d3c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:02.708Z [INFO] creating new snapshot: path=/tmp/raft4221624487/snapshots/2-15-1774937822708.tmp -2026-03-31T06:17:02.708Z [INFO] server-32ca7425-0dbf-7b0b-4a45-09c2c64f6d3c: copied to local snapshot: bytes=71 -2026-03-31T06:17:02.708Z [INFO] server-2c2a2976-cfce-78ce-ffb3-d5a52e7d2882: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:02.709Z [INFO] server-32ca7425-0dbf-7b0b-4a45-09c2c64f6d3c: snapshot restore progress: id=2-15-1774937822708 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:02.709Z [INFO] server-32ca7425-0dbf-7b0b-4a45-09c2c64f6d3c: Installed remote snapshot -2026-03-31T06:17:02.709Z [INFO] server-2c2a2976-cfce-78ce-ffb3-d5a52e7d2882: copied to local snapshot: bytes=71 -2026-03-31T06:17:02.709Z [INFO] server-266bf53f-bab9-bd0b-2137-1dbe55e1d964: pipelining replication: peer="{Voter server-32ca7425-0dbf-7b0b-4a45-09c2c64f6d3c 32ca7425-0dbf-7b0b-4a45-09c2c64f6d3c}" -2026-03-31T06:17:02.709Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:17:02.709Z [INFO] server-2c2a2976-cfce-78ce-ffb3-d5a52e7d2882: snapshot restore progress: id=2-15-1774937822708 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:02.709Z [INFO] server-2c2a2976-cfce-78ce-ffb3-d5a52e7d2882: Installed remote snapshot -2026-03-31T06:17:02.709Z [INFO] server-266bf53f-bab9-bd0b-2137-1dbe55e1d964: pipelining replication: peer="{Voter server-2c2a2976-cfce-78ce-ffb3-d5a52e7d2882 2c2a2976-cfce-78ce-ffb3-d5a52e7d2882}" -2026-03-31T06:17:03.715Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:17:02.70940002 +0000 UTC m=+50.987007401 (2 nodes), 920ns from start of poll, 1.71983112s from cluster start. Timeout at 2026-03-31 06:17:03.71440738 +0000 UTC m=+51.992014761, 1.00500736s after stability -2026-03-31T06:17:03.727Z [INFO] server-b864b142-50a9-d4b8-a398-cfefc5fc2e3b: aborting pipeline replication: peer="{Voter server-4d59777c-b92d-9cf3-d7d1-43fc8ab92bd8 4d59777c-b92d-9cf3-d7d1-43fc8ab92bd8}" -2026-03-31T06:17:03.727Z [INFO] server-b864b142-50a9-d4b8-a398-cfefc5fc2e3b: aborting pipeline replication: peer="{Voter server-d120d7d0-ae87-696c-a3d3-a07c0c87f0f9 d120d7d0-ae87-696c-a3d3-a07c0c87f0f9}" +2025-02-25T23:58:21.138Z [INFO] server-473f7812-fe09-18ee-0ad4-4110ffb04eeb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-473f7812-fe09-18ee-0ad4-4110ffb04eeb Address:473f7812-fe09-18ee-0ad4-4110ffb04eeb} {Suffrage:Voter ID:server-f8954ffc-33b1-c919-e681-0a00c14934e9 Address:f8954ffc-33b1-c919-e681-0a00c14934e9} {Suffrage:Voter ID:server-034b4b6f-7102-ae92-cd9a-1e36732a0d21 Address:034b4b6f-7102-ae92-cd9a-1e36732a0d21}]" +2025-02-25T23:58:21.138Z [INFO] server-f8954ffc-33b1-c919-e681-0a00c14934e9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-473f7812-fe09-18ee-0ad4-4110ffb04eeb Address:473f7812-fe09-18ee-0ad4-4110ffb04eeb} {Suffrage:Voter ID:server-f8954ffc-33b1-c919-e681-0a00c14934e9 Address:f8954ffc-33b1-c919-e681-0a00c14934e9} {Suffrage:Voter ID:server-034b4b6f-7102-ae92-cd9a-1e36732a0d21 Address:034b4b6f-7102-ae92-cd9a-1e36732a0d21}]" +2025-02-25T23:58:21.138Z [INFO] server-034b4b6f-7102-ae92-cd9a-1e36732a0d21: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-473f7812-fe09-18ee-0ad4-4110ffb04eeb Address:473f7812-fe09-18ee-0ad4-4110ffb04eeb} {Suffrage:Voter ID:server-f8954ffc-33b1-c919-e681-0a00c14934e9 Address:f8954ffc-33b1-c919-e681-0a00c14934e9} {Suffrage:Voter ID:server-034b4b6f-7102-ae92-cd9a-1e36732a0d21 Address:034b4b6f-7102-ae92-cd9a-1e36732a0d21}]" +2025-02-25T23:58:21.138Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:58:21.138Z [INFO] server-473f7812-fe09-18ee-0ad4-4110ffb04eeb: entering follower state: follower="Node at 473f7812-fe09-18ee-0ad4-4110ffb04eeb [Follower]" leader-address= leader-id= +2025-02-25T23:58:21.138Z [INFO] server-f8954ffc-33b1-c919-e681-0a00c14934e9: entering follower state: follower="Node at f8954ffc-33b1-c919-e681-0a00c14934e9 [Follower]" leader-address= leader-id= +2025-02-25T23:58:21.138Z [INFO] server-034b4b6f-7102-ae92-cd9a-1e36732a0d21: entering follower state: follower="Node at 034b4b6f-7102-ae92-cd9a-1e36732a0d21 [Follower]" leader-address= leader-id= +2025-02-25T23:58:21.905Z [WARN] server-473f7812-fe09-18ee-0ad4-4110ffb04eeb: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:58:21.905Z [INFO] server-473f7812-fe09-18ee-0ad4-4110ffb04eeb: entering candidate state: node="Node at 473f7812-fe09-18ee-0ad4-4110ffb04eeb [Candidate]" term=2 +2025-02-25T23:58:21.906Z [INFO] server-473f7812-fe09-18ee-0ad4-4110ffb04eeb: election won: term=2 tally=2 +2025-02-25T23:58:21.906Z [INFO] server-473f7812-fe09-18ee-0ad4-4110ffb04eeb: entering leader state: leader="Node at 473f7812-fe09-18ee-0ad4-4110ffb04eeb [Leader]" +2025-02-25T23:58:21.906Z [INFO] server-473f7812-fe09-18ee-0ad4-4110ffb04eeb: added peer, starting replication: peer=server-f8954ffc-33b1-c919-e681-0a00c14934e9 +2025-02-25T23:58:21.906Z [INFO] server-473f7812-fe09-18ee-0ad4-4110ffb04eeb: added peer, starting replication: peer=server-034b4b6f-7102-ae92-cd9a-1e36732a0d21 +2025-02-25T23:58:21.906Z [INFO] server-473f7812-fe09-18ee-0ad4-4110ffb04eeb: pipelining replication: peer="{Voter server-034b4b6f-7102-ae92-cd9a-1e36732a0d21 034b4b6f-7102-ae92-cd9a-1e36732a0d21}" +2025-02-25T23:58:21.906Z [INFO] server-473f7812-fe09-18ee-0ad4-4110ffb04eeb: pipelining replication: peer="{Voter server-f8954ffc-33b1-c919-e681-0a00c14934e9 f8954ffc-33b1-c919-e681-0a00c14934e9}" +2025-02-25T23:58:22.914Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:58:21.906219858 +0000 UTC m=+49.515235265 (1 nodes), 767.411527ms from start of poll, 767.587808ms from cluster start. Timeout at 2025-02-25 23:58:22.911222258 +0000 UTC m=+50.520237625, 1.00500236s after stability +2025-02-25T23:58:22.914Z [INFO] server-473f7812-fe09-18ee-0ad4-4110ffb04eeb: starting snapshot up to: index=12 +2025-02-25T23:58:22.914Z [INFO] creating new snapshot: path=/tmp/raft535964428/snapshots/2-12-1740527902914.tmp +2025-02-25T23:58:22.914Z [INFO] server-473f7812-fe09-18ee-0ad4-4110ffb04eeb: snapshot complete up to: index=12 +2025-02-25T23:58:22.915Z [INFO] server-08182ba6-a906-c995-4386-9c8321297c15: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-08182ba6-a906-c995-4386-9c8321297c15 Address:08182ba6-a906-c995-4386-9c8321297c15} {Suffrage:Voter ID:server-bad6ac21-f55b-3812-b7ad-eeca6aff7c84 Address:bad6ac21-f55b-3812-b7ad-eeca6aff7c84} {Suffrage:Voter ID:server-91cad574-3f2b-cc48-c039-0a161c756872 Address:91cad574-3f2b-cc48-c039-0a161c756872}]" +2025-02-25T23:58:22.915Z [INFO] server-bad6ac21-f55b-3812-b7ad-eeca6aff7c84: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-08182ba6-a906-c995-4386-9c8321297c15 Address:08182ba6-a906-c995-4386-9c8321297c15} {Suffrage:Voter ID:server-bad6ac21-f55b-3812-b7ad-eeca6aff7c84 Address:bad6ac21-f55b-3812-b7ad-eeca6aff7c84} {Suffrage:Voter ID:server-91cad574-3f2b-cc48-c039-0a161c756872 Address:91cad574-3f2b-cc48-c039-0a161c756872}]" +2025-02-25T23:58:22.915Z [INFO] server-91cad574-3f2b-cc48-c039-0a161c756872: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-08182ba6-a906-c995-4386-9c8321297c15 Address:08182ba6-a906-c995-4386-9c8321297c15} {Suffrage:Voter ID:server-bad6ac21-f55b-3812-b7ad-eeca6aff7c84 Address:bad6ac21-f55b-3812-b7ad-eeca6aff7c84} {Suffrage:Voter ID:server-91cad574-3f2b-cc48-c039-0a161c756872 Address:91cad574-3f2b-cc48-c039-0a161c756872}]" +2025-02-25T23:58:22.915Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:58:22.915Z [INFO] server-08182ba6-a906-c995-4386-9c8321297c15: entering follower state: follower="Node at 08182ba6-a906-c995-4386-9c8321297c15 [Follower]" leader-address= leader-id= +2025-02-25T23:58:22.915Z [INFO] server-bad6ac21-f55b-3812-b7ad-eeca6aff7c84: entering follower state: follower="Node at bad6ac21-f55b-3812-b7ad-eeca6aff7c84 [Follower]" leader-address= leader-id= +2025-02-25T23:58:22.915Z [INFO] server-91cad574-3f2b-cc48-c039-0a161c756872: entering follower state: follower="Node at 91cad574-3f2b-cc48-c039-0a161c756872 [Follower]" leader-address= leader-id= +2025-02-25T23:58:23.422Z [WARN] server-91cad574-3f2b-cc48-c039-0a161c756872: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:58:23.422Z [INFO] server-91cad574-3f2b-cc48-c039-0a161c756872: entering candidate state: node="Node at 91cad574-3f2b-cc48-c039-0a161c756872 [Candidate]" term=2 +2025-02-25T23:58:23.422Z [INFO] server-91cad574-3f2b-cc48-c039-0a161c756872: election won: term=2 tally=2 +2025-02-25T23:58:23.422Z [INFO] server-91cad574-3f2b-cc48-c039-0a161c756872: entering leader state: leader="Node at 91cad574-3f2b-cc48-c039-0a161c756872 [Leader]" +2025-02-25T23:58:23.422Z [INFO] server-91cad574-3f2b-cc48-c039-0a161c756872: added peer, starting replication: peer=server-08182ba6-a906-c995-4386-9c8321297c15 +2025-02-25T23:58:23.422Z [INFO] server-91cad574-3f2b-cc48-c039-0a161c756872: added peer, starting replication: peer=server-bad6ac21-f55b-3812-b7ad-eeca6aff7c84 +2025-02-25T23:58:23.422Z [INFO] server-91cad574-3f2b-cc48-c039-0a161c756872: pipelining replication: peer="{Voter server-bad6ac21-f55b-3812-b7ad-eeca6aff7c84 bad6ac21-f55b-3812-b7ad-eeca6aff7c84}" +2025-02-25T23:58:23.422Z [INFO] server-91cad574-3f2b-cc48-c039-0a161c756872: pipelining replication: peer="{Voter server-08182ba6-a906-c995-4386-9c8321297c15 08182ba6-a906-c995-4386-9c8321297c15}" +2025-02-25T23:58:24.430Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:58:23.422397153 +0000 UTC m=+51.031412560 (1 nodes), 506.916068ms from start of poll, 507.079829ms from cluster start. Timeout at 2025-02-25 23:58:24.427399993 +0000 UTC m=+52.036415360, 1.0050028s after stability +2025-02-25T23:58:24.430Z [INFO] creating new snapshot: path=/tmp/raft3876923816/snapshots/2-15-1740527904430.tmp +2025-02-25T23:58:24.430Z [INFO] server-91cad574-3f2b-cc48-c039-0a161c756872: copied to local snapshot: bytes=71 +2025-02-25T23:58:24.430Z [INFO] server-91cad574-3f2b-cc48-c039-0a161c756872: snapshot restore progress: id=2-15-1740527904430 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:24.430Z [INFO] server-91cad574-3f2b-cc48-c039-0a161c756872: restored user snapshot: index=15 +2025-02-25T23:58:24.430Z [ERROR] server-91cad574-3f2b-cc48-c039-0a161c756872: failed to get log: index=3 error="log not found" +2025-02-25T23:58:24.430Z [INFO] server-91cad574-3f2b-cc48-c039-0a161c756872: aborting pipeline replication: peer="{Voter server-bad6ac21-f55b-3812-b7ad-eeca6aff7c84 bad6ac21-f55b-3812-b7ad-eeca6aff7c84}" +2025-02-25T23:58:24.430Z [ERROR] server-91cad574-3f2b-cc48-c039-0a161c756872: failed to get log: index=3 error="log not found" +2025-02-25T23:58:24.430Z [INFO] server-91cad574-3f2b-cc48-c039-0a161c756872: aborting pipeline replication: peer="{Voter server-08182ba6-a906-c995-4386-9c8321297c15 08182ba6-a906-c995-4386-9c8321297c15}" +2025-02-25T23:58:24.438Z [ERROR] server-91cad574-3f2b-cc48-c039-0a161c756872: failed to get log: index=3 error="log not found" +2025-02-25T23:58:24.438Z [ERROR] server-91cad574-3f2b-cc48-c039-0a161c756872: failed to get log: index=3 error="log not found" +2025-02-25T23:58:24.438Z [INFO] creating new snapshot: path=/tmp/raft379092617/snapshots/2-15-1740527904438.tmp +2025-02-25T23:58:24.438Z [INFO] server-08182ba6-a906-c995-4386-9c8321297c15: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:24.439Z [INFO] server-08182ba6-a906-c995-4386-9c8321297c15: copied to local snapshot: bytes=71 +2025-02-25T23:58:24.439Z [INFO] server-08182ba6-a906-c995-4386-9c8321297c15: snapshot restore progress: id=2-15-1740527904438 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:24.439Z [INFO] server-08182ba6-a906-c995-4386-9c8321297c15: Installed remote snapshot +2025-02-25T23:58:24.439Z [INFO] server-91cad574-3f2b-cc48-c039-0a161c756872: pipelining replication: peer="{Voter server-08182ba6-a906-c995-4386-9c8321297c15 08182ba6-a906-c995-4386-9c8321297c15}" +2025-02-25T23:58:24.439Z [INFO] creating new snapshot: path=/tmp/raft3070002826/snapshots/2-15-1740527904439.tmp +2025-02-25T23:58:24.439Z [INFO] server-bad6ac21-f55b-3812-b7ad-eeca6aff7c84: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:24.439Z [INFO] server-bad6ac21-f55b-3812-b7ad-eeca6aff7c84: copied to local snapshot: bytes=71 +2025-02-25T23:58:24.439Z [INFO] server-bad6ac21-f55b-3812-b7ad-eeca6aff7c84: snapshot restore progress: id=2-15-1740527904439 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:24.439Z [INFO] server-bad6ac21-f55b-3812-b7ad-eeca6aff7c84: Installed remote snapshot +2025-02-25T23:58:24.439Z [INFO] server-91cad574-3f2b-cc48-c039-0a161c756872: pipelining replication: peer="{Voter server-bad6ac21-f55b-3812-b7ad-eeca6aff7c84 bad6ac21-f55b-3812-b7ad-eeca6aff7c84}" +2025-02-25T23:58:24.439Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:58:25.445Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:58:24.43963735 +0000 UTC m=+52.048652717 (2 nodes), 320ns from start of poll, 1.524319986s from cluster start. Timeout at 2025-02-25 23:58:25.44465971 +0000 UTC m=+53.053675357, 1.00502264s after stability +2025-02-25T23:58:25.462Z [INFO] server-473f7812-fe09-18ee-0ad4-4110ffb04eeb: aborting pipeline replication: peer="{Voter server-034b4b6f-7102-ae92-cd9a-1e36732a0d21 034b4b6f-7102-ae92-cd9a-1e36732a0d21}" +2025-02-25T23:58:25.462Z [INFO] server-473f7812-fe09-18ee-0ad4-4110ffb04eeb: aborting pipeline replication: peer="{Voter server-f8954ffc-33b1-c919-e681-0a00c14934e9 f8954ffc-33b1-c919-e681-0a00c14934e9}" === RUN TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_true -2026-03-31T06:17:03.728Z [INFO] server-3678e8fd-f7b7-a5ef-b366-356c085ad0c1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3678e8fd-f7b7-a5ef-b366-356c085ad0c1 Address:3678e8fd-f7b7-a5ef-b366-356c085ad0c1} {Suffrage:Voter ID:server-0479d444-945e-d40a-8c10-c1ce56887f57 Address:0479d444-945e-d40a-8c10-c1ce56887f57} {Suffrage:Voter ID:server-0d44823f-aba0-e4f1-9be1-886d4b964c71 Address:0d44823f-aba0-e4f1-9be1-886d4b964c71}]" -2026-03-31T06:17:03.728Z [INFO] server-0479d444-945e-d40a-8c10-c1ce56887f57: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3678e8fd-f7b7-a5ef-b366-356c085ad0c1 Address:3678e8fd-f7b7-a5ef-b366-356c085ad0c1} {Suffrage:Voter ID:server-0479d444-945e-d40a-8c10-c1ce56887f57 Address:0479d444-945e-d40a-8c10-c1ce56887f57} {Suffrage:Voter ID:server-0d44823f-aba0-e4f1-9be1-886d4b964c71 Address:0d44823f-aba0-e4f1-9be1-886d4b964c71}]" -2026-03-31T06:17:03.728Z [INFO] server-3678e8fd-f7b7-a5ef-b366-356c085ad0c1: entering follower state: follower="Node at 3678e8fd-f7b7-a5ef-b366-356c085ad0c1 [Follower]" leader-address= leader-id= -2026-03-31T06:17:03.728Z [INFO] server-0479d444-945e-d40a-8c10-c1ce56887f57: entering follower state: follower="Node at 0479d444-945e-d40a-8c10-c1ce56887f57 [Follower]" leader-address= leader-id= -2026-03-31T06:17:03.728Z [INFO] server-0d44823f-aba0-e4f1-9be1-886d4b964c71: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3678e8fd-f7b7-a5ef-b366-356c085ad0c1 Address:3678e8fd-f7b7-a5ef-b366-356c085ad0c1} {Suffrage:Voter ID:server-0479d444-945e-d40a-8c10-c1ce56887f57 Address:0479d444-945e-d40a-8c10-c1ce56887f57} {Suffrage:Voter ID:server-0d44823f-aba0-e4f1-9be1-886d4b964c71 Address:0d44823f-aba0-e4f1-9be1-886d4b964c71}]" -2026-03-31T06:17:03.728Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:03.728Z [INFO] server-0d44823f-aba0-e4f1-9be1-886d4b964c71: entering follower state: follower="Node at 0d44823f-aba0-e4f1-9be1-886d4b964c71 [Follower]" leader-address= leader-id= -2026-03-31T06:17:04.509Z [WARN] server-0479d444-945e-d40a-8c10-c1ce56887f57: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:04.509Z [INFO] server-0479d444-945e-d40a-8c10-c1ce56887f57: entering candidate state: node="Node at 0479d444-945e-d40a-8c10-c1ce56887f57 [Candidate]" term=2 -2026-03-31T06:17:04.509Z [INFO] server-0479d444-945e-d40a-8c10-c1ce56887f57: election won: term=2 tally=2 -2026-03-31T06:17:04.509Z [INFO] server-0479d444-945e-d40a-8c10-c1ce56887f57: entering leader state: leader="Node at 0479d444-945e-d40a-8c10-c1ce56887f57 [Leader]" -2026-03-31T06:17:04.509Z [INFO] server-0479d444-945e-d40a-8c10-c1ce56887f57: added peer, starting replication: peer=server-3678e8fd-f7b7-a5ef-b366-356c085ad0c1 -2026-03-31T06:17:04.509Z [INFO] server-0479d444-945e-d40a-8c10-c1ce56887f57: added peer, starting replication: peer=server-0d44823f-aba0-e4f1-9be1-886d4b964c71 -2026-03-31T06:17:04.509Z [INFO] server-0479d444-945e-d40a-8c10-c1ce56887f57: pipelining replication: peer="{Voter server-0d44823f-aba0-e4f1-9be1-886d4b964c71 0d44823f-aba0-e4f1-9be1-886d4b964c71}" -2026-03-31T06:17:04.509Z [INFO] server-0479d444-945e-d40a-8c10-c1ce56887f57: pipelining replication: peer="{Voter server-3678e8fd-f7b7-a5ef-b366-356c085ad0c1 3678e8fd-f7b7-a5ef-b366-356c085ad0c1}" -2026-03-31T06:17:05.514Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:04.50960038 +0000 UTC m=+52.787207761 (1 nodes), 780.77304ms from start of poll, 781.22464ms from cluster start. Timeout at 2026-03-31 06:17:05.51460294 +0000 UTC m=+53.792210361, 1.0050026s after stability -2026-03-31T06:17:05.515Z [INFO] server-0479d444-945e-d40a-8c10-c1ce56887f57: starting snapshot up to: index=12 -2026-03-31T06:17:05.515Z [INFO] creating new snapshot: path=/tmp/raft4132315654/snapshots/2-12-1774937825515.tmp -2026-03-31T06:17:05.516Z [INFO] server-0479d444-945e-d40a-8c10-c1ce56887f57: snapshot complete up to: index=12 -2026-03-31T06:17:05.517Z [INFO] server-22940dab-7423-c433-e58c-c83986124b3e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-22940dab-7423-c433-e58c-c83986124b3e Address:22940dab-7423-c433-e58c-c83986124b3e} {Suffrage:Voter ID:server-b955cc52-c412-9694-f487-0708caadc00f Address:b955cc52-c412-9694-f487-0708caadc00f} {Suffrage:Voter ID:server-24043e19-27b6-b585-1d4d-86d38bb889f3 Address:24043e19-27b6-b585-1d4d-86d38bb889f3}]" -2026-03-31T06:17:05.517Z [INFO] server-b955cc52-c412-9694-f487-0708caadc00f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-22940dab-7423-c433-e58c-c83986124b3e Address:22940dab-7423-c433-e58c-c83986124b3e} {Suffrage:Voter ID:server-b955cc52-c412-9694-f487-0708caadc00f Address:b955cc52-c412-9694-f487-0708caadc00f} {Suffrage:Voter ID:server-24043e19-27b6-b585-1d4d-86d38bb889f3 Address:24043e19-27b6-b585-1d4d-86d38bb889f3}]" -2026-03-31T06:17:05.517Z [INFO] server-22940dab-7423-c433-e58c-c83986124b3e: entering follower state: follower="Node at 22940dab-7423-c433-e58c-c83986124b3e [Follower]" leader-address= leader-id= -2026-03-31T06:17:05.517Z [INFO] server-b955cc52-c412-9694-f487-0708caadc00f: entering follower state: follower="Node at b955cc52-c412-9694-f487-0708caadc00f [Follower]" leader-address= leader-id= -2026-03-31T06:17:05.517Z [INFO] server-24043e19-27b6-b585-1d4d-86d38bb889f3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-22940dab-7423-c433-e58c-c83986124b3e Address:22940dab-7423-c433-e58c-c83986124b3e} {Suffrage:Voter ID:server-b955cc52-c412-9694-f487-0708caadc00f Address:b955cc52-c412-9694-f487-0708caadc00f} {Suffrage:Voter ID:server-24043e19-27b6-b585-1d4d-86d38bb889f3 Address:24043e19-27b6-b585-1d4d-86d38bb889f3}]" -2026-03-31T06:17:05.517Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:05.517Z [INFO] server-24043e19-27b6-b585-1d4d-86d38bb889f3: entering follower state: follower="Node at 24043e19-27b6-b585-1d4d-86d38bb889f3 [Follower]" leader-address= leader-id= -2026-03-31T06:17:06.019Z [WARN] server-22940dab-7423-c433-e58c-c83986124b3e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:06.019Z [INFO] server-22940dab-7423-c433-e58c-c83986124b3e: entering candidate state: node="Node at 22940dab-7423-c433-e58c-c83986124b3e [Candidate]" term=2 -2026-03-31T06:17:06.019Z [INFO] server-22940dab-7423-c433-e58c-c83986124b3e: election won: term=2 tally=2 -2026-03-31T06:17:06.019Z [INFO] server-22940dab-7423-c433-e58c-c83986124b3e: entering leader state: leader="Node at 22940dab-7423-c433-e58c-c83986124b3e [Leader]" -2026-03-31T06:17:06.019Z [INFO] server-22940dab-7423-c433-e58c-c83986124b3e: added peer, starting replication: peer=server-b955cc52-c412-9694-f487-0708caadc00f -2026-03-31T06:17:06.019Z [INFO] server-22940dab-7423-c433-e58c-c83986124b3e: added peer, starting replication: peer=server-24043e19-27b6-b585-1d4d-86d38bb889f3 -2026-03-31T06:17:06.019Z [INFO] server-22940dab-7423-c433-e58c-c83986124b3e: pipelining replication: peer="{Voter server-24043e19-27b6-b585-1d4d-86d38bb889f3 24043e19-27b6-b585-1d4d-86d38bb889f3}" -2026-03-31T06:17:06.020Z [INFO] server-22940dab-7423-c433-e58c-c83986124b3e: pipelining replication: peer="{Voter server-b955cc52-c412-9694-f487-0708caadc00f b955cc52-c412-9694-f487-0708caadc00f}" -2026-03-31T06:17:07.025Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:06.0195635 +0000 UTC m=+54.297170881 (1 nodes), 502.08776ms from start of poll, 502.56564ms from cluster start. Timeout at 2026-03-31 06:17:07.0245677 +0000 UTC m=+55.302175121, 1.00500424s after stability -2026-03-31T06:17:07.025Z [INFO] creating new snapshot: path=/tmp/raft1803540490/snapshots/2-15-1774937827025.tmp -2026-03-31T06:17:07.026Z [INFO] server-22940dab-7423-c433-e58c-c83986124b3e: copied to local snapshot: bytes=71 -2026-03-31T06:17:07.027Z [INFO] server-22940dab-7423-c433-e58c-c83986124b3e: snapshot restore progress: id=2-15-1774937827025 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:07.027Z [INFO] server-22940dab-7423-c433-e58c-c83986124b3e: removing all old logs from log store -2026-03-31T06:17:07.027Z [INFO] server-22940dab-7423-c433-e58c-c83986124b3e: compacting logs: from=1 to=2 -2026-03-31T06:17:07.027Z [INFO] server-22940dab-7423-c433-e58c-c83986124b3e: restored user snapshot: index=15 -2026-03-31T06:17:07.027Z [ERROR] server-22940dab-7423-c433-e58c-c83986124b3e: failed to get log: index=2 error="log not found" -2026-03-31T06:17:07.027Z [INFO] server-22940dab-7423-c433-e58c-c83986124b3e: aborting pipeline replication: peer="{Voter server-b955cc52-c412-9694-f487-0708caadc00f b955cc52-c412-9694-f487-0708caadc00f}" -2026-03-31T06:17:07.027Z [ERROR] server-22940dab-7423-c433-e58c-c83986124b3e: failed to get log: index=2 error="log not found" -2026-03-31T06:17:07.027Z [INFO] server-22940dab-7423-c433-e58c-c83986124b3e: aborting pipeline replication: peer="{Voter server-24043e19-27b6-b585-1d4d-86d38bb889f3 24043e19-27b6-b585-1d4d-86d38bb889f3}" -2026-03-31T06:17:07.034Z [ERROR] server-22940dab-7423-c433-e58c-c83986124b3e: failed to get log: index=2 error="log not found" -2026-03-31T06:17:07.035Z [ERROR] server-22940dab-7423-c433-e58c-c83986124b3e: failed to get log: index=2 error="log not found" -2026-03-31T06:17:07.035Z [INFO] creating new snapshot: path=/tmp/raft864742639/snapshots/2-15-1774937827035.tmp -2026-03-31T06:17:07.035Z [INFO] creating new snapshot: path=/tmp/raft1848266708/snapshots/2-15-1774937827035.tmp -2026-03-31T06:17:07.035Z [INFO] server-24043e19-27b6-b585-1d4d-86d38bb889f3: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:07.035Z [INFO] server-b955cc52-c412-9694-f487-0708caadc00f: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:07.035Z [INFO] server-24043e19-27b6-b585-1d4d-86d38bb889f3: copied to local snapshot: bytes=71 -2026-03-31T06:17:07.036Z [INFO] server-b955cc52-c412-9694-f487-0708caadc00f: copied to local snapshot: bytes=71 -2026-03-31T06:17:07.036Z [INFO] server-24043e19-27b6-b585-1d4d-86d38bb889f3: snapshot restore progress: id=2-15-1774937827035 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:07.036Z [INFO] server-24043e19-27b6-b585-1d4d-86d38bb889f3: removing all old logs from log store -2026-03-31T06:17:07.036Z [INFO] server-24043e19-27b6-b585-1d4d-86d38bb889f3: compacting logs: from=1 to=2 -2026-03-31T06:17:07.036Z [INFO] server-24043e19-27b6-b585-1d4d-86d38bb889f3: Installed remote snapshot -2026-03-31T06:17:07.036Z [INFO] server-b955cc52-c412-9694-f487-0708caadc00f: snapshot restore progress: id=2-15-1774937827035 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:07.036Z [INFO] server-b955cc52-c412-9694-f487-0708caadc00f: removing all old logs from log store -2026-03-31T06:17:07.036Z [INFO] server-b955cc52-c412-9694-f487-0708caadc00f: compacting logs: from=1 to=2 -2026-03-31T06:17:07.036Z [INFO] server-b955cc52-c412-9694-f487-0708caadc00f: Installed remote snapshot -2026-03-31T06:17:07.036Z [INFO] server-22940dab-7423-c433-e58c-c83986124b3e: pipelining replication: peer="{Voter server-24043e19-27b6-b585-1d4d-86d38bb889f3 24043e19-27b6-b585-1d4d-86d38bb889f3}" -2026-03-31T06:17:07.036Z [INFO] server-22940dab-7423-c433-e58c-c83986124b3e: pipelining replication: peer="{Voter server-b955cc52-c412-9694-f487-0708caadc00f b955cc52-c412-9694-f487-0708caadc00f}" -2026-03-31T06:17:07.036Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:17:08.041Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:17:07.03656026 +0000 UTC m=+55.314167641 (2 nodes), 1.12µs from start of poll, 1.5195624s from cluster start. Timeout at 2026-03-31 06:17:08.04158302 +0000 UTC m=+56.319190401, 1.00502276s after stability -2026-03-31T06:17:08.054Z [INFO] server-0479d444-945e-d40a-8c10-c1ce56887f57: aborting pipeline replication: peer="{Voter server-0d44823f-aba0-e4f1-9be1-886d4b964c71 0d44823f-aba0-e4f1-9be1-886d4b964c71}" -2026-03-31T06:17:08.054Z [INFO] server-0479d444-945e-d40a-8c10-c1ce56887f57: aborting pipeline replication: peer="{Voter server-3678e8fd-f7b7-a5ef-b366-356c085ad0c1 3678e8fd-f7b7-a5ef-b366-356c085ad0c1}" +2025-02-25T23:58:25.462Z [INFO] server-c4adf526-29ec-b8e3-77c8-bff13f88c8c6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c4adf526-29ec-b8e3-77c8-bff13f88c8c6 Address:c4adf526-29ec-b8e3-77c8-bff13f88c8c6} {Suffrage:Voter ID:server-1465332a-9e14-8b30-ab5b-101a5d4ad838 Address:1465332a-9e14-8b30-ab5b-101a5d4ad838} {Suffrage:Voter ID:server-509ca44e-b0f5-6aa4-7b1e-899ea88fbf6c Address:509ca44e-b0f5-6aa4-7b1e-899ea88fbf6c}]" +2025-02-25T23:58:25.462Z [INFO] server-1465332a-9e14-8b30-ab5b-101a5d4ad838: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c4adf526-29ec-b8e3-77c8-bff13f88c8c6 Address:c4adf526-29ec-b8e3-77c8-bff13f88c8c6} {Suffrage:Voter ID:server-1465332a-9e14-8b30-ab5b-101a5d4ad838 Address:1465332a-9e14-8b30-ab5b-101a5d4ad838} {Suffrage:Voter ID:server-509ca44e-b0f5-6aa4-7b1e-899ea88fbf6c Address:509ca44e-b0f5-6aa4-7b1e-899ea88fbf6c}]" +2025-02-25T23:58:25.462Z [INFO] server-509ca44e-b0f5-6aa4-7b1e-899ea88fbf6c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c4adf526-29ec-b8e3-77c8-bff13f88c8c6 Address:c4adf526-29ec-b8e3-77c8-bff13f88c8c6} {Suffrage:Voter ID:server-1465332a-9e14-8b30-ab5b-101a5d4ad838 Address:1465332a-9e14-8b30-ab5b-101a5d4ad838} {Suffrage:Voter ID:server-509ca44e-b0f5-6aa4-7b1e-899ea88fbf6c Address:509ca44e-b0f5-6aa4-7b1e-899ea88fbf6c}]" +2025-02-25T23:58:25.462Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:58:25.462Z [INFO] server-c4adf526-29ec-b8e3-77c8-bff13f88c8c6: entering follower state: follower="Node at c4adf526-29ec-b8e3-77c8-bff13f88c8c6 [Follower]" leader-address= leader-id= +2025-02-25T23:58:25.463Z [INFO] server-1465332a-9e14-8b30-ab5b-101a5d4ad838: entering follower state: follower="Node at 1465332a-9e14-8b30-ab5b-101a5d4ad838 [Follower]" leader-address= leader-id= +2025-02-25T23:58:25.463Z [INFO] server-509ca44e-b0f5-6aa4-7b1e-899ea88fbf6c: entering follower state: follower="Node at 509ca44e-b0f5-6aa4-7b1e-899ea88fbf6c [Follower]" leader-address= leader-id= +2025-02-25T23:58:26.121Z [WARN] server-c4adf526-29ec-b8e3-77c8-bff13f88c8c6: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:58:26.121Z [INFO] server-c4adf526-29ec-b8e3-77c8-bff13f88c8c6: entering candidate state: node="Node at c4adf526-29ec-b8e3-77c8-bff13f88c8c6 [Candidate]" term=2 +2025-02-25T23:58:26.122Z [INFO] server-c4adf526-29ec-b8e3-77c8-bff13f88c8c6: election won: term=2 tally=2 +2025-02-25T23:58:26.122Z [INFO] server-c4adf526-29ec-b8e3-77c8-bff13f88c8c6: entering leader state: leader="Node at c4adf526-29ec-b8e3-77c8-bff13f88c8c6 [Leader]" +2025-02-25T23:58:26.122Z [INFO] server-c4adf526-29ec-b8e3-77c8-bff13f88c8c6: added peer, starting replication: peer=server-1465332a-9e14-8b30-ab5b-101a5d4ad838 +2025-02-25T23:58:26.122Z [INFO] server-c4adf526-29ec-b8e3-77c8-bff13f88c8c6: added peer, starting replication: peer=server-509ca44e-b0f5-6aa4-7b1e-899ea88fbf6c +2025-02-25T23:58:26.122Z [INFO] server-c4adf526-29ec-b8e3-77c8-bff13f88c8c6: pipelining replication: peer="{Voter server-509ca44e-b0f5-6aa4-7b1e-899ea88fbf6c 509ca44e-b0f5-6aa4-7b1e-899ea88fbf6c}" +2025-02-25T23:58:26.122Z [INFO] server-c4adf526-29ec-b8e3-77c8-bff13f88c8c6: pipelining replication: peer="{Voter server-1465332a-9e14-8b30-ab5b-101a5d4ad838 1465332a-9e14-8b30-ab5b-101a5d4ad838}" +2025-02-25T23:58:27.129Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:58:26.122286785 +0000 UTC m=+53.731302192 (1 nodes), 659.326438ms from start of poll, 659.51268ms from cluster start. Timeout at 2025-02-25 23:58:27.127309145 +0000 UTC m=+54.736324512, 1.00502232s after stability +2025-02-25T23:58:27.130Z [INFO] server-c4adf526-29ec-b8e3-77c8-bff13f88c8c6: starting snapshot up to: index=12 +2025-02-25T23:58:27.130Z [INFO] creating new snapshot: path=/tmp/raft3617047752/snapshots/2-12-1740527907130.tmp +2025-02-25T23:58:27.130Z [INFO] server-c4adf526-29ec-b8e3-77c8-bff13f88c8c6: snapshot complete up to: index=12 +2025-02-25T23:58:27.131Z [INFO] server-369d285c-2dc7-906b-dcca-1bec9089e348: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-369d285c-2dc7-906b-dcca-1bec9089e348 Address:369d285c-2dc7-906b-dcca-1bec9089e348} {Suffrage:Voter ID:server-e25e6453-af01-2ed8-20e2-19573038182b Address:e25e6453-af01-2ed8-20e2-19573038182b} {Suffrage:Voter ID:server-b2ae76e9-f056-92c4-bec3-939b4a0e4bef Address:b2ae76e9-f056-92c4-bec3-939b4a0e4bef}]" +2025-02-25T23:58:27.131Z [INFO] server-e25e6453-af01-2ed8-20e2-19573038182b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-369d285c-2dc7-906b-dcca-1bec9089e348 Address:369d285c-2dc7-906b-dcca-1bec9089e348} {Suffrage:Voter ID:server-e25e6453-af01-2ed8-20e2-19573038182b Address:e25e6453-af01-2ed8-20e2-19573038182b} {Suffrage:Voter ID:server-b2ae76e9-f056-92c4-bec3-939b4a0e4bef Address:b2ae76e9-f056-92c4-bec3-939b4a0e4bef}]" +2025-02-25T23:58:27.131Z [INFO] server-b2ae76e9-f056-92c4-bec3-939b4a0e4bef: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-369d285c-2dc7-906b-dcca-1bec9089e348 Address:369d285c-2dc7-906b-dcca-1bec9089e348} {Suffrage:Voter ID:server-e25e6453-af01-2ed8-20e2-19573038182b Address:e25e6453-af01-2ed8-20e2-19573038182b} {Suffrage:Voter ID:server-b2ae76e9-f056-92c4-bec3-939b4a0e4bef Address:b2ae76e9-f056-92c4-bec3-939b4a0e4bef}]" +2025-02-25T23:58:27.131Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:58:27.131Z [INFO] server-369d285c-2dc7-906b-dcca-1bec9089e348: entering follower state: follower="Node at 369d285c-2dc7-906b-dcca-1bec9089e348 [Follower]" leader-address= leader-id= +2025-02-25T23:58:27.131Z [INFO] server-e25e6453-af01-2ed8-20e2-19573038182b: entering follower state: follower="Node at e25e6453-af01-2ed8-20e2-19573038182b [Follower]" leader-address= leader-id= +2025-02-25T23:58:27.131Z [INFO] server-b2ae76e9-f056-92c4-bec3-939b4a0e4bef: entering follower state: follower="Node at b2ae76e9-f056-92c4-bec3-939b4a0e4bef [Follower]" leader-address= leader-id= +2025-02-25T23:58:27.753Z [WARN] server-b2ae76e9-f056-92c4-bec3-939b4a0e4bef: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:58:27.753Z [INFO] server-b2ae76e9-f056-92c4-bec3-939b4a0e4bef: entering candidate state: node="Node at b2ae76e9-f056-92c4-bec3-939b4a0e4bef [Candidate]" term=2 +2025-02-25T23:58:27.754Z [INFO] server-b2ae76e9-f056-92c4-bec3-939b4a0e4bef: election won: term=2 tally=2 +2025-02-25T23:58:27.754Z [INFO] server-b2ae76e9-f056-92c4-bec3-939b4a0e4bef: entering leader state: leader="Node at b2ae76e9-f056-92c4-bec3-939b4a0e4bef [Leader]" +2025-02-25T23:58:27.754Z [INFO] server-b2ae76e9-f056-92c4-bec3-939b4a0e4bef: added peer, starting replication: peer=server-369d285c-2dc7-906b-dcca-1bec9089e348 +2025-02-25T23:58:27.754Z [INFO] server-b2ae76e9-f056-92c4-bec3-939b4a0e4bef: added peer, starting replication: peer=server-e25e6453-af01-2ed8-20e2-19573038182b +2025-02-25T23:58:27.754Z [INFO] server-b2ae76e9-f056-92c4-bec3-939b4a0e4bef: pipelining replication: peer="{Voter server-e25e6453-af01-2ed8-20e2-19573038182b e25e6453-af01-2ed8-20e2-19573038182b}" +2025-02-25T23:58:27.754Z [INFO] server-b2ae76e9-f056-92c4-bec3-939b4a0e4bef: pipelining replication: peer="{Voter server-369d285c-2dc7-906b-dcca-1bec9089e348 369d285c-2dc7-906b-dcca-1bec9089e348}" +2025-02-25T23:58:28.760Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:58:27.754205617 +0000 UTC m=+55.363221024 (1 nodes), 622.896687ms from start of poll, 623.052448ms from cluster start. Timeout at 2025-02-25 23:58:28.759209457 +0000 UTC m=+56.368224864, 1.00500384s after stability +2025-02-25T23:58:28.760Z [INFO] creating new snapshot: path=/tmp/raft1029219871/snapshots/2-15-1740527908760.tmp +2025-02-25T23:58:28.760Z [INFO] server-b2ae76e9-f056-92c4-bec3-939b4a0e4bef: copied to local snapshot: bytes=71 +2025-02-25T23:58:28.760Z [INFO] server-b2ae76e9-f056-92c4-bec3-939b4a0e4bef: snapshot restore progress: id=2-15-1740527908760 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:28.760Z [INFO] server-b2ae76e9-f056-92c4-bec3-939b4a0e4bef: removing all old logs from log store +2025-02-25T23:58:28.760Z [INFO] server-b2ae76e9-f056-92c4-bec3-939b4a0e4bef: compacting logs: from=1 to=2 +2025-02-25T23:58:28.760Z [INFO] server-b2ae76e9-f056-92c4-bec3-939b4a0e4bef: restored user snapshot: index=15 +2025-02-25T23:58:28.760Z [ERROR] server-b2ae76e9-f056-92c4-bec3-939b4a0e4bef: failed to get log: index=2 error="log not found" +2025-02-25T23:58:28.760Z [INFO] server-b2ae76e9-f056-92c4-bec3-939b4a0e4bef: aborting pipeline replication: peer="{Voter server-e25e6453-af01-2ed8-20e2-19573038182b e25e6453-af01-2ed8-20e2-19573038182b}" +2025-02-25T23:58:28.760Z [ERROR] server-b2ae76e9-f056-92c4-bec3-939b4a0e4bef: failed to get log: index=2 error="log not found" +2025-02-25T23:58:28.760Z [INFO] server-b2ae76e9-f056-92c4-bec3-939b4a0e4bef: aborting pipeline replication: peer="{Voter server-369d285c-2dc7-906b-dcca-1bec9089e348 369d285c-2dc7-906b-dcca-1bec9089e348}" +2025-02-25T23:58:28.769Z [ERROR] server-b2ae76e9-f056-92c4-bec3-939b4a0e4bef: failed to get log: index=2 error="log not found" +2025-02-25T23:58:28.770Z [ERROR] server-b2ae76e9-f056-92c4-bec3-939b4a0e4bef: failed to get log: index=2 error="log not found" +2025-02-25T23:58:28.770Z [INFO] creating new snapshot: path=/tmp/raft2106962350/snapshots/2-15-1740527908770.tmp +2025-02-25T23:58:28.770Z [INFO] server-e25e6453-af01-2ed8-20e2-19573038182b: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:28.770Z [INFO] server-e25e6453-af01-2ed8-20e2-19573038182b: copied to local snapshot: bytes=71 +2025-02-25T23:58:28.770Z [INFO] server-e25e6453-af01-2ed8-20e2-19573038182b: snapshot restore progress: id=2-15-1740527908770 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:28.770Z [INFO] server-e25e6453-af01-2ed8-20e2-19573038182b: removing all old logs from log store +2025-02-25T23:58:28.770Z [INFO] server-e25e6453-af01-2ed8-20e2-19573038182b: compacting logs: from=1 to=2 +2025-02-25T23:58:28.770Z [INFO] server-e25e6453-af01-2ed8-20e2-19573038182b: Installed remote snapshot +2025-02-25T23:58:28.770Z [INFO] server-b2ae76e9-f056-92c4-bec3-939b4a0e4bef: pipelining replication: peer="{Voter server-e25e6453-af01-2ed8-20e2-19573038182b e25e6453-af01-2ed8-20e2-19573038182b}" +2025-02-25T23:58:28.770Z [INFO] creating new snapshot: path=/tmp/raft3662857056/snapshots/2-15-1740527908770.tmp +2025-02-25T23:58:28.770Z [INFO] server-369d285c-2dc7-906b-dcca-1bec9089e348: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:28.770Z [INFO] server-369d285c-2dc7-906b-dcca-1bec9089e348: copied to local snapshot: bytes=71 +2025-02-25T23:58:28.771Z [INFO] server-369d285c-2dc7-906b-dcca-1bec9089e348: snapshot restore progress: id=2-15-1740527908770 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:28.771Z [INFO] server-369d285c-2dc7-906b-dcca-1bec9089e348: removing all old logs from log store +2025-02-25T23:58:28.771Z [INFO] server-369d285c-2dc7-906b-dcca-1bec9089e348: compacting logs: from=1 to=2 +2025-02-25T23:58:28.771Z [INFO] server-369d285c-2dc7-906b-dcca-1bec9089e348: Installed remote snapshot +2025-02-25T23:58:28.771Z [INFO] server-b2ae76e9-f056-92c4-bec3-939b4a0e4bef: pipelining replication: peer="{Voter server-369d285c-2dc7-906b-dcca-1bec9089e348 369d285c-2dc7-906b-dcca-1bec9089e348}" +2025-02-25T23:58:28.771Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:58:29.778Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:58:28.771164892 +0000 UTC m=+56.380180259 (2 nodes), 360ns from start of poll, 1.640011683s from cluster start. Timeout at 2025-02-25 23:58:29.776187132 +0000 UTC m=+57.385202539, 1.00502228s after stability +2025-02-25T23:58:29.794Z [INFO] server-c4adf526-29ec-b8e3-77c8-bff13f88c8c6: aborting pipeline replication: peer="{Voter server-509ca44e-b0f5-6aa4-7b1e-899ea88fbf6c 509ca44e-b0f5-6aa4-7b1e-899ea88fbf6c}" +2025-02-25T23:58:29.794Z [INFO] server-c4adf526-29ec-b8e3-77c8-bff13f88c8c6: aborting pipeline replication: peer="{Voter server-1465332a-9e14-8b30-ab5b-101a5d4ad838 1465332a-9e14-8b30-ab5b-101a5d4ad838}" === RUN TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_false -2026-03-31T06:17:08.055Z [INFO] server-a4743941-6289-1233-5944-02dcb5626528: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a4743941-6289-1233-5944-02dcb5626528 Address:a4743941-6289-1233-5944-02dcb5626528} {Suffrage:Voter ID:server-1beb164c-d829-99d5-a391-4ba9596a5688 Address:1beb164c-d829-99d5-a391-4ba9596a5688} {Suffrage:Voter ID:server-b6bc123e-1ffc-5060-96a6-2df4c8ddf8a8 Address:b6bc123e-1ffc-5060-96a6-2df4c8ddf8a8}]" -2026-03-31T06:17:08.055Z [INFO] server-1beb164c-d829-99d5-a391-4ba9596a5688: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a4743941-6289-1233-5944-02dcb5626528 Address:a4743941-6289-1233-5944-02dcb5626528} {Suffrage:Voter ID:server-1beb164c-d829-99d5-a391-4ba9596a5688 Address:1beb164c-d829-99d5-a391-4ba9596a5688} {Suffrage:Voter ID:server-b6bc123e-1ffc-5060-96a6-2df4c8ddf8a8 Address:b6bc123e-1ffc-5060-96a6-2df4c8ddf8a8}]" -2026-03-31T06:17:08.055Z [INFO] server-a4743941-6289-1233-5944-02dcb5626528: entering follower state: follower="Node at a4743941-6289-1233-5944-02dcb5626528 [Follower]" leader-address= leader-id= -2026-03-31T06:17:08.056Z [INFO] server-1beb164c-d829-99d5-a391-4ba9596a5688: entering follower state: follower="Node at 1beb164c-d829-99d5-a391-4ba9596a5688 [Follower]" leader-address= leader-id= -2026-03-31T06:17:08.056Z [INFO] server-b6bc123e-1ffc-5060-96a6-2df4c8ddf8a8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a4743941-6289-1233-5944-02dcb5626528 Address:a4743941-6289-1233-5944-02dcb5626528} {Suffrage:Voter ID:server-1beb164c-d829-99d5-a391-4ba9596a5688 Address:1beb164c-d829-99d5-a391-4ba9596a5688} {Suffrage:Voter ID:server-b6bc123e-1ffc-5060-96a6-2df4c8ddf8a8 Address:b6bc123e-1ffc-5060-96a6-2df4c8ddf8a8}]" -2026-03-31T06:17:08.056Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:08.056Z [INFO] server-b6bc123e-1ffc-5060-96a6-2df4c8ddf8a8: entering follower state: follower="Node at b6bc123e-1ffc-5060-96a6-2df4c8ddf8a8 [Follower]" leader-address= leader-id= -2026-03-31T06:17:08.597Z [WARN] server-b6bc123e-1ffc-5060-96a6-2df4c8ddf8a8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:08.597Z [INFO] server-b6bc123e-1ffc-5060-96a6-2df4c8ddf8a8: entering candidate state: node="Node at b6bc123e-1ffc-5060-96a6-2df4c8ddf8a8 [Candidate]" term=2 -2026-03-31T06:17:08.597Z [INFO] server-b6bc123e-1ffc-5060-96a6-2df4c8ddf8a8: election won: term=2 tally=2 -2026-03-31T06:17:08.597Z [INFO] server-b6bc123e-1ffc-5060-96a6-2df4c8ddf8a8: entering leader state: leader="Node at b6bc123e-1ffc-5060-96a6-2df4c8ddf8a8 [Leader]" -2026-03-31T06:17:08.597Z [INFO] server-b6bc123e-1ffc-5060-96a6-2df4c8ddf8a8: added peer, starting replication: peer=server-a4743941-6289-1233-5944-02dcb5626528 -2026-03-31T06:17:08.597Z [INFO] server-b6bc123e-1ffc-5060-96a6-2df4c8ddf8a8: added peer, starting replication: peer=server-1beb164c-d829-99d5-a391-4ba9596a5688 -2026-03-31T06:17:08.597Z [INFO] server-b6bc123e-1ffc-5060-96a6-2df4c8ddf8a8: pipelining replication: peer="{Voter server-1beb164c-d829-99d5-a391-4ba9596a5688 1beb164c-d829-99d5-a391-4ba9596a5688}" -2026-03-31T06:17:08.598Z [INFO] server-b6bc123e-1ffc-5060-96a6-2df4c8ddf8a8: pipelining replication: peer="{Voter server-a4743941-6289-1233-5944-02dcb5626528 a4743941-6289-1233-5944-02dcb5626528}" -2026-03-31T06:17:09.603Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:08.59791574 +0000 UTC m=+56.875523121 (1 nodes), 541.76404ms from start of poll, 542.24132ms from cluster start. Timeout at 2026-03-31 06:17:09.60291998 +0000 UTC m=+57.880527361, 1.00500424s after stability -2026-03-31T06:17:09.604Z [INFO] server-b6bc123e-1ffc-5060-96a6-2df4c8ddf8a8: starting snapshot up to: index=12 -2026-03-31T06:17:09.604Z [INFO] creating new snapshot: path=/tmp/raft2260129171/snapshots/2-12-1774937829604.tmp -2026-03-31T06:17:09.605Z [INFO] server-b6bc123e-1ffc-5060-96a6-2df4c8ddf8a8: snapshot complete up to: index=12 -2026-03-31T06:17:09.607Z [INFO] creating new snapshot: path=/tmp/raft2260129171/snapshots/2-113-1774937829607.tmp -2026-03-31T06:17:09.608Z [INFO] server-b6bc123e-1ffc-5060-96a6-2df4c8ddf8a8: copied to local snapshot: bytes=71 -2026-03-31T06:17:09.608Z [INFO] server-b6bc123e-1ffc-5060-96a6-2df4c8ddf8a8: snapshot restore progress: id=2-113-1774937829607 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:09.608Z [INFO] server-b6bc123e-1ffc-5060-96a6-2df4c8ddf8a8: restored user snapshot: index=113 -2026-03-31T06:17:09.608Z [ERROR] server-b6bc123e-1ffc-5060-96a6-2df4c8ddf8a8: failed to get log: index=23 error="log not found" -2026-03-31T06:17:09.608Z [INFO] server-b6bc123e-1ffc-5060-96a6-2df4c8ddf8a8: aborting pipeline replication: peer="{Voter server-1beb164c-d829-99d5-a391-4ba9596a5688 1beb164c-d829-99d5-a391-4ba9596a5688}" -2026-03-31T06:17:09.608Z [ERROR] server-b6bc123e-1ffc-5060-96a6-2df4c8ddf8a8: failed to get log: index=23 error="log not found" -2026-03-31T06:17:09.608Z [INFO] server-b6bc123e-1ffc-5060-96a6-2df4c8ddf8a8: aborting pipeline replication: peer="{Voter server-a4743941-6289-1233-5944-02dcb5626528 a4743941-6289-1233-5944-02dcb5626528}" -2026-03-31T06:17:09.614Z [ERROR] server-b6bc123e-1ffc-5060-96a6-2df4c8ddf8a8: failed to get log: index=23 error="log not found" -2026-03-31T06:17:09.615Z [ERROR] server-b6bc123e-1ffc-5060-96a6-2df4c8ddf8a8: failed to get log: index=23 error="log not found" -2026-03-31T06:17:09.615Z [INFO] creating new snapshot: path=/tmp/raft1721862110/snapshots/2-113-1774937829615.tmp -2026-03-31T06:17:09.615Z [INFO] creating new snapshot: path=/tmp/raft2912833754/snapshots/2-113-1774937829615.tmp -2026-03-31T06:17:09.615Z [INFO] server-a4743941-6289-1233-5944-02dcb5626528: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:09.616Z [INFO] server-1beb164c-d829-99d5-a391-4ba9596a5688: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:09.616Z [INFO] server-a4743941-6289-1233-5944-02dcb5626528: copied to local snapshot: bytes=71 -2026-03-31T06:17:09.616Z [INFO] server-1beb164c-d829-99d5-a391-4ba9596a5688: copied to local snapshot: bytes=71 -2026-03-31T06:17:09.616Z [INFO] server-a4743941-6289-1233-5944-02dcb5626528: snapshot restore progress: id=2-113-1774937829615 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:09.616Z [INFO] server-a4743941-6289-1233-5944-02dcb5626528: Installed remote snapshot -2026-03-31T06:17:09.616Z [WARN] server-a4743941-6289-1233-5944-02dcb5626528: skipping application of old log: index=22 -2026-03-31T06:17:09.616Z [INFO] server-1beb164c-d829-99d5-a391-4ba9596a5688: snapshot restore progress: id=2-113-1774937829615 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:09.616Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:17:09.616Z [INFO] server-b6bc123e-1ffc-5060-96a6-2df4c8ddf8a8: pipelining replication: peer="{Voter server-a4743941-6289-1233-5944-02dcb5626528 a4743941-6289-1233-5944-02dcb5626528}" -2026-03-31T06:17:09.616Z [INFO] server-1beb164c-d829-99d5-a391-4ba9596a5688: Installed remote snapshot -2026-03-31T06:17:09.616Z [INFO] server-b6bc123e-1ffc-5060-96a6-2df4c8ddf8a8: pipelining replication: peer="{Voter server-1beb164c-d829-99d5-a391-4ba9596a5688 1beb164c-d829-99d5-a391-4ba9596a5688}" -2026-03-31T06:17:10.622Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:17:09.6167285 +0000 UTC m=+57.894335881 (2 nodes), 920ns from start of poll, 1.56105408s from cluster start. Timeout at 2026-03-31 06:17:10.62174494 +0000 UTC m=+58.899352321, 1.00501644s after stability -2026-03-31T06:17:10.634Z [INFO] server-b6bc123e-1ffc-5060-96a6-2df4c8ddf8a8: aborting pipeline replication: peer="{Voter server-1beb164c-d829-99d5-a391-4ba9596a5688 1beb164c-d829-99d5-a391-4ba9596a5688}" -2026-03-31T06:17:10.634Z [INFO] server-b6bc123e-1ffc-5060-96a6-2df4c8ddf8a8: aborting pipeline replication: peer="{Voter server-a4743941-6289-1233-5944-02dcb5626528 a4743941-6289-1233-5944-02dcb5626528}" +2025-02-25T23:58:29.794Z [INFO] server-b2eb8c56-b0e1-091a-51e2-fbbbff1eec79: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b2eb8c56-b0e1-091a-51e2-fbbbff1eec79 Address:b2eb8c56-b0e1-091a-51e2-fbbbff1eec79} {Suffrage:Voter ID:server-2a6eba49-a229-8182-fb17-1a5b44bbfa32 Address:2a6eba49-a229-8182-fb17-1a5b44bbfa32} {Suffrage:Voter ID:server-e35d10a8-5194-b254-fc98-ce5333cbf746 Address:e35d10a8-5194-b254-fc98-ce5333cbf746}]" +2025-02-25T23:58:29.794Z [INFO] server-2a6eba49-a229-8182-fb17-1a5b44bbfa32: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b2eb8c56-b0e1-091a-51e2-fbbbff1eec79 Address:b2eb8c56-b0e1-091a-51e2-fbbbff1eec79} {Suffrage:Voter ID:server-2a6eba49-a229-8182-fb17-1a5b44bbfa32 Address:2a6eba49-a229-8182-fb17-1a5b44bbfa32} {Suffrage:Voter ID:server-e35d10a8-5194-b254-fc98-ce5333cbf746 Address:e35d10a8-5194-b254-fc98-ce5333cbf746}]" +2025-02-25T23:58:29.794Z [INFO] server-e35d10a8-5194-b254-fc98-ce5333cbf746: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b2eb8c56-b0e1-091a-51e2-fbbbff1eec79 Address:b2eb8c56-b0e1-091a-51e2-fbbbff1eec79} {Suffrage:Voter ID:server-2a6eba49-a229-8182-fb17-1a5b44bbfa32 Address:2a6eba49-a229-8182-fb17-1a5b44bbfa32} {Suffrage:Voter ID:server-e35d10a8-5194-b254-fc98-ce5333cbf746 Address:e35d10a8-5194-b254-fc98-ce5333cbf746}]" +2025-02-25T23:58:29.794Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:58:29.794Z [INFO] server-b2eb8c56-b0e1-091a-51e2-fbbbff1eec79: entering follower state: follower="Node at b2eb8c56-b0e1-091a-51e2-fbbbff1eec79 [Follower]" leader-address= leader-id= +2025-02-25T23:58:29.794Z [INFO] server-2a6eba49-a229-8182-fb17-1a5b44bbfa32: entering follower state: follower="Node at 2a6eba49-a229-8182-fb17-1a5b44bbfa32 [Follower]" leader-address= leader-id= +2025-02-25T23:58:29.794Z [INFO] server-e35d10a8-5194-b254-fc98-ce5333cbf746: entering follower state: follower="Node at e35d10a8-5194-b254-fc98-ce5333cbf746 [Follower]" leader-address= leader-id= +2025-02-25T23:58:30.637Z [WARN] server-e35d10a8-5194-b254-fc98-ce5333cbf746: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:58:30.637Z [INFO] server-e35d10a8-5194-b254-fc98-ce5333cbf746: entering candidate state: node="Node at e35d10a8-5194-b254-fc98-ce5333cbf746 [Candidate]" term=2 +2025-02-25T23:58:30.638Z [INFO] server-e35d10a8-5194-b254-fc98-ce5333cbf746: election won: term=2 tally=2 +2025-02-25T23:58:30.638Z [INFO] server-e35d10a8-5194-b254-fc98-ce5333cbf746: entering leader state: leader="Node at e35d10a8-5194-b254-fc98-ce5333cbf746 [Leader]" +2025-02-25T23:58:30.638Z [INFO] server-e35d10a8-5194-b254-fc98-ce5333cbf746: added peer, starting replication: peer=server-b2eb8c56-b0e1-091a-51e2-fbbbff1eec79 +2025-02-25T23:58:30.638Z [INFO] server-e35d10a8-5194-b254-fc98-ce5333cbf746: added peer, starting replication: peer=server-2a6eba49-a229-8182-fb17-1a5b44bbfa32 +2025-02-25T23:58:30.640Z [INFO] server-e35d10a8-5194-b254-fc98-ce5333cbf746: pipelining replication: peer="{Voter server-b2eb8c56-b0e1-091a-51e2-fbbbff1eec79 b2eb8c56-b0e1-091a-51e2-fbbbff1eec79}" +2025-02-25T23:58:30.640Z [INFO] server-e35d10a8-5194-b254-fc98-ce5333cbf746: pipelining replication: peer="{Voter server-2a6eba49-a229-8182-fb17-1a5b44bbfa32 2a6eba49-a229-8182-fb17-1a5b44bbfa32}" +2025-02-25T23:58:31.645Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:58:30.640382075 +0000 UTC m=+58.249397482 (1 nodes), 845.656225ms from start of poll, 845.826386ms from cluster start. Timeout at 2025-02-25 23:58:31.645399475 +0000 UTC m=+59.254414882, 1.0050174s after stability +2025-02-25T23:58:31.646Z [INFO] server-e35d10a8-5194-b254-fc98-ce5333cbf746: starting snapshot up to: index=12 +2025-02-25T23:58:31.646Z [INFO] creating new snapshot: path=/tmp/raft2806516659/snapshots/2-12-1740527911646.tmp +2025-02-25T23:58:31.646Z [INFO] server-e35d10a8-5194-b254-fc98-ce5333cbf746: snapshot complete up to: index=12 +2025-02-25T23:58:31.647Z [INFO] creating new snapshot: path=/tmp/raft2806516659/snapshots/2-113-1740527911647.tmp +2025-02-25T23:58:31.647Z [INFO] server-e35d10a8-5194-b254-fc98-ce5333cbf746: copied to local snapshot: bytes=71 +2025-02-25T23:58:31.647Z [INFO] server-e35d10a8-5194-b254-fc98-ce5333cbf746: snapshot restore progress: id=2-113-1740527911647 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:31.647Z [INFO] server-e35d10a8-5194-b254-fc98-ce5333cbf746: restored user snapshot: index=113 +2025-02-25T23:58:31.647Z [ERROR] server-e35d10a8-5194-b254-fc98-ce5333cbf746: failed to get log: index=23 error="log not found" +2025-02-25T23:58:31.647Z [INFO] server-e35d10a8-5194-b254-fc98-ce5333cbf746: aborting pipeline replication: peer="{Voter server-b2eb8c56-b0e1-091a-51e2-fbbbff1eec79 b2eb8c56-b0e1-091a-51e2-fbbbff1eec79}" +2025-02-25T23:58:31.647Z [ERROR] server-e35d10a8-5194-b254-fc98-ce5333cbf746: failed to get log: index=23 error="log not found" +2025-02-25T23:58:31.647Z [INFO] server-e35d10a8-5194-b254-fc98-ce5333cbf746: aborting pipeline replication: peer="{Voter server-2a6eba49-a229-8182-fb17-1a5b44bbfa32 2a6eba49-a229-8182-fb17-1a5b44bbfa32}" +2025-02-25T23:58:31.654Z [ERROR] server-e35d10a8-5194-b254-fc98-ce5333cbf746: failed to get log: index=23 error="log not found" +2025-02-25T23:58:31.656Z [ERROR] server-e35d10a8-5194-b254-fc98-ce5333cbf746: failed to get log: index=23 error="log not found" +2025-02-25T23:58:31.662Z [INFO] creating new snapshot: path=/tmp/raft2425492770/snapshots/2-113-1740527911662.tmp +2025-02-25T23:58:31.662Z [INFO] server-2a6eba49-a229-8182-fb17-1a5b44bbfa32: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:31.662Z [INFO] server-2a6eba49-a229-8182-fb17-1a5b44bbfa32: copied to local snapshot: bytes=71 +2025-02-25T23:58:31.662Z [INFO] server-2a6eba49-a229-8182-fb17-1a5b44bbfa32: snapshot restore progress: id=2-113-1740527911662 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:31.662Z [INFO] server-2a6eba49-a229-8182-fb17-1a5b44bbfa32: Installed remote snapshot +2025-02-25T23:58:31.662Z [WARN] server-2a6eba49-a229-8182-fb17-1a5b44bbfa32: skipping application of old log: index=22 +2025-02-25T23:58:31.662Z [INFO] server-e35d10a8-5194-b254-fc98-ce5333cbf746: pipelining replication: peer="{Voter server-2a6eba49-a229-8182-fb17-1a5b44bbfa32 2a6eba49-a229-8182-fb17-1a5b44bbfa32}" +2025-02-25T23:58:31.662Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:58:31.662Z [INFO] creating new snapshot: path=/tmp/raft3808701251/snapshots/2-113-1740527911662.tmp +2025-02-25T23:58:31.662Z [INFO] server-b2eb8c56-b0e1-091a-51e2-fbbbff1eec79: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:31.663Z [INFO] server-b2eb8c56-b0e1-091a-51e2-fbbbff1eec79: copied to local snapshot: bytes=71 +2025-02-25T23:58:31.663Z [INFO] server-b2eb8c56-b0e1-091a-51e2-fbbbff1eec79: snapshot restore progress: id=2-113-1740527911662 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:31.663Z [INFO] server-b2eb8c56-b0e1-091a-51e2-fbbbff1eec79: Installed remote snapshot +2025-02-25T23:58:31.663Z [INFO] server-e35d10a8-5194-b254-fc98-ce5333cbf746: pipelining replication: peer="{Voter server-b2eb8c56-b0e1-091a-51e2-fbbbff1eec79 b2eb8c56-b0e1-091a-51e2-fbbbff1eec79}" +2025-02-25T23:58:32.669Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:58:31.662758705 +0000 UTC m=+59.271774112 (2 nodes), 480ns from start of poll, 1.868203016s from cluster start. Timeout at 2025-02-25 23:58:32.667777586 +0000 UTC m=+60.276792993, 1.005018881s after stability +2025-02-25T23:58:32.698Z [INFO] server-e35d10a8-5194-b254-fc98-ce5333cbf746: aborting pipeline replication: peer="{Voter server-b2eb8c56-b0e1-091a-51e2-fbbbff1eec79 b2eb8c56-b0e1-091a-51e2-fbbbff1eec79}" +2025-02-25T23:58:32.698Z [INFO] server-e35d10a8-5194-b254-fc98-ce5333cbf746: aborting pipeline replication: peer="{Voter server-2a6eba49-a229-8182-fb17-1a5b44bbfa32 2a6eba49-a229-8182-fb17-1a5b44bbfa32}" === RUN TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_false -2026-03-31T06:17:10.635Z [INFO] server-bfa88748-dc20-34e3-96d2-4431baabae9d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bfa88748-dc20-34e3-96d2-4431baabae9d Address:bfa88748-dc20-34e3-96d2-4431baabae9d} {Suffrage:Voter ID:server-4417bb25-3a54-caa0-765b-5d7b287cc1cf Address:4417bb25-3a54-caa0-765b-5d7b287cc1cf} {Suffrage:Voter ID:server-f0f2588a-bd40-55a3-ca48-4b05b3c48927 Address:f0f2588a-bd40-55a3-ca48-4b05b3c48927}]" -2026-03-31T06:17:10.635Z [INFO] server-4417bb25-3a54-caa0-765b-5d7b287cc1cf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bfa88748-dc20-34e3-96d2-4431baabae9d Address:bfa88748-dc20-34e3-96d2-4431baabae9d} {Suffrage:Voter ID:server-4417bb25-3a54-caa0-765b-5d7b287cc1cf Address:4417bb25-3a54-caa0-765b-5d7b287cc1cf} {Suffrage:Voter ID:server-f0f2588a-bd40-55a3-ca48-4b05b3c48927 Address:f0f2588a-bd40-55a3-ca48-4b05b3c48927}]" -2026-03-31T06:17:10.635Z [INFO] server-bfa88748-dc20-34e3-96d2-4431baabae9d: entering follower state: follower="Node at bfa88748-dc20-34e3-96d2-4431baabae9d [Follower]" leader-address= leader-id= -2026-03-31T06:17:10.636Z [INFO] server-f0f2588a-bd40-55a3-ca48-4b05b3c48927: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bfa88748-dc20-34e3-96d2-4431baabae9d Address:bfa88748-dc20-34e3-96d2-4431baabae9d} {Suffrage:Voter ID:server-4417bb25-3a54-caa0-765b-5d7b287cc1cf Address:4417bb25-3a54-caa0-765b-5d7b287cc1cf} {Suffrage:Voter ID:server-f0f2588a-bd40-55a3-ca48-4b05b3c48927 Address:f0f2588a-bd40-55a3-ca48-4b05b3c48927}]" -2026-03-31T06:17:10.636Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:10.636Z [INFO] server-4417bb25-3a54-caa0-765b-5d7b287cc1cf: entering follower state: follower="Node at 4417bb25-3a54-caa0-765b-5d7b287cc1cf [Follower]" leader-address= leader-id= -2026-03-31T06:17:10.636Z [INFO] server-f0f2588a-bd40-55a3-ca48-4b05b3c48927: entering follower state: follower="Node at f0f2588a-bd40-55a3-ca48-4b05b3c48927 [Follower]" leader-address= leader-id= -2026-03-31T06:17:11.168Z [WARN] server-f0f2588a-bd40-55a3-ca48-4b05b3c48927: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:11.168Z [INFO] server-f0f2588a-bd40-55a3-ca48-4b05b3c48927: entering candidate state: node="Node at f0f2588a-bd40-55a3-ca48-4b05b3c48927 [Candidate]" term=2 -2026-03-31T06:17:11.168Z [INFO] server-f0f2588a-bd40-55a3-ca48-4b05b3c48927: election won: term=2 tally=2 -2026-03-31T06:17:11.168Z [INFO] server-f0f2588a-bd40-55a3-ca48-4b05b3c48927: entering leader state: leader="Node at f0f2588a-bd40-55a3-ca48-4b05b3c48927 [Leader]" -2026-03-31T06:17:11.168Z [INFO] server-f0f2588a-bd40-55a3-ca48-4b05b3c48927: added peer, starting replication: peer=server-bfa88748-dc20-34e3-96d2-4431baabae9d -2026-03-31T06:17:11.168Z [INFO] server-f0f2588a-bd40-55a3-ca48-4b05b3c48927: added peer, starting replication: peer=server-4417bb25-3a54-caa0-765b-5d7b287cc1cf -2026-03-31T06:17:11.168Z [INFO] server-f0f2588a-bd40-55a3-ca48-4b05b3c48927: pipelining replication: peer="{Voter server-4417bb25-3a54-caa0-765b-5d7b287cc1cf 4417bb25-3a54-caa0-765b-5d7b287cc1cf}" -2026-03-31T06:17:11.168Z [INFO] server-f0f2588a-bd40-55a3-ca48-4b05b3c48927: pipelining replication: peer="{Voter server-bfa88748-dc20-34e3-96d2-4431baabae9d bfa88748-dc20-34e3-96d2-4431baabae9d}" -2026-03-31T06:17:12.174Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:11.16860082 +0000 UTC m=+59.446208201 (1 nodes), 532.47936ms from start of poll, 532.89128ms from cluster start. Timeout at 2026-03-31 06:17:12.1736041 +0000 UTC m=+60.451211481, 1.00500328s after stability -2026-03-31T06:17:12.175Z [INFO] server-f0f2588a-bd40-55a3-ca48-4b05b3c48927: starting snapshot up to: index=12 -2026-03-31T06:17:12.175Z [INFO] creating new snapshot: path=/tmp/raft3945630257/snapshots/2-12-1774937832175.tmp -2026-03-31T06:17:12.175Z [INFO] server-f0f2588a-bd40-55a3-ca48-4b05b3c48927: snapshot complete up to: index=12 -2026-03-31T06:17:12.176Z [INFO] creating new snapshot: path=/tmp/raft3945630257/snapshots/2-113-1774937832176.tmp -2026-03-31T06:17:12.177Z [INFO] server-f0f2588a-bd40-55a3-ca48-4b05b3c48927: copied to local snapshot: bytes=71 -2026-03-31T06:17:12.178Z [INFO] server-f0f2588a-bd40-55a3-ca48-4b05b3c48927: snapshot restore progress: id=2-113-1774937832176 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:12.178Z [INFO] server-f0f2588a-bd40-55a3-ca48-4b05b3c48927: removing all old logs from log store -2026-03-31T06:17:12.178Z [INFO] server-f0f2588a-bd40-55a3-ca48-4b05b3c48927: compacting logs: from=1 to=22 -2026-03-31T06:17:12.178Z [INFO] server-f0f2588a-bd40-55a3-ca48-4b05b3c48927: restored user snapshot: index=113 -2026-03-31T06:17:12.178Z [ERROR] server-f0f2588a-bd40-55a3-ca48-4b05b3c48927: failed to get log: index=22 error="log not found" -2026-03-31T06:17:12.178Z [INFO] server-f0f2588a-bd40-55a3-ca48-4b05b3c48927: aborting pipeline replication: peer="{Voter server-4417bb25-3a54-caa0-765b-5d7b287cc1cf 4417bb25-3a54-caa0-765b-5d7b287cc1cf}" -2026-03-31T06:17:12.178Z [ERROR] server-f0f2588a-bd40-55a3-ca48-4b05b3c48927: failed to get log: index=22 error="log not found" -2026-03-31T06:17:12.178Z [INFO] server-f0f2588a-bd40-55a3-ca48-4b05b3c48927: aborting pipeline replication: peer="{Voter server-bfa88748-dc20-34e3-96d2-4431baabae9d bfa88748-dc20-34e3-96d2-4431baabae9d}" -2026-03-31T06:17:12.188Z [ERROR] server-f0f2588a-bd40-55a3-ca48-4b05b3c48927: failed to get log: index=22 error="log not found" -2026-03-31T06:17:12.188Z [ERROR] server-f0f2588a-bd40-55a3-ca48-4b05b3c48927: failed to get log: index=22 error="log not found" -2026-03-31T06:17:12.189Z [INFO] creating new snapshot: path=/tmp/raft3254952146/snapshots/2-113-1774937832189.tmp -2026-03-31T06:17:12.189Z [INFO] creating new snapshot: path=/tmp/raft3786439934/snapshots/2-113-1774937832189.tmp -2026-03-31T06:17:12.189Z [INFO] server-4417bb25-3a54-caa0-765b-5d7b287cc1cf: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:12.189Z [INFO] server-bfa88748-dc20-34e3-96d2-4431baabae9d: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:12.189Z [INFO] server-4417bb25-3a54-caa0-765b-5d7b287cc1cf: copied to local snapshot: bytes=71 -2026-03-31T06:17:12.189Z [INFO] server-bfa88748-dc20-34e3-96d2-4431baabae9d: copied to local snapshot: bytes=71 -2026-03-31T06:17:12.189Z [INFO] server-4417bb25-3a54-caa0-765b-5d7b287cc1cf: snapshot restore progress: id=2-113-1774937832189 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:12.189Z [INFO] server-4417bb25-3a54-caa0-765b-5d7b287cc1cf: removing all old logs from log store -2026-03-31T06:17:12.189Z [INFO] server-4417bb25-3a54-caa0-765b-5d7b287cc1cf: compacting logs: from=1 to=22 -2026-03-31T06:17:12.189Z [INFO] server-4417bb25-3a54-caa0-765b-5d7b287cc1cf: Installed remote snapshot -2026-03-31T06:17:12.189Z [WARN] server-4417bb25-3a54-caa0-765b-5d7b287cc1cf: skipping application of old log: index=22 -2026-03-31T06:17:12.189Z [INFO] server-f0f2588a-bd40-55a3-ca48-4b05b3c48927: pipelining replication: peer="{Voter server-4417bb25-3a54-caa0-765b-5d7b287cc1cf 4417bb25-3a54-caa0-765b-5d7b287cc1cf}" -2026-03-31T06:17:12.189Z [INFO] server-bfa88748-dc20-34e3-96d2-4431baabae9d: snapshot restore progress: id=2-113-1774937832189 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:12.190Z [INFO] server-bfa88748-dc20-34e3-96d2-4431baabae9d: removing all old logs from log store -2026-03-31T06:17:12.190Z [INFO] server-bfa88748-dc20-34e3-96d2-4431baabae9d: compacting logs: from=1 to=22 -2026-03-31T06:17:12.190Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:17:12.190Z [INFO] server-bfa88748-dc20-34e3-96d2-4431baabae9d: Installed remote snapshot -2026-03-31T06:17:12.190Z [INFO] server-f0f2588a-bd40-55a3-ca48-4b05b3c48927: pipelining replication: peer="{Voter server-bfa88748-dc20-34e3-96d2-4431baabae9d bfa88748-dc20-34e3-96d2-4431baabae9d}" -2026-03-31T06:17:13.195Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:17:12.19003154 +0000 UTC m=+60.467638881 (2 nodes), 560ns from start of poll, 1.55432196s from cluster start. Timeout at 2026-03-31 06:17:13.19504794 +0000 UTC m=+61.472655321, 1.00501644s after stability -2026-03-31T06:17:13.206Z [INFO] server-f0f2588a-bd40-55a3-ca48-4b05b3c48927: aborting pipeline replication: peer="{Voter server-4417bb25-3a54-caa0-765b-5d7b287cc1cf 4417bb25-3a54-caa0-765b-5d7b287cc1cf}" -2026-03-31T06:17:13.206Z [INFO] server-f0f2588a-bd40-55a3-ca48-4b05b3c48927: aborting pipeline replication: peer="{Voter server-bfa88748-dc20-34e3-96d2-4431baabae9d bfa88748-dc20-34e3-96d2-4431baabae9d}" +2025-02-25T23:58:32.698Z [INFO] server-e23a1651-9ab5-a680-b008-43db46ec37ff: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e23a1651-9ab5-a680-b008-43db46ec37ff Address:e23a1651-9ab5-a680-b008-43db46ec37ff} {Suffrage:Voter ID:server-508476fb-6211-7893-4b1c-a1af0412763c Address:508476fb-6211-7893-4b1c-a1af0412763c} {Suffrage:Voter ID:server-25b081bc-3d15-9b4a-4954-33bb78681e1c Address:25b081bc-3d15-9b4a-4954-33bb78681e1c}]" +2025-02-25T23:58:32.698Z [INFO] server-508476fb-6211-7893-4b1c-a1af0412763c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e23a1651-9ab5-a680-b008-43db46ec37ff Address:e23a1651-9ab5-a680-b008-43db46ec37ff} {Suffrage:Voter ID:server-508476fb-6211-7893-4b1c-a1af0412763c Address:508476fb-6211-7893-4b1c-a1af0412763c} {Suffrage:Voter ID:server-25b081bc-3d15-9b4a-4954-33bb78681e1c Address:25b081bc-3d15-9b4a-4954-33bb78681e1c}]" +2025-02-25T23:58:32.698Z [INFO] server-25b081bc-3d15-9b4a-4954-33bb78681e1c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e23a1651-9ab5-a680-b008-43db46ec37ff Address:e23a1651-9ab5-a680-b008-43db46ec37ff} {Suffrage:Voter ID:server-508476fb-6211-7893-4b1c-a1af0412763c Address:508476fb-6211-7893-4b1c-a1af0412763c} {Suffrage:Voter ID:server-25b081bc-3d15-9b4a-4954-33bb78681e1c Address:25b081bc-3d15-9b4a-4954-33bb78681e1c}]" +2025-02-25T23:58:32.698Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:58:32.698Z [INFO] server-e23a1651-9ab5-a680-b008-43db46ec37ff: entering follower state: follower="Node at e23a1651-9ab5-a680-b008-43db46ec37ff [Follower]" leader-address= leader-id= +2025-02-25T23:58:32.698Z [INFO] server-508476fb-6211-7893-4b1c-a1af0412763c: entering follower state: follower="Node at 508476fb-6211-7893-4b1c-a1af0412763c [Follower]" leader-address= leader-id= +2025-02-25T23:58:32.698Z [INFO] server-25b081bc-3d15-9b4a-4954-33bb78681e1c: entering follower state: follower="Node at 25b081bc-3d15-9b4a-4954-33bb78681e1c [Follower]" leader-address= leader-id= +2025-02-25T23:58:33.289Z [WARN] server-e23a1651-9ab5-a680-b008-43db46ec37ff: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:58:33.289Z [INFO] server-e23a1651-9ab5-a680-b008-43db46ec37ff: entering candidate state: node="Node at e23a1651-9ab5-a680-b008-43db46ec37ff [Candidate]" term=2 +2025-02-25T23:58:33.290Z [INFO] server-e23a1651-9ab5-a680-b008-43db46ec37ff: election won: term=2 tally=2 +2025-02-25T23:58:33.290Z [INFO] server-e23a1651-9ab5-a680-b008-43db46ec37ff: entering leader state: leader="Node at e23a1651-9ab5-a680-b008-43db46ec37ff [Leader]" +2025-02-25T23:58:33.290Z [INFO] server-e23a1651-9ab5-a680-b008-43db46ec37ff: added peer, starting replication: peer=server-508476fb-6211-7893-4b1c-a1af0412763c +2025-02-25T23:58:33.290Z [INFO] server-e23a1651-9ab5-a680-b008-43db46ec37ff: added peer, starting replication: peer=server-25b081bc-3d15-9b4a-4954-33bb78681e1c +2025-02-25T23:58:33.290Z [INFO] server-e23a1651-9ab5-a680-b008-43db46ec37ff: pipelining replication: peer="{Voter server-25b081bc-3d15-9b4a-4954-33bb78681e1c 25b081bc-3d15-9b4a-4954-33bb78681e1c}" +2025-02-25T23:58:33.290Z [INFO] server-e23a1651-9ab5-a680-b008-43db46ec37ff: pipelining replication: peer="{Voter server-508476fb-6211-7893-4b1c-a1af0412763c 508476fb-6211-7893-4b1c-a1af0412763c}" +2025-02-25T23:58:34.298Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:58:33.290256908 +0000 UTC m=+60.899272275 (1 nodes), 591.552927ms from start of poll, 591.749808ms from cluster start. Timeout at 2025-02-25 23:58:34.295280868 +0000 UTC m=+61.904296235, 1.00502396s after stability +2025-02-25T23:58:34.298Z [INFO] server-e23a1651-9ab5-a680-b008-43db46ec37ff: starting snapshot up to: index=12 +2025-02-25T23:58:34.298Z [INFO] creating new snapshot: path=/tmp/raft1443577756/snapshots/2-12-1740527914298.tmp +2025-02-25T23:58:34.298Z [INFO] server-e23a1651-9ab5-a680-b008-43db46ec37ff: snapshot complete up to: index=12 +2025-02-25T23:58:34.299Z [INFO] creating new snapshot: path=/tmp/raft1443577756/snapshots/2-113-1740527914299.tmp +2025-02-25T23:58:34.299Z [INFO] server-e23a1651-9ab5-a680-b008-43db46ec37ff: copied to local snapshot: bytes=71 +2025-02-25T23:58:34.299Z [INFO] server-e23a1651-9ab5-a680-b008-43db46ec37ff: snapshot restore progress: id=2-113-1740527914299 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:34.299Z [INFO] server-e23a1651-9ab5-a680-b008-43db46ec37ff: removing all old logs from log store +2025-02-25T23:58:34.299Z [INFO] server-e23a1651-9ab5-a680-b008-43db46ec37ff: compacting logs: from=1 to=22 +2025-02-25T23:58:34.299Z [INFO] server-e23a1651-9ab5-a680-b008-43db46ec37ff: restored user snapshot: index=113 +2025-02-25T23:58:34.299Z [ERROR] server-e23a1651-9ab5-a680-b008-43db46ec37ff: failed to get log: index=22 error="log not found" +2025-02-25T23:58:34.299Z [INFO] server-e23a1651-9ab5-a680-b008-43db46ec37ff: aborting pipeline replication: peer="{Voter server-25b081bc-3d15-9b4a-4954-33bb78681e1c 25b081bc-3d15-9b4a-4954-33bb78681e1c}" +2025-02-25T23:58:34.299Z [ERROR] server-e23a1651-9ab5-a680-b008-43db46ec37ff: failed to get log: index=22 error="log not found" +2025-02-25T23:58:34.299Z [INFO] server-e23a1651-9ab5-a680-b008-43db46ec37ff: aborting pipeline replication: peer="{Voter server-508476fb-6211-7893-4b1c-a1af0412763c 508476fb-6211-7893-4b1c-a1af0412763c}" +2025-02-25T23:58:34.305Z [ERROR] server-e23a1651-9ab5-a680-b008-43db46ec37ff: failed to get log: index=22 error="log not found" +2025-02-25T23:58:34.306Z [INFO] creating new snapshot: path=/tmp/raft769079184/snapshots/2-113-1740527914306.tmp +2025-02-25T23:58:34.306Z [INFO] server-508476fb-6211-7893-4b1c-a1af0412763c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:34.306Z [INFO] server-508476fb-6211-7893-4b1c-a1af0412763c: copied to local snapshot: bytes=71 +2025-02-25T23:58:34.306Z [INFO] server-508476fb-6211-7893-4b1c-a1af0412763c: snapshot restore progress: id=2-113-1740527914306 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:34.306Z [INFO] server-508476fb-6211-7893-4b1c-a1af0412763c: removing all old logs from log store +2025-02-25T23:58:34.306Z [INFO] server-508476fb-6211-7893-4b1c-a1af0412763c: compacting logs: from=1 to=22 +2025-02-25T23:58:34.306Z [INFO] server-508476fb-6211-7893-4b1c-a1af0412763c: Installed remote snapshot +2025-02-25T23:58:34.306Z [WARN] server-508476fb-6211-7893-4b1c-a1af0412763c: skipping application of old log: index=22 +2025-02-25T23:58:34.306Z [INFO] server-e23a1651-9ab5-a680-b008-43db46ec37ff: pipelining replication: peer="{Voter server-508476fb-6211-7893-4b1c-a1af0412763c 508476fb-6211-7893-4b1c-a1af0412763c}" +2025-02-25T23:58:34.306Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:58:34.309Z [ERROR] server-e23a1651-9ab5-a680-b008-43db46ec37ff: failed to get log: index=22 error="log not found" +2025-02-25T23:58:34.310Z [INFO] creating new snapshot: path=/tmp/raft701570311/snapshots/2-113-1740527914310.tmp +2025-02-25T23:58:34.310Z [INFO] server-25b081bc-3d15-9b4a-4954-33bb78681e1c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:34.310Z [INFO] server-25b081bc-3d15-9b4a-4954-33bb78681e1c: copied to local snapshot: bytes=71 +2025-02-25T23:58:34.310Z [INFO] server-25b081bc-3d15-9b4a-4954-33bb78681e1c: snapshot restore progress: id=2-113-1740527914310 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:34.310Z [INFO] server-25b081bc-3d15-9b4a-4954-33bb78681e1c: removing all old logs from log store +2025-02-25T23:58:34.310Z [INFO] server-25b081bc-3d15-9b4a-4954-33bb78681e1c: compacting logs: from=1 to=22 +2025-02-25T23:58:34.310Z [INFO] server-25b081bc-3d15-9b4a-4954-33bb78681e1c: Installed remote snapshot +2025-02-25T23:58:34.310Z [INFO] server-e23a1651-9ab5-a680-b008-43db46ec37ff: pipelining replication: peer="{Voter server-25b081bc-3d15-9b4a-4954-33bb78681e1c 25b081bc-3d15-9b4a-4954-33bb78681e1c}" +2025-02-25T23:58:35.313Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:58:34.30662562 +0000 UTC m=+61.915641027 (2 nodes), 680ns from start of poll, 1.60811856s from cluster start. Timeout at 2025-02-25 23:58:35.3116451 +0000 UTC m=+62.920660507, 1.00501948s after stability +2025-02-25T23:58:35.330Z [INFO] server-e23a1651-9ab5-a680-b008-43db46ec37ff: aborting pipeline replication: peer="{Voter server-25b081bc-3d15-9b4a-4954-33bb78681e1c 25b081bc-3d15-9b4a-4954-33bb78681e1c}" +2025-02-25T23:58:35.330Z [INFO] server-e23a1651-9ab5-a680-b008-43db46ec37ff: aborting pipeline replication: peer="{Voter server-508476fb-6211-7893-4b1c-a1af0412763c 508476fb-6211-7893-4b1c-a1af0412763c}" === RUN TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_true -2026-03-31T06:17:13.207Z [INFO] server-dc0db525-da3a-2171-f9ad-bebffcc6b586: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dc0db525-da3a-2171-f9ad-bebffcc6b586 Address:dc0db525-da3a-2171-f9ad-bebffcc6b586} {Suffrage:Voter ID:server-2246677e-332c-c3bb-f928-7bcf7ea7d52e Address:2246677e-332c-c3bb-f928-7bcf7ea7d52e} {Suffrage:Voter ID:server-02dd54c7-7f0e-14ef-d6f4-d8caf098a278 Address:02dd54c7-7f0e-14ef-d6f4-d8caf098a278}]" -2026-03-31T06:17:13.207Z [INFO] server-dc0db525-da3a-2171-f9ad-bebffcc6b586: entering follower state: follower="Node at dc0db525-da3a-2171-f9ad-bebffcc6b586 [Follower]" leader-address= leader-id= -2026-03-31T06:17:13.207Z [INFO] server-2246677e-332c-c3bb-f928-7bcf7ea7d52e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dc0db525-da3a-2171-f9ad-bebffcc6b586 Address:dc0db525-da3a-2171-f9ad-bebffcc6b586} {Suffrage:Voter ID:server-2246677e-332c-c3bb-f928-7bcf7ea7d52e Address:2246677e-332c-c3bb-f928-7bcf7ea7d52e} {Suffrage:Voter ID:server-02dd54c7-7f0e-14ef-d6f4-d8caf098a278 Address:02dd54c7-7f0e-14ef-d6f4-d8caf098a278}]" -2026-03-31T06:17:13.207Z [INFO] server-2246677e-332c-c3bb-f928-7bcf7ea7d52e: entering follower state: follower="Node at 2246677e-332c-c3bb-f928-7bcf7ea7d52e [Follower]" leader-address= leader-id= -2026-03-31T06:17:13.207Z [INFO] server-02dd54c7-7f0e-14ef-d6f4-d8caf098a278: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dc0db525-da3a-2171-f9ad-bebffcc6b586 Address:dc0db525-da3a-2171-f9ad-bebffcc6b586} {Suffrage:Voter ID:server-2246677e-332c-c3bb-f928-7bcf7ea7d52e Address:2246677e-332c-c3bb-f928-7bcf7ea7d52e} {Suffrage:Voter ID:server-02dd54c7-7f0e-14ef-d6f4-d8caf098a278 Address:02dd54c7-7f0e-14ef-d6f4-d8caf098a278}]" -2026-03-31T06:17:13.207Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:13.207Z [INFO] server-02dd54c7-7f0e-14ef-d6f4-d8caf098a278: entering follower state: follower="Node at 02dd54c7-7f0e-14ef-d6f4-d8caf098a278 [Follower]" leader-address= leader-id= -2026-03-31T06:17:13.865Z [WARN] server-dc0db525-da3a-2171-f9ad-bebffcc6b586: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:13.866Z [INFO] server-dc0db525-da3a-2171-f9ad-bebffcc6b586: entering candidate state: node="Node at dc0db525-da3a-2171-f9ad-bebffcc6b586 [Candidate]" term=2 -2026-03-31T06:17:13.866Z [INFO] server-dc0db525-da3a-2171-f9ad-bebffcc6b586: election won: term=2 tally=2 -2026-03-31T06:17:13.866Z [INFO] server-dc0db525-da3a-2171-f9ad-bebffcc6b586: entering leader state: leader="Node at dc0db525-da3a-2171-f9ad-bebffcc6b586 [Leader]" -2026-03-31T06:17:13.866Z [INFO] server-dc0db525-da3a-2171-f9ad-bebffcc6b586: added peer, starting replication: peer=server-2246677e-332c-c3bb-f928-7bcf7ea7d52e -2026-03-31T06:17:13.866Z [INFO] server-dc0db525-da3a-2171-f9ad-bebffcc6b586: added peer, starting replication: peer=server-02dd54c7-7f0e-14ef-d6f4-d8caf098a278 -2026-03-31T06:17:13.866Z [INFO] server-dc0db525-da3a-2171-f9ad-bebffcc6b586: pipelining replication: peer="{Voter server-02dd54c7-7f0e-14ef-d6f4-d8caf098a278 02dd54c7-7f0e-14ef-d6f4-d8caf098a278}" -2026-03-31T06:17:13.866Z [INFO] server-dc0db525-da3a-2171-f9ad-bebffcc6b586: pipelining replication: peer="{Voter server-2246677e-332c-c3bb-f928-7bcf7ea7d52e 2246677e-332c-c3bb-f928-7bcf7ea7d52e}" -2026-03-31T06:17:14.871Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:13.86644078 +0000 UTC m=+62.144048121 (1 nodes), 658.59768ms from start of poll, 659.11952ms from cluster start. Timeout at 2026-03-31 06:17:14.87144194 +0000 UTC m=+63.149049281, 1.00500116s after stability -2026-03-31T06:17:14.872Z [INFO] server-dc0db525-da3a-2171-f9ad-bebffcc6b586: starting snapshot up to: index=12 -2026-03-31T06:17:14.872Z [INFO] creating new snapshot: path=/tmp/raft1211106798/snapshots/2-12-1774937834872.tmp -2026-03-31T06:17:14.872Z [INFO] server-dc0db525-da3a-2171-f9ad-bebffcc6b586: snapshot complete up to: index=12 -2026-03-31T06:17:14.873Z [INFO] server-c33788ab-1405-f9aa-f225-f6977710add8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c33788ab-1405-f9aa-f225-f6977710add8 Address:c33788ab-1405-f9aa-f225-f6977710add8} {Suffrage:Voter ID:server-3c76c0d2-0bb9-ad86-76e0-ec71e7dd490a Address:3c76c0d2-0bb9-ad86-76e0-ec71e7dd490a} {Suffrage:Voter ID:server-7103a22b-9cb3-50c6-d80e-148e3fd73724 Address:7103a22b-9cb3-50c6-d80e-148e3fd73724}]" -2026-03-31T06:17:14.873Z [INFO] server-3c76c0d2-0bb9-ad86-76e0-ec71e7dd490a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c33788ab-1405-f9aa-f225-f6977710add8 Address:c33788ab-1405-f9aa-f225-f6977710add8} {Suffrage:Voter ID:server-3c76c0d2-0bb9-ad86-76e0-ec71e7dd490a Address:3c76c0d2-0bb9-ad86-76e0-ec71e7dd490a} {Suffrage:Voter ID:server-7103a22b-9cb3-50c6-d80e-148e3fd73724 Address:7103a22b-9cb3-50c6-d80e-148e3fd73724}]" -2026-03-31T06:17:14.873Z [INFO] server-7103a22b-9cb3-50c6-d80e-148e3fd73724: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c33788ab-1405-f9aa-f225-f6977710add8 Address:c33788ab-1405-f9aa-f225-f6977710add8} {Suffrage:Voter ID:server-3c76c0d2-0bb9-ad86-76e0-ec71e7dd490a Address:3c76c0d2-0bb9-ad86-76e0-ec71e7dd490a} {Suffrage:Voter ID:server-7103a22b-9cb3-50c6-d80e-148e3fd73724 Address:7103a22b-9cb3-50c6-d80e-148e3fd73724}]" -2026-03-31T06:17:14.873Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:14.873Z [INFO] server-c33788ab-1405-f9aa-f225-f6977710add8: entering follower state: follower="Node at c33788ab-1405-f9aa-f225-f6977710add8 [Follower]" leader-address= leader-id= -2026-03-31T06:17:14.873Z [INFO] server-3c76c0d2-0bb9-ad86-76e0-ec71e7dd490a: entering follower state: follower="Node at 3c76c0d2-0bb9-ad86-76e0-ec71e7dd490a [Follower]" leader-address= leader-id= -2026-03-31T06:17:14.873Z [INFO] server-7103a22b-9cb3-50c6-d80e-148e3fd73724: entering follower state: follower="Node at 7103a22b-9cb3-50c6-d80e-148e3fd73724 [Follower]" leader-address= leader-id= -2026-03-31T06:17:15.520Z [WARN] server-7103a22b-9cb3-50c6-d80e-148e3fd73724: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:15.520Z [INFO] server-7103a22b-9cb3-50c6-d80e-148e3fd73724: entering candidate state: node="Node at 7103a22b-9cb3-50c6-d80e-148e3fd73724 [Candidate]" term=2 -2026-03-31T06:17:15.520Z [INFO] server-7103a22b-9cb3-50c6-d80e-148e3fd73724: election won: term=2 tally=2 -2026-03-31T06:17:15.520Z [INFO] server-7103a22b-9cb3-50c6-d80e-148e3fd73724: entering leader state: leader="Node at 7103a22b-9cb3-50c6-d80e-148e3fd73724 [Leader]" -2026-03-31T06:17:15.520Z [INFO] server-7103a22b-9cb3-50c6-d80e-148e3fd73724: added peer, starting replication: peer=server-c33788ab-1405-f9aa-f225-f6977710add8 -2026-03-31T06:17:15.520Z [INFO] server-7103a22b-9cb3-50c6-d80e-148e3fd73724: added peer, starting replication: peer=server-3c76c0d2-0bb9-ad86-76e0-ec71e7dd490a -2026-03-31T06:17:15.520Z [INFO] server-7103a22b-9cb3-50c6-d80e-148e3fd73724: pipelining replication: peer="{Voter server-c33788ab-1405-f9aa-f225-f6977710add8 c33788ab-1405-f9aa-f225-f6977710add8}" -2026-03-31T06:17:15.520Z [INFO] server-7103a22b-9cb3-50c6-d80e-148e3fd73724: pipelining replication: peer="{Voter server-3c76c0d2-0bb9-ad86-76e0-ec71e7dd490a 3c76c0d2-0bb9-ad86-76e0-ec71e7dd490a}" -2026-03-31T06:17:16.526Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:15.52054198 +0000 UTC m=+63.798149361 (1 nodes), 647.16308ms from start of poll, 647.4322ms from cluster start. Timeout at 2026-03-31 06:17:16.52554382 +0000 UTC m=+64.803151201, 1.00500184s after stability -2026-03-31T06:17:16.526Z [INFO] creating new snapshot: path=/tmp/raft279774614/snapshots/2-113-1774937836526.tmp -2026-03-31T06:17:16.526Z [INFO] server-7103a22b-9cb3-50c6-d80e-148e3fd73724: copied to local snapshot: bytes=71 -2026-03-31T06:17:16.527Z [INFO] server-7103a22b-9cb3-50c6-d80e-148e3fd73724: snapshot restore progress: id=2-113-1774937836526 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:16.527Z [INFO] server-7103a22b-9cb3-50c6-d80e-148e3fd73724: restored user snapshot: index=113 -2026-03-31T06:17:16.527Z [ERROR] server-7103a22b-9cb3-50c6-d80e-148e3fd73724: failed to get log: index=3 error="log not found" -2026-03-31T06:17:16.527Z [INFO] server-7103a22b-9cb3-50c6-d80e-148e3fd73724: aborting pipeline replication: peer="{Voter server-3c76c0d2-0bb9-ad86-76e0-ec71e7dd490a 3c76c0d2-0bb9-ad86-76e0-ec71e7dd490a}" -2026-03-31T06:17:16.527Z [ERROR] server-7103a22b-9cb3-50c6-d80e-148e3fd73724: failed to get log: index=3 error="log not found" -2026-03-31T06:17:16.527Z [INFO] server-7103a22b-9cb3-50c6-d80e-148e3fd73724: aborting pipeline replication: peer="{Voter server-c33788ab-1405-f9aa-f225-f6977710add8 c33788ab-1405-f9aa-f225-f6977710add8}" -2026-03-31T06:17:16.534Z [ERROR] server-7103a22b-9cb3-50c6-d80e-148e3fd73724: failed to get log: index=3 error="log not found" -2026-03-31T06:17:16.535Z [INFO] creating new snapshot: path=/tmp/raft3000024908/snapshots/2-113-1774937836535.tmp -2026-03-31T06:17:16.535Z [ERROR] server-7103a22b-9cb3-50c6-d80e-148e3fd73724: failed to get log: index=3 error="log not found" -2026-03-31T06:17:16.535Z [INFO] server-c33788ab-1405-f9aa-f225-f6977710add8: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:16.535Z [INFO] creating new snapshot: path=/tmp/raft2889474590/snapshots/2-113-1774937836535.tmp -2026-03-31T06:17:16.536Z [INFO] server-c33788ab-1405-f9aa-f225-f6977710add8: copied to local snapshot: bytes=71 -2026-03-31T06:17:16.536Z [INFO] server-3c76c0d2-0bb9-ad86-76e0-ec71e7dd490a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:16.536Z [INFO] server-c33788ab-1405-f9aa-f225-f6977710add8: snapshot restore progress: id=2-113-1774937836535 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:16.537Z [INFO] server-c33788ab-1405-f9aa-f225-f6977710add8: Installed remote snapshot -2026-03-31T06:17:16.537Z [INFO] server-3c76c0d2-0bb9-ad86-76e0-ec71e7dd490a: copied to local snapshot: bytes=71 -2026-03-31T06:17:16.537Z [INFO] server-7103a22b-9cb3-50c6-d80e-148e3fd73724: pipelining replication: peer="{Voter server-c33788ab-1405-f9aa-f225-f6977710add8 c33788ab-1405-f9aa-f225-f6977710add8}" -2026-03-31T06:17:16.537Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:17:16.537Z [INFO] server-3c76c0d2-0bb9-ad86-76e0-ec71e7dd490a: snapshot restore progress: id=2-113-1774937836535 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:16.537Z [INFO] server-3c76c0d2-0bb9-ad86-76e0-ec71e7dd490a: Installed remote snapshot -2026-03-31T06:17:16.537Z [INFO] server-7103a22b-9cb3-50c6-d80e-148e3fd73724: pipelining replication: peer="{Voter server-3c76c0d2-0bb9-ad86-76e0-ec71e7dd490a 3c76c0d2-0bb9-ad86-76e0-ec71e7dd490a}" -2026-03-31T06:17:17.543Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:17:16.53732302 +0000 UTC m=+64.814930401 (2 nodes), 880ns from start of poll, 1.66421324s from cluster start. Timeout at 2026-03-31 06:17:17.54235126 +0000 UTC m=+65.819958641, 1.00502824s after stability -2026-03-31T06:17:17.554Z [INFO] server-dc0db525-da3a-2171-f9ad-bebffcc6b586: aborting pipeline replication: peer="{Voter server-02dd54c7-7f0e-14ef-d6f4-d8caf098a278 02dd54c7-7f0e-14ef-d6f4-d8caf098a278}" -2026-03-31T06:17:17.554Z [INFO] server-dc0db525-da3a-2171-f9ad-bebffcc6b586: aborting pipeline replication: peer="{Voter server-2246677e-332c-c3bb-f928-7bcf7ea7d52e 2246677e-332c-c3bb-f928-7bcf7ea7d52e}" +2025-02-25T23:58:35.330Z [INFO] server-08b3d66f-5b31-ca43-cc7e-efdaff7e0107: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-08b3d66f-5b31-ca43-cc7e-efdaff7e0107 Address:08b3d66f-5b31-ca43-cc7e-efdaff7e0107} {Suffrage:Voter ID:server-e5187b5a-8935-4aaf-60cc-c23485693173 Address:e5187b5a-8935-4aaf-60cc-c23485693173} {Suffrage:Voter ID:server-d71379be-c6d9-91e5-a185-d0e1a9feb89d Address:d71379be-c6d9-91e5-a185-d0e1a9feb89d}]" +2025-02-25T23:58:35.330Z [INFO] server-e5187b5a-8935-4aaf-60cc-c23485693173: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-08b3d66f-5b31-ca43-cc7e-efdaff7e0107 Address:08b3d66f-5b31-ca43-cc7e-efdaff7e0107} {Suffrage:Voter ID:server-e5187b5a-8935-4aaf-60cc-c23485693173 Address:e5187b5a-8935-4aaf-60cc-c23485693173} {Suffrage:Voter ID:server-d71379be-c6d9-91e5-a185-d0e1a9feb89d Address:d71379be-c6d9-91e5-a185-d0e1a9feb89d}]" +2025-02-25T23:58:35.330Z [INFO] server-d71379be-c6d9-91e5-a185-d0e1a9feb89d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-08b3d66f-5b31-ca43-cc7e-efdaff7e0107 Address:08b3d66f-5b31-ca43-cc7e-efdaff7e0107} {Suffrage:Voter ID:server-e5187b5a-8935-4aaf-60cc-c23485693173 Address:e5187b5a-8935-4aaf-60cc-c23485693173} {Suffrage:Voter ID:server-d71379be-c6d9-91e5-a185-d0e1a9feb89d Address:d71379be-c6d9-91e5-a185-d0e1a9feb89d}]" +2025-02-25T23:58:35.330Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:58:35.330Z [INFO] server-08b3d66f-5b31-ca43-cc7e-efdaff7e0107: entering follower state: follower="Node at 08b3d66f-5b31-ca43-cc7e-efdaff7e0107 [Follower]" leader-address= leader-id= +2025-02-25T23:58:35.330Z [INFO] server-e5187b5a-8935-4aaf-60cc-c23485693173: entering follower state: follower="Node at e5187b5a-8935-4aaf-60cc-c23485693173 [Follower]" leader-address= leader-id= +2025-02-25T23:58:35.330Z [INFO] server-d71379be-c6d9-91e5-a185-d0e1a9feb89d: entering follower state: follower="Node at d71379be-c6d9-91e5-a185-d0e1a9feb89d [Follower]" leader-address= leader-id= +2025-02-25T23:58:36.025Z [WARN] server-e5187b5a-8935-4aaf-60cc-c23485693173: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:58:36.025Z [INFO] server-e5187b5a-8935-4aaf-60cc-c23485693173: entering candidate state: node="Node at e5187b5a-8935-4aaf-60cc-c23485693173 [Candidate]" term=2 +2025-02-25T23:58:36.025Z [INFO] server-e5187b5a-8935-4aaf-60cc-c23485693173: election won: term=2 tally=2 +2025-02-25T23:58:36.025Z [INFO] server-e5187b5a-8935-4aaf-60cc-c23485693173: entering leader state: leader="Node at e5187b5a-8935-4aaf-60cc-c23485693173 [Leader]" +2025-02-25T23:58:36.026Z [INFO] server-e5187b5a-8935-4aaf-60cc-c23485693173: added peer, starting replication: peer=server-08b3d66f-5b31-ca43-cc7e-efdaff7e0107 +2025-02-25T23:58:36.026Z [INFO] server-e5187b5a-8935-4aaf-60cc-c23485693173: added peer, starting replication: peer=server-d71379be-c6d9-91e5-a185-d0e1a9feb89d +2025-02-25T23:58:36.026Z [INFO] server-e5187b5a-8935-4aaf-60cc-c23485693173: pipelining replication: peer="{Voter server-08b3d66f-5b31-ca43-cc7e-efdaff7e0107 08b3d66f-5b31-ca43-cc7e-efdaff7e0107}" +2025-02-25T23:58:36.026Z [INFO] server-e5187b5a-8935-4aaf-60cc-c23485693173: pipelining replication: peer="{Voter server-d71379be-c6d9-91e5-a185-d0e1a9feb89d d71379be-c6d9-91e5-a185-d0e1a9feb89d}" +2025-02-25T23:58:37.034Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:58:36.026007208 +0000 UTC m=+63.635022615 (1 nodes), 695.375948ms from start of poll, 695.543229ms from cluster start. Timeout at 2025-02-25 23:58:37.031009488 +0000 UTC m=+64.640024895, 1.00500228s after stability +2025-02-25T23:58:37.034Z [INFO] server-e5187b5a-8935-4aaf-60cc-c23485693173: starting snapshot up to: index=12 +2025-02-25T23:58:37.034Z [INFO] creating new snapshot: path=/tmp/raft3526584335/snapshots/2-12-1740527917034.tmp +2025-02-25T23:58:37.034Z [INFO] server-e5187b5a-8935-4aaf-60cc-c23485693173: snapshot complete up to: index=12 +2025-02-25T23:58:37.035Z [INFO] server-38a7a0ad-ce85-a761-6ae2-16c9f48786c3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-38a7a0ad-ce85-a761-6ae2-16c9f48786c3 Address:38a7a0ad-ce85-a761-6ae2-16c9f48786c3} {Suffrage:Voter ID:server-8c4476b9-d092-f285-c79f-f3f46979bdc0 Address:8c4476b9-d092-f285-c79f-f3f46979bdc0} {Suffrage:Voter ID:server-d5ee769a-342b-cff7-6fee-21a574bab308 Address:d5ee769a-342b-cff7-6fee-21a574bab308}]" +2025-02-25T23:58:37.035Z [INFO] server-8c4476b9-d092-f285-c79f-f3f46979bdc0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-38a7a0ad-ce85-a761-6ae2-16c9f48786c3 Address:38a7a0ad-ce85-a761-6ae2-16c9f48786c3} {Suffrage:Voter ID:server-8c4476b9-d092-f285-c79f-f3f46979bdc0 Address:8c4476b9-d092-f285-c79f-f3f46979bdc0} {Suffrage:Voter ID:server-d5ee769a-342b-cff7-6fee-21a574bab308 Address:d5ee769a-342b-cff7-6fee-21a574bab308}]" +2025-02-25T23:58:37.035Z [INFO] server-d5ee769a-342b-cff7-6fee-21a574bab308: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-38a7a0ad-ce85-a761-6ae2-16c9f48786c3 Address:38a7a0ad-ce85-a761-6ae2-16c9f48786c3} {Suffrage:Voter ID:server-8c4476b9-d092-f285-c79f-f3f46979bdc0 Address:8c4476b9-d092-f285-c79f-f3f46979bdc0} {Suffrage:Voter ID:server-d5ee769a-342b-cff7-6fee-21a574bab308 Address:d5ee769a-342b-cff7-6fee-21a574bab308}]" +2025-02-25T23:58:37.035Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:58:37.035Z [INFO] server-38a7a0ad-ce85-a761-6ae2-16c9f48786c3: entering follower state: follower="Node at 38a7a0ad-ce85-a761-6ae2-16c9f48786c3 [Follower]" leader-address= leader-id= +2025-02-25T23:58:37.035Z [INFO] server-8c4476b9-d092-f285-c79f-f3f46979bdc0: entering follower state: follower="Node at 8c4476b9-d092-f285-c79f-f3f46979bdc0 [Follower]" leader-address= leader-id= +2025-02-25T23:58:37.035Z [INFO] server-d5ee769a-342b-cff7-6fee-21a574bab308: entering follower state: follower="Node at d5ee769a-342b-cff7-6fee-21a574bab308 [Follower]" leader-address= leader-id= +2025-02-25T23:58:37.566Z [WARN] server-d5ee769a-342b-cff7-6fee-21a574bab308: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:58:37.566Z [INFO] server-d5ee769a-342b-cff7-6fee-21a574bab308: entering candidate state: node="Node at d5ee769a-342b-cff7-6fee-21a574bab308 [Candidate]" term=2 +2025-02-25T23:58:37.566Z [INFO] server-d5ee769a-342b-cff7-6fee-21a574bab308: election won: term=2 tally=2 +2025-02-25T23:58:37.566Z [INFO] server-d5ee769a-342b-cff7-6fee-21a574bab308: entering leader state: leader="Node at d5ee769a-342b-cff7-6fee-21a574bab308 [Leader]" +2025-02-25T23:58:37.566Z [INFO] server-d5ee769a-342b-cff7-6fee-21a574bab308: added peer, starting replication: peer=server-38a7a0ad-ce85-a761-6ae2-16c9f48786c3 +2025-02-25T23:58:37.566Z [INFO] server-d5ee769a-342b-cff7-6fee-21a574bab308: added peer, starting replication: peer=server-8c4476b9-d092-f285-c79f-f3f46979bdc0 +2025-02-25T23:58:37.566Z [INFO] server-d5ee769a-342b-cff7-6fee-21a574bab308: pipelining replication: peer="{Voter server-38a7a0ad-ce85-a761-6ae2-16c9f48786c3 38a7a0ad-ce85-a761-6ae2-16c9f48786c3}" +2025-02-25T23:58:37.566Z [WARN] server-8c4476b9-d092-f285-c79f-f3f46979bdc0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:58:37.566Z [INFO] server-8c4476b9-d092-f285-c79f-f3f46979bdc0: entering candidate state: node="Node at 8c4476b9-d092-f285-c79f-f3f46979bdc0 [Candidate]" term=2 +2025-02-25T23:58:37.566Z [INFO] server-8c4476b9-d092-f285-c79f-f3f46979bdc0: duplicate requestVote for same term: term=2 +2025-02-25T23:58:37.566Z [INFO] server-8c4476b9-d092-f285-c79f-f3f46979bdc0: entering follower state: follower="Node at 8c4476b9-d092-f285-c79f-f3f46979bdc0 [Follower]" leader-address=d5ee769a-342b-cff7-6fee-21a574bab308 leader-id=server-d5ee769a-342b-cff7-6fee-21a574bab308 +2025-02-25T23:58:37.566Z [INFO] server-d5ee769a-342b-cff7-6fee-21a574bab308: pipelining replication: peer="{Voter server-8c4476b9-d092-f285-c79f-f3f46979bdc0 8c4476b9-d092-f285-c79f-f3f46979bdc0}" +2025-02-25T23:58:37.566Z [WARN] server-38a7a0ad-ce85-a761-6ae2-16c9f48786c3: rejecting vote request since we have a leader: from=8c4476b9-d092-f285-c79f-f3f46979bdc0 leader=d5ee769a-342b-cff7-6fee-21a574bab308 leader-id=server-d5ee769a-342b-cff7-6fee-21a574bab308 +2025-02-25T23:58:37.566Z [WARN] server-d5ee769a-342b-cff7-6fee-21a574bab308: rejecting vote request since we have a leader: from=8c4476b9-d092-f285-c79f-f3f46979bdc0 leader=d5ee769a-342b-cff7-6fee-21a574bab308 leader-id=server-d5ee769a-342b-cff7-6fee-21a574bab308 +2025-02-25T23:58:38.574Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:58:37.566805939 +0000 UTC m=+65.175821346 (1 nodes), 531.096982ms from start of poll, 531.255663ms from cluster start. Timeout at 2025-02-25 23:58:38.571809619 +0000 UTC m=+66.180825026, 1.00500368s after stability +2025-02-25T23:58:38.574Z [INFO] creating new snapshot: path=/tmp/raft3951854045/snapshots/2-113-1740527918574.tmp +2025-02-25T23:58:38.574Z [INFO] server-d5ee769a-342b-cff7-6fee-21a574bab308: copied to local snapshot: bytes=71 +2025-02-25T23:58:38.574Z [INFO] server-d5ee769a-342b-cff7-6fee-21a574bab308: snapshot restore progress: id=2-113-1740527918574 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:38.574Z [INFO] server-d5ee769a-342b-cff7-6fee-21a574bab308: restored user snapshot: index=113 +2025-02-25T23:58:38.574Z [ERROR] server-d5ee769a-342b-cff7-6fee-21a574bab308: failed to get log: index=3 error="log not found" +2025-02-25T23:58:38.574Z [INFO] server-d5ee769a-342b-cff7-6fee-21a574bab308: aborting pipeline replication: peer="{Voter server-38a7a0ad-ce85-a761-6ae2-16c9f48786c3 38a7a0ad-ce85-a761-6ae2-16c9f48786c3}" +2025-02-25T23:58:38.574Z [ERROR] server-d5ee769a-342b-cff7-6fee-21a574bab308: failed to get log: index=3 error="log not found" +2025-02-25T23:58:38.574Z [INFO] server-d5ee769a-342b-cff7-6fee-21a574bab308: aborting pipeline replication: peer="{Voter server-8c4476b9-d092-f285-c79f-f3f46979bdc0 8c4476b9-d092-f285-c79f-f3f46979bdc0}" +2025-02-25T23:58:38.585Z [ERROR] server-d5ee769a-342b-cff7-6fee-21a574bab308: failed to get log: index=3 error="log not found" +2025-02-25T23:58:38.586Z [ERROR] server-d5ee769a-342b-cff7-6fee-21a574bab308: failed to get log: index=3 error="log not found" +2025-02-25T23:58:38.586Z [INFO] creating new snapshot: path=/tmp/raft358520855/snapshots/2-113-1740527918586.tmp +2025-02-25T23:58:38.586Z [INFO] server-8c4476b9-d092-f285-c79f-f3f46979bdc0: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:38.586Z [INFO] server-8c4476b9-d092-f285-c79f-f3f46979bdc0: copied to local snapshot: bytes=71 +2025-02-25T23:58:38.586Z [INFO] server-8c4476b9-d092-f285-c79f-f3f46979bdc0: snapshot restore progress: id=2-113-1740527918586 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:38.586Z [INFO] server-8c4476b9-d092-f285-c79f-f3f46979bdc0: Installed remote snapshot +2025-02-25T23:58:38.586Z [INFO] server-d5ee769a-342b-cff7-6fee-21a574bab308: pipelining replication: peer="{Voter server-8c4476b9-d092-f285-c79f-f3f46979bdc0 8c4476b9-d092-f285-c79f-f3f46979bdc0}" +2025-02-25T23:58:38.586Z [INFO] creating new snapshot: path=/tmp/raft2785256801/snapshots/2-113-1740527918586.tmp +2025-02-25T23:58:38.586Z [INFO] server-38a7a0ad-ce85-a761-6ae2-16c9f48786c3: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:38.586Z [INFO] server-38a7a0ad-ce85-a761-6ae2-16c9f48786c3: copied to local snapshot: bytes=71 +2025-02-25T23:58:38.586Z [INFO] server-38a7a0ad-ce85-a761-6ae2-16c9f48786c3: snapshot restore progress: id=2-113-1740527918586 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:38.587Z [INFO] server-38a7a0ad-ce85-a761-6ae2-16c9f48786c3: Installed remote snapshot +2025-02-25T23:58:38.587Z [INFO] server-d5ee769a-342b-cff7-6fee-21a574bab308: pipelining replication: peer="{Voter server-38a7a0ad-ce85-a761-6ae2-16c9f48786c3 38a7a0ad-ce85-a761-6ae2-16c9f48786c3}" +2025-02-25T23:58:38.587Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:58:39.593Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:58:38.587123195 +0000 UTC m=+66.196138602 (2 nodes), 400ns from start of poll, 1.551572919s from cluster start. Timeout at 2025-02-25 23:58:39.592156035 +0000 UTC m=+67.201171402, 1.0050328s after stability +2025-02-25T23:58:39.614Z [INFO] server-e5187b5a-8935-4aaf-60cc-c23485693173: aborting pipeline replication: peer="{Voter server-d71379be-c6d9-91e5-a185-d0e1a9feb89d d71379be-c6d9-91e5-a185-d0e1a9feb89d}" +2025-02-25T23:58:39.614Z [INFO] server-e5187b5a-8935-4aaf-60cc-c23485693173: aborting pipeline replication: peer="{Voter server-08b3d66f-5b31-ca43-cc7e-efdaff7e0107 08b3d66f-5b31-ca43-cc7e-efdaff7e0107}" === RUN TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_true -2026-03-31T06:17:17.555Z [INFO] server-7b2946a3-9dce-c8ef-e6f8-65a28bdef6c5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b2946a3-9dce-c8ef-e6f8-65a28bdef6c5 Address:7b2946a3-9dce-c8ef-e6f8-65a28bdef6c5} {Suffrage:Voter ID:server-6903bdac-4d6e-0229-4552-b866adb4f266 Address:6903bdac-4d6e-0229-4552-b866adb4f266} {Suffrage:Voter ID:server-d382be12-0329-41de-5a8f-ec900400de82 Address:d382be12-0329-41de-5a8f-ec900400de82}]" -2026-03-31T06:17:17.555Z [INFO] server-6903bdac-4d6e-0229-4552-b866adb4f266: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b2946a3-9dce-c8ef-e6f8-65a28bdef6c5 Address:7b2946a3-9dce-c8ef-e6f8-65a28bdef6c5} {Suffrage:Voter ID:server-6903bdac-4d6e-0229-4552-b866adb4f266 Address:6903bdac-4d6e-0229-4552-b866adb4f266} {Suffrage:Voter ID:server-d382be12-0329-41de-5a8f-ec900400de82 Address:d382be12-0329-41de-5a8f-ec900400de82}]" -2026-03-31T06:17:17.555Z [INFO] server-7b2946a3-9dce-c8ef-e6f8-65a28bdef6c5: entering follower state: follower="Node at 7b2946a3-9dce-c8ef-e6f8-65a28bdef6c5 [Follower]" leader-address= leader-id= -2026-03-31T06:17:17.555Z [INFO] server-6903bdac-4d6e-0229-4552-b866adb4f266: entering follower state: follower="Node at 6903bdac-4d6e-0229-4552-b866adb4f266 [Follower]" leader-address= leader-id= -2026-03-31T06:17:17.555Z [INFO] server-d382be12-0329-41de-5a8f-ec900400de82: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b2946a3-9dce-c8ef-e6f8-65a28bdef6c5 Address:7b2946a3-9dce-c8ef-e6f8-65a28bdef6c5} {Suffrage:Voter ID:server-6903bdac-4d6e-0229-4552-b866adb4f266 Address:6903bdac-4d6e-0229-4552-b866adb4f266} {Suffrage:Voter ID:server-d382be12-0329-41de-5a8f-ec900400de82 Address:d382be12-0329-41de-5a8f-ec900400de82}]" -2026-03-31T06:17:17.555Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:17.555Z [INFO] server-d382be12-0329-41de-5a8f-ec900400de82: entering follower state: follower="Node at d382be12-0329-41de-5a8f-ec900400de82 [Follower]" leader-address= leader-id= -2026-03-31T06:17:18.078Z [WARN] server-7b2946a3-9dce-c8ef-e6f8-65a28bdef6c5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:18.078Z [INFO] server-7b2946a3-9dce-c8ef-e6f8-65a28bdef6c5: entering candidate state: node="Node at 7b2946a3-9dce-c8ef-e6f8-65a28bdef6c5 [Candidate]" term=2 -2026-03-31T06:17:18.078Z [INFO] server-7b2946a3-9dce-c8ef-e6f8-65a28bdef6c5: election won: term=2 tally=2 -2026-03-31T06:17:18.078Z [INFO] server-7b2946a3-9dce-c8ef-e6f8-65a28bdef6c5: entering leader state: leader="Node at 7b2946a3-9dce-c8ef-e6f8-65a28bdef6c5 [Leader]" -2026-03-31T06:17:18.078Z [INFO] server-7b2946a3-9dce-c8ef-e6f8-65a28bdef6c5: added peer, starting replication: peer=server-6903bdac-4d6e-0229-4552-b866adb4f266 -2026-03-31T06:17:18.078Z [INFO] server-7b2946a3-9dce-c8ef-e6f8-65a28bdef6c5: added peer, starting replication: peer=server-d382be12-0329-41de-5a8f-ec900400de82 -2026-03-31T06:17:18.078Z [INFO] server-7b2946a3-9dce-c8ef-e6f8-65a28bdef6c5: pipelining replication: peer="{Voter server-d382be12-0329-41de-5a8f-ec900400de82 d382be12-0329-41de-5a8f-ec900400de82}" -2026-03-31T06:17:18.078Z [INFO] server-7b2946a3-9dce-c8ef-e6f8-65a28bdef6c5: pipelining replication: peer="{Voter server-6903bdac-4d6e-0229-4552-b866adb4f266 6903bdac-4d6e-0229-4552-b866adb4f266}" -2026-03-31T06:17:19.084Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:18.07859478 +0000 UTC m=+66.356202161 (1 nodes), 522.74712ms from start of poll, 523.24292ms from cluster start. Timeout at 2026-03-31 06:17:19.08360922 +0000 UTC m=+67.361216601, 1.00501444s after stability -2026-03-31T06:17:19.085Z [INFO] server-7b2946a3-9dce-c8ef-e6f8-65a28bdef6c5: starting snapshot up to: index=12 -2026-03-31T06:17:19.085Z [INFO] creating new snapshot: path=/tmp/raft250992349/snapshots/2-12-1774937839085.tmp -2026-03-31T06:17:19.085Z [INFO] server-7b2946a3-9dce-c8ef-e6f8-65a28bdef6c5: snapshot complete up to: index=12 -2026-03-31T06:17:19.086Z [INFO] server-1ea97fd9-8802-7f70-8534-9cde4f63a859: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1ea97fd9-8802-7f70-8534-9cde4f63a859 Address:1ea97fd9-8802-7f70-8534-9cde4f63a859} {Suffrage:Voter ID:server-6fdcb464-59b9-f125-72bc-0c50ecd06dfc Address:6fdcb464-59b9-f125-72bc-0c50ecd06dfc} {Suffrage:Voter ID:server-7147a64a-30f0-2739-5437-bd868944a92f Address:7147a64a-30f0-2739-5437-bd868944a92f}]" -2026-03-31T06:17:19.087Z [INFO] server-6fdcb464-59b9-f125-72bc-0c50ecd06dfc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1ea97fd9-8802-7f70-8534-9cde4f63a859 Address:1ea97fd9-8802-7f70-8534-9cde4f63a859} {Suffrage:Voter ID:server-6fdcb464-59b9-f125-72bc-0c50ecd06dfc Address:6fdcb464-59b9-f125-72bc-0c50ecd06dfc} {Suffrage:Voter ID:server-7147a64a-30f0-2739-5437-bd868944a92f Address:7147a64a-30f0-2739-5437-bd868944a92f}]" -2026-03-31T06:17:19.087Z [INFO] server-1ea97fd9-8802-7f70-8534-9cde4f63a859: entering follower state: follower="Node at 1ea97fd9-8802-7f70-8534-9cde4f63a859 [Follower]" leader-address= leader-id= -2026-03-31T06:17:19.087Z [INFO] server-6fdcb464-59b9-f125-72bc-0c50ecd06dfc: entering follower state: follower="Node at 6fdcb464-59b9-f125-72bc-0c50ecd06dfc [Follower]" leader-address= leader-id= -2026-03-31T06:17:19.087Z [INFO] server-7147a64a-30f0-2739-5437-bd868944a92f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1ea97fd9-8802-7f70-8534-9cde4f63a859 Address:1ea97fd9-8802-7f70-8534-9cde4f63a859} {Suffrage:Voter ID:server-6fdcb464-59b9-f125-72bc-0c50ecd06dfc Address:6fdcb464-59b9-f125-72bc-0c50ecd06dfc} {Suffrage:Voter ID:server-7147a64a-30f0-2739-5437-bd868944a92f Address:7147a64a-30f0-2739-5437-bd868944a92f}]" -2026-03-31T06:17:19.087Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:19.087Z [INFO] server-7147a64a-30f0-2739-5437-bd868944a92f: entering follower state: follower="Node at 7147a64a-30f0-2739-5437-bd868944a92f [Follower]" leader-address= leader-id= -2026-03-31T06:17:19.817Z [WARN] server-7147a64a-30f0-2739-5437-bd868944a92f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:19.817Z [INFO] server-7147a64a-30f0-2739-5437-bd868944a92f: entering candidate state: node="Node at 7147a64a-30f0-2739-5437-bd868944a92f [Candidate]" term=2 -2026-03-31T06:17:19.817Z [INFO] server-7147a64a-30f0-2739-5437-bd868944a92f: election won: term=2 tally=2 -2026-03-31T06:17:19.817Z [INFO] server-7147a64a-30f0-2739-5437-bd868944a92f: entering leader state: leader="Node at 7147a64a-30f0-2739-5437-bd868944a92f [Leader]" -2026-03-31T06:17:19.817Z [INFO] server-7147a64a-30f0-2739-5437-bd868944a92f: added peer, starting replication: peer=server-1ea97fd9-8802-7f70-8534-9cde4f63a859 -2026-03-31T06:17:19.817Z [INFO] server-7147a64a-30f0-2739-5437-bd868944a92f: added peer, starting replication: peer=server-6fdcb464-59b9-f125-72bc-0c50ecd06dfc -2026-03-31T06:17:19.817Z [INFO] server-7147a64a-30f0-2739-5437-bd868944a92f: pipelining replication: peer="{Voter server-1ea97fd9-8802-7f70-8534-9cde4f63a859 1ea97fd9-8802-7f70-8534-9cde4f63a859}" -2026-03-31T06:17:19.817Z [INFO] server-7147a64a-30f0-2739-5437-bd868944a92f: pipelining replication: peer="{Voter server-6fdcb464-59b9-f125-72bc-0c50ecd06dfc 6fdcb464-59b9-f125-72bc-0c50ecd06dfc}" -2026-03-31T06:17:20.823Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:19.8175289 +0000 UTC m=+68.095136281 (1 nodes), 730.24484ms from start of poll, 730.68816ms from cluster start. Timeout at 2026-03-31 06:17:20.82253094 +0000 UTC m=+69.100138361, 1.00500208s after stability -2026-03-31T06:17:20.823Z [INFO] creating new snapshot: path=/tmp/raft2637456169/snapshots/2-113-1774937840823.tmp -2026-03-31T06:17:20.824Z [INFO] server-7147a64a-30f0-2739-5437-bd868944a92f: copied to local snapshot: bytes=71 -2026-03-31T06:17:20.824Z [INFO] server-7147a64a-30f0-2739-5437-bd868944a92f: snapshot restore progress: id=2-113-1774937840823 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:20.824Z [INFO] server-7147a64a-30f0-2739-5437-bd868944a92f: removing all old logs from log store -2026-03-31T06:17:20.824Z [INFO] server-7147a64a-30f0-2739-5437-bd868944a92f: compacting logs: from=1 to=2 -2026-03-31T06:17:20.824Z [INFO] server-7147a64a-30f0-2739-5437-bd868944a92f: restored user snapshot: index=113 -2026-03-31T06:17:20.824Z [ERROR] server-7147a64a-30f0-2739-5437-bd868944a92f: failed to get log: index=2 error="log not found" -2026-03-31T06:17:20.824Z [ERROR] server-7147a64a-30f0-2739-5437-bd868944a92f: failed to get log: index=2 error="log not found" -2026-03-31T06:17:20.824Z [INFO] server-7147a64a-30f0-2739-5437-bd868944a92f: aborting pipeline replication: peer="{Voter server-1ea97fd9-8802-7f70-8534-9cde4f63a859 1ea97fd9-8802-7f70-8534-9cde4f63a859}" -2026-03-31T06:17:20.824Z [INFO] server-7147a64a-30f0-2739-5437-bd868944a92f: aborting pipeline replication: peer="{Voter server-6fdcb464-59b9-f125-72bc-0c50ecd06dfc 6fdcb464-59b9-f125-72bc-0c50ecd06dfc}" -2026-03-31T06:17:20.831Z [ERROR] server-7147a64a-30f0-2739-5437-bd868944a92f: failed to get log: index=2 error="log not found" -2026-03-31T06:17:20.832Z [INFO] creating new snapshot: path=/tmp/raft2657706990/snapshots/2-113-1774937840832.tmp -2026-03-31T06:17:20.832Z [INFO] server-1ea97fd9-8802-7f70-8534-9cde4f63a859: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:20.832Z [INFO] server-1ea97fd9-8802-7f70-8534-9cde4f63a859: copied to local snapshot: bytes=71 -2026-03-31T06:17:20.833Z [INFO] server-1ea97fd9-8802-7f70-8534-9cde4f63a859: snapshot restore progress: id=2-113-1774937840832 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:20.833Z [INFO] server-1ea97fd9-8802-7f70-8534-9cde4f63a859: removing all old logs from log store -2026-03-31T06:17:20.833Z [INFO] server-1ea97fd9-8802-7f70-8534-9cde4f63a859: compacting logs: from=1 to=2 -2026-03-31T06:17:20.833Z [INFO] server-1ea97fd9-8802-7f70-8534-9cde4f63a859: Installed remote snapshot -2026-03-31T06:17:20.833Z [INFO] server-7147a64a-30f0-2739-5437-bd868944a92f: pipelining replication: peer="{Voter server-1ea97fd9-8802-7f70-8534-9cde4f63a859 1ea97fd9-8802-7f70-8534-9cde4f63a859}" -2026-03-31T06:17:20.833Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:17:20.834Z [ERROR] server-7147a64a-30f0-2739-5437-bd868944a92f: failed to get log: index=2 error="log not found" -2026-03-31T06:17:20.834Z [INFO] creating new snapshot: path=/tmp/raft220395483/snapshots/2-113-1774937840834.tmp -2026-03-31T06:17:20.834Z [INFO] server-6fdcb464-59b9-f125-72bc-0c50ecd06dfc: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:20.835Z [INFO] server-6fdcb464-59b9-f125-72bc-0c50ecd06dfc: copied to local snapshot: bytes=71 -2026-03-31T06:17:20.835Z [INFO] server-6fdcb464-59b9-f125-72bc-0c50ecd06dfc: snapshot restore progress: id=2-113-1774937840834 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:20.835Z [INFO] server-6fdcb464-59b9-f125-72bc-0c50ecd06dfc: removing all old logs from log store -2026-03-31T06:17:20.835Z [INFO] server-6fdcb464-59b9-f125-72bc-0c50ecd06dfc: compacting logs: from=1 to=2 -2026-03-31T06:17:20.835Z [INFO] server-6fdcb464-59b9-f125-72bc-0c50ecd06dfc: Installed remote snapshot -2026-03-31T06:17:20.835Z [INFO] server-7147a64a-30f0-2739-5437-bd868944a92f: pipelining replication: peer="{Voter server-6fdcb464-59b9-f125-72bc-0c50ecd06dfc 6fdcb464-59b9-f125-72bc-0c50ecd06dfc}" -2026-03-31T06:17:21.838Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:17:20.83340282 +0000 UTC m=+69.111010201 (2 nodes), 880ns from start of poll, 1.74656208s from cluster start. Timeout at 2026-03-31 06:17:21.83843322 +0000 UTC m=+70.116040641, 1.00503044s after stability -2026-03-31T06:17:21.851Z [INFO] server-7b2946a3-9dce-c8ef-e6f8-65a28bdef6c5: aborting pipeline replication: peer="{Voter server-d382be12-0329-41de-5a8f-ec900400de82 d382be12-0329-41de-5a8f-ec900400de82}" -2026-03-31T06:17:21.851Z [INFO] server-7b2946a3-9dce-c8ef-e6f8-65a28bdef6c5: aborting pipeline replication: peer="{Voter server-6903bdac-4d6e-0229-4552-b866adb4f266 6903bdac-4d6e-0229-4552-b866adb4f266}" +2025-02-25T23:58:39.614Z [INFO] server-c1ec2ee6-a69c-65ff-840a-fc5dc9177e78: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c1ec2ee6-a69c-65ff-840a-fc5dc9177e78 Address:c1ec2ee6-a69c-65ff-840a-fc5dc9177e78} {Suffrage:Voter ID:server-320d0415-9050-f052-e2d3-da513565965f Address:320d0415-9050-f052-e2d3-da513565965f} {Suffrage:Voter ID:server-7b831873-ca53-f41a-2119-ed9344552829 Address:7b831873-ca53-f41a-2119-ed9344552829}]" +2025-02-25T23:58:39.614Z [INFO] server-320d0415-9050-f052-e2d3-da513565965f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c1ec2ee6-a69c-65ff-840a-fc5dc9177e78 Address:c1ec2ee6-a69c-65ff-840a-fc5dc9177e78} {Suffrage:Voter ID:server-320d0415-9050-f052-e2d3-da513565965f Address:320d0415-9050-f052-e2d3-da513565965f} {Suffrage:Voter ID:server-7b831873-ca53-f41a-2119-ed9344552829 Address:7b831873-ca53-f41a-2119-ed9344552829}]" +2025-02-25T23:58:39.614Z [INFO] server-7b831873-ca53-f41a-2119-ed9344552829: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c1ec2ee6-a69c-65ff-840a-fc5dc9177e78 Address:c1ec2ee6-a69c-65ff-840a-fc5dc9177e78} {Suffrage:Voter ID:server-320d0415-9050-f052-e2d3-da513565965f Address:320d0415-9050-f052-e2d3-da513565965f} {Suffrage:Voter ID:server-7b831873-ca53-f41a-2119-ed9344552829 Address:7b831873-ca53-f41a-2119-ed9344552829}]" +2025-02-25T23:58:39.614Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:58:39.614Z [INFO] server-c1ec2ee6-a69c-65ff-840a-fc5dc9177e78: entering follower state: follower="Node at c1ec2ee6-a69c-65ff-840a-fc5dc9177e78 [Follower]" leader-address= leader-id= +2025-02-25T23:58:39.615Z [INFO] server-320d0415-9050-f052-e2d3-da513565965f: entering follower state: follower="Node at 320d0415-9050-f052-e2d3-da513565965f [Follower]" leader-address= leader-id= +2025-02-25T23:58:39.615Z [INFO] server-7b831873-ca53-f41a-2119-ed9344552829: entering follower state: follower="Node at 7b831873-ca53-f41a-2119-ed9344552829 [Follower]" leader-address= leader-id= +2025-02-25T23:58:40.254Z [WARN] server-c1ec2ee6-a69c-65ff-840a-fc5dc9177e78: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:58:40.254Z [INFO] server-c1ec2ee6-a69c-65ff-840a-fc5dc9177e78: entering candidate state: node="Node at c1ec2ee6-a69c-65ff-840a-fc5dc9177e78 [Candidate]" term=2 +2025-02-25T23:58:40.254Z [INFO] server-c1ec2ee6-a69c-65ff-840a-fc5dc9177e78: election won: term=2 tally=2 +2025-02-25T23:58:40.254Z [INFO] server-c1ec2ee6-a69c-65ff-840a-fc5dc9177e78: entering leader state: leader="Node at c1ec2ee6-a69c-65ff-840a-fc5dc9177e78 [Leader]" +2025-02-25T23:58:40.254Z [INFO] server-c1ec2ee6-a69c-65ff-840a-fc5dc9177e78: added peer, starting replication: peer=server-320d0415-9050-f052-e2d3-da513565965f +2025-02-25T23:58:40.254Z [INFO] server-c1ec2ee6-a69c-65ff-840a-fc5dc9177e78: added peer, starting replication: peer=server-7b831873-ca53-f41a-2119-ed9344552829 +2025-02-25T23:58:40.254Z [INFO] server-c1ec2ee6-a69c-65ff-840a-fc5dc9177e78: pipelining replication: peer="{Voter server-7b831873-ca53-f41a-2119-ed9344552829 7b831873-ca53-f41a-2119-ed9344552829}" +2025-02-25T23:58:40.254Z [INFO] server-c1ec2ee6-a69c-65ff-840a-fc5dc9177e78: pipelining replication: peer="{Voter server-320d0415-9050-f052-e2d3-da513565965f 320d0415-9050-f052-e2d3-da513565965f}" +2025-02-25T23:58:41.262Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:58:40.254500012 +0000 UTC m=+67.863515419 (1 nodes), 639.542952ms from start of poll, 639.715273ms from cluster start. Timeout at 2025-02-25 23:58:41.259502172 +0000 UTC m=+68.868517579, 1.00500216s after stability +2025-02-25T23:58:41.262Z [INFO] server-c1ec2ee6-a69c-65ff-840a-fc5dc9177e78: starting snapshot up to: index=12 +2025-02-25T23:58:41.262Z [INFO] creating new snapshot: path=/tmp/raft2206063063/snapshots/2-12-1740527921262.tmp +2025-02-25T23:58:41.262Z [INFO] server-c1ec2ee6-a69c-65ff-840a-fc5dc9177e78: snapshot complete up to: index=12 +2025-02-25T23:58:41.263Z [INFO] server-fb7cf3c3-d1ad-4b97-c6f6-3ba9998f4d7e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fb7cf3c3-d1ad-4b97-c6f6-3ba9998f4d7e Address:fb7cf3c3-d1ad-4b97-c6f6-3ba9998f4d7e} {Suffrage:Voter ID:server-1842defb-12c5-b181-815f-7aa3fa315583 Address:1842defb-12c5-b181-815f-7aa3fa315583} {Suffrage:Voter ID:server-2276eb37-0b4d-4254-1a9b-f6572116f335 Address:2276eb37-0b4d-4254-1a9b-f6572116f335}]" +2025-02-25T23:58:41.263Z [INFO] server-1842defb-12c5-b181-815f-7aa3fa315583: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fb7cf3c3-d1ad-4b97-c6f6-3ba9998f4d7e Address:fb7cf3c3-d1ad-4b97-c6f6-3ba9998f4d7e} {Suffrage:Voter ID:server-1842defb-12c5-b181-815f-7aa3fa315583 Address:1842defb-12c5-b181-815f-7aa3fa315583} {Suffrage:Voter ID:server-2276eb37-0b4d-4254-1a9b-f6572116f335 Address:2276eb37-0b4d-4254-1a9b-f6572116f335}]" +2025-02-25T23:58:41.263Z [INFO] server-2276eb37-0b4d-4254-1a9b-f6572116f335: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fb7cf3c3-d1ad-4b97-c6f6-3ba9998f4d7e Address:fb7cf3c3-d1ad-4b97-c6f6-3ba9998f4d7e} {Suffrage:Voter ID:server-1842defb-12c5-b181-815f-7aa3fa315583 Address:1842defb-12c5-b181-815f-7aa3fa315583} {Suffrage:Voter ID:server-2276eb37-0b4d-4254-1a9b-f6572116f335 Address:2276eb37-0b4d-4254-1a9b-f6572116f335}]" +2025-02-25T23:58:41.263Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:58:41.263Z [INFO] server-fb7cf3c3-d1ad-4b97-c6f6-3ba9998f4d7e: entering follower state: follower="Node at fb7cf3c3-d1ad-4b97-c6f6-3ba9998f4d7e [Follower]" leader-address= leader-id= +2025-02-25T23:58:41.263Z [INFO] server-1842defb-12c5-b181-815f-7aa3fa315583: entering follower state: follower="Node at 1842defb-12c5-b181-815f-7aa3fa315583 [Follower]" leader-address= leader-id= +2025-02-25T23:58:41.263Z [INFO] server-2276eb37-0b4d-4254-1a9b-f6572116f335: entering follower state: follower="Node at 2276eb37-0b4d-4254-1a9b-f6572116f335 [Follower]" leader-address= leader-id= +2025-02-25T23:58:41.806Z [WARN] server-1842defb-12c5-b181-815f-7aa3fa315583: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:58:41.806Z [INFO] server-1842defb-12c5-b181-815f-7aa3fa315583: entering candidate state: node="Node at 1842defb-12c5-b181-815f-7aa3fa315583 [Candidate]" term=2 +2025-02-25T23:58:41.806Z [INFO] server-1842defb-12c5-b181-815f-7aa3fa315583: election won: term=2 tally=2 +2025-02-25T23:58:41.806Z [INFO] server-1842defb-12c5-b181-815f-7aa3fa315583: entering leader state: leader="Node at 1842defb-12c5-b181-815f-7aa3fa315583 [Leader]" +2025-02-25T23:58:41.806Z [INFO] server-1842defb-12c5-b181-815f-7aa3fa315583: added peer, starting replication: peer=server-fb7cf3c3-d1ad-4b97-c6f6-3ba9998f4d7e +2025-02-25T23:58:41.806Z [INFO] server-1842defb-12c5-b181-815f-7aa3fa315583: added peer, starting replication: peer=server-2276eb37-0b4d-4254-1a9b-f6572116f335 +2025-02-25T23:58:41.806Z [INFO] server-1842defb-12c5-b181-815f-7aa3fa315583: pipelining replication: peer="{Voter server-2276eb37-0b4d-4254-1a9b-f6572116f335 2276eb37-0b4d-4254-1a9b-f6572116f335}" +2025-02-25T23:58:41.806Z [INFO] server-1842defb-12c5-b181-815f-7aa3fa315583: pipelining replication: peer="{Voter server-fb7cf3c3-d1ad-4b97-c6f6-3ba9998f4d7e fb7cf3c3-d1ad-4b97-c6f6-3ba9998f4d7e}" +2025-02-25T23:58:42.814Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:58:41.806628695 +0000 UTC m=+69.415644102 (1 nodes), 543.044578ms from start of poll, 543.239299ms from cluster start. Timeout at 2025-02-25 23:58:42.811630455 +0000 UTC m=+70.420645862, 1.00500176s after stability +2025-02-25T23:58:42.814Z [INFO] creating new snapshot: path=/tmp/raft319085918/snapshots/2-113-1740527922814.tmp +2025-02-25T23:58:42.814Z [INFO] server-1842defb-12c5-b181-815f-7aa3fa315583: copied to local snapshot: bytes=71 +2025-02-25T23:58:42.814Z [INFO] server-1842defb-12c5-b181-815f-7aa3fa315583: snapshot restore progress: id=2-113-1740527922814 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:42.814Z [INFO] server-1842defb-12c5-b181-815f-7aa3fa315583: removing all old logs from log store +2025-02-25T23:58:42.814Z [INFO] server-1842defb-12c5-b181-815f-7aa3fa315583: compacting logs: from=1 to=2 +2025-02-25T23:58:42.814Z [INFO] server-1842defb-12c5-b181-815f-7aa3fa315583: restored user snapshot: index=113 +2025-02-25T23:58:42.814Z [ERROR] server-1842defb-12c5-b181-815f-7aa3fa315583: failed to get log: index=2 error="log not found" +2025-02-25T23:58:42.814Z [INFO] server-1842defb-12c5-b181-815f-7aa3fa315583: aborting pipeline replication: peer="{Voter server-2276eb37-0b4d-4254-1a9b-f6572116f335 2276eb37-0b4d-4254-1a9b-f6572116f335}" +2025-02-25T23:58:42.814Z [ERROR] server-1842defb-12c5-b181-815f-7aa3fa315583: failed to get log: index=2 error="log not found" +2025-02-25T23:58:42.814Z [INFO] server-1842defb-12c5-b181-815f-7aa3fa315583: aborting pipeline replication: peer="{Voter server-fb7cf3c3-d1ad-4b97-c6f6-3ba9998f4d7e fb7cf3c3-d1ad-4b97-c6f6-3ba9998f4d7e}" +2025-02-25T23:58:42.822Z [ERROR] server-1842defb-12c5-b181-815f-7aa3fa315583: failed to get log: index=2 error="log not found" +2025-02-25T23:58:42.822Z [INFO] creating new snapshot: path=/tmp/raft1153103012/snapshots/2-113-1740527922822.tmp +2025-02-25T23:58:42.822Z [INFO] server-fb7cf3c3-d1ad-4b97-c6f6-3ba9998f4d7e: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:42.822Z [INFO] server-fb7cf3c3-d1ad-4b97-c6f6-3ba9998f4d7e: copied to local snapshot: bytes=71 +2025-02-25T23:58:42.822Z [INFO] server-fb7cf3c3-d1ad-4b97-c6f6-3ba9998f4d7e: snapshot restore progress: id=2-113-1740527922822 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:42.822Z [INFO] server-fb7cf3c3-d1ad-4b97-c6f6-3ba9998f4d7e: removing all old logs from log store +2025-02-25T23:58:42.822Z [INFO] server-fb7cf3c3-d1ad-4b97-c6f6-3ba9998f4d7e: compacting logs: from=1 to=2 +2025-02-25T23:58:42.822Z [INFO] server-fb7cf3c3-d1ad-4b97-c6f6-3ba9998f4d7e: Installed remote snapshot +2025-02-25T23:58:42.822Z [INFO] server-1842defb-12c5-b181-815f-7aa3fa315583: pipelining replication: peer="{Voter server-fb7cf3c3-d1ad-4b97-c6f6-3ba9998f4d7e fb7cf3c3-d1ad-4b97-c6f6-3ba9998f4d7e}" +2025-02-25T23:58:42.822Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:58:42.830Z [ERROR] server-1842defb-12c5-b181-815f-7aa3fa315583: failed to get log: index=2 error="log not found" +2025-02-25T23:58:42.830Z [INFO] creating new snapshot: path=/tmp/raft762204979/snapshots/2-113-1740527922830.tmp +2025-02-25T23:58:42.830Z [INFO] server-2276eb37-0b4d-4254-1a9b-f6572116f335: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:42.830Z [INFO] server-2276eb37-0b4d-4254-1a9b-f6572116f335: copied to local snapshot: bytes=71 +2025-02-25T23:58:42.830Z [INFO] server-2276eb37-0b4d-4254-1a9b-f6572116f335: snapshot restore progress: id=2-113-1740527922830 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:42.830Z [INFO] server-2276eb37-0b4d-4254-1a9b-f6572116f335: removing all old logs from log store +2025-02-25T23:58:42.830Z [INFO] server-2276eb37-0b4d-4254-1a9b-f6572116f335: compacting logs: from=1 to=2 +2025-02-25T23:58:42.830Z [INFO] server-2276eb37-0b4d-4254-1a9b-f6572116f335: Installed remote snapshot +2025-02-25T23:58:42.830Z [INFO] server-1842defb-12c5-b181-815f-7aa3fa315583: pipelining replication: peer="{Voter server-2276eb37-0b4d-4254-1a9b-f6572116f335 2276eb37-0b4d-4254-1a9b-f6572116f335}" +2025-02-25T23:58:43.830Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:58:42.822772685 +0000 UTC m=+70.431788092 (2 nodes), 480ns from start of poll, 1.559383289s from cluster start. Timeout at 2025-02-25 23:58:43.827792725 +0000 UTC m=+71.436808132, 1.00502004s after stability +2025-02-25T23:58:43.850Z [INFO] server-c1ec2ee6-a69c-65ff-840a-fc5dc9177e78: aborting pipeline replication: peer="{Voter server-7b831873-ca53-f41a-2119-ed9344552829 7b831873-ca53-f41a-2119-ed9344552829}" +2025-02-25T23:58:43.850Z [INFO] server-c1ec2ee6-a69c-65ff-840a-fc5dc9177e78: aborting pipeline replication: peer="{Voter server-320d0415-9050-f052-e2d3-da513565965f 320d0415-9050-f052-e2d3-da513565965f}" === RUN TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_false -2026-03-31T06:17:21.852Z [INFO] server-01c982eb-e05b-b25a-14e8-f026c0e6be91: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-01c982eb-e05b-b25a-14e8-f026c0e6be91 Address:01c982eb-e05b-b25a-14e8-f026c0e6be91} {Suffrage:Voter ID:server-04f53e83-a7bb-1e95-46e1-792cd39035b4 Address:04f53e83-a7bb-1e95-46e1-792cd39035b4} {Suffrage:Voter ID:server-6091ddde-ff76-fd65-1c4e-209863160515 Address:6091ddde-ff76-fd65-1c4e-209863160515}]" -2026-03-31T06:17:21.852Z [INFO] server-04f53e83-a7bb-1e95-46e1-792cd39035b4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-01c982eb-e05b-b25a-14e8-f026c0e6be91 Address:01c982eb-e05b-b25a-14e8-f026c0e6be91} {Suffrage:Voter ID:server-04f53e83-a7bb-1e95-46e1-792cd39035b4 Address:04f53e83-a7bb-1e95-46e1-792cd39035b4} {Suffrage:Voter ID:server-6091ddde-ff76-fd65-1c4e-209863160515 Address:6091ddde-ff76-fd65-1c4e-209863160515}]" -2026-03-31T06:17:21.852Z [INFO] server-01c982eb-e05b-b25a-14e8-f026c0e6be91: entering follower state: follower="Node at 01c982eb-e05b-b25a-14e8-f026c0e6be91 [Follower]" leader-address= leader-id= -2026-03-31T06:17:21.852Z [INFO] server-04f53e83-a7bb-1e95-46e1-792cd39035b4: entering follower state: follower="Node at 04f53e83-a7bb-1e95-46e1-792cd39035b4 [Follower]" leader-address= leader-id= -2026-03-31T06:17:21.852Z [INFO] server-6091ddde-ff76-fd65-1c4e-209863160515: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-01c982eb-e05b-b25a-14e8-f026c0e6be91 Address:01c982eb-e05b-b25a-14e8-f026c0e6be91} {Suffrage:Voter ID:server-04f53e83-a7bb-1e95-46e1-792cd39035b4 Address:04f53e83-a7bb-1e95-46e1-792cd39035b4} {Suffrage:Voter ID:server-6091ddde-ff76-fd65-1c4e-209863160515 Address:6091ddde-ff76-fd65-1c4e-209863160515}]" -2026-03-31T06:17:21.852Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:21.852Z [INFO] server-6091ddde-ff76-fd65-1c4e-209863160515: entering follower state: follower="Node at 6091ddde-ff76-fd65-1c4e-209863160515 [Follower]" leader-address= leader-id= -2026-03-31T06:17:22.533Z [WARN] server-04f53e83-a7bb-1e95-46e1-792cd39035b4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:22.533Z [INFO] server-04f53e83-a7bb-1e95-46e1-792cd39035b4: entering candidate state: node="Node at 04f53e83-a7bb-1e95-46e1-792cd39035b4 [Candidate]" term=2 -2026-03-31T06:17:22.534Z [INFO] server-04f53e83-a7bb-1e95-46e1-792cd39035b4: election won: term=2 tally=2 -2026-03-31T06:17:22.534Z [INFO] server-04f53e83-a7bb-1e95-46e1-792cd39035b4: entering leader state: leader="Node at 04f53e83-a7bb-1e95-46e1-792cd39035b4 [Leader]" -2026-03-31T06:17:22.534Z [INFO] server-04f53e83-a7bb-1e95-46e1-792cd39035b4: added peer, starting replication: peer=server-01c982eb-e05b-b25a-14e8-f026c0e6be91 -2026-03-31T06:17:22.534Z [INFO] server-04f53e83-a7bb-1e95-46e1-792cd39035b4: added peer, starting replication: peer=server-6091ddde-ff76-fd65-1c4e-209863160515 -2026-03-31T06:17:22.534Z [INFO] server-04f53e83-a7bb-1e95-46e1-792cd39035b4: pipelining replication: peer="{Voter server-6091ddde-ff76-fd65-1c4e-209863160515 6091ddde-ff76-fd65-1c4e-209863160515}" -2026-03-31T06:17:22.534Z [INFO] server-04f53e83-a7bb-1e95-46e1-792cd39035b4: pipelining replication: peer="{Voter server-01c982eb-e05b-b25a-14e8-f026c0e6be91 01c982eb-e05b-b25a-14e8-f026c0e6be91}" -2026-03-31T06:17:23.539Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:22.53416798 +0000 UTC m=+70.811775361 (1 nodes), 681.207ms from start of poll, 681.63516ms from cluster start. Timeout at 2026-03-31 06:17:23.53917134 +0000 UTC m=+71.816778721, 1.00500336s after stability -2026-03-31T06:17:23.540Z [INFO] server-04f53e83-a7bb-1e95-46e1-792cd39035b4: starting snapshot up to: index=12 -2026-03-31T06:17:23.540Z [INFO] creating new snapshot: path=/tmp/raft3474682345/snapshots/2-12-1774937843540.tmp -2026-03-31T06:17:23.540Z [INFO] server-04f53e83-a7bb-1e95-46e1-792cd39035b4: snapshot complete up to: index=12 -2026-03-31T06:17:23.541Z [INFO] creating new snapshot: path=/tmp/raft3474682345/snapshots/2-1013-1774937843541.tmp -2026-03-31T06:17:23.542Z [INFO] server-04f53e83-a7bb-1e95-46e1-792cd39035b4: copied to local snapshot: bytes=71 -2026-03-31T06:17:23.542Z [INFO] server-04f53e83-a7bb-1e95-46e1-792cd39035b4: snapshot restore progress: id=2-1013-1774937843541 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:23.542Z [INFO] server-04f53e83-a7bb-1e95-46e1-792cd39035b4: restored user snapshot: index=1013 -2026-03-31T06:17:23.542Z [ERROR] server-04f53e83-a7bb-1e95-46e1-792cd39035b4: failed to get log: index=23 error="log not found" -2026-03-31T06:17:23.542Z [INFO] server-04f53e83-a7bb-1e95-46e1-792cd39035b4: aborting pipeline replication: peer="{Voter server-6091ddde-ff76-fd65-1c4e-209863160515 6091ddde-ff76-fd65-1c4e-209863160515}" -2026-03-31T06:17:23.542Z [ERROR] server-04f53e83-a7bb-1e95-46e1-792cd39035b4: failed to get log: index=23 error="log not found" -2026-03-31T06:17:23.542Z [INFO] server-04f53e83-a7bb-1e95-46e1-792cd39035b4: aborting pipeline replication: peer="{Voter server-01c982eb-e05b-b25a-14e8-f026c0e6be91 01c982eb-e05b-b25a-14e8-f026c0e6be91}" -2026-03-31T06:17:23.549Z [ERROR] server-04f53e83-a7bb-1e95-46e1-792cd39035b4: failed to get log: index=23 error="log not found" -2026-03-31T06:17:23.549Z [ERROR] server-04f53e83-a7bb-1e95-46e1-792cd39035b4: failed to get log: index=23 error="log not found" -2026-03-31T06:17:23.550Z [INFO] creating new snapshot: path=/tmp/raft242502708/snapshots/2-1013-1774937843550.tmp -2026-03-31T06:17:23.550Z [INFO] creating new snapshot: path=/tmp/raft4168585622/snapshots/2-1013-1774937843550.tmp -2026-03-31T06:17:23.550Z [INFO] server-6091ddde-ff76-fd65-1c4e-209863160515: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:23.550Z [INFO] server-01c982eb-e05b-b25a-14e8-f026c0e6be91: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:23.550Z [INFO] server-6091ddde-ff76-fd65-1c4e-209863160515: copied to local snapshot: bytes=71 -2026-03-31T06:17:23.551Z [INFO] server-6091ddde-ff76-fd65-1c4e-209863160515: snapshot restore progress: id=2-1013-1774937843550 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:23.551Z [INFO] server-6091ddde-ff76-fd65-1c4e-209863160515: Installed remote snapshot -2026-03-31T06:17:23.551Z [WARN] server-6091ddde-ff76-fd65-1c4e-209863160515: skipping application of old log: index=22 -2026-03-31T06:17:23.551Z [INFO] server-01c982eb-e05b-b25a-14e8-f026c0e6be91: copied to local snapshot: bytes=71 -2026-03-31T06:17:23.552Z [INFO] server-04f53e83-a7bb-1e95-46e1-792cd39035b4: pipelining replication: peer="{Voter server-6091ddde-ff76-fd65-1c4e-209863160515 6091ddde-ff76-fd65-1c4e-209863160515}" -2026-03-31T06:17:23.552Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:17:23.552Z [INFO] server-01c982eb-e05b-b25a-14e8-f026c0e6be91: snapshot restore progress: id=2-1013-1774937843550 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:23.552Z [INFO] server-01c982eb-e05b-b25a-14e8-f026c0e6be91: Installed remote snapshot -2026-03-31T06:17:23.552Z [INFO] server-04f53e83-a7bb-1e95-46e1-792cd39035b4: pipelining replication: peer="{Voter server-01c982eb-e05b-b25a-14e8-f026c0e6be91 01c982eb-e05b-b25a-14e8-f026c0e6be91}" -2026-03-31T06:17:24.558Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:17:23.55245846 +0000 UTC m=+71.830065841 (2 nodes), 1.64µs from start of poll, 1.69992564s from cluster start. Timeout at 2026-03-31 06:17:24.5574745 +0000 UTC m=+72.835081881, 1.00501604s after stability -2026-03-31T06:17:24.569Z [INFO] server-04f53e83-a7bb-1e95-46e1-792cd39035b4: aborting pipeline replication: peer="{Voter server-6091ddde-ff76-fd65-1c4e-209863160515 6091ddde-ff76-fd65-1c4e-209863160515}" -2026-03-31T06:17:24.569Z [INFO] server-04f53e83-a7bb-1e95-46e1-792cd39035b4: aborting pipeline replication: peer="{Voter server-01c982eb-e05b-b25a-14e8-f026c0e6be91 01c982eb-e05b-b25a-14e8-f026c0e6be91}" +2025-02-25T23:58:43.850Z [INFO] server-8df0b924-6cbe-bbd4-70d5-2a8e642e963e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8df0b924-6cbe-bbd4-70d5-2a8e642e963e Address:8df0b924-6cbe-bbd4-70d5-2a8e642e963e} {Suffrage:Voter ID:server-cdae21b7-4c3e-ee1d-79ab-57ef32f9f6fb Address:cdae21b7-4c3e-ee1d-79ab-57ef32f9f6fb} {Suffrage:Voter ID:server-bc496d4a-77ca-83a8-30b1-bdcc97934a03 Address:bc496d4a-77ca-83a8-30b1-bdcc97934a03}]" +2025-02-25T23:58:43.850Z [INFO] server-cdae21b7-4c3e-ee1d-79ab-57ef32f9f6fb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8df0b924-6cbe-bbd4-70d5-2a8e642e963e Address:8df0b924-6cbe-bbd4-70d5-2a8e642e963e} {Suffrage:Voter ID:server-cdae21b7-4c3e-ee1d-79ab-57ef32f9f6fb Address:cdae21b7-4c3e-ee1d-79ab-57ef32f9f6fb} {Suffrage:Voter ID:server-bc496d4a-77ca-83a8-30b1-bdcc97934a03 Address:bc496d4a-77ca-83a8-30b1-bdcc97934a03}]" +2025-02-25T23:58:43.850Z [INFO] server-bc496d4a-77ca-83a8-30b1-bdcc97934a03: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8df0b924-6cbe-bbd4-70d5-2a8e642e963e Address:8df0b924-6cbe-bbd4-70d5-2a8e642e963e} {Suffrage:Voter ID:server-cdae21b7-4c3e-ee1d-79ab-57ef32f9f6fb Address:cdae21b7-4c3e-ee1d-79ab-57ef32f9f6fb} {Suffrage:Voter ID:server-bc496d4a-77ca-83a8-30b1-bdcc97934a03 Address:bc496d4a-77ca-83a8-30b1-bdcc97934a03}]" +2025-02-25T23:58:43.850Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:58:43.850Z [INFO] server-8df0b924-6cbe-bbd4-70d5-2a8e642e963e: entering follower state: follower="Node at 8df0b924-6cbe-bbd4-70d5-2a8e642e963e [Follower]" leader-address= leader-id= +2025-02-25T23:58:43.851Z [INFO] server-cdae21b7-4c3e-ee1d-79ab-57ef32f9f6fb: entering follower state: follower="Node at cdae21b7-4c3e-ee1d-79ab-57ef32f9f6fb [Follower]" leader-address= leader-id= +2025-02-25T23:58:43.851Z [INFO] server-bc496d4a-77ca-83a8-30b1-bdcc97934a03: entering follower state: follower="Node at bc496d4a-77ca-83a8-30b1-bdcc97934a03 [Follower]" leader-address= leader-id= +2025-02-25T23:58:44.633Z [WARN] server-bc496d4a-77ca-83a8-30b1-bdcc97934a03: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:58:44.633Z [INFO] server-bc496d4a-77ca-83a8-30b1-bdcc97934a03: entering candidate state: node="Node at bc496d4a-77ca-83a8-30b1-bdcc97934a03 [Candidate]" term=2 +2025-02-25T23:58:44.634Z [WARN] server-cdae21b7-4c3e-ee1d-79ab-57ef32f9f6fb: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:58:44.634Z [INFO] server-cdae21b7-4c3e-ee1d-79ab-57ef32f9f6fb: entering candidate state: node="Node at cdae21b7-4c3e-ee1d-79ab-57ef32f9f6fb [Candidate]" term=2 +2025-02-25T23:58:44.634Z [INFO] server-cdae21b7-4c3e-ee1d-79ab-57ef32f9f6fb: duplicate requestVote for same term: term=2 +2025-02-25T23:58:44.634Z [INFO] server-bc496d4a-77ca-83a8-30b1-bdcc97934a03: election won: term=2 tally=2 +2025-02-25T23:58:44.634Z [INFO] server-bc496d4a-77ca-83a8-30b1-bdcc97934a03: entering leader state: leader="Node at bc496d4a-77ca-83a8-30b1-bdcc97934a03 [Leader]" +2025-02-25T23:58:44.634Z [INFO] server-bc496d4a-77ca-83a8-30b1-bdcc97934a03: added peer, starting replication: peer=server-8df0b924-6cbe-bbd4-70d5-2a8e642e963e +2025-02-25T23:58:44.634Z [INFO] server-bc496d4a-77ca-83a8-30b1-bdcc97934a03: added peer, starting replication: peer=server-cdae21b7-4c3e-ee1d-79ab-57ef32f9f6fb +2025-02-25T23:58:44.634Z [INFO] server-cdae21b7-4c3e-ee1d-79ab-57ef32f9f6fb: entering follower state: follower="Node at cdae21b7-4c3e-ee1d-79ab-57ef32f9f6fb [Follower]" leader-address=bc496d4a-77ca-83a8-30b1-bdcc97934a03 leader-id=server-bc496d4a-77ca-83a8-30b1-bdcc97934a03 +2025-02-25T23:58:44.634Z [INFO] server-bc496d4a-77ca-83a8-30b1-bdcc97934a03: pipelining replication: peer="{Voter server-cdae21b7-4c3e-ee1d-79ab-57ef32f9f6fb cdae21b7-4c3e-ee1d-79ab-57ef32f9f6fb}" +2025-02-25T23:58:44.634Z [INFO] server-8df0b924-6cbe-bbd4-70d5-2a8e642e963e: duplicate requestVote for same term: term=2 +2025-02-25T23:58:44.634Z [INFO] server-bc496d4a-77ca-83a8-30b1-bdcc97934a03: pipelining replication: peer="{Voter server-8df0b924-6cbe-bbd4-70d5-2a8e642e963e 8df0b924-6cbe-bbd4-70d5-2a8e642e963e}" +2025-02-25T23:58:44.634Z [WARN] server-bc496d4a-77ca-83a8-30b1-bdcc97934a03: rejecting vote request since we have a leader: from=cdae21b7-4c3e-ee1d-79ab-57ef32f9f6fb leader=bc496d4a-77ca-83a8-30b1-bdcc97934a03 leader-id=server-bc496d4a-77ca-83a8-30b1-bdcc97934a03 +2025-02-25T23:58:45.642Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:58:44.63442698 +0000 UTC m=+72.243442387 (1 nodes), 783.465748ms from start of poll, 783.634669ms from cluster start. Timeout at 2025-02-25 23:58:45.63942942 +0000 UTC m=+73.248444827, 1.00500244s after stability +2025-02-25T23:58:45.645Z [INFO] server-bc496d4a-77ca-83a8-30b1-bdcc97934a03: starting snapshot up to: index=12 +2025-02-25T23:58:45.645Z [INFO] creating new snapshot: path=/tmp/raft2711524892/snapshots/2-12-1740527925645.tmp +2025-02-25T23:58:45.645Z [INFO] server-bc496d4a-77ca-83a8-30b1-bdcc97934a03: snapshot complete up to: index=12 +2025-02-25T23:58:45.650Z [INFO] creating new snapshot: path=/tmp/raft2711524892/snapshots/2-1013-1740527925650.tmp +2025-02-25T23:58:45.650Z [INFO] server-bc496d4a-77ca-83a8-30b1-bdcc97934a03: copied to local snapshot: bytes=71 +2025-02-25T23:58:45.650Z [INFO] server-bc496d4a-77ca-83a8-30b1-bdcc97934a03: snapshot restore progress: id=2-1013-1740527925650 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:45.650Z [INFO] server-bc496d4a-77ca-83a8-30b1-bdcc97934a03: restored user snapshot: index=1013 +2025-02-25T23:58:45.650Z [ERROR] server-bc496d4a-77ca-83a8-30b1-bdcc97934a03: failed to get log: index=23 error="log not found" +2025-02-25T23:58:45.650Z [ERROR] server-bc496d4a-77ca-83a8-30b1-bdcc97934a03: failed to get log: index=23 error="log not found" +2025-02-25T23:58:45.650Z [INFO] server-bc496d4a-77ca-83a8-30b1-bdcc97934a03: aborting pipeline replication: peer="{Voter server-cdae21b7-4c3e-ee1d-79ab-57ef32f9f6fb cdae21b7-4c3e-ee1d-79ab-57ef32f9f6fb}" +2025-02-25T23:58:45.650Z [INFO] server-bc496d4a-77ca-83a8-30b1-bdcc97934a03: aborting pipeline replication: peer="{Voter server-8df0b924-6cbe-bbd4-70d5-2a8e642e963e 8df0b924-6cbe-bbd4-70d5-2a8e642e963e}" +2025-02-25T23:58:45.661Z [ERROR] server-bc496d4a-77ca-83a8-30b1-bdcc97934a03: failed to get log: index=23 error="log not found" +2025-02-25T23:58:45.662Z [INFO] creating new snapshot: path=/tmp/raft4238841917/snapshots/2-1013-1740527925662.tmp +2025-02-25T23:58:45.662Z [INFO] server-cdae21b7-4c3e-ee1d-79ab-57ef32f9f6fb: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:45.662Z [INFO] server-cdae21b7-4c3e-ee1d-79ab-57ef32f9f6fb: copied to local snapshot: bytes=71 +2025-02-25T23:58:45.662Z [INFO] server-cdae21b7-4c3e-ee1d-79ab-57ef32f9f6fb: snapshot restore progress: id=2-1013-1740527925662 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:45.662Z [INFO] server-cdae21b7-4c3e-ee1d-79ab-57ef32f9f6fb: Installed remote snapshot +2025-02-25T23:58:45.662Z [WARN] server-cdae21b7-4c3e-ee1d-79ab-57ef32f9f6fb: skipping application of old log: index=22 +2025-02-25T23:58:45.662Z [INFO] server-bc496d4a-77ca-83a8-30b1-bdcc97934a03: pipelining replication: peer="{Voter server-cdae21b7-4c3e-ee1d-79ab-57ef32f9f6fb cdae21b7-4c3e-ee1d-79ab-57ef32f9f6fb}" +2025-02-25T23:58:45.662Z [ERROR] server-bc496d4a-77ca-83a8-30b1-bdcc97934a03: failed to get log: index=23 error="log not found" +2025-02-25T23:58:45.662Z [INFO] creating new snapshot: path=/tmp/raft3466979559/snapshots/2-1013-1740527925662.tmp +2025-02-25T23:58:45.662Z [INFO] server-8df0b924-6cbe-bbd4-70d5-2a8e642e963e: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:45.663Z [INFO] server-8df0b924-6cbe-bbd4-70d5-2a8e642e963e: copied to local snapshot: bytes=71 +2025-02-25T23:58:45.663Z [INFO] server-8df0b924-6cbe-bbd4-70d5-2a8e642e963e: snapshot restore progress: id=2-1013-1740527925662 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:45.663Z [INFO] server-8df0b924-6cbe-bbd4-70d5-2a8e642e963e: Installed remote snapshot +2025-02-25T23:58:45.663Z [WARN] server-8df0b924-6cbe-bbd4-70d5-2a8e642e963e: skipping application of old log: index=22 +2025-02-25T23:58:45.663Z [INFO] server-bc496d4a-77ca-83a8-30b1-bdcc97934a03: pipelining replication: peer="{Voter server-8df0b924-6cbe-bbd4-70d5-2a8e642e963e 8df0b924-6cbe-bbd4-70d5-2a8e642e963e}" +2025-02-25T23:58:45.663Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:58:46.670Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:58:45.663356452 +0000 UTC m=+73.272371859 (2 nodes), 440ns from start of poll, 1.812564141s from cluster start. Timeout at 2025-02-25 23:58:46.668362532 +0000 UTC m=+74.277377939, 1.00500608s after stability +2025-02-25T23:58:46.686Z [INFO] server-bc496d4a-77ca-83a8-30b1-bdcc97934a03: aborting pipeline replication: peer="{Voter server-cdae21b7-4c3e-ee1d-79ab-57ef32f9f6fb cdae21b7-4c3e-ee1d-79ab-57ef32f9f6fb}" +2025-02-25T23:58:46.686Z [INFO] server-bc496d4a-77ca-83a8-30b1-bdcc97934a03: aborting pipeline replication: peer="{Voter server-8df0b924-6cbe-bbd4-70d5-2a8e642e963e 8df0b924-6cbe-bbd4-70d5-2a8e642e963e}" === RUN TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_false -2026-03-31T06:17:24.571Z [INFO] server-32f71c9e-2a0e-5da4-829b-7c61ce435bf6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-32f71c9e-2a0e-5da4-829b-7c61ce435bf6 Address:32f71c9e-2a0e-5da4-829b-7c61ce435bf6} {Suffrage:Voter ID:server-48f913ad-39bc-88e0-7156-8f530c0a31d0 Address:48f913ad-39bc-88e0-7156-8f530c0a31d0} {Suffrage:Voter ID:server-0dd973e7-df15-9326-3d5e-f0151bedc105 Address:0dd973e7-df15-9326-3d5e-f0151bedc105}]" -2026-03-31T06:17:24.571Z [INFO] server-48f913ad-39bc-88e0-7156-8f530c0a31d0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-32f71c9e-2a0e-5da4-829b-7c61ce435bf6 Address:32f71c9e-2a0e-5da4-829b-7c61ce435bf6} {Suffrage:Voter ID:server-48f913ad-39bc-88e0-7156-8f530c0a31d0 Address:48f913ad-39bc-88e0-7156-8f530c0a31d0} {Suffrage:Voter ID:server-0dd973e7-df15-9326-3d5e-f0151bedc105 Address:0dd973e7-df15-9326-3d5e-f0151bedc105}]" -2026-03-31T06:17:24.571Z [INFO] server-32f71c9e-2a0e-5da4-829b-7c61ce435bf6: entering follower state: follower="Node at 32f71c9e-2a0e-5da4-829b-7c61ce435bf6 [Follower]" leader-address= leader-id= -2026-03-31T06:17:24.571Z [INFO] server-0dd973e7-df15-9326-3d5e-f0151bedc105: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-32f71c9e-2a0e-5da4-829b-7c61ce435bf6 Address:32f71c9e-2a0e-5da4-829b-7c61ce435bf6} {Suffrage:Voter ID:server-48f913ad-39bc-88e0-7156-8f530c0a31d0 Address:48f913ad-39bc-88e0-7156-8f530c0a31d0} {Suffrage:Voter ID:server-0dd973e7-df15-9326-3d5e-f0151bedc105 Address:0dd973e7-df15-9326-3d5e-f0151bedc105}]" -2026-03-31T06:17:24.571Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:24.571Z [INFO] server-48f913ad-39bc-88e0-7156-8f530c0a31d0: entering follower state: follower="Node at 48f913ad-39bc-88e0-7156-8f530c0a31d0 [Follower]" leader-address= leader-id= -2026-03-31T06:17:24.571Z [INFO] server-0dd973e7-df15-9326-3d5e-f0151bedc105: entering follower state: follower="Node at 0dd973e7-df15-9326-3d5e-f0151bedc105 [Follower]" leader-address= leader-id= -2026-03-31T06:17:25.214Z [WARN] server-0dd973e7-df15-9326-3d5e-f0151bedc105: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:25.214Z [INFO] server-0dd973e7-df15-9326-3d5e-f0151bedc105: entering candidate state: node="Node at 0dd973e7-df15-9326-3d5e-f0151bedc105 [Candidate]" term=2 -2026-03-31T06:17:25.214Z [INFO] server-0dd973e7-df15-9326-3d5e-f0151bedc105: election won: term=2 tally=2 -2026-03-31T06:17:25.214Z [INFO] server-0dd973e7-df15-9326-3d5e-f0151bedc105: entering leader state: leader="Node at 0dd973e7-df15-9326-3d5e-f0151bedc105 [Leader]" -2026-03-31T06:17:25.214Z [INFO] server-0dd973e7-df15-9326-3d5e-f0151bedc105: added peer, starting replication: peer=server-32f71c9e-2a0e-5da4-829b-7c61ce435bf6 -2026-03-31T06:17:25.214Z [INFO] server-0dd973e7-df15-9326-3d5e-f0151bedc105: added peer, starting replication: peer=server-48f913ad-39bc-88e0-7156-8f530c0a31d0 -2026-03-31T06:17:25.214Z [INFO] server-0dd973e7-df15-9326-3d5e-f0151bedc105: pipelining replication: peer="{Voter server-48f913ad-39bc-88e0-7156-8f530c0a31d0 48f913ad-39bc-88e0-7156-8f530c0a31d0}" -2026-03-31T06:17:25.214Z [INFO] server-0dd973e7-df15-9326-3d5e-f0151bedc105: pipelining replication: peer="{Voter server-32f71c9e-2a0e-5da4-829b-7c61ce435bf6 32f71c9e-2a0e-5da4-829b-7c61ce435bf6}" -2026-03-31T06:17:26.220Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:25.21468022 +0000 UTC m=+73.492287601 (1 nodes), 643.33576ms from start of poll, 643.756ms from cluster start. Timeout at 2026-03-31 06:17:26.21968406 +0000 UTC m=+74.497291441, 1.00500384s after stability -2026-03-31T06:17:26.221Z [INFO] server-0dd973e7-df15-9326-3d5e-f0151bedc105: starting snapshot up to: index=12 -2026-03-31T06:17:26.221Z [INFO] creating new snapshot: path=/tmp/raft1012957094/snapshots/2-12-1774937846221.tmp -2026-03-31T06:17:26.221Z [INFO] server-0dd973e7-df15-9326-3d5e-f0151bedc105: snapshot complete up to: index=12 -2026-03-31T06:17:26.224Z [INFO] creating new snapshot: path=/tmp/raft1012957094/snapshots/2-1013-1774937846224.tmp -2026-03-31T06:17:26.225Z [INFO] server-0dd973e7-df15-9326-3d5e-f0151bedc105: copied to local snapshot: bytes=71 -2026-03-31T06:17:26.225Z [INFO] server-0dd973e7-df15-9326-3d5e-f0151bedc105: snapshot restore progress: id=2-1013-1774937846224 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:26.225Z [INFO] server-0dd973e7-df15-9326-3d5e-f0151bedc105: removing all old logs from log store -2026-03-31T06:17:26.225Z [INFO] server-0dd973e7-df15-9326-3d5e-f0151bedc105: compacting logs: from=1 to=22 -2026-03-31T06:17:26.225Z [INFO] server-0dd973e7-df15-9326-3d5e-f0151bedc105: restored user snapshot: index=1013 -2026-03-31T06:17:26.225Z [ERROR] server-0dd973e7-df15-9326-3d5e-f0151bedc105: failed to get log: index=22 error="log not found" -2026-03-31T06:17:26.225Z [INFO] server-0dd973e7-df15-9326-3d5e-f0151bedc105: aborting pipeline replication: peer="{Voter server-32f71c9e-2a0e-5da4-829b-7c61ce435bf6 32f71c9e-2a0e-5da4-829b-7c61ce435bf6}" -2026-03-31T06:17:26.225Z [ERROR] server-0dd973e7-df15-9326-3d5e-f0151bedc105: failed to get log: index=22 error="log not found" -2026-03-31T06:17:26.225Z [INFO] server-0dd973e7-df15-9326-3d5e-f0151bedc105: aborting pipeline replication: peer="{Voter server-48f913ad-39bc-88e0-7156-8f530c0a31d0 48f913ad-39bc-88e0-7156-8f530c0a31d0}" -2026-03-31T06:17:26.234Z [ERROR] server-0dd973e7-df15-9326-3d5e-f0151bedc105: failed to get log: index=22 error="log not found" -2026-03-31T06:17:26.234Z [ERROR] server-0dd973e7-df15-9326-3d5e-f0151bedc105: failed to get log: index=22 error="log not found" -2026-03-31T06:17:26.235Z [INFO] creating new snapshot: path=/tmp/raft857983343/snapshots/2-1013-1774937846235.tmp -2026-03-31T06:17:26.235Z [INFO] server-32f71c9e-2a0e-5da4-829b-7c61ce435bf6: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:26.235Z [INFO] creating new snapshot: path=/tmp/raft587305711/snapshots/2-1013-1774937846235.tmp -2026-03-31T06:17:26.235Z [INFO] server-48f913ad-39bc-88e0-7156-8f530c0a31d0: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:26.235Z [INFO] server-32f71c9e-2a0e-5da4-829b-7c61ce435bf6: copied to local snapshot: bytes=71 -2026-03-31T06:17:26.235Z [INFO] server-48f913ad-39bc-88e0-7156-8f530c0a31d0: copied to local snapshot: bytes=71 -2026-03-31T06:17:26.236Z [INFO] server-32f71c9e-2a0e-5da4-829b-7c61ce435bf6: snapshot restore progress: id=2-1013-1774937846235 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:26.236Z [INFO] server-32f71c9e-2a0e-5da4-829b-7c61ce435bf6: removing all old logs from log store -2026-03-31T06:17:26.236Z [INFO] server-32f71c9e-2a0e-5da4-829b-7c61ce435bf6: compacting logs: from=1 to=22 -2026-03-31T06:17:26.236Z [INFO] server-32f71c9e-2a0e-5da4-829b-7c61ce435bf6: Installed remote snapshot -2026-03-31T06:17:26.236Z [INFO] server-48f913ad-39bc-88e0-7156-8f530c0a31d0: snapshot restore progress: id=2-1013-1774937846235 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:26.236Z [INFO] server-48f913ad-39bc-88e0-7156-8f530c0a31d0: removing all old logs from log store -2026-03-31T06:17:26.236Z [INFO] server-48f913ad-39bc-88e0-7156-8f530c0a31d0: compacting logs: from=1 to=22 -2026-03-31T06:17:26.236Z [INFO] server-48f913ad-39bc-88e0-7156-8f530c0a31d0: Installed remote snapshot -2026-03-31T06:17:26.236Z [WARN] server-32f71c9e-2a0e-5da4-829b-7c61ce435bf6: skipping application of old log: index=22 -2026-03-31T06:17:26.236Z [INFO] server-0dd973e7-df15-9326-3d5e-f0151bedc105: pipelining replication: peer="{Voter server-32f71c9e-2a0e-5da4-829b-7c61ce435bf6 32f71c9e-2a0e-5da4-829b-7c61ce435bf6}" -2026-03-31T06:17:26.236Z [WARN] server-48f913ad-39bc-88e0-7156-8f530c0a31d0: skipping application of old log: index=1013 -2026-03-31T06:17:26.236Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:17:26.236Z [INFO] server-0dd973e7-df15-9326-3d5e-f0151bedc105: pipelining replication: peer="{Voter server-48f913ad-39bc-88e0-7156-8f530c0a31d0 48f913ad-39bc-88e0-7156-8f530c0a31d0}" -2026-03-31T06:17:27.242Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:17:26.23648926 +0000 UTC m=+74.514096641 (2 nodes), 880ns from start of poll, 1.66556504s from cluster start. Timeout at 2026-03-31 06:17:27.24150802 +0000 UTC m=+75.519115401, 1.00501876s after stability -2026-03-31T06:17:27.254Z [INFO] server-0dd973e7-df15-9326-3d5e-f0151bedc105: aborting pipeline replication: peer="{Voter server-48f913ad-39bc-88e0-7156-8f530c0a31d0 48f913ad-39bc-88e0-7156-8f530c0a31d0}" -2026-03-31T06:17:27.254Z [INFO] server-0dd973e7-df15-9326-3d5e-f0151bedc105: aborting pipeline replication: peer="{Voter server-32f71c9e-2a0e-5da4-829b-7c61ce435bf6 32f71c9e-2a0e-5da4-829b-7c61ce435bf6}" +2025-02-25T23:58:46.686Z [INFO] server-c22ea190-d273-0b5f-9ad9-8bc2d549d5ed: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c22ea190-d273-0b5f-9ad9-8bc2d549d5ed Address:c22ea190-d273-0b5f-9ad9-8bc2d549d5ed} {Suffrage:Voter ID:server-c4beb647-608a-17ac-f85d-096114ba7228 Address:c4beb647-608a-17ac-f85d-096114ba7228} {Suffrage:Voter ID:server-f95172c5-dce1-5c0e-7027-6b4e931859af Address:f95172c5-dce1-5c0e-7027-6b4e931859af}]" +2025-02-25T23:58:46.686Z [INFO] server-c4beb647-608a-17ac-f85d-096114ba7228: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c22ea190-d273-0b5f-9ad9-8bc2d549d5ed Address:c22ea190-d273-0b5f-9ad9-8bc2d549d5ed} {Suffrage:Voter ID:server-c4beb647-608a-17ac-f85d-096114ba7228 Address:c4beb647-608a-17ac-f85d-096114ba7228} {Suffrage:Voter ID:server-f95172c5-dce1-5c0e-7027-6b4e931859af Address:f95172c5-dce1-5c0e-7027-6b4e931859af}]" +2025-02-25T23:58:46.686Z [INFO] server-f95172c5-dce1-5c0e-7027-6b4e931859af: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c22ea190-d273-0b5f-9ad9-8bc2d549d5ed Address:c22ea190-d273-0b5f-9ad9-8bc2d549d5ed} {Suffrage:Voter ID:server-c4beb647-608a-17ac-f85d-096114ba7228 Address:c4beb647-608a-17ac-f85d-096114ba7228} {Suffrage:Voter ID:server-f95172c5-dce1-5c0e-7027-6b4e931859af Address:f95172c5-dce1-5c0e-7027-6b4e931859af}]" +2025-02-25T23:58:46.686Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:58:46.686Z [INFO] server-c22ea190-d273-0b5f-9ad9-8bc2d549d5ed: entering follower state: follower="Node at c22ea190-d273-0b5f-9ad9-8bc2d549d5ed [Follower]" leader-address= leader-id= +2025-02-25T23:58:46.686Z [INFO] server-c4beb647-608a-17ac-f85d-096114ba7228: entering follower state: follower="Node at c4beb647-608a-17ac-f85d-096114ba7228 [Follower]" leader-address= leader-id= +2025-02-25T23:58:46.686Z [INFO] server-f95172c5-dce1-5c0e-7027-6b4e931859af: entering follower state: follower="Node at f95172c5-dce1-5c0e-7027-6b4e931859af [Follower]" leader-address= leader-id= +2025-02-25T23:58:47.370Z [WARN] server-f95172c5-dce1-5c0e-7027-6b4e931859af: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:58:47.370Z [INFO] server-f95172c5-dce1-5c0e-7027-6b4e931859af: entering candidate state: node="Node at f95172c5-dce1-5c0e-7027-6b4e931859af [Candidate]" term=2 +2025-02-25T23:58:47.370Z [INFO] server-f95172c5-dce1-5c0e-7027-6b4e931859af: election won: term=2 tally=2 +2025-02-25T23:58:47.370Z [INFO] server-f95172c5-dce1-5c0e-7027-6b4e931859af: entering leader state: leader="Node at f95172c5-dce1-5c0e-7027-6b4e931859af [Leader]" +2025-02-25T23:58:47.370Z [INFO] server-f95172c5-dce1-5c0e-7027-6b4e931859af: added peer, starting replication: peer=server-c22ea190-d273-0b5f-9ad9-8bc2d549d5ed +2025-02-25T23:58:47.370Z [INFO] server-f95172c5-dce1-5c0e-7027-6b4e931859af: added peer, starting replication: peer=server-c4beb647-608a-17ac-f85d-096114ba7228 +2025-02-25T23:58:47.370Z [INFO] server-f95172c5-dce1-5c0e-7027-6b4e931859af: pipelining replication: peer="{Voter server-c4beb647-608a-17ac-f85d-096114ba7228 c4beb647-608a-17ac-f85d-096114ba7228}" +2025-02-25T23:58:47.370Z [WARN] server-c22ea190-d273-0b5f-9ad9-8bc2d549d5ed: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:58:47.370Z [INFO] server-c22ea190-d273-0b5f-9ad9-8bc2d549d5ed: entering candidate state: node="Node at c22ea190-d273-0b5f-9ad9-8bc2d549d5ed [Candidate]" term=2 +2025-02-25T23:58:47.370Z [INFO] server-c22ea190-d273-0b5f-9ad9-8bc2d549d5ed: duplicate requestVote for same term: term=2 +2025-02-25T23:58:47.370Z [INFO] server-c22ea190-d273-0b5f-9ad9-8bc2d549d5ed: entering follower state: follower="Node at c22ea190-d273-0b5f-9ad9-8bc2d549d5ed [Follower]" leader-address=f95172c5-dce1-5c0e-7027-6b4e931859af leader-id=server-f95172c5-dce1-5c0e-7027-6b4e931859af +2025-02-25T23:58:47.370Z [INFO] server-f95172c5-dce1-5c0e-7027-6b4e931859af: pipelining replication: peer="{Voter server-c22ea190-d273-0b5f-9ad9-8bc2d549d5ed c22ea190-d273-0b5f-9ad9-8bc2d549d5ed}" +2025-02-25T23:58:47.370Z [WARN] server-f95172c5-dce1-5c0e-7027-6b4e931859af: rejecting vote request since we have a leader: from=c22ea190-d273-0b5f-9ad9-8bc2d549d5ed leader=f95172c5-dce1-5c0e-7027-6b4e931859af leader-id=server-f95172c5-dce1-5c0e-7027-6b4e931859af +2025-02-25T23:58:47.370Z [WARN] server-c4beb647-608a-17ac-f85d-096114ba7228: rejecting vote request since we have a leader: from=c22ea190-d273-0b5f-9ad9-8bc2d549d5ed leader=f95172c5-dce1-5c0e-7027-6b4e931859af leader-id=server-f95172c5-dce1-5c0e-7027-6b4e931859af +2025-02-25T23:58:48.377Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:58:47.370612882 +0000 UTC m=+74.979628289 (1 nodes), 683.795154ms from start of poll, 683.973155ms from cluster start. Timeout at 2025-02-25 23:58:48.375628162 +0000 UTC m=+75.984643569, 1.00501528s after stability +2025-02-25T23:58:48.378Z [INFO] server-f95172c5-dce1-5c0e-7027-6b4e931859af: starting snapshot up to: index=12 +2025-02-25T23:58:48.378Z [INFO] creating new snapshot: path=/tmp/raft2216329897/snapshots/2-12-1740527928378.tmp +2025-02-25T23:58:48.379Z [INFO] server-f95172c5-dce1-5c0e-7027-6b4e931859af: snapshot complete up to: index=12 +2025-02-25T23:58:48.390Z [INFO] creating new snapshot: path=/tmp/raft2216329897/snapshots/2-1013-1740527928390.tmp +2025-02-25T23:58:48.390Z [INFO] server-f95172c5-dce1-5c0e-7027-6b4e931859af: copied to local snapshot: bytes=71 +2025-02-25T23:58:48.390Z [INFO] server-f95172c5-dce1-5c0e-7027-6b4e931859af: snapshot restore progress: id=2-1013-1740527928390 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:48.390Z [INFO] server-f95172c5-dce1-5c0e-7027-6b4e931859af: removing all old logs from log store +2025-02-25T23:58:48.390Z [INFO] server-f95172c5-dce1-5c0e-7027-6b4e931859af: compacting logs: from=1 to=22 +2025-02-25T23:58:48.390Z [INFO] server-f95172c5-dce1-5c0e-7027-6b4e931859af: restored user snapshot: index=1013 +2025-02-25T23:58:48.390Z [ERROR] server-f95172c5-dce1-5c0e-7027-6b4e931859af: failed to get log: index=22 error="log not found" +2025-02-25T23:58:48.390Z [INFO] server-f95172c5-dce1-5c0e-7027-6b4e931859af: aborting pipeline replication: peer="{Voter server-c4beb647-608a-17ac-f85d-096114ba7228 c4beb647-608a-17ac-f85d-096114ba7228}" +2025-02-25T23:58:48.390Z [ERROR] server-f95172c5-dce1-5c0e-7027-6b4e931859af: failed to get log: index=22 error="log not found" +2025-02-25T23:58:48.390Z [INFO] server-f95172c5-dce1-5c0e-7027-6b4e931859af: aborting pipeline replication: peer="{Voter server-c22ea190-d273-0b5f-9ad9-8bc2d549d5ed c22ea190-d273-0b5f-9ad9-8bc2d549d5ed}" +2025-02-25T23:58:48.397Z [ERROR] server-f95172c5-dce1-5c0e-7027-6b4e931859af: failed to get log: index=22 error="log not found" +2025-02-25T23:58:48.398Z [INFO] creating new snapshot: path=/tmp/raft2178002190/snapshots/2-1013-1740527928398.tmp +2025-02-25T23:58:48.398Z [INFO] server-c4beb647-608a-17ac-f85d-096114ba7228: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:48.398Z [INFO] server-c4beb647-608a-17ac-f85d-096114ba7228: copied to local snapshot: bytes=71 +2025-02-25T23:58:48.398Z [INFO] server-c4beb647-608a-17ac-f85d-096114ba7228: snapshot restore progress: id=2-1013-1740527928398 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:48.398Z [INFO] server-c4beb647-608a-17ac-f85d-096114ba7228: removing all old logs from log store +2025-02-25T23:58:48.398Z [INFO] server-c4beb647-608a-17ac-f85d-096114ba7228: compacting logs: from=1 to=22 +2025-02-25T23:58:48.398Z [INFO] server-c4beb647-608a-17ac-f85d-096114ba7228: Installed remote snapshot +2025-02-25T23:58:48.398Z [WARN] server-c4beb647-608a-17ac-f85d-096114ba7228: skipping application of old log: index=22 +2025-02-25T23:58:48.398Z [INFO] server-f95172c5-dce1-5c0e-7027-6b4e931859af: pipelining replication: peer="{Voter server-c4beb647-608a-17ac-f85d-096114ba7228 c4beb647-608a-17ac-f85d-096114ba7228}" +2025-02-25T23:58:48.398Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:58:48.405Z [ERROR] server-f95172c5-dce1-5c0e-7027-6b4e931859af: failed to get log: index=22 error="log not found" +2025-02-25T23:58:48.406Z [INFO] creating new snapshot: path=/tmp/raft412455806/snapshots/2-1013-1740527928406.tmp +2025-02-25T23:58:48.406Z [INFO] server-c22ea190-d273-0b5f-9ad9-8bc2d549d5ed: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:48.406Z [INFO] server-c22ea190-d273-0b5f-9ad9-8bc2d549d5ed: copied to local snapshot: bytes=71 +2025-02-25T23:58:48.406Z [INFO] server-c22ea190-d273-0b5f-9ad9-8bc2d549d5ed: snapshot restore progress: id=2-1013-1740527928406 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:48.406Z [INFO] server-c22ea190-d273-0b5f-9ad9-8bc2d549d5ed: removing all old logs from log store +2025-02-25T23:58:48.406Z [INFO] server-c22ea190-d273-0b5f-9ad9-8bc2d549d5ed: compacting logs: from=1 to=22 +2025-02-25T23:58:48.406Z [INFO] server-c22ea190-d273-0b5f-9ad9-8bc2d549d5ed: Installed remote snapshot +2025-02-25T23:58:48.406Z [INFO] server-f95172c5-dce1-5c0e-7027-6b4e931859af: pipelining replication: peer="{Voter server-c22ea190-d273-0b5f-9ad9-8bc2d549d5ed c22ea190-d273-0b5f-9ad9-8bc2d549d5ed}" +2025-02-25T23:58:49.406Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:58:48.398726508 +0000 UTC m=+76.007741915 (2 nodes), 560ns from start of poll, 1.712086781s from cluster start. Timeout at 2025-02-25 23:58:49.403728828 +0000 UTC m=+77.012744235, 1.00500232s after stability +2025-02-25T23:58:49.422Z [INFO] server-f95172c5-dce1-5c0e-7027-6b4e931859af: aborting pipeline replication: peer="{Voter server-c4beb647-608a-17ac-f85d-096114ba7228 c4beb647-608a-17ac-f85d-096114ba7228}" +2025-02-25T23:58:49.422Z [INFO] server-f95172c5-dce1-5c0e-7027-6b4e931859af: aborting pipeline replication: peer="{Voter server-c22ea190-d273-0b5f-9ad9-8bc2d549d5ed c22ea190-d273-0b5f-9ad9-8bc2d549d5ed}" === RUN TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_true -2026-03-31T06:17:27.256Z [INFO] server-b04e078b-7e6f-c2f8-51cf-b3f479c47700: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b04e078b-7e6f-c2f8-51cf-b3f479c47700 Address:b04e078b-7e6f-c2f8-51cf-b3f479c47700} {Suffrage:Voter ID:server-b0ccbb7f-42ac-62c4-c57c-c184fc16bc2a Address:b0ccbb7f-42ac-62c4-c57c-c184fc16bc2a} {Suffrage:Voter ID:server-ce49da3b-259e-dcac-37f9-8e3e0df07a5f Address:ce49da3b-259e-dcac-37f9-8e3e0df07a5f}]" -2026-03-31T06:17:27.256Z [INFO] server-b0ccbb7f-42ac-62c4-c57c-c184fc16bc2a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b04e078b-7e6f-c2f8-51cf-b3f479c47700 Address:b04e078b-7e6f-c2f8-51cf-b3f479c47700} {Suffrage:Voter ID:server-b0ccbb7f-42ac-62c4-c57c-c184fc16bc2a Address:b0ccbb7f-42ac-62c4-c57c-c184fc16bc2a} {Suffrage:Voter ID:server-ce49da3b-259e-dcac-37f9-8e3e0df07a5f Address:ce49da3b-259e-dcac-37f9-8e3e0df07a5f}]" -2026-03-31T06:17:27.256Z [INFO] server-b04e078b-7e6f-c2f8-51cf-b3f479c47700: entering follower state: follower="Node at b04e078b-7e6f-c2f8-51cf-b3f479c47700 [Follower]" leader-address= leader-id= -2026-03-31T06:17:27.256Z [INFO] server-b0ccbb7f-42ac-62c4-c57c-c184fc16bc2a: entering follower state: follower="Node at b0ccbb7f-42ac-62c4-c57c-c184fc16bc2a [Follower]" leader-address= leader-id= -2026-03-31T06:17:27.256Z [INFO] server-ce49da3b-259e-dcac-37f9-8e3e0df07a5f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b04e078b-7e6f-c2f8-51cf-b3f479c47700 Address:b04e078b-7e6f-c2f8-51cf-b3f479c47700} {Suffrage:Voter ID:server-b0ccbb7f-42ac-62c4-c57c-c184fc16bc2a Address:b0ccbb7f-42ac-62c4-c57c-c184fc16bc2a} {Suffrage:Voter ID:server-ce49da3b-259e-dcac-37f9-8e3e0df07a5f Address:ce49da3b-259e-dcac-37f9-8e3e0df07a5f}]" -2026-03-31T06:17:27.256Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:27.256Z [INFO] server-ce49da3b-259e-dcac-37f9-8e3e0df07a5f: entering follower state: follower="Node at ce49da3b-259e-dcac-37f9-8e3e0df07a5f [Follower]" leader-address= leader-id= -2026-03-31T06:17:27.813Z [WARN] server-b0ccbb7f-42ac-62c4-c57c-c184fc16bc2a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:27.813Z [INFO] server-b0ccbb7f-42ac-62c4-c57c-c184fc16bc2a: entering candidate state: node="Node at b0ccbb7f-42ac-62c4-c57c-c184fc16bc2a [Candidate]" term=2 -2026-03-31T06:17:27.813Z [INFO] server-b0ccbb7f-42ac-62c4-c57c-c184fc16bc2a: election won: term=2 tally=2 -2026-03-31T06:17:27.814Z [INFO] server-b0ccbb7f-42ac-62c4-c57c-c184fc16bc2a: entering leader state: leader="Node at b0ccbb7f-42ac-62c4-c57c-c184fc16bc2a [Leader]" -2026-03-31T06:17:27.814Z [INFO] server-b0ccbb7f-42ac-62c4-c57c-c184fc16bc2a: added peer, starting replication: peer=server-b04e078b-7e6f-c2f8-51cf-b3f479c47700 -2026-03-31T06:17:27.814Z [INFO] server-b0ccbb7f-42ac-62c4-c57c-c184fc16bc2a: added peer, starting replication: peer=server-ce49da3b-259e-dcac-37f9-8e3e0df07a5f -2026-03-31T06:17:27.814Z [INFO] server-b0ccbb7f-42ac-62c4-c57c-c184fc16bc2a: pipelining replication: peer="{Voter server-ce49da3b-259e-dcac-37f9-8e3e0df07a5f ce49da3b-259e-dcac-37f9-8e3e0df07a5f}" -2026-03-31T06:17:27.814Z [INFO] server-b0ccbb7f-42ac-62c4-c57c-c184fc16bc2a: pipelining replication: peer="{Voter server-b04e078b-7e6f-c2f8-51cf-b3f479c47700 b04e078b-7e6f-c2f8-51cf-b3f479c47700}" -2026-03-31T06:17:28.819Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:27.8140397 +0000 UTC m=+76.091647081 (1 nodes), 557.54036ms from start of poll, 558.1398ms from cluster start. Timeout at 2026-03-31 06:17:28.81904298 +0000 UTC m=+77.096650401, 1.00500332s after stability -2026-03-31T06:17:28.820Z [INFO] server-b0ccbb7f-42ac-62c4-c57c-c184fc16bc2a: starting snapshot up to: index=12 -2026-03-31T06:17:28.820Z [INFO] creating new snapshot: path=/tmp/raft1723467927/snapshots/2-12-1774937848820.tmp -2026-03-31T06:17:28.820Z [INFO] server-b0ccbb7f-42ac-62c4-c57c-c184fc16bc2a: snapshot complete up to: index=12 -2026-03-31T06:17:28.821Z [INFO] server-5e78dc5d-334c-dfb1-a343-2cf89b7999c1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5e78dc5d-334c-dfb1-a343-2cf89b7999c1 Address:5e78dc5d-334c-dfb1-a343-2cf89b7999c1} {Suffrage:Voter ID:server-de9ce7ab-af71-0f4c-032f-1ed2f0d097f0 Address:de9ce7ab-af71-0f4c-032f-1ed2f0d097f0} {Suffrage:Voter ID:server-e5ab2f62-b4b7-26a5-2bbc-f96793927052 Address:e5ab2f62-b4b7-26a5-2bbc-f96793927052}]" -2026-03-31T06:17:28.821Z [INFO] server-5e78dc5d-334c-dfb1-a343-2cf89b7999c1: entering follower state: follower="Node at 5e78dc5d-334c-dfb1-a343-2cf89b7999c1 [Follower]" leader-address= leader-id= -2026-03-31T06:17:28.821Z [INFO] server-de9ce7ab-af71-0f4c-032f-1ed2f0d097f0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5e78dc5d-334c-dfb1-a343-2cf89b7999c1 Address:5e78dc5d-334c-dfb1-a343-2cf89b7999c1} {Suffrage:Voter ID:server-de9ce7ab-af71-0f4c-032f-1ed2f0d097f0 Address:de9ce7ab-af71-0f4c-032f-1ed2f0d097f0} {Suffrage:Voter ID:server-e5ab2f62-b4b7-26a5-2bbc-f96793927052 Address:e5ab2f62-b4b7-26a5-2bbc-f96793927052}]" -2026-03-31T06:17:28.821Z [INFO] server-de9ce7ab-af71-0f4c-032f-1ed2f0d097f0: entering follower state: follower="Node at de9ce7ab-af71-0f4c-032f-1ed2f0d097f0 [Follower]" leader-address= leader-id= -2026-03-31T06:17:28.821Z [INFO] server-e5ab2f62-b4b7-26a5-2bbc-f96793927052: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5e78dc5d-334c-dfb1-a343-2cf89b7999c1 Address:5e78dc5d-334c-dfb1-a343-2cf89b7999c1} {Suffrage:Voter ID:server-de9ce7ab-af71-0f4c-032f-1ed2f0d097f0 Address:de9ce7ab-af71-0f4c-032f-1ed2f0d097f0} {Suffrage:Voter ID:server-e5ab2f62-b4b7-26a5-2bbc-f96793927052 Address:e5ab2f62-b4b7-26a5-2bbc-f96793927052}]" -2026-03-31T06:17:28.821Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:28.822Z [INFO] server-e5ab2f62-b4b7-26a5-2bbc-f96793927052: entering follower state: follower="Node at e5ab2f62-b4b7-26a5-2bbc-f96793927052 [Follower]" leader-address= leader-id= -2026-03-31T06:17:29.425Z [WARN] server-e5ab2f62-b4b7-26a5-2bbc-f96793927052: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:29.425Z [INFO] server-e5ab2f62-b4b7-26a5-2bbc-f96793927052: entering candidate state: node="Node at e5ab2f62-b4b7-26a5-2bbc-f96793927052 [Candidate]" term=2 -2026-03-31T06:17:29.425Z [INFO] server-e5ab2f62-b4b7-26a5-2bbc-f96793927052: election won: term=2 tally=2 -2026-03-31T06:17:29.425Z [INFO] server-e5ab2f62-b4b7-26a5-2bbc-f96793927052: entering leader state: leader="Node at e5ab2f62-b4b7-26a5-2bbc-f96793927052 [Leader]" -2026-03-31T06:17:29.425Z [INFO] server-e5ab2f62-b4b7-26a5-2bbc-f96793927052: added peer, starting replication: peer=server-5e78dc5d-334c-dfb1-a343-2cf89b7999c1 -2026-03-31T06:17:29.425Z [INFO] server-e5ab2f62-b4b7-26a5-2bbc-f96793927052: added peer, starting replication: peer=server-de9ce7ab-af71-0f4c-032f-1ed2f0d097f0 -2026-03-31T06:17:29.425Z [INFO] server-e5ab2f62-b4b7-26a5-2bbc-f96793927052: pipelining replication: peer="{Voter server-5e78dc5d-334c-dfb1-a343-2cf89b7999c1 5e78dc5d-334c-dfb1-a343-2cf89b7999c1}" -2026-03-31T06:17:29.425Z [INFO] server-e5ab2f62-b4b7-26a5-2bbc-f96793927052: pipelining replication: peer="{Voter server-de9ce7ab-af71-0f4c-032f-1ed2f0d097f0 de9ce7ab-af71-0f4c-032f-1ed2f0d097f0}" -2026-03-31T06:17:30.430Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:29.42530562 +0000 UTC m=+77.702913001 (1 nodes), 603.31344ms from start of poll, 603.75548ms from cluster start. Timeout at 2026-03-31 06:17:30.43030962 +0000 UTC m=+78.707917041, 1.00500404s after stability -2026-03-31T06:17:30.431Z [INFO] creating new snapshot: path=/tmp/raft1546839382/snapshots/2-1013-1774937850431.tmp -2026-03-31T06:17:30.431Z [INFO] server-e5ab2f62-b4b7-26a5-2bbc-f96793927052: copied to local snapshot: bytes=71 -2026-03-31T06:17:30.431Z [INFO] server-e5ab2f62-b4b7-26a5-2bbc-f96793927052: snapshot restore progress: id=2-1013-1774937850431 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:30.431Z [INFO] server-e5ab2f62-b4b7-26a5-2bbc-f96793927052: restored user snapshot: index=1013 -2026-03-31T06:17:30.431Z [ERROR] server-e5ab2f62-b4b7-26a5-2bbc-f96793927052: failed to get log: index=3 error="log not found" -2026-03-31T06:17:30.431Z [INFO] server-e5ab2f62-b4b7-26a5-2bbc-f96793927052: aborting pipeline replication: peer="{Voter server-de9ce7ab-af71-0f4c-032f-1ed2f0d097f0 de9ce7ab-af71-0f4c-032f-1ed2f0d097f0}" -2026-03-31T06:17:30.431Z [ERROR] server-e5ab2f62-b4b7-26a5-2bbc-f96793927052: failed to get log: index=3 error="log not found" -2026-03-31T06:17:30.431Z [INFO] server-e5ab2f62-b4b7-26a5-2bbc-f96793927052: aborting pipeline replication: peer="{Voter server-5e78dc5d-334c-dfb1-a343-2cf89b7999c1 5e78dc5d-334c-dfb1-a343-2cf89b7999c1}" -2026-03-31T06:17:30.437Z [ERROR] server-e5ab2f62-b4b7-26a5-2bbc-f96793927052: failed to get log: index=3 error="log not found" -2026-03-31T06:17:30.438Z [INFO] creating new snapshot: path=/tmp/raft1406951030/snapshots/2-1013-1774937850438.tmp -2026-03-31T06:17:30.438Z [INFO] server-de9ce7ab-af71-0f4c-032f-1ed2f0d097f0: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:30.438Z [INFO] server-de9ce7ab-af71-0f4c-032f-1ed2f0d097f0: copied to local snapshot: bytes=71 -2026-03-31T06:17:30.439Z [INFO] server-de9ce7ab-af71-0f4c-032f-1ed2f0d097f0: snapshot restore progress: id=2-1013-1774937850438 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:30.439Z [INFO] server-de9ce7ab-af71-0f4c-032f-1ed2f0d097f0: Installed remote snapshot -2026-03-31T06:17:30.439Z [INFO] server-e5ab2f62-b4b7-26a5-2bbc-f96793927052: pipelining replication: peer="{Voter server-de9ce7ab-af71-0f4c-032f-1ed2f0d097f0 de9ce7ab-af71-0f4c-032f-1ed2f0d097f0}" -2026-03-31T06:17:30.439Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:17:30.441Z [ERROR] server-e5ab2f62-b4b7-26a5-2bbc-f96793927052: failed to get log: index=3 error="log not found" -2026-03-31T06:17:30.441Z [INFO] creating new snapshot: path=/tmp/raft93875238/snapshots/2-1013-1774937850441.tmp -2026-03-31T06:17:30.442Z [INFO] server-5e78dc5d-334c-dfb1-a343-2cf89b7999c1: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:30.442Z [INFO] server-5e78dc5d-334c-dfb1-a343-2cf89b7999c1: copied to local snapshot: bytes=71 -2026-03-31T06:17:30.442Z [INFO] server-5e78dc5d-334c-dfb1-a343-2cf89b7999c1: snapshot restore progress: id=2-1013-1774937850441 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:30.442Z [INFO] server-5e78dc5d-334c-dfb1-a343-2cf89b7999c1: Installed remote snapshot -2026-03-31T06:17:30.442Z [INFO] server-e5ab2f62-b4b7-26a5-2bbc-f96793927052: pipelining replication: peer="{Voter server-5e78dc5d-334c-dfb1-a343-2cf89b7999c1 5e78dc5d-334c-dfb1-a343-2cf89b7999c1}" -2026-03-31T06:17:31.445Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:17:30.43943306 +0000 UTC m=+78.717040441 (2 nodes), 920ns from start of poll, 1.61788292s from cluster start. Timeout at 2026-03-31 06:17:31.44444586 +0000 UTC m=+79.722053241, 1.0050128s after stability -2026-03-31T06:17:31.456Z [INFO] server-b0ccbb7f-42ac-62c4-c57c-c184fc16bc2a: aborting pipeline replication: peer="{Voter server-b04e078b-7e6f-c2f8-51cf-b3f479c47700 b04e078b-7e6f-c2f8-51cf-b3f479c47700}" -2026-03-31T06:17:31.456Z [INFO] server-b0ccbb7f-42ac-62c4-c57c-c184fc16bc2a: aborting pipeline replication: peer="{Voter server-ce49da3b-259e-dcac-37f9-8e3e0df07a5f ce49da3b-259e-dcac-37f9-8e3e0df07a5f}" +2025-02-25T23:58:49.423Z [INFO] server-31f4ff2b-a29d-c044-a1dc-6d6a6fe77481: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31f4ff2b-a29d-c044-a1dc-6d6a6fe77481 Address:31f4ff2b-a29d-c044-a1dc-6d6a6fe77481} {Suffrage:Voter ID:server-b50ae21a-b494-bc62-070a-3437d696086a Address:b50ae21a-b494-bc62-070a-3437d696086a} {Suffrage:Voter ID:server-2f93f415-d145-475f-4556-1a0681b57859 Address:2f93f415-d145-475f-4556-1a0681b57859}]" +2025-02-25T23:58:49.423Z [INFO] server-b50ae21a-b494-bc62-070a-3437d696086a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31f4ff2b-a29d-c044-a1dc-6d6a6fe77481 Address:31f4ff2b-a29d-c044-a1dc-6d6a6fe77481} {Suffrage:Voter ID:server-b50ae21a-b494-bc62-070a-3437d696086a Address:b50ae21a-b494-bc62-070a-3437d696086a} {Suffrage:Voter ID:server-2f93f415-d145-475f-4556-1a0681b57859 Address:2f93f415-d145-475f-4556-1a0681b57859}]" +2025-02-25T23:58:49.423Z [INFO] server-2f93f415-d145-475f-4556-1a0681b57859: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31f4ff2b-a29d-c044-a1dc-6d6a6fe77481 Address:31f4ff2b-a29d-c044-a1dc-6d6a6fe77481} {Suffrage:Voter ID:server-b50ae21a-b494-bc62-070a-3437d696086a Address:b50ae21a-b494-bc62-070a-3437d696086a} {Suffrage:Voter ID:server-2f93f415-d145-475f-4556-1a0681b57859 Address:2f93f415-d145-475f-4556-1a0681b57859}]" +2025-02-25T23:58:49.423Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:58:49.423Z [INFO] server-31f4ff2b-a29d-c044-a1dc-6d6a6fe77481: entering follower state: follower="Node at 31f4ff2b-a29d-c044-a1dc-6d6a6fe77481 [Follower]" leader-address= leader-id= +2025-02-25T23:58:49.423Z [INFO] server-b50ae21a-b494-bc62-070a-3437d696086a: entering follower state: follower="Node at b50ae21a-b494-bc62-070a-3437d696086a [Follower]" leader-address= leader-id= +2025-02-25T23:58:49.423Z [INFO] server-2f93f415-d145-475f-4556-1a0681b57859: entering follower state: follower="Node at 2f93f415-d145-475f-4556-1a0681b57859 [Follower]" leader-address= leader-id= +2025-02-25T23:58:49.966Z [WARN] server-2f93f415-d145-475f-4556-1a0681b57859: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:58:49.966Z [INFO] server-2f93f415-d145-475f-4556-1a0681b57859: entering candidate state: node="Node at 2f93f415-d145-475f-4556-1a0681b57859 [Candidate]" term=2 +2025-02-25T23:58:49.966Z [INFO] server-2f93f415-d145-475f-4556-1a0681b57859: election won: term=2 tally=2 +2025-02-25T23:58:49.966Z [INFO] server-2f93f415-d145-475f-4556-1a0681b57859: entering leader state: leader="Node at 2f93f415-d145-475f-4556-1a0681b57859 [Leader]" +2025-02-25T23:58:49.966Z [INFO] server-2f93f415-d145-475f-4556-1a0681b57859: added peer, starting replication: peer=server-31f4ff2b-a29d-c044-a1dc-6d6a6fe77481 +2025-02-25T23:58:49.966Z [INFO] server-2f93f415-d145-475f-4556-1a0681b57859: added peer, starting replication: peer=server-b50ae21a-b494-bc62-070a-3437d696086a +2025-02-25T23:58:49.966Z [INFO] server-2f93f415-d145-475f-4556-1a0681b57859: pipelining replication: peer="{Voter server-b50ae21a-b494-bc62-070a-3437d696086a b50ae21a-b494-bc62-070a-3437d696086a}" +2025-02-25T23:58:49.966Z [INFO] server-2f93f415-d145-475f-4556-1a0681b57859: pipelining replication: peer="{Voter server-31f4ff2b-a29d-c044-a1dc-6d6a6fe77481 31f4ff2b-a29d-c044-a1dc-6d6a6fe77481}" +2025-02-25T23:58:50.973Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:58:49.966638931 +0000 UTC m=+77.575654338 (1 nodes), 543.4095ms from start of poll, 543.585821ms from cluster start. Timeout at 2025-02-25 23:58:50.971642651 +0000 UTC m=+78.580658058, 1.00500372s after stability +2025-02-25T23:58:50.974Z [INFO] server-2f93f415-d145-475f-4556-1a0681b57859: starting snapshot up to: index=12 +2025-02-25T23:58:50.974Z [INFO] creating new snapshot: path=/tmp/raft1971939157/snapshots/2-12-1740527930974.tmp +2025-02-25T23:58:50.974Z [INFO] server-2f93f415-d145-475f-4556-1a0681b57859: snapshot complete up to: index=12 +2025-02-25T23:58:50.975Z [INFO] server-921c8fda-cc35-800c-1d2b-17d015987542: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-921c8fda-cc35-800c-1d2b-17d015987542 Address:921c8fda-cc35-800c-1d2b-17d015987542} {Suffrage:Voter ID:server-644faf4e-7593-23d4-c7d5-5f1b8bb6eec4 Address:644faf4e-7593-23d4-c7d5-5f1b8bb6eec4} {Suffrage:Voter ID:server-db629e71-7ea9-de28-0fc4-b15c9c042915 Address:db629e71-7ea9-de28-0fc4-b15c9c042915}]" +2025-02-25T23:58:50.975Z [INFO] server-644faf4e-7593-23d4-c7d5-5f1b8bb6eec4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-921c8fda-cc35-800c-1d2b-17d015987542 Address:921c8fda-cc35-800c-1d2b-17d015987542} {Suffrage:Voter ID:server-644faf4e-7593-23d4-c7d5-5f1b8bb6eec4 Address:644faf4e-7593-23d4-c7d5-5f1b8bb6eec4} {Suffrage:Voter ID:server-db629e71-7ea9-de28-0fc4-b15c9c042915 Address:db629e71-7ea9-de28-0fc4-b15c9c042915}]" +2025-02-25T23:58:50.975Z [INFO] server-db629e71-7ea9-de28-0fc4-b15c9c042915: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-921c8fda-cc35-800c-1d2b-17d015987542 Address:921c8fda-cc35-800c-1d2b-17d015987542} {Suffrage:Voter ID:server-644faf4e-7593-23d4-c7d5-5f1b8bb6eec4 Address:644faf4e-7593-23d4-c7d5-5f1b8bb6eec4} {Suffrage:Voter ID:server-db629e71-7ea9-de28-0fc4-b15c9c042915 Address:db629e71-7ea9-de28-0fc4-b15c9c042915}]" +2025-02-25T23:58:50.975Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:58:50.975Z [INFO] server-921c8fda-cc35-800c-1d2b-17d015987542: entering follower state: follower="Node at 921c8fda-cc35-800c-1d2b-17d015987542 [Follower]" leader-address= leader-id= +2025-02-25T23:58:50.975Z [INFO] server-644faf4e-7593-23d4-c7d5-5f1b8bb6eec4: entering follower state: follower="Node at 644faf4e-7593-23d4-c7d5-5f1b8bb6eec4 [Follower]" leader-address= leader-id= +2025-02-25T23:58:50.975Z [INFO] server-db629e71-7ea9-de28-0fc4-b15c9c042915: entering follower state: follower="Node at db629e71-7ea9-de28-0fc4-b15c9c042915 [Follower]" leader-address= leader-id= +2025-02-25T23:58:51.590Z [WARN] server-db629e71-7ea9-de28-0fc4-b15c9c042915: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:58:51.590Z [INFO] server-db629e71-7ea9-de28-0fc4-b15c9c042915: entering candidate state: node="Node at db629e71-7ea9-de28-0fc4-b15c9c042915 [Candidate]" term=2 +2025-02-25T23:58:51.590Z [INFO] server-db629e71-7ea9-de28-0fc4-b15c9c042915: election won: term=2 tally=2 +2025-02-25T23:58:51.590Z [INFO] server-db629e71-7ea9-de28-0fc4-b15c9c042915: entering leader state: leader="Node at db629e71-7ea9-de28-0fc4-b15c9c042915 [Leader]" +2025-02-25T23:58:51.590Z [INFO] server-db629e71-7ea9-de28-0fc4-b15c9c042915: added peer, starting replication: peer=server-921c8fda-cc35-800c-1d2b-17d015987542 +2025-02-25T23:58:51.590Z [INFO] server-db629e71-7ea9-de28-0fc4-b15c9c042915: added peer, starting replication: peer=server-644faf4e-7593-23d4-c7d5-5f1b8bb6eec4 +2025-02-25T23:58:51.590Z [INFO] server-db629e71-7ea9-de28-0fc4-b15c9c042915: pipelining replication: peer="{Voter server-644faf4e-7593-23d4-c7d5-5f1b8bb6eec4 644faf4e-7593-23d4-c7d5-5f1b8bb6eec4}" +2025-02-25T23:58:51.590Z [INFO] server-db629e71-7ea9-de28-0fc4-b15c9c042915: pipelining replication: peer="{Voter server-921c8fda-cc35-800c-1d2b-17d015987542 921c8fda-cc35-800c-1d2b-17d015987542}" +2025-02-25T23:58:52.597Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:58:51.59054887 +0000 UTC m=+79.199564277 (1 nodes), 615.245197ms from start of poll, 615.413758ms from cluster start. Timeout at 2025-02-25 23:58:52.59555559 +0000 UTC m=+80.204570957, 1.00500668s after stability +2025-02-25T23:58:52.598Z [INFO] creating new snapshot: path=/tmp/raft2316868673/snapshots/2-1013-1740527932598.tmp +2025-02-25T23:58:52.598Z [INFO] server-db629e71-7ea9-de28-0fc4-b15c9c042915: copied to local snapshot: bytes=71 +2025-02-25T23:58:52.598Z [INFO] server-db629e71-7ea9-de28-0fc4-b15c9c042915: snapshot restore progress: id=2-1013-1740527932598 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:52.598Z [INFO] server-db629e71-7ea9-de28-0fc4-b15c9c042915: restored user snapshot: index=1013 +2025-02-25T23:58:52.598Z [ERROR] server-db629e71-7ea9-de28-0fc4-b15c9c042915: failed to get log: index=3 error="log not found" +2025-02-25T23:58:52.598Z [INFO] server-db629e71-7ea9-de28-0fc4-b15c9c042915: aborting pipeline replication: peer="{Voter server-644faf4e-7593-23d4-c7d5-5f1b8bb6eec4 644faf4e-7593-23d4-c7d5-5f1b8bb6eec4}" +2025-02-25T23:58:52.598Z [ERROR] server-db629e71-7ea9-de28-0fc4-b15c9c042915: failed to get log: index=3 error="log not found" +2025-02-25T23:58:52.598Z [INFO] server-db629e71-7ea9-de28-0fc4-b15c9c042915: aborting pipeline replication: peer="{Voter server-921c8fda-cc35-800c-1d2b-17d015987542 921c8fda-cc35-800c-1d2b-17d015987542}" +2025-02-25T23:58:52.605Z [ERROR] server-db629e71-7ea9-de28-0fc4-b15c9c042915: failed to get log: index=3 error="log not found" +2025-02-25T23:58:52.606Z [INFO] creating new snapshot: path=/tmp/raft373161415/snapshots/2-1013-1740527932606.tmp +2025-02-25T23:58:52.606Z [INFO] server-921c8fda-cc35-800c-1d2b-17d015987542: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:52.606Z [INFO] server-921c8fda-cc35-800c-1d2b-17d015987542: copied to local snapshot: bytes=71 +2025-02-25T23:58:52.606Z [INFO] server-921c8fda-cc35-800c-1d2b-17d015987542: snapshot restore progress: id=2-1013-1740527932606 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:52.606Z [INFO] server-921c8fda-cc35-800c-1d2b-17d015987542: Installed remote snapshot +2025-02-25T23:58:52.606Z [INFO] server-db629e71-7ea9-de28-0fc4-b15c9c042915: pipelining replication: peer="{Voter server-921c8fda-cc35-800c-1d2b-17d015987542 921c8fda-cc35-800c-1d2b-17d015987542}" +2025-02-25T23:58:52.606Z [ERROR] server-db629e71-7ea9-de28-0fc4-b15c9c042915: failed to get log: index=3 error="log not found" +2025-02-25T23:58:52.606Z [INFO] creating new snapshot: path=/tmp/raft2560350992/snapshots/2-1013-1740527932606.tmp +2025-02-25T23:58:52.606Z [INFO] server-644faf4e-7593-23d4-c7d5-5f1b8bb6eec4: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:52.607Z [INFO] server-644faf4e-7593-23d4-c7d5-5f1b8bb6eec4: copied to local snapshot: bytes=71 +2025-02-25T23:58:52.607Z [INFO] server-644faf4e-7593-23d4-c7d5-5f1b8bb6eec4: snapshot restore progress: id=2-1013-1740527932606 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:52.607Z [INFO] server-644faf4e-7593-23d4-c7d5-5f1b8bb6eec4: Installed remote snapshot +2025-02-25T23:58:52.607Z [INFO] server-db629e71-7ea9-de28-0fc4-b15c9c042915: pipelining replication: peer="{Voter server-644faf4e-7593-23d4-c7d5-5f1b8bb6eec4 644faf4e-7593-23d4-c7d5-5f1b8bb6eec4}" +2025-02-25T23:58:52.607Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:58:53.614Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:58:52.607273664 +0000 UTC m=+80.216289031 (2 nodes), 440ns from start of poll, 1.632138512s from cluster start. Timeout at 2025-02-25 23:58:53.612291224 +0000 UTC m=+81.221306631, 1.0050176s after stability +2025-02-25T23:58:53.637Z [INFO] server-2f93f415-d145-475f-4556-1a0681b57859: aborting pipeline replication: peer="{Voter server-31f4ff2b-a29d-c044-a1dc-6d6a6fe77481 31f4ff2b-a29d-c044-a1dc-6d6a6fe77481}" +2025-02-25T23:58:53.637Z [INFO] server-2f93f415-d145-475f-4556-1a0681b57859: aborting pipeline replication: peer="{Voter server-b50ae21a-b494-bc62-070a-3437d696086a b50ae21a-b494-bc62-070a-3437d696086a}" === RUN TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_true -2026-03-31T06:17:31.457Z [INFO] server-95d8e299-d28d-9fbc-2197-0a286385dd9c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-95d8e299-d28d-9fbc-2197-0a286385dd9c Address:95d8e299-d28d-9fbc-2197-0a286385dd9c} {Suffrage:Voter ID:server-63e6bab9-a3fb-f266-d4a2-b02f380b07d5 Address:63e6bab9-a3fb-f266-d4a2-b02f380b07d5} {Suffrage:Voter ID:server-06bf44c7-7190-e36e-350b-48fa5debf5b9 Address:06bf44c7-7190-e36e-350b-48fa5debf5b9}]" -2026-03-31T06:17:31.457Z [INFO] server-63e6bab9-a3fb-f266-d4a2-b02f380b07d5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-95d8e299-d28d-9fbc-2197-0a286385dd9c Address:95d8e299-d28d-9fbc-2197-0a286385dd9c} {Suffrage:Voter ID:server-63e6bab9-a3fb-f266-d4a2-b02f380b07d5 Address:63e6bab9-a3fb-f266-d4a2-b02f380b07d5} {Suffrage:Voter ID:server-06bf44c7-7190-e36e-350b-48fa5debf5b9 Address:06bf44c7-7190-e36e-350b-48fa5debf5b9}]" -2026-03-31T06:17:31.457Z [INFO] server-95d8e299-d28d-9fbc-2197-0a286385dd9c: entering follower state: follower="Node at 95d8e299-d28d-9fbc-2197-0a286385dd9c [Follower]" leader-address= leader-id= -2026-03-31T06:17:31.457Z [INFO] server-63e6bab9-a3fb-f266-d4a2-b02f380b07d5: entering follower state: follower="Node at 63e6bab9-a3fb-f266-d4a2-b02f380b07d5 [Follower]" leader-address= leader-id= -2026-03-31T06:17:31.457Z [INFO] server-06bf44c7-7190-e36e-350b-48fa5debf5b9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-95d8e299-d28d-9fbc-2197-0a286385dd9c Address:95d8e299-d28d-9fbc-2197-0a286385dd9c} {Suffrage:Voter ID:server-63e6bab9-a3fb-f266-d4a2-b02f380b07d5 Address:63e6bab9-a3fb-f266-d4a2-b02f380b07d5} {Suffrage:Voter ID:server-06bf44c7-7190-e36e-350b-48fa5debf5b9 Address:06bf44c7-7190-e36e-350b-48fa5debf5b9}]" -2026-03-31T06:17:31.457Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:31.457Z [INFO] server-06bf44c7-7190-e36e-350b-48fa5debf5b9: entering follower state: follower="Node at 06bf44c7-7190-e36e-350b-48fa5debf5b9 [Follower]" leader-address= leader-id= -2026-03-31T06:17:32.186Z [WARN] server-06bf44c7-7190-e36e-350b-48fa5debf5b9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:32.186Z [INFO] server-06bf44c7-7190-e36e-350b-48fa5debf5b9: entering candidate state: node="Node at 06bf44c7-7190-e36e-350b-48fa5debf5b9 [Candidate]" term=2 -2026-03-31T06:17:32.186Z [INFO] server-06bf44c7-7190-e36e-350b-48fa5debf5b9: election won: term=2 tally=2 -2026-03-31T06:17:32.186Z [INFO] server-06bf44c7-7190-e36e-350b-48fa5debf5b9: entering leader state: leader="Node at 06bf44c7-7190-e36e-350b-48fa5debf5b9 [Leader]" -2026-03-31T06:17:32.186Z [INFO] server-06bf44c7-7190-e36e-350b-48fa5debf5b9: added peer, starting replication: peer=server-95d8e299-d28d-9fbc-2197-0a286385dd9c -2026-03-31T06:17:32.186Z [INFO] server-06bf44c7-7190-e36e-350b-48fa5debf5b9: added peer, starting replication: peer=server-63e6bab9-a3fb-f266-d4a2-b02f380b07d5 -2026-03-31T06:17:32.186Z [INFO] server-06bf44c7-7190-e36e-350b-48fa5debf5b9: pipelining replication: peer="{Voter server-63e6bab9-a3fb-f266-d4a2-b02f380b07d5 63e6bab9-a3fb-f266-d4a2-b02f380b07d5}" -2026-03-31T06:17:32.186Z [INFO] server-06bf44c7-7190-e36e-350b-48fa5debf5b9: pipelining replication: peer="{Voter server-95d8e299-d28d-9fbc-2197-0a286385dd9c 95d8e299-d28d-9fbc-2197-0a286385dd9c}" -2026-03-31T06:17:33.192Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:32.18671458 +0000 UTC m=+80.464321961 (1 nodes), 729.15724ms from start of poll, 729.59016ms from cluster start. Timeout at 2026-03-31 06:17:33.1917381 +0000 UTC m=+81.469345481, 1.00502352s after stability -2026-03-31T06:17:33.192Z [INFO] server-06bf44c7-7190-e36e-350b-48fa5debf5b9: starting snapshot up to: index=12 -2026-03-31T06:17:33.192Z [INFO] creating new snapshot: path=/tmp/raft901330092/snapshots/2-12-1774937853192.tmp -2026-03-31T06:17:33.193Z [INFO] server-06bf44c7-7190-e36e-350b-48fa5debf5b9: snapshot complete up to: index=12 -2026-03-31T06:17:33.194Z [INFO] server-138684a6-3969-fa11-0ec2-b6ae697f77de: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-138684a6-3969-fa11-0ec2-b6ae697f77de Address:138684a6-3969-fa11-0ec2-b6ae697f77de} {Suffrage:Voter ID:server-27a15905-12a1-0071-aec6-5346e06f87f9 Address:27a15905-12a1-0071-aec6-5346e06f87f9} {Suffrage:Voter ID:server-8b4ad946-840c-fba6-e77c-d74b2a44e3c5 Address:8b4ad946-840c-fba6-e77c-d74b2a44e3c5}]" -2026-03-31T06:17:33.194Z [INFO] server-27a15905-12a1-0071-aec6-5346e06f87f9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-138684a6-3969-fa11-0ec2-b6ae697f77de Address:138684a6-3969-fa11-0ec2-b6ae697f77de} {Suffrage:Voter ID:server-27a15905-12a1-0071-aec6-5346e06f87f9 Address:27a15905-12a1-0071-aec6-5346e06f87f9} {Suffrage:Voter ID:server-8b4ad946-840c-fba6-e77c-d74b2a44e3c5 Address:8b4ad946-840c-fba6-e77c-d74b2a44e3c5}]" -2026-03-31T06:17:33.194Z [INFO] server-138684a6-3969-fa11-0ec2-b6ae697f77de: entering follower state: follower="Node at 138684a6-3969-fa11-0ec2-b6ae697f77de [Follower]" leader-address= leader-id= -2026-03-31T06:17:33.194Z [INFO] server-27a15905-12a1-0071-aec6-5346e06f87f9: entering follower state: follower="Node at 27a15905-12a1-0071-aec6-5346e06f87f9 [Follower]" leader-address= leader-id= -2026-03-31T06:17:33.194Z [INFO] server-8b4ad946-840c-fba6-e77c-d74b2a44e3c5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-138684a6-3969-fa11-0ec2-b6ae697f77de Address:138684a6-3969-fa11-0ec2-b6ae697f77de} {Suffrage:Voter ID:server-27a15905-12a1-0071-aec6-5346e06f87f9 Address:27a15905-12a1-0071-aec6-5346e06f87f9} {Suffrage:Voter ID:server-8b4ad946-840c-fba6-e77c-d74b2a44e3c5 Address:8b4ad946-840c-fba6-e77c-d74b2a44e3c5}]" -2026-03-31T06:17:33.194Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:33.194Z [INFO] server-8b4ad946-840c-fba6-e77c-d74b2a44e3c5: entering follower state: follower="Node at 8b4ad946-840c-fba6-e77c-d74b2a44e3c5 [Follower]" leader-address= leader-id= -2026-03-31T06:17:33.805Z [WARN] server-8b4ad946-840c-fba6-e77c-d74b2a44e3c5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:33.805Z [INFO] server-8b4ad946-840c-fba6-e77c-d74b2a44e3c5: entering candidate state: node="Node at 8b4ad946-840c-fba6-e77c-d74b2a44e3c5 [Candidate]" term=2 -2026-03-31T06:17:33.805Z [INFO] server-8b4ad946-840c-fba6-e77c-d74b2a44e3c5: election won: term=2 tally=2 -2026-03-31T06:17:33.805Z [INFO] server-8b4ad946-840c-fba6-e77c-d74b2a44e3c5: entering leader state: leader="Node at 8b4ad946-840c-fba6-e77c-d74b2a44e3c5 [Leader]" -2026-03-31T06:17:33.806Z [INFO] server-8b4ad946-840c-fba6-e77c-d74b2a44e3c5: added peer, starting replication: peer=server-138684a6-3969-fa11-0ec2-b6ae697f77de -2026-03-31T06:17:33.806Z [INFO] server-8b4ad946-840c-fba6-e77c-d74b2a44e3c5: added peer, starting replication: peer=server-27a15905-12a1-0071-aec6-5346e06f87f9 -2026-03-31T06:17:33.806Z [INFO] server-8b4ad946-840c-fba6-e77c-d74b2a44e3c5: pipelining replication: peer="{Voter server-138684a6-3969-fa11-0ec2-b6ae697f77de 138684a6-3969-fa11-0ec2-b6ae697f77de}" -2026-03-31T06:17:33.806Z [INFO] server-8b4ad946-840c-fba6-e77c-d74b2a44e3c5: pipelining replication: peer="{Voter server-27a15905-12a1-0071-aec6-5346e06f87f9 27a15905-12a1-0071-aec6-5346e06f87f9}" -2026-03-31T06:17:34.811Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:33.80607538 +0000 UTC m=+82.083682801 (1 nodes), 611.34296ms from start of poll, 611.75772ms from cluster start. Timeout at 2026-03-31 06:17:34.81107898 +0000 UTC m=+83.088686361, 1.00500356s after stability -2026-03-31T06:17:34.811Z [INFO] creating new snapshot: path=/tmp/raft86392860/snapshots/2-1013-1774937854811.tmp -2026-03-31T06:17:34.812Z [INFO] server-8b4ad946-840c-fba6-e77c-d74b2a44e3c5: copied to local snapshot: bytes=71 -2026-03-31T06:17:34.812Z [INFO] server-8b4ad946-840c-fba6-e77c-d74b2a44e3c5: snapshot restore progress: id=2-1013-1774937854811 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:34.812Z [INFO] server-8b4ad946-840c-fba6-e77c-d74b2a44e3c5: removing all old logs from log store -2026-03-31T06:17:34.812Z [INFO] server-8b4ad946-840c-fba6-e77c-d74b2a44e3c5: compacting logs: from=1 to=2 -2026-03-31T06:17:34.812Z [INFO] server-8b4ad946-840c-fba6-e77c-d74b2a44e3c5: restored user snapshot: index=1013 -2026-03-31T06:17:34.812Z [ERROR] server-8b4ad946-840c-fba6-e77c-d74b2a44e3c5: failed to get log: index=2 error="log not found" -2026-03-31T06:17:34.812Z [INFO] server-8b4ad946-840c-fba6-e77c-d74b2a44e3c5: aborting pipeline replication: peer="{Voter server-27a15905-12a1-0071-aec6-5346e06f87f9 27a15905-12a1-0071-aec6-5346e06f87f9}" -2026-03-31T06:17:34.812Z [ERROR] server-8b4ad946-840c-fba6-e77c-d74b2a44e3c5: failed to get log: index=2 error="log not found" -2026-03-31T06:17:34.812Z [INFO] server-8b4ad946-840c-fba6-e77c-d74b2a44e3c5: aborting pipeline replication: peer="{Voter server-138684a6-3969-fa11-0ec2-b6ae697f77de 138684a6-3969-fa11-0ec2-b6ae697f77de}" -2026-03-31T06:17:34.818Z [ERROR] server-8b4ad946-840c-fba6-e77c-d74b2a44e3c5: failed to get log: index=2 error="log not found" -2026-03-31T06:17:34.818Z [INFO] creating new snapshot: path=/tmp/raft1839776734/snapshots/2-1013-1774937854818.tmp -2026-03-31T06:17:34.818Z [INFO] server-138684a6-3969-fa11-0ec2-b6ae697f77de: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:34.819Z [INFO] server-138684a6-3969-fa11-0ec2-b6ae697f77de: copied to local snapshot: bytes=71 -2026-03-31T06:17:34.819Z [ERROR] server-8b4ad946-840c-fba6-e77c-d74b2a44e3c5: failed to get log: index=2 error="log not found" -2026-03-31T06:17:34.819Z [INFO] server-138684a6-3969-fa11-0ec2-b6ae697f77de: snapshot restore progress: id=2-1013-1774937854818 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:34.819Z [INFO] server-138684a6-3969-fa11-0ec2-b6ae697f77de: removing all old logs from log store -2026-03-31T06:17:34.819Z [INFO] server-138684a6-3969-fa11-0ec2-b6ae697f77de: compacting logs: from=1 to=2 -2026-03-31T06:17:34.819Z [INFO] server-138684a6-3969-fa11-0ec2-b6ae697f77de: Installed remote snapshot -2026-03-31T06:17:34.819Z [INFO] server-8b4ad946-840c-fba6-e77c-d74b2a44e3c5: pipelining replication: peer="{Voter server-138684a6-3969-fa11-0ec2-b6ae697f77de 138684a6-3969-fa11-0ec2-b6ae697f77de}" -2026-03-31T06:17:34.819Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:17:34.819Z [INFO] creating new snapshot: path=/tmp/raft1188761005/snapshots/2-1013-1774937854819.tmp -2026-03-31T06:17:34.819Z [INFO] server-27a15905-12a1-0071-aec6-5346e06f87f9: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:34.820Z [INFO] server-27a15905-12a1-0071-aec6-5346e06f87f9: copied to local snapshot: bytes=71 -2026-03-31T06:17:34.820Z [INFO] server-27a15905-12a1-0071-aec6-5346e06f87f9: snapshot restore progress: id=2-1013-1774937854819 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:34.820Z [INFO] server-27a15905-12a1-0071-aec6-5346e06f87f9: removing all old logs from log store -2026-03-31T06:17:34.820Z [INFO] server-27a15905-12a1-0071-aec6-5346e06f87f9: compacting logs: from=1 to=2 -2026-03-31T06:17:34.820Z [INFO] server-27a15905-12a1-0071-aec6-5346e06f87f9: Installed remote snapshot -2026-03-31T06:17:34.820Z [INFO] server-8b4ad946-840c-fba6-e77c-d74b2a44e3c5: pipelining replication: peer="{Voter server-27a15905-12a1-0071-aec6-5346e06f87f9 27a15905-12a1-0071-aec6-5346e06f87f9}" -2026-03-31T06:17:35.825Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:17:34.8197535 +0000 UTC m=+83.097360881 (2 nodes), 880ns from start of poll, 1.6254358s from cluster start. Timeout at 2026-03-31 06:17:35.8247813 +0000 UTC m=+84.102388681, 1.0050278s after stability -2026-03-31T06:17:35.841Z [INFO] server-06bf44c7-7190-e36e-350b-48fa5debf5b9: aborting pipeline replication: peer="{Voter server-63e6bab9-a3fb-f266-d4a2-b02f380b07d5 63e6bab9-a3fb-f266-d4a2-b02f380b07d5}" -2026-03-31T06:17:35.841Z [INFO] server-06bf44c7-7190-e36e-350b-48fa5debf5b9: aborting pipeline replication: peer="{Voter server-95d8e299-d28d-9fbc-2197-0a286385dd9c 95d8e299-d28d-9fbc-2197-0a286385dd9c}" +2025-02-25T23:58:53.638Z [INFO] server-30a660d8-4c34-9517-539d-3c846a44711c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-30a660d8-4c34-9517-539d-3c846a44711c Address:30a660d8-4c34-9517-539d-3c846a44711c} {Suffrage:Voter ID:server-b5dd7154-f004-fa55-22c4-338e9dd1d82c Address:b5dd7154-f004-fa55-22c4-338e9dd1d82c} {Suffrage:Voter ID:server-ecceb8e0-f663-0b65-981b-8c706be35dd0 Address:ecceb8e0-f663-0b65-981b-8c706be35dd0}]" +2025-02-25T23:58:53.638Z [INFO] server-b5dd7154-f004-fa55-22c4-338e9dd1d82c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-30a660d8-4c34-9517-539d-3c846a44711c Address:30a660d8-4c34-9517-539d-3c846a44711c} {Suffrage:Voter ID:server-b5dd7154-f004-fa55-22c4-338e9dd1d82c Address:b5dd7154-f004-fa55-22c4-338e9dd1d82c} {Suffrage:Voter ID:server-ecceb8e0-f663-0b65-981b-8c706be35dd0 Address:ecceb8e0-f663-0b65-981b-8c706be35dd0}]" +2025-02-25T23:58:53.638Z [INFO] server-ecceb8e0-f663-0b65-981b-8c706be35dd0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-30a660d8-4c34-9517-539d-3c846a44711c Address:30a660d8-4c34-9517-539d-3c846a44711c} {Suffrage:Voter ID:server-b5dd7154-f004-fa55-22c4-338e9dd1d82c Address:b5dd7154-f004-fa55-22c4-338e9dd1d82c} {Suffrage:Voter ID:server-ecceb8e0-f663-0b65-981b-8c706be35dd0 Address:ecceb8e0-f663-0b65-981b-8c706be35dd0}]" +2025-02-25T23:58:53.638Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:58:53.638Z [INFO] server-30a660d8-4c34-9517-539d-3c846a44711c: entering follower state: follower="Node at 30a660d8-4c34-9517-539d-3c846a44711c [Follower]" leader-address= leader-id= +2025-02-25T23:58:53.638Z [INFO] server-b5dd7154-f004-fa55-22c4-338e9dd1d82c: entering follower state: follower="Node at b5dd7154-f004-fa55-22c4-338e9dd1d82c [Follower]" leader-address= leader-id= +2025-02-25T23:58:53.638Z [INFO] server-ecceb8e0-f663-0b65-981b-8c706be35dd0: entering follower state: follower="Node at ecceb8e0-f663-0b65-981b-8c706be35dd0 [Follower]" leader-address= leader-id= +2025-02-25T23:58:54.142Z [WARN] server-b5dd7154-f004-fa55-22c4-338e9dd1d82c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:58:54.142Z [INFO] server-b5dd7154-f004-fa55-22c4-338e9dd1d82c: entering candidate state: node="Node at b5dd7154-f004-fa55-22c4-338e9dd1d82c [Candidate]" term=2 +2025-02-25T23:58:54.142Z [INFO] server-b5dd7154-f004-fa55-22c4-338e9dd1d82c: election won: term=2 tally=2 +2025-02-25T23:58:54.142Z [INFO] server-b5dd7154-f004-fa55-22c4-338e9dd1d82c: entering leader state: leader="Node at b5dd7154-f004-fa55-22c4-338e9dd1d82c [Leader]" +2025-02-25T23:58:54.142Z [INFO] server-b5dd7154-f004-fa55-22c4-338e9dd1d82c: added peer, starting replication: peer=server-30a660d8-4c34-9517-539d-3c846a44711c +2025-02-25T23:58:54.142Z [INFO] server-b5dd7154-f004-fa55-22c4-338e9dd1d82c: added peer, starting replication: peer=server-ecceb8e0-f663-0b65-981b-8c706be35dd0 +2025-02-25T23:58:54.142Z [INFO] server-b5dd7154-f004-fa55-22c4-338e9dd1d82c: pipelining replication: peer="{Voter server-ecceb8e0-f663-0b65-981b-8c706be35dd0 ecceb8e0-f663-0b65-981b-8c706be35dd0}" +2025-02-25T23:58:54.142Z [INFO] server-b5dd7154-f004-fa55-22c4-338e9dd1d82c: pipelining replication: peer="{Voter server-30a660d8-4c34-9517-539d-3c846a44711c 30a660d8-4c34-9517-539d-3c846a44711c}" +2025-02-25T23:58:55.153Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:58:54.142560839 +0000 UTC m=+81.751576246 (1 nodes), 503.910609ms from start of poll, 504.08849ms from cluster start. Timeout at 2025-02-25 23:58:55.147568639 +0000 UTC m=+82.756584046, 1.0050078s after stability +2025-02-25T23:58:55.154Z [INFO] server-b5dd7154-f004-fa55-22c4-338e9dd1d82c: starting snapshot up to: index=12 +2025-02-25T23:58:55.154Z [INFO] creating new snapshot: path=/tmp/raft2442645803/snapshots/2-12-1740527935154.tmp +2025-02-25T23:58:55.154Z [INFO] server-b5dd7154-f004-fa55-22c4-338e9dd1d82c: snapshot complete up to: index=12 +2025-02-25T23:58:55.155Z [INFO] server-b582e096-238d-c0d5-4e10-24d838d372e7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b582e096-238d-c0d5-4e10-24d838d372e7 Address:b582e096-238d-c0d5-4e10-24d838d372e7} {Suffrage:Voter ID:server-e49c13e6-e00e-5de1-e7ea-c431bcaec2c3 Address:e49c13e6-e00e-5de1-e7ea-c431bcaec2c3} {Suffrage:Voter ID:server-55f35811-1f9e-be5b-eb70-acd0231c4416 Address:55f35811-1f9e-be5b-eb70-acd0231c4416}]" +2025-02-25T23:58:55.155Z [INFO] server-e49c13e6-e00e-5de1-e7ea-c431bcaec2c3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b582e096-238d-c0d5-4e10-24d838d372e7 Address:b582e096-238d-c0d5-4e10-24d838d372e7} {Suffrage:Voter ID:server-e49c13e6-e00e-5de1-e7ea-c431bcaec2c3 Address:e49c13e6-e00e-5de1-e7ea-c431bcaec2c3} {Suffrage:Voter ID:server-55f35811-1f9e-be5b-eb70-acd0231c4416 Address:55f35811-1f9e-be5b-eb70-acd0231c4416}]" +2025-02-25T23:58:55.155Z [INFO] server-55f35811-1f9e-be5b-eb70-acd0231c4416: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b582e096-238d-c0d5-4e10-24d838d372e7 Address:b582e096-238d-c0d5-4e10-24d838d372e7} {Suffrage:Voter ID:server-e49c13e6-e00e-5de1-e7ea-c431bcaec2c3 Address:e49c13e6-e00e-5de1-e7ea-c431bcaec2c3} {Suffrage:Voter ID:server-55f35811-1f9e-be5b-eb70-acd0231c4416 Address:55f35811-1f9e-be5b-eb70-acd0231c4416}]" +2025-02-25T23:58:55.155Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:58:55.155Z [INFO] server-b582e096-238d-c0d5-4e10-24d838d372e7: entering follower state: follower="Node at b582e096-238d-c0d5-4e10-24d838d372e7 [Follower]" leader-address= leader-id= +2025-02-25T23:58:55.155Z [INFO] server-e49c13e6-e00e-5de1-e7ea-c431bcaec2c3: entering follower state: follower="Node at e49c13e6-e00e-5de1-e7ea-c431bcaec2c3 [Follower]" leader-address= leader-id= +2025-02-25T23:58:55.155Z [INFO] server-55f35811-1f9e-be5b-eb70-acd0231c4416: entering follower state: follower="Node at 55f35811-1f9e-be5b-eb70-acd0231c4416 [Follower]" leader-address= leader-id= +2025-02-25T23:58:55.869Z [WARN] server-e49c13e6-e00e-5de1-e7ea-c431bcaec2c3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:58:55.869Z [INFO] server-e49c13e6-e00e-5de1-e7ea-c431bcaec2c3: entering candidate state: node="Node at e49c13e6-e00e-5de1-e7ea-c431bcaec2c3 [Candidate]" term=2 +2025-02-25T23:58:55.869Z [INFO] server-e49c13e6-e00e-5de1-e7ea-c431bcaec2c3: election won: term=2 tally=2 +2025-02-25T23:58:55.869Z [INFO] server-e49c13e6-e00e-5de1-e7ea-c431bcaec2c3: entering leader state: leader="Node at e49c13e6-e00e-5de1-e7ea-c431bcaec2c3 [Leader]" +2025-02-25T23:58:55.870Z [INFO] server-e49c13e6-e00e-5de1-e7ea-c431bcaec2c3: added peer, starting replication: peer=server-b582e096-238d-c0d5-4e10-24d838d372e7 +2025-02-25T23:58:55.870Z [INFO] server-e49c13e6-e00e-5de1-e7ea-c431bcaec2c3: added peer, starting replication: peer=server-55f35811-1f9e-be5b-eb70-acd0231c4416 +2025-02-25T23:58:55.870Z [INFO] server-e49c13e6-e00e-5de1-e7ea-c431bcaec2c3: pipelining replication: peer="{Voter server-55f35811-1f9e-be5b-eb70-acd0231c4416 55f35811-1f9e-be5b-eb70-acd0231c4416}" +2025-02-25T23:58:55.870Z [INFO] server-e49c13e6-e00e-5de1-e7ea-c431bcaec2c3: pipelining replication: peer="{Voter server-b582e096-238d-c0d5-4e10-24d838d372e7 b582e096-238d-c0d5-4e10-24d838d372e7}" +2025-02-25T23:58:56.877Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:58:55.870193758 +0000 UTC m=+83.479209125 (1 nodes), 714.405148ms from start of poll, 714.569949ms from cluster start. Timeout at 2025-02-25 23:58:56.875209838 +0000 UTC m=+84.484225245, 1.00501612s after stability +2025-02-25T23:58:56.878Z [INFO] creating new snapshot: path=/tmp/raft2719651489/snapshots/2-1013-1740527936878.tmp +2025-02-25T23:58:56.878Z [INFO] server-e49c13e6-e00e-5de1-e7ea-c431bcaec2c3: copied to local snapshot: bytes=71 +2025-02-25T23:58:56.878Z [INFO] server-e49c13e6-e00e-5de1-e7ea-c431bcaec2c3: snapshot restore progress: id=2-1013-1740527936878 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:56.878Z [INFO] server-e49c13e6-e00e-5de1-e7ea-c431bcaec2c3: removing all old logs from log store +2025-02-25T23:58:56.878Z [INFO] server-e49c13e6-e00e-5de1-e7ea-c431bcaec2c3: compacting logs: from=1 to=2 +2025-02-25T23:58:56.878Z [INFO] server-e49c13e6-e00e-5de1-e7ea-c431bcaec2c3: restored user snapshot: index=1013 +2025-02-25T23:58:56.878Z [ERROR] server-e49c13e6-e00e-5de1-e7ea-c431bcaec2c3: failed to get log: index=2 error="log not found" +2025-02-25T23:58:56.878Z [INFO] server-e49c13e6-e00e-5de1-e7ea-c431bcaec2c3: aborting pipeline replication: peer="{Voter server-b582e096-238d-c0d5-4e10-24d838d372e7 b582e096-238d-c0d5-4e10-24d838d372e7}" +2025-02-25T23:58:56.878Z [ERROR] server-e49c13e6-e00e-5de1-e7ea-c431bcaec2c3: failed to get log: index=2 error="log not found" +2025-02-25T23:58:56.878Z [INFO] server-e49c13e6-e00e-5de1-e7ea-c431bcaec2c3: aborting pipeline replication: peer="{Voter server-55f35811-1f9e-be5b-eb70-acd0231c4416 55f35811-1f9e-be5b-eb70-acd0231c4416}" +2025-02-25T23:58:56.878Z [ERROR] server-e49c13e6-e00e-5de1-e7ea-c431bcaec2c3: failed to get log: index=2 error="log not found" +2025-02-25T23:58:56.886Z [INFO] creating new snapshot: path=/tmp/raft3073409587/snapshots/2-1013-1740527936886.tmp +2025-02-25T23:58:56.887Z [INFO] server-55f35811-1f9e-be5b-eb70-acd0231c4416: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:56.887Z [INFO] server-55f35811-1f9e-be5b-eb70-acd0231c4416: copied to local snapshot: bytes=71 +2025-02-25T23:58:56.887Z [INFO] server-55f35811-1f9e-be5b-eb70-acd0231c4416: snapshot restore progress: id=2-1013-1740527936886 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:56.887Z [INFO] server-55f35811-1f9e-be5b-eb70-acd0231c4416: removing all old logs from log store +2025-02-25T23:58:56.887Z [INFO] server-55f35811-1f9e-be5b-eb70-acd0231c4416: compacting logs: from=1 to=2 +2025-02-25T23:58:56.887Z [INFO] server-55f35811-1f9e-be5b-eb70-acd0231c4416: Installed remote snapshot +2025-02-25T23:58:56.887Z [INFO] server-e49c13e6-e00e-5de1-e7ea-c431bcaec2c3: pipelining replication: peer="{Voter server-55f35811-1f9e-be5b-eb70-acd0231c4416 55f35811-1f9e-be5b-eb70-acd0231c4416}" +2025-02-25T23:58:56.887Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:58:56.893Z [ERROR] server-e49c13e6-e00e-5de1-e7ea-c431bcaec2c3: failed to get log: index=2 error="log not found" +2025-02-25T23:58:56.894Z [INFO] creating new snapshot: path=/tmp/raft2854794220/snapshots/2-1013-1740527936894.tmp +2025-02-25T23:58:56.894Z [INFO] server-b582e096-238d-c0d5-4e10-24d838d372e7: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:56.894Z [INFO] server-b582e096-238d-c0d5-4e10-24d838d372e7: copied to local snapshot: bytes=71 +2025-02-25T23:58:56.894Z [INFO] server-b582e096-238d-c0d5-4e10-24d838d372e7: snapshot restore progress: id=2-1013-1740527936894 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:56.894Z [INFO] server-b582e096-238d-c0d5-4e10-24d838d372e7: removing all old logs from log store +2025-02-25T23:58:56.894Z [INFO] server-b582e096-238d-c0d5-4e10-24d838d372e7: compacting logs: from=1 to=2 +2025-02-25T23:58:56.894Z [INFO] server-b582e096-238d-c0d5-4e10-24d838d372e7: Installed remote snapshot +2025-02-25T23:58:56.894Z [INFO] server-e49c13e6-e00e-5de1-e7ea-c431bcaec2c3: pipelining replication: peer="{Voter server-b582e096-238d-c0d5-4e10-24d838d372e7 b582e096-238d-c0d5-4e10-24d838d372e7}" +2025-02-25T23:58:57.894Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:58:56.887552876 +0000 UTC m=+84.496568283 (2 nodes), 600ns from start of poll, 1.731929107s from cluster start. Timeout at 2025-02-25 23:58:57.892589956 +0000 UTC m=+85.501605323, 1.00503704s after stability +2025-02-25T23:58:57.918Z [INFO] server-b5dd7154-f004-fa55-22c4-338e9dd1d82c: aborting pipeline replication: peer="{Voter server-ecceb8e0-f663-0b65-981b-8c706be35dd0 ecceb8e0-f663-0b65-981b-8c706be35dd0}" +2025-02-25T23:58:57.918Z [INFO] server-b5dd7154-f004-fa55-22c4-338e9dd1d82c: aborting pipeline replication: peer="{Voter server-30a660d8-4c34-9517-539d-3c846a44711c 30a660d8-4c34-9517-539d-3c846a44711c}" === RUN TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_false -2026-03-31T06:17:35.842Z [INFO] server-662a5961-f839-31f5-77fa-5808ccf8d8c3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-662a5961-f839-31f5-77fa-5808ccf8d8c3 Address:662a5961-f839-31f5-77fa-5808ccf8d8c3} {Suffrage:Voter ID:server-9d54cb97-cfc0-9b3a-c60d-edc4f7b5d6c8 Address:9d54cb97-cfc0-9b3a-c60d-edc4f7b5d6c8} {Suffrage:Voter ID:server-6e8797f1-8dd7-a675-2e1f-c01a4e03d5fb Address:6e8797f1-8dd7-a675-2e1f-c01a4e03d5fb}]" -2026-03-31T06:17:35.842Z [INFO] server-9d54cb97-cfc0-9b3a-c60d-edc4f7b5d6c8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-662a5961-f839-31f5-77fa-5808ccf8d8c3 Address:662a5961-f839-31f5-77fa-5808ccf8d8c3} {Suffrage:Voter ID:server-9d54cb97-cfc0-9b3a-c60d-edc4f7b5d6c8 Address:9d54cb97-cfc0-9b3a-c60d-edc4f7b5d6c8} {Suffrage:Voter ID:server-6e8797f1-8dd7-a675-2e1f-c01a4e03d5fb Address:6e8797f1-8dd7-a675-2e1f-c01a4e03d5fb}]" -2026-03-31T06:17:35.842Z [INFO] server-662a5961-f839-31f5-77fa-5808ccf8d8c3: entering follower state: follower="Node at 662a5961-f839-31f5-77fa-5808ccf8d8c3 [Follower]" leader-address= leader-id= -2026-03-31T06:17:35.842Z [INFO] server-9d54cb97-cfc0-9b3a-c60d-edc4f7b5d6c8: entering follower state: follower="Node at 9d54cb97-cfc0-9b3a-c60d-edc4f7b5d6c8 [Follower]" leader-address= leader-id= -2026-03-31T06:17:35.842Z [INFO] server-6e8797f1-8dd7-a675-2e1f-c01a4e03d5fb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-662a5961-f839-31f5-77fa-5808ccf8d8c3 Address:662a5961-f839-31f5-77fa-5808ccf8d8c3} {Suffrage:Voter ID:server-9d54cb97-cfc0-9b3a-c60d-edc4f7b5d6c8 Address:9d54cb97-cfc0-9b3a-c60d-edc4f7b5d6c8} {Suffrage:Voter ID:server-6e8797f1-8dd7-a675-2e1f-c01a4e03d5fb Address:6e8797f1-8dd7-a675-2e1f-c01a4e03d5fb}]" -2026-03-31T06:17:35.842Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:35.842Z [INFO] server-6e8797f1-8dd7-a675-2e1f-c01a4e03d5fb: entering follower state: follower="Node at 6e8797f1-8dd7-a675-2e1f-c01a4e03d5fb [Follower]" leader-address= leader-id= -2026-03-31T06:17:36.347Z [WARN] server-9d54cb97-cfc0-9b3a-c60d-edc4f7b5d6c8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:36.347Z [INFO] server-9d54cb97-cfc0-9b3a-c60d-edc4f7b5d6c8: entering candidate state: node="Node at 9d54cb97-cfc0-9b3a-c60d-edc4f7b5d6c8 [Candidate]" term=2 -2026-03-31T06:17:36.347Z [INFO] server-9d54cb97-cfc0-9b3a-c60d-edc4f7b5d6c8: election won: term=2 tally=2 -2026-03-31T06:17:36.347Z [INFO] server-9d54cb97-cfc0-9b3a-c60d-edc4f7b5d6c8: entering leader state: leader="Node at 9d54cb97-cfc0-9b3a-c60d-edc4f7b5d6c8 [Leader]" -2026-03-31T06:17:36.347Z [INFO] server-9d54cb97-cfc0-9b3a-c60d-edc4f7b5d6c8: added peer, starting replication: peer=server-662a5961-f839-31f5-77fa-5808ccf8d8c3 -2026-03-31T06:17:36.347Z [INFO] server-9d54cb97-cfc0-9b3a-c60d-edc4f7b5d6c8: added peer, starting replication: peer=server-6e8797f1-8dd7-a675-2e1f-c01a4e03d5fb -2026-03-31T06:17:36.347Z [INFO] server-9d54cb97-cfc0-9b3a-c60d-edc4f7b5d6c8: pipelining replication: peer="{Voter server-6e8797f1-8dd7-a675-2e1f-c01a4e03d5fb 6e8797f1-8dd7-a675-2e1f-c01a4e03d5fb}" -2026-03-31T06:17:36.347Z [INFO] server-9d54cb97-cfc0-9b3a-c60d-edc4f7b5d6c8: pipelining replication: peer="{Voter server-662a5961-f839-31f5-77fa-5808ccf8d8c3 662a5961-f839-31f5-77fa-5808ccf8d8c3}" -2026-03-31T06:17:37.352Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:36.34744726 +0000 UTC m=+84.625054681 (1 nodes), 504.62996ms from start of poll, 505.04ms from cluster start. Timeout at 2026-03-31 06:17:37.3524549 +0000 UTC m=+85.630062281, 1.0050076s after stability -2026-03-31T06:17:37.353Z [INFO] server-9d54cb97-cfc0-9b3a-c60d-edc4f7b5d6c8: starting snapshot up to: index=12 -2026-03-31T06:17:37.353Z [INFO] creating new snapshot: path=/tmp/raft392822834/snapshots/2-12-1774937857353.tmp -2026-03-31T06:17:37.353Z [INFO] server-9d54cb97-cfc0-9b3a-c60d-edc4f7b5d6c8: snapshot complete up to: index=12 -2026-03-31T06:17:37.354Z [INFO] creating new snapshot: path=/tmp/raft392822834/snapshots/2-10013-1774937857354.tmp -2026-03-31T06:17:37.355Z [INFO] server-9d54cb97-cfc0-9b3a-c60d-edc4f7b5d6c8: copied to local snapshot: bytes=71 -2026-03-31T06:17:37.355Z [INFO] server-9d54cb97-cfc0-9b3a-c60d-edc4f7b5d6c8: snapshot restore progress: id=2-10013-1774937857354 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:37.355Z [INFO] server-9d54cb97-cfc0-9b3a-c60d-edc4f7b5d6c8: restored user snapshot: index=10013 -2026-03-31T06:17:37.355Z [ERROR] server-9d54cb97-cfc0-9b3a-c60d-edc4f7b5d6c8: failed to get log: index=23 error="log not found" -2026-03-31T06:17:37.355Z [ERROR] server-9d54cb97-cfc0-9b3a-c60d-edc4f7b5d6c8: failed to get log: index=23 error="log not found" -2026-03-31T06:17:37.355Z [INFO] server-9d54cb97-cfc0-9b3a-c60d-edc4f7b5d6c8: aborting pipeline replication: peer="{Voter server-6e8797f1-8dd7-a675-2e1f-c01a4e03d5fb 6e8797f1-8dd7-a675-2e1f-c01a4e03d5fb}" -2026-03-31T06:17:37.355Z [INFO] server-9d54cb97-cfc0-9b3a-c60d-edc4f7b5d6c8: aborting pipeline replication: peer="{Voter server-662a5961-f839-31f5-77fa-5808ccf8d8c3 662a5961-f839-31f5-77fa-5808ccf8d8c3}" -2026-03-31T06:17:37.362Z [ERROR] server-9d54cb97-cfc0-9b3a-c60d-edc4f7b5d6c8: failed to get log: index=23 error="log not found" -2026-03-31T06:17:37.362Z [INFO] creating new snapshot: path=/tmp/raft2564497719/snapshots/2-10013-1774937857362.tmp -2026-03-31T06:17:37.362Z [INFO] server-662a5961-f839-31f5-77fa-5808ccf8d8c3: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:37.363Z [INFO] server-662a5961-f839-31f5-77fa-5808ccf8d8c3: copied to local snapshot: bytes=71 -2026-03-31T06:17:37.363Z [ERROR] server-9d54cb97-cfc0-9b3a-c60d-edc4f7b5d6c8: failed to get log: index=23 error="log not found" -2026-03-31T06:17:37.363Z [INFO] server-662a5961-f839-31f5-77fa-5808ccf8d8c3: snapshot restore progress: id=2-10013-1774937857362 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:37.363Z [INFO] server-662a5961-f839-31f5-77fa-5808ccf8d8c3: Installed remote snapshot -2026-03-31T06:17:37.363Z [WARN] server-662a5961-f839-31f5-77fa-5808ccf8d8c3: skipping application of old log: index=22 -2026-03-31T06:17:37.363Z [INFO] server-9d54cb97-cfc0-9b3a-c60d-edc4f7b5d6c8: pipelining replication: peer="{Voter server-662a5961-f839-31f5-77fa-5808ccf8d8c3 662a5961-f839-31f5-77fa-5808ccf8d8c3}" -2026-03-31T06:17:37.363Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:17:37.363Z [INFO] creating new snapshot: path=/tmp/raft288657270/snapshots/2-10013-1774937857363.tmp -2026-03-31T06:17:37.364Z [INFO] server-6e8797f1-8dd7-a675-2e1f-c01a4e03d5fb: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:37.364Z [INFO] server-6e8797f1-8dd7-a675-2e1f-c01a4e03d5fb: copied to local snapshot: bytes=71 -2026-03-31T06:17:37.364Z [INFO] server-6e8797f1-8dd7-a675-2e1f-c01a4e03d5fb: snapshot restore progress: id=2-10013-1774937857363 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:37.364Z [INFO] server-6e8797f1-8dd7-a675-2e1f-c01a4e03d5fb: Installed remote snapshot -2026-03-31T06:17:37.364Z [INFO] server-9d54cb97-cfc0-9b3a-c60d-edc4f7b5d6c8: pipelining replication: peer="{Voter server-6e8797f1-8dd7-a675-2e1f-c01a4e03d5fb 6e8797f1-8dd7-a675-2e1f-c01a4e03d5fb}" -2026-03-31T06:17:38.369Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:17:37.36378762 +0000 UTC m=+85.641395001 (2 nodes), 880ns from start of poll, 1.52138032s from cluster start. Timeout at 2026-03-31 06:17:38.36881086 +0000 UTC m=+86.646418241, 1.00502324s after stability -2026-03-31T06:17:38.380Z [INFO] server-9d54cb97-cfc0-9b3a-c60d-edc4f7b5d6c8: aborting pipeline replication: peer="{Voter server-662a5961-f839-31f5-77fa-5808ccf8d8c3 662a5961-f839-31f5-77fa-5808ccf8d8c3}" -2026-03-31T06:17:38.380Z [INFO] server-9d54cb97-cfc0-9b3a-c60d-edc4f7b5d6c8: aborting pipeline replication: peer="{Voter server-6e8797f1-8dd7-a675-2e1f-c01a4e03d5fb 6e8797f1-8dd7-a675-2e1f-c01a4e03d5fb}" +2025-02-25T23:58:57.918Z [INFO] server-8dd59ebd-701d-46ab-7560-a9e320179f5a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8dd59ebd-701d-46ab-7560-a9e320179f5a Address:8dd59ebd-701d-46ab-7560-a9e320179f5a} {Suffrage:Voter ID:server-f0c190be-c7b3-7fcc-002e-0ebc39c7caec Address:f0c190be-c7b3-7fcc-002e-0ebc39c7caec} {Suffrage:Voter ID:server-996df080-691b-df08-d640-3ebbcc2a76fc Address:996df080-691b-df08-d640-3ebbcc2a76fc}]" +2025-02-25T23:58:57.918Z [INFO] server-f0c190be-c7b3-7fcc-002e-0ebc39c7caec: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8dd59ebd-701d-46ab-7560-a9e320179f5a Address:8dd59ebd-701d-46ab-7560-a9e320179f5a} {Suffrage:Voter ID:server-f0c190be-c7b3-7fcc-002e-0ebc39c7caec Address:f0c190be-c7b3-7fcc-002e-0ebc39c7caec} {Suffrage:Voter ID:server-996df080-691b-df08-d640-3ebbcc2a76fc Address:996df080-691b-df08-d640-3ebbcc2a76fc}]" +2025-02-25T23:58:57.918Z [INFO] server-996df080-691b-df08-d640-3ebbcc2a76fc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8dd59ebd-701d-46ab-7560-a9e320179f5a Address:8dd59ebd-701d-46ab-7560-a9e320179f5a} {Suffrage:Voter ID:server-f0c190be-c7b3-7fcc-002e-0ebc39c7caec Address:f0c190be-c7b3-7fcc-002e-0ebc39c7caec} {Suffrage:Voter ID:server-996df080-691b-df08-d640-3ebbcc2a76fc Address:996df080-691b-df08-d640-3ebbcc2a76fc}]" +2025-02-25T23:58:57.918Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:58:57.918Z [INFO] server-8dd59ebd-701d-46ab-7560-a9e320179f5a: entering follower state: follower="Node at 8dd59ebd-701d-46ab-7560-a9e320179f5a [Follower]" leader-address= leader-id= +2025-02-25T23:58:57.918Z [INFO] server-f0c190be-c7b3-7fcc-002e-0ebc39c7caec: entering follower state: follower="Node at f0c190be-c7b3-7fcc-002e-0ebc39c7caec [Follower]" leader-address= leader-id= +2025-02-25T23:58:57.918Z [INFO] server-996df080-691b-df08-d640-3ebbcc2a76fc: entering follower state: follower="Node at 996df080-691b-df08-d640-3ebbcc2a76fc [Follower]" leader-address= leader-id= +2025-02-25T23:58:58.646Z [WARN] server-8dd59ebd-701d-46ab-7560-a9e320179f5a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:58:58.646Z [INFO] server-8dd59ebd-701d-46ab-7560-a9e320179f5a: entering candidate state: node="Node at 8dd59ebd-701d-46ab-7560-a9e320179f5a [Candidate]" term=2 +2025-02-25T23:58:58.646Z [INFO] server-8dd59ebd-701d-46ab-7560-a9e320179f5a: election won: term=2 tally=2 +2025-02-25T23:58:58.646Z [INFO] server-8dd59ebd-701d-46ab-7560-a9e320179f5a: entering leader state: leader="Node at 8dd59ebd-701d-46ab-7560-a9e320179f5a [Leader]" +2025-02-25T23:58:58.646Z [INFO] server-8dd59ebd-701d-46ab-7560-a9e320179f5a: added peer, starting replication: peer=server-f0c190be-c7b3-7fcc-002e-0ebc39c7caec +2025-02-25T23:58:58.646Z [INFO] server-8dd59ebd-701d-46ab-7560-a9e320179f5a: added peer, starting replication: peer=server-996df080-691b-df08-d640-3ebbcc2a76fc +2025-02-25T23:58:58.646Z [INFO] server-8dd59ebd-701d-46ab-7560-a9e320179f5a: pipelining replication: peer="{Voter server-f0c190be-c7b3-7fcc-002e-0ebc39c7caec f0c190be-c7b3-7fcc-002e-0ebc39c7caec}" +2025-02-25T23:58:58.646Z [INFO] server-8dd59ebd-701d-46ab-7560-a9e320179f5a: pipelining replication: peer="{Voter server-996df080-691b-df08-d640-3ebbcc2a76fc 996df080-691b-df08-d640-3ebbcc2a76fc}" +2025-02-25T23:58:59.654Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:58:58.646731356 +0000 UTC m=+86.255746763 (1 nodes), 728.089676ms from start of poll, 728.258477ms from cluster start. Timeout at 2025-02-25 23:58:59.651737996 +0000 UTC m=+87.260753363, 1.0050066s after stability +2025-02-25T23:58:59.654Z [INFO] server-8dd59ebd-701d-46ab-7560-a9e320179f5a: starting snapshot up to: index=12 +2025-02-25T23:58:59.654Z [INFO] creating new snapshot: path=/tmp/raft2610584988/snapshots/2-12-1740527939654.tmp +2025-02-25T23:58:59.655Z [INFO] server-8dd59ebd-701d-46ab-7560-a9e320179f5a: snapshot complete up to: index=12 +2025-02-25T23:58:59.655Z [INFO] creating new snapshot: path=/tmp/raft2610584988/snapshots/2-10013-1740527939655.tmp +2025-02-25T23:58:59.655Z [INFO] server-8dd59ebd-701d-46ab-7560-a9e320179f5a: copied to local snapshot: bytes=71 +2025-02-25T23:58:59.655Z [INFO] server-8dd59ebd-701d-46ab-7560-a9e320179f5a: snapshot restore progress: id=2-10013-1740527939655 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:59.656Z [INFO] server-8dd59ebd-701d-46ab-7560-a9e320179f5a: restored user snapshot: index=10013 +2025-02-25T23:58:59.656Z [ERROR] server-8dd59ebd-701d-46ab-7560-a9e320179f5a: failed to get log: index=23 error="log not found" +2025-02-25T23:58:59.656Z [INFO] server-8dd59ebd-701d-46ab-7560-a9e320179f5a: aborting pipeline replication: peer="{Voter server-f0c190be-c7b3-7fcc-002e-0ebc39c7caec f0c190be-c7b3-7fcc-002e-0ebc39c7caec}" +2025-02-25T23:58:59.656Z [ERROR] server-8dd59ebd-701d-46ab-7560-a9e320179f5a: failed to get log: index=23 error="log not found" +2025-02-25T23:58:59.656Z [INFO] server-8dd59ebd-701d-46ab-7560-a9e320179f5a: aborting pipeline replication: peer="{Voter server-996df080-691b-df08-d640-3ebbcc2a76fc 996df080-691b-df08-d640-3ebbcc2a76fc}" +2025-02-25T23:58:59.661Z [ERROR] server-8dd59ebd-701d-46ab-7560-a9e320179f5a: failed to get log: index=23 error="log not found" +2025-02-25T23:58:59.662Z [INFO] creating new snapshot: path=/tmp/raft4079384129/snapshots/2-10013-1740527939662.tmp +2025-02-25T23:58:59.668Z [INFO] server-f0c190be-c7b3-7fcc-002e-0ebc39c7caec: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:59.668Z [INFO] server-f0c190be-c7b3-7fcc-002e-0ebc39c7caec: copied to local snapshot: bytes=71 +2025-02-25T23:58:59.668Z [INFO] server-f0c190be-c7b3-7fcc-002e-0ebc39c7caec: snapshot restore progress: id=2-10013-1740527939662 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:59.668Z [INFO] server-f0c190be-c7b3-7fcc-002e-0ebc39c7caec: Installed remote snapshot +2025-02-25T23:58:59.668Z [WARN] server-f0c190be-c7b3-7fcc-002e-0ebc39c7caec: skipping application of old log: index=22 +2025-02-25T23:58:59.668Z [INFO] server-8dd59ebd-701d-46ab-7560-a9e320179f5a: pipelining replication: peer="{Voter server-f0c190be-c7b3-7fcc-002e-0ebc39c7caec f0c190be-c7b3-7fcc-002e-0ebc39c7caec}" +2025-02-25T23:58:59.668Z [ERROR] server-8dd59ebd-701d-46ab-7560-a9e320179f5a: failed to get log: index=23 error="log not found" +2025-02-25T23:58:59.669Z [INFO] creating new snapshot: path=/tmp/raft2139315136/snapshots/2-10013-1740527939669.tmp +2025-02-25T23:58:59.669Z [INFO] server-996df080-691b-df08-d640-3ebbcc2a76fc: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:59.669Z [INFO] server-996df080-691b-df08-d640-3ebbcc2a76fc: copied to local snapshot: bytes=71 +2025-02-25T23:58:59.669Z [INFO] server-996df080-691b-df08-d640-3ebbcc2a76fc: snapshot restore progress: id=2-10013-1740527939669 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:58:59.669Z [INFO] server-996df080-691b-df08-d640-3ebbcc2a76fc: Installed remote snapshot +2025-02-25T23:58:59.669Z [WARN] server-996df080-691b-df08-d640-3ebbcc2a76fc: skipping application of old log: index=22 +2025-02-25T23:58:59.669Z [INFO] server-8dd59ebd-701d-46ab-7560-a9e320179f5a: pipelining replication: peer="{Voter server-996df080-691b-df08-d640-3ebbcc2a76fc 996df080-691b-df08-d640-3ebbcc2a76fc}" +2025-02-25T23:58:59.669Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:59:00.678Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:58:59.669467987 +0000 UTC m=+87.278483394 (2 nodes), 400ns from start of poll, 1.750995108s from cluster start. Timeout at 2025-02-25 23:59:00.674476507 +0000 UTC m=+88.283491874, 1.00500848s after stability +2025-02-25T23:59:00.694Z [INFO] server-8dd59ebd-701d-46ab-7560-a9e320179f5a: aborting pipeline replication: peer="{Voter server-f0c190be-c7b3-7fcc-002e-0ebc39c7caec f0c190be-c7b3-7fcc-002e-0ebc39c7caec}" +2025-02-25T23:59:00.694Z [INFO] server-8dd59ebd-701d-46ab-7560-a9e320179f5a: aborting pipeline replication: peer="{Voter server-996df080-691b-df08-d640-3ebbcc2a76fc 996df080-691b-df08-d640-3ebbcc2a76fc}" === RUN TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_false -2026-03-31T06:17:38.381Z [INFO] server-070fdaaf-c568-b25c-5836-a8617e1a27c5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-070fdaaf-c568-b25c-5836-a8617e1a27c5 Address:070fdaaf-c568-b25c-5836-a8617e1a27c5} {Suffrage:Voter ID:server-e77d88a3-122c-4fd3-0110-6d5866fcf60a Address:e77d88a3-122c-4fd3-0110-6d5866fcf60a} {Suffrage:Voter ID:server-38d1d610-9da4-b706-e060-2a2bd571957f Address:38d1d610-9da4-b706-e060-2a2bd571957f}]" -2026-03-31T06:17:38.381Z [INFO] server-e77d88a3-122c-4fd3-0110-6d5866fcf60a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-070fdaaf-c568-b25c-5836-a8617e1a27c5 Address:070fdaaf-c568-b25c-5836-a8617e1a27c5} {Suffrage:Voter ID:server-e77d88a3-122c-4fd3-0110-6d5866fcf60a Address:e77d88a3-122c-4fd3-0110-6d5866fcf60a} {Suffrage:Voter ID:server-38d1d610-9da4-b706-e060-2a2bd571957f Address:38d1d610-9da4-b706-e060-2a2bd571957f}]" -2026-03-31T06:17:38.381Z [INFO] server-070fdaaf-c568-b25c-5836-a8617e1a27c5: entering follower state: follower="Node at 070fdaaf-c568-b25c-5836-a8617e1a27c5 [Follower]" leader-address= leader-id= -2026-03-31T06:17:38.381Z [INFO] server-38d1d610-9da4-b706-e060-2a2bd571957f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-070fdaaf-c568-b25c-5836-a8617e1a27c5 Address:070fdaaf-c568-b25c-5836-a8617e1a27c5} {Suffrage:Voter ID:server-e77d88a3-122c-4fd3-0110-6d5866fcf60a Address:e77d88a3-122c-4fd3-0110-6d5866fcf60a} {Suffrage:Voter ID:server-38d1d610-9da4-b706-e060-2a2bd571957f Address:38d1d610-9da4-b706-e060-2a2bd571957f}]" -2026-03-31T06:17:38.381Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:38.381Z [INFO] server-e77d88a3-122c-4fd3-0110-6d5866fcf60a: entering follower state: follower="Node at e77d88a3-122c-4fd3-0110-6d5866fcf60a [Follower]" leader-address= leader-id= -2026-03-31T06:17:38.381Z [INFO] server-38d1d610-9da4-b706-e060-2a2bd571957f: entering follower state: follower="Node at 38d1d610-9da4-b706-e060-2a2bd571957f [Follower]" leader-address= leader-id= -2026-03-31T06:17:39.089Z [WARN] server-38d1d610-9da4-b706-e060-2a2bd571957f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:39.089Z [INFO] server-38d1d610-9da4-b706-e060-2a2bd571957f: entering candidate state: node="Node at 38d1d610-9da4-b706-e060-2a2bd571957f [Candidate]" term=2 -2026-03-31T06:17:39.089Z [INFO] server-38d1d610-9da4-b706-e060-2a2bd571957f: election won: term=2 tally=2 -2026-03-31T06:17:39.089Z [INFO] server-38d1d610-9da4-b706-e060-2a2bd571957f: entering leader state: leader="Node at 38d1d610-9da4-b706-e060-2a2bd571957f [Leader]" -2026-03-31T06:17:39.089Z [INFO] server-38d1d610-9da4-b706-e060-2a2bd571957f: added peer, starting replication: peer=server-070fdaaf-c568-b25c-5836-a8617e1a27c5 -2026-03-31T06:17:39.089Z [INFO] server-38d1d610-9da4-b706-e060-2a2bd571957f: added peer, starting replication: peer=server-e77d88a3-122c-4fd3-0110-6d5866fcf60a -2026-03-31T06:17:39.089Z [INFO] server-38d1d610-9da4-b706-e060-2a2bd571957f: pipelining replication: peer="{Voter server-070fdaaf-c568-b25c-5836-a8617e1a27c5 070fdaaf-c568-b25c-5836-a8617e1a27c5}" -2026-03-31T06:17:39.089Z [INFO] server-38d1d610-9da4-b706-e060-2a2bd571957f: pipelining replication: peer="{Voter server-e77d88a3-122c-4fd3-0110-6d5866fcf60a e77d88a3-122c-4fd3-0110-6d5866fcf60a}" -2026-03-31T06:17:40.095Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:39.0894793 +0000 UTC m=+87.367086681 (1 nodes), 707.68024ms from start of poll, 708.14324ms from cluster start. Timeout at 2026-03-31 06:17:40.09448262 +0000 UTC m=+88.372090001, 1.00500332s after stability -2026-03-31T06:17:40.095Z [INFO] server-38d1d610-9da4-b706-e060-2a2bd571957f: starting snapshot up to: index=12 -2026-03-31T06:17:40.095Z [INFO] creating new snapshot: path=/tmp/raft3738045731/snapshots/2-12-1774937860095.tmp -2026-03-31T06:17:40.096Z [INFO] server-38d1d610-9da4-b706-e060-2a2bd571957f: snapshot complete up to: index=12 -2026-03-31T06:17:40.097Z [INFO] creating new snapshot: path=/tmp/raft3738045731/snapshots/2-10013-1774937860097.tmp -2026-03-31T06:17:40.098Z [INFO] server-38d1d610-9da4-b706-e060-2a2bd571957f: copied to local snapshot: bytes=71 -2026-03-31T06:17:40.099Z [INFO] server-38d1d610-9da4-b706-e060-2a2bd571957f: snapshot restore progress: id=2-10013-1774937860097 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:40.099Z [INFO] server-38d1d610-9da4-b706-e060-2a2bd571957f: removing all old logs from log store -2026-03-31T06:17:40.099Z [INFO] server-38d1d610-9da4-b706-e060-2a2bd571957f: compacting logs: from=1 to=22 -2026-03-31T06:17:40.099Z [INFO] server-38d1d610-9da4-b706-e060-2a2bd571957f: restored user snapshot: index=10013 -2026-03-31T06:17:40.099Z [ERROR] server-38d1d610-9da4-b706-e060-2a2bd571957f: failed to get log: index=22 error="log not found" -2026-03-31T06:17:40.099Z [INFO] server-38d1d610-9da4-b706-e060-2a2bd571957f: aborting pipeline replication: peer="{Voter server-e77d88a3-122c-4fd3-0110-6d5866fcf60a e77d88a3-122c-4fd3-0110-6d5866fcf60a}" -2026-03-31T06:17:40.099Z [ERROR] server-38d1d610-9da4-b706-e060-2a2bd571957f: failed to get log: index=22 error="log not found" -2026-03-31T06:17:40.099Z [INFO] server-38d1d610-9da4-b706-e060-2a2bd571957f: aborting pipeline replication: peer="{Voter server-070fdaaf-c568-b25c-5836-a8617e1a27c5 070fdaaf-c568-b25c-5836-a8617e1a27c5}" -2026-03-31T06:17:40.106Z [ERROR] server-38d1d610-9da4-b706-e060-2a2bd571957f: failed to get log: index=22 error="log not found" -2026-03-31T06:17:40.106Z [INFO] creating new snapshot: path=/tmp/raft188702885/snapshots/2-10013-1774937860106.tmp -2026-03-31T06:17:40.107Z [INFO] server-e77d88a3-122c-4fd3-0110-6d5866fcf60a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:40.107Z [INFO] server-e77d88a3-122c-4fd3-0110-6d5866fcf60a: copied to local snapshot: bytes=71 -2026-03-31T06:17:40.107Z [INFO] server-e77d88a3-122c-4fd3-0110-6d5866fcf60a: snapshot restore progress: id=2-10013-1774937860106 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:40.107Z [INFO] server-e77d88a3-122c-4fd3-0110-6d5866fcf60a: removing all old logs from log store -2026-03-31T06:17:40.107Z [INFO] server-e77d88a3-122c-4fd3-0110-6d5866fcf60a: compacting logs: from=1 to=22 -2026-03-31T06:17:40.107Z [INFO] server-e77d88a3-122c-4fd3-0110-6d5866fcf60a: Installed remote snapshot -2026-03-31T06:17:40.107Z [WARN] server-e77d88a3-122c-4fd3-0110-6d5866fcf60a: skipping application of old log: index=22 -2026-03-31T06:17:40.107Z [INFO] server-38d1d610-9da4-b706-e060-2a2bd571957f: pipelining replication: peer="{Voter server-e77d88a3-122c-4fd3-0110-6d5866fcf60a e77d88a3-122c-4fd3-0110-6d5866fcf60a}" -2026-03-31T06:17:40.107Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:17:40.109Z [ERROR] server-38d1d610-9da4-b706-e060-2a2bd571957f: failed to get log: index=22 error="log not found" -2026-03-31T06:17:40.110Z [INFO] creating new snapshot: path=/tmp/raft1808133049/snapshots/2-10013-1774937860110.tmp -2026-03-31T06:17:40.110Z [INFO] server-070fdaaf-c568-b25c-5836-a8617e1a27c5: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:40.110Z [INFO] server-070fdaaf-c568-b25c-5836-a8617e1a27c5: copied to local snapshot: bytes=71 -2026-03-31T06:17:40.110Z [INFO] server-070fdaaf-c568-b25c-5836-a8617e1a27c5: snapshot restore progress: id=2-10013-1774937860110 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:40.110Z [INFO] server-070fdaaf-c568-b25c-5836-a8617e1a27c5: removing all old logs from log store -2026-03-31T06:17:40.110Z [INFO] server-070fdaaf-c568-b25c-5836-a8617e1a27c5: compacting logs: from=1 to=22 -2026-03-31T06:17:40.110Z [INFO] server-070fdaaf-c568-b25c-5836-a8617e1a27c5: Installed remote snapshot -2026-03-31T06:17:40.111Z [INFO] server-38d1d610-9da4-b706-e060-2a2bd571957f: pipelining replication: peer="{Voter server-070fdaaf-c568-b25c-5836-a8617e1a27c5 070fdaaf-c568-b25c-5836-a8617e1a27c5}" -2026-03-31T06:17:41.113Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:17:40.10786374 +0000 UTC m=+88.385471121 (2 nodes), 840ns from start of poll, 1.72652768s from cluster start. Timeout at 2026-03-31 06:17:41.11286998 +0000 UTC m=+89.390477361, 1.00500624s after stability -2026-03-31T06:17:41.125Z [INFO] server-38d1d610-9da4-b706-e060-2a2bd571957f: aborting pipeline replication: peer="{Voter server-e77d88a3-122c-4fd3-0110-6d5866fcf60a e77d88a3-122c-4fd3-0110-6d5866fcf60a}" -2026-03-31T06:17:41.125Z [INFO] server-38d1d610-9da4-b706-e060-2a2bd571957f: aborting pipeline replication: peer="{Voter server-070fdaaf-c568-b25c-5836-a8617e1a27c5 070fdaaf-c568-b25c-5836-a8617e1a27c5}" +2025-02-25T23:59:00.694Z [INFO] server-c2da4d72-9ee9-09d7-d6d8-5e9370dc981a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c2da4d72-9ee9-09d7-d6d8-5e9370dc981a Address:c2da4d72-9ee9-09d7-d6d8-5e9370dc981a} {Suffrage:Voter ID:server-7ee67907-7ed0-deee-8512-20a8be84524b Address:7ee67907-7ed0-deee-8512-20a8be84524b} {Suffrage:Voter ID:server-23af0c94-d61e-f5d8-a1fe-679e1b3bc0f7 Address:23af0c94-d61e-f5d8-a1fe-679e1b3bc0f7}]" +2025-02-25T23:59:00.694Z [INFO] server-7ee67907-7ed0-deee-8512-20a8be84524b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c2da4d72-9ee9-09d7-d6d8-5e9370dc981a Address:c2da4d72-9ee9-09d7-d6d8-5e9370dc981a} {Suffrage:Voter ID:server-7ee67907-7ed0-deee-8512-20a8be84524b Address:7ee67907-7ed0-deee-8512-20a8be84524b} {Suffrage:Voter ID:server-23af0c94-d61e-f5d8-a1fe-679e1b3bc0f7 Address:23af0c94-d61e-f5d8-a1fe-679e1b3bc0f7}]" +2025-02-25T23:59:00.694Z [INFO] server-23af0c94-d61e-f5d8-a1fe-679e1b3bc0f7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c2da4d72-9ee9-09d7-d6d8-5e9370dc981a Address:c2da4d72-9ee9-09d7-d6d8-5e9370dc981a} {Suffrage:Voter ID:server-7ee67907-7ed0-deee-8512-20a8be84524b Address:7ee67907-7ed0-deee-8512-20a8be84524b} {Suffrage:Voter ID:server-23af0c94-d61e-f5d8-a1fe-679e1b3bc0f7 Address:23af0c94-d61e-f5d8-a1fe-679e1b3bc0f7}]" +2025-02-25T23:59:00.694Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:00.694Z [INFO] server-c2da4d72-9ee9-09d7-d6d8-5e9370dc981a: entering follower state: follower="Node at c2da4d72-9ee9-09d7-d6d8-5e9370dc981a [Follower]" leader-address= leader-id= +2025-02-25T23:59:00.694Z [INFO] server-7ee67907-7ed0-deee-8512-20a8be84524b: entering follower state: follower="Node at 7ee67907-7ed0-deee-8512-20a8be84524b [Follower]" leader-address= leader-id= +2025-02-25T23:59:00.694Z [INFO] server-23af0c94-d61e-f5d8-a1fe-679e1b3bc0f7: entering follower state: follower="Node at 23af0c94-d61e-f5d8-a1fe-679e1b3bc0f7 [Follower]" leader-address= leader-id= +2025-02-25T23:59:01.362Z [WARN] server-7ee67907-7ed0-deee-8512-20a8be84524b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:01.362Z [INFO] server-7ee67907-7ed0-deee-8512-20a8be84524b: entering candidate state: node="Node at 7ee67907-7ed0-deee-8512-20a8be84524b [Candidate]" term=2 +2025-02-25T23:59:01.362Z [INFO] server-7ee67907-7ed0-deee-8512-20a8be84524b: election won: term=2 tally=2 +2025-02-25T23:59:01.362Z [INFO] server-7ee67907-7ed0-deee-8512-20a8be84524b: entering leader state: leader="Node at 7ee67907-7ed0-deee-8512-20a8be84524b [Leader]" +2025-02-25T23:59:01.362Z [INFO] server-7ee67907-7ed0-deee-8512-20a8be84524b: added peer, starting replication: peer=server-c2da4d72-9ee9-09d7-d6d8-5e9370dc981a +2025-02-25T23:59:01.362Z [INFO] server-7ee67907-7ed0-deee-8512-20a8be84524b: added peer, starting replication: peer=server-23af0c94-d61e-f5d8-a1fe-679e1b3bc0f7 +2025-02-25T23:59:01.362Z [INFO] server-7ee67907-7ed0-deee-8512-20a8be84524b: pipelining replication: peer="{Voter server-23af0c94-d61e-f5d8-a1fe-679e1b3bc0f7 23af0c94-d61e-f5d8-a1fe-679e1b3bc0f7}" +2025-02-25T23:59:01.362Z [INFO] server-7ee67907-7ed0-deee-8512-20a8be84524b: pipelining replication: peer="{Voter server-c2da4d72-9ee9-09d7-d6d8-5e9370dc981a c2da4d72-9ee9-09d7-d6d8-5e9370dc981a}" +2025-02-25T23:59:02.370Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:01.362477006 +0000 UTC m=+88.971492413 (1 nodes), 667.684091ms from start of poll, 667.855412ms from cluster start. Timeout at 2025-02-25 23:59:02.367479166 +0000 UTC m=+89.976494573, 1.00500216s after stability +2025-02-25T23:59:02.370Z [INFO] server-7ee67907-7ed0-deee-8512-20a8be84524b: starting snapshot up to: index=12 +2025-02-25T23:59:02.370Z [INFO] creating new snapshot: path=/tmp/raft1740634107/snapshots/2-12-1740527942370.tmp +2025-02-25T23:59:02.371Z [INFO] server-7ee67907-7ed0-deee-8512-20a8be84524b: snapshot complete up to: index=12 +2025-02-25T23:59:02.371Z [INFO] creating new snapshot: path=/tmp/raft1740634107/snapshots/2-10013-1740527942371.tmp +2025-02-25T23:59:02.371Z [INFO] server-7ee67907-7ed0-deee-8512-20a8be84524b: copied to local snapshot: bytes=71 +2025-02-25T23:59:02.371Z [INFO] server-7ee67907-7ed0-deee-8512-20a8be84524b: snapshot restore progress: id=2-10013-1740527942371 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:59:02.371Z [INFO] server-7ee67907-7ed0-deee-8512-20a8be84524b: removing all old logs from log store +2025-02-25T23:59:02.371Z [INFO] server-7ee67907-7ed0-deee-8512-20a8be84524b: compacting logs: from=1 to=22 +2025-02-25T23:59:02.371Z [INFO] server-7ee67907-7ed0-deee-8512-20a8be84524b: restored user snapshot: index=10013 +2025-02-25T23:59:02.371Z [ERROR] server-7ee67907-7ed0-deee-8512-20a8be84524b: failed to get log: index=22 error="log not found" +2025-02-25T23:59:02.371Z [INFO] server-7ee67907-7ed0-deee-8512-20a8be84524b: aborting pipeline replication: peer="{Voter server-23af0c94-d61e-f5d8-a1fe-679e1b3bc0f7 23af0c94-d61e-f5d8-a1fe-679e1b3bc0f7}" +2025-02-25T23:59:02.371Z [ERROR] server-7ee67907-7ed0-deee-8512-20a8be84524b: failed to get log: index=22 error="log not found" +2025-02-25T23:59:02.371Z [INFO] server-7ee67907-7ed0-deee-8512-20a8be84524b: aborting pipeline replication: peer="{Voter server-c2da4d72-9ee9-09d7-d6d8-5e9370dc981a c2da4d72-9ee9-09d7-d6d8-5e9370dc981a}" +2025-02-25T23:59:02.382Z [ERROR] server-7ee67907-7ed0-deee-8512-20a8be84524b: failed to get log: index=12 error="log not found" +2025-02-25T23:59:02.382Z [INFO] creating new snapshot: path=/tmp/raft3010313633/snapshots/2-10013-1740527942382.tmp +2025-02-25T23:59:02.382Z [INFO] server-23af0c94-d61e-f5d8-a1fe-679e1b3bc0f7: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:59:02.382Z [INFO] server-23af0c94-d61e-f5d8-a1fe-679e1b3bc0f7: copied to local snapshot: bytes=71 +2025-02-25T23:59:02.382Z [INFO] server-23af0c94-d61e-f5d8-a1fe-679e1b3bc0f7: snapshot restore progress: id=2-10013-1740527942382 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:59:02.382Z [INFO] server-23af0c94-d61e-f5d8-a1fe-679e1b3bc0f7: removing all old logs from log store +2025-02-25T23:59:02.382Z [INFO] server-23af0c94-d61e-f5d8-a1fe-679e1b3bc0f7: compacting logs: from=1 to=22 +2025-02-25T23:59:02.382Z [INFO] server-23af0c94-d61e-f5d8-a1fe-679e1b3bc0f7: Installed remote snapshot +2025-02-25T23:59:02.382Z [WARN] server-23af0c94-d61e-f5d8-a1fe-679e1b3bc0f7: skipping application of old log: index=22 +2025-02-25T23:59:02.382Z [INFO] server-7ee67907-7ed0-deee-8512-20a8be84524b: pipelining replication: peer="{Voter server-23af0c94-d61e-f5d8-a1fe-679e1b3bc0f7 23af0c94-d61e-f5d8-a1fe-679e1b3bc0f7}" +2025-02-25T23:59:02.382Z [ERROR] server-7ee67907-7ed0-deee-8512-20a8be84524b: failed to get log: index=22 error="log not found" +2025-02-25T23:59:02.382Z [INFO] creating new snapshot: path=/tmp/raft584807664/snapshots/2-10013-1740527942382.tmp +2025-02-25T23:59:02.383Z [INFO] server-c2da4d72-9ee9-09d7-d6d8-5e9370dc981a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:59:02.383Z [INFO] server-c2da4d72-9ee9-09d7-d6d8-5e9370dc981a: copied to local snapshot: bytes=71 +2025-02-25T23:59:02.383Z [INFO] server-c2da4d72-9ee9-09d7-d6d8-5e9370dc981a: snapshot restore progress: id=2-10013-1740527942382 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:59:02.383Z [INFO] server-c2da4d72-9ee9-09d7-d6d8-5e9370dc981a: removing all old logs from log store +2025-02-25T23:59:02.383Z [INFO] server-c2da4d72-9ee9-09d7-d6d8-5e9370dc981a: compacting logs: from=1 to=22 +2025-02-25T23:59:02.383Z [INFO] server-c2da4d72-9ee9-09d7-d6d8-5e9370dc981a: Installed remote snapshot +2025-02-25T23:59:02.383Z [WARN] server-c2da4d72-9ee9-09d7-d6d8-5e9370dc981a: skipping application of old log: index=22 +2025-02-25T23:59:02.383Z [INFO] server-7ee67907-7ed0-deee-8512-20a8be84524b: pipelining replication: peer="{Voter server-c2da4d72-9ee9-09d7-d6d8-5e9370dc981a c2da4d72-9ee9-09d7-d6d8-5e9370dc981a}" +2025-02-25T23:59:02.383Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:59:03.390Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:59:02.383428146 +0000 UTC m=+89.992443553 (2 nodes), 600ns from start of poll, 1.688806552s from cluster start. Timeout at 2025-02-25 23:59:03.388457547 +0000 UTC m=+90.997472954, 1.005029401s after stability +2025-02-25T23:59:03.406Z [INFO] server-7ee67907-7ed0-deee-8512-20a8be84524b: aborting pipeline replication: peer="{Voter server-23af0c94-d61e-f5d8-a1fe-679e1b3bc0f7 23af0c94-d61e-f5d8-a1fe-679e1b3bc0f7}" +2025-02-25T23:59:03.406Z [INFO] server-7ee67907-7ed0-deee-8512-20a8be84524b: aborting pipeline replication: peer="{Voter server-c2da4d72-9ee9-09d7-d6d8-5e9370dc981a c2da4d72-9ee9-09d7-d6d8-5e9370dc981a}" === RUN TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_true -2026-03-31T06:17:41.126Z [INFO] server-7d198484-1da5-d3e9-67ee-b5e0c7ff966d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7d198484-1da5-d3e9-67ee-b5e0c7ff966d Address:7d198484-1da5-d3e9-67ee-b5e0c7ff966d} {Suffrage:Voter ID:server-bcbbbe14-3fe5-5155-f2d2-03a19db7ca51 Address:bcbbbe14-3fe5-5155-f2d2-03a19db7ca51} {Suffrage:Voter ID:server-fb346e13-1906-433b-70f8-b4bc4d163f27 Address:fb346e13-1906-433b-70f8-b4bc4d163f27}]" -2026-03-31T06:17:41.126Z [INFO] server-bcbbbe14-3fe5-5155-f2d2-03a19db7ca51: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7d198484-1da5-d3e9-67ee-b5e0c7ff966d Address:7d198484-1da5-d3e9-67ee-b5e0c7ff966d} {Suffrage:Voter ID:server-bcbbbe14-3fe5-5155-f2d2-03a19db7ca51 Address:bcbbbe14-3fe5-5155-f2d2-03a19db7ca51} {Suffrage:Voter ID:server-fb346e13-1906-433b-70f8-b4bc4d163f27 Address:fb346e13-1906-433b-70f8-b4bc4d163f27}]" -2026-03-31T06:17:41.126Z [INFO] server-7d198484-1da5-d3e9-67ee-b5e0c7ff966d: entering follower state: follower="Node at 7d198484-1da5-d3e9-67ee-b5e0c7ff966d [Follower]" leader-address= leader-id= -2026-03-31T06:17:41.127Z [INFO] server-fb346e13-1906-433b-70f8-b4bc4d163f27: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7d198484-1da5-d3e9-67ee-b5e0c7ff966d Address:7d198484-1da5-d3e9-67ee-b5e0c7ff966d} {Suffrage:Voter ID:server-bcbbbe14-3fe5-5155-f2d2-03a19db7ca51 Address:bcbbbe14-3fe5-5155-f2d2-03a19db7ca51} {Suffrage:Voter ID:server-fb346e13-1906-433b-70f8-b4bc4d163f27 Address:fb346e13-1906-433b-70f8-b4bc4d163f27}]" -2026-03-31T06:17:41.127Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:41.127Z [INFO] server-bcbbbe14-3fe5-5155-f2d2-03a19db7ca51: entering follower state: follower="Node at bcbbbe14-3fe5-5155-f2d2-03a19db7ca51 [Follower]" leader-address= leader-id= -2026-03-31T06:17:41.127Z [INFO] server-fb346e13-1906-433b-70f8-b4bc4d163f27: entering follower state: follower="Node at fb346e13-1906-433b-70f8-b4bc4d163f27 [Follower]" leader-address= leader-id= -2026-03-31T06:17:41.853Z [WARN] server-fb346e13-1906-433b-70f8-b4bc4d163f27: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:41.853Z [INFO] server-fb346e13-1906-433b-70f8-b4bc4d163f27: entering candidate state: node="Node at fb346e13-1906-433b-70f8-b4bc4d163f27 [Candidate]" term=2 -2026-03-31T06:17:41.854Z [INFO] server-fb346e13-1906-433b-70f8-b4bc4d163f27: election won: term=2 tally=2 -2026-03-31T06:17:41.854Z [INFO] server-fb346e13-1906-433b-70f8-b4bc4d163f27: entering leader state: leader="Node at fb346e13-1906-433b-70f8-b4bc4d163f27 [Leader]" -2026-03-31T06:17:41.854Z [INFO] server-fb346e13-1906-433b-70f8-b4bc4d163f27: added peer, starting replication: peer=server-7d198484-1da5-d3e9-67ee-b5e0c7ff966d -2026-03-31T06:17:41.854Z [INFO] server-fb346e13-1906-433b-70f8-b4bc4d163f27: added peer, starting replication: peer=server-bcbbbe14-3fe5-5155-f2d2-03a19db7ca51 -2026-03-31T06:17:41.854Z [INFO] server-fb346e13-1906-433b-70f8-b4bc4d163f27: pipelining replication: peer="{Voter server-bcbbbe14-3fe5-5155-f2d2-03a19db7ca51 bcbbbe14-3fe5-5155-f2d2-03a19db7ca51}" -2026-03-31T06:17:41.854Z [INFO] server-fb346e13-1906-433b-70f8-b4bc4d163f27: pipelining replication: peer="{Voter server-7d198484-1da5-d3e9-67ee-b5e0c7ff966d 7d198484-1da5-d3e9-67ee-b5e0c7ff966d}" -2026-03-31T06:17:42.860Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:41.85419886 +0000 UTC m=+90.131806241 (1 nodes), 727.10584ms from start of poll, 727.5166ms from cluster start. Timeout at 2026-03-31 06:17:42.85920158 +0000 UTC m=+91.136808961, 1.00500272s after stability -2026-03-31T06:17:42.861Z [INFO] server-fb346e13-1906-433b-70f8-b4bc4d163f27: starting snapshot up to: index=12 -2026-03-31T06:17:42.861Z [INFO] creating new snapshot: path=/tmp/raft1544705166/snapshots/2-12-1774937862861.tmp -2026-03-31T06:17:42.861Z [INFO] server-fb346e13-1906-433b-70f8-b4bc4d163f27: snapshot complete up to: index=12 -2026-03-31T06:17:42.863Z [INFO] server-64677f9e-4fe3-9c57-bb25-edb3b9bc8403: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-64677f9e-4fe3-9c57-bb25-edb3b9bc8403 Address:64677f9e-4fe3-9c57-bb25-edb3b9bc8403} {Suffrage:Voter ID:server-2718e9e8-abc3-7046-450d-cd403200c67b Address:2718e9e8-abc3-7046-450d-cd403200c67b} {Suffrage:Voter ID:server-3bdb1798-ffdc-2974-b381-f6ab4c9c537d Address:3bdb1798-ffdc-2974-b381-f6ab4c9c537d}]" -2026-03-31T06:17:42.863Z [INFO] server-64677f9e-4fe3-9c57-bb25-edb3b9bc8403: entering follower state: follower="Node at 64677f9e-4fe3-9c57-bb25-edb3b9bc8403 [Follower]" leader-address= leader-id= -2026-03-31T06:17:42.863Z [INFO] server-2718e9e8-abc3-7046-450d-cd403200c67b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-64677f9e-4fe3-9c57-bb25-edb3b9bc8403 Address:64677f9e-4fe3-9c57-bb25-edb3b9bc8403} {Suffrage:Voter ID:server-2718e9e8-abc3-7046-450d-cd403200c67b Address:2718e9e8-abc3-7046-450d-cd403200c67b} {Suffrage:Voter ID:server-3bdb1798-ffdc-2974-b381-f6ab4c9c537d Address:3bdb1798-ffdc-2974-b381-f6ab4c9c537d}]" -2026-03-31T06:17:42.863Z [INFO] server-2718e9e8-abc3-7046-450d-cd403200c67b: entering follower state: follower="Node at 2718e9e8-abc3-7046-450d-cd403200c67b [Follower]" leader-address= leader-id= -2026-03-31T06:17:42.863Z [INFO] server-3bdb1798-ffdc-2974-b381-f6ab4c9c537d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-64677f9e-4fe3-9c57-bb25-edb3b9bc8403 Address:64677f9e-4fe3-9c57-bb25-edb3b9bc8403} {Suffrage:Voter ID:server-2718e9e8-abc3-7046-450d-cd403200c67b Address:2718e9e8-abc3-7046-450d-cd403200c67b} {Suffrage:Voter ID:server-3bdb1798-ffdc-2974-b381-f6ab4c9c537d Address:3bdb1798-ffdc-2974-b381-f6ab4c9c537d}]" -2026-03-31T06:17:42.863Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:42.863Z [INFO] server-3bdb1798-ffdc-2974-b381-f6ab4c9c537d: entering follower state: follower="Node at 3bdb1798-ffdc-2974-b381-f6ab4c9c537d [Follower]" leader-address= leader-id= -2026-03-31T06:17:43.400Z [WARN] server-3bdb1798-ffdc-2974-b381-f6ab4c9c537d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:43.400Z [INFO] server-3bdb1798-ffdc-2974-b381-f6ab4c9c537d: entering candidate state: node="Node at 3bdb1798-ffdc-2974-b381-f6ab4c9c537d [Candidate]" term=2 -2026-03-31T06:17:43.400Z [INFO] server-3bdb1798-ffdc-2974-b381-f6ab4c9c537d: election won: term=2 tally=2 -2026-03-31T06:17:43.400Z [INFO] server-3bdb1798-ffdc-2974-b381-f6ab4c9c537d: entering leader state: leader="Node at 3bdb1798-ffdc-2974-b381-f6ab4c9c537d [Leader]" -2026-03-31T06:17:43.400Z [INFO] server-3bdb1798-ffdc-2974-b381-f6ab4c9c537d: added peer, starting replication: peer=server-64677f9e-4fe3-9c57-bb25-edb3b9bc8403 -2026-03-31T06:17:43.400Z [INFO] server-3bdb1798-ffdc-2974-b381-f6ab4c9c537d: added peer, starting replication: peer=server-2718e9e8-abc3-7046-450d-cd403200c67b -2026-03-31T06:17:43.400Z [INFO] server-3bdb1798-ffdc-2974-b381-f6ab4c9c537d: pipelining replication: peer="{Voter server-64677f9e-4fe3-9c57-bb25-edb3b9bc8403 64677f9e-4fe3-9c57-bb25-edb3b9bc8403}" -2026-03-31T06:17:43.400Z [INFO] server-3bdb1798-ffdc-2974-b381-f6ab4c9c537d: pipelining replication: peer="{Voter server-2718e9e8-abc3-7046-450d-cd403200c67b 2718e9e8-abc3-7046-450d-cd403200c67b}" -2026-03-31T06:17:44.406Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:43.40056254 +0000 UTC m=+91.678169961 (1 nodes), 536.57456ms from start of poll, 537.047ms from cluster start. Timeout at 2026-03-31 06:17:44.40557166 +0000 UTC m=+92.683179041, 1.00500908s after stability -2026-03-31T06:17:44.406Z [INFO] creating new snapshot: path=/tmp/raft1598952552/snapshots/2-10013-1774937864406.tmp -2026-03-31T06:17:44.406Z [INFO] server-3bdb1798-ffdc-2974-b381-f6ab4c9c537d: copied to local snapshot: bytes=71 -2026-03-31T06:17:44.407Z [INFO] server-3bdb1798-ffdc-2974-b381-f6ab4c9c537d: snapshot restore progress: id=2-10013-1774937864406 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:44.407Z [INFO] server-3bdb1798-ffdc-2974-b381-f6ab4c9c537d: restored user snapshot: index=10013 -2026-03-31T06:17:44.407Z [ERROR] server-3bdb1798-ffdc-2974-b381-f6ab4c9c537d: failed to get log: index=3 error="log not found" -2026-03-31T06:17:44.407Z [INFO] server-3bdb1798-ffdc-2974-b381-f6ab4c9c537d: aborting pipeline replication: peer="{Voter server-2718e9e8-abc3-7046-450d-cd403200c67b 2718e9e8-abc3-7046-450d-cd403200c67b}" -2026-03-31T06:17:44.407Z [ERROR] server-3bdb1798-ffdc-2974-b381-f6ab4c9c537d: failed to get log: index=3 error="log not found" -2026-03-31T06:17:44.407Z [INFO] server-3bdb1798-ffdc-2974-b381-f6ab4c9c537d: aborting pipeline replication: peer="{Voter server-64677f9e-4fe3-9c57-bb25-edb3b9bc8403 64677f9e-4fe3-9c57-bb25-edb3b9bc8403}" -2026-03-31T06:17:44.414Z [ERROR] server-3bdb1798-ffdc-2974-b381-f6ab4c9c537d: failed to get log: index=3 error="log not found" -2026-03-31T06:17:44.414Z [ERROR] server-3bdb1798-ffdc-2974-b381-f6ab4c9c537d: failed to get log: index=3 error="log not found" -2026-03-31T06:17:44.415Z [INFO] creating new snapshot: path=/tmp/raft4253972539/snapshots/2-10013-1774937864415.tmp -2026-03-31T06:17:44.415Z [INFO] creating new snapshot: path=/tmp/raft1472649542/snapshots/2-10013-1774937864415.tmp -2026-03-31T06:17:44.415Z [INFO] server-64677f9e-4fe3-9c57-bb25-edb3b9bc8403: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:44.415Z [INFO] server-2718e9e8-abc3-7046-450d-cd403200c67b: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:44.415Z [INFO] server-64677f9e-4fe3-9c57-bb25-edb3b9bc8403: copied to local snapshot: bytes=71 -2026-03-31T06:17:44.415Z [INFO] server-2718e9e8-abc3-7046-450d-cd403200c67b: copied to local snapshot: bytes=71 -2026-03-31T06:17:44.415Z [INFO] server-64677f9e-4fe3-9c57-bb25-edb3b9bc8403: snapshot restore progress: id=2-10013-1774937864415 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:44.416Z [INFO] server-64677f9e-4fe3-9c57-bb25-edb3b9bc8403: Installed remote snapshot -2026-03-31T06:17:44.416Z [INFO] server-3bdb1798-ffdc-2974-b381-f6ab4c9c537d: pipelining replication: peer="{Voter server-64677f9e-4fe3-9c57-bb25-edb3b9bc8403 64677f9e-4fe3-9c57-bb25-edb3b9bc8403}" -2026-03-31T06:17:44.416Z [INFO] server-2718e9e8-abc3-7046-450d-cd403200c67b: snapshot restore progress: id=2-10013-1774937864415 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:44.416Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:17:44.416Z [INFO] server-2718e9e8-abc3-7046-450d-cd403200c67b: Installed remote snapshot -2026-03-31T06:17:44.416Z [INFO] server-3bdb1798-ffdc-2974-b381-f6ab4c9c537d: pipelining replication: peer="{Voter server-2718e9e8-abc3-7046-450d-cd403200c67b 2718e9e8-abc3-7046-450d-cd403200c67b}" -2026-03-31T06:17:45.422Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:17:44.41619858 +0000 UTC m=+92.693805961 (2 nodes), 920ns from start of poll, 1.552683s from cluster start. Timeout at 2026-03-31 06:17:45.42122686 +0000 UTC m=+93.698834241, 1.00502828s after stability -2026-03-31T06:17:45.433Z [INFO] server-fb346e13-1906-433b-70f8-b4bc4d163f27: aborting pipeline replication: peer="{Voter server-bcbbbe14-3fe5-5155-f2d2-03a19db7ca51 bcbbbe14-3fe5-5155-f2d2-03a19db7ca51}" -2026-03-31T06:17:45.433Z [INFO] server-fb346e13-1906-433b-70f8-b4bc4d163f27: aborting pipeline replication: peer="{Voter server-7d198484-1da5-d3e9-67ee-b5e0c7ff966d 7d198484-1da5-d3e9-67ee-b5e0c7ff966d}" +2025-02-25T23:59:03.406Z [INFO] server-58c16bf5-f633-b037-6efb-ac471ce47c8f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-58c16bf5-f633-b037-6efb-ac471ce47c8f Address:58c16bf5-f633-b037-6efb-ac471ce47c8f} {Suffrage:Voter ID:server-6dc4e20d-89a9-ca73-002c-0a7de09e2479 Address:6dc4e20d-89a9-ca73-002c-0a7de09e2479} {Suffrage:Voter ID:server-f3089311-d91f-5fc5-a145-380962bdee87 Address:f3089311-d91f-5fc5-a145-380962bdee87}]" +2025-02-25T23:59:03.406Z [INFO] server-6dc4e20d-89a9-ca73-002c-0a7de09e2479: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-58c16bf5-f633-b037-6efb-ac471ce47c8f Address:58c16bf5-f633-b037-6efb-ac471ce47c8f} {Suffrage:Voter ID:server-6dc4e20d-89a9-ca73-002c-0a7de09e2479 Address:6dc4e20d-89a9-ca73-002c-0a7de09e2479} {Suffrage:Voter ID:server-f3089311-d91f-5fc5-a145-380962bdee87 Address:f3089311-d91f-5fc5-a145-380962bdee87}]" +2025-02-25T23:59:03.406Z [INFO] server-f3089311-d91f-5fc5-a145-380962bdee87: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-58c16bf5-f633-b037-6efb-ac471ce47c8f Address:58c16bf5-f633-b037-6efb-ac471ce47c8f} {Suffrage:Voter ID:server-6dc4e20d-89a9-ca73-002c-0a7de09e2479 Address:6dc4e20d-89a9-ca73-002c-0a7de09e2479} {Suffrage:Voter ID:server-f3089311-d91f-5fc5-a145-380962bdee87 Address:f3089311-d91f-5fc5-a145-380962bdee87}]" +2025-02-25T23:59:03.406Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:03.406Z [INFO] server-58c16bf5-f633-b037-6efb-ac471ce47c8f: entering follower state: follower="Node at 58c16bf5-f633-b037-6efb-ac471ce47c8f [Follower]" leader-address= leader-id= +2025-02-25T23:59:03.406Z [INFO] server-6dc4e20d-89a9-ca73-002c-0a7de09e2479: entering follower state: follower="Node at 6dc4e20d-89a9-ca73-002c-0a7de09e2479 [Follower]" leader-address= leader-id= +2025-02-25T23:59:03.406Z [INFO] server-f3089311-d91f-5fc5-a145-380962bdee87: entering follower state: follower="Node at f3089311-d91f-5fc5-a145-380962bdee87 [Follower]" leader-address= leader-id= +2025-02-25T23:59:03.977Z [WARN] server-f3089311-d91f-5fc5-a145-380962bdee87: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:03.977Z [INFO] server-f3089311-d91f-5fc5-a145-380962bdee87: entering candidate state: node="Node at f3089311-d91f-5fc5-a145-380962bdee87 [Candidate]" term=2 +2025-02-25T23:59:03.977Z [INFO] server-f3089311-d91f-5fc5-a145-380962bdee87: election won: term=2 tally=2 +2025-02-25T23:59:03.977Z [INFO] server-f3089311-d91f-5fc5-a145-380962bdee87: entering leader state: leader="Node at f3089311-d91f-5fc5-a145-380962bdee87 [Leader]" +2025-02-25T23:59:03.977Z [INFO] server-f3089311-d91f-5fc5-a145-380962bdee87: added peer, starting replication: peer=server-58c16bf5-f633-b037-6efb-ac471ce47c8f +2025-02-25T23:59:03.978Z [INFO] server-f3089311-d91f-5fc5-a145-380962bdee87: added peer, starting replication: peer=server-6dc4e20d-89a9-ca73-002c-0a7de09e2479 +2025-02-25T23:59:03.978Z [INFO] server-f3089311-d91f-5fc5-a145-380962bdee87: pipelining replication: peer="{Voter server-6dc4e20d-89a9-ca73-002c-0a7de09e2479 6dc4e20d-89a9-ca73-002c-0a7de09e2479}" +2025-02-25T23:59:03.978Z [INFO] server-f3089311-d91f-5fc5-a145-380962bdee87: pipelining replication: peer="{Voter server-58c16bf5-f633-b037-6efb-ac471ce47c8f 58c16bf5-f633-b037-6efb-ac471ce47c8f}" +2025-02-25T23:59:04.985Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:03.978384821 +0000 UTC m=+91.587400188 (1 nodes), 571.571159ms from start of poll, 571.751ms from cluster start. Timeout at 2025-02-25 23:59:04.983392261 +0000 UTC m=+92.592407668, 1.00500748s after stability +2025-02-25T23:59:04.986Z [INFO] server-f3089311-d91f-5fc5-a145-380962bdee87: starting snapshot up to: index=12 +2025-02-25T23:59:04.986Z [INFO] creating new snapshot: path=/tmp/raft422550068/snapshots/2-12-1740527944986.tmp +2025-02-25T23:59:04.986Z [INFO] server-f3089311-d91f-5fc5-a145-380962bdee87: snapshot complete up to: index=12 +2025-02-25T23:59:04.987Z [INFO] server-a90d62ed-76ca-2d76-745d-38015fc95a2c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a90d62ed-76ca-2d76-745d-38015fc95a2c Address:a90d62ed-76ca-2d76-745d-38015fc95a2c} {Suffrage:Voter ID:server-227f3c22-aa0b-141b-dc6f-700b27990672 Address:227f3c22-aa0b-141b-dc6f-700b27990672} {Suffrage:Voter ID:server-7a72f73d-286c-0d21-a227-48c2008139fc Address:7a72f73d-286c-0d21-a227-48c2008139fc}]" +2025-02-25T23:59:04.987Z [INFO] server-227f3c22-aa0b-141b-dc6f-700b27990672: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a90d62ed-76ca-2d76-745d-38015fc95a2c Address:a90d62ed-76ca-2d76-745d-38015fc95a2c} {Suffrage:Voter ID:server-227f3c22-aa0b-141b-dc6f-700b27990672 Address:227f3c22-aa0b-141b-dc6f-700b27990672} {Suffrage:Voter ID:server-7a72f73d-286c-0d21-a227-48c2008139fc Address:7a72f73d-286c-0d21-a227-48c2008139fc}]" +2025-02-25T23:59:04.987Z [INFO] server-7a72f73d-286c-0d21-a227-48c2008139fc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a90d62ed-76ca-2d76-745d-38015fc95a2c Address:a90d62ed-76ca-2d76-745d-38015fc95a2c} {Suffrage:Voter ID:server-227f3c22-aa0b-141b-dc6f-700b27990672 Address:227f3c22-aa0b-141b-dc6f-700b27990672} {Suffrage:Voter ID:server-7a72f73d-286c-0d21-a227-48c2008139fc Address:7a72f73d-286c-0d21-a227-48c2008139fc}]" +2025-02-25T23:59:04.987Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:04.987Z [INFO] server-a90d62ed-76ca-2d76-745d-38015fc95a2c: entering follower state: follower="Node at a90d62ed-76ca-2d76-745d-38015fc95a2c [Follower]" leader-address= leader-id= +2025-02-25T23:59:04.987Z [INFO] server-227f3c22-aa0b-141b-dc6f-700b27990672: entering follower state: follower="Node at 227f3c22-aa0b-141b-dc6f-700b27990672 [Follower]" leader-address= leader-id= +2025-02-25T23:59:04.987Z [INFO] server-7a72f73d-286c-0d21-a227-48c2008139fc: entering follower state: follower="Node at 7a72f73d-286c-0d21-a227-48c2008139fc [Follower]" leader-address= leader-id= +2025-02-25T23:59:05.646Z [WARN] server-7a72f73d-286c-0d21-a227-48c2008139fc: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:05.646Z [INFO] server-7a72f73d-286c-0d21-a227-48c2008139fc: entering candidate state: node="Node at 7a72f73d-286c-0d21-a227-48c2008139fc [Candidate]" term=2 +2025-02-25T23:59:05.646Z [INFO] server-7a72f73d-286c-0d21-a227-48c2008139fc: election won: term=2 tally=2 +2025-02-25T23:59:05.646Z [INFO] server-7a72f73d-286c-0d21-a227-48c2008139fc: entering leader state: leader="Node at 7a72f73d-286c-0d21-a227-48c2008139fc [Leader]" +2025-02-25T23:59:05.646Z [INFO] server-7a72f73d-286c-0d21-a227-48c2008139fc: added peer, starting replication: peer=server-a90d62ed-76ca-2d76-745d-38015fc95a2c +2025-02-25T23:59:05.646Z [INFO] server-7a72f73d-286c-0d21-a227-48c2008139fc: added peer, starting replication: peer=server-227f3c22-aa0b-141b-dc6f-700b27990672 +2025-02-25T23:59:05.646Z [INFO] server-7a72f73d-286c-0d21-a227-48c2008139fc: pipelining replication: peer="{Voter server-227f3c22-aa0b-141b-dc6f-700b27990672 227f3c22-aa0b-141b-dc6f-700b27990672}" +2025-02-25T23:59:05.646Z [INFO] server-7a72f73d-286c-0d21-a227-48c2008139fc: pipelining replication: peer="{Voter server-a90d62ed-76ca-2d76-745d-38015fc95a2c a90d62ed-76ca-2d76-745d-38015fc95a2c}" +2025-02-25T23:59:06.653Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:05.646984284 +0000 UTC m=+93.255999691 (1 nodes), 659.508839ms from start of poll, 659.6628ms from cluster start. Timeout at 2025-02-25 23:59:06.651995724 +0000 UTC m=+94.261011131, 1.00501144s after stability +2025-02-25T23:59:06.654Z [INFO] creating new snapshot: path=/tmp/raft693749038/snapshots/2-10013-1740527946654.tmp +2025-02-25T23:59:06.654Z [INFO] server-7a72f73d-286c-0d21-a227-48c2008139fc: copied to local snapshot: bytes=71 +2025-02-25T23:59:06.654Z [INFO] server-7a72f73d-286c-0d21-a227-48c2008139fc: snapshot restore progress: id=2-10013-1740527946654 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:59:06.654Z [INFO] server-7a72f73d-286c-0d21-a227-48c2008139fc: restored user snapshot: index=10013 +2025-02-25T23:59:06.654Z [ERROR] server-7a72f73d-286c-0d21-a227-48c2008139fc: failed to get log: index=3 error="log not found" +2025-02-25T23:59:06.654Z [INFO] server-7a72f73d-286c-0d21-a227-48c2008139fc: aborting pipeline replication: peer="{Voter server-227f3c22-aa0b-141b-dc6f-700b27990672 227f3c22-aa0b-141b-dc6f-700b27990672}" +2025-02-25T23:59:06.654Z [ERROR] server-7a72f73d-286c-0d21-a227-48c2008139fc: failed to get log: index=3 error="log not found" +2025-02-25T23:59:06.654Z [INFO] server-7a72f73d-286c-0d21-a227-48c2008139fc: aborting pipeline replication: peer="{Voter server-a90d62ed-76ca-2d76-745d-38015fc95a2c a90d62ed-76ca-2d76-745d-38015fc95a2c}" +2025-02-25T23:59:06.661Z [ERROR] server-7a72f73d-286c-0d21-a227-48c2008139fc: failed to get log: index=3 error="log not found" +2025-02-25T23:59:06.662Z [INFO] creating new snapshot: path=/tmp/raft4046205003/snapshots/2-10013-1740527946662.tmp +2025-02-25T23:59:06.662Z [INFO] server-227f3c22-aa0b-141b-dc6f-700b27990672: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:59:06.662Z [INFO] server-227f3c22-aa0b-141b-dc6f-700b27990672: copied to local snapshot: bytes=71 +2025-02-25T23:59:06.662Z [INFO] server-227f3c22-aa0b-141b-dc6f-700b27990672: snapshot restore progress: id=2-10013-1740527946662 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:59:06.662Z [INFO] server-227f3c22-aa0b-141b-dc6f-700b27990672: Installed remote snapshot +2025-02-25T23:59:06.662Z [INFO] server-7a72f73d-286c-0d21-a227-48c2008139fc: pipelining replication: peer="{Voter server-227f3c22-aa0b-141b-dc6f-700b27990672 227f3c22-aa0b-141b-dc6f-700b27990672}" +2025-02-25T23:59:06.662Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:59:06.665Z [ERROR] server-7a72f73d-286c-0d21-a227-48c2008139fc: failed to get log: index=3 error="log not found" +2025-02-25T23:59:06.666Z [INFO] creating new snapshot: path=/tmp/raft247661177/snapshots/2-10013-1740527946666.tmp +2025-02-25T23:59:06.666Z [INFO] server-a90d62ed-76ca-2d76-745d-38015fc95a2c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:59:06.666Z [INFO] server-a90d62ed-76ca-2d76-745d-38015fc95a2c: copied to local snapshot: bytes=71 +2025-02-25T23:59:06.666Z [INFO] server-a90d62ed-76ca-2d76-745d-38015fc95a2c: snapshot restore progress: id=2-10013-1740527946666 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:59:06.666Z [INFO] server-a90d62ed-76ca-2d76-745d-38015fc95a2c: Installed remote snapshot +2025-02-25T23:59:06.666Z [INFO] server-7a72f73d-286c-0d21-a227-48c2008139fc: pipelining replication: peer="{Voter server-a90d62ed-76ca-2d76-745d-38015fc95a2c a90d62ed-76ca-2d76-745d-38015fc95a2c}" +2025-02-25T23:59:07.670Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:59:06.66265439 +0000 UTC m=+94.271669797 (2 nodes), 640ns from start of poll, 1.675332906s from cluster start. Timeout at 2025-02-25 23:59:07.66765599 +0000 UTC m=+95.276671397, 1.0050016s after stability +2025-02-25T23:59:07.686Z [INFO] server-f3089311-d91f-5fc5-a145-380962bdee87: aborting pipeline replication: peer="{Voter server-6dc4e20d-89a9-ca73-002c-0a7de09e2479 6dc4e20d-89a9-ca73-002c-0a7de09e2479}" +2025-02-25T23:59:07.686Z [INFO] server-f3089311-d91f-5fc5-a145-380962bdee87: aborting pipeline replication: peer="{Voter server-58c16bf5-f633-b037-6efb-ac471ce47c8f 58c16bf5-f633-b037-6efb-ac471ce47c8f}" === RUN TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_true -2026-03-31T06:17:45.435Z [INFO] server-7ebe48ce-1c6c-e92c-7179-691658b6c4fd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7ebe48ce-1c6c-e92c-7179-691658b6c4fd Address:7ebe48ce-1c6c-e92c-7179-691658b6c4fd} {Suffrage:Voter ID:server-f59039a3-393c-9f23-6ff5-3022b4943eed Address:f59039a3-393c-9f23-6ff5-3022b4943eed} {Suffrage:Voter ID:server-a02ac7cc-899e-2ff3-83a3-346f2c92b4d0 Address:a02ac7cc-899e-2ff3-83a3-346f2c92b4d0}]" -2026-03-31T06:17:45.435Z [INFO] server-f59039a3-393c-9f23-6ff5-3022b4943eed: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7ebe48ce-1c6c-e92c-7179-691658b6c4fd Address:7ebe48ce-1c6c-e92c-7179-691658b6c4fd} {Suffrage:Voter ID:server-f59039a3-393c-9f23-6ff5-3022b4943eed Address:f59039a3-393c-9f23-6ff5-3022b4943eed} {Suffrage:Voter ID:server-a02ac7cc-899e-2ff3-83a3-346f2c92b4d0 Address:a02ac7cc-899e-2ff3-83a3-346f2c92b4d0}]" -2026-03-31T06:17:45.435Z [INFO] server-7ebe48ce-1c6c-e92c-7179-691658b6c4fd: entering follower state: follower="Node at 7ebe48ce-1c6c-e92c-7179-691658b6c4fd [Follower]" leader-address= leader-id= -2026-03-31T06:17:45.436Z [INFO] server-a02ac7cc-899e-2ff3-83a3-346f2c92b4d0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7ebe48ce-1c6c-e92c-7179-691658b6c4fd Address:7ebe48ce-1c6c-e92c-7179-691658b6c4fd} {Suffrage:Voter ID:server-f59039a3-393c-9f23-6ff5-3022b4943eed Address:f59039a3-393c-9f23-6ff5-3022b4943eed} {Suffrage:Voter ID:server-a02ac7cc-899e-2ff3-83a3-346f2c92b4d0 Address:a02ac7cc-899e-2ff3-83a3-346f2c92b4d0}]" -2026-03-31T06:17:45.436Z [INFO] server-f59039a3-393c-9f23-6ff5-3022b4943eed: entering follower state: follower="Node at f59039a3-393c-9f23-6ff5-3022b4943eed [Follower]" leader-address= leader-id= -2026-03-31T06:17:45.436Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:45.436Z [INFO] server-a02ac7cc-899e-2ff3-83a3-346f2c92b4d0: entering follower state: follower="Node at a02ac7cc-899e-2ff3-83a3-346f2c92b4d0 [Follower]" leader-address= leader-id= -2026-03-31T06:17:46.077Z [WARN] server-a02ac7cc-899e-2ff3-83a3-346f2c92b4d0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:46.077Z [INFO] server-a02ac7cc-899e-2ff3-83a3-346f2c92b4d0: entering candidate state: node="Node at a02ac7cc-899e-2ff3-83a3-346f2c92b4d0 [Candidate]" term=2 -2026-03-31T06:17:46.077Z [INFO] server-a02ac7cc-899e-2ff3-83a3-346f2c92b4d0: election won: term=2 tally=2 -2026-03-31T06:17:46.077Z [INFO] server-a02ac7cc-899e-2ff3-83a3-346f2c92b4d0: entering leader state: leader="Node at a02ac7cc-899e-2ff3-83a3-346f2c92b4d0 [Leader]" -2026-03-31T06:17:46.077Z [INFO] server-a02ac7cc-899e-2ff3-83a3-346f2c92b4d0: added peer, starting replication: peer=server-7ebe48ce-1c6c-e92c-7179-691658b6c4fd -2026-03-31T06:17:46.077Z [INFO] server-a02ac7cc-899e-2ff3-83a3-346f2c92b4d0: added peer, starting replication: peer=server-f59039a3-393c-9f23-6ff5-3022b4943eed -2026-03-31T06:17:46.077Z [INFO] server-a02ac7cc-899e-2ff3-83a3-346f2c92b4d0: pipelining replication: peer="{Voter server-7ebe48ce-1c6c-e92c-7179-691658b6c4fd 7ebe48ce-1c6c-e92c-7179-691658b6c4fd}" -2026-03-31T06:17:46.077Z [INFO] server-a02ac7cc-899e-2ff3-83a3-346f2c92b4d0: pipelining replication: peer="{Voter server-f59039a3-393c-9f23-6ff5-3022b4943eed f59039a3-393c-9f23-6ff5-3022b4943eed}" -2026-03-31T06:17:47.083Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:46.07746898 +0000 UTC m=+94.355076361 (1 nodes), 641.33208ms from start of poll, 641.8792ms from cluster start. Timeout at 2026-03-31 06:17:47.08248222 +0000 UTC m=+95.360089601, 1.00501324s after stability -2026-03-31T06:17:47.084Z [INFO] server-a02ac7cc-899e-2ff3-83a3-346f2c92b4d0: starting snapshot up to: index=12 -2026-03-31T06:17:47.084Z [INFO] creating new snapshot: path=/tmp/raft99171621/snapshots/2-12-1774937867084.tmp -2026-03-31T06:17:47.084Z [INFO] server-a02ac7cc-899e-2ff3-83a3-346f2c92b4d0: snapshot complete up to: index=12 -2026-03-31T06:17:47.086Z [INFO] server-da2f6390-895e-7c71-7fa1-78a3d93381b5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-da2f6390-895e-7c71-7fa1-78a3d93381b5 Address:da2f6390-895e-7c71-7fa1-78a3d93381b5} {Suffrage:Voter ID:server-26ca502a-b3b7-bc56-c689-5c17590b14a6 Address:26ca502a-b3b7-bc56-c689-5c17590b14a6} {Suffrage:Voter ID:server-67081f75-4633-6a81-dcec-133fa7aefac9 Address:67081f75-4633-6a81-dcec-133fa7aefac9}]" -2026-03-31T06:17:47.086Z [INFO] server-da2f6390-895e-7c71-7fa1-78a3d93381b5: entering follower state: follower="Node at da2f6390-895e-7c71-7fa1-78a3d93381b5 [Follower]" leader-address= leader-id= -2026-03-31T06:17:47.086Z [INFO] server-26ca502a-b3b7-bc56-c689-5c17590b14a6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-da2f6390-895e-7c71-7fa1-78a3d93381b5 Address:da2f6390-895e-7c71-7fa1-78a3d93381b5} {Suffrage:Voter ID:server-26ca502a-b3b7-bc56-c689-5c17590b14a6 Address:26ca502a-b3b7-bc56-c689-5c17590b14a6} {Suffrage:Voter ID:server-67081f75-4633-6a81-dcec-133fa7aefac9 Address:67081f75-4633-6a81-dcec-133fa7aefac9}]" -2026-03-31T06:17:47.087Z [INFO] server-26ca502a-b3b7-bc56-c689-5c17590b14a6: entering follower state: follower="Node at 26ca502a-b3b7-bc56-c689-5c17590b14a6 [Follower]" leader-address= leader-id= -2026-03-31T06:17:47.087Z [INFO] server-67081f75-4633-6a81-dcec-133fa7aefac9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-da2f6390-895e-7c71-7fa1-78a3d93381b5 Address:da2f6390-895e-7c71-7fa1-78a3d93381b5} {Suffrage:Voter ID:server-26ca502a-b3b7-bc56-c689-5c17590b14a6 Address:26ca502a-b3b7-bc56-c689-5c17590b14a6} {Suffrage:Voter ID:server-67081f75-4633-6a81-dcec-133fa7aefac9 Address:67081f75-4633-6a81-dcec-133fa7aefac9}]" -2026-03-31T06:17:47.087Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:47.087Z [INFO] server-67081f75-4633-6a81-dcec-133fa7aefac9: entering follower state: follower="Node at 67081f75-4633-6a81-dcec-133fa7aefac9 [Follower]" leader-address= leader-id= -2026-03-31T06:17:47.616Z [WARN] server-da2f6390-895e-7c71-7fa1-78a3d93381b5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:47.616Z [INFO] server-da2f6390-895e-7c71-7fa1-78a3d93381b5: entering candidate state: node="Node at da2f6390-895e-7c71-7fa1-78a3d93381b5 [Candidate]" term=2 -2026-03-31T06:17:47.616Z [INFO] server-da2f6390-895e-7c71-7fa1-78a3d93381b5: election won: term=2 tally=2 -2026-03-31T06:17:47.616Z [INFO] server-da2f6390-895e-7c71-7fa1-78a3d93381b5: entering leader state: leader="Node at da2f6390-895e-7c71-7fa1-78a3d93381b5 [Leader]" -2026-03-31T06:17:47.616Z [INFO] server-da2f6390-895e-7c71-7fa1-78a3d93381b5: added peer, starting replication: peer=server-26ca502a-b3b7-bc56-c689-5c17590b14a6 -2026-03-31T06:17:47.616Z [INFO] server-da2f6390-895e-7c71-7fa1-78a3d93381b5: added peer, starting replication: peer=server-67081f75-4633-6a81-dcec-133fa7aefac9 -2026-03-31T06:17:47.616Z [INFO] server-da2f6390-895e-7c71-7fa1-78a3d93381b5: pipelining replication: peer="{Voter server-26ca502a-b3b7-bc56-c689-5c17590b14a6 26ca502a-b3b7-bc56-c689-5c17590b14a6}" -2026-03-31T06:17:47.616Z [INFO] server-da2f6390-895e-7c71-7fa1-78a3d93381b5: pipelining replication: peer="{Voter server-67081f75-4633-6a81-dcec-133fa7aefac9 67081f75-4633-6a81-dcec-133fa7aefac9}" -2026-03-31T06:17:48.622Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:47.61659094 +0000 UTC m=+95.894198321 (1 nodes), 528.80156ms from start of poll, 530.26848ms from cluster start. Timeout at 2026-03-31 06:17:48.6216011 +0000 UTC m=+96.899208481, 1.00501016s after stability -2026-03-31T06:17:48.622Z [INFO] creating new snapshot: path=/tmp/raft2797902974/snapshots/2-10013-1774937868622.tmp -2026-03-31T06:17:48.622Z [INFO] server-da2f6390-895e-7c71-7fa1-78a3d93381b5: copied to local snapshot: bytes=71 -2026-03-31T06:17:48.623Z [INFO] server-da2f6390-895e-7c71-7fa1-78a3d93381b5: snapshot restore progress: id=2-10013-1774937868622 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:48.623Z [INFO] server-da2f6390-895e-7c71-7fa1-78a3d93381b5: removing all old logs from log store -2026-03-31T06:17:48.623Z [INFO] server-da2f6390-895e-7c71-7fa1-78a3d93381b5: compacting logs: from=1 to=2 -2026-03-31T06:17:48.623Z [INFO] server-da2f6390-895e-7c71-7fa1-78a3d93381b5: restored user snapshot: index=10013 -2026-03-31T06:17:48.623Z [ERROR] server-da2f6390-895e-7c71-7fa1-78a3d93381b5: failed to get log: index=2 error="log not found" -2026-03-31T06:17:48.623Z [INFO] server-da2f6390-895e-7c71-7fa1-78a3d93381b5: aborting pipeline replication: peer="{Voter server-67081f75-4633-6a81-dcec-133fa7aefac9 67081f75-4633-6a81-dcec-133fa7aefac9}" -2026-03-31T06:17:48.623Z [ERROR] server-da2f6390-895e-7c71-7fa1-78a3d93381b5: failed to get log: index=2 error="log not found" -2026-03-31T06:17:48.623Z [INFO] server-da2f6390-895e-7c71-7fa1-78a3d93381b5: aborting pipeline replication: peer="{Voter server-26ca502a-b3b7-bc56-c689-5c17590b14a6 26ca502a-b3b7-bc56-c689-5c17590b14a6}" -2026-03-31T06:17:48.630Z [ERROR] server-da2f6390-895e-7c71-7fa1-78a3d93381b5: failed to get log: index=2 error="log not found" -2026-03-31T06:17:48.630Z [INFO] creating new snapshot: path=/tmp/raft4274536991/snapshots/2-10013-1774937868630.tmp -2026-03-31T06:17:48.631Z [INFO] server-26ca502a-b3b7-bc56-c689-5c17590b14a6: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:48.631Z [INFO] server-26ca502a-b3b7-bc56-c689-5c17590b14a6: copied to local snapshot: bytes=71 -2026-03-31T06:17:48.631Z [INFO] server-26ca502a-b3b7-bc56-c689-5c17590b14a6: snapshot restore progress: id=2-10013-1774937868630 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:48.631Z [INFO] server-26ca502a-b3b7-bc56-c689-5c17590b14a6: removing all old logs from log store -2026-03-31T06:17:48.631Z [INFO] server-26ca502a-b3b7-bc56-c689-5c17590b14a6: compacting logs: from=1 to=2 -2026-03-31T06:17:48.631Z [INFO] server-26ca502a-b3b7-bc56-c689-5c17590b14a6: Installed remote snapshot -2026-03-31T06:17:48.631Z [INFO] server-da2f6390-895e-7c71-7fa1-78a3d93381b5: pipelining replication: peer="{Voter server-26ca502a-b3b7-bc56-c689-5c17590b14a6 26ca502a-b3b7-bc56-c689-5c17590b14a6}" -2026-03-31T06:17:48.631Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:17:48.632Z [ERROR] server-da2f6390-895e-7c71-7fa1-78a3d93381b5: failed to get log: index=2 error="log not found" -2026-03-31T06:17:48.633Z [INFO] creating new snapshot: path=/tmp/raft2817878173/snapshots/2-10013-1774937868633.tmp -2026-03-31T06:17:48.633Z [INFO] server-67081f75-4633-6a81-dcec-133fa7aefac9: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:48.633Z [INFO] server-67081f75-4633-6a81-dcec-133fa7aefac9: copied to local snapshot: bytes=71 -2026-03-31T06:17:48.634Z [INFO] server-67081f75-4633-6a81-dcec-133fa7aefac9: snapshot restore progress: id=2-10013-1774937868633 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2026-03-31T06:17:48.634Z [INFO] server-67081f75-4633-6a81-dcec-133fa7aefac9: removing all old logs from log store -2026-03-31T06:17:48.634Z [INFO] server-67081f75-4633-6a81-dcec-133fa7aefac9: compacting logs: from=1 to=2 -2026-03-31T06:17:48.634Z [INFO] server-67081f75-4633-6a81-dcec-133fa7aefac9: Installed remote snapshot -2026-03-31T06:17:48.634Z [INFO] server-da2f6390-895e-7c71-7fa1-78a3d93381b5: pipelining replication: peer="{Voter server-67081f75-4633-6a81-dcec-133fa7aefac9 67081f75-4633-6a81-dcec-133fa7aefac9}" -2026-03-31T06:17:49.637Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:17:48.6317991 +0000 UTC m=+96.909406481 (2 nodes), 1µs from start of poll, 1.54547664s from cluster start. Timeout at 2026-03-31 06:17:49.6368217 +0000 UTC m=+97.914429121, 1.00502264s after stability -2026-03-31T06:17:49.649Z [INFO] server-a02ac7cc-899e-2ff3-83a3-346f2c92b4d0: aborting pipeline replication: peer="{Voter server-f59039a3-393c-9f23-6ff5-3022b4943eed f59039a3-393c-9f23-6ff5-3022b4943eed}" -2026-03-31T06:17:49.649Z [INFO] server-a02ac7cc-899e-2ff3-83a3-346f2c92b4d0: aborting pipeline replication: peer="{Voter server-7ebe48ce-1c6c-e92c-7179-691658b6c4fd 7ebe48ce-1c6c-e92c-7179-691658b6c4fd}" ---- PASS: TestRaft_UserRestore (83.82s) - --- PASS: TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_false (2.55s) - --- PASS: TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_false (2.56s) - --- PASS: TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_true (4.22s) - --- PASS: TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_true (4.66s) - --- PASS: TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_false (2.72s) - --- PASS: TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_false (2.63s) - --- PASS: TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_true (4.45s) - --- PASS: TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_true (4.32s) - --- PASS: TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_false (2.88s) - --- PASS: TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_false (2.61s) - --- PASS: TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_true (4.31s) +2025-02-25T23:59:07.686Z [INFO] server-01886301-af05-677b-7135-68929affb5fb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-01886301-af05-677b-7135-68929affb5fb Address:01886301-af05-677b-7135-68929affb5fb} {Suffrage:Voter ID:server-1c987459-00ac-6a70-f170-270824606518 Address:1c987459-00ac-6a70-f170-270824606518} {Suffrage:Voter ID:server-94aa56a4-ca75-1211-88ff-9a0a1ebbd83b Address:94aa56a4-ca75-1211-88ff-9a0a1ebbd83b}]" +2025-02-25T23:59:07.686Z [INFO] server-1c987459-00ac-6a70-f170-270824606518: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-01886301-af05-677b-7135-68929affb5fb Address:01886301-af05-677b-7135-68929affb5fb} {Suffrage:Voter ID:server-1c987459-00ac-6a70-f170-270824606518 Address:1c987459-00ac-6a70-f170-270824606518} {Suffrage:Voter ID:server-94aa56a4-ca75-1211-88ff-9a0a1ebbd83b Address:94aa56a4-ca75-1211-88ff-9a0a1ebbd83b}]" +2025-02-25T23:59:07.686Z [INFO] server-94aa56a4-ca75-1211-88ff-9a0a1ebbd83b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-01886301-af05-677b-7135-68929affb5fb Address:01886301-af05-677b-7135-68929affb5fb} {Suffrage:Voter ID:server-1c987459-00ac-6a70-f170-270824606518 Address:1c987459-00ac-6a70-f170-270824606518} {Suffrage:Voter ID:server-94aa56a4-ca75-1211-88ff-9a0a1ebbd83b Address:94aa56a4-ca75-1211-88ff-9a0a1ebbd83b}]" +2025-02-25T23:59:07.686Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:07.686Z [INFO] server-01886301-af05-677b-7135-68929affb5fb: entering follower state: follower="Node at 01886301-af05-677b-7135-68929affb5fb [Follower]" leader-address= leader-id= +2025-02-25T23:59:07.686Z [INFO] server-1c987459-00ac-6a70-f170-270824606518: entering follower state: follower="Node at 1c987459-00ac-6a70-f170-270824606518 [Follower]" leader-address= leader-id= +2025-02-25T23:59:07.686Z [INFO] server-94aa56a4-ca75-1211-88ff-9a0a1ebbd83b: entering follower state: follower="Node at 94aa56a4-ca75-1211-88ff-9a0a1ebbd83b [Follower]" leader-address= leader-id= +2025-02-25T23:59:08.426Z [WARN] server-1c987459-00ac-6a70-f170-270824606518: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:08.426Z [INFO] server-1c987459-00ac-6a70-f170-270824606518: entering candidate state: node="Node at 1c987459-00ac-6a70-f170-270824606518 [Candidate]" term=2 +2025-02-25T23:59:08.426Z [INFO] server-1c987459-00ac-6a70-f170-270824606518: election won: term=2 tally=2 +2025-02-25T23:59:08.426Z [INFO] server-1c987459-00ac-6a70-f170-270824606518: entering leader state: leader="Node at 1c987459-00ac-6a70-f170-270824606518 [Leader]" +2025-02-25T23:59:08.426Z [INFO] server-1c987459-00ac-6a70-f170-270824606518: added peer, starting replication: peer=server-01886301-af05-677b-7135-68929affb5fb +2025-02-25T23:59:08.426Z [INFO] server-1c987459-00ac-6a70-f170-270824606518: added peer, starting replication: peer=server-94aa56a4-ca75-1211-88ff-9a0a1ebbd83b +2025-02-25T23:59:08.426Z [INFO] server-1c987459-00ac-6a70-f170-270824606518: pipelining replication: peer="{Voter server-94aa56a4-ca75-1211-88ff-9a0a1ebbd83b 94aa56a4-ca75-1211-88ff-9a0a1ebbd83b}" +2025-02-25T23:59:08.426Z [INFO] server-1c987459-00ac-6a70-f170-270824606518: pipelining replication: peer="{Voter server-01886301-af05-677b-7135-68929affb5fb 01886301-af05-677b-7135-68929affb5fb}" +2025-02-25T23:59:09.433Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:08.4264719 +0000 UTC m=+96.035487307 (1 nodes), 739.564989ms from start of poll, 739.74135ms from cluster start. Timeout at 2025-02-25 23:59:09.43148942 +0000 UTC m=+97.040504787, 1.00501748s after stability +2025-02-25T23:59:09.434Z [INFO] server-1c987459-00ac-6a70-f170-270824606518: starting snapshot up to: index=12 +2025-02-25T23:59:09.434Z [INFO] creating new snapshot: path=/tmp/raft2988785581/snapshots/2-12-1740527949434.tmp +2025-02-25T23:59:09.434Z [INFO] server-1c987459-00ac-6a70-f170-270824606518: snapshot complete up to: index=12 +2025-02-25T23:59:09.435Z [INFO] server-8a5edf56-a061-fee2-a3dd-093c062a24c4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8a5edf56-a061-fee2-a3dd-093c062a24c4 Address:8a5edf56-a061-fee2-a3dd-093c062a24c4} {Suffrage:Voter ID:server-3dcaf9ca-256f-87f0-fde3-766062889b51 Address:3dcaf9ca-256f-87f0-fde3-766062889b51} {Suffrage:Voter ID:server-4cee3d37-1ba3-0ad7-28d7-149218c7859c Address:4cee3d37-1ba3-0ad7-28d7-149218c7859c}]" +2025-02-25T23:59:09.435Z [INFO] server-3dcaf9ca-256f-87f0-fde3-766062889b51: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8a5edf56-a061-fee2-a3dd-093c062a24c4 Address:8a5edf56-a061-fee2-a3dd-093c062a24c4} {Suffrage:Voter ID:server-3dcaf9ca-256f-87f0-fde3-766062889b51 Address:3dcaf9ca-256f-87f0-fde3-766062889b51} {Suffrage:Voter ID:server-4cee3d37-1ba3-0ad7-28d7-149218c7859c Address:4cee3d37-1ba3-0ad7-28d7-149218c7859c}]" +2025-02-25T23:59:09.435Z [INFO] server-4cee3d37-1ba3-0ad7-28d7-149218c7859c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8a5edf56-a061-fee2-a3dd-093c062a24c4 Address:8a5edf56-a061-fee2-a3dd-093c062a24c4} {Suffrage:Voter ID:server-3dcaf9ca-256f-87f0-fde3-766062889b51 Address:3dcaf9ca-256f-87f0-fde3-766062889b51} {Suffrage:Voter ID:server-4cee3d37-1ba3-0ad7-28d7-149218c7859c Address:4cee3d37-1ba3-0ad7-28d7-149218c7859c}]" +2025-02-25T23:59:09.435Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:09.435Z [INFO] server-8a5edf56-a061-fee2-a3dd-093c062a24c4: entering follower state: follower="Node at 8a5edf56-a061-fee2-a3dd-093c062a24c4 [Follower]" leader-address= leader-id= +2025-02-25T23:59:09.435Z [INFO] server-3dcaf9ca-256f-87f0-fde3-766062889b51: entering follower state: follower="Node at 3dcaf9ca-256f-87f0-fde3-766062889b51 [Follower]" leader-address= leader-id= +2025-02-25T23:59:09.435Z [INFO] server-4cee3d37-1ba3-0ad7-28d7-149218c7859c: entering follower state: follower="Node at 4cee3d37-1ba3-0ad7-28d7-149218c7859c [Follower]" leader-address= leader-id= +2025-02-25T23:59:09.970Z [WARN] server-8a5edf56-a061-fee2-a3dd-093c062a24c4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:09.970Z [INFO] server-8a5edf56-a061-fee2-a3dd-093c062a24c4: entering candidate state: node="Node at 8a5edf56-a061-fee2-a3dd-093c062a24c4 [Candidate]" term=2 +2025-02-25T23:59:09.970Z [INFO] server-8a5edf56-a061-fee2-a3dd-093c062a24c4: election won: term=2 tally=2 +2025-02-25T23:59:09.970Z [INFO] server-8a5edf56-a061-fee2-a3dd-093c062a24c4: entering leader state: leader="Node at 8a5edf56-a061-fee2-a3dd-093c062a24c4 [Leader]" +2025-02-25T23:59:09.970Z [INFO] server-8a5edf56-a061-fee2-a3dd-093c062a24c4: added peer, starting replication: peer=server-3dcaf9ca-256f-87f0-fde3-766062889b51 +2025-02-25T23:59:09.970Z [INFO] server-8a5edf56-a061-fee2-a3dd-093c062a24c4: added peer, starting replication: peer=server-4cee3d37-1ba3-0ad7-28d7-149218c7859c +2025-02-25T23:59:09.970Z [INFO] server-8a5edf56-a061-fee2-a3dd-093c062a24c4: pipelining replication: peer="{Voter server-4cee3d37-1ba3-0ad7-28d7-149218c7859c 4cee3d37-1ba3-0ad7-28d7-149218c7859c}" +2025-02-25T23:59:09.970Z [INFO] server-8a5edf56-a061-fee2-a3dd-093c062a24c4: pipelining replication: peer="{Voter server-3dcaf9ca-256f-87f0-fde3-766062889b51 3dcaf9ca-256f-87f0-fde3-766062889b51}" +2025-02-25T23:59:10.978Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:09.970461209 +0000 UTC m=+97.579476616 (1 nodes), 534.947925ms from start of poll, 535.104606ms from cluster start. Timeout at 2025-02-25 23:59:10.975464209 +0000 UTC m=+98.584479616, 1.005003s after stability +2025-02-25T23:59:10.978Z [INFO] creating new snapshot: path=/tmp/raft3690334492/snapshots/2-10013-1740527950978.tmp +2025-02-25T23:59:10.978Z [INFO] server-8a5edf56-a061-fee2-a3dd-093c062a24c4: copied to local snapshot: bytes=71 +2025-02-25T23:59:10.978Z [INFO] server-8a5edf56-a061-fee2-a3dd-093c062a24c4: snapshot restore progress: id=2-10013-1740527950978 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:59:10.978Z [INFO] server-8a5edf56-a061-fee2-a3dd-093c062a24c4: removing all old logs from log store +2025-02-25T23:59:10.978Z [INFO] server-8a5edf56-a061-fee2-a3dd-093c062a24c4: compacting logs: from=1 to=2 +2025-02-25T23:59:10.978Z [INFO] server-8a5edf56-a061-fee2-a3dd-093c062a24c4: restored user snapshot: index=10013 +2025-02-25T23:59:10.979Z [ERROR] server-8a5edf56-a061-fee2-a3dd-093c062a24c4: failed to get log: index=2 error="log not found" +2025-02-25T23:59:10.979Z [INFO] server-8a5edf56-a061-fee2-a3dd-093c062a24c4: aborting pipeline replication: peer="{Voter server-4cee3d37-1ba3-0ad7-28d7-149218c7859c 4cee3d37-1ba3-0ad7-28d7-149218c7859c}" +2025-02-25T23:59:10.979Z [ERROR] server-8a5edf56-a061-fee2-a3dd-093c062a24c4: failed to get log: index=2 error="log not found" +2025-02-25T23:59:10.979Z [INFO] server-8a5edf56-a061-fee2-a3dd-093c062a24c4: aborting pipeline replication: peer="{Voter server-3dcaf9ca-256f-87f0-fde3-766062889b51 3dcaf9ca-256f-87f0-fde3-766062889b51}" +2025-02-25T23:59:10.989Z [ERROR] server-8a5edf56-a061-fee2-a3dd-093c062a24c4: failed to get log: index=2 error="log not found" +2025-02-25T23:59:10.990Z [INFO] creating new snapshot: path=/tmp/raft4211834623/snapshots/2-10013-1740527950990.tmp +2025-02-25T23:59:10.990Z [INFO] server-3dcaf9ca-256f-87f0-fde3-766062889b51: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:59:10.990Z [INFO] server-3dcaf9ca-256f-87f0-fde3-766062889b51: copied to local snapshot: bytes=71 +2025-02-25T23:59:10.990Z [INFO] server-3dcaf9ca-256f-87f0-fde3-766062889b51: snapshot restore progress: id=2-10013-1740527950990 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:59:10.990Z [INFO] server-3dcaf9ca-256f-87f0-fde3-766062889b51: removing all old logs from log store +2025-02-25T23:59:10.990Z [INFO] server-3dcaf9ca-256f-87f0-fde3-766062889b51: compacting logs: from=1 to=2 +2025-02-25T23:59:10.990Z [INFO] server-3dcaf9ca-256f-87f0-fde3-766062889b51: Installed remote snapshot +2025-02-25T23:59:10.990Z [ERROR] server-8a5edf56-a061-fee2-a3dd-093c062a24c4: failed to get log: index=2 error="log not found" +2025-02-25T23:59:10.990Z [INFO] creating new snapshot: path=/tmp/raft115835298/snapshots/2-10013-1740527950990.tmp +2025-02-25T23:59:10.990Z [INFO] server-4cee3d37-1ba3-0ad7-28d7-149218c7859c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-02-25T23:59:10.990Z [INFO] server-4cee3d37-1ba3-0ad7-28d7-149218c7859c: copied to local snapshot: bytes=71 +2025-02-25T23:59:10.991Z [INFO] server-8a5edf56-a061-fee2-a3dd-093c062a24c4: pipelining replication: peer="{Voter server-3dcaf9ca-256f-87f0-fde3-766062889b51 3dcaf9ca-256f-87f0-fde3-766062889b51}" +2025-02-25T23:59:10.991Z [INFO] server-4cee3d37-1ba3-0ad7-28d7-149218c7859c: snapshot restore progress: id=2-10013-1740527950990 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-02-25T23:59:10.991Z [INFO] server-4cee3d37-1ba3-0ad7-28d7-149218c7859c: removing all old logs from log store +2025-02-25T23:59:10.991Z [INFO] server-4cee3d37-1ba3-0ad7-28d7-149218c7859c: compacting logs: from=1 to=2 +2025-02-25T23:59:10.991Z [INFO] server-4cee3d37-1ba3-0ad7-28d7-149218c7859c: Installed remote snapshot +2025-02-25T23:59:10.991Z [WARN] server-4cee3d37-1ba3-0ad7-28d7-149218c7859c: skipping application of old log: index=10013 +2025-02-25T23:59:10.991Z [INFO] server-8a5edf56-a061-fee2-a3dd-093c062a24c4: pipelining replication: peer="{Voter server-4cee3d37-1ba3-0ad7-28d7-149218c7859c 4cee3d37-1ba3-0ad7-28d7-149218c7859c}" +2025-02-25T23:59:10.991Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:59:11.998Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:59:10.991365749 +0000 UTC m=+98.600381156 (2 nodes), 520ns from start of poll, 1.556009146s from cluster start. Timeout at 2025-02-25 23:59:11.99638143 +0000 UTC m=+99.605396797, 1.005015641s after stability +2025-02-25T23:59:12.014Z [INFO] server-1c987459-00ac-6a70-f170-270824606518: aborting pipeline replication: peer="{Voter server-94aa56a4-ca75-1211-88ff-9a0a1ebbd83b 94aa56a4-ca75-1211-88ff-9a0a1ebbd83b}" +2025-02-25T23:59:12.014Z [INFO] server-1c987459-00ac-6a70-f170-270824606518: aborting pipeline replication: peer="{Voter server-01886301-af05-677b-7135-68929affb5fb 01886301-af05-677b-7135-68929affb5fb}" +--- PASS: TestRaft_UserRestore (84.68s) + --- PASS: TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_false (2.92s) + --- PASS: TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_false (2.67s) + --- PASS: TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_true (4.40s) + --- PASS: TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_true (4.23s) + --- PASS: TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_false (2.73s) + --- PASS: TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_false (2.55s) + --- PASS: TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_true (4.32s) + --- PASS: TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_true (4.59s) + --- PASS: TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_false (2.64s) + --- PASS: TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_false (2.76s) + --- PASS: TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_true (4.32s) --- PASS: TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_true (4.33s) - --- PASS: TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_false (2.58s) - --- PASS: TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_false (2.57s) - --- PASS: TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_true (4.35s) - --- PASS: TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_true (4.30s) - --- PASS: TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_false (2.72s) - --- PASS: TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_false (2.68s) - --- PASS: TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_true (4.20s) - --- PASS: TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_true (4.39s) - --- PASS: TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_false (2.54s) - --- PASS: TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_false (2.75s) - --- PASS: TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_true (4.31s) - --- PASS: TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_true (4.21s) + --- PASS: TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_false (2.90s) + --- PASS: TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_false (2.63s) + --- PASS: TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_true (4.28s) + --- PASS: TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_true (4.24s) + --- PASS: TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_false (2.84s) + --- PASS: TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_false (2.74s) + --- PASS: TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_true (4.22s) + --- PASS: TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_true (4.28s) + --- PASS: TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_false (2.78s) + --- PASS: TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_false (2.71s) + --- PASS: TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_true (4.28s) + --- PASS: TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_true (4.33s) === RUN TestRaft_SendSnapshotFollower -2026-03-31T06:17:49.650Z [INFO] server-a2cbb12f-846a-e8c9-ba36-d3436e519df8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a2cbb12f-846a-e8c9-ba36-d3436e519df8 Address:a2cbb12f-846a-e8c9-ba36-d3436e519df8} {Suffrage:Voter ID:server-8ee4efb1-6c7d-f6f2-353c-07328eb60cbd Address:8ee4efb1-6c7d-f6f2-353c-07328eb60cbd} {Suffrage:Voter ID:server-b45d2b32-2a18-365a-9803-64d02476f42d Address:b45d2b32-2a18-365a-9803-64d02476f42d}]" -2026-03-31T06:17:49.650Z [INFO] server-8ee4efb1-6c7d-f6f2-353c-07328eb60cbd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a2cbb12f-846a-e8c9-ba36-d3436e519df8 Address:a2cbb12f-846a-e8c9-ba36-d3436e519df8} {Suffrage:Voter ID:server-8ee4efb1-6c7d-f6f2-353c-07328eb60cbd Address:8ee4efb1-6c7d-f6f2-353c-07328eb60cbd} {Suffrage:Voter ID:server-b45d2b32-2a18-365a-9803-64d02476f42d Address:b45d2b32-2a18-365a-9803-64d02476f42d}]" -2026-03-31T06:17:49.650Z [INFO] server-a2cbb12f-846a-e8c9-ba36-d3436e519df8: entering follower state: follower="Node at a2cbb12f-846a-e8c9-ba36-d3436e519df8 [Follower]" leader-address= leader-id= -2026-03-31T06:17:49.650Z [INFO] server-8ee4efb1-6c7d-f6f2-353c-07328eb60cbd: entering follower state: follower="Node at 8ee4efb1-6c7d-f6f2-353c-07328eb60cbd [Follower]" leader-address= leader-id= -2026-03-31T06:17:49.650Z [INFO] server-b45d2b32-2a18-365a-9803-64d02476f42d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a2cbb12f-846a-e8c9-ba36-d3436e519df8 Address:a2cbb12f-846a-e8c9-ba36-d3436e519df8} {Suffrage:Voter ID:server-8ee4efb1-6c7d-f6f2-353c-07328eb60cbd Address:8ee4efb1-6c7d-f6f2-353c-07328eb60cbd} {Suffrage:Voter ID:server-b45d2b32-2a18-365a-9803-64d02476f42d Address:b45d2b32-2a18-365a-9803-64d02476f42d}]" -2026-03-31T06:17:49.650Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:17:49.650Z [INFO] server-b45d2b32-2a18-365a-9803-64d02476f42d: entering follower state: follower="Node at b45d2b32-2a18-365a-9803-64d02476f42d [Follower]" leader-address= leader-id= -2026-03-31T06:17:49.716Z [WARN] server-b45d2b32-2a18-365a-9803-64d02476f42d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:49.716Z [INFO] server-b45d2b32-2a18-365a-9803-64d02476f42d: entering candidate state: node="Node at b45d2b32-2a18-365a-9803-64d02476f42d [Candidate]" term=2 -2026-03-31T06:17:49.716Z [INFO] server-b45d2b32-2a18-365a-9803-64d02476f42d: election won: term=2 tally=2 -2026-03-31T06:17:49.716Z [INFO] server-b45d2b32-2a18-365a-9803-64d02476f42d: entering leader state: leader="Node at b45d2b32-2a18-365a-9803-64d02476f42d [Leader]" -2026-03-31T06:17:49.716Z [INFO] server-b45d2b32-2a18-365a-9803-64d02476f42d: added peer, starting replication: peer=server-a2cbb12f-846a-e8c9-ba36-d3436e519df8 -2026-03-31T06:17:49.716Z [INFO] server-b45d2b32-2a18-365a-9803-64d02476f42d: added peer, starting replication: peer=server-8ee4efb1-6c7d-f6f2-353c-07328eb60cbd -2026-03-31T06:17:49.716Z [INFO] server-b45d2b32-2a18-365a-9803-64d02476f42d: pipelining replication: peer="{Voter server-a2cbb12f-846a-e8c9-ba36-d3436e519df8 a2cbb12f-846a-e8c9-ba36-d3436e519df8}" -2026-03-31T06:17:49.716Z [INFO] server-b45d2b32-2a18-365a-9803-64d02476f42d: pipelining replication: peer="{Voter server-8ee4efb1-6c7d-f6f2-353c-07328eb60cbd 8ee4efb1-6c7d-f6f2-353c-07328eb60cbd}" -2026-03-31T06:17:49.822Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:17:49.71663382 +0000 UTC m=+97.994241201 (2 nodes), 66.04228ms from start of poll, 66.4384ms from cluster start. Timeout at 2026-03-31 06:17:49.8216785 +0000 UTC m=+98.099285881, 105.04468ms after stability -2026-03-31T06:17:49.822Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:49.928Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:49.8226477 +0000 UTC m=+98.100255081 (1 nodes), 1.88µs from start of poll, 172.45228ms from cluster start. Timeout at 2026-03-31 06:17:49.92773646 +0000 UTC m=+98.205343961, 105.08888ms after stability -2026-03-31T06:17:49.928Z [ERROR] server-b45d2b32-2a18-365a-9803-64d02476f42d: failed to pipeline appendEntries: peer="{Voter server-a2cbb12f-846a-e8c9-ba36-d3436e519df8 a2cbb12f-846a-e8c9-ba36-d3436e519df8}" error="append pipeline closed" -2026-03-31T06:17:49.928Z [INFO] server-b45d2b32-2a18-365a-9803-64d02476f42d: aborting pipeline replication: peer="{Voter server-a2cbb12f-846a-e8c9-ba36-d3436e519df8 a2cbb12f-846a-e8c9-ba36-d3436e519df8}" -2026-03-31T06:17:49.928Z [ERROR] server-b45d2b32-2a18-365a-9803-64d02476f42d: failed to appendEntries to: peer="{Voter server-a2cbb12f-846a-e8c9-ba36-d3436e519df8 a2cbb12f-846a-e8c9-ba36-d3436e519df8}" error="failed to connect to peer: a2cbb12f-846a-e8c9-ba36-d3436e519df8" -2026-03-31T06:17:49.929Z [ERROR] server-b45d2b32-2a18-365a-9803-64d02476f42d: failed to heartbeat to: peer=a2cbb12f-846a-e8c9-ba36-d3436e519df8 backoff time=10ms error="failed to connect to peer: a2cbb12f-846a-e8c9-ba36-d3436e519df8" +2025-02-25T23:59:12.014Z [INFO] server-16d74bcb-2f93-abdf-f9df-08eb5d66b300: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-16d74bcb-2f93-abdf-f9df-08eb5d66b300 Address:16d74bcb-2f93-abdf-f9df-08eb5d66b300} {Suffrage:Voter ID:server-98852d83-cf8a-1c18-3b6e-e8f589008095 Address:98852d83-cf8a-1c18-3b6e-e8f589008095} {Suffrage:Voter ID:server-086933d2-ee6d-ade9-ebf5-bebbd83fd475 Address:086933d2-ee6d-ade9-ebf5-bebbd83fd475}]" +2025-02-25T23:59:12.014Z [INFO] server-98852d83-cf8a-1c18-3b6e-e8f589008095: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-16d74bcb-2f93-abdf-f9df-08eb5d66b300 Address:16d74bcb-2f93-abdf-f9df-08eb5d66b300} {Suffrage:Voter ID:server-98852d83-cf8a-1c18-3b6e-e8f589008095 Address:98852d83-cf8a-1c18-3b6e-e8f589008095} {Suffrage:Voter ID:server-086933d2-ee6d-ade9-ebf5-bebbd83fd475 Address:086933d2-ee6d-ade9-ebf5-bebbd83fd475}]" +2025-02-25T23:59:12.014Z [INFO] server-086933d2-ee6d-ade9-ebf5-bebbd83fd475: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-16d74bcb-2f93-abdf-f9df-08eb5d66b300 Address:16d74bcb-2f93-abdf-f9df-08eb5d66b300} {Suffrage:Voter ID:server-98852d83-cf8a-1c18-3b6e-e8f589008095 Address:98852d83-cf8a-1c18-3b6e-e8f589008095} {Suffrage:Voter ID:server-086933d2-ee6d-ade9-ebf5-bebbd83fd475 Address:086933d2-ee6d-ade9-ebf5-bebbd83fd475}]" +2025-02-25T23:59:12.014Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:59:12.014Z [INFO] server-16d74bcb-2f93-abdf-f9df-08eb5d66b300: entering follower state: follower="Node at 16d74bcb-2f93-abdf-f9df-08eb5d66b300 [Follower]" leader-address= leader-id= +2025-02-25T23:59:12.014Z [INFO] server-98852d83-cf8a-1c18-3b6e-e8f589008095: entering follower state: follower="Node at 98852d83-cf8a-1c18-3b6e-e8f589008095 [Follower]" leader-address= leader-id= +2025-02-25T23:59:12.014Z [INFO] server-086933d2-ee6d-ade9-ebf5-bebbd83fd475: entering follower state: follower="Node at 086933d2-ee6d-ade9-ebf5-bebbd83fd475 [Follower]" leader-address= leader-id= +2025-02-25T23:59:12.094Z [WARN] server-98852d83-cf8a-1c18-3b6e-e8f589008095: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:12.094Z [INFO] server-98852d83-cf8a-1c18-3b6e-e8f589008095: entering candidate state: node="Node at 98852d83-cf8a-1c18-3b6e-e8f589008095 [Candidate]" term=2 +2025-02-25T23:59:12.094Z [INFO] server-98852d83-cf8a-1c18-3b6e-e8f589008095: election won: term=2 tally=2 +2025-02-25T23:59:12.094Z [INFO] server-98852d83-cf8a-1c18-3b6e-e8f589008095: entering leader state: leader="Node at 98852d83-cf8a-1c18-3b6e-e8f589008095 [Leader]" +2025-02-25T23:59:12.094Z [INFO] server-98852d83-cf8a-1c18-3b6e-e8f589008095: added peer, starting replication: peer=server-16d74bcb-2f93-abdf-f9df-08eb5d66b300 +2025-02-25T23:59:12.094Z [INFO] server-98852d83-cf8a-1c18-3b6e-e8f589008095: added peer, starting replication: peer=server-086933d2-ee6d-ade9-ebf5-bebbd83fd475 +2025-02-25T23:59:12.094Z [INFO] server-98852d83-cf8a-1c18-3b6e-e8f589008095: pipelining replication: peer="{Voter server-086933d2-ee6d-ade9-ebf5-bebbd83fd475 086933d2-ee6d-ade9-ebf5-bebbd83fd475}" +2025-02-25T23:59:12.094Z [WARN] server-16d74bcb-2f93-abdf-f9df-08eb5d66b300: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:12.094Z [INFO] server-16d74bcb-2f93-abdf-f9df-08eb5d66b300: entering candidate state: node="Node at 16d74bcb-2f93-abdf-f9df-08eb5d66b300 [Candidate]" term=2 +2025-02-25T23:59:12.094Z [WARN] server-086933d2-ee6d-ade9-ebf5-bebbd83fd475: rejecting vote request since we have a leader: from=16d74bcb-2f93-abdf-f9df-08eb5d66b300 leader=98852d83-cf8a-1c18-3b6e-e8f589008095 leader-id=server-98852d83-cf8a-1c18-3b6e-e8f589008095 +2025-02-25T23:59:12.094Z [INFO] server-16d74bcb-2f93-abdf-f9df-08eb5d66b300: duplicate requestVote for same term: term=2 +2025-02-25T23:59:12.094Z [INFO] server-16d74bcb-2f93-abdf-f9df-08eb5d66b300: entering follower state: follower="Node at 16d74bcb-2f93-abdf-f9df-08eb5d66b300 [Follower]" leader-address=98852d83-cf8a-1c18-3b6e-e8f589008095 leader-id=server-98852d83-cf8a-1c18-3b6e-e8f589008095 +2025-02-25T23:59:12.094Z [INFO] server-98852d83-cf8a-1c18-3b6e-e8f589008095: pipelining replication: peer="{Voter server-16d74bcb-2f93-abdf-f9df-08eb5d66b300 16d74bcb-2f93-abdf-f9df-08eb5d66b300}" +2025-02-25T23:59:12.094Z [WARN] server-98852d83-cf8a-1c18-3b6e-e8f589008095: rejecting vote request since we have a leader: from=16d74bcb-2f93-abdf-f9df-08eb5d66b300 leader=98852d83-cf8a-1c18-3b6e-e8f589008095 leader-id=server-98852d83-cf8a-1c18-3b6e-e8f589008095 +2025-02-25T23:59:12.202Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:59:12.094625373 +0000 UTC m=+99.703640780 (2 nodes), 79.879549ms from start of poll, 80.04855ms from cluster start. Timeout at 2025-02-25 23:59:12.199627973 +0000 UTC m=+99.808643340, 105.00256ms after stability +2025-02-25T23:59:12.202Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:12.309Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:12.202288498 +0000 UTC m=+99.811303905 (1 nodes), 1.24µs from start of poll, 187.711675ms from cluster start. Timeout at 2025-02-25 23:59:12.307333539 +0000 UTC m=+99.916348946, 105.045041ms after stability +2025-02-25T23:59:12.309Z [ERROR] server-98852d83-cf8a-1c18-3b6e-e8f589008095: failed to pipeline appendEntries: peer="{Voter server-16d74bcb-2f93-abdf-f9df-08eb5d66b300 16d74bcb-2f93-abdf-f9df-08eb5d66b300}" error="append pipeline closed" +2025-02-25T23:59:12.309Z [INFO] server-98852d83-cf8a-1c18-3b6e-e8f589008095: aborting pipeline replication: peer="{Voter server-16d74bcb-2f93-abdf-f9df-08eb5d66b300 16d74bcb-2f93-abdf-f9df-08eb5d66b300}" +2025-02-25T23:59:12.309Z [ERROR] server-98852d83-cf8a-1c18-3b6e-e8f589008095: failed to appendEntries to: peer="{Voter server-16d74bcb-2f93-abdf-f9df-08eb5d66b300 16d74bcb-2f93-abdf-f9df-08eb5d66b300}" error="failed to connect to peer: 16d74bcb-2f93-abdf-f9df-08eb5d66b300" +2025-02-25T23:59:12.314Z [ERROR] server-98852d83-cf8a-1c18-3b6e-e8f589008095: failed to heartbeat to: peer=16d74bcb-2f93-abdf-f9df-08eb5d66b300 backoff time=10ms error="failed to connect to peer: 16d74bcb-2f93-abdf-f9df-08eb5d66b300" raft_test.go:1627: [INFO] Finished apply without behind follower -2026-03-31T06:17:49.932Z [ERROR] server-a2cbb12f-846a-e8c9-ba36-d3436e519df8: failed to take snapshot: error="nothing new to snapshot" -2026-03-31T06:17:49.932Z [INFO] server-8ee4efb1-6c7d-f6f2-353c-07328eb60cbd: starting snapshot up to: index=98 -2026-03-31T06:17:49.932Z [INFO] creating new snapshot: path=/tmp/raft1606399254/snapshots/2-98-1774937869932.tmp -2026-03-31T06:17:49.933Z [INFO] server-8ee4efb1-6c7d-f6f2-353c-07328eb60cbd: compacting logs: from=1 to=92 -2026-03-31T06:17:49.933Z [INFO] server-8ee4efb1-6c7d-f6f2-353c-07328eb60cbd: snapshot complete up to: index=98 -2026-03-31T06:17:49.933Z [INFO] server-b45d2b32-2a18-365a-9803-64d02476f42d: starting snapshot up to: index=102 -2026-03-31T06:17:49.933Z [INFO] creating new snapshot: path=/tmp/raft3252572363/snapshots/2-102-1774937869933.tmp -2026-03-31T06:17:49.933Z [INFO] server-b45d2b32-2a18-365a-9803-64d02476f42d: compacting logs: from=1 to=92 -2026-03-31T06:17:49.933Z [INFO] server-b45d2b32-2a18-365a-9803-64d02476f42d: snapshot complete up to: index=102 -2026-03-31T06:17:49.939Z [ERROR] server-b45d2b32-2a18-365a-9803-64d02476f42d: failed to get log: index=2 error="log not found" -2026-03-31T06:17:49.940Z [INFO] creating new snapshot: path=/tmp/raft2753935963/snapshots/2-102-1774937869940.tmp -2026-03-31T06:17:49.940Z [INFO] server-a2cbb12f-846a-e8c9-ba36-d3436e519df8: snapshot network transfer progress: read-bytes=693 percent-complete=100.00% -2026-03-31T06:17:49.941Z [INFO] server-a2cbb12f-846a-e8c9-ba36-d3436e519df8: copied to local snapshot: bytes=693 -2026-03-31T06:17:49.941Z [INFO] server-a2cbb12f-846a-e8c9-ba36-d3436e519df8: snapshot restore progress: id=2-102-1774937869940 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% -2026-03-31T06:17:49.941Z [INFO] server-a2cbb12f-846a-e8c9-ba36-d3436e519df8: Installed remote snapshot -2026-03-31T06:17:49.941Z [WARN] server-a2cbb12f-846a-e8c9-ba36-d3436e519df8: skipping application of old log: index=102 -2026-03-31T06:17:49.941Z [INFO] server-b45d2b32-2a18-365a-9803-64d02476f42d: pipelining replication: peer="{Voter server-a2cbb12f-846a-e8c9-ba36-d3436e519df8 a2cbb12f-846a-e8c9-ba36-d3436e519df8}" -2026-03-31T06:17:49.944Z [INFO] server-b45d2b32-2a18-365a-9803-64d02476f42d: aborting pipeline replication: peer="{Voter server-a2cbb12f-846a-e8c9-ba36-d3436e519df8 a2cbb12f-846a-e8c9-ba36-d3436e519df8}" -2026-03-31T06:17:49.944Z [INFO] server-b45d2b32-2a18-365a-9803-64d02476f42d: aborting pipeline replication: peer="{Voter server-8ee4efb1-6c7d-f6f2-353c-07328eb60cbd 8ee4efb1-6c7d-f6f2-353c-07328eb60cbd}" ---- PASS: TestRaft_SendSnapshotFollower (0.30s) +2025-02-25T23:59:12.316Z [ERROR] server-16d74bcb-2f93-abdf-f9df-08eb5d66b300: failed to take snapshot: error="nothing new to snapshot" +2025-02-25T23:59:12.316Z [INFO] server-98852d83-cf8a-1c18-3b6e-e8f589008095: starting snapshot up to: index=102 +2025-02-25T23:59:12.316Z [INFO] creating new snapshot: path=/tmp/raft2383542156/snapshots/2-102-1740527952316.tmp +2025-02-25T23:59:12.316Z [INFO] server-98852d83-cf8a-1c18-3b6e-e8f589008095: compacting logs: from=1 to=92 +2025-02-25T23:59:12.316Z [INFO] server-98852d83-cf8a-1c18-3b6e-e8f589008095: snapshot complete up to: index=102 +2025-02-25T23:59:12.316Z [INFO] server-086933d2-ee6d-ade9-ebf5-bebbd83fd475: starting snapshot up to: index=99 +2025-02-25T23:59:12.316Z [INFO] creating new snapshot: path=/tmp/raft1278528375/snapshots/2-99-1740527952316.tmp +2025-02-25T23:59:12.317Z [INFO] server-086933d2-ee6d-ade9-ebf5-bebbd83fd475: compacting logs: from=1 to=92 +2025-02-25T23:59:12.317Z [INFO] server-086933d2-ee6d-ade9-ebf5-bebbd83fd475: snapshot complete up to: index=99 +2025-02-25T23:59:12.322Z [ERROR] server-98852d83-cf8a-1c18-3b6e-e8f589008095: failed to get log: index=2 error="log not found" +2025-02-25T23:59:12.322Z [INFO] creating new snapshot: path=/tmp/raft2707567490/snapshots/2-102-1740527952322.tmp +2025-02-25T23:59:12.322Z [INFO] server-16d74bcb-2f93-abdf-f9df-08eb5d66b300: snapshot network transfer progress: read-bytes=693 percent-complete=100.00% +2025-02-25T23:59:12.322Z [INFO] server-16d74bcb-2f93-abdf-f9df-08eb5d66b300: copied to local snapshot: bytes=693 +2025-02-25T23:59:12.322Z [INFO] server-16d74bcb-2f93-abdf-f9df-08eb5d66b300: snapshot restore progress: id=2-102-1740527952322 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% +2025-02-25T23:59:12.322Z [INFO] server-16d74bcb-2f93-abdf-f9df-08eb5d66b300: Installed remote snapshot +2025-02-25T23:59:12.322Z [WARN] server-16d74bcb-2f93-abdf-f9df-08eb5d66b300: skipping application of old log: index=102 +2025-02-25T23:59:12.322Z [INFO] server-98852d83-cf8a-1c18-3b6e-e8f589008095: pipelining replication: peer="{Voter server-16d74bcb-2f93-abdf-f9df-08eb5d66b300 16d74bcb-2f93-abdf-f9df-08eb5d66b300}" +2025-02-25T23:59:12.334Z [INFO] server-98852d83-cf8a-1c18-3b6e-e8f589008095: aborting pipeline replication: peer="{Voter server-086933d2-ee6d-ade9-ebf5-bebbd83fd475 086933d2-ee6d-ade9-ebf5-bebbd83fd475}" +2025-02-25T23:59:12.334Z [INFO] server-98852d83-cf8a-1c18-3b6e-e8f589008095: aborting pipeline replication: peer="{Voter server-16d74bcb-2f93-abdf-f9df-08eb5d66b300 16d74bcb-2f93-abdf-f9df-08eb5d66b300}" +2025-02-25T23:59:12.838Z [ERROR] server-98852d83-cf8a-1c18-3b6e-e8f589008095: failed to heartbeat to: peer=086933d2-ee6d-ade9-ebf5-bebbd83fd475 backoff time=10ms error="command timed out" +2025-02-25T23:59:12.838Z [ERROR] server-98852d83-cf8a-1c18-3b6e-e8f589008095: failed to heartbeat to: peer=16d74bcb-2f93-abdf-f9df-08eb5d66b300 backoff time=10ms error="command timed out" +--- PASS: TestRaft_SendSnapshotFollower (0.82s) === RUN TestRaft_SendSnapshotAndLogsFollower -2026-03-31T06:17:49.945Z [INFO] server-e9b0f684-efef-534a-26c5-684612eb504c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e9b0f684-efef-534a-26c5-684612eb504c Address:e9b0f684-efef-534a-26c5-684612eb504c} {Suffrage:Voter ID:server-7c27d8a9-9ca6-22ce-351d-9ce8522e65f2 Address:7c27d8a9-9ca6-22ce-351d-9ce8522e65f2} {Suffrage:Voter ID:server-801241bb-5066-f390-cc76-d3f8dbd8fb67 Address:801241bb-5066-f390-cc76-d3f8dbd8fb67}]" -2026-03-31T06:17:49.945Z [INFO] server-e9b0f684-efef-534a-26c5-684612eb504c: entering follower state: follower="Node at e9b0f684-efef-534a-26c5-684612eb504c [Follower]" leader-address= leader-id= -2026-03-31T06:17:49.945Z [INFO] server-7c27d8a9-9ca6-22ce-351d-9ce8522e65f2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e9b0f684-efef-534a-26c5-684612eb504c Address:e9b0f684-efef-534a-26c5-684612eb504c} {Suffrage:Voter ID:server-7c27d8a9-9ca6-22ce-351d-9ce8522e65f2 Address:7c27d8a9-9ca6-22ce-351d-9ce8522e65f2} {Suffrage:Voter ID:server-801241bb-5066-f390-cc76-d3f8dbd8fb67 Address:801241bb-5066-f390-cc76-d3f8dbd8fb67}]" -2026-03-31T06:17:49.946Z [INFO] server-7c27d8a9-9ca6-22ce-351d-9ce8522e65f2: entering follower state: follower="Node at 7c27d8a9-9ca6-22ce-351d-9ce8522e65f2 [Follower]" leader-address= leader-id= -2026-03-31T06:17:49.946Z [INFO] server-801241bb-5066-f390-cc76-d3f8dbd8fb67: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e9b0f684-efef-534a-26c5-684612eb504c Address:e9b0f684-efef-534a-26c5-684612eb504c} {Suffrage:Voter ID:server-7c27d8a9-9ca6-22ce-351d-9ce8522e65f2 Address:7c27d8a9-9ca6-22ce-351d-9ce8522e65f2} {Suffrage:Voter ID:server-801241bb-5066-f390-cc76-d3f8dbd8fb67 Address:801241bb-5066-f390-cc76-d3f8dbd8fb67}]" -2026-03-31T06:17:49.946Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:17:49.946Z [INFO] server-801241bb-5066-f390-cc76-d3f8dbd8fb67: entering follower state: follower="Node at 801241bb-5066-f390-cc76-d3f8dbd8fb67 [Follower]" leader-address= leader-id= -2026-03-31T06:17:50.008Z [WARN] server-e9b0f684-efef-534a-26c5-684612eb504c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:50.008Z [INFO] server-e9b0f684-efef-534a-26c5-684612eb504c: entering candidate state: node="Node at e9b0f684-efef-534a-26c5-684612eb504c [Candidate]" term=2 -2026-03-31T06:17:50.008Z [INFO] server-e9b0f684-efef-534a-26c5-684612eb504c: election won: term=2 tally=2 -2026-03-31T06:17:50.008Z [INFO] server-e9b0f684-efef-534a-26c5-684612eb504c: entering leader state: leader="Node at e9b0f684-efef-534a-26c5-684612eb504c [Leader]" -2026-03-31T06:17:50.008Z [INFO] server-e9b0f684-efef-534a-26c5-684612eb504c: added peer, starting replication: peer=server-7c27d8a9-9ca6-22ce-351d-9ce8522e65f2 -2026-03-31T06:17:50.008Z [INFO] server-e9b0f684-efef-534a-26c5-684612eb504c: added peer, starting replication: peer=server-801241bb-5066-f390-cc76-d3f8dbd8fb67 -2026-03-31T06:17:50.009Z [INFO] server-e9b0f684-efef-534a-26c5-684612eb504c: pipelining replication: peer="{Voter server-801241bb-5066-f390-cc76-d3f8dbd8fb67 801241bb-5066-f390-cc76-d3f8dbd8fb67}" -2026-03-31T06:17:50.009Z [INFO] server-e9b0f684-efef-534a-26c5-684612eb504c: pipelining replication: peer="{Voter server-7c27d8a9-9ca6-22ce-351d-9ce8522e65f2 7c27d8a9-9ca6-22ce-351d-9ce8522e65f2}" -2026-03-31T06:17:50.114Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:17:50.00896598 +0000 UTC m=+98.286573401 (2 nodes), 62.70888ms from start of poll, 63.24508ms from cluster start. Timeout at 2026-03-31 06:17:50.11397342 +0000 UTC m=+98.391580841, 105.00744ms after stability -2026-03-31T06:17:50.114Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:50.219Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:50.11469518 +0000 UTC m=+98.392302561 (1 nodes), 2.44µs from start of poll, 168.97424ms from cluster start. Timeout at 2026-03-31 06:17:50.21972342 +0000 UTC m=+98.497330841, 105.02828ms after stability -2026-03-31T06:17:50.219Z [ERROR] server-e9b0f684-efef-534a-26c5-684612eb504c: failed to pipeline appendEntries: peer="{Voter server-7c27d8a9-9ca6-22ce-351d-9ce8522e65f2 7c27d8a9-9ca6-22ce-351d-9ce8522e65f2}" error="append pipeline closed" -2026-03-31T06:17:50.219Z [INFO] server-e9b0f684-efef-534a-26c5-684612eb504c: aborting pipeline replication: peer="{Voter server-7c27d8a9-9ca6-22ce-351d-9ce8522e65f2 7c27d8a9-9ca6-22ce-351d-9ce8522e65f2}" -2026-03-31T06:17:50.219Z [ERROR] server-e9b0f684-efef-534a-26c5-684612eb504c: failed to appendEntries to: peer="{Voter server-7c27d8a9-9ca6-22ce-351d-9ce8522e65f2 7c27d8a9-9ca6-22ce-351d-9ce8522e65f2}" error="failed to connect to peer: 7c27d8a9-9ca6-22ce-351d-9ce8522e65f2" -2026-03-31T06:17:50.223Z [ERROR] server-e9b0f684-efef-534a-26c5-684612eb504c: failed to heartbeat to: peer=7c27d8a9-9ca6-22ce-351d-9ce8522e65f2 backoff time=10ms error="failed to connect to peer: 7c27d8a9-9ca6-22ce-351d-9ce8522e65f2" +2025-02-25T23:59:12.838Z [INFO] server-eba1543c-c7e5-5e8d-6fe5-fe816c0325d4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eba1543c-c7e5-5e8d-6fe5-fe816c0325d4 Address:eba1543c-c7e5-5e8d-6fe5-fe816c0325d4} {Suffrage:Voter ID:server-e144e729-d5fe-619c-8762-a975832dd98b Address:e144e729-d5fe-619c-8762-a975832dd98b} {Suffrage:Voter ID:server-60735b1e-f2ac-65b7-96c1-1ea6b42728f7 Address:60735b1e-f2ac-65b7-96c1-1ea6b42728f7}]" +2025-02-25T23:59:12.838Z [INFO] server-e144e729-d5fe-619c-8762-a975832dd98b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eba1543c-c7e5-5e8d-6fe5-fe816c0325d4 Address:eba1543c-c7e5-5e8d-6fe5-fe816c0325d4} {Suffrage:Voter ID:server-e144e729-d5fe-619c-8762-a975832dd98b Address:e144e729-d5fe-619c-8762-a975832dd98b} {Suffrage:Voter ID:server-60735b1e-f2ac-65b7-96c1-1ea6b42728f7 Address:60735b1e-f2ac-65b7-96c1-1ea6b42728f7}]" +2025-02-25T23:59:12.838Z [INFO] server-60735b1e-f2ac-65b7-96c1-1ea6b42728f7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eba1543c-c7e5-5e8d-6fe5-fe816c0325d4 Address:eba1543c-c7e5-5e8d-6fe5-fe816c0325d4} {Suffrage:Voter ID:server-e144e729-d5fe-619c-8762-a975832dd98b Address:e144e729-d5fe-619c-8762-a975832dd98b} {Suffrage:Voter ID:server-60735b1e-f2ac-65b7-96c1-1ea6b42728f7 Address:60735b1e-f2ac-65b7-96c1-1ea6b42728f7}]" +2025-02-25T23:59:12.838Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:59:12.839Z [INFO] server-eba1543c-c7e5-5e8d-6fe5-fe816c0325d4: entering follower state: follower="Node at eba1543c-c7e5-5e8d-6fe5-fe816c0325d4 [Follower]" leader-address= leader-id= +2025-02-25T23:59:12.839Z [INFO] server-e144e729-d5fe-619c-8762-a975832dd98b: entering follower state: follower="Node at e144e729-d5fe-619c-8762-a975832dd98b [Follower]" leader-address= leader-id= +2025-02-25T23:59:12.839Z [INFO] server-60735b1e-f2ac-65b7-96c1-1ea6b42728f7: entering follower state: follower="Node at 60735b1e-f2ac-65b7-96c1-1ea6b42728f7 [Follower]" leader-address= leader-id= +2025-02-25T23:59:12.898Z [WARN] server-60735b1e-f2ac-65b7-96c1-1ea6b42728f7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:12.898Z [INFO] server-60735b1e-f2ac-65b7-96c1-1ea6b42728f7: entering candidate state: node="Node at 60735b1e-f2ac-65b7-96c1-1ea6b42728f7 [Candidate]" term=2 +2025-02-25T23:59:12.898Z [WARN] server-e144e729-d5fe-619c-8762-a975832dd98b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:12.898Z [INFO] server-e144e729-d5fe-619c-8762-a975832dd98b: entering candidate state: node="Node at e144e729-d5fe-619c-8762-a975832dd98b [Candidate]" term=2 +2025-02-25T23:59:12.898Z [INFO] server-e144e729-d5fe-619c-8762-a975832dd98b: duplicate requestVote for same term: term=2 +2025-02-25T23:59:12.898Z [INFO] server-60735b1e-f2ac-65b7-96c1-1ea6b42728f7: election won: term=2 tally=2 +2025-02-25T23:59:12.898Z [INFO] server-60735b1e-f2ac-65b7-96c1-1ea6b42728f7: entering leader state: leader="Node at 60735b1e-f2ac-65b7-96c1-1ea6b42728f7 [Leader]" +2025-02-25T23:59:12.898Z [INFO] server-60735b1e-f2ac-65b7-96c1-1ea6b42728f7: added peer, starting replication: peer=server-eba1543c-c7e5-5e8d-6fe5-fe816c0325d4 +2025-02-25T23:59:12.898Z [INFO] server-60735b1e-f2ac-65b7-96c1-1ea6b42728f7: added peer, starting replication: peer=server-e144e729-d5fe-619c-8762-a975832dd98b +2025-02-25T23:59:12.898Z [INFO] server-e144e729-d5fe-619c-8762-a975832dd98b: entering follower state: follower="Node at e144e729-d5fe-619c-8762-a975832dd98b [Follower]" leader-address=60735b1e-f2ac-65b7-96c1-1ea6b42728f7 leader-id=server-60735b1e-f2ac-65b7-96c1-1ea6b42728f7 +2025-02-25T23:59:12.898Z [INFO] server-60735b1e-f2ac-65b7-96c1-1ea6b42728f7: pipelining replication: peer="{Voter server-e144e729-d5fe-619c-8762-a975832dd98b e144e729-d5fe-619c-8762-a975832dd98b}" +2025-02-25T23:59:12.898Z [INFO] server-eba1543c-c7e5-5e8d-6fe5-fe816c0325d4: duplicate requestVote for same term: term=2 +2025-02-25T23:59:12.898Z [INFO] server-60735b1e-f2ac-65b7-96c1-1ea6b42728f7: pipelining replication: peer="{Voter server-eba1543c-c7e5-5e8d-6fe5-fe816c0325d4 eba1543c-c7e5-5e8d-6fe5-fe816c0325d4}" +2025-02-25T23:59:12.898Z [WARN] server-60735b1e-f2ac-65b7-96c1-1ea6b42728f7: rejecting vote request since we have a leader: from=e144e729-d5fe-619c-8762-a975832dd98b leader=60735b1e-f2ac-65b7-96c1-1ea6b42728f7 leader-id=server-60735b1e-f2ac-65b7-96c1-1ea6b42728f7 +2025-02-25T23:59:13.009Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:59:12.89851417 +0000 UTC m=+100.507529577 (2 nodes), 59.571499ms from start of poll, 59.74618ms from cluster start. Timeout at 2025-02-25 23:59:13.00352877 +0000 UTC m=+100.612544177, 105.0146ms after stability +2025-02-25T23:59:13.010Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:13.118Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:13.01000836 +0000 UTC m=+100.619023767 (1 nodes), 560ns from start of poll, 171.24037ms from cluster start. Timeout at 2025-02-25 23:59:13.11507812 +0000 UTC m=+100.724093527, 105.06976ms after stability +2025-02-25T23:59:13.118Z [ERROR] server-60735b1e-f2ac-65b7-96c1-1ea6b42728f7: failed to pipeline appendEntries: peer="{Voter server-eba1543c-c7e5-5e8d-6fe5-fe816c0325d4 eba1543c-c7e5-5e8d-6fe5-fe816c0325d4}" error="append pipeline closed" +2025-02-25T23:59:13.118Z [INFO] server-60735b1e-f2ac-65b7-96c1-1ea6b42728f7: aborting pipeline replication: peer="{Voter server-eba1543c-c7e5-5e8d-6fe5-fe816c0325d4 eba1543c-c7e5-5e8d-6fe5-fe816c0325d4}" +2025-02-25T23:59:13.118Z [ERROR] server-60735b1e-f2ac-65b7-96c1-1ea6b42728f7: failed to appendEntries to: peer="{Voter server-eba1543c-c7e5-5e8d-6fe5-fe816c0325d4 eba1543c-c7e5-5e8d-6fe5-fe816c0325d4}" error="failed to connect to peer: eba1543c-c7e5-5e8d-6fe5-fe816c0325d4" raft_test.go:1669: [INFO] Finished apply without behind follower -2026-03-31T06:17:50.223Z [INFO] server-e9b0f684-efef-534a-26c5-684612eb504c: starting snapshot up to: index=102 -2026-03-31T06:17:50.223Z [INFO] creating new snapshot: path=/tmp/raft3704746840/snapshots/2-102-1774937870223.tmp -2026-03-31T06:17:50.224Z [INFO] server-e9b0f684-efef-534a-26c5-684612eb504c: compacting logs: from=1 to=92 -2026-03-31T06:17:50.224Z [INFO] server-e9b0f684-efef-534a-26c5-684612eb504c: snapshot complete up to: index=102 -2026-03-31T06:17:50.225Z [ERROR] server-7c27d8a9-9ca6-22ce-351d-9ce8522e65f2: failed to take snapshot: error="nothing new to snapshot" -2026-03-31T06:17:50.225Z [INFO] server-801241bb-5066-f390-cc76-d3f8dbd8fb67: starting snapshot up to: index=96 -2026-03-31T06:17:50.225Z [INFO] creating new snapshot: path=/tmp/raft1809535513/snapshots/2-96-1774937870225.tmp -2026-03-31T06:17:50.226Z [INFO] server-801241bb-5066-f390-cc76-d3f8dbd8fb67: compacting logs: from=1 to=92 -2026-03-31T06:17:50.226Z [INFO] server-801241bb-5066-f390-cc76-d3f8dbd8fb67: snapshot complete up to: index=96 -2026-03-31T06:17:50.230Z [ERROR] server-e9b0f684-efef-534a-26c5-684612eb504c: failed to get log: index=2 error="log not found" -2026-03-31T06:17:50.230Z [ERROR] server-e9b0f684-efef-534a-26c5-684612eb504c: failed to install snapshot: id=2-102-1774937870223 error="failed to connect to peer: 7c27d8a9-9ca6-22ce-351d-9ce8522e65f2" -2026-03-31T06:17:50.230Z [ERROR] server-e9b0f684-efef-534a-26c5-684612eb504c: failed to send snapshot to: peer="{Voter server-7c27d8a9-9ca6-22ce-351d-9ce8522e65f2 7c27d8a9-9ca6-22ce-351d-9ce8522e65f2}" error="failed to connect to peer: 7c27d8a9-9ca6-22ce-351d-9ce8522e65f2" +2025-02-25T23:59:13.120Z [ERROR] server-eba1543c-c7e5-5e8d-6fe5-fe816c0325d4: failed to take snapshot: error="nothing new to snapshot" +2025-02-25T23:59:13.120Z [INFO] server-e144e729-d5fe-619c-8762-a975832dd98b: starting snapshot up to: index=98 +2025-02-25T23:59:13.120Z [INFO] creating new snapshot: path=/tmp/raft2104306269/snapshots/2-98-1740527953120.tmp +2025-02-25T23:59:13.120Z [INFO] server-e144e729-d5fe-619c-8762-a975832dd98b: compacting logs: from=1 to=92 +2025-02-25T23:59:13.120Z [INFO] server-e144e729-d5fe-619c-8762-a975832dd98b: snapshot complete up to: index=98 +2025-02-25T23:59:13.120Z [INFO] server-60735b1e-f2ac-65b7-96c1-1ea6b42728f7: starting snapshot up to: index=102 +2025-02-25T23:59:13.120Z [INFO] creating new snapshot: path=/tmp/raft3012430312/snapshots/2-102-1740527953120.tmp +2025-02-25T23:59:13.120Z [INFO] server-60735b1e-f2ac-65b7-96c1-1ea6b42728f7: compacting logs: from=1 to=92 +2025-02-25T23:59:13.120Z [INFO] server-60735b1e-f2ac-65b7-96c1-1ea6b42728f7: snapshot complete up to: index=102 +2025-02-25T23:59:13.121Z [ERROR] server-60735b1e-f2ac-65b7-96c1-1ea6b42728f7: failed to heartbeat to: peer=eba1543c-c7e5-5e8d-6fe5-fe816c0325d4 backoff time=10ms error="failed to connect to peer: eba1543c-c7e5-5e8d-6fe5-fe816c0325d4" raft_test.go:1690: [INFO] Finished apply without behind follower -2026-03-31T06:17:50.240Z [ERROR] server-e9b0f684-efef-534a-26c5-684612eb504c: failed to get log: index=2 error="log not found" -2026-03-31T06:17:50.241Z [INFO] creating new snapshot: path=/tmp/raft3671030991/snapshots/2-102-1774937870241.tmp -2026-03-31T06:17:50.241Z [INFO] server-7c27d8a9-9ca6-22ce-351d-9ce8522e65f2: snapshot network transfer progress: read-bytes=693 percent-complete=100.00% -2026-03-31T06:17:50.241Z [INFO] server-7c27d8a9-9ca6-22ce-351d-9ce8522e65f2: copied to local snapshot: bytes=693 -2026-03-31T06:17:50.242Z [INFO] server-7c27d8a9-9ca6-22ce-351d-9ce8522e65f2: snapshot restore progress: id=2-102-1774937870241 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% -2026-03-31T06:17:50.242Z [INFO] server-7c27d8a9-9ca6-22ce-351d-9ce8522e65f2: Installed remote snapshot -2026-03-31T06:17:50.242Z [INFO] server-e9b0f684-efef-534a-26c5-684612eb504c: pipelining replication: peer="{Voter server-7c27d8a9-9ca6-22ce-351d-9ce8522e65f2 7c27d8a9-9ca6-22ce-351d-9ce8522e65f2}" -2026-03-31T06:17:50.248Z [INFO] server-e9b0f684-efef-534a-26c5-684612eb504c: aborting pipeline replication: peer="{Voter server-801241bb-5066-f390-cc76-d3f8dbd8fb67 801241bb-5066-f390-cc76-d3f8dbd8fb67}" -2026-03-31T06:17:50.248Z [INFO] server-e9b0f684-efef-534a-26c5-684612eb504c: aborting pipeline replication: peer="{Voter server-7c27d8a9-9ca6-22ce-351d-9ce8522e65f2 7c27d8a9-9ca6-22ce-351d-9ce8522e65f2}" +2025-02-25T23:59:13.134Z [ERROR] server-60735b1e-f2ac-65b7-96c1-1ea6b42728f7: failed to get log: index=2 error="log not found" +2025-02-25T23:59:13.134Z [INFO] creating new snapshot: path=/tmp/raft3002081467/snapshots/2-102-1740527953134.tmp +2025-02-25T23:59:13.134Z [INFO] server-eba1543c-c7e5-5e8d-6fe5-fe816c0325d4: snapshot network transfer progress: read-bytes=693 percent-complete=100.00% +2025-02-25T23:59:13.134Z [INFO] server-eba1543c-c7e5-5e8d-6fe5-fe816c0325d4: copied to local snapshot: bytes=693 +2025-02-25T23:59:13.134Z [INFO] server-eba1543c-c7e5-5e8d-6fe5-fe816c0325d4: snapshot restore progress: id=2-102-1740527953134 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% +2025-02-25T23:59:13.134Z [INFO] server-eba1543c-c7e5-5e8d-6fe5-fe816c0325d4: Installed remote snapshot +2025-02-25T23:59:13.135Z [INFO] server-60735b1e-f2ac-65b7-96c1-1ea6b42728f7: pipelining replication: peer="{Voter server-eba1543c-c7e5-5e8d-6fe5-fe816c0325d4 eba1543c-c7e5-5e8d-6fe5-fe816c0325d4}" +2025-02-25T23:59:13.142Z [INFO] server-60735b1e-f2ac-65b7-96c1-1ea6b42728f7: aborting pipeline replication: peer="{Voter server-e144e729-d5fe-619c-8762-a975832dd98b e144e729-d5fe-619c-8762-a975832dd98b}" +2025-02-25T23:59:13.142Z [INFO] server-60735b1e-f2ac-65b7-96c1-1ea6b42728f7: aborting pipeline replication: peer="{Voter server-eba1543c-c7e5-5e8d-6fe5-fe816c0325d4 eba1543c-c7e5-5e8d-6fe5-fe816c0325d4}" --- PASS: TestRaft_SendSnapshotAndLogsFollower (0.30s) === RUN TestRaft_ReJoinFollower -2026-03-31T06:17:50.249Z [INFO] server-a1597eb8-a47c-21a0-99f8-c805d54978f6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a1597eb8-a47c-21a0-99f8-c805d54978f6 Address:a1597eb8-a47c-21a0-99f8-c805d54978f6} {Suffrage:Voter ID:server-d0eb7b68-a89b-fc4c-732a-b6b19965d047 Address:d0eb7b68-a89b-fc4c-732a-b6b19965d047} {Suffrage:Voter ID:server-f0b0c6ec-7413-242e-8987-4caa199e4719 Address:f0b0c6ec-7413-242e-8987-4caa199e4719}]" -2026-03-31T06:17:50.249Z [INFO] server-d0eb7b68-a89b-fc4c-732a-b6b19965d047: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a1597eb8-a47c-21a0-99f8-c805d54978f6 Address:a1597eb8-a47c-21a0-99f8-c805d54978f6} {Suffrage:Voter ID:server-d0eb7b68-a89b-fc4c-732a-b6b19965d047 Address:d0eb7b68-a89b-fc4c-732a-b6b19965d047} {Suffrage:Voter ID:server-f0b0c6ec-7413-242e-8987-4caa199e4719 Address:f0b0c6ec-7413-242e-8987-4caa199e4719}]" -2026-03-31T06:17:50.249Z [INFO] server-a1597eb8-a47c-21a0-99f8-c805d54978f6: entering follower state: follower="Node at a1597eb8-a47c-21a0-99f8-c805d54978f6 [Follower]" leader-address= leader-id= -2026-03-31T06:17:50.249Z [INFO] server-d0eb7b68-a89b-fc4c-732a-b6b19965d047: entering follower state: follower="Node at d0eb7b68-a89b-fc4c-732a-b6b19965d047 [Follower]" leader-address= leader-id= -2026-03-31T06:17:50.249Z [INFO] server-f0b0c6ec-7413-242e-8987-4caa199e4719: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a1597eb8-a47c-21a0-99f8-c805d54978f6 Address:a1597eb8-a47c-21a0-99f8-c805d54978f6} {Suffrage:Voter ID:server-d0eb7b68-a89b-fc4c-732a-b6b19965d047 Address:d0eb7b68-a89b-fc4c-732a-b6b19965d047} {Suffrage:Voter ID:server-f0b0c6ec-7413-242e-8987-4caa199e4719 Address:f0b0c6ec-7413-242e-8987-4caa199e4719}]" -2026-03-31T06:17:50.249Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:50.249Z [INFO] server-f0b0c6ec-7413-242e-8987-4caa199e4719: entering follower state: follower="Node at f0b0c6ec-7413-242e-8987-4caa199e4719 [Follower]" leader-address= leader-id= -2026-03-31T06:17:50.332Z [WARN] server-d0eb7b68-a89b-fc4c-732a-b6b19965d047: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:50.332Z [INFO] server-d0eb7b68-a89b-fc4c-732a-b6b19965d047: entering candidate state: node="Node at d0eb7b68-a89b-fc4c-732a-b6b19965d047 [Candidate]" term=2 -2026-03-31T06:17:50.332Z [INFO] server-d0eb7b68-a89b-fc4c-732a-b6b19965d047: election won: term=2 tally=2 -2026-03-31T06:17:50.332Z [INFO] server-d0eb7b68-a89b-fc4c-732a-b6b19965d047: entering leader state: leader="Node at d0eb7b68-a89b-fc4c-732a-b6b19965d047 [Leader]" -2026-03-31T06:17:50.332Z [INFO] server-d0eb7b68-a89b-fc4c-732a-b6b19965d047: added peer, starting replication: peer=server-a1597eb8-a47c-21a0-99f8-c805d54978f6 -2026-03-31T06:17:50.332Z [INFO] server-d0eb7b68-a89b-fc4c-732a-b6b19965d047: added peer, starting replication: peer=server-f0b0c6ec-7413-242e-8987-4caa199e4719 -2026-03-31T06:17:50.332Z [INFO] server-d0eb7b68-a89b-fc4c-732a-b6b19965d047: pipelining replication: peer="{Voter server-a1597eb8-a47c-21a0-99f8-c805d54978f6 a1597eb8-a47c-21a0-99f8-c805d54978f6}" -2026-03-31T06:17:50.332Z [INFO] server-d0eb7b68-a89b-fc4c-732a-b6b19965d047: pipelining replication: peer="{Voter server-f0b0c6ec-7413-242e-8987-4caa199e4719 f0b0c6ec-7413-242e-8987-4caa199e4719}" -2026-03-31T06:17:50.438Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:50.33282978 +0000 UTC m=+98.610437161 (1 nodes), 83.08992ms from start of poll, 83.54456ms from cluster start. Timeout at 2026-03-31 06:17:50.4378345 +0000 UTC m=+98.715441881, 105.00472ms after stability -2026-03-31T06:17:50.444Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:17:50.550Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:17:50.4450013 +0000 UTC m=+98.722608681 (2 nodes), 1.4µs from start of poll, 195.71608ms from cluster start. Timeout at 2026-03-31 06:17:50.55001666 +0000 UTC m=+98.827624081, 105.0154ms after stability -2026-03-31T06:17:50.550Z [INFO] server-d0eb7b68-a89b-fc4c-732a-b6b19965d047: updating configuration: command=RemoveServer server-id=server-a1597eb8-a47c-21a0-99f8-c805d54978f6 server-addr= servers="[{Suffrage:Voter ID:server-d0eb7b68-a89b-fc4c-732a-b6b19965d047 Address:d0eb7b68-a89b-fc4c-732a-b6b19965d047} {Suffrage:Voter ID:server-f0b0c6ec-7413-242e-8987-4caa199e4719 Address:f0b0c6ec-7413-242e-8987-4caa199e4719}]" -2026-03-31T06:17:50.550Z [INFO] server-d0eb7b68-a89b-fc4c-732a-b6b19965d047: removed peer, stopping replication: peer=server-a1597eb8-a47c-21a0-99f8-c805d54978f6 last-index=3 -2026-03-31T06:17:50.550Z [INFO] server-d0eb7b68-a89b-fc4c-732a-b6b19965d047: aborting pipeline replication: peer="{Voter server-a1597eb8-a47c-21a0-99f8-c805d54978f6 a1597eb8-a47c-21a0-99f8-c805d54978f6}" -2026-03-31T06:17:50.633Z [WARN] server-a1597eb8-a47c-21a0-99f8-c805d54978f6: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election -2026-03-31T06:17:50.656Z [INFO] server-d0eb7b68-a89b-fc4c-732a-b6b19965d047: updating configuration: command=AddVoter server-id=server-a1597eb8-a47c-21a0-99f8-c805d54978f6 server-addr=a1597eb8-a47c-21a0-99f8-c805d54978f6 servers="[{Suffrage:Voter ID:server-d0eb7b68-a89b-fc4c-732a-b6b19965d047 Address:d0eb7b68-a89b-fc4c-732a-b6b19965d047} {Suffrage:Voter ID:server-f0b0c6ec-7413-242e-8987-4caa199e4719 Address:f0b0c6ec-7413-242e-8987-4caa199e4719} {Suffrage:Voter ID:server-a1597eb8-a47c-21a0-99f8-c805d54978f6 Address:a1597eb8-a47c-21a0-99f8-c805d54978f6}]" -2026-03-31T06:17:50.656Z [INFO] server-d0eb7b68-a89b-fc4c-732a-b6b19965d047: added peer, starting replication: peer=server-a1597eb8-a47c-21a0-99f8-c805d54978f6 -2026-03-31T06:17:50.656Z [WARN] server-a1597eb8-a47c-21a0-99f8-c805d54978f6: failed to get previous log: previous-index=4 last-index=3 error="log not found" -2026-03-31T06:17:50.657Z [WARN] server-d0eb7b68-a89b-fc4c-732a-b6b19965d047: appendEntries rejected, sending older logs: peer="{Voter server-a1597eb8-a47c-21a0-99f8-c805d54978f6 a1597eb8-a47c-21a0-99f8-c805d54978f6}" next=4 -2026-03-31T06:17:50.657Z [INFO] server-d0eb7b68-a89b-fc4c-732a-b6b19965d047: pipelining replication: peer="{Voter server-a1597eb8-a47c-21a0-99f8-c805d54978f6 a1597eb8-a47c-21a0-99f8-c805d54978f6}" -2026-03-31T06:17:50.657Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:50.762Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:50.65722478 +0000 UTC m=+98.934832161 (1 nodes), 2.28µs from start of poll, 407.93956ms from cluster start. Timeout at 2026-03-31 06:17:50.76226394 +0000 UTC m=+99.039871321, 105.03916ms after stability -2026-03-31T06:17:50.762Z [INFO] server-d0eb7b68-a89b-fc4c-732a-b6b19965d047: aborting pipeline replication: peer="{Voter server-a1597eb8-a47c-21a0-99f8-c805d54978f6 a1597eb8-a47c-21a0-99f8-c805d54978f6}" -2026-03-31T06:17:50.762Z [INFO] server-d0eb7b68-a89b-fc4c-732a-b6b19965d047: aborting pipeline replication: peer="{Voter server-f0b0c6ec-7413-242e-8987-4caa199e4719 f0b0c6ec-7413-242e-8987-4caa199e4719}" ---- PASS: TestRaft_ReJoinFollower (0.51s) +2025-02-25T23:59:13.142Z [INFO] server-fff8a9d7-58f1-6b27-6b4a-6ff7d022cc56: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fff8a9d7-58f1-6b27-6b4a-6ff7d022cc56 Address:fff8a9d7-58f1-6b27-6b4a-6ff7d022cc56} {Suffrage:Voter ID:server-5297d41d-37e9-5661-31ca-235da52a312b Address:5297d41d-37e9-5661-31ca-235da52a312b} {Suffrage:Voter ID:server-749f09f2-c8dd-5217-5ce3-b423d3ce0353 Address:749f09f2-c8dd-5217-5ce3-b423d3ce0353}]" +2025-02-25T23:59:13.142Z [INFO] server-5297d41d-37e9-5661-31ca-235da52a312b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fff8a9d7-58f1-6b27-6b4a-6ff7d022cc56 Address:fff8a9d7-58f1-6b27-6b4a-6ff7d022cc56} {Suffrage:Voter ID:server-5297d41d-37e9-5661-31ca-235da52a312b Address:5297d41d-37e9-5661-31ca-235da52a312b} {Suffrage:Voter ID:server-749f09f2-c8dd-5217-5ce3-b423d3ce0353 Address:749f09f2-c8dd-5217-5ce3-b423d3ce0353}]" +2025-02-25T23:59:13.142Z [INFO] server-749f09f2-c8dd-5217-5ce3-b423d3ce0353: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fff8a9d7-58f1-6b27-6b4a-6ff7d022cc56 Address:fff8a9d7-58f1-6b27-6b4a-6ff7d022cc56} {Suffrage:Voter ID:server-5297d41d-37e9-5661-31ca-235da52a312b Address:5297d41d-37e9-5661-31ca-235da52a312b} {Suffrage:Voter ID:server-749f09f2-c8dd-5217-5ce3-b423d3ce0353 Address:749f09f2-c8dd-5217-5ce3-b423d3ce0353}]" +2025-02-25T23:59:13.142Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:13.142Z [INFO] server-fff8a9d7-58f1-6b27-6b4a-6ff7d022cc56: entering follower state: follower="Node at fff8a9d7-58f1-6b27-6b4a-6ff7d022cc56 [Follower]" leader-address= leader-id= +2025-02-25T23:59:13.142Z [INFO] server-5297d41d-37e9-5661-31ca-235da52a312b: entering follower state: follower="Node at 5297d41d-37e9-5661-31ca-235da52a312b [Follower]" leader-address= leader-id= +2025-02-25T23:59:13.142Z [INFO] server-749f09f2-c8dd-5217-5ce3-b423d3ce0353: entering follower state: follower="Node at 749f09f2-c8dd-5217-5ce3-b423d3ce0353 [Follower]" leader-address= leader-id= +2025-02-25T23:59:13.206Z [WARN] server-fff8a9d7-58f1-6b27-6b4a-6ff7d022cc56: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:13.206Z [INFO] server-fff8a9d7-58f1-6b27-6b4a-6ff7d022cc56: entering candidate state: node="Node at fff8a9d7-58f1-6b27-6b4a-6ff7d022cc56 [Candidate]" term=2 +2025-02-25T23:59:13.206Z [INFO] server-fff8a9d7-58f1-6b27-6b4a-6ff7d022cc56: election won: term=2 tally=2 +2025-02-25T23:59:13.206Z [INFO] server-fff8a9d7-58f1-6b27-6b4a-6ff7d022cc56: entering leader state: leader="Node at fff8a9d7-58f1-6b27-6b4a-6ff7d022cc56 [Leader]" +2025-02-25T23:59:13.206Z [INFO] server-fff8a9d7-58f1-6b27-6b4a-6ff7d022cc56: added peer, starting replication: peer=server-5297d41d-37e9-5661-31ca-235da52a312b +2025-02-25T23:59:13.206Z [INFO] server-fff8a9d7-58f1-6b27-6b4a-6ff7d022cc56: added peer, starting replication: peer=server-749f09f2-c8dd-5217-5ce3-b423d3ce0353 +2025-02-25T23:59:13.206Z [INFO] server-fff8a9d7-58f1-6b27-6b4a-6ff7d022cc56: pipelining replication: peer="{Voter server-749f09f2-c8dd-5217-5ce3-b423d3ce0353 749f09f2-c8dd-5217-5ce3-b423d3ce0353}" +2025-02-25T23:59:13.206Z [INFO] server-fff8a9d7-58f1-6b27-6b4a-6ff7d022cc56: pipelining replication: peer="{Voter server-5297d41d-37e9-5661-31ca-235da52a312b 5297d41d-37e9-5661-31ca-235da52a312b}" +2025-02-25T23:59:13.313Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:13.20626405 +0000 UTC m=+100.815279457 (1 nodes), 63.539685ms from start of poll, 63.722326ms from cluster start. Timeout at 2025-02-25 23:59:13.311277771 +0000 UTC m=+100.920293178, 105.013721ms after stability +2025-02-25T23:59:13.322Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:59:13.430Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:59:13.322069747 +0000 UTC m=+100.931085154 (2 nodes), 1.52µs from start of poll, 179.528023ms from cluster start. Timeout at 2025-02-25 23:59:13.427115707 +0000 UTC m=+101.036131074, 105.04592ms after stability +2025-02-25T23:59:13.430Z [INFO] server-fff8a9d7-58f1-6b27-6b4a-6ff7d022cc56: updating configuration: command=RemoveServer server-id=server-5297d41d-37e9-5661-31ca-235da52a312b server-addr= servers="[{Suffrage:Voter ID:server-fff8a9d7-58f1-6b27-6b4a-6ff7d022cc56 Address:fff8a9d7-58f1-6b27-6b4a-6ff7d022cc56} {Suffrage:Voter ID:server-749f09f2-c8dd-5217-5ce3-b423d3ce0353 Address:749f09f2-c8dd-5217-5ce3-b423d3ce0353}]" +2025-02-25T23:59:13.430Z [INFO] server-fff8a9d7-58f1-6b27-6b4a-6ff7d022cc56: removed peer, stopping replication: peer=server-5297d41d-37e9-5661-31ca-235da52a312b last-index=3 +2025-02-25T23:59:13.430Z [INFO] server-fff8a9d7-58f1-6b27-6b4a-6ff7d022cc56: aborting pipeline replication: peer="{Voter server-5297d41d-37e9-5661-31ca-235da52a312b 5297d41d-37e9-5661-31ca-235da52a312b}" +2025-02-25T23:59:13.530Z [WARN] server-5297d41d-37e9-5661-31ca-235da52a312b: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +2025-02-25T23:59:13.538Z [INFO] server-fff8a9d7-58f1-6b27-6b4a-6ff7d022cc56: updating configuration: command=AddVoter server-id=server-5297d41d-37e9-5661-31ca-235da52a312b server-addr=5297d41d-37e9-5661-31ca-235da52a312b servers="[{Suffrage:Voter ID:server-fff8a9d7-58f1-6b27-6b4a-6ff7d022cc56 Address:fff8a9d7-58f1-6b27-6b4a-6ff7d022cc56} {Suffrage:Voter ID:server-749f09f2-c8dd-5217-5ce3-b423d3ce0353 Address:749f09f2-c8dd-5217-5ce3-b423d3ce0353} {Suffrage:Voter ID:server-5297d41d-37e9-5661-31ca-235da52a312b Address:5297d41d-37e9-5661-31ca-235da52a312b}]" +2025-02-25T23:59:13.538Z [INFO] server-fff8a9d7-58f1-6b27-6b4a-6ff7d022cc56: added peer, starting replication: peer=server-5297d41d-37e9-5661-31ca-235da52a312b +2025-02-25T23:59:13.538Z [WARN] server-5297d41d-37e9-5661-31ca-235da52a312b: failed to get previous log: previous-index=4 last-index=3 error="log not found" +2025-02-25T23:59:13.538Z [WARN] server-fff8a9d7-58f1-6b27-6b4a-6ff7d022cc56: appendEntries rejected, sending older logs: peer="{Voter server-5297d41d-37e9-5661-31ca-235da52a312b 5297d41d-37e9-5661-31ca-235da52a312b}" next=4 +2025-02-25T23:59:13.538Z [INFO] server-fff8a9d7-58f1-6b27-6b4a-6ff7d022cc56: pipelining replication: peer="{Voter server-5297d41d-37e9-5661-31ca-235da52a312b 5297d41d-37e9-5661-31ca-235da52a312b}" +2025-02-25T23:59:13.538Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:13.646Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:13.538669406 +0000 UTC m=+101.147684773 (1 nodes), 640ns from start of poll, 396.127642ms from cluster start. Timeout at 2025-02-25 23:59:13.643699566 +0000 UTC m=+101.252714973, 105.0302ms after stability +2025-02-25T23:59:13.646Z [INFO] server-fff8a9d7-58f1-6b27-6b4a-6ff7d022cc56: aborting pipeline replication: peer="{Voter server-749f09f2-c8dd-5217-5ce3-b423d3ce0353 749f09f2-c8dd-5217-5ce3-b423d3ce0353}" +2025-02-25T23:59:13.646Z [INFO] server-fff8a9d7-58f1-6b27-6b4a-6ff7d022cc56: aborting pipeline replication: peer="{Voter server-5297d41d-37e9-5661-31ca-235da52a312b 5297d41d-37e9-5661-31ca-235da52a312b}" +2025-02-25T23:59:14.146Z [ERROR] server-fff8a9d7-58f1-6b27-6b4a-6ff7d022cc56: failed to heartbeat to: peer=749f09f2-c8dd-5217-5ce3-b423d3ce0353 backoff time=10ms error="command timed out" +--- PASS: TestRaft_ReJoinFollower (1.00s) === RUN TestRaft_LeaderLeaseExpire -2026-03-31T06:17:50.763Z [INFO] server-71b20d2e-cd70-0e29-74a5-a54159011605: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-71b20d2e-cd70-0e29-74a5-a54159011605 Address:71b20d2e-cd70-0e29-74a5-a54159011605} {Suffrage:Voter ID:server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98 Address:a7f38cf4-329f-9cc0-3dff-ef43ab723d98}]" -2026-03-31T06:17:50.763Z [INFO] server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-71b20d2e-cd70-0e29-74a5-a54159011605 Address:71b20d2e-cd70-0e29-74a5-a54159011605} {Suffrage:Voter ID:server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98 Address:a7f38cf4-329f-9cc0-3dff-ef43ab723d98}]" -2026-03-31T06:17:50.763Z [INFO] server-71b20d2e-cd70-0e29-74a5-a54159011605: entering follower state: follower="Node at 71b20d2e-cd70-0e29-74a5-a54159011605 [Follower]" leader-address= leader-id= -2026-03-31T06:17:50.763Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:50.763Z [INFO] server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98: entering follower state: follower="Node at a7f38cf4-329f-9cc0-3dff-ef43ab723d98 [Follower]" leader-address= leader-id= -2026-03-31T06:17:50.825Z [WARN] server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:50.825Z [INFO] server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98: entering candidate state: node="Node at a7f38cf4-329f-9cc0-3dff-ef43ab723d98 [Candidate]" term=2 -2026-03-31T06:17:50.825Z [INFO] server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98: election won: term=2 tally=2 -2026-03-31T06:17:50.825Z [INFO] server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98: entering leader state: leader="Node at a7f38cf4-329f-9cc0-3dff-ef43ab723d98 [Leader]" -2026-03-31T06:17:50.825Z [INFO] server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98: added peer, starting replication: peer=server-71b20d2e-cd70-0e29-74a5-a54159011605 -2026-03-31T06:17:50.825Z [INFO] server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98: pipelining replication: peer="{Voter server-71b20d2e-cd70-0e29-74a5-a54159011605 71b20d2e-cd70-0e29-74a5-a54159011605}" -2026-03-31T06:17:50.931Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:50.82577278 +0000 UTC m=+99.103380161 (1 nodes), 62.1074ms from start of poll, 62.52144ms from cluster start. Timeout at 2026-03-31 06:17:50.93079406 +0000 UTC m=+99.208401441, 105.02128ms after stability -2026-03-31T06:17:50.936Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:17:51.041Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:17:50.93653534 +0000 UTC m=+99.214142721 (1 nodes), 1.76µs from start of poll, 173.284ms from cluster start. Timeout at 2026-03-31 06:17:51.04156346 +0000 UTC m=+99.319170881, 105.02816ms after stability - raft_test.go:1799: [INFO] Disconnecting Node at 71b20d2e-cd70-0e29-74a5-a54159011605 [Follower] -2026-03-31T06:17:51.049Z [ERROR] server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98: failed to pipeline appendEntries: peer="{Voter server-71b20d2e-cd70-0e29-74a5-a54159011605 71b20d2e-cd70-0e29-74a5-a54159011605}" error="append pipeline closed" -2026-03-31T06:17:51.049Z [INFO] server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98: aborting pipeline replication: peer="{Voter server-71b20d2e-cd70-0e29-74a5-a54159011605 71b20d2e-cd70-0e29-74a5-a54159011605}" -2026-03-31T06:17:51.051Z [ERROR] server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98: failed to heartbeat to: peer=71b20d2e-cd70-0e29-74a5-a54159011605 backoff time=10ms error="failed to connect to peer: 71b20d2e-cd70-0e29-74a5-a54159011605" -2026-03-31T06:17:51.057Z [ERROR] server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98: failed to appendEntries to: peer="{Voter server-71b20d2e-cd70-0e29-74a5-a54159011605 71b20d2e-cd70-0e29-74a5-a54159011605}" error="failed to connect to peer: 71b20d2e-cd70-0e29-74a5-a54159011605" -2026-03-31T06:17:51.068Z [ERROR] server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98: failed to heartbeat to: peer=71b20d2e-cd70-0e29-74a5-a54159011605 backoff time=10ms error="failed to connect to peer: 71b20d2e-cd70-0e29-74a5-a54159011605" -2026-03-31T06:17:51.075Z [ERROR] server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98: failed to appendEntries to: peer="{Voter server-71b20d2e-cd70-0e29-74a5-a54159011605 71b20d2e-cd70-0e29-74a5-a54159011605}" error="failed to connect to peer: 71b20d2e-cd70-0e29-74a5-a54159011605" -2026-03-31T06:17:51.089Z [ERROR] server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98: failed to heartbeat to: peer=71b20d2e-cd70-0e29-74a5-a54159011605 backoff time=10ms error="failed to connect to peer: 71b20d2e-cd70-0e29-74a5-a54159011605" -2026-03-31T06:17:51.092Z [WARN] server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98: failed to contact: server-id=server-71b20d2e-cd70-0e29-74a5-a54159011605 time=50.91724ms -2026-03-31T06:17:51.092Z [WARN] server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98: failed to contact quorum of nodes, stepping down -2026-03-31T06:17:51.093Z [INFO] server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98: entering follower state: follower="Node at a7f38cf4-329f-9cc0-3dff-ef43ab723d98 [Follower]" leader-address= leader-id= -2026-03-31T06:17:51.093Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:51.093Z [ERROR] server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98: failed to appendEntries to: peer="{Voter server-71b20d2e-cd70-0e29-74a5-a54159011605 71b20d2e-cd70-0e29-74a5-a54159011605}" error="failed to connect to peer: 71b20d2e-cd70-0e29-74a5-a54159011605" -2026-03-31T06:17:51.156Z [WARN] server-71b20d2e-cd70-0e29-74a5-a54159011605: heartbeat timeout reached, starting election: last-leader-addr=a7f38cf4-329f-9cc0-3dff-ef43ab723d98 last-leader-id=server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98 -2026-03-31T06:17:51.156Z [INFO] server-71b20d2e-cd70-0e29-74a5-a54159011605: entering candidate state: node="Node at 71b20d2e-cd70-0e29-74a5-a54159011605 [Candidate]" term=3 -2026-03-31T06:17:51.156Z [ERROR] server-71b20d2e-cd70-0e29-74a5-a54159011605: failed to make requestVote RPC: target="{Voter server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98 a7f38cf4-329f-9cc0-3dff-ef43ab723d98}" error="failed to connect to peer: a7f38cf4-329f-9cc0-3dff-ef43ab723d98" term=3 -2026-03-31T06:17:51.183Z [WARN] server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:51.183Z [INFO] server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98: entering candidate state: node="Node at a7f38cf4-329f-9cc0-3dff-ef43ab723d98 [Candidate]" term=3 -2026-03-31T06:17:51.183Z [ERROR] server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98: failed to make requestVote RPC: target="{Voter server-71b20d2e-cd70-0e29-74a5-a54159011605 71b20d2e-cd70-0e29-74a5-a54159011605}" error="failed to connect to peer: 71b20d2e-cd70-0e29-74a5-a54159011605" term=3 -2026-03-31T06:17:51.227Z [WARN] server-71b20d2e-cd70-0e29-74a5-a54159011605: Election timeout reached, restarting election -2026-03-31T06:17:51.227Z [INFO] server-71b20d2e-cd70-0e29-74a5-a54159011605: entering candidate state: node="Node at 71b20d2e-cd70-0e29-74a5-a54159011605 [Candidate]" term=4 -2026-03-31T06:17:51.227Z [ERROR] server-71b20d2e-cd70-0e29-74a5-a54159011605: failed to make requestVote RPC: target="{Voter server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98 a7f38cf4-329f-9cc0-3dff-ef43ab723d98}" error="failed to connect to peer: a7f38cf4-329f-9cc0-3dff-ef43ab723d98" term=4 -2026-03-31T06:17:51.242Z [WARN] server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98: Election timeout reached, restarting election -2026-03-31T06:17:51.242Z [INFO] server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98: entering candidate state: node="Node at a7f38cf4-329f-9cc0-3dff-ef43ab723d98 [Candidate]" term=4 -2026-03-31T06:17:51.242Z [ERROR] server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98: failed to make requestVote RPC: target="{Voter server-71b20d2e-cd70-0e29-74a5-a54159011605 71b20d2e-cd70-0e29-74a5-a54159011605}" error="failed to connect to peer: 71b20d2e-cd70-0e29-74a5-a54159011605" term=4 -2026-03-31T06:17:51.288Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:51.18339026 +0000 UTC m=+99.460997681 (0 nodes), 89.88912ms from start of poll, 420.13896ms from cluster start. Timeout at 2026-03-31 06:17:51.28840082 +0000 UTC m=+99.566008201, 105.01052ms after stability -2026-03-31T06:17:51.306Z [WARN] server-71b20d2e-cd70-0e29-74a5-a54159011605: Election timeout reached, restarting election -2026-03-31T06:17:51.306Z [INFO] server-71b20d2e-cd70-0e29-74a5-a54159011605: entering candidate state: node="Node at 71b20d2e-cd70-0e29-74a5-a54159011605 [Candidate]" term=5 -2026-03-31T06:17:51.306Z [ERROR] server-71b20d2e-cd70-0e29-74a5-a54159011605: failed to make requestVote RPC: target="{Voter server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98 a7f38cf4-329f-9cc0-3dff-ef43ab723d98}" error="failed to connect to peer: a7f38cf4-329f-9cc0-3dff-ef43ab723d98" term=5 -2026-03-31T06:17:51.337Z [WARN] server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98: Election timeout reached, restarting election -2026-03-31T06:17:51.337Z [INFO] server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98: entering candidate state: node="Node at a7f38cf4-329f-9cc0-3dff-ef43ab723d98 [Candidate]" term=5 -2026-03-31T06:17:51.337Z [ERROR] server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98: failed to make requestVote RPC: target="{Voter server-71b20d2e-cd70-0e29-74a5-a54159011605 71b20d2e-cd70-0e29-74a5-a54159011605}" error="failed to connect to peer: 71b20d2e-cd70-0e29-74a5-a54159011605" term=5 -2026-03-31T06:17:51.363Z [WARN] server-71b20d2e-cd70-0e29-74a5-a54159011605: Election timeout reached, restarting election -2026-03-31T06:17:51.363Z [INFO] server-71b20d2e-cd70-0e29-74a5-a54159011605: entering candidate state: node="Node at 71b20d2e-cd70-0e29-74a5-a54159011605 [Candidate]" term=6 -2026-03-31T06:17:51.363Z [ERROR] server-71b20d2e-cd70-0e29-74a5-a54159011605: failed to make requestVote RPC: target="{Voter server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98 a7f38cf4-329f-9cc0-3dff-ef43ab723d98}" error="failed to connect to peer: a7f38cf4-329f-9cc0-3dff-ef43ab723d98" term=6 -2026-03-31T06:17:51.392Z [WARN] server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98: Election timeout reached, restarting election -2026-03-31T06:17:51.392Z [INFO] server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98: entering candidate state: node="Node at a7f38cf4-329f-9cc0-3dff-ef43ab723d98 [Candidate]" term=6 -2026-03-31T06:17:51.392Z [ERROR] server-a7f38cf4-329f-9cc0-3dff-ef43ab723d98: failed to make requestVote RPC: target="{Voter server-71b20d2e-cd70-0e29-74a5-a54159011605 71b20d2e-cd70-0e29-74a5-a54159011605}" error="failed to connect to peer: 71b20d2e-cd70-0e29-74a5-a54159011605" term=6 ---- PASS: TestRaft_LeaderLeaseExpire (0.63s) +2025-02-25T23:59:14.146Z [INFO] server-951961d7-efeb-be57-ffb2-04f1c912f77f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-951961d7-efeb-be57-ffb2-04f1c912f77f Address:951961d7-efeb-be57-ffb2-04f1c912f77f} {Suffrage:Voter ID:server-1d5e9e6c-519d-79d6-4621-9dd80c2a77c7 Address:1d5e9e6c-519d-79d6-4621-9dd80c2a77c7}]" +2025-02-25T23:59:14.146Z [INFO] server-1d5e9e6c-519d-79d6-4621-9dd80c2a77c7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-951961d7-efeb-be57-ffb2-04f1c912f77f Address:951961d7-efeb-be57-ffb2-04f1c912f77f} {Suffrage:Voter ID:server-1d5e9e6c-519d-79d6-4621-9dd80c2a77c7 Address:1d5e9e6c-519d-79d6-4621-9dd80c2a77c7}]" +2025-02-25T23:59:14.146Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:14.146Z [INFO] server-951961d7-efeb-be57-ffb2-04f1c912f77f: entering follower state: follower="Node at 951961d7-efeb-be57-ffb2-04f1c912f77f [Follower]" leader-address= leader-id= +2025-02-25T23:59:14.146Z [INFO] server-1d5e9e6c-519d-79d6-4621-9dd80c2a77c7: entering follower state: follower="Node at 1d5e9e6c-519d-79d6-4621-9dd80c2a77c7 [Follower]" leader-address= leader-id= +2025-02-25T23:59:14.222Z [WARN] server-1d5e9e6c-519d-79d6-4621-9dd80c2a77c7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:14.222Z [INFO] server-1d5e9e6c-519d-79d6-4621-9dd80c2a77c7: entering candidate state: node="Node at 1d5e9e6c-519d-79d6-4621-9dd80c2a77c7 [Candidate]" term=2 +2025-02-25T23:59:14.222Z [INFO] server-1d5e9e6c-519d-79d6-4621-9dd80c2a77c7: election won: term=2 tally=2 +2025-02-25T23:59:14.222Z [INFO] server-1d5e9e6c-519d-79d6-4621-9dd80c2a77c7: entering leader state: leader="Node at 1d5e9e6c-519d-79d6-4621-9dd80c2a77c7 [Leader]" +2025-02-25T23:59:14.222Z [INFO] server-1d5e9e6c-519d-79d6-4621-9dd80c2a77c7: added peer, starting replication: peer=server-951961d7-efeb-be57-ffb2-04f1c912f77f +2025-02-25T23:59:14.222Z [INFO] server-1d5e9e6c-519d-79d6-4621-9dd80c2a77c7: pipelining replication: peer="{Voter server-951961d7-efeb-be57-ffb2-04f1c912f77f 951961d7-efeb-be57-ffb2-04f1c912f77f}" +2025-02-25T23:59:14.332Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:14.222451519 +0000 UTC m=+101.831466926 (1 nodes), 75.664202ms from start of poll, 75.797723ms from cluster start. Timeout at 2025-02-25 23:59:14.327465119 +0000 UTC m=+101.936480526, 105.0136ms after stability +2025-02-25T23:59:14.337Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:59:14.446Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:59:14.337923454 +0000 UTC m=+101.946938861 (1 nodes), 1.24µs from start of poll, 191.269658ms from cluster start. Timeout at 2025-02-25 23:59:14.442956934 +0000 UTC m=+102.051972301, 105.03344ms after stability + raft_test.go:1799: [INFO] Disconnecting Node at 951961d7-efeb-be57-ffb2-04f1c912f77f [Follower] +2025-02-25T23:59:14.446Z [ERROR] server-1d5e9e6c-519d-79d6-4621-9dd80c2a77c7: failed to heartbeat to: peer=951961d7-efeb-be57-ffb2-04f1c912f77f backoff time=10ms error="failed to connect to peer: 951961d7-efeb-be57-ffb2-04f1c912f77f" +2025-02-25T23:59:14.454Z [ERROR] server-1d5e9e6c-519d-79d6-4621-9dd80c2a77c7: failed to pipeline appendEntries: peer="{Voter server-951961d7-efeb-be57-ffb2-04f1c912f77f 951961d7-efeb-be57-ffb2-04f1c912f77f}" error="append pipeline closed" +2025-02-25T23:59:14.454Z [INFO] server-1d5e9e6c-519d-79d6-4621-9dd80c2a77c7: aborting pipeline replication: peer="{Voter server-951961d7-efeb-be57-ffb2-04f1c912f77f 951961d7-efeb-be57-ffb2-04f1c912f77f}" +2025-02-25T23:59:14.462Z [ERROR] server-1d5e9e6c-519d-79d6-4621-9dd80c2a77c7: failed to appendEntries to: peer="{Voter server-951961d7-efeb-be57-ffb2-04f1c912f77f 951961d7-efeb-be57-ffb2-04f1c912f77f}" error="failed to connect to peer: 951961d7-efeb-be57-ffb2-04f1c912f77f" +2025-02-25T23:59:14.465Z [ERROR] server-1d5e9e6c-519d-79d6-4621-9dd80c2a77c7: failed to heartbeat to: peer=951961d7-efeb-be57-ffb2-04f1c912f77f backoff time=10ms error="failed to connect to peer: 951961d7-efeb-be57-ffb2-04f1c912f77f" +2025-02-25T23:59:14.482Z [ERROR] server-1d5e9e6c-519d-79d6-4621-9dd80c2a77c7: failed to appendEntries to: peer="{Voter server-951961d7-efeb-be57-ffb2-04f1c912f77f 951961d7-efeb-be57-ffb2-04f1c912f77f}" error="failed to connect to peer: 951961d7-efeb-be57-ffb2-04f1c912f77f" +2025-02-25T23:59:14.485Z [ERROR] server-1d5e9e6c-519d-79d6-4621-9dd80c2a77c7: failed to heartbeat to: peer=951961d7-efeb-be57-ffb2-04f1c912f77f backoff time=10ms error="failed to connect to peer: 951961d7-efeb-be57-ffb2-04f1c912f77f" +2025-02-25T23:59:14.493Z [WARN] server-1d5e9e6c-519d-79d6-4621-9dd80c2a77c7: failed to contact: server-id=server-951961d7-efeb-be57-ffb2-04f1c912f77f time=51.87757ms +2025-02-25T23:59:14.493Z [WARN] server-1d5e9e6c-519d-79d6-4621-9dd80c2a77c7: failed to contact quorum of nodes, stepping down +2025-02-25T23:59:14.493Z [INFO] server-1d5e9e6c-519d-79d6-4621-9dd80c2a77c7: entering follower state: follower="Node at 1d5e9e6c-519d-79d6-4621-9dd80c2a77c7 [Follower]" leader-address= leader-id= +2025-02-25T23:59:14.493Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:14.533Z [WARN] server-951961d7-efeb-be57-ffb2-04f1c912f77f: heartbeat timeout reached, starting election: last-leader-addr=1d5e9e6c-519d-79d6-4621-9dd80c2a77c7 last-leader-id=server-1d5e9e6c-519d-79d6-4621-9dd80c2a77c7 +2025-02-25T23:59:14.533Z [INFO] server-951961d7-efeb-be57-ffb2-04f1c912f77f: entering candidate state: node="Node at 951961d7-efeb-be57-ffb2-04f1c912f77f [Candidate]" term=3 +2025-02-25T23:59:14.534Z [ERROR] server-951961d7-efeb-be57-ffb2-04f1c912f77f: failed to make requestVote RPC: target="{Voter server-1d5e9e6c-519d-79d6-4621-9dd80c2a77c7 1d5e9e6c-519d-79d6-4621-9dd80c2a77c7}" error="failed to connect to peer: 1d5e9e6c-519d-79d6-4621-9dd80c2a77c7" term=3 +2025-02-25T23:59:14.573Z [WARN] server-1d5e9e6c-519d-79d6-4621-9dd80c2a77c7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:14.574Z [INFO] server-1d5e9e6c-519d-79d6-4621-9dd80c2a77c7: entering candidate state: node="Node at 1d5e9e6c-519d-79d6-4621-9dd80c2a77c7 [Candidate]" term=3 +2025-02-25T23:59:14.574Z [ERROR] server-1d5e9e6c-519d-79d6-4621-9dd80c2a77c7: failed to make requestVote RPC: target="{Voter server-951961d7-efeb-be57-ffb2-04f1c912f77f 951961d7-efeb-be57-ffb2-04f1c912f77f}" error="failed to connect to peer: 951961d7-efeb-be57-ffb2-04f1c912f77f" term=3 +2025-02-25T23:59:14.605Z [WARN] server-951961d7-efeb-be57-ffb2-04f1c912f77f: Election timeout reached, restarting election +2025-02-25T23:59:14.605Z [INFO] server-951961d7-efeb-be57-ffb2-04f1c912f77f: entering candidate state: node="Node at 951961d7-efeb-be57-ffb2-04f1c912f77f [Candidate]" term=4 +2025-02-25T23:59:14.605Z [ERROR] server-951961d7-efeb-be57-ffb2-04f1c912f77f: failed to make requestVote RPC: target="{Voter server-1d5e9e6c-519d-79d6-4621-9dd80c2a77c7 1d5e9e6c-519d-79d6-4621-9dd80c2a77c7}" error="failed to connect to peer: 1d5e9e6c-519d-79d6-4621-9dd80c2a77c7" term=4 +2025-02-25T23:59:14.670Z [WARN] server-1d5e9e6c-519d-79d6-4621-9dd80c2a77c7: Election timeout reached, restarting election +2025-02-25T23:59:14.670Z [INFO] server-1d5e9e6c-519d-79d6-4621-9dd80c2a77c7: entering candidate state: node="Node at 1d5e9e6c-519d-79d6-4621-9dd80c2a77c7 [Candidate]" term=4 +2025-02-25T23:59:14.670Z [ERROR] server-1d5e9e6c-519d-79d6-4621-9dd80c2a77c7: failed to make requestVote RPC: target="{Voter server-951961d7-efeb-be57-ffb2-04f1c912f77f 951961d7-efeb-be57-ffb2-04f1c912f77f}" error="failed to connect to peer: 951961d7-efeb-be57-ffb2-04f1c912f77f" term=4 +2025-02-25T23:59:14.674Z [WARN] server-951961d7-efeb-be57-ffb2-04f1c912f77f: Election timeout reached, restarting election +2025-02-25T23:59:14.674Z [INFO] server-951961d7-efeb-be57-ffb2-04f1c912f77f: entering candidate state: node="Node at 951961d7-efeb-be57-ffb2-04f1c912f77f [Candidate]" term=5 +2025-02-25T23:59:14.674Z [ERROR] server-951961d7-efeb-be57-ffb2-04f1c912f77f: failed to make requestVote RPC: target="{Voter server-1d5e9e6c-519d-79d6-4621-9dd80c2a77c7 1d5e9e6c-519d-79d6-4621-9dd80c2a77c7}" error="failed to connect to peer: 1d5e9e6c-519d-79d6-4621-9dd80c2a77c7" term=5 +2025-02-25T23:59:14.682Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:14.574163838 +0000 UTC m=+102.183179245 (0 nodes), 80.201791ms from start of poll, 427.510042ms from cluster start. Timeout at 2025-02-25 23:59:14.679174878 +0000 UTC m=+102.288190245, 105.011ms after stability +2025-02-25T23:59:14.745Z [WARN] server-1d5e9e6c-519d-79d6-4621-9dd80c2a77c7: Election timeout reached, restarting election +2025-02-25T23:59:14.745Z [INFO] server-1d5e9e6c-519d-79d6-4621-9dd80c2a77c7: entering candidate state: node="Node at 1d5e9e6c-519d-79d6-4621-9dd80c2a77c7 [Candidate]" term=5 +2025-02-25T23:59:14.746Z [ERROR] server-1d5e9e6c-519d-79d6-4621-9dd80c2a77c7: failed to make requestVote RPC: target="{Voter server-951961d7-efeb-be57-ffb2-04f1c912f77f 951961d7-efeb-be57-ffb2-04f1c912f77f}" error="failed to connect to peer: 951961d7-efeb-be57-ffb2-04f1c912f77f" term=5 +2025-02-25T23:59:14.774Z [WARN] server-951961d7-efeb-be57-ffb2-04f1c912f77f: Election timeout reached, restarting election +2025-02-25T23:59:14.774Z [INFO] server-951961d7-efeb-be57-ffb2-04f1c912f77f: entering candidate state: node="Node at 951961d7-efeb-be57-ffb2-04f1c912f77f [Candidate]" term=6 +2025-02-25T23:59:14.774Z [ERROR] server-951961d7-efeb-be57-ffb2-04f1c912f77f: failed to make requestVote RPC: target="{Voter server-1d5e9e6c-519d-79d6-4621-9dd80c2a77c7 1d5e9e6c-519d-79d6-4621-9dd80c2a77c7}" error="failed to connect to peer: 1d5e9e6c-519d-79d6-4621-9dd80c2a77c7" term=6 +--- PASS: TestRaft_LeaderLeaseExpire (0.64s) === RUN TestRaft_Barrier -2026-03-31T06:17:51.395Z [INFO] server-34f0a963-2d6c-59ca-96d0-b9f75dc86b2b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-34f0a963-2d6c-59ca-96d0-b9f75dc86b2b Address:34f0a963-2d6c-59ca-96d0-b9f75dc86b2b} {Suffrage:Voter ID:server-3f6f0386-ee87-fa1e-e03e-dfe8ae0502cb Address:3f6f0386-ee87-fa1e-e03e-dfe8ae0502cb} {Suffrage:Voter ID:server-c1a650a5-e22b-b074-15db-e605969ca9bb Address:c1a650a5-e22b-b074-15db-e605969ca9bb}]" -2026-03-31T06:17:51.395Z [INFO] server-34f0a963-2d6c-59ca-96d0-b9f75dc86b2b: entering follower state: follower="Node at 34f0a963-2d6c-59ca-96d0-b9f75dc86b2b [Follower]" leader-address= leader-id= -2026-03-31T06:17:51.395Z [INFO] server-3f6f0386-ee87-fa1e-e03e-dfe8ae0502cb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-34f0a963-2d6c-59ca-96d0-b9f75dc86b2b Address:34f0a963-2d6c-59ca-96d0-b9f75dc86b2b} {Suffrage:Voter ID:server-3f6f0386-ee87-fa1e-e03e-dfe8ae0502cb Address:3f6f0386-ee87-fa1e-e03e-dfe8ae0502cb} {Suffrage:Voter ID:server-c1a650a5-e22b-b074-15db-e605969ca9bb Address:c1a650a5-e22b-b074-15db-e605969ca9bb}]" -2026-03-31T06:17:51.395Z [INFO] server-3f6f0386-ee87-fa1e-e03e-dfe8ae0502cb: entering follower state: follower="Node at 3f6f0386-ee87-fa1e-e03e-dfe8ae0502cb [Follower]" leader-address= leader-id= -2026-03-31T06:17:51.395Z [INFO] server-c1a650a5-e22b-b074-15db-e605969ca9bb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-34f0a963-2d6c-59ca-96d0-b9f75dc86b2b Address:34f0a963-2d6c-59ca-96d0-b9f75dc86b2b} {Suffrage:Voter ID:server-3f6f0386-ee87-fa1e-e03e-dfe8ae0502cb Address:3f6f0386-ee87-fa1e-e03e-dfe8ae0502cb} {Suffrage:Voter ID:server-c1a650a5-e22b-b074-15db-e605969ca9bb Address:c1a650a5-e22b-b074-15db-e605969ca9bb}]" -2026-03-31T06:17:51.395Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:51.395Z [INFO] server-c1a650a5-e22b-b074-15db-e605969ca9bb: entering follower state: follower="Node at c1a650a5-e22b-b074-15db-e605969ca9bb [Follower]" leader-address= leader-id= -2026-03-31T06:17:51.449Z [WARN] server-34f0a963-2d6c-59ca-96d0-b9f75dc86b2b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:51.449Z [INFO] server-34f0a963-2d6c-59ca-96d0-b9f75dc86b2b: entering candidate state: node="Node at 34f0a963-2d6c-59ca-96d0-b9f75dc86b2b [Candidate]" term=2 -2026-03-31T06:17:51.449Z [INFO] server-34f0a963-2d6c-59ca-96d0-b9f75dc86b2b: election won: term=2 tally=2 -2026-03-31T06:17:51.449Z [INFO] server-34f0a963-2d6c-59ca-96d0-b9f75dc86b2b: entering leader state: leader="Node at 34f0a963-2d6c-59ca-96d0-b9f75dc86b2b [Leader]" -2026-03-31T06:17:51.449Z [INFO] server-34f0a963-2d6c-59ca-96d0-b9f75dc86b2b: added peer, starting replication: peer=server-3f6f0386-ee87-fa1e-e03e-dfe8ae0502cb -2026-03-31T06:17:51.449Z [INFO] server-34f0a963-2d6c-59ca-96d0-b9f75dc86b2b: added peer, starting replication: peer=server-c1a650a5-e22b-b074-15db-e605969ca9bb -2026-03-31T06:17:51.450Z [INFO] server-34f0a963-2d6c-59ca-96d0-b9f75dc86b2b: pipelining replication: peer="{Voter server-3f6f0386-ee87-fa1e-e03e-dfe8ae0502cb 3f6f0386-ee87-fa1e-e03e-dfe8ae0502cb}" -2026-03-31T06:17:51.450Z [INFO] server-34f0a963-2d6c-59ca-96d0-b9f75dc86b2b: pipelining replication: peer="{Voter server-c1a650a5-e22b-b074-15db-e605969ca9bb c1a650a5-e22b-b074-15db-e605969ca9bb}" -2026-03-31T06:17:51.554Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:51.44989866 +0000 UTC m=+99.727506041 (1 nodes), 54.11ms from start of poll, 54.58356ms from cluster start. Timeout at 2026-03-31 06:17:51.55490866 +0000 UTC m=+99.832516081, 105.01004ms after stability -2026-03-31T06:17:51.573Z [INFO] server-34f0a963-2d6c-59ca-96d0-b9f75dc86b2b: aborting pipeline replication: peer="{Voter server-c1a650a5-e22b-b074-15db-e605969ca9bb c1a650a5-e22b-b074-15db-e605969ca9bb}" -2026-03-31T06:17:51.573Z [INFO] server-34f0a963-2d6c-59ca-96d0-b9f75dc86b2b: aborting pipeline replication: peer="{Voter server-3f6f0386-ee87-fa1e-e03e-dfe8ae0502cb 3f6f0386-ee87-fa1e-e03e-dfe8ae0502cb}" +2025-02-25T23:59:14.790Z [INFO] server-a68bf0b6-912e-5c1c-1ec9-bfa40b7d1a25: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a68bf0b6-912e-5c1c-1ec9-bfa40b7d1a25 Address:a68bf0b6-912e-5c1c-1ec9-bfa40b7d1a25} {Suffrage:Voter ID:server-b7fd16bd-4431-4fb1-88a2-8007215d613b Address:b7fd16bd-4431-4fb1-88a2-8007215d613b} {Suffrage:Voter ID:server-329e9973-56e7-09c5-4f98-1f47c5b3df86 Address:329e9973-56e7-09c5-4f98-1f47c5b3df86}]" +2025-02-25T23:59:14.790Z [INFO] server-b7fd16bd-4431-4fb1-88a2-8007215d613b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a68bf0b6-912e-5c1c-1ec9-bfa40b7d1a25 Address:a68bf0b6-912e-5c1c-1ec9-bfa40b7d1a25} {Suffrage:Voter ID:server-b7fd16bd-4431-4fb1-88a2-8007215d613b Address:b7fd16bd-4431-4fb1-88a2-8007215d613b} {Suffrage:Voter ID:server-329e9973-56e7-09c5-4f98-1f47c5b3df86 Address:329e9973-56e7-09c5-4f98-1f47c5b3df86}]" +2025-02-25T23:59:14.790Z [INFO] server-329e9973-56e7-09c5-4f98-1f47c5b3df86: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a68bf0b6-912e-5c1c-1ec9-bfa40b7d1a25 Address:a68bf0b6-912e-5c1c-1ec9-bfa40b7d1a25} {Suffrage:Voter ID:server-b7fd16bd-4431-4fb1-88a2-8007215d613b Address:b7fd16bd-4431-4fb1-88a2-8007215d613b} {Suffrage:Voter ID:server-329e9973-56e7-09c5-4f98-1f47c5b3df86 Address:329e9973-56e7-09c5-4f98-1f47c5b3df86}]" +2025-02-25T23:59:14.790Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:14.790Z [INFO] server-a68bf0b6-912e-5c1c-1ec9-bfa40b7d1a25: entering follower state: follower="Node at a68bf0b6-912e-5c1c-1ec9-bfa40b7d1a25 [Follower]" leader-address= leader-id= +2025-02-25T23:59:14.790Z [INFO] server-b7fd16bd-4431-4fb1-88a2-8007215d613b: entering follower state: follower="Node at b7fd16bd-4431-4fb1-88a2-8007215d613b [Follower]" leader-address= leader-id= +2025-02-25T23:59:14.790Z [INFO] server-329e9973-56e7-09c5-4f98-1f47c5b3df86: entering follower state: follower="Node at 329e9973-56e7-09c5-4f98-1f47c5b3df86 [Follower]" leader-address= leader-id= +2025-02-25T23:59:14.850Z [WARN] server-a68bf0b6-912e-5c1c-1ec9-bfa40b7d1a25: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:14.850Z [INFO] server-a68bf0b6-912e-5c1c-1ec9-bfa40b7d1a25: entering candidate state: node="Node at a68bf0b6-912e-5c1c-1ec9-bfa40b7d1a25 [Candidate]" term=2 +2025-02-25T23:59:14.850Z [WARN] server-329e9973-56e7-09c5-4f98-1f47c5b3df86: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:14.850Z [INFO] server-329e9973-56e7-09c5-4f98-1f47c5b3df86: entering candidate state: node="Node at 329e9973-56e7-09c5-4f98-1f47c5b3df86 [Candidate]" term=2 +2025-02-25T23:59:14.850Z [INFO] server-329e9973-56e7-09c5-4f98-1f47c5b3df86: duplicate requestVote for same term: term=2 +2025-02-25T23:59:14.850Z [INFO] server-a68bf0b6-912e-5c1c-1ec9-bfa40b7d1a25: election won: term=2 tally=2 +2025-02-25T23:59:14.850Z [INFO] server-a68bf0b6-912e-5c1c-1ec9-bfa40b7d1a25: entering leader state: leader="Node at a68bf0b6-912e-5c1c-1ec9-bfa40b7d1a25 [Leader]" +2025-02-25T23:59:14.850Z [INFO] server-a68bf0b6-912e-5c1c-1ec9-bfa40b7d1a25: added peer, starting replication: peer=server-b7fd16bd-4431-4fb1-88a2-8007215d613b +2025-02-25T23:59:14.850Z [INFO] server-a68bf0b6-912e-5c1c-1ec9-bfa40b7d1a25: added peer, starting replication: peer=server-329e9973-56e7-09c5-4f98-1f47c5b3df86 +2025-02-25T23:59:14.850Z [INFO] server-329e9973-56e7-09c5-4f98-1f47c5b3df86: entering follower state: follower="Node at 329e9973-56e7-09c5-4f98-1f47c5b3df86 [Follower]" leader-address=a68bf0b6-912e-5c1c-1ec9-bfa40b7d1a25 leader-id=server-a68bf0b6-912e-5c1c-1ec9-bfa40b7d1a25 +2025-02-25T23:59:14.850Z [INFO] server-a68bf0b6-912e-5c1c-1ec9-bfa40b7d1a25: pipelining replication: peer="{Voter server-329e9973-56e7-09c5-4f98-1f47c5b3df86 329e9973-56e7-09c5-4f98-1f47c5b3df86}" +2025-02-25T23:59:14.850Z [INFO] server-b7fd16bd-4431-4fb1-88a2-8007215d613b: duplicate requestVote for same term: term=2 +2025-02-25T23:59:14.850Z [INFO] server-a68bf0b6-912e-5c1c-1ec9-bfa40b7d1a25: pipelining replication: peer="{Voter server-b7fd16bd-4431-4fb1-88a2-8007215d613b b7fd16bd-4431-4fb1-88a2-8007215d613b}" +2025-02-25T23:59:14.850Z [WARN] server-a68bf0b6-912e-5c1c-1ec9-bfa40b7d1a25: rejecting vote request since we have a leader: from=329e9973-56e7-09c5-4f98-1f47c5b3df86 leader=a68bf0b6-912e-5c1c-1ec9-bfa40b7d1a25 leader-id=server-a68bf0b6-912e-5c1c-1ec9-bfa40b7d1a25 +2025-02-25T23:59:14.962Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:14.850576958 +0000 UTC m=+102.459592365 (1 nodes), 60.021102ms from start of poll, 60.206223ms from cluster start. Timeout at 2025-02-25 23:59:14.955578478 +0000 UTC m=+102.564593885, 105.00152ms after stability +2025-02-25T23:59:14.974Z [INFO] server-a68bf0b6-912e-5c1c-1ec9-bfa40b7d1a25: aborting pipeline replication: peer="{Voter server-b7fd16bd-4431-4fb1-88a2-8007215d613b b7fd16bd-4431-4fb1-88a2-8007215d613b}" +2025-02-25T23:59:14.974Z [INFO] server-a68bf0b6-912e-5c1c-1ec9-bfa40b7d1a25: aborting pipeline replication: peer="{Voter server-329e9973-56e7-09c5-4f98-1f47c5b3df86 329e9973-56e7-09c5-4f98-1f47c5b3df86}" --- PASS: TestRaft_Barrier (0.18s) === RUN TestRaft_VerifyLeader -2026-03-31T06:17:51.574Z [INFO] server-06c85147-c5a6-5cba-b5c7-e5d14d114d07: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-06c85147-c5a6-5cba-b5c7-e5d14d114d07 Address:06c85147-c5a6-5cba-b5c7-e5d14d114d07} {Suffrage:Voter ID:server-9124fbc9-38f7-5775-26e0-d782db1645f5 Address:9124fbc9-38f7-5775-26e0-d782db1645f5} {Suffrage:Voter ID:server-9c164685-f2f0-af6c-b2a8-1bb3a69d9ca1 Address:9c164685-f2f0-af6c-b2a8-1bb3a69d9ca1}]" -2026-03-31T06:17:51.574Z [INFO] server-9124fbc9-38f7-5775-26e0-d782db1645f5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-06c85147-c5a6-5cba-b5c7-e5d14d114d07 Address:06c85147-c5a6-5cba-b5c7-e5d14d114d07} {Suffrage:Voter ID:server-9124fbc9-38f7-5775-26e0-d782db1645f5 Address:9124fbc9-38f7-5775-26e0-d782db1645f5} {Suffrage:Voter ID:server-9c164685-f2f0-af6c-b2a8-1bb3a69d9ca1 Address:9c164685-f2f0-af6c-b2a8-1bb3a69d9ca1}]" -2026-03-31T06:17:51.574Z [INFO] server-06c85147-c5a6-5cba-b5c7-e5d14d114d07: entering follower state: follower="Node at 06c85147-c5a6-5cba-b5c7-e5d14d114d07 [Follower]" leader-address= leader-id= -2026-03-31T06:17:51.574Z [INFO] server-9124fbc9-38f7-5775-26e0-d782db1645f5: entering follower state: follower="Node at 9124fbc9-38f7-5775-26e0-d782db1645f5 [Follower]" leader-address= leader-id= -2026-03-31T06:17:51.574Z [INFO] server-9c164685-f2f0-af6c-b2a8-1bb3a69d9ca1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-06c85147-c5a6-5cba-b5c7-e5d14d114d07 Address:06c85147-c5a6-5cba-b5c7-e5d14d114d07} {Suffrage:Voter ID:server-9124fbc9-38f7-5775-26e0-d782db1645f5 Address:9124fbc9-38f7-5775-26e0-d782db1645f5} {Suffrage:Voter ID:server-9c164685-f2f0-af6c-b2a8-1bb3a69d9ca1 Address:9c164685-f2f0-af6c-b2a8-1bb3a69d9ca1}]" -2026-03-31T06:17:51.574Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:51.574Z [INFO] server-9c164685-f2f0-af6c-b2a8-1bb3a69d9ca1: entering follower state: follower="Node at 9c164685-f2f0-af6c-b2a8-1bb3a69d9ca1 [Follower]" leader-address= leader-id= -2026-03-31T06:17:51.642Z [WARN] server-06c85147-c5a6-5cba-b5c7-e5d14d114d07: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:51.642Z [INFO] server-06c85147-c5a6-5cba-b5c7-e5d14d114d07: entering candidate state: node="Node at 06c85147-c5a6-5cba-b5c7-e5d14d114d07 [Candidate]" term=2 -2026-03-31T06:17:51.642Z [INFO] server-06c85147-c5a6-5cba-b5c7-e5d14d114d07: election won: term=2 tally=2 -2026-03-31T06:17:51.642Z [INFO] server-06c85147-c5a6-5cba-b5c7-e5d14d114d07: entering leader state: leader="Node at 06c85147-c5a6-5cba-b5c7-e5d14d114d07 [Leader]" -2026-03-31T06:17:51.642Z [INFO] server-06c85147-c5a6-5cba-b5c7-e5d14d114d07: added peer, starting replication: peer=server-9124fbc9-38f7-5775-26e0-d782db1645f5 -2026-03-31T06:17:51.642Z [INFO] server-06c85147-c5a6-5cba-b5c7-e5d14d114d07: added peer, starting replication: peer=server-9c164685-f2f0-af6c-b2a8-1bb3a69d9ca1 -2026-03-31T06:17:51.642Z [INFO] server-06c85147-c5a6-5cba-b5c7-e5d14d114d07: pipelining replication: peer="{Voter server-9c164685-f2f0-af6c-b2a8-1bb3a69d9ca1 9c164685-f2f0-af6c-b2a8-1bb3a69d9ca1}" -2026-03-31T06:17:51.642Z [INFO] server-06c85147-c5a6-5cba-b5c7-e5d14d114d07: pipelining replication: peer="{Voter server-9124fbc9-38f7-5775-26e0-d782db1645f5 9124fbc9-38f7-5775-26e0-d782db1645f5}" -2026-03-31T06:17:51.747Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:51.64263514 +0000 UTC m=+99.920242561 (1 nodes), 68.15388ms from start of poll, 68.57048ms from cluster start. Timeout at 2026-03-31 06:17:51.74763914 +0000 UTC m=+100.025246521, 105.00396ms after stability -2026-03-31T06:17:51.748Z [INFO] server-06c85147-c5a6-5cba-b5c7-e5d14d114d07: aborting pipeline replication: peer="{Voter server-9c164685-f2f0-af6c-b2a8-1bb3a69d9ca1 9c164685-f2f0-af6c-b2a8-1bb3a69d9ca1}" -2026-03-31T06:17:51.748Z [INFO] server-06c85147-c5a6-5cba-b5c7-e5d14d114d07: aborting pipeline replication: peer="{Voter server-9124fbc9-38f7-5775-26e0-d782db1645f5 9124fbc9-38f7-5775-26e0-d782db1645f5}" ---- PASS: TestRaft_VerifyLeader (0.17s) +2025-02-25T23:59:14.974Z [INFO] server-f343f53f-ed4e-8116-a1fd-3d73f454111c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f343f53f-ed4e-8116-a1fd-3d73f454111c Address:f343f53f-ed4e-8116-a1fd-3d73f454111c} {Suffrage:Voter ID:server-75fcde8c-4547-034c-0563-5fb126462e61 Address:75fcde8c-4547-034c-0563-5fb126462e61} {Suffrage:Voter ID:server-df337323-f61a-521c-e1d1-6cb4714c720a Address:df337323-f61a-521c-e1d1-6cb4714c720a}]" +2025-02-25T23:59:14.974Z [INFO] server-75fcde8c-4547-034c-0563-5fb126462e61: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f343f53f-ed4e-8116-a1fd-3d73f454111c Address:f343f53f-ed4e-8116-a1fd-3d73f454111c} {Suffrage:Voter ID:server-75fcde8c-4547-034c-0563-5fb126462e61 Address:75fcde8c-4547-034c-0563-5fb126462e61} {Suffrage:Voter ID:server-df337323-f61a-521c-e1d1-6cb4714c720a Address:df337323-f61a-521c-e1d1-6cb4714c720a}]" +2025-02-25T23:59:14.974Z [INFO] server-df337323-f61a-521c-e1d1-6cb4714c720a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f343f53f-ed4e-8116-a1fd-3d73f454111c Address:f343f53f-ed4e-8116-a1fd-3d73f454111c} {Suffrage:Voter ID:server-75fcde8c-4547-034c-0563-5fb126462e61 Address:75fcde8c-4547-034c-0563-5fb126462e61} {Suffrage:Voter ID:server-df337323-f61a-521c-e1d1-6cb4714c720a Address:df337323-f61a-521c-e1d1-6cb4714c720a}]" +2025-02-25T23:59:14.974Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:14.974Z [INFO] server-f343f53f-ed4e-8116-a1fd-3d73f454111c: entering follower state: follower="Node at f343f53f-ed4e-8116-a1fd-3d73f454111c [Follower]" leader-address= leader-id= +2025-02-25T23:59:14.974Z [INFO] server-75fcde8c-4547-034c-0563-5fb126462e61: entering follower state: follower="Node at 75fcde8c-4547-034c-0563-5fb126462e61 [Follower]" leader-address= leader-id= +2025-02-25T23:59:14.974Z [INFO] server-df337323-f61a-521c-e1d1-6cb4714c720a: entering follower state: follower="Node at df337323-f61a-521c-e1d1-6cb4714c720a [Follower]" leader-address= leader-id= +2025-02-25T23:59:15.033Z [WARN] server-df337323-f61a-521c-e1d1-6cb4714c720a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:15.033Z [INFO] server-df337323-f61a-521c-e1d1-6cb4714c720a: entering candidate state: node="Node at df337323-f61a-521c-e1d1-6cb4714c720a [Candidate]" term=2 +2025-02-25T23:59:15.034Z [INFO] server-df337323-f61a-521c-e1d1-6cb4714c720a: election won: term=2 tally=2 +2025-02-25T23:59:15.034Z [INFO] server-df337323-f61a-521c-e1d1-6cb4714c720a: entering leader state: leader="Node at df337323-f61a-521c-e1d1-6cb4714c720a [Leader]" +2025-02-25T23:59:15.034Z [INFO] server-df337323-f61a-521c-e1d1-6cb4714c720a: added peer, starting replication: peer=server-f343f53f-ed4e-8116-a1fd-3d73f454111c +2025-02-25T23:59:15.034Z [INFO] server-df337323-f61a-521c-e1d1-6cb4714c720a: added peer, starting replication: peer=server-75fcde8c-4547-034c-0563-5fb126462e61 +2025-02-25T23:59:15.034Z [INFO] server-df337323-f61a-521c-e1d1-6cb4714c720a: pipelining replication: peer="{Voter server-75fcde8c-4547-034c-0563-5fb126462e61 75fcde8c-4547-034c-0563-5fb126462e61}" +2025-02-25T23:59:15.034Z [INFO] server-df337323-f61a-521c-e1d1-6cb4714c720a: pipelining replication: peer="{Voter server-f343f53f-ed4e-8116-a1fd-3d73f454111c f343f53f-ed4e-8116-a1fd-3d73f454111c}" +2025-02-25T23:59:15.141Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:15.034226487 +0000 UTC m=+102.643241894 (1 nodes), 59.400938ms from start of poll, 59.60862ms from cluster start. Timeout at 2025-02-25 23:59:15.139229567 +0000 UTC m=+102.748244974, 105.00308ms after stability +2025-02-25T23:59:15.142Z [INFO] server-df337323-f61a-521c-e1d1-6cb4714c720a: aborting pipeline replication: peer="{Voter server-75fcde8c-4547-034c-0563-5fb126462e61 75fcde8c-4547-034c-0563-5fb126462e61}" +2025-02-25T23:59:15.142Z [INFO] server-df337323-f61a-521c-e1d1-6cb4714c720a: aborting pipeline replication: peer="{Voter server-f343f53f-ed4e-8116-a1fd-3d73f454111c f343f53f-ed4e-8116-a1fd-3d73f454111c}" +2025-02-25T23:59:15.642Z [ERROR] server-df337323-f61a-521c-e1d1-6cb4714c720a: failed to heartbeat to: peer=75fcde8c-4547-034c-0563-5fb126462e61 backoff time=10ms error="command timed out" +--- PASS: TestRaft_VerifyLeader (0.67s) === RUN TestRaft_VerifyLeader_Single -2026-03-31T06:17:51.748Z [INFO] server-0c901148-d2fa-f628-e258-3274be0ccd3b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0c901148-d2fa-f628-e258-3274be0ccd3b Address:0c901148-d2fa-f628-e258-3274be0ccd3b}]" -2026-03-31T06:17:51.749Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:51.749Z [INFO] server-0c901148-d2fa-f628-e258-3274be0ccd3b: entering follower state: follower="Node at 0c901148-d2fa-f628-e258-3274be0ccd3b [Follower]" leader-address= leader-id= -2026-03-31T06:17:51.820Z [WARN] server-0c901148-d2fa-f628-e258-3274be0ccd3b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:51.821Z [INFO] server-0c901148-d2fa-f628-e258-3274be0ccd3b: entering candidate state: node="Node at 0c901148-d2fa-f628-e258-3274be0ccd3b [Candidate]" term=2 -2026-03-31T06:17:51.821Z [INFO] server-0c901148-d2fa-f628-e258-3274be0ccd3b: election won: term=2 tally=1 -2026-03-31T06:17:51.821Z [INFO] server-0c901148-d2fa-f628-e258-3274be0ccd3b: entering leader state: leader="Node at 0c901148-d2fa-f628-e258-3274be0ccd3b [Leader]" -2026-03-31T06:17:51.927Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:51.82235826 +0000 UTC m=+100.099965681 (1 nodes), 73.31152ms from start of poll, 73.53468ms from cluster start. Timeout at 2026-03-31 06:17:51.92738034 +0000 UTC m=+100.204987721, 105.02204ms after stability ---- PASS: TestRaft_VerifyLeader_Single (0.18s) +2025-02-25T23:59:15.642Z [INFO] server-da7d3b7b-194b-2f2c-7c03-33de548626ed: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-da7d3b7b-194b-2f2c-7c03-33de548626ed Address:da7d3b7b-194b-2f2c-7c03-33de548626ed}]" +2025-02-25T23:59:15.642Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:15.642Z [INFO] server-da7d3b7b-194b-2f2c-7c03-33de548626ed: entering follower state: follower="Node at da7d3b7b-194b-2f2c-7c03-33de548626ed [Follower]" leader-address= leader-id= +2025-02-25T23:59:15.698Z [WARN] server-da7d3b7b-194b-2f2c-7c03-33de548626ed: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:15.698Z [INFO] server-da7d3b7b-194b-2f2c-7c03-33de548626ed: entering candidate state: node="Node at da7d3b7b-194b-2f2c-7c03-33de548626ed [Candidate]" term=2 +2025-02-25T23:59:15.698Z [INFO] server-da7d3b7b-194b-2f2c-7c03-33de548626ed: election won: term=2 tally=1 +2025-02-25T23:59:15.698Z [INFO] server-da7d3b7b-194b-2f2c-7c03-33de548626ed: entering leader state: leader="Node at da7d3b7b-194b-2f2c-7c03-33de548626ed [Leader]" +2025-02-25T23:59:15.806Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:15.698454596 +0000 UTC m=+103.307470003 (1 nodes), 55.731715ms from start of poll, 55.862316ms from cluster start. Timeout at 2025-02-25 23:59:15.803457796 +0000 UTC m=+103.412473163, 105.00316ms after stability +--- PASS: TestRaft_VerifyLeader_Single (0.16s) === RUN TestRaft_VerifyLeader_Fail -2026-03-31T06:17:51.928Z [INFO] server-efc6e291-a578-0069-368f-41258eaf6e28: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-efc6e291-a578-0069-368f-41258eaf6e28 Address:efc6e291-a578-0069-368f-41258eaf6e28} {Suffrage:Voter ID:server-5e9563d4-1a6d-1fa3-b7d8-c8f63f652082 Address:5e9563d4-1a6d-1fa3-b7d8-c8f63f652082}]" -2026-03-31T06:17:51.928Z [INFO] server-5e9563d4-1a6d-1fa3-b7d8-c8f63f652082: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-efc6e291-a578-0069-368f-41258eaf6e28 Address:efc6e291-a578-0069-368f-41258eaf6e28} {Suffrage:Voter ID:server-5e9563d4-1a6d-1fa3-b7d8-c8f63f652082 Address:5e9563d4-1a6d-1fa3-b7d8-c8f63f652082}]" -2026-03-31T06:17:51.928Z [INFO] server-efc6e291-a578-0069-368f-41258eaf6e28: entering follower state: follower="Node at efc6e291-a578-0069-368f-41258eaf6e28 [Follower]" leader-address= leader-id= -2026-03-31T06:17:51.928Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:51.928Z [INFO] server-5e9563d4-1a6d-1fa3-b7d8-c8f63f652082: entering follower state: follower="Node at 5e9563d4-1a6d-1fa3-b7d8-c8f63f652082 [Follower]" leader-address= leader-id= -2026-03-31T06:17:51.978Z [WARN] server-efc6e291-a578-0069-368f-41258eaf6e28: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:51.978Z [INFO] server-efc6e291-a578-0069-368f-41258eaf6e28: entering candidate state: node="Node at efc6e291-a578-0069-368f-41258eaf6e28 [Candidate]" term=2 -2026-03-31T06:17:51.979Z [INFO] server-efc6e291-a578-0069-368f-41258eaf6e28: election won: term=2 tally=2 -2026-03-31T06:17:51.979Z [INFO] server-efc6e291-a578-0069-368f-41258eaf6e28: entering leader state: leader="Node at efc6e291-a578-0069-368f-41258eaf6e28 [Leader]" -2026-03-31T06:17:51.979Z [INFO] server-efc6e291-a578-0069-368f-41258eaf6e28: added peer, starting replication: peer=server-5e9563d4-1a6d-1fa3-b7d8-c8f63f652082 -2026-03-31T06:17:51.979Z [INFO] server-efc6e291-a578-0069-368f-41258eaf6e28: pipelining replication: peer="{Voter server-5e9563d4-1a6d-1fa3-b7d8-c8f63f652082 5e9563d4-1a6d-1fa3-b7d8-c8f63f652082}" -2026-03-31T06:17:52.085Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:51.97921158 +0000 UTC m=+100.256818961 (1 nodes), 50.6304ms from start of poll, 50.93872ms from cluster start. Timeout at 2026-03-31 06:17:52.08421498 +0000 UTC m=+100.361822401, 105.00344ms after stability -2026-03-31T06:17:52.085Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:17:52.190Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:17:52.08540646 +0000 UTC m=+100.363013841 (1 nodes), 2.64µs from start of poll, 157.1336ms from cluster start. Timeout at 2026-03-31 06:17:52.19043466 +0000 UTC m=+100.468042081, 105.02824ms after stability -2026-03-31T06:17:52.190Z [ERROR] server-efc6e291-a578-0069-368f-41258eaf6e28: peer has newer term, stopping replication: peer="{Voter server-5e9563d4-1a6d-1fa3-b7d8-c8f63f652082 5e9563d4-1a6d-1fa3-b7d8-c8f63f652082}" -2026-03-31T06:17:52.190Z [INFO] server-efc6e291-a578-0069-368f-41258eaf6e28: aborting pipeline replication: peer="{Voter server-5e9563d4-1a6d-1fa3-b7d8-c8f63f652082 5e9563d4-1a6d-1fa3-b7d8-c8f63f652082}" -2026-03-31T06:17:52.191Z [INFO] server-efc6e291-a578-0069-368f-41258eaf6e28: entering follower state: follower="Node at efc6e291-a578-0069-368f-41258eaf6e28 [Follower]" leader-address= leader-id= ---- PASS: TestRaft_VerifyLeader_Fail (0.26s) +2025-02-25T23:59:15.806Z [INFO] server-dc33d13b-b3db-1636-9c30-bce348ae89df: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dc33d13b-b3db-1636-9c30-bce348ae89df Address:dc33d13b-b3db-1636-9c30-bce348ae89df} {Suffrage:Voter ID:server-a4914f51-5f3e-4764-2b7a-366c33d30915 Address:a4914f51-5f3e-4764-2b7a-366c33d30915}]" +2025-02-25T23:59:15.806Z [INFO] server-a4914f51-5f3e-4764-2b7a-366c33d30915: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dc33d13b-b3db-1636-9c30-bce348ae89df Address:dc33d13b-b3db-1636-9c30-bce348ae89df} {Suffrage:Voter ID:server-a4914f51-5f3e-4764-2b7a-366c33d30915 Address:a4914f51-5f3e-4764-2b7a-366c33d30915}]" +2025-02-25T23:59:15.806Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:15.806Z [INFO] server-dc33d13b-b3db-1636-9c30-bce348ae89df: entering follower state: follower="Node at dc33d13b-b3db-1636-9c30-bce348ae89df [Follower]" leader-address= leader-id= +2025-02-25T23:59:15.806Z [INFO] server-a4914f51-5f3e-4764-2b7a-366c33d30915: entering follower state: follower="Node at a4914f51-5f3e-4764-2b7a-366c33d30915 [Follower]" leader-address= leader-id= +2025-02-25T23:59:15.862Z [WARN] server-a4914f51-5f3e-4764-2b7a-366c33d30915: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:15.862Z [INFO] server-a4914f51-5f3e-4764-2b7a-366c33d30915: entering candidate state: node="Node at a4914f51-5f3e-4764-2b7a-366c33d30915 [Candidate]" term=2 +2025-02-25T23:59:15.862Z [INFO] server-a4914f51-5f3e-4764-2b7a-366c33d30915: election won: term=2 tally=2 +2025-02-25T23:59:15.862Z [INFO] server-a4914f51-5f3e-4764-2b7a-366c33d30915: entering leader state: leader="Node at a4914f51-5f3e-4764-2b7a-366c33d30915 [Leader]" +2025-02-25T23:59:15.862Z [INFO] server-a4914f51-5f3e-4764-2b7a-366c33d30915: added peer, starting replication: peer=server-dc33d13b-b3db-1636-9c30-bce348ae89df +2025-02-25T23:59:15.862Z [INFO] server-a4914f51-5f3e-4764-2b7a-366c33d30915: pipelining replication: peer="{Voter server-dc33d13b-b3db-1636-9c30-bce348ae89df dc33d13b-b3db-1636-9c30-bce348ae89df}" +2025-02-25T23:59:15.970Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:15.8625032 +0000 UTC m=+103.471518607 (1 nodes), 55.840716ms from start of poll, 55.989556ms from cluster start. Timeout at 2025-02-25 23:59:15.967512281 +0000 UTC m=+103.576527688, 105.009081ms after stability +2025-02-25T23:59:15.970Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:59:16.078Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:59:15.970468287 +0000 UTC m=+103.579483694 (1 nodes), 800ns from start of poll, 163.954643ms from cluster start. Timeout at 2025-02-25 23:59:16.075513888 +0000 UTC m=+103.684529295, 105.045601ms after stability +2025-02-25T23:59:16.086Z [ERROR] server-a4914f51-5f3e-4764-2b7a-366c33d30915: peer has newer term, stopping replication: peer="{Voter server-dc33d13b-b3db-1636-9c30-bce348ae89df dc33d13b-b3db-1636-9c30-bce348ae89df}" +2025-02-25T23:59:16.086Z [INFO] server-a4914f51-5f3e-4764-2b7a-366c33d30915: aborting pipeline replication: peer="{Voter server-dc33d13b-b3db-1636-9c30-bce348ae89df dc33d13b-b3db-1636-9c30-bce348ae89df}" +2025-02-25T23:59:16.086Z [INFO] server-a4914f51-5f3e-4764-2b7a-366c33d30915: entering follower state: follower="Node at a4914f51-5f3e-4764-2b7a-366c33d30915 [Follower]" leader-address= leader-id= +--- PASS: TestRaft_VerifyLeader_Fail (0.28s) === RUN TestRaft_VerifyLeader_PartialConnect -2026-03-31T06:17:52.191Z [INFO] server-5c84d6b3-b164-8151-16b5-adb6f5d2f633: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5c84d6b3-b164-8151-16b5-adb6f5d2f633 Address:5c84d6b3-b164-8151-16b5-adb6f5d2f633} {Suffrage:Voter ID:server-1d4f6e84-e27e-f44d-e253-fcfed5c8395c Address:1d4f6e84-e27e-f44d-e253-fcfed5c8395c} {Suffrage:Voter ID:server-f1811d58-2110-798e-957e-feefe2a74fca Address:f1811d58-2110-798e-957e-feefe2a74fca}]" -2026-03-31T06:17:52.191Z [INFO] server-5c84d6b3-b164-8151-16b5-adb6f5d2f633: entering follower state: follower="Node at 5c84d6b3-b164-8151-16b5-adb6f5d2f633 [Follower]" leader-address= leader-id= -2026-03-31T06:17:52.192Z [INFO] server-1d4f6e84-e27e-f44d-e253-fcfed5c8395c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5c84d6b3-b164-8151-16b5-adb6f5d2f633 Address:5c84d6b3-b164-8151-16b5-adb6f5d2f633} {Suffrage:Voter ID:server-1d4f6e84-e27e-f44d-e253-fcfed5c8395c Address:1d4f6e84-e27e-f44d-e253-fcfed5c8395c} {Suffrage:Voter ID:server-f1811d58-2110-798e-957e-feefe2a74fca Address:f1811d58-2110-798e-957e-feefe2a74fca}]" -2026-03-31T06:17:52.192Z [INFO] server-1d4f6e84-e27e-f44d-e253-fcfed5c8395c: entering follower state: follower="Node at 1d4f6e84-e27e-f44d-e253-fcfed5c8395c [Follower]" leader-address= leader-id= -2026-03-31T06:17:52.192Z [INFO] server-f1811d58-2110-798e-957e-feefe2a74fca: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5c84d6b3-b164-8151-16b5-adb6f5d2f633 Address:5c84d6b3-b164-8151-16b5-adb6f5d2f633} {Suffrage:Voter ID:server-1d4f6e84-e27e-f44d-e253-fcfed5c8395c Address:1d4f6e84-e27e-f44d-e253-fcfed5c8395c} {Suffrage:Voter ID:server-f1811d58-2110-798e-957e-feefe2a74fca Address:f1811d58-2110-798e-957e-feefe2a74fca}]" -2026-03-31T06:17:52.192Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:52.192Z [INFO] server-f1811d58-2110-798e-957e-feefe2a74fca: entering follower state: follower="Node at f1811d58-2110-798e-957e-feefe2a74fca [Follower]" leader-address= leader-id= -2026-03-31T06:17:52.255Z [WARN] server-f1811d58-2110-798e-957e-feefe2a74fca: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:52.255Z [INFO] server-f1811d58-2110-798e-957e-feefe2a74fca: entering candidate state: node="Node at f1811d58-2110-798e-957e-feefe2a74fca [Candidate]" term=2 -2026-03-31T06:17:52.255Z [INFO] server-f1811d58-2110-798e-957e-feefe2a74fca: election won: term=2 tally=2 -2026-03-31T06:17:52.255Z [INFO] server-f1811d58-2110-798e-957e-feefe2a74fca: entering leader state: leader="Node at f1811d58-2110-798e-957e-feefe2a74fca [Leader]" -2026-03-31T06:17:52.255Z [INFO] server-f1811d58-2110-798e-957e-feefe2a74fca: added peer, starting replication: peer=server-5c84d6b3-b164-8151-16b5-adb6f5d2f633 -2026-03-31T06:17:52.255Z [INFO] server-f1811d58-2110-798e-957e-feefe2a74fca: added peer, starting replication: peer=server-1d4f6e84-e27e-f44d-e253-fcfed5c8395c -2026-03-31T06:17:52.255Z [INFO] server-f1811d58-2110-798e-957e-feefe2a74fca: pipelining replication: peer="{Voter server-5c84d6b3-b164-8151-16b5-adb6f5d2f633 5c84d6b3-b164-8151-16b5-adb6f5d2f633}" -2026-03-31T06:17:52.255Z [INFO] server-f1811d58-2110-798e-957e-feefe2a74fca: pipelining replication: peer="{Voter server-1d4f6e84-e27e-f44d-e253-fcfed5c8395c 1d4f6e84-e27e-f44d-e253-fcfed5c8395c}" -2026-03-31T06:17:52.360Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:52.25558254 +0000 UTC m=+100.533189961 (1 nodes), 63.2532ms from start of poll, 63.8298ms from cluster start. Timeout at 2026-03-31 06:17:52.3605857 +0000 UTC m=+100.638193081, 105.00312ms after stability -2026-03-31T06:17:52.365Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:17:52.471Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:17:52.36599006 +0000 UTC m=+100.643597441 (2 nodes), 2.48µs from start of poll, 174.23728ms from cluster start. Timeout at 2026-03-31 06:17:52.47101458 +0000 UTC m=+100.748621961, 105.02452ms after stability - raft_test.go:1968: [INFO] Disconnecting Node at 5c84d6b3-b164-8151-16b5-adb6f5d2f633 [Follower] -2026-03-31T06:17:52.471Z [ERROR] server-f1811d58-2110-798e-957e-feefe2a74fca: failed to heartbeat to: peer=5c84d6b3-b164-8151-16b5-adb6f5d2f633 backoff time=10ms error="failed to connect to peer: 5c84d6b3-b164-8151-16b5-adb6f5d2f633" -2026-03-31T06:17:52.471Z [INFO] server-f1811d58-2110-798e-957e-feefe2a74fca: aborting pipeline replication: peer="{Voter server-1d4f6e84-e27e-f44d-e253-fcfed5c8395c 1d4f6e84-e27e-f44d-e253-fcfed5c8395c}" -2026-03-31T06:17:52.471Z [INFO] server-f1811d58-2110-798e-957e-feefe2a74fca: aborting pipeline replication: peer="{Voter server-5c84d6b3-b164-8151-16b5-adb6f5d2f633 5c84d6b3-b164-8151-16b5-adb6f5d2f633}" ---- PASS: TestRaft_VerifyLeader_PartialConnect (0.28s) +2025-02-25T23:59:16.086Z [INFO] server-1902d4f0-b28a-1a33-7a59-c9521f544447: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1902d4f0-b28a-1a33-7a59-c9521f544447 Address:1902d4f0-b28a-1a33-7a59-c9521f544447} {Suffrage:Voter ID:server-dfcb8f14-07a2-2592-e1b8-a49ec78a09bf Address:dfcb8f14-07a2-2592-e1b8-a49ec78a09bf} {Suffrage:Voter ID:server-6a6074d5-7b15-fe61-a451-dde57fff1c17 Address:6a6074d5-7b15-fe61-a451-dde57fff1c17}]" +2025-02-25T23:59:16.086Z [INFO] server-dfcb8f14-07a2-2592-e1b8-a49ec78a09bf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1902d4f0-b28a-1a33-7a59-c9521f544447 Address:1902d4f0-b28a-1a33-7a59-c9521f544447} {Suffrage:Voter ID:server-dfcb8f14-07a2-2592-e1b8-a49ec78a09bf Address:dfcb8f14-07a2-2592-e1b8-a49ec78a09bf} {Suffrage:Voter ID:server-6a6074d5-7b15-fe61-a451-dde57fff1c17 Address:6a6074d5-7b15-fe61-a451-dde57fff1c17}]" +2025-02-25T23:59:16.086Z [INFO] server-6a6074d5-7b15-fe61-a451-dde57fff1c17: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1902d4f0-b28a-1a33-7a59-c9521f544447 Address:1902d4f0-b28a-1a33-7a59-c9521f544447} {Suffrage:Voter ID:server-dfcb8f14-07a2-2592-e1b8-a49ec78a09bf Address:dfcb8f14-07a2-2592-e1b8-a49ec78a09bf} {Suffrage:Voter ID:server-6a6074d5-7b15-fe61-a451-dde57fff1c17 Address:6a6074d5-7b15-fe61-a451-dde57fff1c17}]" +2025-02-25T23:59:16.086Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:16.086Z [INFO] server-1902d4f0-b28a-1a33-7a59-c9521f544447: entering follower state: follower="Node at 1902d4f0-b28a-1a33-7a59-c9521f544447 [Follower]" leader-address= leader-id= +2025-02-25T23:59:16.086Z [INFO] server-dfcb8f14-07a2-2592-e1b8-a49ec78a09bf: entering follower state: follower="Node at dfcb8f14-07a2-2592-e1b8-a49ec78a09bf [Follower]" leader-address= leader-id= +2025-02-25T23:59:16.086Z [INFO] server-6a6074d5-7b15-fe61-a451-dde57fff1c17: entering follower state: follower="Node at 6a6074d5-7b15-fe61-a451-dde57fff1c17 [Follower]" leader-address= leader-id= +2025-02-25T23:59:16.158Z [WARN] server-1902d4f0-b28a-1a33-7a59-c9521f544447: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:16.158Z [INFO] server-1902d4f0-b28a-1a33-7a59-c9521f544447: entering candidate state: node="Node at 1902d4f0-b28a-1a33-7a59-c9521f544447 [Candidate]" term=2 +2025-02-25T23:59:16.158Z [INFO] server-1902d4f0-b28a-1a33-7a59-c9521f544447: election won: term=2 tally=2 +2025-02-25T23:59:16.158Z [INFO] server-1902d4f0-b28a-1a33-7a59-c9521f544447: entering leader state: leader="Node at 1902d4f0-b28a-1a33-7a59-c9521f544447 [Leader]" +2025-02-25T23:59:16.158Z [INFO] server-1902d4f0-b28a-1a33-7a59-c9521f544447: added peer, starting replication: peer=server-dfcb8f14-07a2-2592-e1b8-a49ec78a09bf +2025-02-25T23:59:16.158Z [INFO] server-1902d4f0-b28a-1a33-7a59-c9521f544447: added peer, starting replication: peer=server-6a6074d5-7b15-fe61-a451-dde57fff1c17 +2025-02-25T23:59:16.158Z [INFO] server-1902d4f0-b28a-1a33-7a59-c9521f544447: pipelining replication: peer="{Voter server-6a6074d5-7b15-fe61-a451-dde57fff1c17 6a6074d5-7b15-fe61-a451-dde57fff1c17}" +2025-02-25T23:59:16.158Z [INFO] server-1902d4f0-b28a-1a33-7a59-c9521f544447: pipelining replication: peer="{Voter server-dfcb8f14-07a2-2592-e1b8-a49ec78a09bf dfcb8f14-07a2-2592-e1b8-a49ec78a09bf}" +2025-02-25T23:59:16.265Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:16.158738166 +0000 UTC m=+103.767753573 (1 nodes), 71.928858ms from start of poll, 72.105899ms from cluster start. Timeout at 2025-02-25 23:59:16.263744726 +0000 UTC m=+103.872760093, 105.00652ms after stability +2025-02-25T23:59:16.273Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:59:16.382Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:59:16.273961219 +0000 UTC m=+103.882976626 (2 nodes), 3.36µs from start of poll, 187.328952ms from cluster start. Timeout at 2025-02-25 23:59:16.37900082 +0000 UTC m=+103.988016307, 105.039681ms after stability + raft_test.go:1968: [INFO] Disconnecting Node at dfcb8f14-07a2-2592-e1b8-a49ec78a09bf [Follower] +2025-02-25T23:59:16.382Z [ERROR] server-1902d4f0-b28a-1a33-7a59-c9521f544447: failed to heartbeat to: peer=dfcb8f14-07a2-2592-e1b8-a49ec78a09bf backoff time=10ms error="failed to connect to peer: dfcb8f14-07a2-2592-e1b8-a49ec78a09bf" +2025-02-25T23:59:16.382Z [INFO] server-1902d4f0-b28a-1a33-7a59-c9521f544447: aborting pipeline replication: peer="{Voter server-dfcb8f14-07a2-2592-e1b8-a49ec78a09bf dfcb8f14-07a2-2592-e1b8-a49ec78a09bf}" +2025-02-25T23:59:16.382Z [INFO] server-1902d4f0-b28a-1a33-7a59-c9521f544447: aborting pipeline replication: peer="{Voter server-6a6074d5-7b15-fe61-a451-dde57fff1c17 6a6074d5-7b15-fe61-a451-dde57fff1c17}" +--- PASS: TestRaft_VerifyLeader_PartialConnect (0.30s) === RUN TestRaft_NotifyCh -2026-03-31T06:17:52.472Z [INFO] server-232020e1-56ca-e0b2-2225-ca6a73b39f37: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-232020e1-56ca-e0b2-2225-ca6a73b39f37 Address:232020e1-56ca-e0b2-2225-ca6a73b39f37}]" -2026-03-31T06:17:52.472Z [INFO] server-232020e1-56ca-e0b2-2225-ca6a73b39f37: entering follower state: follower="Node at 232020e1-56ca-e0b2-2225-ca6a73b39f37 [Follower]" leader-address= leader-id= -2026-03-31T06:17:52.542Z [WARN] server-232020e1-56ca-e0b2-2225-ca6a73b39f37: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:52.542Z [INFO] server-232020e1-56ca-e0b2-2225-ca6a73b39f37: entering candidate state: node="Node at 232020e1-56ca-e0b2-2225-ca6a73b39f37 [Candidate]" term=2 -2026-03-31T06:17:52.542Z [INFO] server-232020e1-56ca-e0b2-2225-ca6a73b39f37: election won: term=2 tally=1 -2026-03-31T06:17:52.542Z [INFO] server-232020e1-56ca-e0b2-2225-ca6a73b39f37: entering leader state: leader="Node at 232020e1-56ca-e0b2-2225-ca6a73b39f37 [Leader]" ---- PASS: TestRaft_NotifyCh (0.07s) +2025-02-25T23:59:16.382Z [INFO] server-00fc91c1-3e41-3e42-85b6-6c70f59ddddd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-00fc91c1-3e41-3e42-85b6-6c70f59ddddd Address:00fc91c1-3e41-3e42-85b6-6c70f59ddddd}]" +2025-02-25T23:59:16.382Z [INFO] server-00fc91c1-3e41-3e42-85b6-6c70f59ddddd: entering follower state: follower="Node at 00fc91c1-3e41-3e42-85b6-6c70f59ddddd [Follower]" leader-address= leader-id= +2025-02-25T23:59:16.438Z [WARN] server-00fc91c1-3e41-3e42-85b6-6c70f59ddddd: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:16.438Z [INFO] server-00fc91c1-3e41-3e42-85b6-6c70f59ddddd: entering candidate state: node="Node at 00fc91c1-3e41-3e42-85b6-6c70f59ddddd [Candidate]" term=2 +2025-02-25T23:59:16.438Z [INFO] server-00fc91c1-3e41-3e42-85b6-6c70f59ddddd: election won: term=2 tally=1 +2025-02-25T23:59:16.438Z [INFO] server-00fc91c1-3e41-3e42-85b6-6c70f59ddddd: entering leader state: leader="Node at 00fc91c1-3e41-3e42-85b6-6c70f59ddddd [Leader]" +--- PASS: TestRaft_NotifyCh (0.06s) === RUN TestRaft_AppendEntry -2026-03-31T06:17:52.543Z [INFO] server-13117c58-ed7b-7f03-2abb-a6368f6c285e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-13117c58-ed7b-7f03-2abb-a6368f6c285e Address:13117c58-ed7b-7f03-2abb-a6368f6c285e} {Suffrage:Voter ID:server-38e0d323-0955-2963-6f5c-a788e75a9575 Address:38e0d323-0955-2963-6f5c-a788e75a9575} {Suffrage:Voter ID:server-1db37322-0578-6728-0ae6-98133a9eec9c Address:1db37322-0578-6728-0ae6-98133a9eec9c}]" -2026-03-31T06:17:52.543Z [INFO] server-38e0d323-0955-2963-6f5c-a788e75a9575: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-13117c58-ed7b-7f03-2abb-a6368f6c285e Address:13117c58-ed7b-7f03-2abb-a6368f6c285e} {Suffrage:Voter ID:server-38e0d323-0955-2963-6f5c-a788e75a9575 Address:38e0d323-0955-2963-6f5c-a788e75a9575} {Suffrage:Voter ID:server-1db37322-0578-6728-0ae6-98133a9eec9c Address:1db37322-0578-6728-0ae6-98133a9eec9c}]" -2026-03-31T06:17:52.543Z [INFO] server-13117c58-ed7b-7f03-2abb-a6368f6c285e: entering follower state: follower="Node at 13117c58-ed7b-7f03-2abb-a6368f6c285e [Follower]" leader-address= leader-id= -2026-03-31T06:17:52.543Z [INFO] server-38e0d323-0955-2963-6f5c-a788e75a9575: entering follower state: follower="Node at 38e0d323-0955-2963-6f5c-a788e75a9575 [Follower]" leader-address= leader-id= -2026-03-31T06:17:52.543Z [INFO] server-1db37322-0578-6728-0ae6-98133a9eec9c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-13117c58-ed7b-7f03-2abb-a6368f6c285e Address:13117c58-ed7b-7f03-2abb-a6368f6c285e} {Suffrage:Voter ID:server-38e0d323-0955-2963-6f5c-a788e75a9575 Address:38e0d323-0955-2963-6f5c-a788e75a9575} {Suffrage:Voter ID:server-1db37322-0578-6728-0ae6-98133a9eec9c Address:1db37322-0578-6728-0ae6-98133a9eec9c}]" -2026-03-31T06:17:52.543Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:17:52.543Z [INFO] server-1db37322-0578-6728-0ae6-98133a9eec9c: entering follower state: follower="Node at 1db37322-0578-6728-0ae6-98133a9eec9c [Follower]" leader-address= leader-id= -2026-03-31T06:17:52.598Z [WARN] server-38e0d323-0955-2963-6f5c-a788e75a9575: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:52.598Z [INFO] server-38e0d323-0955-2963-6f5c-a788e75a9575: entering candidate state: node="Node at 38e0d323-0955-2963-6f5c-a788e75a9575 [Candidate]" term=2 -2026-03-31T06:17:52.598Z [WARN] server-13117c58-ed7b-7f03-2abb-a6368f6c285e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:52.598Z [INFO] server-38e0d323-0955-2963-6f5c-a788e75a9575: election won: term=2 tally=2 -2026-03-31T06:17:52.598Z [INFO] server-13117c58-ed7b-7f03-2abb-a6368f6c285e: entering candidate state: node="Node at 13117c58-ed7b-7f03-2abb-a6368f6c285e [Candidate]" term=2 -2026-03-31T06:17:52.598Z [INFO] server-38e0d323-0955-2963-6f5c-a788e75a9575: entering leader state: leader="Node at 38e0d323-0955-2963-6f5c-a788e75a9575 [Leader]" -2026-03-31T06:17:52.598Z [INFO] server-38e0d323-0955-2963-6f5c-a788e75a9575: added peer, starting replication: peer=server-13117c58-ed7b-7f03-2abb-a6368f6c285e -2026-03-31T06:17:52.598Z [INFO] server-13117c58-ed7b-7f03-2abb-a6368f6c285e: duplicate requestVote for same term: term=2 -2026-03-31T06:17:52.598Z [INFO] server-38e0d323-0955-2963-6f5c-a788e75a9575: added peer, starting replication: peer=server-1db37322-0578-6728-0ae6-98133a9eec9c -2026-03-31T06:17:52.598Z [INFO] server-1db37322-0578-6728-0ae6-98133a9eec9c: duplicate requestVote for same term: term=2 -2026-03-31T06:17:52.598Z [WARN] server-38e0d323-0955-2963-6f5c-a788e75a9575: rejecting vote request since we have a leader: from=13117c58-ed7b-7f03-2abb-a6368f6c285e leader=38e0d323-0955-2963-6f5c-a788e75a9575 leader-id=server-38e0d323-0955-2963-6f5c-a788e75a9575 -2026-03-31T06:17:52.598Z [INFO] server-13117c58-ed7b-7f03-2abb-a6368f6c285e: entering follower state: follower="Node at 13117c58-ed7b-7f03-2abb-a6368f6c285e [Follower]" leader-address=38e0d323-0955-2963-6f5c-a788e75a9575 leader-id=server-38e0d323-0955-2963-6f5c-a788e75a9575 -2026-03-31T06:17:52.598Z [INFO] server-38e0d323-0955-2963-6f5c-a788e75a9575: pipelining replication: peer="{Voter server-1db37322-0578-6728-0ae6-98133a9eec9c 1db37322-0578-6728-0ae6-98133a9eec9c}" -2026-03-31T06:17:52.598Z [INFO] server-38e0d323-0955-2963-6f5c-a788e75a9575: pipelining replication: peer="{Voter server-13117c58-ed7b-7f03-2abb-a6368f6c285e 13117c58-ed7b-7f03-2abb-a6368f6c285e}" -2026-03-31T06:17:52.703Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:17:52.59839506 +0000 UTC m=+100.876002441 (2 nodes), 54.6056ms from start of poll, 55.05252ms from cluster start. Timeout at 2026-03-31 06:17:52.70339638 +0000 UTC m=+100.981003761, 105.00132ms after stability -2026-03-31T06:17:52.703Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:52.809Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:52.70350334 +0000 UTC m=+100.981110721 (1 nodes), 1µs from start of poll, 160.1608ms from cluster start. Timeout at 2026-03-31 06:17:52.80852314 +0000 UTC m=+101.086130561, 105.01984ms after stability -2026-03-31T06:17:52.809Z [WARN] server-13117c58-ed7b-7f03-2abb-a6368f6c285e: clearing log suffix: from=1 to=2 -2026-03-31T06:17:52.809Z [WARN] server-13117c58-ed7b-7f03-2abb-a6368f6c285e: skipping application of old log: index=1 -2026-03-31T06:17:52.809Z [WARN] server-13117c58-ed7b-7f03-2abb-a6368f6c285e: skipping application of old log: index=1 -2026-03-31T06:17:52.809Z [INFO] server-38e0d323-0955-2963-6f5c-a788e75a9575: aborting pipeline replication: peer="{Voter server-1db37322-0578-6728-0ae6-98133a9eec9c 1db37322-0578-6728-0ae6-98133a9eec9c}" -2026-03-31T06:17:52.809Z [INFO] server-38e0d323-0955-2963-6f5c-a788e75a9575: aborting pipeline replication: peer="{Voter server-13117c58-ed7b-7f03-2abb-a6368f6c285e 13117c58-ed7b-7f03-2abb-a6368f6c285e}" ---- PASS: TestRaft_AppendEntry (0.27s) +2025-02-25T23:59:16.438Z [INFO] server-268e63a4-d25e-4a25-dd90-838d84ed9397: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-268e63a4-d25e-4a25-dd90-838d84ed9397 Address:268e63a4-d25e-4a25-dd90-838d84ed9397} {Suffrage:Voter ID:server-9f83f189-277c-7dcc-8276-9276b484e516 Address:9f83f189-277c-7dcc-8276-9276b484e516} {Suffrage:Voter ID:server-2d6d1381-92dd-3b4b-1859-b1f80002ef4d Address:2d6d1381-92dd-3b4b-1859-b1f80002ef4d}]" +2025-02-25T23:59:16.438Z [INFO] server-9f83f189-277c-7dcc-8276-9276b484e516: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-268e63a4-d25e-4a25-dd90-838d84ed9397 Address:268e63a4-d25e-4a25-dd90-838d84ed9397} {Suffrage:Voter ID:server-9f83f189-277c-7dcc-8276-9276b484e516 Address:9f83f189-277c-7dcc-8276-9276b484e516} {Suffrage:Voter ID:server-2d6d1381-92dd-3b4b-1859-b1f80002ef4d Address:2d6d1381-92dd-3b4b-1859-b1f80002ef4d}]" +2025-02-25T23:59:16.438Z [INFO] server-2d6d1381-92dd-3b4b-1859-b1f80002ef4d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-268e63a4-d25e-4a25-dd90-838d84ed9397 Address:268e63a4-d25e-4a25-dd90-838d84ed9397} {Suffrage:Voter ID:server-9f83f189-277c-7dcc-8276-9276b484e516 Address:9f83f189-277c-7dcc-8276-9276b484e516} {Suffrage:Voter ID:server-2d6d1381-92dd-3b4b-1859-b1f80002ef4d Address:2d6d1381-92dd-3b4b-1859-b1f80002ef4d}]" +2025-02-25T23:59:16.438Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:59:16.438Z [INFO] server-268e63a4-d25e-4a25-dd90-838d84ed9397: entering follower state: follower="Node at 268e63a4-d25e-4a25-dd90-838d84ed9397 [Follower]" leader-address= leader-id= +2025-02-25T23:59:16.438Z [INFO] server-9f83f189-277c-7dcc-8276-9276b484e516: entering follower state: follower="Node at 9f83f189-277c-7dcc-8276-9276b484e516 [Follower]" leader-address= leader-id= +2025-02-25T23:59:16.438Z [INFO] server-2d6d1381-92dd-3b4b-1859-b1f80002ef4d: entering follower state: follower="Node at 2d6d1381-92dd-3b4b-1859-b1f80002ef4d [Follower]" leader-address= leader-id= +2025-02-25T23:59:16.502Z [WARN] server-9f83f189-277c-7dcc-8276-9276b484e516: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:16.502Z [INFO] server-9f83f189-277c-7dcc-8276-9276b484e516: entering candidate state: node="Node at 9f83f189-277c-7dcc-8276-9276b484e516 [Candidate]" term=2 +2025-02-25T23:59:16.502Z [INFO] server-9f83f189-277c-7dcc-8276-9276b484e516: election won: term=2 tally=2 +2025-02-25T23:59:16.502Z [INFO] server-9f83f189-277c-7dcc-8276-9276b484e516: entering leader state: leader="Node at 9f83f189-277c-7dcc-8276-9276b484e516 [Leader]" +2025-02-25T23:59:16.502Z [INFO] server-9f83f189-277c-7dcc-8276-9276b484e516: added peer, starting replication: peer=server-268e63a4-d25e-4a25-dd90-838d84ed9397 +2025-02-25T23:59:16.502Z [INFO] server-9f83f189-277c-7dcc-8276-9276b484e516: added peer, starting replication: peer=server-2d6d1381-92dd-3b4b-1859-b1f80002ef4d +2025-02-25T23:59:16.502Z [INFO] server-9f83f189-277c-7dcc-8276-9276b484e516: pipelining replication: peer="{Voter server-2d6d1381-92dd-3b4b-1859-b1f80002ef4d 2d6d1381-92dd-3b4b-1859-b1f80002ef4d}" +2025-02-25T23:59:16.502Z [INFO] server-9f83f189-277c-7dcc-8276-9276b484e516: pipelining replication: peer="{Voter server-268e63a4-d25e-4a25-dd90-838d84ed9397 268e63a4-d25e-4a25-dd90-838d84ed9397}" +2025-02-25T23:59:16.610Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:59:16.502533235 +0000 UTC m=+104.111548642 (2 nodes), 63.732166ms from start of poll, 63.905847ms from cluster start. Timeout at 2025-02-25 23:59:16.607536955 +0000 UTC m=+104.216552362, 105.00372ms after stability +2025-02-25T23:59:16.610Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:16.718Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:16.610288361 +0000 UTC m=+104.219303728 (1 nodes), 600ns from start of poll, 171.660933ms from cluster start. Timeout at 2025-02-25 23:59:16.715345402 +0000 UTC m=+104.324360809, 105.057081ms after stability +2025-02-25T23:59:16.718Z [WARN] server-268e63a4-d25e-4a25-dd90-838d84ed9397: clearing log suffix: from=1 to=2 +2025-02-25T23:59:16.718Z [WARN] server-268e63a4-d25e-4a25-dd90-838d84ed9397: skipping application of old log: index=1 +2025-02-25T23:59:16.718Z [WARN] server-268e63a4-d25e-4a25-dd90-838d84ed9397: skipping application of old log: index=1 +2025-02-25T23:59:16.718Z [INFO] server-9f83f189-277c-7dcc-8276-9276b484e516: aborting pipeline replication: peer="{Voter server-268e63a4-d25e-4a25-dd90-838d84ed9397 268e63a4-d25e-4a25-dd90-838d84ed9397}" +2025-02-25T23:59:16.718Z [INFO] server-9f83f189-277c-7dcc-8276-9276b484e516: aborting pipeline replication: peer="{Voter server-2d6d1381-92dd-3b4b-1859-b1f80002ef4d 2d6d1381-92dd-3b4b-1859-b1f80002ef4d}" +--- PASS: TestRaft_AppendEntry (0.28s) === RUN TestRaft_VotingGrant_WhenLeaderAvailable -2026-03-31T06:17:52.810Z [INFO] server-8a580dde-e09a-6ab6-bebd-0cf78571a2ca: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8a580dde-e09a-6ab6-bebd-0cf78571a2ca Address:8a580dde-e09a-6ab6-bebd-0cf78571a2ca} {Suffrage:Voter ID:server-143813ea-fa67-2096-4511-2c63c049437d Address:143813ea-fa67-2096-4511-2c63c049437d} {Suffrage:Voter ID:server-f48eacd1-ba0f-f5ae-d84d-85fb723fc820 Address:f48eacd1-ba0f-f5ae-d84d-85fb723fc820}]" -2026-03-31T06:17:52.810Z [INFO] server-143813ea-fa67-2096-4511-2c63c049437d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8a580dde-e09a-6ab6-bebd-0cf78571a2ca Address:8a580dde-e09a-6ab6-bebd-0cf78571a2ca} {Suffrage:Voter ID:server-143813ea-fa67-2096-4511-2c63c049437d Address:143813ea-fa67-2096-4511-2c63c049437d} {Suffrage:Voter ID:server-f48eacd1-ba0f-f5ae-d84d-85fb723fc820 Address:f48eacd1-ba0f-f5ae-d84d-85fb723fc820}]" -2026-03-31T06:17:52.810Z [INFO] server-8a580dde-e09a-6ab6-bebd-0cf78571a2ca: entering follower state: follower="Node at 8a580dde-e09a-6ab6-bebd-0cf78571a2ca [Follower]" leader-address= leader-id= -2026-03-31T06:17:52.810Z [INFO] server-143813ea-fa67-2096-4511-2c63c049437d: entering follower state: follower="Node at 143813ea-fa67-2096-4511-2c63c049437d [Follower]" leader-address= leader-id= -2026-03-31T06:17:52.810Z [INFO] server-f48eacd1-ba0f-f5ae-d84d-85fb723fc820: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8a580dde-e09a-6ab6-bebd-0cf78571a2ca Address:8a580dde-e09a-6ab6-bebd-0cf78571a2ca} {Suffrage:Voter ID:server-143813ea-fa67-2096-4511-2c63c049437d Address:143813ea-fa67-2096-4511-2c63c049437d} {Suffrage:Voter ID:server-f48eacd1-ba0f-f5ae-d84d-85fb723fc820 Address:f48eacd1-ba0f-f5ae-d84d-85fb723fc820}]" -2026-03-31T06:17:52.810Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:17:52.810Z [INFO] server-f48eacd1-ba0f-f5ae-d84d-85fb723fc820: entering follower state: follower="Node at f48eacd1-ba0f-f5ae-d84d-85fb723fc820 [Follower]" leader-address= leader-id= -2026-03-31T06:17:52.869Z [WARN] server-143813ea-fa67-2096-4511-2c63c049437d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:52.869Z [INFO] server-143813ea-fa67-2096-4511-2c63c049437d: entering candidate state: node="Node at 143813ea-fa67-2096-4511-2c63c049437d [Candidate]" term=2 -2026-03-31T06:17:52.869Z [INFO] server-143813ea-fa67-2096-4511-2c63c049437d: election won: term=2 tally=2 -2026-03-31T06:17:52.869Z [INFO] server-143813ea-fa67-2096-4511-2c63c049437d: entering leader state: leader="Node at 143813ea-fa67-2096-4511-2c63c049437d [Leader]" -2026-03-31T06:17:52.869Z [INFO] server-143813ea-fa67-2096-4511-2c63c049437d: added peer, starting replication: peer=server-8a580dde-e09a-6ab6-bebd-0cf78571a2ca -2026-03-31T06:17:52.869Z [INFO] server-143813ea-fa67-2096-4511-2c63c049437d: added peer, starting replication: peer=server-f48eacd1-ba0f-f5ae-d84d-85fb723fc820 -2026-03-31T06:17:52.869Z [INFO] server-143813ea-fa67-2096-4511-2c63c049437d: pipelining replication: peer="{Voter server-f48eacd1-ba0f-f5ae-d84d-85fb723fc820 f48eacd1-ba0f-f5ae-d84d-85fb723fc820}" -2026-03-31T06:17:52.869Z [INFO] server-143813ea-fa67-2096-4511-2c63c049437d: pipelining replication: peer="{Voter server-8a580dde-e09a-6ab6-bebd-0cf78571a2ca 8a580dde-e09a-6ab6-bebd-0cf78571a2ca}" -2026-03-31T06:17:52.974Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:17:52.8693937 +0000 UTC m=+101.147001121 (2 nodes), 58.8408ms from start of poll, 59.24272ms from cluster start. Timeout at 2026-03-31 06:17:52.9743969 +0000 UTC m=+101.252004321, 105.0032ms after stability -2026-03-31T06:17:52.974Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:53.080Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:52.97452866 +0000 UTC m=+101.252136081 (1 nodes), 2µs from start of poll, 164.37768ms from cluster start. Timeout at 2026-03-31 06:17:53.07953118 +0000 UTC m=+101.357138601, 105.00252ms after stability -2026-03-31T06:17:53.080Z [WARN] server-f48eacd1-ba0f-f5ae-d84d-85fb723fc820: rejecting vote request since we have a leader: from=8a580dde-e09a-6ab6-bebd-0cf78571a2ca leader=143813ea-fa67-2096-4511-2c63c049437d leader-id=server-143813ea-fa67-2096-4511-2c63c049437d -2026-03-31T06:17:53.080Z [INFO] server-143813ea-fa67-2096-4511-2c63c049437d: aborting pipeline replication: peer="{Voter server-f48eacd1-ba0f-f5ae-d84d-85fb723fc820 f48eacd1-ba0f-f5ae-d84d-85fb723fc820}" -2026-03-31T06:17:53.080Z [INFO] server-143813ea-fa67-2096-4511-2c63c049437d: aborting pipeline replication: peer="{Voter server-8a580dde-e09a-6ab6-bebd-0cf78571a2ca 8a580dde-e09a-6ab6-bebd-0cf78571a2ca}" +2025-02-25T23:59:16.718Z [INFO] server-e7ce65af-ea80-dd65-d5a2-7ed10ec742c5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e7ce65af-ea80-dd65-d5a2-7ed10ec742c5 Address:e7ce65af-ea80-dd65-d5a2-7ed10ec742c5} {Suffrage:Voter ID:server-b7da79d8-8680-37cd-2f65-e8686636a9de Address:b7da79d8-8680-37cd-2f65-e8686636a9de} {Suffrage:Voter ID:server-26b60e68-ce90-0a11-d5c2-a83cb9cbbd92 Address:26b60e68-ce90-0a11-d5c2-a83cb9cbbd92}]" +2025-02-25T23:59:16.718Z [INFO] server-b7da79d8-8680-37cd-2f65-e8686636a9de: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e7ce65af-ea80-dd65-d5a2-7ed10ec742c5 Address:e7ce65af-ea80-dd65-d5a2-7ed10ec742c5} {Suffrage:Voter ID:server-b7da79d8-8680-37cd-2f65-e8686636a9de Address:b7da79d8-8680-37cd-2f65-e8686636a9de} {Suffrage:Voter ID:server-26b60e68-ce90-0a11-d5c2-a83cb9cbbd92 Address:26b60e68-ce90-0a11-d5c2-a83cb9cbbd92}]" +2025-02-25T23:59:16.718Z [INFO] server-26b60e68-ce90-0a11-d5c2-a83cb9cbbd92: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e7ce65af-ea80-dd65-d5a2-7ed10ec742c5 Address:e7ce65af-ea80-dd65-d5a2-7ed10ec742c5} {Suffrage:Voter ID:server-b7da79d8-8680-37cd-2f65-e8686636a9de Address:b7da79d8-8680-37cd-2f65-e8686636a9de} {Suffrage:Voter ID:server-26b60e68-ce90-0a11-d5c2-a83cb9cbbd92 Address:26b60e68-ce90-0a11-d5c2-a83cb9cbbd92}]" +2025-02-25T23:59:16.718Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:59:16.718Z [INFO] server-e7ce65af-ea80-dd65-d5a2-7ed10ec742c5: entering follower state: follower="Node at e7ce65af-ea80-dd65-d5a2-7ed10ec742c5 [Follower]" leader-address= leader-id= +2025-02-25T23:59:16.718Z [INFO] server-b7da79d8-8680-37cd-2f65-e8686636a9de: entering follower state: follower="Node at b7da79d8-8680-37cd-2f65-e8686636a9de [Follower]" leader-address= leader-id= +2025-02-25T23:59:16.718Z [INFO] server-26b60e68-ce90-0a11-d5c2-a83cb9cbbd92: entering follower state: follower="Node at 26b60e68-ce90-0a11-d5c2-a83cb9cbbd92 [Follower]" leader-address= leader-id= +2025-02-25T23:59:16.774Z [WARN] server-26b60e68-ce90-0a11-d5c2-a83cb9cbbd92: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:16.774Z [INFO] server-26b60e68-ce90-0a11-d5c2-a83cb9cbbd92: entering candidate state: node="Node at 26b60e68-ce90-0a11-d5c2-a83cb9cbbd92 [Candidate]" term=2 +2025-02-25T23:59:16.774Z [INFO] server-26b60e68-ce90-0a11-d5c2-a83cb9cbbd92: election won: term=2 tally=2 +2025-02-25T23:59:16.774Z [INFO] server-26b60e68-ce90-0a11-d5c2-a83cb9cbbd92: entering leader state: leader="Node at 26b60e68-ce90-0a11-d5c2-a83cb9cbbd92 [Leader]" +2025-02-25T23:59:16.774Z [INFO] server-26b60e68-ce90-0a11-d5c2-a83cb9cbbd92: added peer, starting replication: peer=server-e7ce65af-ea80-dd65-d5a2-7ed10ec742c5 +2025-02-25T23:59:16.774Z [INFO] server-26b60e68-ce90-0a11-d5c2-a83cb9cbbd92: added peer, starting replication: peer=server-b7da79d8-8680-37cd-2f65-e8686636a9de +2025-02-25T23:59:16.774Z [INFO] server-26b60e68-ce90-0a11-d5c2-a83cb9cbbd92: pipelining replication: peer="{Voter server-b7da79d8-8680-37cd-2f65-e8686636a9de b7da79d8-8680-37cd-2f65-e8686636a9de}" +2025-02-25T23:59:16.774Z [INFO] server-26b60e68-ce90-0a11-d5c2-a83cb9cbbd92: pipelining replication: peer="{Voter server-e7ce65af-ea80-dd65-d5a2-7ed10ec742c5 e7ce65af-ea80-dd65-d5a2-7ed10ec742c5}" +2025-02-25T23:59:16.881Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:59:16.774515126 +0000 UTC m=+104.383530533 (2 nodes), 55.854516ms from start of poll, 56.020677ms from cluster start. Timeout at 2025-02-25 23:59:16.879518206 +0000 UTC m=+104.488533573, 105.00304ms after stability +2025-02-25T23:59:16.881Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:16.989Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:16.88193801 +0000 UTC m=+104.490953417 (1 nodes), 1.12µs from start of poll, 163.443561ms from cluster start. Timeout at 2025-02-25 23:59:16.98698513 +0000 UTC m=+104.596000497, 105.04708ms after stability +2025-02-25T23:59:16.990Z [WARN] server-b7da79d8-8680-37cd-2f65-e8686636a9de: rejecting vote request since we have a leader: from=e7ce65af-ea80-dd65-d5a2-7ed10ec742c5 leader=26b60e68-ce90-0a11-d5c2-a83cb9cbbd92 leader-id=server-26b60e68-ce90-0a11-d5c2-a83cb9cbbd92 +2025-02-25T23:59:16.990Z [INFO] server-26b60e68-ce90-0a11-d5c2-a83cb9cbbd92: aborting pipeline replication: peer="{Voter server-e7ce65af-ea80-dd65-d5a2-7ed10ec742c5 e7ce65af-ea80-dd65-d5a2-7ed10ec742c5}" +2025-02-25T23:59:16.990Z [INFO] server-26b60e68-ce90-0a11-d5c2-a83cb9cbbd92: aborting pipeline replication: peer="{Voter server-b7da79d8-8680-37cd-2f65-e8686636a9de b7da79d8-8680-37cd-2f65-e8686636a9de}" --- PASS: TestRaft_VotingGrant_WhenLeaderAvailable (0.27s) === RUN TestRaft_ProtocolVersion_RejectRPC -2026-03-31T06:17:53.081Z [INFO] server-f9be55d4-f3b1-825e-a181-fa5eb5af30e3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f9be55d4-f3b1-825e-a181-fa5eb5af30e3 Address:f9be55d4-f3b1-825e-a181-fa5eb5af30e3} {Suffrage:Voter ID:server-54149d2a-7229-85d0-1708-24bcf87330d6 Address:54149d2a-7229-85d0-1708-24bcf87330d6} {Suffrage:Voter ID:server-83f25b2e-0c19-1ec4-0369-ffc49f72e10a Address:83f25b2e-0c19-1ec4-0369-ffc49f72e10a}]" -2026-03-31T06:17:53.081Z [INFO] server-f9be55d4-f3b1-825e-a181-fa5eb5af30e3: entering follower state: follower="Node at f9be55d4-f3b1-825e-a181-fa5eb5af30e3 [Follower]" leader-address= leader-id= -2026-03-31T06:17:53.081Z [INFO] server-54149d2a-7229-85d0-1708-24bcf87330d6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f9be55d4-f3b1-825e-a181-fa5eb5af30e3 Address:f9be55d4-f3b1-825e-a181-fa5eb5af30e3} {Suffrage:Voter ID:server-54149d2a-7229-85d0-1708-24bcf87330d6 Address:54149d2a-7229-85d0-1708-24bcf87330d6} {Suffrage:Voter ID:server-83f25b2e-0c19-1ec4-0369-ffc49f72e10a Address:83f25b2e-0c19-1ec4-0369-ffc49f72e10a}]" -2026-03-31T06:17:53.081Z [INFO] server-54149d2a-7229-85d0-1708-24bcf87330d6: entering follower state: follower="Node at 54149d2a-7229-85d0-1708-24bcf87330d6 [Follower]" leader-address= leader-id= -2026-03-31T06:17:53.081Z [INFO] server-83f25b2e-0c19-1ec4-0369-ffc49f72e10a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f9be55d4-f3b1-825e-a181-fa5eb5af30e3 Address:f9be55d4-f3b1-825e-a181-fa5eb5af30e3} {Suffrage:Voter ID:server-54149d2a-7229-85d0-1708-24bcf87330d6 Address:54149d2a-7229-85d0-1708-24bcf87330d6} {Suffrage:Voter ID:server-83f25b2e-0c19-1ec4-0369-ffc49f72e10a Address:83f25b2e-0c19-1ec4-0369-ffc49f72e10a}]" -2026-03-31T06:17:53.081Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:17:53.081Z [INFO] server-83f25b2e-0c19-1ec4-0369-ffc49f72e10a: entering follower state: follower="Node at 83f25b2e-0c19-1ec4-0369-ffc49f72e10a [Follower]" leader-address= leader-id= -2026-03-31T06:17:53.139Z [WARN] server-83f25b2e-0c19-1ec4-0369-ffc49f72e10a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:53.139Z [INFO] server-83f25b2e-0c19-1ec4-0369-ffc49f72e10a: entering candidate state: node="Node at 83f25b2e-0c19-1ec4-0369-ffc49f72e10a [Candidate]" term=2 -2026-03-31T06:17:53.139Z [INFO] server-83f25b2e-0c19-1ec4-0369-ffc49f72e10a: election won: term=2 tally=2 -2026-03-31T06:17:53.139Z [INFO] server-83f25b2e-0c19-1ec4-0369-ffc49f72e10a: entering leader state: leader="Node at 83f25b2e-0c19-1ec4-0369-ffc49f72e10a [Leader]" -2026-03-31T06:17:53.139Z [INFO] server-83f25b2e-0c19-1ec4-0369-ffc49f72e10a: added peer, starting replication: peer=server-f9be55d4-f3b1-825e-a181-fa5eb5af30e3 -2026-03-31T06:17:53.139Z [INFO] server-83f25b2e-0c19-1ec4-0369-ffc49f72e10a: added peer, starting replication: peer=server-54149d2a-7229-85d0-1708-24bcf87330d6 -2026-03-31T06:17:53.139Z [INFO] server-83f25b2e-0c19-1ec4-0369-ffc49f72e10a: pipelining replication: peer="{Voter server-f9be55d4-f3b1-825e-a181-fa5eb5af30e3 f9be55d4-f3b1-825e-a181-fa5eb5af30e3}" -2026-03-31T06:17:53.139Z [INFO] server-83f25b2e-0c19-1ec4-0369-ffc49f72e10a: pipelining replication: peer="{Voter server-54149d2a-7229-85d0-1708-24bcf87330d6 54149d2a-7229-85d0-1708-24bcf87330d6}" -2026-03-31T06:17:53.244Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:17:53.13930058 +0000 UTC m=+101.416907961 (2 nodes), 57.32572ms from start of poll, 57.72376ms from cluster start. Timeout at 2026-03-31 06:17:53.24430442 +0000 UTC m=+101.521911801, 105.00384ms after stability -2026-03-31T06:17:53.244Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:53.350Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:53.24449466 +0000 UTC m=+101.522102081 (1 nodes), 2.44µs from start of poll, 162.91788ms from cluster start. Timeout at 2026-03-31 06:17:53.3495183 +0000 UTC m=+101.627125641, 105.02356ms after stability -2026-03-31T06:17:53.350Z [INFO] server-83f25b2e-0c19-1ec4-0369-ffc49f72e10a: aborting pipeline replication: peer="{Voter server-54149d2a-7229-85d0-1708-24bcf87330d6 54149d2a-7229-85d0-1708-24bcf87330d6}" -2026-03-31T06:17:53.350Z [INFO] server-83f25b2e-0c19-1ec4-0369-ffc49f72e10a: aborting pipeline replication: peer="{Voter server-f9be55d4-f3b1-825e-a181-fa5eb5af30e3 f9be55d4-f3b1-825e-a181-fa5eb5af30e3}" ---- PASS: TestRaft_ProtocolVersion_RejectRPC (0.27s) +2025-02-25T23:59:16.991Z [INFO] server-15c7c86e-89a1-0ca5-4c46-09751704746a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-15c7c86e-89a1-0ca5-4c46-09751704746a Address:15c7c86e-89a1-0ca5-4c46-09751704746a} {Suffrage:Voter ID:server-48442462-3709-e4d1-a8e5-f9afe2c66a15 Address:48442462-3709-e4d1-a8e5-f9afe2c66a15} {Suffrage:Voter ID:server-60c64c04-15b7-c0ca-7833-b3247d1f7dd1 Address:60c64c04-15b7-c0ca-7833-b3247d1f7dd1}]" +2025-02-25T23:59:16.991Z [INFO] server-48442462-3709-e4d1-a8e5-f9afe2c66a15: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-15c7c86e-89a1-0ca5-4c46-09751704746a Address:15c7c86e-89a1-0ca5-4c46-09751704746a} {Suffrage:Voter ID:server-48442462-3709-e4d1-a8e5-f9afe2c66a15 Address:48442462-3709-e4d1-a8e5-f9afe2c66a15} {Suffrage:Voter ID:server-60c64c04-15b7-c0ca-7833-b3247d1f7dd1 Address:60c64c04-15b7-c0ca-7833-b3247d1f7dd1}]" +2025-02-25T23:59:16.992Z [INFO] server-60c64c04-15b7-c0ca-7833-b3247d1f7dd1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-15c7c86e-89a1-0ca5-4c46-09751704746a Address:15c7c86e-89a1-0ca5-4c46-09751704746a} {Suffrage:Voter ID:server-48442462-3709-e4d1-a8e5-f9afe2c66a15 Address:48442462-3709-e4d1-a8e5-f9afe2c66a15} {Suffrage:Voter ID:server-60c64c04-15b7-c0ca-7833-b3247d1f7dd1 Address:60c64c04-15b7-c0ca-7833-b3247d1f7dd1}]" +2025-02-25T23:59:16.992Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:59:17.002Z [INFO] server-60c64c04-15b7-c0ca-7833-b3247d1f7dd1: entering follower state: follower="Node at 60c64c04-15b7-c0ca-7833-b3247d1f7dd1 [Follower]" leader-address= leader-id= +2025-02-25T23:59:17.002Z [INFO] server-15c7c86e-89a1-0ca5-4c46-09751704746a: entering follower state: follower="Node at 15c7c86e-89a1-0ca5-4c46-09751704746a [Follower]" leader-address= leader-id= +2025-02-25T23:59:17.002Z [INFO] server-48442462-3709-e4d1-a8e5-f9afe2c66a15: entering follower state: follower="Node at 48442462-3709-e4d1-a8e5-f9afe2c66a15 [Follower]" leader-address= leader-id= +2025-02-25T23:59:17.070Z [WARN] server-60c64c04-15b7-c0ca-7833-b3247d1f7dd1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:17.070Z [INFO] server-60c64c04-15b7-c0ca-7833-b3247d1f7dd1: entering candidate state: node="Node at 60c64c04-15b7-c0ca-7833-b3247d1f7dd1 [Candidate]" term=2 +2025-02-25T23:59:17.070Z [INFO] server-60c64c04-15b7-c0ca-7833-b3247d1f7dd1: election won: term=2 tally=2 +2025-02-25T23:59:17.070Z [INFO] server-60c64c04-15b7-c0ca-7833-b3247d1f7dd1: entering leader state: leader="Node at 60c64c04-15b7-c0ca-7833-b3247d1f7dd1 [Leader]" +2025-02-25T23:59:17.070Z [INFO] server-60c64c04-15b7-c0ca-7833-b3247d1f7dd1: added peer, starting replication: peer=server-15c7c86e-89a1-0ca5-4c46-09751704746a +2025-02-25T23:59:17.070Z [INFO] server-60c64c04-15b7-c0ca-7833-b3247d1f7dd1: added peer, starting replication: peer=server-48442462-3709-e4d1-a8e5-f9afe2c66a15 +2025-02-25T23:59:17.070Z [INFO] server-60c64c04-15b7-c0ca-7833-b3247d1f7dd1: pipelining replication: peer="{Voter server-48442462-3709-e4d1-a8e5-f9afe2c66a15 48442462-3709-e4d1-a8e5-f9afe2c66a15}" +2025-02-25T23:59:17.070Z [INFO] server-60c64c04-15b7-c0ca-7833-b3247d1f7dd1: pipelining replication: peer="{Voter server-15c7c86e-89a1-0ca5-4c46-09751704746a 15c7c86e-89a1-0ca5-4c46-09751704746a}" +2025-02-25T23:59:17.178Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:59:17.070676571 +0000 UTC m=+104.679691978 (2 nodes), 78.461339ms from start of poll, 79.680067ms from cluster start. Timeout at 2025-02-25 23:59:17.175678971 +0000 UTC m=+104.784694378, 105.0024ms after stability +2025-02-25T23:59:17.178Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:17.286Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:17.178407897 +0000 UTC m=+104.787423304 (1 nodes), 1.44µs from start of poll, 187.411393ms from cluster start. Timeout at 2025-02-25 23:59:17.283427658 +0000 UTC m=+104.892443065, 105.019761ms after stability +2025-02-25T23:59:17.286Z [INFO] server-60c64c04-15b7-c0ca-7833-b3247d1f7dd1: aborting pipeline replication: peer="{Voter server-48442462-3709-e4d1-a8e5-f9afe2c66a15 48442462-3709-e4d1-a8e5-f9afe2c66a15}" +2025-02-25T23:59:17.286Z [INFO] server-60c64c04-15b7-c0ca-7833-b3247d1f7dd1: aborting pipeline replication: peer="{Voter server-15c7c86e-89a1-0ca5-4c46-09751704746a 15c7c86e-89a1-0ca5-4c46-09751704746a}" +--- PASS: TestRaft_ProtocolVersion_RejectRPC (0.30s) === RUN TestRaft_ProtocolVersion_Upgrade_1_2 raft_test.go:2151: Skipping unreliable test --- SKIP: TestRaft_ProtocolVersion_Upgrade_1_2 (0.00s) @@ -3592,279 +3673,272 @@ raft_test.go:2196: Skipping unreliable test --- SKIP: TestRaft_ProtocolVersion_Upgrade_2_3 (0.00s) === RUN TestRaft_LeaderID_Propagated -2026-03-31T06:17:53.351Z [INFO] server-9ad06214-66a0-99eb-f673-8adbf19e1e74: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9ad06214-66a0-99eb-f673-8adbf19e1e74 Address:9ad06214-66a0-99eb-f673-8adbf19e1e74} {Suffrage:Voter ID:server-238cff00-2b22-e068-d7a4-a961a616da7f Address:238cff00-2b22-e068-d7a4-a961a616da7f} {Suffrage:Voter ID:server-312d2f75-792a-52a3-f3e7-59f6c644f871 Address:312d2f75-792a-52a3-f3e7-59f6c644f871}]" -2026-03-31T06:17:53.351Z [INFO] server-9ad06214-66a0-99eb-f673-8adbf19e1e74: entering follower state: follower="Node at 9ad06214-66a0-99eb-f673-8adbf19e1e74 [Follower]" leader-address= leader-id= -2026-03-31T06:17:53.351Z [INFO] server-238cff00-2b22-e068-d7a4-a961a616da7f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9ad06214-66a0-99eb-f673-8adbf19e1e74 Address:9ad06214-66a0-99eb-f673-8adbf19e1e74} {Suffrage:Voter ID:server-238cff00-2b22-e068-d7a4-a961a616da7f Address:238cff00-2b22-e068-d7a4-a961a616da7f} {Suffrage:Voter ID:server-312d2f75-792a-52a3-f3e7-59f6c644f871 Address:312d2f75-792a-52a3-f3e7-59f6c644f871}]" -2026-03-31T06:17:53.351Z [INFO] server-238cff00-2b22-e068-d7a4-a961a616da7f: entering follower state: follower="Node at 238cff00-2b22-e068-d7a4-a961a616da7f [Follower]" leader-address= leader-id= -2026-03-31T06:17:53.351Z [INFO] server-312d2f75-792a-52a3-f3e7-59f6c644f871: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9ad06214-66a0-99eb-f673-8adbf19e1e74 Address:9ad06214-66a0-99eb-f673-8adbf19e1e74} {Suffrage:Voter ID:server-238cff00-2b22-e068-d7a4-a961a616da7f Address:238cff00-2b22-e068-d7a4-a961a616da7f} {Suffrage:Voter ID:server-312d2f75-792a-52a3-f3e7-59f6c644f871 Address:312d2f75-792a-52a3-f3e7-59f6c644f871}]" -2026-03-31T06:17:53.351Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:53.351Z [INFO] server-312d2f75-792a-52a3-f3e7-59f6c644f871: entering follower state: follower="Node at 312d2f75-792a-52a3-f3e7-59f6c644f871 [Follower]" leader-address= leader-id= -2026-03-31T06:17:53.415Z [WARN] server-238cff00-2b22-e068-d7a4-a961a616da7f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:53.415Z [INFO] server-238cff00-2b22-e068-d7a4-a961a616da7f: entering candidate state: node="Node at 238cff00-2b22-e068-d7a4-a961a616da7f [Candidate]" term=2 -2026-03-31T06:17:53.415Z [INFO] server-238cff00-2b22-e068-d7a4-a961a616da7f: election won: term=2 tally=2 -2026-03-31T06:17:53.415Z [INFO] server-238cff00-2b22-e068-d7a4-a961a616da7f: entering leader state: leader="Node at 238cff00-2b22-e068-d7a4-a961a616da7f [Leader]" -2026-03-31T06:17:53.415Z [INFO] server-238cff00-2b22-e068-d7a4-a961a616da7f: added peer, starting replication: peer=server-9ad06214-66a0-99eb-f673-8adbf19e1e74 -2026-03-31T06:17:53.415Z [INFO] server-238cff00-2b22-e068-d7a4-a961a616da7f: added peer, starting replication: peer=server-312d2f75-792a-52a3-f3e7-59f6c644f871 -2026-03-31T06:17:53.415Z [INFO] server-238cff00-2b22-e068-d7a4-a961a616da7f: pipelining replication: peer="{Voter server-312d2f75-792a-52a3-f3e7-59f6c644f871 312d2f75-792a-52a3-f3e7-59f6c644f871}" -2026-03-31T06:17:53.415Z [INFO] server-238cff00-2b22-e068-d7a4-a961a616da7f: pipelining replication: peer="{Voter server-9ad06214-66a0-99eb-f673-8adbf19e1e74 9ad06214-66a0-99eb-f673-8adbf19e1e74}" -2026-03-31T06:17:53.521Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:53.41589786 +0000 UTC m=+101.693505281 (1 nodes), 64.2676ms from start of poll, 64.70696ms from cluster start. Timeout at 2026-03-31 06:17:53.52090242 +0000 UTC m=+101.798509801, 105.00452ms after stability -2026-03-31T06:17:53.521Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:53.626Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:53.52110262 +0000 UTC m=+101.798710001 (1 nodes), 1.6µs from start of poll, 169.91168ms from cluster start. Timeout at 2026-03-31 06:17:53.62613534 +0000 UTC m=+101.903742761, 105.03276ms after stability -2026-03-31T06:17:53.626Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:53.732Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:53.62644658 +0000 UTC m=+101.904053961 (1 nodes), 1.12µs from start of poll, 275.25564ms from cluster start. Timeout at 2026-03-31 06:17:53.73145758 +0000 UTC m=+102.009065001, 105.01104ms after stability -2026-03-31T06:17:53.732Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:53.838Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:53.73273042 +0000 UTC m=+102.010337801 (1 nodes), 1.12µs from start of poll, 381.53948ms from cluster start. Timeout at 2026-03-31 06:17:53.8378405 +0000 UTC m=+102.115447921, 105.11012ms after stability -2026-03-31T06:17:53.839Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:53.946Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:53.83991818 +0000 UTC m=+102.117525561 (1 nodes), 1.44µs from start of poll, 488.72724ms from cluster start. Timeout at 2026-03-31 06:17:53.94494674 +0000 UTC m=+102.222554121, 105.02856ms after stability -2026-03-31T06:17:53.946Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:54.051Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:53.94634642 +0000 UTC m=+102.223953801 (1 nodes), 920ns from start of poll, 595.15548ms from cluster start. Timeout at 2026-03-31 06:17:54.05137686 +0000 UTC m=+102.328984241, 105.03044ms after stability -2026-03-31T06:17:54.157Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:54.263Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:54.15784026 +0000 UTC m=+102.435447641 (1 nodes), 1.48µs from start of poll, 806.64932ms from cluster start. Timeout at 2026-03-31 06:17:54.26287306 +0000 UTC m=+102.540480441, 105.0328ms after stability -2026-03-31T06:17:54.263Z [INFO] server-238cff00-2b22-e068-d7a4-a961a616da7f: aborting pipeline replication: peer="{Voter server-312d2f75-792a-52a3-f3e7-59f6c644f871 312d2f75-792a-52a3-f3e7-59f6c644f871}" -2026-03-31T06:17:54.263Z [INFO] server-238cff00-2b22-e068-d7a4-a961a616da7f: aborting pipeline replication: peer="{Voter server-9ad06214-66a0-99eb-f673-8adbf19e1e74 9ad06214-66a0-99eb-f673-8adbf19e1e74}" ---- PASS: TestRaft_LeaderID_Propagated (0.91s) +2025-02-25T23:59:17.286Z [INFO] server-07d39c37-f8f6-7cfc-23d8-7539c3b81344: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-07d39c37-f8f6-7cfc-23d8-7539c3b81344 Address:07d39c37-f8f6-7cfc-23d8-7539c3b81344} {Suffrage:Voter ID:server-a539abbd-22f0-da35-af99-bdc702f46fb2 Address:a539abbd-22f0-da35-af99-bdc702f46fb2} {Suffrage:Voter ID:server-3275cbf9-17bc-a706-ccd6-7bb5956e4ecd Address:3275cbf9-17bc-a706-ccd6-7bb5956e4ecd}]" +2025-02-25T23:59:17.286Z [INFO] server-a539abbd-22f0-da35-af99-bdc702f46fb2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-07d39c37-f8f6-7cfc-23d8-7539c3b81344 Address:07d39c37-f8f6-7cfc-23d8-7539c3b81344} {Suffrage:Voter ID:server-a539abbd-22f0-da35-af99-bdc702f46fb2 Address:a539abbd-22f0-da35-af99-bdc702f46fb2} {Suffrage:Voter ID:server-3275cbf9-17bc-a706-ccd6-7bb5956e4ecd Address:3275cbf9-17bc-a706-ccd6-7bb5956e4ecd}]" +2025-02-25T23:59:17.286Z [INFO] server-3275cbf9-17bc-a706-ccd6-7bb5956e4ecd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-07d39c37-f8f6-7cfc-23d8-7539c3b81344 Address:07d39c37-f8f6-7cfc-23d8-7539c3b81344} {Suffrage:Voter ID:server-a539abbd-22f0-da35-af99-bdc702f46fb2 Address:a539abbd-22f0-da35-af99-bdc702f46fb2} {Suffrage:Voter ID:server-3275cbf9-17bc-a706-ccd6-7bb5956e4ecd Address:3275cbf9-17bc-a706-ccd6-7bb5956e4ecd}]" +2025-02-25T23:59:17.286Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:17.286Z [INFO] server-07d39c37-f8f6-7cfc-23d8-7539c3b81344: entering follower state: follower="Node at 07d39c37-f8f6-7cfc-23d8-7539c3b81344 [Follower]" leader-address= leader-id= +2025-02-25T23:59:17.286Z [INFO] server-a539abbd-22f0-da35-af99-bdc702f46fb2: entering follower state: follower="Node at a539abbd-22f0-da35-af99-bdc702f46fb2 [Follower]" leader-address= leader-id= +2025-02-25T23:59:17.286Z [INFO] server-3275cbf9-17bc-a706-ccd6-7bb5956e4ecd: entering follower state: follower="Node at 3275cbf9-17bc-a706-ccd6-7bb5956e4ecd [Follower]" leader-address= leader-id= +2025-02-25T23:59:17.350Z [WARN] server-3275cbf9-17bc-a706-ccd6-7bb5956e4ecd: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:17.350Z [INFO] server-3275cbf9-17bc-a706-ccd6-7bb5956e4ecd: entering candidate state: node="Node at 3275cbf9-17bc-a706-ccd6-7bb5956e4ecd [Candidate]" term=2 +2025-02-25T23:59:17.350Z [INFO] server-3275cbf9-17bc-a706-ccd6-7bb5956e4ecd: election won: term=2 tally=2 +2025-02-25T23:59:17.350Z [INFO] server-3275cbf9-17bc-a706-ccd6-7bb5956e4ecd: entering leader state: leader="Node at 3275cbf9-17bc-a706-ccd6-7bb5956e4ecd [Leader]" +2025-02-25T23:59:17.350Z [INFO] server-3275cbf9-17bc-a706-ccd6-7bb5956e4ecd: added peer, starting replication: peer=server-07d39c37-f8f6-7cfc-23d8-7539c3b81344 +2025-02-25T23:59:17.350Z [INFO] server-3275cbf9-17bc-a706-ccd6-7bb5956e4ecd: added peer, starting replication: peer=server-a539abbd-22f0-da35-af99-bdc702f46fb2 +2025-02-25T23:59:17.350Z [INFO] server-3275cbf9-17bc-a706-ccd6-7bb5956e4ecd: pipelining replication: peer="{Voter server-a539abbd-22f0-da35-af99-bdc702f46fb2 a539abbd-22f0-da35-af99-bdc702f46fb2}" +2025-02-25T23:59:17.350Z [INFO] server-3275cbf9-17bc-a706-ccd6-7bb5956e4ecd: pipelining replication: peer="{Voter server-07d39c37-f8f6-7cfc-23d8-7539c3b81344 07d39c37-f8f6-7cfc-23d8-7539c3b81344}" +2025-02-25T23:59:17.457Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:17.350527793 +0000 UTC m=+104.959543200 (1 nodes), 63.840126ms from start of poll, 63.999167ms from cluster start. Timeout at 2025-02-25 23:59:17.455535193 +0000 UTC m=+105.064550600, 105.0074ms after stability +2025-02-25T23:59:17.458Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:17.566Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:17.458018997 +0000 UTC m=+105.067034404 (1 nodes), 480ns from start of poll, 171.490371ms from cluster start. Timeout at 2025-02-25 23:59:17.563089878 +0000 UTC m=+105.172105285, 105.070881ms after stability +2025-02-25T23:59:17.566Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:17.674Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:17.566228966 +0000 UTC m=+105.175244373 (1 nodes), 400ns from start of poll, 279.70034ms from cluster start. Timeout at 2025-02-25 23:59:17.671261726 +0000 UTC m=+105.280277133, 105.03276ms after stability +2025-02-25T23:59:17.674Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:17.781Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:17.674391775 +0000 UTC m=+105.283407182 (1 nodes), 600ns from start of poll, 387.863149ms from cluster start. Timeout at 2025-02-25 23:59:17.779423135 +0000 UTC m=+105.388438502, 105.03132ms after stability +2025-02-25T23:59:17.782Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:17.894Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:17.782181021 +0000 UTC m=+105.391196428 (1 nodes), 880ns from start of poll, 495.652395ms from cluster start. Timeout at 2025-02-25 23:59:17.887213822 +0000 UTC m=+105.496229229, 105.032801ms after stability +2025-02-25T23:59:17.894Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:18.001Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:17.894833418 +0000 UTC m=+105.503848785 (1 nodes), 480ns from start of poll, 608.304752ms from cluster start. Timeout at 2025-02-25 23:59:17.999878219 +0000 UTC m=+105.608893626, 105.044841ms after stability +2025-02-25T23:59:18.109Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:18.217Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:18.110005628 +0000 UTC m=+105.719021035 (1 nodes), 1.28µs from start of poll, 823.477002ms from cluster start. Timeout at 2025-02-25 23:59:18.215044189 +0000 UTC m=+105.824059596, 105.038561ms after stability +2025-02-25T23:59:18.218Z [INFO] server-3275cbf9-17bc-a706-ccd6-7bb5956e4ecd: aborting pipeline replication: peer="{Voter server-a539abbd-22f0-da35-af99-bdc702f46fb2 a539abbd-22f0-da35-af99-bdc702f46fb2}" +2025-02-25T23:59:18.218Z [INFO] server-3275cbf9-17bc-a706-ccd6-7bb5956e4ecd: aborting pipeline replication: peer="{Voter server-07d39c37-f8f6-7cfc-23d8-7539c3b81344 07d39c37-f8f6-7cfc-23d8-7539c3b81344}" +2025-02-25T23:59:18.718Z [ERROR] server-3275cbf9-17bc-a706-ccd6-7bb5956e4ecd: failed to heartbeat to: peer=07d39c37-f8f6-7cfc-23d8-7539c3b81344 backoff time=10ms error="command timed out" +--- PASS: TestRaft_LeaderID_Propagated (1.43s) === RUN TestRaft_LeadershipTransferInProgress --- PASS: TestRaft_LeadershipTransferInProgress (0.00s) === RUN TestRaft_LeadershipTransferPickServer --- PASS: TestRaft_LeadershipTransferPickServer (0.00s) === RUN TestRaft_LeadershipTransfer -2026-03-31T06:17:54.264Z [INFO] server-28bafaf5-b703-a348-8ff2-d64ce13d2a7e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-28bafaf5-b703-a348-8ff2-d64ce13d2a7e Address:28bafaf5-b703-a348-8ff2-d64ce13d2a7e} {Suffrage:Voter ID:server-42321323-0935-74f8-14d4-11529617ffd5 Address:42321323-0935-74f8-14d4-11529617ffd5} {Suffrage:Voter ID:server-5f4161dd-7402-be8e-b193-c378fe7341f4 Address:5f4161dd-7402-be8e-b193-c378fe7341f4}]" -2026-03-31T06:17:54.264Z [INFO] server-42321323-0935-74f8-14d4-11529617ffd5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-28bafaf5-b703-a348-8ff2-d64ce13d2a7e Address:28bafaf5-b703-a348-8ff2-d64ce13d2a7e} {Suffrage:Voter ID:server-42321323-0935-74f8-14d4-11529617ffd5 Address:42321323-0935-74f8-14d4-11529617ffd5} {Suffrage:Voter ID:server-5f4161dd-7402-be8e-b193-c378fe7341f4 Address:5f4161dd-7402-be8e-b193-c378fe7341f4}]" -2026-03-31T06:17:54.264Z [INFO] server-28bafaf5-b703-a348-8ff2-d64ce13d2a7e: entering follower state: follower="Node at 28bafaf5-b703-a348-8ff2-d64ce13d2a7e [Follower]" leader-address= leader-id= -2026-03-31T06:17:54.264Z [INFO] server-42321323-0935-74f8-14d4-11529617ffd5: entering follower state: follower="Node at 42321323-0935-74f8-14d4-11529617ffd5 [Follower]" leader-address= leader-id= -2026-03-31T06:17:54.264Z [INFO] server-5f4161dd-7402-be8e-b193-c378fe7341f4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-28bafaf5-b703-a348-8ff2-d64ce13d2a7e Address:28bafaf5-b703-a348-8ff2-d64ce13d2a7e} {Suffrage:Voter ID:server-42321323-0935-74f8-14d4-11529617ffd5 Address:42321323-0935-74f8-14d4-11529617ffd5} {Suffrage:Voter ID:server-5f4161dd-7402-be8e-b193-c378fe7341f4 Address:5f4161dd-7402-be8e-b193-c378fe7341f4}]" -2026-03-31T06:17:54.264Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:54.264Z [INFO] server-5f4161dd-7402-be8e-b193-c378fe7341f4: entering follower state: follower="Node at 5f4161dd-7402-be8e-b193-c378fe7341f4 [Follower]" leader-address= leader-id= -2026-03-31T06:17:54.318Z [WARN] server-42321323-0935-74f8-14d4-11529617ffd5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:54.318Z [INFO] server-42321323-0935-74f8-14d4-11529617ffd5: entering candidate state: node="Node at 42321323-0935-74f8-14d4-11529617ffd5 [Candidate]" term=2 -2026-03-31T06:17:54.318Z [INFO] server-42321323-0935-74f8-14d4-11529617ffd5: election won: term=2 tally=2 -2026-03-31T06:17:54.318Z [INFO] server-42321323-0935-74f8-14d4-11529617ffd5: entering leader state: leader="Node at 42321323-0935-74f8-14d4-11529617ffd5 [Leader]" -2026-03-31T06:17:54.318Z [INFO] server-42321323-0935-74f8-14d4-11529617ffd5: added peer, starting replication: peer=server-28bafaf5-b703-a348-8ff2-d64ce13d2a7e -2026-03-31T06:17:54.319Z [INFO] server-42321323-0935-74f8-14d4-11529617ffd5: added peer, starting replication: peer=server-5f4161dd-7402-be8e-b193-c378fe7341f4 -2026-03-31T06:17:54.319Z [INFO] server-42321323-0935-74f8-14d4-11529617ffd5: pipelining replication: peer="{Voter server-28bafaf5-b703-a348-8ff2-d64ce13d2a7e 28bafaf5-b703-a348-8ff2-d64ce13d2a7e}" -2026-03-31T06:17:54.319Z [INFO] server-42321323-0935-74f8-14d4-11529617ffd5: pipelining replication: peer="{Voter server-5f4161dd-7402-be8e-b193-c378fe7341f4 5f4161dd-7402-be8e-b193-c378fe7341f4}" -2026-03-31T06:17:54.424Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:54.3190529 +0000 UTC m=+102.596660241 (1 nodes), 54.33736ms from start of poll, 54.74944ms from cluster start. Timeout at 2026-03-31 06:17:54.4240541 +0000 UTC m=+102.701661481, 105.00124ms after stability -2026-03-31T06:17:54.424Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:54.532Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:54.42417582 +0000 UTC m=+102.701783201 (1 nodes), 2.24µs from start of poll, 159.8724ms from cluster start. Timeout at 2026-03-31 06:17:54.52920934 +0000 UTC m=+102.806816721, 105.03352ms after stability -2026-03-31T06:17:54.532Z [INFO] server-28bafaf5-b703-a348-8ff2-d64ce13d2a7e: entering candidate state: node="Node at 28bafaf5-b703-a348-8ff2-d64ce13d2a7e [Candidate]" term=3 -2026-03-31T06:17:54.532Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:54.533Z [INFO] server-28bafaf5-b703-a348-8ff2-d64ce13d2a7e: election won: term=3 tally=2 -2026-03-31T06:17:54.533Z [INFO] server-28bafaf5-b703-a348-8ff2-d64ce13d2a7e: entering leader state: leader="Node at 28bafaf5-b703-a348-8ff2-d64ce13d2a7e [Leader]" -2026-03-31T06:17:54.533Z [INFO] server-28bafaf5-b703-a348-8ff2-d64ce13d2a7e: added peer, starting replication: peer=server-42321323-0935-74f8-14d4-11529617ffd5 -2026-03-31T06:17:54.533Z [INFO] server-28bafaf5-b703-a348-8ff2-d64ce13d2a7e: added peer, starting replication: peer=server-5f4161dd-7402-be8e-b193-c378fe7341f4 -2026-03-31T06:17:54.533Z [INFO] server-42321323-0935-74f8-14d4-11529617ffd5: entering follower state: follower="Node at 42321323-0935-74f8-14d4-11529617ffd5 [Follower]" leader-address= leader-id= -2026-03-31T06:17:54.533Z [INFO] server-42321323-0935-74f8-14d4-11529617ffd5: aborting pipeline replication: peer="{Voter server-5f4161dd-7402-be8e-b193-c378fe7341f4 5f4161dd-7402-be8e-b193-c378fe7341f4}" -2026-03-31T06:17:54.533Z [INFO] server-42321323-0935-74f8-14d4-11529617ffd5: aborting pipeline replication: peer="{Voter server-28bafaf5-b703-a348-8ff2-d64ce13d2a7e 28bafaf5-b703-a348-8ff2-d64ce13d2a7e}" -2026-03-31T06:17:54.533Z [INFO] server-28bafaf5-b703-a348-8ff2-d64ce13d2a7e: pipelining replication: peer="{Voter server-5f4161dd-7402-be8e-b193-c378fe7341f4 5f4161dd-7402-be8e-b193-c378fe7341f4}" -2026-03-31T06:17:54.533Z [INFO] server-28bafaf5-b703-a348-8ff2-d64ce13d2a7e: pipelining replication: peer="{Voter server-42321323-0935-74f8-14d4-11529617ffd5 42321323-0935-74f8-14d4-11529617ffd5}" -2026-03-31T06:17:54.639Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:54.53317618 +0000 UTC m=+102.810783561 (1 nodes), 159.16µs from start of poll, 268.87276ms from cluster start. Timeout at 2026-03-31 06:17:54.63822706 +0000 UTC m=+102.915834481, 105.05092ms after stability -2026-03-31T06:17:54.639Z [INFO] server-28bafaf5-b703-a348-8ff2-d64ce13d2a7e: aborting pipeline replication: peer="{Voter server-5f4161dd-7402-be8e-b193-c378fe7341f4 5f4161dd-7402-be8e-b193-c378fe7341f4}" -2026-03-31T06:17:54.639Z [INFO] server-28bafaf5-b703-a348-8ff2-d64ce13d2a7e: aborting pipeline replication: peer="{Voter server-42321323-0935-74f8-14d4-11529617ffd5 42321323-0935-74f8-14d4-11529617ffd5}" -2026-03-31T06:17:55.140Z [ERROR] server-28bafaf5-b703-a348-8ff2-d64ce13d2a7e: failed to heartbeat to: peer=42321323-0935-74f8-14d4-11529617ffd5 backoff time=10ms error="command timed out" ---- PASS: TestRaft_LeadershipTransfer (0.88s) +2025-02-25T23:59:18.718Z [INFO] server-afd0c230-041e-e0f9-a21d-b91a4bae217f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-afd0c230-041e-e0f9-a21d-b91a4bae217f Address:afd0c230-041e-e0f9-a21d-b91a4bae217f} {Suffrage:Voter ID:server-0ecfdc60-ced8-36b5-8db9-9d9c49905d97 Address:0ecfdc60-ced8-36b5-8db9-9d9c49905d97} {Suffrage:Voter ID:server-0e70bf46-f521-7b45-4ffd-dc2a0778d8ca Address:0e70bf46-f521-7b45-4ffd-dc2a0778d8ca}]" +2025-02-25T23:59:18.718Z [INFO] server-0ecfdc60-ced8-36b5-8db9-9d9c49905d97: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-afd0c230-041e-e0f9-a21d-b91a4bae217f Address:afd0c230-041e-e0f9-a21d-b91a4bae217f} {Suffrage:Voter ID:server-0ecfdc60-ced8-36b5-8db9-9d9c49905d97 Address:0ecfdc60-ced8-36b5-8db9-9d9c49905d97} {Suffrage:Voter ID:server-0e70bf46-f521-7b45-4ffd-dc2a0778d8ca Address:0e70bf46-f521-7b45-4ffd-dc2a0778d8ca}]" +2025-02-25T23:59:18.718Z [INFO] server-0e70bf46-f521-7b45-4ffd-dc2a0778d8ca: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-afd0c230-041e-e0f9-a21d-b91a4bae217f Address:afd0c230-041e-e0f9-a21d-b91a4bae217f} {Suffrage:Voter ID:server-0ecfdc60-ced8-36b5-8db9-9d9c49905d97 Address:0ecfdc60-ced8-36b5-8db9-9d9c49905d97} {Suffrage:Voter ID:server-0e70bf46-f521-7b45-4ffd-dc2a0778d8ca Address:0e70bf46-f521-7b45-4ffd-dc2a0778d8ca}]" +2025-02-25T23:59:18.718Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:18.718Z [INFO] server-afd0c230-041e-e0f9-a21d-b91a4bae217f: entering follower state: follower="Node at afd0c230-041e-e0f9-a21d-b91a4bae217f [Follower]" leader-address= leader-id= +2025-02-25T23:59:18.718Z [INFO] server-0ecfdc60-ced8-36b5-8db9-9d9c49905d97: entering follower state: follower="Node at 0ecfdc60-ced8-36b5-8db9-9d9c49905d97 [Follower]" leader-address= leader-id= +2025-02-25T23:59:18.718Z [INFO] server-0e70bf46-f521-7b45-4ffd-dc2a0778d8ca: entering follower state: follower="Node at 0e70bf46-f521-7b45-4ffd-dc2a0778d8ca [Follower]" leader-address= leader-id= +2025-02-25T23:59:18.798Z [WARN] server-afd0c230-041e-e0f9-a21d-b91a4bae217f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:18.798Z [INFO] server-afd0c230-041e-e0f9-a21d-b91a4bae217f: entering candidate state: node="Node at afd0c230-041e-e0f9-a21d-b91a4bae217f [Candidate]" term=2 +2025-02-25T23:59:18.798Z [INFO] server-afd0c230-041e-e0f9-a21d-b91a4bae217f: election won: term=2 tally=2 +2025-02-25T23:59:18.798Z [INFO] server-afd0c230-041e-e0f9-a21d-b91a4bae217f: entering leader state: leader="Node at afd0c230-041e-e0f9-a21d-b91a4bae217f [Leader]" +2025-02-25T23:59:18.798Z [INFO] server-afd0c230-041e-e0f9-a21d-b91a4bae217f: added peer, starting replication: peer=server-0ecfdc60-ced8-36b5-8db9-9d9c49905d97 +2025-02-25T23:59:18.798Z [INFO] server-afd0c230-041e-e0f9-a21d-b91a4bae217f: added peer, starting replication: peer=server-0e70bf46-f521-7b45-4ffd-dc2a0778d8ca +2025-02-25T23:59:18.798Z [INFO] server-afd0c230-041e-e0f9-a21d-b91a4bae217f: pipelining replication: peer="{Voter server-0e70bf46-f521-7b45-4ffd-dc2a0778d8ca 0e70bf46-f521-7b45-4ffd-dc2a0778d8ca}" +2025-02-25T23:59:18.798Z [INFO] server-afd0c230-041e-e0f9-a21d-b91a4bae217f: pipelining replication: peer="{Voter server-0ecfdc60-ced8-36b5-8db9-9d9c49905d97 0ecfdc60-ced8-36b5-8db9-9d9c49905d97}" +2025-02-25T23:59:18.906Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:18.798986454 +0000 UTC m=+106.408001861 (1 nodes), 80.350111ms from start of poll, 80.539393ms from cluster start. Timeout at 2025-02-25 23:59:18.903991534 +0000 UTC m=+106.513006941, 105.00508ms after stability +2025-02-25T23:59:18.906Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:19.013Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:18.906225537 +0000 UTC m=+106.515240944 (1 nodes), 1.36µs from start of poll, 187.778476ms from cluster start. Timeout at 2025-02-25 23:59:19.011283857 +0000 UTC m=+106.620299264, 105.05832ms after stability +2025-02-25T23:59:19.014Z [INFO] server-0ecfdc60-ced8-36b5-8db9-9d9c49905d97: entering candidate state: node="Node at 0ecfdc60-ced8-36b5-8db9-9d9c49905d97 [Candidate]" term=3 +2025-02-25T23:59:19.014Z [INFO] server-0ecfdc60-ced8-36b5-8db9-9d9c49905d97: election won: term=3 tally=2 +2025-02-25T23:59:19.014Z [INFO] server-0ecfdc60-ced8-36b5-8db9-9d9c49905d97: entering leader state: leader="Node at 0ecfdc60-ced8-36b5-8db9-9d9c49905d97 [Leader]" +2025-02-25T23:59:19.014Z [INFO] server-0ecfdc60-ced8-36b5-8db9-9d9c49905d97: added peer, starting replication: peer=server-afd0c230-041e-e0f9-a21d-b91a4bae217f +2025-02-25T23:59:19.014Z [INFO] server-0ecfdc60-ced8-36b5-8db9-9d9c49905d97: added peer, starting replication: peer=server-0e70bf46-f521-7b45-4ffd-dc2a0778d8ca +2025-02-25T23:59:19.014Z [INFO] server-0ecfdc60-ced8-36b5-8db9-9d9c49905d97: pipelining replication: peer="{Voter server-0e70bf46-f521-7b45-4ffd-dc2a0778d8ca 0e70bf46-f521-7b45-4ffd-dc2a0778d8ca}" +2025-02-25T23:59:19.014Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:19.014Z [INFO] server-afd0c230-041e-e0f9-a21d-b91a4bae217f: entering follower state: follower="Node at afd0c230-041e-e0f9-a21d-b91a4bae217f [Follower]" leader-address= leader-id= +2025-02-25T23:59:19.014Z [INFO] server-afd0c230-041e-e0f9-a21d-b91a4bae217f: aborting pipeline replication: peer="{Voter server-0ecfdc60-ced8-36b5-8db9-9d9c49905d97 0ecfdc60-ced8-36b5-8db9-9d9c49905d97}" +2025-02-25T23:59:19.014Z [INFO] server-0ecfdc60-ced8-36b5-8db9-9d9c49905d97: pipelining replication: peer="{Voter server-afd0c230-041e-e0f9-a21d-b91a4bae217f afd0c230-041e-e0f9-a21d-b91a4bae217f}" +2025-02-25T23:59:19.014Z [INFO] server-afd0c230-041e-e0f9-a21d-b91a4bae217f: aborting pipeline replication: peer="{Voter server-0e70bf46-f521-7b45-4ffd-dc2a0778d8ca 0e70bf46-f521-7b45-4ffd-dc2a0778d8ca}" +2025-02-25T23:59:19.121Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:19.014338265 +0000 UTC m=+106.623353672 (1 nodes), 91.08µs from start of poll, 295.891204ms from cluster start. Timeout at 2025-02-25 23:59:19.119398346 +0000 UTC m=+106.728413753, 105.060081ms after stability +2025-02-25T23:59:19.122Z [INFO] server-0ecfdc60-ced8-36b5-8db9-9d9c49905d97: aborting pipeline replication: peer="{Voter server-0e70bf46-f521-7b45-4ffd-dc2a0778d8ca 0e70bf46-f521-7b45-4ffd-dc2a0778d8ca}" +2025-02-25T23:59:19.122Z [INFO] server-0ecfdc60-ced8-36b5-8db9-9d9c49905d97: aborting pipeline replication: peer="{Voter server-afd0c230-041e-e0f9-a21d-b91a4bae217f afd0c230-041e-e0f9-a21d-b91a4bae217f}" +--- PASS: TestRaft_LeadershipTransfer (0.40s) === RUN TestRaft_LeadershipTransferWithOneNode -2026-03-31T06:17:55.140Z [INFO] server-c87bd9b4-9167-0128-e20e-1c0c45390350: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c87bd9b4-9167-0128-e20e-1c0c45390350 Address:c87bd9b4-9167-0128-e20e-1c0c45390350}]" -2026-03-31T06:17:55.140Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:55.140Z [INFO] server-c87bd9b4-9167-0128-e20e-1c0c45390350: entering follower state: follower="Node at c87bd9b4-9167-0128-e20e-1c0c45390350 [Follower]" leader-address= leader-id= -2026-03-31T06:17:55.230Z [WARN] server-c87bd9b4-9167-0128-e20e-1c0c45390350: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:55.230Z [INFO] server-c87bd9b4-9167-0128-e20e-1c0c45390350: entering candidate state: node="Node at c87bd9b4-9167-0128-e20e-1c0c45390350 [Candidate]" term=2 -2026-03-31T06:17:55.230Z [INFO] server-c87bd9b4-9167-0128-e20e-1c0c45390350: election won: term=2 tally=1 -2026-03-31T06:17:55.230Z [INFO] server-c87bd9b4-9167-0128-e20e-1c0c45390350: entering leader state: leader="Node at c87bd9b4-9167-0128-e20e-1c0c45390350 [Leader]" -2026-03-31T06:17:55.336Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:55.23077618 +0000 UTC m=+103.508383561 (1 nodes), 89.97208ms from start of poll, 90.17472ms from cluster start. Timeout at 2026-03-31 06:17:55.33578742 +0000 UTC m=+103.613394841, 105.01128ms after stability ---- PASS: TestRaft_LeadershipTransferWithOneNode (0.20s) +2025-02-25T23:59:19.122Z [INFO] server-7c172fb9-67d0-db23-c801-db80a7e78b6f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7c172fb9-67d0-db23-c801-db80a7e78b6f Address:7c172fb9-67d0-db23-c801-db80a7e78b6f}]" +2025-02-25T23:59:19.122Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:19.122Z [INFO] server-7c172fb9-67d0-db23-c801-db80a7e78b6f: entering follower state: follower="Node at 7c172fb9-67d0-db23-c801-db80a7e78b6f [Follower]" leader-address= leader-id= +2025-02-25T23:59:19.185Z [WARN] server-7c172fb9-67d0-db23-c801-db80a7e78b6f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:19.185Z [INFO] server-7c172fb9-67d0-db23-c801-db80a7e78b6f: entering candidate state: node="Node at 7c172fb9-67d0-db23-c801-db80a7e78b6f [Candidate]" term=2 +2025-02-25T23:59:19.185Z [INFO] server-7c172fb9-67d0-db23-c801-db80a7e78b6f: election won: term=2 tally=1 +2025-02-25T23:59:19.185Z [INFO] server-7c172fb9-67d0-db23-c801-db80a7e78b6f: entering leader state: leader="Node at 7c172fb9-67d0-db23-c801-db80a7e78b6f [Leader]" +2025-02-25T23:59:19.294Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:19.186192199 +0000 UTC m=+106.795207606 (1 nodes), 63.629485ms from start of poll, 63.720365ms from cluster start. Timeout at 2025-02-25 23:59:19.291194599 +0000 UTC m=+106.900209966, 105.00236ms after stability +--- PASS: TestRaft_LeadershipTransferWithOneNode (0.17s) === RUN TestRaft_LeadershipTransferWithSevenNodes -2026-03-31T06:17:55.337Z [INFO] server-9a7ed411-0692-79f0-8010-6607591de316: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9a7ed411-0692-79f0-8010-6607591de316 Address:9a7ed411-0692-79f0-8010-6607591de316} {Suffrage:Voter ID:server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e Address:feae006a-0b80-ffd2-cb0c-c56f84f3a37e} {Suffrage:Voter ID:server-cb7301ff-2b28-8211-f4b1-8cfd8f000433 Address:cb7301ff-2b28-8211-f4b1-8cfd8f000433} {Suffrage:Voter ID:server-c43c11f2-dcf5-530a-0d7f-c8b7802c61c0 Address:c43c11f2-dcf5-530a-0d7f-c8b7802c61c0} {Suffrage:Voter ID:server-4138e958-1dbb-9700-a44a-c7ee468a4105 Address:4138e958-1dbb-9700-a44a-c7ee468a4105} {Suffrage:Voter ID:server-5439fe31-2692-fca0-5c05-cba1f545aaad Address:5439fe31-2692-fca0-5c05-cba1f545aaad} {Suffrage:Voter ID:server-61802f6b-b1ae-9489-8ef7-908db117bfe4 Address:61802f6b-b1ae-9489-8ef7-908db117bfe4}]" -2026-03-31T06:17:55.337Z [INFO] server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9a7ed411-0692-79f0-8010-6607591de316 Address:9a7ed411-0692-79f0-8010-6607591de316} {Suffrage:Voter ID:server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e Address:feae006a-0b80-ffd2-cb0c-c56f84f3a37e} {Suffrage:Voter ID:server-cb7301ff-2b28-8211-f4b1-8cfd8f000433 Address:cb7301ff-2b28-8211-f4b1-8cfd8f000433} {Suffrage:Voter ID:server-c43c11f2-dcf5-530a-0d7f-c8b7802c61c0 Address:c43c11f2-dcf5-530a-0d7f-c8b7802c61c0} {Suffrage:Voter ID:server-4138e958-1dbb-9700-a44a-c7ee468a4105 Address:4138e958-1dbb-9700-a44a-c7ee468a4105} {Suffrage:Voter ID:server-5439fe31-2692-fca0-5c05-cba1f545aaad Address:5439fe31-2692-fca0-5c05-cba1f545aaad} {Suffrage:Voter ID:server-61802f6b-b1ae-9489-8ef7-908db117bfe4 Address:61802f6b-b1ae-9489-8ef7-908db117bfe4}]" -2026-03-31T06:17:55.337Z [INFO] server-9a7ed411-0692-79f0-8010-6607591de316: entering follower state: follower="Node at 9a7ed411-0692-79f0-8010-6607591de316 [Follower]" leader-address= leader-id= -2026-03-31T06:17:55.337Z [INFO] server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e: entering follower state: follower="Node at feae006a-0b80-ffd2-cb0c-c56f84f3a37e [Follower]" leader-address= leader-id= -2026-03-31T06:17:55.337Z [INFO] server-cb7301ff-2b28-8211-f4b1-8cfd8f000433: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9a7ed411-0692-79f0-8010-6607591de316 Address:9a7ed411-0692-79f0-8010-6607591de316} {Suffrage:Voter ID:server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e Address:feae006a-0b80-ffd2-cb0c-c56f84f3a37e} {Suffrage:Voter ID:server-cb7301ff-2b28-8211-f4b1-8cfd8f000433 Address:cb7301ff-2b28-8211-f4b1-8cfd8f000433} {Suffrage:Voter ID:server-c43c11f2-dcf5-530a-0d7f-c8b7802c61c0 Address:c43c11f2-dcf5-530a-0d7f-c8b7802c61c0} {Suffrage:Voter ID:server-4138e958-1dbb-9700-a44a-c7ee468a4105 Address:4138e958-1dbb-9700-a44a-c7ee468a4105} {Suffrage:Voter ID:server-5439fe31-2692-fca0-5c05-cba1f545aaad Address:5439fe31-2692-fca0-5c05-cba1f545aaad} {Suffrage:Voter ID:server-61802f6b-b1ae-9489-8ef7-908db117bfe4 Address:61802f6b-b1ae-9489-8ef7-908db117bfe4}]" -2026-03-31T06:17:55.337Z [INFO] server-cb7301ff-2b28-8211-f4b1-8cfd8f000433: entering follower state: follower="Node at cb7301ff-2b28-8211-f4b1-8cfd8f000433 [Follower]" leader-address= leader-id= -2026-03-31T06:17:55.337Z [INFO] server-c43c11f2-dcf5-530a-0d7f-c8b7802c61c0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9a7ed411-0692-79f0-8010-6607591de316 Address:9a7ed411-0692-79f0-8010-6607591de316} {Suffrage:Voter ID:server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e Address:feae006a-0b80-ffd2-cb0c-c56f84f3a37e} {Suffrage:Voter ID:server-cb7301ff-2b28-8211-f4b1-8cfd8f000433 Address:cb7301ff-2b28-8211-f4b1-8cfd8f000433} {Suffrage:Voter ID:server-c43c11f2-dcf5-530a-0d7f-c8b7802c61c0 Address:c43c11f2-dcf5-530a-0d7f-c8b7802c61c0} {Suffrage:Voter ID:server-4138e958-1dbb-9700-a44a-c7ee468a4105 Address:4138e958-1dbb-9700-a44a-c7ee468a4105} {Suffrage:Voter ID:server-5439fe31-2692-fca0-5c05-cba1f545aaad Address:5439fe31-2692-fca0-5c05-cba1f545aaad} {Suffrage:Voter ID:server-61802f6b-b1ae-9489-8ef7-908db117bfe4 Address:61802f6b-b1ae-9489-8ef7-908db117bfe4}]" -2026-03-31T06:17:55.337Z [INFO] server-c43c11f2-dcf5-530a-0d7f-c8b7802c61c0: entering follower state: follower="Node at c43c11f2-dcf5-530a-0d7f-c8b7802c61c0 [Follower]" leader-address= leader-id= -2026-03-31T06:17:55.337Z [INFO] server-4138e958-1dbb-9700-a44a-c7ee468a4105: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9a7ed411-0692-79f0-8010-6607591de316 Address:9a7ed411-0692-79f0-8010-6607591de316} {Suffrage:Voter ID:server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e Address:feae006a-0b80-ffd2-cb0c-c56f84f3a37e} {Suffrage:Voter ID:server-cb7301ff-2b28-8211-f4b1-8cfd8f000433 Address:cb7301ff-2b28-8211-f4b1-8cfd8f000433} {Suffrage:Voter ID:server-c43c11f2-dcf5-530a-0d7f-c8b7802c61c0 Address:c43c11f2-dcf5-530a-0d7f-c8b7802c61c0} {Suffrage:Voter ID:server-4138e958-1dbb-9700-a44a-c7ee468a4105 Address:4138e958-1dbb-9700-a44a-c7ee468a4105} {Suffrage:Voter ID:server-5439fe31-2692-fca0-5c05-cba1f545aaad Address:5439fe31-2692-fca0-5c05-cba1f545aaad} {Suffrage:Voter ID:server-61802f6b-b1ae-9489-8ef7-908db117bfe4 Address:61802f6b-b1ae-9489-8ef7-908db117bfe4}]" -2026-03-31T06:17:55.338Z [INFO] server-4138e958-1dbb-9700-a44a-c7ee468a4105: entering follower state: follower="Node at 4138e958-1dbb-9700-a44a-c7ee468a4105 [Follower]" leader-address= leader-id= -2026-03-31T06:17:55.338Z [INFO] server-5439fe31-2692-fca0-5c05-cba1f545aaad: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9a7ed411-0692-79f0-8010-6607591de316 Address:9a7ed411-0692-79f0-8010-6607591de316} {Suffrage:Voter ID:server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e Address:feae006a-0b80-ffd2-cb0c-c56f84f3a37e} {Suffrage:Voter ID:server-cb7301ff-2b28-8211-f4b1-8cfd8f000433 Address:cb7301ff-2b28-8211-f4b1-8cfd8f000433} {Suffrage:Voter ID:server-c43c11f2-dcf5-530a-0d7f-c8b7802c61c0 Address:c43c11f2-dcf5-530a-0d7f-c8b7802c61c0} {Suffrage:Voter ID:server-4138e958-1dbb-9700-a44a-c7ee468a4105 Address:4138e958-1dbb-9700-a44a-c7ee468a4105} {Suffrage:Voter ID:server-5439fe31-2692-fca0-5c05-cba1f545aaad Address:5439fe31-2692-fca0-5c05-cba1f545aaad} {Suffrage:Voter ID:server-61802f6b-b1ae-9489-8ef7-908db117bfe4 Address:61802f6b-b1ae-9489-8ef7-908db117bfe4}]" -2026-03-31T06:17:55.338Z [INFO] server-5439fe31-2692-fca0-5c05-cba1f545aaad: entering follower state: follower="Node at 5439fe31-2692-fca0-5c05-cba1f545aaad [Follower]" leader-address= leader-id= -2026-03-31T06:17:55.338Z [INFO] server-61802f6b-b1ae-9489-8ef7-908db117bfe4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9a7ed411-0692-79f0-8010-6607591de316 Address:9a7ed411-0692-79f0-8010-6607591de316} {Suffrage:Voter ID:server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e Address:feae006a-0b80-ffd2-cb0c-c56f84f3a37e} {Suffrage:Voter ID:server-cb7301ff-2b28-8211-f4b1-8cfd8f000433 Address:cb7301ff-2b28-8211-f4b1-8cfd8f000433} {Suffrage:Voter ID:server-c43c11f2-dcf5-530a-0d7f-c8b7802c61c0 Address:c43c11f2-dcf5-530a-0d7f-c8b7802c61c0} {Suffrage:Voter ID:server-4138e958-1dbb-9700-a44a-c7ee468a4105 Address:4138e958-1dbb-9700-a44a-c7ee468a4105} {Suffrage:Voter ID:server-5439fe31-2692-fca0-5c05-cba1f545aaad Address:5439fe31-2692-fca0-5c05-cba1f545aaad} {Suffrage:Voter ID:server-61802f6b-b1ae-9489-8ef7-908db117bfe4 Address:61802f6b-b1ae-9489-8ef7-908db117bfe4}]" -2026-03-31T06:17:55.338Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:55.338Z [INFO] server-61802f6b-b1ae-9489-8ef7-908db117bfe4: entering follower state: follower="Node at 61802f6b-b1ae-9489-8ef7-908db117bfe4 [Follower]" leader-address= leader-id= -2026-03-31T06:17:55.390Z [WARN] server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:55.390Z [INFO] server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e: entering candidate state: node="Node at feae006a-0b80-ffd2-cb0c-c56f84f3a37e [Candidate]" term=2 -2026-03-31T06:17:55.390Z [WARN] server-c43c11f2-dcf5-530a-0d7f-c8b7802c61c0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:55.390Z [INFO] server-c43c11f2-dcf5-530a-0d7f-c8b7802c61c0: entering candidate state: node="Node at c43c11f2-dcf5-530a-0d7f-c8b7802c61c0 [Candidate]" term=2 -2026-03-31T06:17:55.390Z [INFO] server-c43c11f2-dcf5-530a-0d7f-c8b7802c61c0: duplicate requestVote for same term: term=2 -2026-03-31T06:17:55.390Z [INFO] server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e: election won: term=2 tally=4 -2026-03-31T06:17:55.390Z [INFO] server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e: entering leader state: leader="Node at feae006a-0b80-ffd2-cb0c-c56f84f3a37e [Leader]" -2026-03-31T06:17:55.390Z [INFO] server-9a7ed411-0692-79f0-8010-6607591de316: duplicate requestVote for same term: term=2 -2026-03-31T06:17:55.390Z [INFO] server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e: added peer, starting replication: peer=server-9a7ed411-0692-79f0-8010-6607591de316 -2026-03-31T06:17:55.390Z [INFO] server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e: added peer, starting replication: peer=server-cb7301ff-2b28-8211-f4b1-8cfd8f000433 -2026-03-31T06:17:55.390Z [INFO] server-5439fe31-2692-fca0-5c05-cba1f545aaad: duplicate requestVote for same term: term=2 -2026-03-31T06:17:55.390Z [INFO] server-61802f6b-b1ae-9489-8ef7-908db117bfe4: duplicate requestVote for same term: term=2 -2026-03-31T06:17:55.390Z [INFO] server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e: added peer, starting replication: peer=server-c43c11f2-dcf5-530a-0d7f-c8b7802c61c0 -2026-03-31T06:17:55.390Z [INFO] server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e: added peer, starting replication: peer=server-4138e958-1dbb-9700-a44a-c7ee468a4105 -2026-03-31T06:17:55.390Z [INFO] server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e: added peer, starting replication: peer=server-5439fe31-2692-fca0-5c05-cba1f545aaad -2026-03-31T06:17:55.390Z [INFO] server-c43c11f2-dcf5-530a-0d7f-c8b7802c61c0: entering follower state: follower="Node at c43c11f2-dcf5-530a-0d7f-c8b7802c61c0 [Follower]" leader-address=feae006a-0b80-ffd2-cb0c-c56f84f3a37e leader-id=server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e -2026-03-31T06:17:55.390Z [INFO] server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e: pipelining replication: peer="{Voter server-9a7ed411-0692-79f0-8010-6607591de316 9a7ed411-0692-79f0-8010-6607591de316}" -2026-03-31T06:17:55.390Z [INFO] server-cb7301ff-2b28-8211-f4b1-8cfd8f000433: duplicate requestVote for same term: term=2 -2026-03-31T06:17:55.390Z [INFO] server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e: added peer, starting replication: peer=server-61802f6b-b1ae-9489-8ef7-908db117bfe4 -2026-03-31T06:17:55.390Z [INFO] server-4138e958-1dbb-9700-a44a-c7ee468a4105: duplicate requestVote for same term: term=2 -2026-03-31T06:17:55.390Z [INFO] server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e: pipelining replication: peer="{Voter server-c43c11f2-dcf5-530a-0d7f-c8b7802c61c0 c43c11f2-dcf5-530a-0d7f-c8b7802c61c0}" -2026-03-31T06:17:55.390Z [INFO] server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e: pipelining replication: peer="{Voter server-5439fe31-2692-fca0-5c05-cba1f545aaad 5439fe31-2692-fca0-5c05-cba1f545aaad}" -2026-03-31T06:17:55.390Z [INFO] server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e: pipelining replication: peer="{Voter server-cb7301ff-2b28-8211-f4b1-8cfd8f000433 cb7301ff-2b28-8211-f4b1-8cfd8f000433}" -2026-03-31T06:17:55.390Z [WARN] server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e: rejecting vote request since we have a leader: from=c43c11f2-dcf5-530a-0d7f-c8b7802c61c0 leader=feae006a-0b80-ffd2-cb0c-c56f84f3a37e leader-id=server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e -2026-03-31T06:17:55.390Z [INFO] server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e: pipelining replication: peer="{Voter server-4138e958-1dbb-9700-a44a-c7ee468a4105 4138e958-1dbb-9700-a44a-c7ee468a4105}" -2026-03-31T06:17:55.390Z [INFO] server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e: pipelining replication: peer="{Voter server-61802f6b-b1ae-9489-8ef7-908db117bfe4 61802f6b-b1ae-9489-8ef7-908db117bfe4}" -2026-03-31T06:17:55.497Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:55.39068054 +0000 UTC m=+103.668287921 (1 nodes), 52.41224ms from start of poll, 53.37788ms from cluster start. Timeout at 2026-03-31 06:17:55.49568286 +0000 UTC m=+103.773290281, 105.00236ms after stability -2026-03-31T06:17:55.497Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:17:55.602Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:17:55.49748122 +0000 UTC m=+103.775088601 (6 nodes), 3.12µs from start of poll, 160.17856ms from cluster start. Timeout at 2026-03-31 06:17:55.60251366 +0000 UTC m=+103.880121041, 105.03244ms after stability -2026-03-31T06:17:55.603Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:55.708Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:55.60302198 +0000 UTC m=+103.880629401 (1 nodes), 2.88µs from start of poll, 265.71936ms from cluster start. Timeout at 2026-03-31 06:17:55.70806326 +0000 UTC m=+103.985670681, 105.04128ms after stability -2026-03-31T06:17:55.708Z [INFO] server-9a7ed411-0692-79f0-8010-6607591de316: entering candidate state: node="Node at 9a7ed411-0692-79f0-8010-6607591de316 [Candidate]" term=3 -2026-03-31T06:17:55.708Z [INFO] server-9a7ed411-0692-79f0-8010-6607591de316: election won: term=3 tally=4 -2026-03-31T06:17:55.708Z [INFO] server-9a7ed411-0692-79f0-8010-6607591de316: entering leader state: leader="Node at 9a7ed411-0692-79f0-8010-6607591de316 [Leader]" -2026-03-31T06:17:55.708Z [INFO] server-9a7ed411-0692-79f0-8010-6607591de316: added peer, starting replication: peer=server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e -2026-03-31T06:17:55.708Z [INFO] server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e: entering follower state: follower="Node at feae006a-0b80-ffd2-cb0c-c56f84f3a37e [Follower]" leader-address= leader-id= -2026-03-31T06:17:55.709Z [INFO] server-9a7ed411-0692-79f0-8010-6607591de316: added peer, starting replication: peer=server-cb7301ff-2b28-8211-f4b1-8cfd8f000433 -2026-03-31T06:17:55.709Z [INFO] server-9a7ed411-0692-79f0-8010-6607591de316: added peer, starting replication: peer=server-c43c11f2-dcf5-530a-0d7f-c8b7802c61c0 -2026-03-31T06:17:55.709Z [INFO] server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e: aborting pipeline replication: peer="{Voter server-4138e958-1dbb-9700-a44a-c7ee468a4105 4138e958-1dbb-9700-a44a-c7ee468a4105}" -2026-03-31T06:17:55.709Z [INFO] server-9a7ed411-0692-79f0-8010-6607591de316: added peer, starting replication: peer=server-4138e958-1dbb-9700-a44a-c7ee468a4105 -2026-03-31T06:17:55.709Z [INFO] server-9a7ed411-0692-79f0-8010-6607591de316: added peer, starting replication: peer=server-5439fe31-2692-fca0-5c05-cba1f545aaad -2026-03-31T06:17:55.709Z [INFO] server-9a7ed411-0692-79f0-8010-6607591de316: added peer, starting replication: peer=server-61802f6b-b1ae-9489-8ef7-908db117bfe4 -2026-03-31T06:17:55.709Z [INFO] server-9a7ed411-0692-79f0-8010-6607591de316: pipelining replication: peer="{Voter server-c43c11f2-dcf5-530a-0d7f-c8b7802c61c0 c43c11f2-dcf5-530a-0d7f-c8b7802c61c0}" -2026-03-31T06:17:55.708Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:55.709Z [INFO] server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e: aborting pipeline replication: peer="{Voter server-9a7ed411-0692-79f0-8010-6607591de316 9a7ed411-0692-79f0-8010-6607591de316}" -2026-03-31T06:17:55.709Z [INFO] server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e: aborting pipeline replication: peer="{Voter server-5439fe31-2692-fca0-5c05-cba1f545aaad 5439fe31-2692-fca0-5c05-cba1f545aaad}" -2026-03-31T06:17:55.709Z [INFO] server-9a7ed411-0692-79f0-8010-6607591de316: pipelining replication: peer="{Voter server-cb7301ff-2b28-8211-f4b1-8cfd8f000433 cb7301ff-2b28-8211-f4b1-8cfd8f000433}" -2026-03-31T06:17:55.709Z [INFO] server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e: aborting pipeline replication: peer="{Voter server-c43c11f2-dcf5-530a-0d7f-c8b7802c61c0 c43c11f2-dcf5-530a-0d7f-c8b7802c61c0}" -2026-03-31T06:17:55.709Z [INFO] server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e: aborting pipeline replication: peer="{Voter server-61802f6b-b1ae-9489-8ef7-908db117bfe4 61802f6b-b1ae-9489-8ef7-908db117bfe4}" -2026-03-31T06:17:55.709Z [INFO] server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e: aborting pipeline replication: peer="{Voter server-cb7301ff-2b28-8211-f4b1-8cfd8f000433 cb7301ff-2b28-8211-f4b1-8cfd8f000433}" -2026-03-31T06:17:55.709Z [INFO] server-9a7ed411-0692-79f0-8010-6607591de316: pipelining replication: peer="{Voter server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e feae006a-0b80-ffd2-cb0c-c56f84f3a37e}" -2026-03-31T06:17:55.709Z [INFO] server-9a7ed411-0692-79f0-8010-6607591de316: pipelining replication: peer="{Voter server-61802f6b-b1ae-9489-8ef7-908db117bfe4 61802f6b-b1ae-9489-8ef7-908db117bfe4}" -2026-03-31T06:17:55.709Z [INFO] server-9a7ed411-0692-79f0-8010-6607591de316: pipelining replication: peer="{Voter server-4138e958-1dbb-9700-a44a-c7ee468a4105 4138e958-1dbb-9700-a44a-c7ee468a4105}" -2026-03-31T06:17:55.709Z [INFO] server-9a7ed411-0692-79f0-8010-6607591de316: pipelining replication: peer="{Voter server-5439fe31-2692-fca0-5c05-cba1f545aaad 5439fe31-2692-fca0-5c05-cba1f545aaad}" -2026-03-31T06:17:55.814Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:55.7092651 +0000 UTC m=+103.986872521 (1 nodes), 67.76µs from start of poll, 371.96248ms from cluster start. Timeout at 2026-03-31 06:17:55.81428674 +0000 UTC m=+104.091894121, 105.0216ms after stability -2026-03-31T06:17:55.815Z [INFO] server-9a7ed411-0692-79f0-8010-6607591de316: aborting pipeline replication: peer="{Voter server-4138e958-1dbb-9700-a44a-c7ee468a4105 4138e958-1dbb-9700-a44a-c7ee468a4105}" -2026-03-31T06:17:55.815Z [INFO] server-9a7ed411-0692-79f0-8010-6607591de316: aborting pipeline replication: peer="{Voter server-feae006a-0b80-ffd2-cb0c-c56f84f3a37e feae006a-0b80-ffd2-cb0c-c56f84f3a37e}" -2026-03-31T06:17:55.815Z [INFO] server-9a7ed411-0692-79f0-8010-6607591de316: aborting pipeline replication: peer="{Voter server-cb7301ff-2b28-8211-f4b1-8cfd8f000433 cb7301ff-2b28-8211-f4b1-8cfd8f000433}" -2026-03-31T06:17:55.815Z [INFO] server-9a7ed411-0692-79f0-8010-6607591de316: aborting pipeline replication: peer="{Voter server-c43c11f2-dcf5-530a-0d7f-c8b7802c61c0 c43c11f2-dcf5-530a-0d7f-c8b7802c61c0}" -2026-03-31T06:17:55.815Z [INFO] server-9a7ed411-0692-79f0-8010-6607591de316: aborting pipeline replication: peer="{Voter server-5439fe31-2692-fca0-5c05-cba1f545aaad 5439fe31-2692-fca0-5c05-cba1f545aaad}" -2026-03-31T06:17:55.815Z [INFO] server-9a7ed411-0692-79f0-8010-6607591de316: aborting pipeline replication: peer="{Voter server-61802f6b-b1ae-9489-8ef7-908db117bfe4 61802f6b-b1ae-9489-8ef7-908db117bfe4}" -2026-03-31T06:17:56.315Z [ERROR] server-9a7ed411-0692-79f0-8010-6607591de316: failed to heartbeat to: peer=61802f6b-b1ae-9489-8ef7-908db117bfe4 backoff time=10ms error="command timed out" ---- PASS: TestRaft_LeadershipTransferWithSevenNodes (0.98s) +2025-02-25T23:59:19.294Z [INFO] server-ac7f5eee-e884-e1b8-4d56-14d0ba2bc981: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ac7f5eee-e884-e1b8-4d56-14d0ba2bc981 Address:ac7f5eee-e884-e1b8-4d56-14d0ba2bc981} {Suffrage:Voter ID:server-670f6f2f-ed1f-1dcb-7a38-b0d57aa370fe Address:670f6f2f-ed1f-1dcb-7a38-b0d57aa370fe} {Suffrage:Voter ID:server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb Address:6369e5cc-ccc7-a799-6959-e525b1a6e7bb} {Suffrage:Voter ID:server-ef8f2d53-a97f-bac5-9bb8-73689d97a436 Address:ef8f2d53-a97f-bac5-9bb8-73689d97a436} {Suffrage:Voter ID:server-d54c2e9e-a8f5-7fe9-ccf8-c7e051b79b81 Address:d54c2e9e-a8f5-7fe9-ccf8-c7e051b79b81} {Suffrage:Voter ID:server-02eacae1-6000-7bdc-9282-f571429af218 Address:02eacae1-6000-7bdc-9282-f571429af218} {Suffrage:Voter ID:server-48144434-6d83-ece7-b902-7e122583bdaf Address:48144434-6d83-ece7-b902-7e122583bdaf}]" +2025-02-25T23:59:19.294Z [INFO] server-670f6f2f-ed1f-1dcb-7a38-b0d57aa370fe: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ac7f5eee-e884-e1b8-4d56-14d0ba2bc981 Address:ac7f5eee-e884-e1b8-4d56-14d0ba2bc981} {Suffrage:Voter ID:server-670f6f2f-ed1f-1dcb-7a38-b0d57aa370fe Address:670f6f2f-ed1f-1dcb-7a38-b0d57aa370fe} {Suffrage:Voter ID:server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb Address:6369e5cc-ccc7-a799-6959-e525b1a6e7bb} {Suffrage:Voter ID:server-ef8f2d53-a97f-bac5-9bb8-73689d97a436 Address:ef8f2d53-a97f-bac5-9bb8-73689d97a436} {Suffrage:Voter ID:server-d54c2e9e-a8f5-7fe9-ccf8-c7e051b79b81 Address:d54c2e9e-a8f5-7fe9-ccf8-c7e051b79b81} {Suffrage:Voter ID:server-02eacae1-6000-7bdc-9282-f571429af218 Address:02eacae1-6000-7bdc-9282-f571429af218} {Suffrage:Voter ID:server-48144434-6d83-ece7-b902-7e122583bdaf Address:48144434-6d83-ece7-b902-7e122583bdaf}]" +2025-02-25T23:59:19.294Z [INFO] server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ac7f5eee-e884-e1b8-4d56-14d0ba2bc981 Address:ac7f5eee-e884-e1b8-4d56-14d0ba2bc981} {Suffrage:Voter ID:server-670f6f2f-ed1f-1dcb-7a38-b0d57aa370fe Address:670f6f2f-ed1f-1dcb-7a38-b0d57aa370fe} {Suffrage:Voter ID:server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb Address:6369e5cc-ccc7-a799-6959-e525b1a6e7bb} {Suffrage:Voter ID:server-ef8f2d53-a97f-bac5-9bb8-73689d97a436 Address:ef8f2d53-a97f-bac5-9bb8-73689d97a436} {Suffrage:Voter ID:server-d54c2e9e-a8f5-7fe9-ccf8-c7e051b79b81 Address:d54c2e9e-a8f5-7fe9-ccf8-c7e051b79b81} {Suffrage:Voter ID:server-02eacae1-6000-7bdc-9282-f571429af218 Address:02eacae1-6000-7bdc-9282-f571429af218} {Suffrage:Voter ID:server-48144434-6d83-ece7-b902-7e122583bdaf Address:48144434-6d83-ece7-b902-7e122583bdaf}]" +2025-02-25T23:59:19.294Z [INFO] server-ef8f2d53-a97f-bac5-9bb8-73689d97a436: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ac7f5eee-e884-e1b8-4d56-14d0ba2bc981 Address:ac7f5eee-e884-e1b8-4d56-14d0ba2bc981} {Suffrage:Voter ID:server-670f6f2f-ed1f-1dcb-7a38-b0d57aa370fe Address:670f6f2f-ed1f-1dcb-7a38-b0d57aa370fe} {Suffrage:Voter ID:server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb Address:6369e5cc-ccc7-a799-6959-e525b1a6e7bb} {Suffrage:Voter ID:server-ef8f2d53-a97f-bac5-9bb8-73689d97a436 Address:ef8f2d53-a97f-bac5-9bb8-73689d97a436} {Suffrage:Voter ID:server-d54c2e9e-a8f5-7fe9-ccf8-c7e051b79b81 Address:d54c2e9e-a8f5-7fe9-ccf8-c7e051b79b81} {Suffrage:Voter ID:server-02eacae1-6000-7bdc-9282-f571429af218 Address:02eacae1-6000-7bdc-9282-f571429af218} {Suffrage:Voter ID:server-48144434-6d83-ece7-b902-7e122583bdaf Address:48144434-6d83-ece7-b902-7e122583bdaf}]" +2025-02-25T23:59:19.294Z [INFO] server-d54c2e9e-a8f5-7fe9-ccf8-c7e051b79b81: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ac7f5eee-e884-e1b8-4d56-14d0ba2bc981 Address:ac7f5eee-e884-e1b8-4d56-14d0ba2bc981} {Suffrage:Voter ID:server-670f6f2f-ed1f-1dcb-7a38-b0d57aa370fe Address:670f6f2f-ed1f-1dcb-7a38-b0d57aa370fe} {Suffrage:Voter ID:server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb Address:6369e5cc-ccc7-a799-6959-e525b1a6e7bb} {Suffrage:Voter ID:server-ef8f2d53-a97f-bac5-9bb8-73689d97a436 Address:ef8f2d53-a97f-bac5-9bb8-73689d97a436} {Suffrage:Voter ID:server-d54c2e9e-a8f5-7fe9-ccf8-c7e051b79b81 Address:d54c2e9e-a8f5-7fe9-ccf8-c7e051b79b81} {Suffrage:Voter ID:server-02eacae1-6000-7bdc-9282-f571429af218 Address:02eacae1-6000-7bdc-9282-f571429af218} {Suffrage:Voter ID:server-48144434-6d83-ece7-b902-7e122583bdaf Address:48144434-6d83-ece7-b902-7e122583bdaf}]" +2025-02-25T23:59:19.294Z [INFO] server-02eacae1-6000-7bdc-9282-f571429af218: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ac7f5eee-e884-e1b8-4d56-14d0ba2bc981 Address:ac7f5eee-e884-e1b8-4d56-14d0ba2bc981} {Suffrage:Voter ID:server-670f6f2f-ed1f-1dcb-7a38-b0d57aa370fe Address:670f6f2f-ed1f-1dcb-7a38-b0d57aa370fe} {Suffrage:Voter ID:server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb Address:6369e5cc-ccc7-a799-6959-e525b1a6e7bb} {Suffrage:Voter ID:server-ef8f2d53-a97f-bac5-9bb8-73689d97a436 Address:ef8f2d53-a97f-bac5-9bb8-73689d97a436} {Suffrage:Voter ID:server-d54c2e9e-a8f5-7fe9-ccf8-c7e051b79b81 Address:d54c2e9e-a8f5-7fe9-ccf8-c7e051b79b81} {Suffrage:Voter ID:server-02eacae1-6000-7bdc-9282-f571429af218 Address:02eacae1-6000-7bdc-9282-f571429af218} {Suffrage:Voter ID:server-48144434-6d83-ece7-b902-7e122583bdaf Address:48144434-6d83-ece7-b902-7e122583bdaf}]" +2025-02-25T23:59:19.295Z [INFO] server-48144434-6d83-ece7-b902-7e122583bdaf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ac7f5eee-e884-e1b8-4d56-14d0ba2bc981 Address:ac7f5eee-e884-e1b8-4d56-14d0ba2bc981} {Suffrage:Voter ID:server-670f6f2f-ed1f-1dcb-7a38-b0d57aa370fe Address:670f6f2f-ed1f-1dcb-7a38-b0d57aa370fe} {Suffrage:Voter ID:server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb Address:6369e5cc-ccc7-a799-6959-e525b1a6e7bb} {Suffrage:Voter ID:server-ef8f2d53-a97f-bac5-9bb8-73689d97a436 Address:ef8f2d53-a97f-bac5-9bb8-73689d97a436} {Suffrage:Voter ID:server-d54c2e9e-a8f5-7fe9-ccf8-c7e051b79b81 Address:d54c2e9e-a8f5-7fe9-ccf8-c7e051b79b81} {Suffrage:Voter ID:server-02eacae1-6000-7bdc-9282-f571429af218 Address:02eacae1-6000-7bdc-9282-f571429af218} {Suffrage:Voter ID:server-48144434-6d83-ece7-b902-7e122583bdaf Address:48144434-6d83-ece7-b902-7e122583bdaf}]" +2025-02-25T23:59:19.295Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:19.295Z [INFO] server-ac7f5eee-e884-e1b8-4d56-14d0ba2bc981: entering follower state: follower="Node at ac7f5eee-e884-e1b8-4d56-14d0ba2bc981 [Follower]" leader-address= leader-id= +2025-02-25T23:59:19.295Z [INFO] server-670f6f2f-ed1f-1dcb-7a38-b0d57aa370fe: entering follower state: follower="Node at 670f6f2f-ed1f-1dcb-7a38-b0d57aa370fe [Follower]" leader-address= leader-id= +2025-02-25T23:59:19.295Z [INFO] server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb: entering follower state: follower="Node at 6369e5cc-ccc7-a799-6959-e525b1a6e7bb [Follower]" leader-address= leader-id= +2025-02-25T23:59:19.295Z [INFO] server-ef8f2d53-a97f-bac5-9bb8-73689d97a436: entering follower state: follower="Node at ef8f2d53-a97f-bac5-9bb8-73689d97a436 [Follower]" leader-address= leader-id= +2025-02-25T23:59:19.295Z [INFO] server-d54c2e9e-a8f5-7fe9-ccf8-c7e051b79b81: entering follower state: follower="Node at d54c2e9e-a8f5-7fe9-ccf8-c7e051b79b81 [Follower]" leader-address= leader-id= +2025-02-25T23:59:19.295Z [INFO] server-02eacae1-6000-7bdc-9282-f571429af218: entering follower state: follower="Node at 02eacae1-6000-7bdc-9282-f571429af218 [Follower]" leader-address= leader-id= +2025-02-25T23:59:19.295Z [INFO] server-48144434-6d83-ece7-b902-7e122583bdaf: entering follower state: follower="Node at 48144434-6d83-ece7-b902-7e122583bdaf [Follower]" leader-address= leader-id= +2025-02-25T23:59:19.349Z [WARN] server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:19.349Z [INFO] server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb: entering candidate state: node="Node at 6369e5cc-ccc7-a799-6959-e525b1a6e7bb [Candidate]" term=2 +2025-02-25T23:59:19.350Z [INFO] server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb: election won: term=2 tally=4 +2025-02-25T23:59:19.350Z [INFO] server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb: entering leader state: leader="Node at 6369e5cc-ccc7-a799-6959-e525b1a6e7bb [Leader]" +2025-02-25T23:59:19.350Z [INFO] server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb: added peer, starting replication: peer=server-ac7f5eee-e884-e1b8-4d56-14d0ba2bc981 +2025-02-25T23:59:19.350Z [INFO] server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb: added peer, starting replication: peer=server-670f6f2f-ed1f-1dcb-7a38-b0d57aa370fe +2025-02-25T23:59:19.350Z [INFO] server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb: added peer, starting replication: peer=server-ef8f2d53-a97f-bac5-9bb8-73689d97a436 +2025-02-25T23:59:19.350Z [INFO] server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb: added peer, starting replication: peer=server-d54c2e9e-a8f5-7fe9-ccf8-c7e051b79b81 +2025-02-25T23:59:19.350Z [INFO] server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb: added peer, starting replication: peer=server-02eacae1-6000-7bdc-9282-f571429af218 +2025-02-25T23:59:19.350Z [INFO] server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb: added peer, starting replication: peer=server-48144434-6d83-ece7-b902-7e122583bdaf +2025-02-25T23:59:19.350Z [INFO] server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb: pipelining replication: peer="{Voter server-48144434-6d83-ece7-b902-7e122583bdaf 48144434-6d83-ece7-b902-7e122583bdaf}" +2025-02-25T23:59:19.350Z [INFO] server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb: pipelining replication: peer="{Voter server-ac7f5eee-e884-e1b8-4d56-14d0ba2bc981 ac7f5eee-e884-e1b8-4d56-14d0ba2bc981}" +2025-02-25T23:59:19.350Z [INFO] server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb: pipelining replication: peer="{Voter server-670f6f2f-ed1f-1dcb-7a38-b0d57aa370fe 670f6f2f-ed1f-1dcb-7a38-b0d57aa370fe}" +2025-02-25T23:59:19.350Z [INFO] server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb: pipelining replication: peer="{Voter server-ef8f2d53-a97f-bac5-9bb8-73689d97a436 ef8f2d53-a97f-bac5-9bb8-73689d97a436}" +2025-02-25T23:59:19.350Z [INFO] server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb: pipelining replication: peer="{Voter server-d54c2e9e-a8f5-7fe9-ccf8-c7e051b79b81 d54c2e9e-a8f5-7fe9-ccf8-c7e051b79b81}" +2025-02-25T23:59:19.350Z [INFO] server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb: pipelining replication: peer="{Voter server-02eacae1-6000-7bdc-9282-f571429af218 02eacae1-6000-7bdc-9282-f571429af218}" +2025-02-25T23:59:19.458Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:19.350313884 +0000 UTC m=+106.959329251 (1 nodes), 55.267032ms from start of poll, 55.675074ms from cluster start. Timeout at 2025-02-25 23:59:19.455316244 +0000 UTC m=+107.064331651, 105.0024ms after stability +2025-02-25T23:59:19.458Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:59:19.566Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:59:19.458545053 +0000 UTC m=+107.067560460 (6 nodes), 1.76µs from start of poll, 163.906283ms from cluster start. Timeout at 2025-02-25 23:59:19.563592133 +0000 UTC m=+107.172607540, 105.04708ms after stability +2025-02-25T23:59:19.566Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:19.671Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:19.566077738 +0000 UTC m=+107.175093145 (1 nodes), 720ns from start of poll, 271.438968ms from cluster start. Timeout at 2025-02-25 23:59:19.671110419 +0000 UTC m=+107.280125826, 105.032681ms after stability +2025-02-25T23:59:19.671Z [INFO] server-ac7f5eee-e884-e1b8-4d56-14d0ba2bc981: entering candidate state: node="Node at ac7f5eee-e884-e1b8-4d56-14d0ba2bc981 [Candidate]" term=3 +2025-02-25T23:59:19.671Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:19.671Z [INFO] server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb: entering follower state: follower="Node at 6369e5cc-ccc7-a799-6959-e525b1a6e7bb [Follower]" leader-address= leader-id= +2025-02-25T23:59:19.671Z [INFO] server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb: aborting pipeline replication: peer="{Voter server-02eacae1-6000-7bdc-9282-f571429af218 02eacae1-6000-7bdc-9282-f571429af218}" +2025-02-25T23:59:19.671Z [INFO] server-ac7f5eee-e884-e1b8-4d56-14d0ba2bc981: election won: term=3 tally=4 +2025-02-25T23:59:19.671Z [INFO] server-ac7f5eee-e884-e1b8-4d56-14d0ba2bc981: entering leader state: leader="Node at ac7f5eee-e884-e1b8-4d56-14d0ba2bc981 [Leader]" +2025-02-25T23:59:19.671Z [INFO] server-ac7f5eee-e884-e1b8-4d56-14d0ba2bc981: added peer, starting replication: peer=server-670f6f2f-ed1f-1dcb-7a38-b0d57aa370fe +2025-02-25T23:59:19.671Z [INFO] server-ac7f5eee-e884-e1b8-4d56-14d0ba2bc981: added peer, starting replication: peer=server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb +2025-02-25T23:59:19.671Z [INFO] server-ac7f5eee-e884-e1b8-4d56-14d0ba2bc981: added peer, starting replication: peer=server-ef8f2d53-a97f-bac5-9bb8-73689d97a436 +2025-02-25T23:59:19.671Z [INFO] server-ac7f5eee-e884-e1b8-4d56-14d0ba2bc981: added peer, starting replication: peer=server-d54c2e9e-a8f5-7fe9-ccf8-c7e051b79b81 +2025-02-25T23:59:19.671Z [INFO] server-ac7f5eee-e884-e1b8-4d56-14d0ba2bc981: added peer, starting replication: peer=server-02eacae1-6000-7bdc-9282-f571429af218 +2025-02-25T23:59:19.671Z [INFO] server-ac7f5eee-e884-e1b8-4d56-14d0ba2bc981: added peer, starting replication: peer=server-48144434-6d83-ece7-b902-7e122583bdaf +2025-02-25T23:59:19.671Z [INFO] server-ac7f5eee-e884-e1b8-4d56-14d0ba2bc981: pipelining replication: peer="{Voter server-48144434-6d83-ece7-b902-7e122583bdaf 48144434-6d83-ece7-b902-7e122583bdaf}" +2025-02-25T23:59:19.671Z [WARN] server-48144434-6d83-ece7-b902-7e122583bdaf: rejecting vote request since our last term is greater: candidate=ac7f5eee-e884-e1b8-4d56-14d0ba2bc981 last-term=3 last-candidate-term=2 +2025-02-25T23:59:19.671Z [INFO] server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb: aborting pipeline replication: peer="{Voter server-48144434-6d83-ece7-b902-7e122583bdaf 48144434-6d83-ece7-b902-7e122583bdaf}" +2025-02-25T23:59:19.671Z [INFO] server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb: aborting pipeline replication: peer="{Voter server-ac7f5eee-e884-e1b8-4d56-14d0ba2bc981 ac7f5eee-e884-e1b8-4d56-14d0ba2bc981}" +2025-02-25T23:59:19.671Z [INFO] server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb: aborting pipeline replication: peer="{Voter server-670f6f2f-ed1f-1dcb-7a38-b0d57aa370fe 670f6f2f-ed1f-1dcb-7a38-b0d57aa370fe}" +2025-02-25T23:59:19.671Z [INFO] server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb: aborting pipeline replication: peer="{Voter server-ef8f2d53-a97f-bac5-9bb8-73689d97a436 ef8f2d53-a97f-bac5-9bb8-73689d97a436}" +2025-02-25T23:59:19.671Z [INFO] server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb: aborting pipeline replication: peer="{Voter server-d54c2e9e-a8f5-7fe9-ccf8-c7e051b79b81 d54c2e9e-a8f5-7fe9-ccf8-c7e051b79b81}" +2025-02-25T23:59:19.671Z [INFO] server-ac7f5eee-e884-e1b8-4d56-14d0ba2bc981: pipelining replication: peer="{Voter server-670f6f2f-ed1f-1dcb-7a38-b0d57aa370fe 670f6f2f-ed1f-1dcb-7a38-b0d57aa370fe}" +2025-02-25T23:59:19.671Z [INFO] server-ac7f5eee-e884-e1b8-4d56-14d0ba2bc981: pipelining replication: peer="{Voter server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb 6369e5cc-ccc7-a799-6959-e525b1a6e7bb}" +2025-02-25T23:59:19.671Z [INFO] server-ac7f5eee-e884-e1b8-4d56-14d0ba2bc981: pipelining replication: peer="{Voter server-ef8f2d53-a97f-bac5-9bb8-73689d97a436 ef8f2d53-a97f-bac5-9bb8-73689d97a436}" +2025-02-25T23:59:19.671Z [INFO] server-ac7f5eee-e884-e1b8-4d56-14d0ba2bc981: pipelining replication: peer="{Voter server-d54c2e9e-a8f5-7fe9-ccf8-c7e051b79b81 d54c2e9e-a8f5-7fe9-ccf8-c7e051b79b81}" +2025-02-25T23:59:19.671Z [INFO] server-ac7f5eee-e884-e1b8-4d56-14d0ba2bc981: pipelining replication: peer="{Voter server-02eacae1-6000-7bdc-9282-f571429af218 02eacae1-6000-7bdc-9282-f571429af218}" +2025-02-25T23:59:19.782Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:19.671608969 +0000 UTC m=+107.280624376 (1 nodes), 284.601µs from start of poll, 376.970199ms from cluster start. Timeout at 2025-02-25 23:59:19.77663885 +0000 UTC m=+107.385654257, 105.029881ms after stability +2025-02-25T23:59:19.782Z [INFO] server-ac7f5eee-e884-e1b8-4d56-14d0ba2bc981: aborting pipeline replication: peer="{Voter server-48144434-6d83-ece7-b902-7e122583bdaf 48144434-6d83-ece7-b902-7e122583bdaf}" +2025-02-25T23:59:19.782Z [INFO] server-ac7f5eee-e884-e1b8-4d56-14d0ba2bc981: aborting pipeline replication: peer="{Voter server-670f6f2f-ed1f-1dcb-7a38-b0d57aa370fe 670f6f2f-ed1f-1dcb-7a38-b0d57aa370fe}" +2025-02-25T23:59:19.782Z [INFO] server-ac7f5eee-e884-e1b8-4d56-14d0ba2bc981: aborting pipeline replication: peer="{Voter server-6369e5cc-ccc7-a799-6959-e525b1a6e7bb 6369e5cc-ccc7-a799-6959-e525b1a6e7bb}" +2025-02-25T23:59:19.783Z [INFO] server-ac7f5eee-e884-e1b8-4d56-14d0ba2bc981: aborting pipeline replication: peer="{Voter server-ef8f2d53-a97f-bac5-9bb8-73689d97a436 ef8f2d53-a97f-bac5-9bb8-73689d97a436}" +2025-02-25T23:59:19.783Z [INFO] server-ac7f5eee-e884-e1b8-4d56-14d0ba2bc981: aborting pipeline replication: peer="{Voter server-d54c2e9e-a8f5-7fe9-ccf8-c7e051b79b81 d54c2e9e-a8f5-7fe9-ccf8-c7e051b79b81}" +2025-02-25T23:59:19.783Z [INFO] server-ac7f5eee-e884-e1b8-4d56-14d0ba2bc981: aborting pipeline replication: peer="{Voter server-02eacae1-6000-7bdc-9282-f571429af218 02eacae1-6000-7bdc-9282-f571429af218}" +--- PASS: TestRaft_LeadershipTransferWithSevenNodes (0.49s) === RUN TestRaft_LeadershipTransferToInvalidID -2026-03-31T06:17:56.317Z [INFO] server-7308c39c-024d-0376-de4b-a2770ef69f2a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7308c39c-024d-0376-de4b-a2770ef69f2a Address:7308c39c-024d-0376-de4b-a2770ef69f2a} {Suffrage:Voter ID:server-a02a1841-0706-856f-0dd0-97446733c13f Address:a02a1841-0706-856f-0dd0-97446733c13f} {Suffrage:Voter ID:server-59d494bb-b3db-67e0-91ae-161345bcab27 Address:59d494bb-b3db-67e0-91ae-161345bcab27}]" -2026-03-31T06:17:56.317Z [INFO] server-a02a1841-0706-856f-0dd0-97446733c13f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7308c39c-024d-0376-de4b-a2770ef69f2a Address:7308c39c-024d-0376-de4b-a2770ef69f2a} {Suffrage:Voter ID:server-a02a1841-0706-856f-0dd0-97446733c13f Address:a02a1841-0706-856f-0dd0-97446733c13f} {Suffrage:Voter ID:server-59d494bb-b3db-67e0-91ae-161345bcab27 Address:59d494bb-b3db-67e0-91ae-161345bcab27}]" -2026-03-31T06:17:56.317Z [INFO] server-7308c39c-024d-0376-de4b-a2770ef69f2a: entering follower state: follower="Node at 7308c39c-024d-0376-de4b-a2770ef69f2a [Follower]" leader-address= leader-id= -2026-03-31T06:17:56.317Z [INFO] server-a02a1841-0706-856f-0dd0-97446733c13f: entering follower state: follower="Node at a02a1841-0706-856f-0dd0-97446733c13f [Follower]" leader-address= leader-id= -2026-03-31T06:17:56.317Z [INFO] server-59d494bb-b3db-67e0-91ae-161345bcab27: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7308c39c-024d-0376-de4b-a2770ef69f2a Address:7308c39c-024d-0376-de4b-a2770ef69f2a} {Suffrage:Voter ID:server-a02a1841-0706-856f-0dd0-97446733c13f Address:a02a1841-0706-856f-0dd0-97446733c13f} {Suffrage:Voter ID:server-59d494bb-b3db-67e0-91ae-161345bcab27 Address:59d494bb-b3db-67e0-91ae-161345bcab27}]" -2026-03-31T06:17:56.317Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:56.317Z [INFO] server-59d494bb-b3db-67e0-91ae-161345bcab27: entering follower state: follower="Node at 59d494bb-b3db-67e0-91ae-161345bcab27 [Follower]" leader-address= leader-id= -2026-03-31T06:17:56.401Z [WARN] server-59d494bb-b3db-67e0-91ae-161345bcab27: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:56.401Z [INFO] server-59d494bb-b3db-67e0-91ae-161345bcab27: entering candidate state: node="Node at 59d494bb-b3db-67e0-91ae-161345bcab27 [Candidate]" term=2 -2026-03-31T06:17:56.401Z [INFO] server-59d494bb-b3db-67e0-91ae-161345bcab27: election won: term=2 tally=2 -2026-03-31T06:17:56.401Z [INFO] server-59d494bb-b3db-67e0-91ae-161345bcab27: entering leader state: leader="Node at 59d494bb-b3db-67e0-91ae-161345bcab27 [Leader]" -2026-03-31T06:17:56.401Z [INFO] server-59d494bb-b3db-67e0-91ae-161345bcab27: added peer, starting replication: peer=server-7308c39c-024d-0376-de4b-a2770ef69f2a -2026-03-31T06:17:56.401Z [INFO] server-59d494bb-b3db-67e0-91ae-161345bcab27: added peer, starting replication: peer=server-a02a1841-0706-856f-0dd0-97446733c13f -2026-03-31T06:17:56.401Z [INFO] server-59d494bb-b3db-67e0-91ae-161345bcab27: pipelining replication: peer="{Voter server-7308c39c-024d-0376-de4b-a2770ef69f2a 7308c39c-024d-0376-de4b-a2770ef69f2a}" -2026-03-31T06:17:56.401Z [INFO] server-59d494bb-b3db-67e0-91ae-161345bcab27: pipelining replication: peer="{Voter server-a02a1841-0706-856f-0dd0-97446733c13f a02a1841-0706-856f-0dd0-97446733c13f}" -2026-03-31T06:17:56.506Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:56.40129838 +0000 UTC m=+104.678905761 (1 nodes), 83.9126ms from start of poll, 84.34808ms from cluster start. Timeout at 2026-03-31 06:17:56.5063025 +0000 UTC m=+104.783909881, 105.00412ms after stability -2026-03-31T06:17:56.506Z [INFO] server-59d494bb-b3db-67e0-91ae-161345bcab27: aborting pipeline replication: peer="{Voter server-a02a1841-0706-856f-0dd0-97446733c13f a02a1841-0706-856f-0dd0-97446733c13f}" -2026-03-31T06:17:56.506Z [INFO] server-59d494bb-b3db-67e0-91ae-161345bcab27: aborting pipeline replication: peer="{Voter server-7308c39c-024d-0376-de4b-a2770ef69f2a 7308c39c-024d-0376-de4b-a2770ef69f2a}" ---- PASS: TestRaft_LeadershipTransferToInvalidID (0.19s) +2025-02-25T23:59:19.783Z [INFO] server-7ad3f2a1-17d1-5d19-1af6-7428c3bc7061: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7ad3f2a1-17d1-5d19-1af6-7428c3bc7061 Address:7ad3f2a1-17d1-5d19-1af6-7428c3bc7061} {Suffrage:Voter ID:server-aa451bac-dc83-9a70-1db1-aa98813e6f5f Address:aa451bac-dc83-9a70-1db1-aa98813e6f5f} {Suffrage:Voter ID:server-8aa65b28-8bff-c99b-3c84-4e636fcf7767 Address:8aa65b28-8bff-c99b-3c84-4e636fcf7767}]" +2025-02-25T23:59:19.783Z [INFO] server-aa451bac-dc83-9a70-1db1-aa98813e6f5f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7ad3f2a1-17d1-5d19-1af6-7428c3bc7061 Address:7ad3f2a1-17d1-5d19-1af6-7428c3bc7061} {Suffrage:Voter ID:server-aa451bac-dc83-9a70-1db1-aa98813e6f5f Address:aa451bac-dc83-9a70-1db1-aa98813e6f5f} {Suffrage:Voter ID:server-8aa65b28-8bff-c99b-3c84-4e636fcf7767 Address:8aa65b28-8bff-c99b-3c84-4e636fcf7767}]" +2025-02-25T23:59:19.783Z [INFO] server-8aa65b28-8bff-c99b-3c84-4e636fcf7767: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7ad3f2a1-17d1-5d19-1af6-7428c3bc7061 Address:7ad3f2a1-17d1-5d19-1af6-7428c3bc7061} {Suffrage:Voter ID:server-aa451bac-dc83-9a70-1db1-aa98813e6f5f Address:aa451bac-dc83-9a70-1db1-aa98813e6f5f} {Suffrage:Voter ID:server-8aa65b28-8bff-c99b-3c84-4e636fcf7767 Address:8aa65b28-8bff-c99b-3c84-4e636fcf7767}]" +2025-02-25T23:59:19.783Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:19.783Z [INFO] server-7ad3f2a1-17d1-5d19-1af6-7428c3bc7061: entering follower state: follower="Node at 7ad3f2a1-17d1-5d19-1af6-7428c3bc7061 [Follower]" leader-address= leader-id= +2025-02-25T23:59:19.783Z [INFO] server-aa451bac-dc83-9a70-1db1-aa98813e6f5f: entering follower state: follower="Node at aa451bac-dc83-9a70-1db1-aa98813e6f5f [Follower]" leader-address= leader-id= +2025-02-25T23:59:19.783Z [INFO] server-8aa65b28-8bff-c99b-3c84-4e636fcf7767: entering follower state: follower="Node at 8aa65b28-8bff-c99b-3c84-4e636fcf7767 [Follower]" leader-address= leader-id= +2025-02-25T23:59:19.842Z [WARN] server-8aa65b28-8bff-c99b-3c84-4e636fcf7767: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:19.842Z [INFO] server-8aa65b28-8bff-c99b-3c84-4e636fcf7767: entering candidate state: node="Node at 8aa65b28-8bff-c99b-3c84-4e636fcf7767 [Candidate]" term=2 +2025-02-25T23:59:19.842Z [INFO] server-8aa65b28-8bff-c99b-3c84-4e636fcf7767: election won: term=2 tally=2 +2025-02-25T23:59:19.842Z [INFO] server-8aa65b28-8bff-c99b-3c84-4e636fcf7767: entering leader state: leader="Node at 8aa65b28-8bff-c99b-3c84-4e636fcf7767 [Leader]" +2025-02-25T23:59:19.842Z [INFO] server-8aa65b28-8bff-c99b-3c84-4e636fcf7767: added peer, starting replication: peer=server-7ad3f2a1-17d1-5d19-1af6-7428c3bc7061 +2025-02-25T23:59:19.842Z [INFO] server-8aa65b28-8bff-c99b-3c84-4e636fcf7767: added peer, starting replication: peer=server-aa451bac-dc83-9a70-1db1-aa98813e6f5f +2025-02-25T23:59:19.842Z [INFO] server-8aa65b28-8bff-c99b-3c84-4e636fcf7767: pipelining replication: peer="{Voter server-aa451bac-dc83-9a70-1db1-aa98813e6f5f aa451bac-dc83-9a70-1db1-aa98813e6f5f}" +2025-02-25T23:59:19.842Z [INFO] server-8aa65b28-8bff-c99b-3c84-4e636fcf7767: pipelining replication: peer="{Voter server-7ad3f2a1-17d1-5d19-1af6-7428c3bc7061 7ad3f2a1-17d1-5d19-1af6-7428c3bc7061}" +2025-02-25T23:59:19.954Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:19.842502017 +0000 UTC m=+107.451517424 (1 nodes), 58.799014ms from start of poll, 59.017575ms from cluster start. Timeout at 2025-02-25 23:59:19.947506937 +0000 UTC m=+107.556522304, 105.00488ms after stability +2025-02-25T23:59:19.954Z [INFO] server-8aa65b28-8bff-c99b-3c84-4e636fcf7767: aborting pipeline replication: peer="{Voter server-aa451bac-dc83-9a70-1db1-aa98813e6f5f aa451bac-dc83-9a70-1db1-aa98813e6f5f}" +2025-02-25T23:59:19.954Z [INFO] server-8aa65b28-8bff-c99b-3c84-4e636fcf7767: aborting pipeline replication: peer="{Voter server-7ad3f2a1-17d1-5d19-1af6-7428c3bc7061 7ad3f2a1-17d1-5d19-1af6-7428c3bc7061}" +2025-02-25T23:59:20.455Z [ERROR] server-8aa65b28-8bff-c99b-3c84-4e636fcf7767: failed to heartbeat to: peer=aa451bac-dc83-9a70-1db1-aa98813e6f5f backoff time=10ms error="command timed out" +--- PASS: TestRaft_LeadershipTransferToInvalidID (0.67s) === RUN TestRaft_LeadershipTransferToInvalidAddress -2026-03-31T06:17:56.507Z [INFO] server-530e477f-8a4c-247a-d538-5773a5ec89a8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-530e477f-8a4c-247a-d538-5773a5ec89a8 Address:530e477f-8a4c-247a-d538-5773a5ec89a8} {Suffrage:Voter ID:server-19ceaed7-d136-909d-247c-48be09838e4e Address:19ceaed7-d136-909d-247c-48be09838e4e} {Suffrage:Voter ID:server-c7d91836-612a-3bfa-4219-9137517ab460 Address:c7d91836-612a-3bfa-4219-9137517ab460}]" -2026-03-31T06:17:56.507Z [INFO] server-19ceaed7-d136-909d-247c-48be09838e4e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-530e477f-8a4c-247a-d538-5773a5ec89a8 Address:530e477f-8a4c-247a-d538-5773a5ec89a8} {Suffrage:Voter ID:server-19ceaed7-d136-909d-247c-48be09838e4e Address:19ceaed7-d136-909d-247c-48be09838e4e} {Suffrage:Voter ID:server-c7d91836-612a-3bfa-4219-9137517ab460 Address:c7d91836-612a-3bfa-4219-9137517ab460}]" -2026-03-31T06:17:56.507Z [INFO] server-530e477f-8a4c-247a-d538-5773a5ec89a8: entering follower state: follower="Node at 530e477f-8a4c-247a-d538-5773a5ec89a8 [Follower]" leader-address= leader-id= -2026-03-31T06:17:56.508Z [INFO] server-c7d91836-612a-3bfa-4219-9137517ab460: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-530e477f-8a4c-247a-d538-5773a5ec89a8 Address:530e477f-8a4c-247a-d538-5773a5ec89a8} {Suffrage:Voter ID:server-19ceaed7-d136-909d-247c-48be09838e4e Address:19ceaed7-d136-909d-247c-48be09838e4e} {Suffrage:Voter ID:server-c7d91836-612a-3bfa-4219-9137517ab460 Address:c7d91836-612a-3bfa-4219-9137517ab460}]" -2026-03-31T06:17:56.508Z [INFO] server-19ceaed7-d136-909d-247c-48be09838e4e: entering follower state: follower="Node at 19ceaed7-d136-909d-247c-48be09838e4e [Follower]" leader-address= leader-id= -2026-03-31T06:17:56.508Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:17:56.508Z [INFO] server-c7d91836-612a-3bfa-4219-9137517ab460: entering follower state: follower="Node at c7d91836-612a-3bfa-4219-9137517ab460 [Follower]" leader-address= leader-id= -2026-03-31T06:17:56.569Z [WARN] server-19ceaed7-d136-909d-247c-48be09838e4e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:56.569Z [INFO] server-19ceaed7-d136-909d-247c-48be09838e4e: entering candidate state: node="Node at 19ceaed7-d136-909d-247c-48be09838e4e [Candidate]" term=2 -2026-03-31T06:17:56.570Z [INFO] server-19ceaed7-d136-909d-247c-48be09838e4e: election won: term=2 tally=2 -2026-03-31T06:17:56.570Z [INFO] server-19ceaed7-d136-909d-247c-48be09838e4e: entering leader state: leader="Node at 19ceaed7-d136-909d-247c-48be09838e4e [Leader]" -2026-03-31T06:17:56.570Z [INFO] server-19ceaed7-d136-909d-247c-48be09838e4e: added peer, starting replication: peer=server-530e477f-8a4c-247a-d538-5773a5ec89a8 -2026-03-31T06:17:56.570Z [INFO] server-19ceaed7-d136-909d-247c-48be09838e4e: added peer, starting replication: peer=server-c7d91836-612a-3bfa-4219-9137517ab460 -2026-03-31T06:17:56.570Z [INFO] server-19ceaed7-d136-909d-247c-48be09838e4e: pipelining replication: peer="{Voter server-c7d91836-612a-3bfa-4219-9137517ab460 c7d91836-612a-3bfa-4219-9137517ab460}" -2026-03-31T06:17:56.570Z [INFO] server-19ceaed7-d136-909d-247c-48be09838e4e: pipelining replication: peer="{Voter server-530e477f-8a4c-247a-d538-5773a5ec89a8 530e477f-8a4c-247a-d538-5773a5ec89a8}" -2026-03-31T06:17:56.676Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:17:56.57008306 +0000 UTC m=+104.847690481 (2 nodes), 61.95264ms from start of poll, 62.40308ms from cluster start. Timeout at 2026-03-31 06:17:56.6750867 +0000 UTC m=+104.952694081, 105.0036ms after stability -2026-03-31T06:17:56.676Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:56.781Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:56.67615486 +0000 UTC m=+104.953762281 (1 nodes), 2.68µs from start of poll, 168.47488ms from cluster start. Timeout at 2026-03-31 06:17:56.7811755 +0000 UTC m=+105.058782881, 105.0206ms after stability -2026-03-31T06:17:56.782Z [INFO] server-19ceaed7-d136-909d-247c-48be09838e4e: aborting pipeline replication: peer="{Voter server-c7d91836-612a-3bfa-4219-9137517ab460 c7d91836-612a-3bfa-4219-9137517ab460}" -2026-03-31T06:17:56.782Z [INFO] server-19ceaed7-d136-909d-247c-48be09838e4e: aborting pipeline replication: peer="{Voter server-530e477f-8a4c-247a-d538-5773a5ec89a8 530e477f-8a4c-247a-d538-5773a5ec89a8}" ---- PASS: TestRaft_LeadershipTransferToInvalidAddress (0.28s) +2025-02-25T23:59:20.455Z [INFO] server-f9f8f2f0-f370-aea0-23ea-4023dabdf66e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f9f8f2f0-f370-aea0-23ea-4023dabdf66e Address:f9f8f2f0-f370-aea0-23ea-4023dabdf66e} {Suffrage:Voter ID:server-8544ce92-c041-a1c2-fb88-bedc46dd31a8 Address:8544ce92-c041-a1c2-fb88-bedc46dd31a8} {Suffrage:Voter ID:server-de07b8e2-d62e-64de-9527-886088a63044 Address:de07b8e2-d62e-64de-9527-886088a63044}]" +2025-02-25T23:59:20.455Z [INFO] server-8544ce92-c041-a1c2-fb88-bedc46dd31a8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f9f8f2f0-f370-aea0-23ea-4023dabdf66e Address:f9f8f2f0-f370-aea0-23ea-4023dabdf66e} {Suffrage:Voter ID:server-8544ce92-c041-a1c2-fb88-bedc46dd31a8 Address:8544ce92-c041-a1c2-fb88-bedc46dd31a8} {Suffrage:Voter ID:server-de07b8e2-d62e-64de-9527-886088a63044 Address:de07b8e2-d62e-64de-9527-886088a63044}]" +2025-02-25T23:59:20.455Z [INFO] server-de07b8e2-d62e-64de-9527-886088a63044: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f9f8f2f0-f370-aea0-23ea-4023dabdf66e Address:f9f8f2f0-f370-aea0-23ea-4023dabdf66e} {Suffrage:Voter ID:server-8544ce92-c041-a1c2-fb88-bedc46dd31a8 Address:8544ce92-c041-a1c2-fb88-bedc46dd31a8} {Suffrage:Voter ID:server-de07b8e2-d62e-64de-9527-886088a63044 Address:de07b8e2-d62e-64de-9527-886088a63044}]" +2025-02-25T23:59:20.455Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:59:20.455Z [INFO] server-f9f8f2f0-f370-aea0-23ea-4023dabdf66e: entering follower state: follower="Node at f9f8f2f0-f370-aea0-23ea-4023dabdf66e [Follower]" leader-address= leader-id= +2025-02-25T23:59:20.455Z [INFO] server-8544ce92-c041-a1c2-fb88-bedc46dd31a8: entering follower state: follower="Node at 8544ce92-c041-a1c2-fb88-bedc46dd31a8 [Follower]" leader-address= leader-id= +2025-02-25T23:59:20.455Z [INFO] server-de07b8e2-d62e-64de-9527-886088a63044: entering follower state: follower="Node at de07b8e2-d62e-64de-9527-886088a63044 [Follower]" leader-address= leader-id= +2025-02-25T23:59:20.509Z [WARN] server-8544ce92-c041-a1c2-fb88-bedc46dd31a8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:20.509Z [INFO] server-8544ce92-c041-a1c2-fb88-bedc46dd31a8: entering candidate state: node="Node at 8544ce92-c041-a1c2-fb88-bedc46dd31a8 [Candidate]" term=2 +2025-02-25T23:59:20.509Z [INFO] server-8544ce92-c041-a1c2-fb88-bedc46dd31a8: election won: term=2 tally=2 +2025-02-25T23:59:20.509Z [INFO] server-8544ce92-c041-a1c2-fb88-bedc46dd31a8: entering leader state: leader="Node at 8544ce92-c041-a1c2-fb88-bedc46dd31a8 [Leader]" +2025-02-25T23:59:20.509Z [INFO] server-8544ce92-c041-a1c2-fb88-bedc46dd31a8: added peer, starting replication: peer=server-f9f8f2f0-f370-aea0-23ea-4023dabdf66e +2025-02-25T23:59:20.509Z [INFO] server-8544ce92-c041-a1c2-fb88-bedc46dd31a8: added peer, starting replication: peer=server-de07b8e2-d62e-64de-9527-886088a63044 +2025-02-25T23:59:20.509Z [INFO] server-8544ce92-c041-a1c2-fb88-bedc46dd31a8: pipelining replication: peer="{Voter server-de07b8e2-d62e-64de-9527-886088a63044 de07b8e2-d62e-64de-9527-886088a63044}" +2025-02-25T23:59:20.509Z [INFO] server-8544ce92-c041-a1c2-fb88-bedc46dd31a8: pipelining replication: peer="{Voter server-f9f8f2f0-f370-aea0-23ea-4023dabdf66e f9f8f2f0-f370-aea0-23ea-4023dabdf66e}" +2025-02-25T23:59:20.618Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:59:20.509538104 +0000 UTC m=+108.118553511 (2 nodes), 53.917184ms from start of poll, 54.103625ms from cluster start. Timeout at 2025-02-25 23:59:20.614539664 +0000 UTC m=+108.223555071, 105.00156ms after stability +2025-02-25T23:59:20.618Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:20.724Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:20.618274276 +0000 UTC m=+108.227289643 (1 nodes), 1.16µs from start of poll, 162.839757ms from cluster start. Timeout at 2025-02-25 23:59:20.723316396 +0000 UTC m=+108.332331803, 105.04216ms after stability +2025-02-25T23:59:20.724Z [INFO] server-8544ce92-c041-a1c2-fb88-bedc46dd31a8: aborting pipeline replication: peer="{Voter server-f9f8f2f0-f370-aea0-23ea-4023dabdf66e f9f8f2f0-f370-aea0-23ea-4023dabdf66e}" +2025-02-25T23:59:20.724Z [INFO] server-8544ce92-c041-a1c2-fb88-bedc46dd31a8: aborting pipeline replication: peer="{Voter server-de07b8e2-d62e-64de-9527-886088a63044 de07b8e2-d62e-64de-9527-886088a63044}" +--- PASS: TestRaft_LeadershipTransferToInvalidAddress (0.27s) === RUN TestRaft_LeadershipTransferToBehindServer -2026-03-31T06:17:56.782Z [INFO] server-410a994b-6a3b-2b28-0bde-68e060f39212: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-410a994b-6a3b-2b28-0bde-68e060f39212 Address:410a994b-6a3b-2b28-0bde-68e060f39212} {Suffrage:Voter ID:server-fdf78898-f326-6af3-f60e-508402f26571 Address:fdf78898-f326-6af3-f60e-508402f26571} {Suffrage:Voter ID:server-b4b0ba40-536d-ab91-160d-32af43e4f87d Address:b4b0ba40-536d-ab91-160d-32af43e4f87d}]" -2026-03-31T06:17:56.783Z [INFO] server-fdf78898-f326-6af3-f60e-508402f26571: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-410a994b-6a3b-2b28-0bde-68e060f39212 Address:410a994b-6a3b-2b28-0bde-68e060f39212} {Suffrage:Voter ID:server-fdf78898-f326-6af3-f60e-508402f26571 Address:fdf78898-f326-6af3-f60e-508402f26571} {Suffrage:Voter ID:server-b4b0ba40-536d-ab91-160d-32af43e4f87d Address:b4b0ba40-536d-ab91-160d-32af43e4f87d}]" -2026-03-31T06:17:56.783Z [INFO] server-410a994b-6a3b-2b28-0bde-68e060f39212: entering follower state: follower="Node at 410a994b-6a3b-2b28-0bde-68e060f39212 [Follower]" leader-address= leader-id= -2026-03-31T06:17:56.783Z [INFO] server-fdf78898-f326-6af3-f60e-508402f26571: entering follower state: follower="Node at fdf78898-f326-6af3-f60e-508402f26571 [Follower]" leader-address= leader-id= -2026-03-31T06:17:56.783Z [INFO] server-b4b0ba40-536d-ab91-160d-32af43e4f87d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-410a994b-6a3b-2b28-0bde-68e060f39212 Address:410a994b-6a3b-2b28-0bde-68e060f39212} {Suffrage:Voter ID:server-fdf78898-f326-6af3-f60e-508402f26571 Address:fdf78898-f326-6af3-f60e-508402f26571} {Suffrage:Voter ID:server-b4b0ba40-536d-ab91-160d-32af43e4f87d Address:b4b0ba40-536d-ab91-160d-32af43e4f87d}]" -2026-03-31T06:17:56.783Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:56.783Z [INFO] server-b4b0ba40-536d-ab91-160d-32af43e4f87d: entering follower state: follower="Node at b4b0ba40-536d-ab91-160d-32af43e4f87d [Follower]" leader-address= leader-id= -2026-03-31T06:17:56.844Z [WARN] server-fdf78898-f326-6af3-f60e-508402f26571: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:56.844Z [INFO] server-fdf78898-f326-6af3-f60e-508402f26571: entering candidate state: node="Node at fdf78898-f326-6af3-f60e-508402f26571 [Candidate]" term=2 -2026-03-31T06:17:56.844Z [INFO] server-fdf78898-f326-6af3-f60e-508402f26571: election won: term=2 tally=2 -2026-03-31T06:17:56.844Z [INFO] server-fdf78898-f326-6af3-f60e-508402f26571: entering leader state: leader="Node at fdf78898-f326-6af3-f60e-508402f26571 [Leader]" -2026-03-31T06:17:56.844Z [INFO] server-fdf78898-f326-6af3-f60e-508402f26571: added peer, starting replication: peer=server-410a994b-6a3b-2b28-0bde-68e060f39212 -2026-03-31T06:17:56.844Z [INFO] server-fdf78898-f326-6af3-f60e-508402f26571: added peer, starting replication: peer=server-b4b0ba40-536d-ab91-160d-32af43e4f87d -2026-03-31T06:17:56.845Z [INFO] server-fdf78898-f326-6af3-f60e-508402f26571: pipelining replication: peer="{Voter server-410a994b-6a3b-2b28-0bde-68e060f39212 410a994b-6a3b-2b28-0bde-68e060f39212}" -2026-03-31T06:17:56.845Z [INFO] server-fdf78898-f326-6af3-f60e-508402f26571: pipelining replication: peer="{Voter server-b4b0ba40-536d-ab91-160d-32af43e4f87d b4b0ba40-536d-ab91-160d-32af43e4f87d}" -2026-03-31T06:17:56.949Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:56.84491154 +0000 UTC m=+105.122518961 (1 nodes), 61.65216ms from start of poll, 62.0702ms from cluster start. Timeout at 2026-03-31 06:17:56.94991474 +0000 UTC m=+105.227522121, 105.00316ms after stability -2026-03-31T06:17:56.949Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:17:57.055Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:17:56.9500099 +0000 UTC m=+105.227617281 (2 nodes), 1.28µs from start of poll, 167.16852ms from cluster start. Timeout at 2026-03-31 06:17:57.05502306 +0000 UTC m=+105.332630441, 105.01316ms after stability -2026-03-31T06:17:57.106Z [INFO] server-410a994b-6a3b-2b28-0bde-68e060f39212: entering candidate state: node="Node at 410a994b-6a3b-2b28-0bde-68e060f39212 [Candidate]" term=3 -2026-03-31T06:17:57.106Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:57.107Z [INFO] server-410a994b-6a3b-2b28-0bde-68e060f39212: election won: term=3 tally=2 -2026-03-31T06:17:57.107Z [INFO] server-410a994b-6a3b-2b28-0bde-68e060f39212: entering leader state: leader="Node at 410a994b-6a3b-2b28-0bde-68e060f39212 [Leader]" -2026-03-31T06:17:57.107Z [INFO] server-410a994b-6a3b-2b28-0bde-68e060f39212: added peer, starting replication: peer=server-fdf78898-f326-6af3-f60e-508402f26571 -2026-03-31T06:17:57.107Z [INFO] server-410a994b-6a3b-2b28-0bde-68e060f39212: added peer, starting replication: peer=server-b4b0ba40-536d-ab91-160d-32af43e4f87d -2026-03-31T06:17:57.107Z [INFO] server-fdf78898-f326-6af3-f60e-508402f26571: entering follower state: follower="Node at fdf78898-f326-6af3-f60e-508402f26571 [Follower]" leader-address= leader-id= -2026-03-31T06:17:57.107Z [INFO] server-fdf78898-f326-6af3-f60e-508402f26571: aborting pipeline replication: peer="{Voter server-410a994b-6a3b-2b28-0bde-68e060f39212 410a994b-6a3b-2b28-0bde-68e060f39212}" -2026-03-31T06:17:57.107Z [INFO] server-fdf78898-f326-6af3-f60e-508402f26571: aborting pipeline replication: peer="{Voter server-b4b0ba40-536d-ab91-160d-32af43e4f87d b4b0ba40-536d-ab91-160d-32af43e4f87d}" -2026-03-31T06:17:57.107Z [INFO] server-410a994b-6a3b-2b28-0bde-68e060f39212: pipelining replication: peer="{Voter server-fdf78898-f326-6af3-f60e-508402f26571 fdf78898-f326-6af3-f60e-508402f26571}" -2026-03-31T06:17:57.107Z [INFO] server-410a994b-6a3b-2b28-0bde-68e060f39212: pipelining replication: peer="{Voter server-b4b0ba40-536d-ab91-160d-32af43e4f87d b4b0ba40-536d-ab91-160d-32af43e4f87d}" -2026-03-31T06:17:57.213Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:57.1071569 +0000 UTC m=+105.384764281 (1 nodes), 160.6µs from start of poll, 324.31552ms from cluster start. Timeout at 2026-03-31 06:17:57.2122197 +0000 UTC m=+105.489827121, 105.06284ms after stability -2026-03-31T06:17:57.213Z [INFO] server-410a994b-6a3b-2b28-0bde-68e060f39212: aborting pipeline replication: peer="{Voter server-b4b0ba40-536d-ab91-160d-32af43e4f87d b4b0ba40-536d-ab91-160d-32af43e4f87d}" -2026-03-31T06:17:57.213Z [INFO] server-410a994b-6a3b-2b28-0bde-68e060f39212: aborting pipeline replication: peer="{Voter server-fdf78898-f326-6af3-f60e-508402f26571 fdf78898-f326-6af3-f60e-508402f26571}" ---- PASS: TestRaft_LeadershipTransferToBehindServer (0.43s) +2025-02-25T23:59:20.725Z [INFO] server-02d56729-0e9d-1cf1-c96b-349f662c42c2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-02d56729-0e9d-1cf1-c96b-349f662c42c2 Address:02d56729-0e9d-1cf1-c96b-349f662c42c2} {Suffrage:Voter ID:server-e2e94cf3-0154-bad3-0c18-41ee1ae722b9 Address:e2e94cf3-0154-bad3-0c18-41ee1ae722b9} {Suffrage:Voter ID:server-de40d4f9-4324-19e2-f2d5-76de3c61c1dd Address:de40d4f9-4324-19e2-f2d5-76de3c61c1dd}]" +2025-02-25T23:59:20.725Z [INFO] server-e2e94cf3-0154-bad3-0c18-41ee1ae722b9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-02d56729-0e9d-1cf1-c96b-349f662c42c2 Address:02d56729-0e9d-1cf1-c96b-349f662c42c2} {Suffrage:Voter ID:server-e2e94cf3-0154-bad3-0c18-41ee1ae722b9 Address:e2e94cf3-0154-bad3-0c18-41ee1ae722b9} {Suffrage:Voter ID:server-de40d4f9-4324-19e2-f2d5-76de3c61c1dd Address:de40d4f9-4324-19e2-f2d5-76de3c61c1dd}]" +2025-02-25T23:59:20.725Z [INFO] server-de40d4f9-4324-19e2-f2d5-76de3c61c1dd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-02d56729-0e9d-1cf1-c96b-349f662c42c2 Address:02d56729-0e9d-1cf1-c96b-349f662c42c2} {Suffrage:Voter ID:server-e2e94cf3-0154-bad3-0c18-41ee1ae722b9 Address:e2e94cf3-0154-bad3-0c18-41ee1ae722b9} {Suffrage:Voter ID:server-de40d4f9-4324-19e2-f2d5-76de3c61c1dd Address:de40d4f9-4324-19e2-f2d5-76de3c61c1dd}]" +2025-02-25T23:59:20.725Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:20.725Z [INFO] server-02d56729-0e9d-1cf1-c96b-349f662c42c2: entering follower state: follower="Node at 02d56729-0e9d-1cf1-c96b-349f662c42c2 [Follower]" leader-address= leader-id= +2025-02-25T23:59:20.725Z [INFO] server-e2e94cf3-0154-bad3-0c18-41ee1ae722b9: entering follower state: follower="Node at e2e94cf3-0154-bad3-0c18-41ee1ae722b9 [Follower]" leader-address= leader-id= +2025-02-25T23:59:20.725Z [INFO] server-de40d4f9-4324-19e2-f2d5-76de3c61c1dd: entering follower state: follower="Node at de40d4f9-4324-19e2-f2d5-76de3c61c1dd [Follower]" leader-address= leader-id= +2025-02-25T23:59:20.786Z [WARN] server-de40d4f9-4324-19e2-f2d5-76de3c61c1dd: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:20.786Z [INFO] server-de40d4f9-4324-19e2-f2d5-76de3c61c1dd: entering candidate state: node="Node at de40d4f9-4324-19e2-f2d5-76de3c61c1dd [Candidate]" term=2 +2025-02-25T23:59:20.786Z [INFO] server-de40d4f9-4324-19e2-f2d5-76de3c61c1dd: election won: term=2 tally=2 +2025-02-25T23:59:20.786Z [INFO] server-de40d4f9-4324-19e2-f2d5-76de3c61c1dd: entering leader state: leader="Node at de40d4f9-4324-19e2-f2d5-76de3c61c1dd [Leader]" +2025-02-25T23:59:20.786Z [INFO] server-de40d4f9-4324-19e2-f2d5-76de3c61c1dd: added peer, starting replication: peer=server-02d56729-0e9d-1cf1-c96b-349f662c42c2 +2025-02-25T23:59:20.786Z [INFO] server-de40d4f9-4324-19e2-f2d5-76de3c61c1dd: added peer, starting replication: peer=server-e2e94cf3-0154-bad3-0c18-41ee1ae722b9 +2025-02-25T23:59:20.786Z [INFO] server-de40d4f9-4324-19e2-f2d5-76de3c61c1dd: pipelining replication: peer="{Voter server-e2e94cf3-0154-bad3-0c18-41ee1ae722b9 e2e94cf3-0154-bad3-0c18-41ee1ae722b9}" +2025-02-25T23:59:20.786Z [INFO] server-de40d4f9-4324-19e2-f2d5-76de3c61c1dd: pipelining replication: peer="{Voter server-02d56729-0e9d-1cf1-c96b-349f662c42c2 02d56729-0e9d-1cf1-c96b-349f662c42c2}" +2025-02-25T23:59:20.898Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:20.786548787 +0000 UTC m=+108.395564194 (1 nodes), 61.34047ms from start of poll, 61.509431ms from cluster start. Timeout at 2025-02-25 23:59:20.891575867 +0000 UTC m=+108.500591274, 105.02708ms after stability +2025-02-25T23:59:20.898Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:59:21.006Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:59:20.898141738 +0000 UTC m=+108.507157145 (2 nodes), 1.12µs from start of poll, 173.102382ms from cluster start. Timeout at 2025-02-25 23:59:21.003201379 +0000 UTC m=+108.612216786, 105.059641ms after stability +2025-02-25T23:59:21.039Z [INFO] server-02d56729-0e9d-1cf1-c96b-349f662c42c2: entering candidate state: node="Node at 02d56729-0e9d-1cf1-c96b-349f662c42c2 [Candidate]" term=3 +2025-02-25T23:59:21.040Z [INFO] server-de40d4f9-4324-19e2-f2d5-76de3c61c1dd: entering follower state: follower="Node at de40d4f9-4324-19e2-f2d5-76de3c61c1dd [Follower]" leader-address= leader-id= +2025-02-25T23:59:21.040Z [INFO] server-de40d4f9-4324-19e2-f2d5-76de3c61c1dd: aborting pipeline replication: peer="{Voter server-e2e94cf3-0154-bad3-0c18-41ee1ae722b9 e2e94cf3-0154-bad3-0c18-41ee1ae722b9}" +2025-02-25T23:59:21.040Z [INFO] server-02d56729-0e9d-1cf1-c96b-349f662c42c2: election won: term=3 tally=2 +2025-02-25T23:59:21.040Z [INFO] server-02d56729-0e9d-1cf1-c96b-349f662c42c2: entering leader state: leader="Node at 02d56729-0e9d-1cf1-c96b-349f662c42c2 [Leader]" +2025-02-25T23:59:21.040Z [INFO] server-02d56729-0e9d-1cf1-c96b-349f662c42c2: added peer, starting replication: peer=server-e2e94cf3-0154-bad3-0c18-41ee1ae722b9 +2025-02-25T23:59:21.040Z [INFO] server-02d56729-0e9d-1cf1-c96b-349f662c42c2: added peer, starting replication: peer=server-de40d4f9-4324-19e2-f2d5-76de3c61c1dd +2025-02-25T23:59:21.040Z [INFO] server-02d56729-0e9d-1cf1-c96b-349f662c42c2: pipelining replication: peer="{Voter server-de40d4f9-4324-19e2-f2d5-76de3c61c1dd de40d4f9-4324-19e2-f2d5-76de3c61c1dd}" +2025-02-25T23:59:21.051Z [WARN] server-e2e94cf3-0154-bad3-0c18-41ee1ae722b9: failed to get previous log: previous-index=1002 last-index=372 error="log not found" +2025-02-25T23:59:21.051Z [WARN] server-02d56729-0e9d-1cf1-c96b-349f662c42c2: appendEntries rejected, sending older logs: peer="{Voter server-e2e94cf3-0154-bad3-0c18-41ee1ae722b9 e2e94cf3-0154-bad3-0c18-41ee1ae722b9}" next=373 +2025-02-25T23:59:21.062Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:21.064Z [INFO] server-02d56729-0e9d-1cf1-c96b-349f662c42c2: pipelining replication: peer="{Voter server-e2e94cf3-0154-bad3-0c18-41ee1ae722b9 e2e94cf3-0154-bad3-0c18-41ee1ae722b9}" +2025-02-25T23:59:21.062Z [INFO] server-de40d4f9-4324-19e2-f2d5-76de3c61c1dd: aborting pipeline replication: peer="{Voter server-02d56729-0e9d-1cf1-c96b-349f662c42c2 02d56729-0e9d-1cf1-c96b-349f662c42c2}" +2025-02-25T23:59:21.174Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:21.066213168 +0000 UTC m=+108.675228575 (1 nodes), 3.530303ms from start of poll, 341.173812ms from cluster start. Timeout at 2025-02-25 23:59:21.171278808 +0000 UTC m=+108.780294215, 105.06564ms after stability +2025-02-25T23:59:21.174Z [INFO] server-02d56729-0e9d-1cf1-c96b-349f662c42c2: aborting pipeline replication: peer="{Voter server-de40d4f9-4324-19e2-f2d5-76de3c61c1dd de40d4f9-4324-19e2-f2d5-76de3c61c1dd}" +2025-02-25T23:59:21.174Z [INFO] server-02d56729-0e9d-1cf1-c96b-349f662c42c2: aborting pipeline replication: peer="{Voter server-e2e94cf3-0154-bad3-0c18-41ee1ae722b9 e2e94cf3-0154-bad3-0c18-41ee1ae722b9}" +--- PASS: TestRaft_LeadershipTransferToBehindServer (0.45s) === RUN TestRaft_LeadershipTransferToItself -2026-03-31T06:17:57.214Z [INFO] server-3cf28049-3696-dfd2-72ae-4ecd2a802741: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3cf28049-3696-dfd2-72ae-4ecd2a802741 Address:3cf28049-3696-dfd2-72ae-4ecd2a802741} {Suffrage:Voter ID:server-914abc7f-d0aa-0e06-e82f-111a51896b80 Address:914abc7f-d0aa-0e06-e82f-111a51896b80} {Suffrage:Voter ID:server-8c04d648-d57b-b00c-805f-0a7300d54212 Address:8c04d648-d57b-b00c-805f-0a7300d54212}]" -2026-03-31T06:17:57.214Z [INFO] server-914abc7f-d0aa-0e06-e82f-111a51896b80: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3cf28049-3696-dfd2-72ae-4ecd2a802741 Address:3cf28049-3696-dfd2-72ae-4ecd2a802741} {Suffrage:Voter ID:server-914abc7f-d0aa-0e06-e82f-111a51896b80 Address:914abc7f-d0aa-0e06-e82f-111a51896b80} {Suffrage:Voter ID:server-8c04d648-d57b-b00c-805f-0a7300d54212 Address:8c04d648-d57b-b00c-805f-0a7300d54212}]" -2026-03-31T06:17:57.214Z [INFO] server-3cf28049-3696-dfd2-72ae-4ecd2a802741: entering follower state: follower="Node at 3cf28049-3696-dfd2-72ae-4ecd2a802741 [Follower]" leader-address= leader-id= -2026-03-31T06:17:57.214Z [INFO] server-8c04d648-d57b-b00c-805f-0a7300d54212: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3cf28049-3696-dfd2-72ae-4ecd2a802741 Address:3cf28049-3696-dfd2-72ae-4ecd2a802741} {Suffrage:Voter ID:server-914abc7f-d0aa-0e06-e82f-111a51896b80 Address:914abc7f-d0aa-0e06-e82f-111a51896b80} {Suffrage:Voter ID:server-8c04d648-d57b-b00c-805f-0a7300d54212 Address:8c04d648-d57b-b00c-805f-0a7300d54212}]" -2026-03-31T06:17:57.214Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:57.214Z [INFO] server-914abc7f-d0aa-0e06-e82f-111a51896b80: entering follower state: follower="Node at 914abc7f-d0aa-0e06-e82f-111a51896b80 [Follower]" leader-address= leader-id= -2026-03-31T06:17:57.214Z [INFO] server-8c04d648-d57b-b00c-805f-0a7300d54212: entering follower state: follower="Node at 8c04d648-d57b-b00c-805f-0a7300d54212 [Follower]" leader-address= leader-id= -2026-03-31T06:17:57.281Z [WARN] server-914abc7f-d0aa-0e06-e82f-111a51896b80: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:57.281Z [INFO] server-914abc7f-d0aa-0e06-e82f-111a51896b80: entering candidate state: node="Node at 914abc7f-d0aa-0e06-e82f-111a51896b80 [Candidate]" term=2 -2026-03-31T06:17:57.281Z [INFO] server-914abc7f-d0aa-0e06-e82f-111a51896b80: election won: term=2 tally=2 -2026-03-31T06:17:57.281Z [INFO] server-914abc7f-d0aa-0e06-e82f-111a51896b80: entering leader state: leader="Node at 914abc7f-d0aa-0e06-e82f-111a51896b80 [Leader]" -2026-03-31T06:17:57.281Z [INFO] server-914abc7f-d0aa-0e06-e82f-111a51896b80: added peer, starting replication: peer=server-3cf28049-3696-dfd2-72ae-4ecd2a802741 -2026-03-31T06:17:57.281Z [INFO] server-914abc7f-d0aa-0e06-e82f-111a51896b80: added peer, starting replication: peer=server-8c04d648-d57b-b00c-805f-0a7300d54212 -2026-03-31T06:17:57.281Z [INFO] server-914abc7f-d0aa-0e06-e82f-111a51896b80: pipelining replication: peer="{Voter server-8c04d648-d57b-b00c-805f-0a7300d54212 8c04d648-d57b-b00c-805f-0a7300d54212}" -2026-03-31T06:17:57.281Z [INFO] server-914abc7f-d0aa-0e06-e82f-111a51896b80: pipelining replication: peer="{Voter server-3cf28049-3696-dfd2-72ae-4ecd2a802741 3cf28049-3696-dfd2-72ae-4ecd2a802741}" -2026-03-31T06:17:57.387Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:57.2818681 +0000 UTC m=+105.559475481 (1 nodes), 67.37676ms from start of poll, 67.74928ms from cluster start. Timeout at 2026-03-31 06:17:57.38687714 +0000 UTC m=+105.664484521, 105.00904ms after stability -2026-03-31T06:17:57.387Z [INFO] server-914abc7f-d0aa-0e06-e82f-111a51896b80: cannot transfer leadership to itself -2026-03-31T06:17:57.387Z [INFO] server-914abc7f-d0aa-0e06-e82f-111a51896b80: aborting pipeline replication: peer="{Voter server-8c04d648-d57b-b00c-805f-0a7300d54212 8c04d648-d57b-b00c-805f-0a7300d54212}" -2026-03-31T06:17:57.387Z [INFO] server-914abc7f-d0aa-0e06-e82f-111a51896b80: aborting pipeline replication: peer="{Voter server-3cf28049-3696-dfd2-72ae-4ecd2a802741 3cf28049-3696-dfd2-72ae-4ecd2a802741}" +2025-02-25T23:59:21.175Z [INFO] server-bddfd723-af45-47fa-c6c0-4deffde5116a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bddfd723-af45-47fa-c6c0-4deffde5116a Address:bddfd723-af45-47fa-c6c0-4deffde5116a} {Suffrage:Voter ID:server-c5f255be-f72f-3cfa-4372-a14e6f9eec73 Address:c5f255be-f72f-3cfa-4372-a14e6f9eec73} {Suffrage:Voter ID:server-7375234d-fb28-cda4-022e-328271fb3677 Address:7375234d-fb28-cda4-022e-328271fb3677}]" +2025-02-25T23:59:21.175Z [INFO] server-c5f255be-f72f-3cfa-4372-a14e6f9eec73: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bddfd723-af45-47fa-c6c0-4deffde5116a Address:bddfd723-af45-47fa-c6c0-4deffde5116a} {Suffrage:Voter ID:server-c5f255be-f72f-3cfa-4372-a14e6f9eec73 Address:c5f255be-f72f-3cfa-4372-a14e6f9eec73} {Suffrage:Voter ID:server-7375234d-fb28-cda4-022e-328271fb3677 Address:7375234d-fb28-cda4-022e-328271fb3677}]" +2025-02-25T23:59:21.175Z [INFO] server-7375234d-fb28-cda4-022e-328271fb3677: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bddfd723-af45-47fa-c6c0-4deffde5116a Address:bddfd723-af45-47fa-c6c0-4deffde5116a} {Suffrage:Voter ID:server-c5f255be-f72f-3cfa-4372-a14e6f9eec73 Address:c5f255be-f72f-3cfa-4372-a14e6f9eec73} {Suffrage:Voter ID:server-7375234d-fb28-cda4-022e-328271fb3677 Address:7375234d-fb28-cda4-022e-328271fb3677}]" +2025-02-25T23:59:21.175Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:21.175Z [INFO] server-bddfd723-af45-47fa-c6c0-4deffde5116a: entering follower state: follower="Node at bddfd723-af45-47fa-c6c0-4deffde5116a [Follower]" leader-address= leader-id= +2025-02-25T23:59:21.175Z [INFO] server-c5f255be-f72f-3cfa-4372-a14e6f9eec73: entering follower state: follower="Node at c5f255be-f72f-3cfa-4372-a14e6f9eec73 [Follower]" leader-address= leader-id= +2025-02-25T23:59:21.175Z [INFO] server-7375234d-fb28-cda4-022e-328271fb3677: entering follower state: follower="Node at 7375234d-fb28-cda4-022e-328271fb3677 [Follower]" leader-address= leader-id= +2025-02-25T23:59:21.234Z [WARN] server-bddfd723-af45-47fa-c6c0-4deffde5116a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:21.234Z [INFO] server-bddfd723-af45-47fa-c6c0-4deffde5116a: entering candidate state: node="Node at bddfd723-af45-47fa-c6c0-4deffde5116a [Candidate]" term=2 +2025-02-25T23:59:21.234Z [INFO] server-bddfd723-af45-47fa-c6c0-4deffde5116a: election won: term=2 tally=2 +2025-02-25T23:59:21.234Z [INFO] server-bddfd723-af45-47fa-c6c0-4deffde5116a: entering leader state: leader="Node at bddfd723-af45-47fa-c6c0-4deffde5116a [Leader]" +2025-02-25T23:59:21.234Z [INFO] server-bddfd723-af45-47fa-c6c0-4deffde5116a: added peer, starting replication: peer=server-c5f255be-f72f-3cfa-4372-a14e6f9eec73 +2025-02-25T23:59:21.234Z [INFO] server-bddfd723-af45-47fa-c6c0-4deffde5116a: added peer, starting replication: peer=server-7375234d-fb28-cda4-022e-328271fb3677 +2025-02-25T23:59:21.234Z [INFO] server-bddfd723-af45-47fa-c6c0-4deffde5116a: pipelining replication: peer="{Voter server-7375234d-fb28-cda4-022e-328271fb3677 7375234d-fb28-cda4-022e-328271fb3677}" +2025-02-25T23:59:21.234Z [INFO] server-bddfd723-af45-47fa-c6c0-4deffde5116a: pipelining replication: peer="{Voter server-c5f255be-f72f-3cfa-4372-a14e6f9eec73 c5f255be-f72f-3cfa-4372-a14e6f9eec73}" +2025-02-25T23:59:21.342Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:21.234509559 +0000 UTC m=+108.843524966 (1 nodes), 59.276057ms from start of poll, 59.452299ms from cluster start. Timeout at 2025-02-25 23:59:21.339524599 +0000 UTC m=+108.948540006, 105.01504ms after stability +2025-02-25T23:59:21.342Z [INFO] server-bddfd723-af45-47fa-c6c0-4deffde5116a: cannot transfer leadership to itself +2025-02-25T23:59:21.342Z [INFO] server-bddfd723-af45-47fa-c6c0-4deffde5116a: aborting pipeline replication: peer="{Voter server-7375234d-fb28-cda4-022e-328271fb3677 7375234d-fb28-cda4-022e-328271fb3677}" +2025-02-25T23:59:21.342Z [INFO] server-bddfd723-af45-47fa-c6c0-4deffde5116a: aborting pipeline replication: peer="{Voter server-c5f255be-f72f-3cfa-4372-a14e6f9eec73 c5f255be-f72f-3cfa-4372-a14e6f9eec73}" --- PASS: TestRaft_LeadershipTransferToItself (0.17s) === RUN TestRaft_LeadershipTransferLeaderRejectsClientRequests -2026-03-31T06:17:57.388Z [INFO] server-e0aa6848-7d91-db0e-48f8-d9663fac06a4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e0aa6848-7d91-db0e-48f8-d9663fac06a4 Address:e0aa6848-7d91-db0e-48f8-d9663fac06a4} {Suffrage:Voter ID:server-49901a17-ee22-d565-ff39-e66eeabb453c Address:49901a17-ee22-d565-ff39-e66eeabb453c} {Suffrage:Voter ID:server-7353fe1b-be16-b5b4-d65d-e26f58773e52 Address:7353fe1b-be16-b5b4-d65d-e26f58773e52}]" -2026-03-31T06:17:57.388Z [INFO] server-49901a17-ee22-d565-ff39-e66eeabb453c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e0aa6848-7d91-db0e-48f8-d9663fac06a4 Address:e0aa6848-7d91-db0e-48f8-d9663fac06a4} {Suffrage:Voter ID:server-49901a17-ee22-d565-ff39-e66eeabb453c Address:49901a17-ee22-d565-ff39-e66eeabb453c} {Suffrage:Voter ID:server-7353fe1b-be16-b5b4-d65d-e26f58773e52 Address:7353fe1b-be16-b5b4-d65d-e26f58773e52}]" -2026-03-31T06:17:57.388Z [INFO] server-e0aa6848-7d91-db0e-48f8-d9663fac06a4: entering follower state: follower="Node at e0aa6848-7d91-db0e-48f8-d9663fac06a4 [Follower]" leader-address= leader-id= -2026-03-31T06:17:57.388Z [INFO] server-49901a17-ee22-d565-ff39-e66eeabb453c: entering follower state: follower="Node at 49901a17-ee22-d565-ff39-e66eeabb453c [Follower]" leader-address= leader-id= -2026-03-31T06:17:57.388Z [INFO] server-7353fe1b-be16-b5b4-d65d-e26f58773e52: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e0aa6848-7d91-db0e-48f8-d9663fac06a4 Address:e0aa6848-7d91-db0e-48f8-d9663fac06a4} {Suffrage:Voter ID:server-49901a17-ee22-d565-ff39-e66eeabb453c Address:49901a17-ee22-d565-ff39-e66eeabb453c} {Suffrage:Voter ID:server-7353fe1b-be16-b5b4-d65d-e26f58773e52 Address:7353fe1b-be16-b5b4-d65d-e26f58773e52}]" -2026-03-31T06:17:57.388Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:57.388Z [INFO] server-7353fe1b-be16-b5b4-d65d-e26f58773e52: entering follower state: follower="Node at 7353fe1b-be16-b5b4-d65d-e26f58773e52 [Follower]" leader-address= leader-id= -2026-03-31T06:17:57.448Z [WARN] server-49901a17-ee22-d565-ff39-e66eeabb453c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:57.448Z [INFO] server-49901a17-ee22-d565-ff39-e66eeabb453c: entering candidate state: node="Node at 49901a17-ee22-d565-ff39-e66eeabb453c [Candidate]" term=2 -2026-03-31T06:17:57.448Z [INFO] server-49901a17-ee22-d565-ff39-e66eeabb453c: election won: term=2 tally=2 -2026-03-31T06:17:57.448Z [INFO] server-49901a17-ee22-d565-ff39-e66eeabb453c: entering leader state: leader="Node at 49901a17-ee22-d565-ff39-e66eeabb453c [Leader]" -2026-03-31T06:17:57.448Z [INFO] server-49901a17-ee22-d565-ff39-e66eeabb453c: added peer, starting replication: peer=server-e0aa6848-7d91-db0e-48f8-d9663fac06a4 -2026-03-31T06:17:57.448Z [INFO] server-49901a17-ee22-d565-ff39-e66eeabb453c: added peer, starting replication: peer=server-7353fe1b-be16-b5b4-d65d-e26f58773e52 -2026-03-31T06:17:57.449Z [INFO] server-49901a17-ee22-d565-ff39-e66eeabb453c: pipelining replication: peer="{Voter server-7353fe1b-be16-b5b4-d65d-e26f58773e52 7353fe1b-be16-b5b4-d65d-e26f58773e52}" -2026-03-31T06:17:57.449Z [INFO] server-49901a17-ee22-d565-ff39-e66eeabb453c: pipelining replication: peer="{Voter server-e0aa6848-7d91-db0e-48f8-d9663fac06a4 e0aa6848-7d91-db0e-48f8-d9663fac06a4}" -2026-03-31T06:17:57.555Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:57.44902482 +0000 UTC m=+105.726632201 (1 nodes), 60.45024ms from start of poll, 60.8544ms from cluster start. Timeout at 2026-03-31 06:17:57.55402694 +0000 UTC m=+105.831634321, 105.00212ms after stability +2025-02-25T23:59:21.342Z [INFO] server-9098644c-0065-036c-3598-310a56b84f63: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9098644c-0065-036c-3598-310a56b84f63 Address:9098644c-0065-036c-3598-310a56b84f63} {Suffrage:Voter ID:server-1c46cff5-e74d-3eec-8b8a-bf55b9eb2c65 Address:1c46cff5-e74d-3eec-8b8a-bf55b9eb2c65} {Suffrage:Voter ID:server-575caa9b-76e2-142f-dcfa-6bd8092d28f1 Address:575caa9b-76e2-142f-dcfa-6bd8092d28f1}]" +2025-02-25T23:59:21.342Z [INFO] server-1c46cff5-e74d-3eec-8b8a-bf55b9eb2c65: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9098644c-0065-036c-3598-310a56b84f63 Address:9098644c-0065-036c-3598-310a56b84f63} {Suffrage:Voter ID:server-1c46cff5-e74d-3eec-8b8a-bf55b9eb2c65 Address:1c46cff5-e74d-3eec-8b8a-bf55b9eb2c65} {Suffrage:Voter ID:server-575caa9b-76e2-142f-dcfa-6bd8092d28f1 Address:575caa9b-76e2-142f-dcfa-6bd8092d28f1}]" +2025-02-25T23:59:21.342Z [INFO] server-575caa9b-76e2-142f-dcfa-6bd8092d28f1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9098644c-0065-036c-3598-310a56b84f63 Address:9098644c-0065-036c-3598-310a56b84f63} {Suffrage:Voter ID:server-1c46cff5-e74d-3eec-8b8a-bf55b9eb2c65 Address:1c46cff5-e74d-3eec-8b8a-bf55b9eb2c65} {Suffrage:Voter ID:server-575caa9b-76e2-142f-dcfa-6bd8092d28f1 Address:575caa9b-76e2-142f-dcfa-6bd8092d28f1}]" +2025-02-25T23:59:21.342Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:21.342Z [INFO] server-9098644c-0065-036c-3598-310a56b84f63: entering follower state: follower="Node at 9098644c-0065-036c-3598-310a56b84f63 [Follower]" leader-address= leader-id= +2025-02-25T23:59:21.342Z [INFO] server-1c46cff5-e74d-3eec-8b8a-bf55b9eb2c65: entering follower state: follower="Node at 1c46cff5-e74d-3eec-8b8a-bf55b9eb2c65 [Follower]" leader-address= leader-id= +2025-02-25T23:59:21.342Z [INFO] server-575caa9b-76e2-142f-dcfa-6bd8092d28f1: entering follower state: follower="Node at 575caa9b-76e2-142f-dcfa-6bd8092d28f1 [Follower]" leader-address= leader-id= +2025-02-25T23:59:21.402Z [WARN] server-575caa9b-76e2-142f-dcfa-6bd8092d28f1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:21.402Z [INFO] server-575caa9b-76e2-142f-dcfa-6bd8092d28f1: entering candidate state: node="Node at 575caa9b-76e2-142f-dcfa-6bd8092d28f1 [Candidate]" term=2 +2025-02-25T23:59:21.402Z [INFO] server-575caa9b-76e2-142f-dcfa-6bd8092d28f1: election won: term=2 tally=2 +2025-02-25T23:59:21.402Z [INFO] server-575caa9b-76e2-142f-dcfa-6bd8092d28f1: entering leader state: leader="Node at 575caa9b-76e2-142f-dcfa-6bd8092d28f1 [Leader]" +2025-02-25T23:59:21.402Z [INFO] server-575caa9b-76e2-142f-dcfa-6bd8092d28f1: added peer, starting replication: peer=server-9098644c-0065-036c-3598-310a56b84f63 +2025-02-25T23:59:21.402Z [INFO] server-575caa9b-76e2-142f-dcfa-6bd8092d28f1: added peer, starting replication: peer=server-1c46cff5-e74d-3eec-8b8a-bf55b9eb2c65 +2025-02-25T23:59:21.402Z [INFO] server-575caa9b-76e2-142f-dcfa-6bd8092d28f1: pipelining replication: peer="{Voter server-1c46cff5-e74d-3eec-8b8a-bf55b9eb2c65 1c46cff5-e74d-3eec-8b8a-bf55b9eb2c65}" +2025-02-25T23:59:21.402Z [INFO] server-575caa9b-76e2-142f-dcfa-6bd8092d28f1: pipelining replication: peer="{Voter server-9098644c-0065-036c-3598-310a56b84f63 9098644c-0065-036c-3598-310a56b84f63}" +2025-02-25T23:59:21.509Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:21.402865031 +0000 UTC m=+109.011880438 (1 nodes), 60.071983ms from start of poll, 60.238664ms from cluster start. Timeout at 2025-02-25 23:59:21.507882751 +0000 UTC m=+109.116898158, 105.01772ms after stability raft_test.go:2457: waiting on future 0 raft_test.go:2457: waiting on future 1 raft_test.go:2457: waiting on future 2 @@ -3872,343 +3946,359 @@ raft_test.go:2457: waiting on future 4 raft_test.go:2457: waiting on future 5 raft_test.go:2457: waiting on future 6 -2026-03-31T06:17:57.555Z [INFO] server-49901a17-ee22-d565-ff39-e66eeabb453c: aborting pipeline replication: peer="{Voter server-e0aa6848-7d91-db0e-48f8-d9663fac06a4 e0aa6848-7d91-db0e-48f8-d9663fac06a4}" -2026-03-31T06:17:57.555Z [INFO] server-49901a17-ee22-d565-ff39-e66eeabb453c: aborting pipeline replication: peer="{Voter server-7353fe1b-be16-b5b4-d65d-e26f58773e52 7353fe1b-be16-b5b4-d65d-e26f58773e52}" +2025-02-25T23:59:21.510Z [INFO] server-575caa9b-76e2-142f-dcfa-6bd8092d28f1: aborting pipeline replication: peer="{Voter server-1c46cff5-e74d-3eec-8b8a-bf55b9eb2c65 1c46cff5-e74d-3eec-8b8a-bf55b9eb2c65}" +2025-02-25T23:59:21.510Z [INFO] server-575caa9b-76e2-142f-dcfa-6bd8092d28f1: aborting pipeline replication: peer="{Voter server-9098644c-0065-036c-3598-310a56b84f63 9098644c-0065-036c-3598-310a56b84f63}" --- PASS: TestRaft_LeadershipTransferLeaderRejectsClientRequests (0.17s) === RUN TestRaft_LeadershipTransferLeaderReplicationTimeout -2026-03-31T06:17:57.556Z [INFO] server-7aaffcd6-7066-9165-8f18-702e92f5771c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7aaffcd6-7066-9165-8f18-702e92f5771c Address:7aaffcd6-7066-9165-8f18-702e92f5771c} {Suffrage:Voter ID:server-98e391dc-371f-67d8-4572-63e0e8c543a8 Address:98e391dc-371f-67d8-4572-63e0e8c543a8} {Suffrage:Voter ID:server-8825253c-5314-a371-19b8-4b749b8d546e Address:8825253c-5314-a371-19b8-4b749b8d546e}]" -2026-03-31T06:17:57.556Z [INFO] server-7aaffcd6-7066-9165-8f18-702e92f5771c: entering follower state: follower="Node at 7aaffcd6-7066-9165-8f18-702e92f5771c [Follower]" leader-address= leader-id= -2026-03-31T06:17:57.556Z [INFO] server-98e391dc-371f-67d8-4572-63e0e8c543a8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7aaffcd6-7066-9165-8f18-702e92f5771c Address:7aaffcd6-7066-9165-8f18-702e92f5771c} {Suffrage:Voter ID:server-98e391dc-371f-67d8-4572-63e0e8c543a8 Address:98e391dc-371f-67d8-4572-63e0e8c543a8} {Suffrage:Voter ID:server-8825253c-5314-a371-19b8-4b749b8d546e Address:8825253c-5314-a371-19b8-4b749b8d546e}]" -2026-03-31T06:17:57.556Z [INFO] server-98e391dc-371f-67d8-4572-63e0e8c543a8: entering follower state: follower="Node at 98e391dc-371f-67d8-4572-63e0e8c543a8 [Follower]" leader-address= leader-id= -2026-03-31T06:17:57.556Z [INFO] server-8825253c-5314-a371-19b8-4b749b8d546e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7aaffcd6-7066-9165-8f18-702e92f5771c Address:7aaffcd6-7066-9165-8f18-702e92f5771c} {Suffrage:Voter ID:server-98e391dc-371f-67d8-4572-63e0e8c543a8 Address:98e391dc-371f-67d8-4572-63e0e8c543a8} {Suffrage:Voter ID:server-8825253c-5314-a371-19b8-4b749b8d546e Address:8825253c-5314-a371-19b8-4b749b8d546e}]" -2026-03-31T06:17:57.556Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:57.556Z [INFO] server-8825253c-5314-a371-19b8-4b749b8d546e: entering follower state: follower="Node at 8825253c-5314-a371-19b8-4b749b8d546e [Follower]" leader-address= leader-id= -2026-03-31T06:17:57.612Z [WARN] server-8825253c-5314-a371-19b8-4b749b8d546e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:57.612Z [INFO] server-8825253c-5314-a371-19b8-4b749b8d546e: entering candidate state: node="Node at 8825253c-5314-a371-19b8-4b749b8d546e [Candidate]" term=2 -2026-03-31T06:17:57.612Z [INFO] server-8825253c-5314-a371-19b8-4b749b8d546e: election won: term=2 tally=2 -2026-03-31T06:17:57.612Z [INFO] server-8825253c-5314-a371-19b8-4b749b8d546e: entering leader state: leader="Node at 8825253c-5314-a371-19b8-4b749b8d546e [Leader]" -2026-03-31T06:17:57.612Z [INFO] server-8825253c-5314-a371-19b8-4b749b8d546e: added peer, starting replication: peer=server-7aaffcd6-7066-9165-8f18-702e92f5771c -2026-03-31T06:17:57.613Z [INFO] server-8825253c-5314-a371-19b8-4b749b8d546e: added peer, starting replication: peer=server-98e391dc-371f-67d8-4572-63e0e8c543a8 -2026-03-31T06:17:57.613Z [INFO] server-8825253c-5314-a371-19b8-4b749b8d546e: pipelining replication: peer="{Voter server-98e391dc-371f-67d8-4572-63e0e8c543a8 98e391dc-371f-67d8-4572-63e0e8c543a8}" -2026-03-31T06:17:57.613Z [INFO] server-8825253c-5314-a371-19b8-4b749b8d546e: pipelining replication: peer="{Voter server-7aaffcd6-7066-9165-8f18-702e92f5771c 7aaffcd6-7066-9165-8f18-702e92f5771c}" -2026-03-31T06:17:57.718Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:57.61301882 +0000 UTC m=+105.890626201 (1 nodes), 56.315ms from start of poll, 56.8678ms from cluster start. Timeout at 2026-03-31 06:17:57.71802374 +0000 UTC m=+105.995631121, 105.00492ms after stability -2026-03-31T06:17:57.718Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:17:57.823Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:17:57.71858594 +0000 UTC m=+105.996193321 (2 nodes), 1.8µs from start of poll, 162.43492ms from cluster start. Timeout at 2026-03-31 06:17:57.8236125 +0000 UTC m=+106.101219881, 105.02656ms after stability -2026-03-31T06:17:58.309Z [INFO] server-8825253c-5314-a371-19b8-4b749b8d546e: aborting pipeline replication: peer="{Voter server-98e391dc-371f-67d8-4572-63e0e8c543a8 98e391dc-371f-67d8-4572-63e0e8c543a8}" -2026-03-31T06:17:58.309Z [INFO] server-8825253c-5314-a371-19b8-4b749b8d546e: aborting pipeline replication: peer="{Voter server-7aaffcd6-7066-9165-8f18-702e92f5771c 7aaffcd6-7066-9165-8f18-702e92f5771c}" ---- PASS: TestRaft_LeadershipTransferLeaderReplicationTimeout (0.75s) +2025-02-25T23:59:21.510Z [INFO] server-7d221cad-46c4-d7b1-2853-f662bc647d70: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7d221cad-46c4-d7b1-2853-f662bc647d70 Address:7d221cad-46c4-d7b1-2853-f662bc647d70} {Suffrage:Voter ID:server-ab4bd848-89a4-1bc8-412b-14c9a7791551 Address:ab4bd848-89a4-1bc8-412b-14c9a7791551} {Suffrage:Voter ID:server-6f3f1865-ff03-894a-3c1a-9d084870ad38 Address:6f3f1865-ff03-894a-3c1a-9d084870ad38}]" +2025-02-25T23:59:21.510Z [INFO] server-ab4bd848-89a4-1bc8-412b-14c9a7791551: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7d221cad-46c4-d7b1-2853-f662bc647d70 Address:7d221cad-46c4-d7b1-2853-f662bc647d70} {Suffrage:Voter ID:server-ab4bd848-89a4-1bc8-412b-14c9a7791551 Address:ab4bd848-89a4-1bc8-412b-14c9a7791551} {Suffrage:Voter ID:server-6f3f1865-ff03-894a-3c1a-9d084870ad38 Address:6f3f1865-ff03-894a-3c1a-9d084870ad38}]" +2025-02-25T23:59:21.510Z [INFO] server-6f3f1865-ff03-894a-3c1a-9d084870ad38: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7d221cad-46c4-d7b1-2853-f662bc647d70 Address:7d221cad-46c4-d7b1-2853-f662bc647d70} {Suffrage:Voter ID:server-ab4bd848-89a4-1bc8-412b-14c9a7791551 Address:ab4bd848-89a4-1bc8-412b-14c9a7791551} {Suffrage:Voter ID:server-6f3f1865-ff03-894a-3c1a-9d084870ad38 Address:6f3f1865-ff03-894a-3c1a-9d084870ad38}]" +2025-02-25T23:59:21.510Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:21.510Z [INFO] server-7d221cad-46c4-d7b1-2853-f662bc647d70: entering follower state: follower="Node at 7d221cad-46c4-d7b1-2853-f662bc647d70 [Follower]" leader-address= leader-id= +2025-02-25T23:59:21.510Z [INFO] server-ab4bd848-89a4-1bc8-412b-14c9a7791551: entering follower state: follower="Node at ab4bd848-89a4-1bc8-412b-14c9a7791551 [Follower]" leader-address= leader-id= +2025-02-25T23:59:21.510Z [INFO] server-6f3f1865-ff03-894a-3c1a-9d084870ad38: entering follower state: follower="Node at 6f3f1865-ff03-894a-3c1a-9d084870ad38 [Follower]" leader-address= leader-id= +2025-02-25T23:59:21.565Z [WARN] server-6f3f1865-ff03-894a-3c1a-9d084870ad38: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:21.565Z [INFO] server-6f3f1865-ff03-894a-3c1a-9d084870ad38: entering candidate state: node="Node at 6f3f1865-ff03-894a-3c1a-9d084870ad38 [Candidate]" term=2 +2025-02-25T23:59:21.565Z [INFO] server-6f3f1865-ff03-894a-3c1a-9d084870ad38: election won: term=2 tally=2 +2025-02-25T23:59:21.565Z [INFO] server-6f3f1865-ff03-894a-3c1a-9d084870ad38: entering leader state: leader="Node at 6f3f1865-ff03-894a-3c1a-9d084870ad38 [Leader]" +2025-02-25T23:59:21.565Z [INFO] server-6f3f1865-ff03-894a-3c1a-9d084870ad38: added peer, starting replication: peer=server-7d221cad-46c4-d7b1-2853-f662bc647d70 +2025-02-25T23:59:21.566Z [INFO] server-6f3f1865-ff03-894a-3c1a-9d084870ad38: added peer, starting replication: peer=server-ab4bd848-89a4-1bc8-412b-14c9a7791551 +2025-02-25T23:59:21.566Z [INFO] server-6f3f1865-ff03-894a-3c1a-9d084870ad38: pipelining replication: peer="{Voter server-ab4bd848-89a4-1bc8-412b-14c9a7791551 ab4bd848-89a4-1bc8-412b-14c9a7791551}" +2025-02-25T23:59:21.566Z [INFO] server-6f3f1865-ff03-894a-3c1a-9d084870ad38: pipelining replication: peer="{Voter server-7d221cad-46c4-d7b1-2853-f662bc647d70 7d221cad-46c4-d7b1-2853-f662bc647d70}" +2025-02-25T23:59:21.674Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:21.566253591 +0000 UTC m=+109.175268998 (1 nodes), 55.602434ms from start of poll, 55.773555ms from cluster start. Timeout at 2025-02-25 23:59:21.671256472 +0000 UTC m=+109.280271879, 105.002881ms after stability +2025-02-25T23:59:21.674Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:59:21.781Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:59:21.674169918 +0000 UTC m=+109.283185325 (2 nodes), 1.44µs from start of poll, 163.689882ms from cluster start. Timeout at 2025-02-25 23:59:21.779215239 +0000 UTC m=+109.388230646, 105.045321ms after stability +2025-02-25T23:59:22.247Z [INFO] server-6f3f1865-ff03-894a-3c1a-9d084870ad38: aborting pipeline replication: peer="{Voter server-7d221cad-46c4-d7b1-2853-f662bc647d70 7d221cad-46c4-d7b1-2853-f662bc647d70}" +2025-02-25T23:59:22.249Z [INFO] server-6f3f1865-ff03-894a-3c1a-9d084870ad38: aborting pipeline replication: peer="{Voter server-ab4bd848-89a4-1bc8-412b-14c9a7791551 ab4bd848-89a4-1bc8-412b-14c9a7791551}" +--- PASS: TestRaft_LeadershipTransferLeaderReplicationTimeout (0.74s) === RUN TestRaft_LeadershipTransferIgnoresNonvoters -2026-03-31T06:17:58.310Z [INFO] server-9f21f33d-f57d-ce75-6d3a-735c4ba3ef42: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9f21f33d-f57d-ce75-6d3a-735c4ba3ef42 Address:9f21f33d-f57d-ce75-6d3a-735c4ba3ef42} {Suffrage:Voter ID:server-e79eff2e-27d0-4e14-0e7e-850918d256fc Address:e79eff2e-27d0-4e14-0e7e-850918d256fc}]" -2026-03-31T06:17:58.310Z [INFO] server-e79eff2e-27d0-4e14-0e7e-850918d256fc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9f21f33d-f57d-ce75-6d3a-735c4ba3ef42 Address:9f21f33d-f57d-ce75-6d3a-735c4ba3ef42} {Suffrage:Voter ID:server-e79eff2e-27d0-4e14-0e7e-850918d256fc Address:e79eff2e-27d0-4e14-0e7e-850918d256fc}]" -2026-03-31T06:17:58.310Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:17:58.310Z [INFO] server-9f21f33d-f57d-ce75-6d3a-735c4ba3ef42: entering follower state: follower="Node at 9f21f33d-f57d-ce75-6d3a-735c4ba3ef42 [Follower]" leader-address= leader-id= -2026-03-31T06:17:58.310Z [INFO] server-e79eff2e-27d0-4e14-0e7e-850918d256fc: entering follower state: follower="Node at e79eff2e-27d0-4e14-0e7e-850918d256fc [Follower]" leader-address= leader-id= -2026-03-31T06:17:58.382Z [WARN] server-9f21f33d-f57d-ce75-6d3a-735c4ba3ef42: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:58.382Z [INFO] server-9f21f33d-f57d-ce75-6d3a-735c4ba3ef42: entering candidate state: node="Node at 9f21f33d-f57d-ce75-6d3a-735c4ba3ef42 [Candidate]" term=2 -2026-03-31T06:17:58.382Z [INFO] server-9f21f33d-f57d-ce75-6d3a-735c4ba3ef42: election won: term=2 tally=2 -2026-03-31T06:17:58.382Z [INFO] server-9f21f33d-f57d-ce75-6d3a-735c4ba3ef42: entering leader state: leader="Node at 9f21f33d-f57d-ce75-6d3a-735c4ba3ef42 [Leader]" -2026-03-31T06:17:58.382Z [INFO] server-9f21f33d-f57d-ce75-6d3a-735c4ba3ef42: added peer, starting replication: peer=server-e79eff2e-27d0-4e14-0e7e-850918d256fc -2026-03-31T06:17:58.382Z [INFO] server-9f21f33d-f57d-ce75-6d3a-735c4ba3ef42: pipelining replication: peer="{Voter server-e79eff2e-27d0-4e14-0e7e-850918d256fc e79eff2e-27d0-4e14-0e7e-850918d256fc}" -2026-03-31T06:17:58.487Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:17:58.38245542 +0000 UTC m=+106.660062841 (1 nodes), 72.10244ms from start of poll, 72.3904ms from cluster start. Timeout at 2026-03-31 06:17:58.4874697 +0000 UTC m=+106.765077081, 105.01424ms after stability -2026-03-31T06:17:58.488Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:58.593Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:58.48804598 +0000 UTC m=+106.765653401 (1 nodes), 760ns from start of poll, 177.98096ms from cluster start. Timeout at 2026-03-31 06:17:58.59313274 +0000 UTC m=+106.870740121, 105.08672ms after stability -2026-03-31T06:17:58.593Z [INFO] server-9f21f33d-f57d-ce75-6d3a-735c4ba3ef42: updating configuration: command=DemoteVoter server-id=server-e79eff2e-27d0-4e14-0e7e-850918d256fc server-addr= servers="[{Suffrage:Voter ID:server-9f21f33d-f57d-ce75-6d3a-735c4ba3ef42 Address:9f21f33d-f57d-ce75-6d3a-735c4ba3ef42} {Suffrage:Nonvoter ID:server-e79eff2e-27d0-4e14-0e7e-850918d256fc Address:e79eff2e-27d0-4e14-0e7e-850918d256fc}]" -2026-03-31T06:17:58.593Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:58.699Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:58.59384682 +0000 UTC m=+106.871454201 (1 nodes), 800ns from start of poll, 283.78176ms from cluster start. Timeout at 2026-03-31 06:17:58.6989115 +0000 UTC m=+106.976518881, 105.06468ms after stability -2026-03-31T06:17:58.699Z [INFO] server-9f21f33d-f57d-ce75-6d3a-735c4ba3ef42: aborting pipeline replication: peer="{Voter server-e79eff2e-27d0-4e14-0e7e-850918d256fc e79eff2e-27d0-4e14-0e7e-850918d256fc}" ---- PASS: TestRaft_LeadershipTransferIgnoresNonvoters (0.39s) +2025-02-25T23:59:22.250Z [INFO] server-34875191-098c-828a-5ed9-cfbce90d551d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-34875191-098c-828a-5ed9-cfbce90d551d Address:34875191-098c-828a-5ed9-cfbce90d551d} {Suffrage:Voter ID:server-d6a08737-dccc-1952-fa10-fa462dedeaf3 Address:d6a08737-dccc-1952-fa10-fa462dedeaf3}]" +2025-02-25T23:59:22.250Z [INFO] server-d6a08737-dccc-1952-fa10-fa462dedeaf3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-34875191-098c-828a-5ed9-cfbce90d551d Address:34875191-098c-828a-5ed9-cfbce90d551d} {Suffrage:Voter ID:server-d6a08737-dccc-1952-fa10-fa462dedeaf3 Address:d6a08737-dccc-1952-fa10-fa462dedeaf3}]" +2025-02-25T23:59:22.250Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:59:22.250Z [INFO] server-34875191-098c-828a-5ed9-cfbce90d551d: entering follower state: follower="Node at 34875191-098c-828a-5ed9-cfbce90d551d [Follower]" leader-address= leader-id= +2025-02-25T23:59:22.250Z [INFO] server-d6a08737-dccc-1952-fa10-fa462dedeaf3: entering follower state: follower="Node at d6a08737-dccc-1952-fa10-fa462dedeaf3 [Follower]" leader-address= leader-id= +2025-02-25T23:59:22.313Z [WARN] server-d6a08737-dccc-1952-fa10-fa462dedeaf3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:22.313Z [INFO] server-d6a08737-dccc-1952-fa10-fa462dedeaf3: entering candidate state: node="Node at d6a08737-dccc-1952-fa10-fa462dedeaf3 [Candidate]" term=2 +2025-02-25T23:59:22.313Z [INFO] server-d6a08737-dccc-1952-fa10-fa462dedeaf3: election won: term=2 tally=2 +2025-02-25T23:59:22.313Z [INFO] server-d6a08737-dccc-1952-fa10-fa462dedeaf3: entering leader state: leader="Node at d6a08737-dccc-1952-fa10-fa462dedeaf3 [Leader]" +2025-02-25T23:59:22.314Z [INFO] server-d6a08737-dccc-1952-fa10-fa462dedeaf3: added peer, starting replication: peer=server-34875191-098c-828a-5ed9-cfbce90d551d +2025-02-25T23:59:22.314Z [INFO] server-d6a08737-dccc-1952-fa10-fa462dedeaf3: pipelining replication: peer="{Voter server-34875191-098c-828a-5ed9-cfbce90d551d 34875191-098c-828a-5ed9-cfbce90d551d}" +2025-02-25T23:59:22.422Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:59:22.314103312 +0000 UTC m=+109.923118719 (1 nodes), 64.001448ms from start of poll, 64.170209ms from cluster start. Timeout at 2025-02-25 23:59:22.419109632 +0000 UTC m=+110.028125039, 105.00632ms after stability +2025-02-25T23:59:22.422Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:22.528Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:22.422754883 +0000 UTC m=+110.031770290 (1 nodes), 600ns from start of poll, 172.82178ms from cluster start. Timeout at 2025-02-25 23:59:22.527792163 +0000 UTC m=+110.136807530, 105.03724ms after stability +2025-02-25T23:59:22.528Z [INFO] server-d6a08737-dccc-1952-fa10-fa462dedeaf3: updating configuration: command=DemoteVoter server-id=server-34875191-098c-828a-5ed9-cfbce90d551d server-addr= servers="[{Suffrage:Nonvoter ID:server-34875191-098c-828a-5ed9-cfbce90d551d Address:34875191-098c-828a-5ed9-cfbce90d551d} {Suffrage:Voter ID:server-d6a08737-dccc-1952-fa10-fa462dedeaf3 Address:d6a08737-dccc-1952-fa10-fa462dedeaf3}]" +2025-02-25T23:59:22.528Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:22.634Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:22.528314395 +0000 UTC m=+110.137329802 (1 nodes), 520ns from start of poll, 278.381292ms from cluster start. Timeout at 2025-02-25 23:59:22.633348235 +0000 UTC m=+110.242363602, 105.0338ms after stability +2025-02-25T23:59:22.634Z [INFO] server-d6a08737-dccc-1952-fa10-fa462dedeaf3: aborting pipeline replication: peer="{Voter server-34875191-098c-828a-5ed9-cfbce90d551d 34875191-098c-828a-5ed9-cfbce90d551d}" +--- PASS: TestRaft_LeadershipTransferIgnoresNonvoters (0.38s) === RUN TestRaft_LeadershipTransferStopRightAway --- PASS: TestRaft_LeadershipTransferStopRightAway (0.00s) === RUN TestRaft_GetConfigurationNoBootstrap -2026-03-31T06:17:58.699Z [INFO] server-22141115-be83-8732-4c77-85cb7fe5263b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-22141115-be83-8732-4c77-85cb7fe5263b Address:22141115-be83-8732-4c77-85cb7fe5263b} {Suffrage:Voter ID:server-a84443d0-d3fa-2389-1710-861e3a302887 Address:a84443d0-d3fa-2389-1710-861e3a302887}]" -2026-03-31T06:17:58.700Z [INFO] server-a84443d0-d3fa-2389-1710-861e3a302887: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-22141115-be83-8732-4c77-85cb7fe5263b Address:22141115-be83-8732-4c77-85cb7fe5263b} {Suffrage:Voter ID:server-a84443d0-d3fa-2389-1710-861e3a302887 Address:a84443d0-d3fa-2389-1710-861e3a302887}]" -2026-03-31T06:17:58.699Z [INFO] server-22141115-be83-8732-4c77-85cb7fe5263b: entering follower state: follower="Node at 22141115-be83-8732-4c77-85cb7fe5263b [Follower]" leader-address= leader-id= -2026-03-31T06:17:58.700Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:17:58.700Z [INFO] server-a84443d0-d3fa-2389-1710-861e3a302887: entering follower state: follower="Node at a84443d0-d3fa-2389-1710-861e3a302887 [Follower]" leader-address= leader-id= -2026-03-31T06:17:58.757Z [WARN] server-22141115-be83-8732-4c77-85cb7fe5263b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:58.757Z [INFO] server-22141115-be83-8732-4c77-85cb7fe5263b: entering candidate state: node="Node at 22141115-be83-8732-4c77-85cb7fe5263b [Candidate]" term=2 -2026-03-31T06:17:58.757Z [INFO] server-22141115-be83-8732-4c77-85cb7fe5263b: election won: term=2 tally=2 -2026-03-31T06:17:58.757Z [INFO] server-22141115-be83-8732-4c77-85cb7fe5263b: entering leader state: leader="Node at 22141115-be83-8732-4c77-85cb7fe5263b [Leader]" -2026-03-31T06:17:58.757Z [INFO] server-22141115-be83-8732-4c77-85cb7fe5263b: added peer, starting replication: peer=server-a84443d0-d3fa-2389-1710-861e3a302887 -2026-03-31T06:17:58.757Z [INFO] server-22141115-be83-8732-4c77-85cb7fe5263b: pipelining replication: peer="{Voter server-a84443d0-d3fa-2389-1710-861e3a302887 a84443d0-d3fa-2389-1710-861e3a302887}" -2026-03-31T06:17:58.864Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:17:58.75787026 +0000 UTC m=+107.035477641 (1 nodes), 57.7766ms from start of poll, 58.09584ms from cluster start. Timeout at 2026-03-31 06:17:58.86287418 +0000 UTC m=+107.140481561, 105.00392ms after stability -2026-03-31T06:17:58.864Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:58.969Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:58.86413506 +0000 UTC m=+107.141742481 (1 nodes), 2.48µs from start of poll, 164.36068ms from cluster start. Timeout at 2026-03-31 06:17:58.96916722 +0000 UTC m=+107.246774601, 105.03212ms after stability -2026-03-31T06:17:58.980Z [INFO] server-a84443d0-d3fa-2389-1710-861e3a302887: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-22141115-be83-8732-4c77-85cb7fe5263b Address:22141115-be83-8732-4c77-85cb7fe5263b} {Suffrage:Voter ID:server-a84443d0-d3fa-2389-1710-861e3a302887 Address:a84443d0-d3fa-2389-1710-861e3a302887}]" -2026-03-31T06:17:58.980Z [INFO] server-22141115-be83-8732-4c77-85cb7fe5263b: aborting pipeline replication: peer="{Voter server-a84443d0-d3fa-2389-1710-861e3a302887 a84443d0-d3fa-2389-1710-861e3a302887}" ---- PASS: TestRaft_GetConfigurationNoBootstrap (0.28s) +2025-02-25T23:59:22.634Z [INFO] server-28c38927-c1a0-d978-00d7-a18d9953516c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-28c38927-c1a0-d978-00d7-a18d9953516c Address:28c38927-c1a0-d978-00d7-a18d9953516c} {Suffrage:Voter ID:server-9921ed00-aac3-3f4e-5f7f-1b0114d19e11 Address:9921ed00-aac3-3f4e-5f7f-1b0114d19e11}]" +2025-02-25T23:59:22.634Z [INFO] server-9921ed00-aac3-3f4e-5f7f-1b0114d19e11: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-28c38927-c1a0-d978-00d7-a18d9953516c Address:28c38927-c1a0-d978-00d7-a18d9953516c} {Suffrage:Voter ID:server-9921ed00-aac3-3f4e-5f7f-1b0114d19e11 Address:9921ed00-aac3-3f4e-5f7f-1b0114d19e11}]" +2025-02-25T23:59:22.634Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:59:22.634Z [INFO] server-28c38927-c1a0-d978-00d7-a18d9953516c: entering follower state: follower="Node at 28c38927-c1a0-d978-00d7-a18d9953516c [Follower]" leader-address= leader-id= +2025-02-25T23:59:22.634Z [INFO] server-9921ed00-aac3-3f4e-5f7f-1b0114d19e11: entering follower state: follower="Node at 9921ed00-aac3-3f4e-5f7f-1b0114d19e11 [Follower]" leader-address= leader-id= +2025-02-25T23:59:22.701Z [WARN] server-28c38927-c1a0-d978-00d7-a18d9953516c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:22.701Z [INFO] server-28c38927-c1a0-d978-00d7-a18d9953516c: entering candidate state: node="Node at 28c38927-c1a0-d978-00d7-a18d9953516c [Candidate]" term=2 +2025-02-25T23:59:22.702Z [INFO] server-28c38927-c1a0-d978-00d7-a18d9953516c: election won: term=2 tally=2 +2025-02-25T23:59:22.702Z [INFO] server-28c38927-c1a0-d978-00d7-a18d9953516c: entering leader state: leader="Node at 28c38927-c1a0-d978-00d7-a18d9953516c [Leader]" +2025-02-25T23:59:22.702Z [INFO] server-28c38927-c1a0-d978-00d7-a18d9953516c: added peer, starting replication: peer=server-9921ed00-aac3-3f4e-5f7f-1b0114d19e11 +2025-02-25T23:59:22.702Z [INFO] server-28c38927-c1a0-d978-00d7-a18d9953516c: pipelining replication: peer="{Voter server-9921ed00-aac3-3f4e-5f7f-1b0114d19e11 9921ed00-aac3-3f4e-5f7f-1b0114d19e11}" +2025-02-25T23:59:22.810Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:59:22.702206382 +0000 UTC m=+110.311221749 (1 nodes), 67.64647ms from start of poll, 67.781391ms from cluster start. Timeout at 2025-02-25 23:59:22.807209262 +0000 UTC m=+110.416224629, 105.00288ms after stability +2025-02-25T23:59:22.810Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:22.916Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:22.81018935 +0000 UTC m=+110.419204757 (1 nodes), 600ns from start of poll, 175.764399ms from cluster start. Timeout at 2025-02-25 23:59:22.91522991 +0000 UTC m=+110.524245317, 105.04056ms after stability +2025-02-25T23:59:22.938Z [INFO] server-9921ed00-aac3-3f4e-5f7f-1b0114d19e11: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-28c38927-c1a0-d978-00d7-a18d9953516c Address:28c38927-c1a0-d978-00d7-a18d9953516c} {Suffrage:Voter ID:server-9921ed00-aac3-3f4e-5f7f-1b0114d19e11 Address:9921ed00-aac3-3f4e-5f7f-1b0114d19e11}]" +2025-02-25T23:59:22.938Z [INFO] server-28c38927-c1a0-d978-00d7-a18d9953516c: aborting pipeline replication: peer="{Voter server-9921ed00-aac3-3f4e-5f7f-1b0114d19e11 9921ed00-aac3-3f4e-5f7f-1b0114d19e11}" +2025-02-25T23:59:23.438Z [ERROR] server-28c38927-c1a0-d978-00d7-a18d9953516c: failed to heartbeat to: peer=9921ed00-aac3-3f4e-5f7f-1b0114d19e11 backoff time=10ms error="command timed out" +--- PASS: TestRaft_GetConfigurationNoBootstrap (0.80s) === RUN TestRaft_LogStoreIsMonotonic -2026-03-31T06:17:58.981Z [INFO] server-e834d2e0-eacf-df0d-6e59-30218780d82f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e834d2e0-eacf-df0d-6e59-30218780d82f Address:e834d2e0-eacf-df0d-6e59-30218780d82f}]" -2026-03-31T06:17:58.981Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:58.981Z [INFO] server-e834d2e0-eacf-df0d-6e59-30218780d82f: entering follower state: follower="Node at e834d2e0-eacf-df0d-6e59-30218780d82f [Follower]" leader-address= leader-id= -2026-03-31T06:17:59.063Z [WARN] server-e834d2e0-eacf-df0d-6e59-30218780d82f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:59.063Z [INFO] server-e834d2e0-eacf-df0d-6e59-30218780d82f: entering candidate state: node="Node at e834d2e0-eacf-df0d-6e59-30218780d82f [Candidate]" term=2 -2026-03-31T06:17:59.063Z [INFO] server-e834d2e0-eacf-df0d-6e59-30218780d82f: election won: term=2 tally=1 -2026-03-31T06:17:59.063Z [INFO] server-e834d2e0-eacf-df0d-6e59-30218780d82f: entering leader state: leader="Node at e834d2e0-eacf-df0d-6e59-30218780d82f [Leader]" -2026-03-31T06:17:59.169Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:59.06402178 +0000 UTC m=+107.341629161 (1 nodes), 82.39224ms from start of poll, 82.59324ms from cluster start. Timeout at 2026-03-31 06:17:59.16902514 +0000 UTC m=+107.446632521, 105.00336ms after stability +2025-02-25T23:59:23.438Z [INFO] server-a2de7abb-cdf6-8991-7756-d4e81d8255c2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a2de7abb-cdf6-8991-7756-d4e81d8255c2 Address:a2de7abb-cdf6-8991-7756-d4e81d8255c2}]" +2025-02-25T23:59:23.438Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:23.438Z [INFO] server-a2de7abb-cdf6-8991-7756-d4e81d8255c2: entering follower state: follower="Node at a2de7abb-cdf6-8991-7756-d4e81d8255c2 [Follower]" leader-address= leader-id= +2025-02-25T23:59:23.513Z [WARN] server-a2de7abb-cdf6-8991-7756-d4e81d8255c2: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:23.513Z [INFO] server-a2de7abb-cdf6-8991-7756-d4e81d8255c2: entering candidate state: node="Node at a2de7abb-cdf6-8991-7756-d4e81d8255c2 [Candidate]" term=2 +2025-02-25T23:59:23.513Z [INFO] server-a2de7abb-cdf6-8991-7756-d4e81d8255c2: election won: term=2 tally=1 +2025-02-25T23:59:23.513Z [INFO] server-a2de7abb-cdf6-8991-7756-d4e81d8255c2: entering leader state: leader="Node at a2de7abb-cdf6-8991-7756-d4e81d8255c2 [Leader]" +2025-02-25T23:59:23.630Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:23.514043151 +0000 UTC m=+111.123058558 (1 nodes), 75.37484ms from start of poll, 75.476321ms from cluster start. Timeout at 2025-02-25 23:59:23.619045031 +0000 UTC m=+111.228060398, 105.00184ms after stability --- PASS: TestRaft_LogStoreIsMonotonic (0.19s) === RUN TestRaft_CacheLogWithStoreError -2026-03-31T06:17:59.170Z [INFO] server-660eb37a-f256-7278-f569-a3cbda40d03f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-660eb37a-f256-7278-f569-a3cbda40d03f Address:660eb37a-f256-7278-f569-a3cbda40d03f} {Suffrage:Voter ID:server-3a63d033-e17a-08cf-8103-c9451d5bfcf4 Address:3a63d033-e17a-08cf-8103-c9451d5bfcf4}]" -2026-03-31T06:17:59.170Z [INFO] server-3a63d033-e17a-08cf-8103-c9451d5bfcf4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-660eb37a-f256-7278-f569-a3cbda40d03f Address:660eb37a-f256-7278-f569-a3cbda40d03f} {Suffrage:Voter ID:server-3a63d033-e17a-08cf-8103-c9451d5bfcf4 Address:3a63d033-e17a-08cf-8103-c9451d5bfcf4}]" -2026-03-31T06:17:59.170Z [INFO] server-660eb37a-f256-7278-f569-a3cbda40d03f: entering follower state: follower="Node at 660eb37a-f256-7278-f569-a3cbda40d03f [Follower]" leader-address= leader-id= -2026-03-31T06:17:59.170Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:17:59.170Z [INFO] server-3a63d033-e17a-08cf-8103-c9451d5bfcf4: entering follower state: follower="Node at 3a63d033-e17a-08cf-8103-c9451d5bfcf4 [Follower]" leader-address= leader-id= -2026-03-31T06:17:59.241Z [WARN] server-660eb37a-f256-7278-f569-a3cbda40d03f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:59.241Z [INFO] server-660eb37a-f256-7278-f569-a3cbda40d03f: entering candidate state: node="Node at 660eb37a-f256-7278-f569-a3cbda40d03f [Candidate]" term=2 -2026-03-31T06:17:59.242Z [INFO] server-660eb37a-f256-7278-f569-a3cbda40d03f: election won: term=2 tally=2 -2026-03-31T06:17:59.242Z [INFO] server-660eb37a-f256-7278-f569-a3cbda40d03f: entering leader state: leader="Node at 660eb37a-f256-7278-f569-a3cbda40d03f [Leader]" -2026-03-31T06:17:59.242Z [INFO] server-660eb37a-f256-7278-f569-a3cbda40d03f: added peer, starting replication: peer=server-3a63d033-e17a-08cf-8103-c9451d5bfcf4 -2026-03-31T06:17:59.242Z [INFO] server-660eb37a-f256-7278-f569-a3cbda40d03f: pipelining replication: peer="{Voter server-3a63d033-e17a-08cf-8103-c9451d5bfcf4 3a63d033-e17a-08cf-8103-c9451d5bfcf4}" -2026-03-31T06:17:59.347Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:17:59.24209958 +0000 UTC m=+107.519706961 (1 nodes), 71.5582ms from start of poll, 71.86396ms from cluster start. Timeout at 2026-03-31 06:17:59.3471043 +0000 UTC m=+107.624711681, 105.00472ms after stability -2026-03-31T06:17:59.347Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:59.452Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:59.3472811 +0000 UTC m=+107.624888481 (1 nodes), 2.08µs from start of poll, 177.04548ms from cluster start. Timeout at 2026-03-31 06:17:59.45230326 +0000 UTC m=+107.729910641, 105.02216ms after stability -2026-03-31T06:17:59.453Z [ERROR] server-3a63d033-e17a-08cf-8103-c9451d5bfcf4: failed to append to logs: error="unable to store logs within log store, err: "some error"" -2026-03-31T06:17:59.453Z [WARN] server-3a63d033-e17a-08cf-8103-c9451d5bfcf4: failed to get previous log: previous-index=9 last-index=7 error="log not found" -2026-03-31T06:17:59.453Z [INFO] server-660eb37a-f256-7278-f569-a3cbda40d03f: aborting pipeline replication: peer="{Voter server-3a63d033-e17a-08cf-8103-c9451d5bfcf4 3a63d033-e17a-08cf-8103-c9451d5bfcf4}" -2026-03-31T06:17:59.461Z [INFO] server-660eb37a-f256-7278-f569-a3cbda40d03f: pipelining replication: peer="{Voter server-3a63d033-e17a-08cf-8103-c9451d5bfcf4 3a63d033-e17a-08cf-8103-c9451d5bfcf4}" -2026-03-31T06:17:59.461Z [INFO] server-3a63d033-e17a-08cf-8103-c9451d5bfcf4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-660eb37a-f256-7278-f569-a3cbda40d03f Address:660eb37a-f256-7278-f569-a3cbda40d03f} {Suffrage:Voter ID:server-3a63d033-e17a-08cf-8103-c9451d5bfcf4 Address:3a63d033-e17a-08cf-8103-c9451d5bfcf4}]" -2026-03-31T06:17:59.461Z [INFO] server-660eb37a-f256-7278-f569-a3cbda40d03f: aborting pipeline replication: peer="{Voter server-3a63d033-e17a-08cf-8103-c9451d5bfcf4 3a63d033-e17a-08cf-8103-c9451d5bfcf4}" ---- PASS: TestRaft_CacheLogWithStoreError (0.29s) +2025-02-25T23:59:23.631Z [INFO] server-e2a029db-7ed6-31e0-9ffa-06278372cd7a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e2a029db-7ed6-31e0-9ffa-06278372cd7a Address:e2a029db-7ed6-31e0-9ffa-06278372cd7a} {Suffrage:Voter ID:server-ee8fbfc6-872e-6545-0a23-933b141cf64c Address:ee8fbfc6-872e-6545-0a23-933b141cf64c}]" +2025-02-25T23:59:23.631Z [INFO] server-ee8fbfc6-872e-6545-0a23-933b141cf64c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e2a029db-7ed6-31e0-9ffa-06278372cd7a Address:e2a029db-7ed6-31e0-9ffa-06278372cd7a} {Suffrage:Voter ID:server-ee8fbfc6-872e-6545-0a23-933b141cf64c Address:ee8fbfc6-872e-6545-0a23-933b141cf64c}]" +2025-02-25T23:59:23.631Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:59:23.631Z [INFO] server-e2a029db-7ed6-31e0-9ffa-06278372cd7a: entering follower state: follower="Node at e2a029db-7ed6-31e0-9ffa-06278372cd7a [Follower]" leader-address= leader-id= +2025-02-25T23:59:23.631Z [INFO] server-ee8fbfc6-872e-6545-0a23-933b141cf64c: entering follower state: follower="Node at ee8fbfc6-872e-6545-0a23-933b141cf64c [Follower]" leader-address= leader-id= +2025-02-25T23:59:23.713Z [WARN] server-ee8fbfc6-872e-6545-0a23-933b141cf64c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:23.714Z [INFO] server-ee8fbfc6-872e-6545-0a23-933b141cf64c: entering candidate state: node="Node at ee8fbfc6-872e-6545-0a23-933b141cf64c [Candidate]" term=2 +2025-02-25T23:59:23.714Z [INFO] server-ee8fbfc6-872e-6545-0a23-933b141cf64c: election won: term=2 tally=2 +2025-02-25T23:59:23.714Z [INFO] server-ee8fbfc6-872e-6545-0a23-933b141cf64c: entering leader state: leader="Node at ee8fbfc6-872e-6545-0a23-933b141cf64c [Leader]" +2025-02-25T23:59:23.714Z [INFO] server-ee8fbfc6-872e-6545-0a23-933b141cf64c: added peer, starting replication: peer=server-e2a029db-7ed6-31e0-9ffa-06278372cd7a +2025-02-25T23:59:23.714Z [INFO] server-ee8fbfc6-872e-6545-0a23-933b141cf64c: pipelining replication: peer="{Voter server-e2a029db-7ed6-31e0-9ffa-06278372cd7a e2a029db-7ed6-31e0-9ffa-06278372cd7a}" +2025-02-25T23:59:23.821Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:59:23.714288705 +0000 UTC m=+111.323304072 (1 nodes), 82.943608ms from start of poll, 83.081689ms from cluster start. Timeout at 2025-02-25 23:59:23.819293625 +0000 UTC m=+111.428309032, 105.00496ms after stability +2025-02-25T23:59:23.822Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:23.930Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:23.822012191 +0000 UTC m=+111.431027598 (1 nodes), 1.4µs from start of poll, 190.805215ms from cluster start. Timeout at 2025-02-25 23:59:23.927068352 +0000 UTC m=+111.536083719, 105.056121ms after stability +2025-02-25T23:59:23.931Z [ERROR] server-e2a029db-7ed6-31e0-9ffa-06278372cd7a: failed to append to logs: error="unable to store logs within log store, err: "some error"" +2025-02-25T23:59:23.931Z [WARN] server-e2a029db-7ed6-31e0-9ffa-06278372cd7a: failed to get previous log: previous-index=9 last-index=7 error="log not found" +2025-02-25T23:59:23.931Z [INFO] server-ee8fbfc6-872e-6545-0a23-933b141cf64c: aborting pipeline replication: peer="{Voter server-e2a029db-7ed6-31e0-9ffa-06278372cd7a e2a029db-7ed6-31e0-9ffa-06278372cd7a}" +2025-02-25T23:59:23.941Z [INFO] server-ee8fbfc6-872e-6545-0a23-933b141cf64c: pipelining replication: peer="{Voter server-e2a029db-7ed6-31e0-9ffa-06278372cd7a e2a029db-7ed6-31e0-9ffa-06278372cd7a}" +2025-02-25T23:59:23.942Z [INFO] server-e2a029db-7ed6-31e0-9ffa-06278372cd7a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e2a029db-7ed6-31e0-9ffa-06278372cd7a Address:e2a029db-7ed6-31e0-9ffa-06278372cd7a} {Suffrage:Voter ID:server-ee8fbfc6-872e-6545-0a23-933b141cf64c Address:ee8fbfc6-872e-6545-0a23-933b141cf64c}]" +2025-02-25T23:59:23.942Z [INFO] server-ee8fbfc6-872e-6545-0a23-933b141cf64c: aborting pipeline replication: peer="{Voter server-e2a029db-7ed6-31e0-9ffa-06278372cd7a e2a029db-7ed6-31e0-9ffa-06278372cd7a}" +--- PASS: TestRaft_CacheLogWithStoreError (0.31s) === RUN TestRaft_ReloadConfig -2026-03-31T06:17:59.462Z [INFO] server-162dd517-40d4-5f0e-d46c-2cab1a292c82: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-162dd517-40d4-5f0e-d46c-2cab1a292c82 Address:162dd517-40d4-5f0e-d46c-2cab1a292c82}]" +2025-02-25T23:59:23.942Z [INFO] server-c0ff80ff-1d99-3cc7-3be6-4a0f5230743c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c0ff80ff-1d99-3cc7-3be6-4a0f5230743c Address:c0ff80ff-1d99-3cc7-3be6-4a0f5230743c}]" --- PASS: TestRaft_ReloadConfig (0.00s) === RUN TestRaft_ReloadConfigValidates -2026-03-31T06:17:59.462Z [INFO] server-554109fb-a6b8-5063-69aa-0a57324dc2bc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-554109fb-a6b8-5063-69aa-0a57324dc2bc Address:554109fb-a6b8-5063-69aa-0a57324dc2bc}]" +2025-02-25T23:59:23.942Z [INFO] server-51f3a010-9abc-966d-27fb-7fd7519ba7d9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-51f3a010-9abc-966d-27fb-7fd7519ba7d9 Address:51f3a010-9abc-966d-27fb-7fd7519ba7d9}]" --- PASS: TestRaft_ReloadConfigValidates (0.00s) === RUN TestRaft_InstallSnapshot_InvalidPeers -2026-03-31T06:17:59.462Z [ERROR] failed to install snapshot: error="failed to decode peers: codec.decoder: Only encoded map or array can be decoded into a slice. (valueType: 2)" +2025-02-25T23:59:23.942Z [ERROR] failed to install snapshot: error="failed to decode peers: codec.decoder: Only encoded map or array can be decoded into a slice. (valueType: 2)" --- PASS: TestRaft_InstallSnapshot_InvalidPeers (0.00s) === RUN TestRaft_VoteNotGranted_WhenNodeNotInCluster -2026-03-31T06:17:59.463Z [INFO] server-4ffc338e-4794-c2a1-5b04-4961401f4d7f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4ffc338e-4794-c2a1-5b04-4961401f4d7f Address:4ffc338e-4794-c2a1-5b04-4961401f4d7f} {Suffrage:Voter ID:server-a3618a08-3134-1c50-85ec-b8d5bec6badd Address:a3618a08-3134-1c50-85ec-b8d5bec6badd} {Suffrage:Voter ID:server-c1094d08-d0f4-666d-def3-8d427748d4ab Address:c1094d08-d0f4-666d-def3-8d427748d4ab}]" -2026-03-31T06:17:59.463Z [INFO] server-a3618a08-3134-1c50-85ec-b8d5bec6badd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4ffc338e-4794-c2a1-5b04-4961401f4d7f Address:4ffc338e-4794-c2a1-5b04-4961401f4d7f} {Suffrage:Voter ID:server-a3618a08-3134-1c50-85ec-b8d5bec6badd Address:a3618a08-3134-1c50-85ec-b8d5bec6badd} {Suffrage:Voter ID:server-c1094d08-d0f4-666d-def3-8d427748d4ab Address:c1094d08-d0f4-666d-def3-8d427748d4ab}]" -2026-03-31T06:17:59.463Z [INFO] server-4ffc338e-4794-c2a1-5b04-4961401f4d7f: entering follower state: follower="Node at 4ffc338e-4794-c2a1-5b04-4961401f4d7f [Follower]" leader-address= leader-id= -2026-03-31T06:17:59.463Z [INFO] server-a3618a08-3134-1c50-85ec-b8d5bec6badd: entering follower state: follower="Node at a3618a08-3134-1c50-85ec-b8d5bec6badd [Follower]" leader-address= leader-id= -2026-03-31T06:17:59.463Z [INFO] server-c1094d08-d0f4-666d-def3-8d427748d4ab: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4ffc338e-4794-c2a1-5b04-4961401f4d7f Address:4ffc338e-4794-c2a1-5b04-4961401f4d7f} {Suffrage:Voter ID:server-a3618a08-3134-1c50-85ec-b8d5bec6badd Address:a3618a08-3134-1c50-85ec-b8d5bec6badd} {Suffrage:Voter ID:server-c1094d08-d0f4-666d-def3-8d427748d4ab Address:c1094d08-d0f4-666d-def3-8d427748d4ab}]" -2026-03-31T06:17:59.463Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:59.463Z [INFO] server-c1094d08-d0f4-666d-def3-8d427748d4ab: entering follower state: follower="Node at c1094d08-d0f4-666d-def3-8d427748d4ab [Follower]" leader-address= leader-id= -2026-03-31T06:17:59.530Z [WARN] server-4ffc338e-4794-c2a1-5b04-4961401f4d7f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:17:59.530Z [INFO] server-4ffc338e-4794-c2a1-5b04-4961401f4d7f: entering candidate state: node="Node at 4ffc338e-4794-c2a1-5b04-4961401f4d7f [Candidate]" term=2 -2026-03-31T06:17:59.530Z [INFO] server-4ffc338e-4794-c2a1-5b04-4961401f4d7f: election won: term=2 tally=2 -2026-03-31T06:17:59.530Z [INFO] server-4ffc338e-4794-c2a1-5b04-4961401f4d7f: entering leader state: leader="Node at 4ffc338e-4794-c2a1-5b04-4961401f4d7f [Leader]" -2026-03-31T06:17:59.530Z [INFO] server-4ffc338e-4794-c2a1-5b04-4961401f4d7f: added peer, starting replication: peer=server-a3618a08-3134-1c50-85ec-b8d5bec6badd -2026-03-31T06:17:59.530Z [INFO] server-4ffc338e-4794-c2a1-5b04-4961401f4d7f: added peer, starting replication: peer=server-c1094d08-d0f4-666d-def3-8d427748d4ab -2026-03-31T06:17:59.530Z [INFO] server-4ffc338e-4794-c2a1-5b04-4961401f4d7f: pipelining replication: peer="{Voter server-a3618a08-3134-1c50-85ec-b8d5bec6badd a3618a08-3134-1c50-85ec-b8d5bec6badd}" -2026-03-31T06:17:59.530Z [INFO] server-4ffc338e-4794-c2a1-5b04-4961401f4d7f: pipelining replication: peer="{Voter server-c1094d08-d0f4-666d-def3-8d427748d4ab c1094d08-d0f4-666d-def3-8d427748d4ab}" -2026-03-31T06:17:59.636Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:17:59.53071866 +0000 UTC m=+107.808326041 (1 nodes), 67.18752ms from start of poll, 67.58488ms from cluster start. Timeout at 2026-03-31 06:17:59.63572934 +0000 UTC m=+107.913336721, 105.01068ms after stability -2026-03-31T06:17:59.642Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:17:59.747Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:17:59.6420537 +0000 UTC m=+107.919661121 (2 nodes), 3.56µs from start of poll, 178.91996ms from cluster start. Timeout at 2026-03-31 06:17:59.7470781 +0000 UTC m=+108.024685481, 105.02436ms after stability -2026-03-31T06:17:59.747Z [INFO] server-4ffc338e-4794-c2a1-5b04-4961401f4d7f: updating configuration: command=RemoveServer server-id=server-a3618a08-3134-1c50-85ec-b8d5bec6badd server-addr= servers="[{Suffrage:Voter ID:server-4ffc338e-4794-c2a1-5b04-4961401f4d7f Address:4ffc338e-4794-c2a1-5b04-4961401f4d7f} {Suffrage:Voter ID:server-c1094d08-d0f4-666d-def3-8d427748d4ab Address:c1094d08-d0f4-666d-def3-8d427748d4ab}]" -2026-03-31T06:17:59.747Z [INFO] server-4ffc338e-4794-c2a1-5b04-4961401f4d7f: removed peer, stopping replication: peer=server-a3618a08-3134-1c50-85ec-b8d5bec6badd last-index=3 -2026-03-31T06:17:59.747Z [INFO] server-4ffc338e-4794-c2a1-5b04-4961401f4d7f: aborting pipeline replication: peer="{Voter server-a3618a08-3134-1c50-85ec-b8d5bec6badd a3618a08-3134-1c50-85ec-b8d5bec6badd}" -2026-03-31T06:17:59.835Z [WARN] server-a3618a08-3134-1c50-85ec-b8d5bec6badd: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election -2026-03-31T06:17:59.854Z [ERROR] server-4ffc338e-4794-c2a1-5b04-4961401f4d7f: failed to heartbeat to: peer=c1094d08-d0f4-666d-def3-8d427748d4ab backoff time=10ms error="failed to connect to peer: c1094d08-d0f4-666d-def3-8d427748d4ab" -2026-03-31T06:17:59.862Z [ERROR] server-4ffc338e-4794-c2a1-5b04-4961401f4d7f: failed to pipeline appendEntries: peer="{Voter server-c1094d08-d0f4-666d-def3-8d427748d4ab c1094d08-d0f4-666d-def3-8d427748d4ab}" error="append pipeline closed" -2026-03-31T06:17:59.862Z [INFO] server-4ffc338e-4794-c2a1-5b04-4961401f4d7f: aborting pipeline replication: peer="{Voter server-c1094d08-d0f4-666d-def3-8d427748d4ab c1094d08-d0f4-666d-def3-8d427748d4ab}" -2026-03-31T06:17:59.869Z [ERROR] server-4ffc338e-4794-c2a1-5b04-4961401f4d7f: failed to appendEntries to: peer="{Voter server-c1094d08-d0f4-666d-def3-8d427748d4ab c1094d08-d0f4-666d-def3-8d427748d4ab}" error="failed to connect to peer: c1094d08-d0f4-666d-def3-8d427748d4ab" -2026-03-31T06:17:59.876Z [ERROR] server-4ffc338e-4794-c2a1-5b04-4961401f4d7f: failed to heartbeat to: peer=c1094d08-d0f4-666d-def3-8d427748d4ab backoff time=10ms error="failed to connect to peer: c1094d08-d0f4-666d-def3-8d427748d4ab" -2026-03-31T06:17:59.889Z [ERROR] server-4ffc338e-4794-c2a1-5b04-4961401f4d7f: failed to appendEntries to: peer="{Voter server-c1094d08-d0f4-666d-def3-8d427748d4ab c1094d08-d0f4-666d-def3-8d427748d4ab}" error="failed to connect to peer: c1094d08-d0f4-666d-def3-8d427748d4ab" -2026-03-31T06:17:59.893Z [ERROR] server-4ffc338e-4794-c2a1-5b04-4961401f4d7f: failed to heartbeat to: peer=c1094d08-d0f4-666d-def3-8d427748d4ab backoff time=10ms error="failed to connect to peer: c1094d08-d0f4-666d-def3-8d427748d4ab" -2026-03-31T06:17:59.903Z [WARN] server-4ffc338e-4794-c2a1-5b04-4961401f4d7f: failed to contact: server-id=server-c1094d08-d0f4-666d-def3-8d427748d4ab time=50.06728ms -2026-03-31T06:17:59.903Z [WARN] server-4ffc338e-4794-c2a1-5b04-4961401f4d7f: failed to contact quorum of nodes, stepping down -2026-03-31T06:17:59.903Z [INFO] server-4ffc338e-4794-c2a1-5b04-4961401f4d7f: entering follower state: follower="Node at 4ffc338e-4794-c2a1-5b04-4961401f4d7f [Follower]" leader-address= leader-id= -2026-03-31T06:17:59.908Z [ERROR] server-4ffc338e-4794-c2a1-5b04-4961401f4d7f: failed to heartbeat to: peer=c1094d08-d0f4-666d-def3-8d427748d4ab backoff time=20ms error="failed to connect to peer: c1094d08-d0f4-666d-def3-8d427748d4ab" -2026-03-31T06:17:59.910Z [ERROR] server-4ffc338e-4794-c2a1-5b04-4961401f4d7f: failed to appendEntries to: peer="{Voter server-c1094d08-d0f4-666d-def3-8d427748d4ab c1094d08-d0f4-666d-def3-8d427748d4ab}" error="failed to connect to peer: c1094d08-d0f4-666d-def3-8d427748d4ab" -2026-03-31T06:17:59.939Z [WARN] server-c1094d08-d0f4-666d-def3-8d427748d4ab: heartbeat timeout reached, starting election: last-leader-addr=4ffc338e-4794-c2a1-5b04-4961401f4d7f last-leader-id=server-4ffc338e-4794-c2a1-5b04-4961401f4d7f -2026-03-31T06:17:59.939Z [INFO] server-c1094d08-d0f4-666d-def3-8d427748d4ab: entering candidate state: node="Node at c1094d08-d0f4-666d-def3-8d427748d4ab [Candidate]" term=3 -2026-03-31T06:17:59.939Z [ERROR] server-c1094d08-d0f4-666d-def3-8d427748d4ab: failed to make requestVote RPC: target="{Voter server-4ffc338e-4794-c2a1-5b04-4961401f4d7f 4ffc338e-4794-c2a1-5b04-4961401f4d7f}" error="failed to connect to peer: 4ffc338e-4794-c2a1-5b04-4961401f4d7f" term=3 -2026-03-31T06:17:59.958Z [WARN] server-c1094d08-d0f4-666d-def3-8d427748d4ab: rejecting vote request since node is not in configuration: from=a3618a08-3134-1c50-85ec-b8d5bec6badd +2025-02-25T23:59:23.943Z [INFO] server-2cccd661-27a5-cfeb-d717-4dd39df40751: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2cccd661-27a5-cfeb-d717-4dd39df40751 Address:2cccd661-27a5-cfeb-d717-4dd39df40751} {Suffrage:Voter ID:server-30786f19-75e0-f837-0654-eebcbfedee28 Address:30786f19-75e0-f837-0654-eebcbfedee28} {Suffrage:Voter ID:server-afd02fb0-7461-ef47-630f-762f2233dba3 Address:afd02fb0-7461-ef47-630f-762f2233dba3}]" +2025-02-25T23:59:23.943Z [INFO] server-30786f19-75e0-f837-0654-eebcbfedee28: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2cccd661-27a5-cfeb-d717-4dd39df40751 Address:2cccd661-27a5-cfeb-d717-4dd39df40751} {Suffrage:Voter ID:server-30786f19-75e0-f837-0654-eebcbfedee28 Address:30786f19-75e0-f837-0654-eebcbfedee28} {Suffrage:Voter ID:server-afd02fb0-7461-ef47-630f-762f2233dba3 Address:afd02fb0-7461-ef47-630f-762f2233dba3}]" +2025-02-25T23:59:23.943Z [INFO] server-afd02fb0-7461-ef47-630f-762f2233dba3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2cccd661-27a5-cfeb-d717-4dd39df40751 Address:2cccd661-27a5-cfeb-d717-4dd39df40751} {Suffrage:Voter ID:server-30786f19-75e0-f837-0654-eebcbfedee28 Address:30786f19-75e0-f837-0654-eebcbfedee28} {Suffrage:Voter ID:server-afd02fb0-7461-ef47-630f-762f2233dba3 Address:afd02fb0-7461-ef47-630f-762f2233dba3}]" +2025-02-25T23:59:23.943Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:23.943Z [INFO] server-2cccd661-27a5-cfeb-d717-4dd39df40751: entering follower state: follower="Node at 2cccd661-27a5-cfeb-d717-4dd39df40751 [Follower]" leader-address= leader-id= +2025-02-25T23:59:23.943Z [INFO] server-30786f19-75e0-f837-0654-eebcbfedee28: entering follower state: follower="Node at 30786f19-75e0-f837-0654-eebcbfedee28 [Follower]" leader-address= leader-id= +2025-02-25T23:59:23.943Z [INFO] server-afd02fb0-7461-ef47-630f-762f2233dba3: entering follower state: follower="Node at afd02fb0-7461-ef47-630f-762f2233dba3 [Follower]" leader-address= leader-id= +2025-02-25T23:59:23.997Z [WARN] server-2cccd661-27a5-cfeb-d717-4dd39df40751: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:23.998Z [INFO] server-2cccd661-27a5-cfeb-d717-4dd39df40751: entering candidate state: node="Node at 2cccd661-27a5-cfeb-d717-4dd39df40751 [Candidate]" term=2 +2025-02-25T23:59:23.998Z [INFO] server-2cccd661-27a5-cfeb-d717-4dd39df40751: election won: term=2 tally=2 +2025-02-25T23:59:23.998Z [INFO] server-2cccd661-27a5-cfeb-d717-4dd39df40751: entering leader state: leader="Node at 2cccd661-27a5-cfeb-d717-4dd39df40751 [Leader]" +2025-02-25T23:59:23.998Z [INFO] server-2cccd661-27a5-cfeb-d717-4dd39df40751: added peer, starting replication: peer=server-30786f19-75e0-f837-0654-eebcbfedee28 +2025-02-25T23:59:23.998Z [INFO] server-2cccd661-27a5-cfeb-d717-4dd39df40751: added peer, starting replication: peer=server-afd02fb0-7461-ef47-630f-762f2233dba3 +2025-02-25T23:59:23.998Z [INFO] server-2cccd661-27a5-cfeb-d717-4dd39df40751: pipelining replication: peer="{Voter server-afd02fb0-7461-ef47-630f-762f2233dba3 afd02fb0-7461-ef47-630f-762f2233dba3}" +2025-02-25T23:59:23.998Z [INFO] server-2cccd661-27a5-cfeb-d717-4dd39df40751: pipelining replication: peer="{Voter server-30786f19-75e0-f837-0654-eebcbfedee28 30786f19-75e0-f837-0654-eebcbfedee28}" +2025-02-25T23:59:24.106Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:23.998429874 +0000 UTC m=+111.607445281 (1 nodes), 55.282952ms from start of poll, 55.432633ms from cluster start. Timeout at 2025-02-25 23:59:24.103439354 +0000 UTC m=+111.712454761, 105.00948ms after stability +2025-02-25T23:59:24.114Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:59:24.221Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:59:24.114359932 +0000 UTC m=+111.723375339 (2 nodes), 1.64µs from start of poll, 171.362691ms from cluster start. Timeout at 2025-02-25 23:59:24.219403573 +0000 UTC m=+111.828418980, 105.043641ms after stability +2025-02-25T23:59:24.221Z [INFO] server-2cccd661-27a5-cfeb-d717-4dd39df40751: updating configuration: command=RemoveServer server-id=server-30786f19-75e0-f837-0654-eebcbfedee28 server-addr= servers="[{Suffrage:Voter ID:server-2cccd661-27a5-cfeb-d717-4dd39df40751 Address:2cccd661-27a5-cfeb-d717-4dd39df40751} {Suffrage:Voter ID:server-afd02fb0-7461-ef47-630f-762f2233dba3 Address:afd02fb0-7461-ef47-630f-762f2233dba3}]" +2025-02-25T23:59:24.222Z [INFO] server-2cccd661-27a5-cfeb-d717-4dd39df40751: removed peer, stopping replication: peer=server-30786f19-75e0-f837-0654-eebcbfedee28 last-index=3 +2025-02-25T23:59:24.222Z [INFO] server-2cccd661-27a5-cfeb-d717-4dd39df40751: aborting pipeline replication: peer="{Voter server-30786f19-75e0-f837-0654-eebcbfedee28 30786f19-75e0-f837-0654-eebcbfedee28}" +2025-02-25T23:59:24.298Z [WARN] server-30786f19-75e0-f837-0654-eebcbfedee28: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +2025-02-25T23:59:24.330Z [ERROR] server-2cccd661-27a5-cfeb-d717-4dd39df40751: failed to heartbeat to: peer=afd02fb0-7461-ef47-630f-762f2233dba3 backoff time=10ms error="failed to connect to peer: afd02fb0-7461-ef47-630f-762f2233dba3" +2025-02-25T23:59:24.338Z [ERROR] server-2cccd661-27a5-cfeb-d717-4dd39df40751: failed to pipeline appendEntries: peer="{Voter server-afd02fb0-7461-ef47-630f-762f2233dba3 afd02fb0-7461-ef47-630f-762f2233dba3}" error="append pipeline closed" +2025-02-25T23:59:24.338Z [INFO] server-2cccd661-27a5-cfeb-d717-4dd39df40751: aborting pipeline replication: peer="{Voter server-afd02fb0-7461-ef47-630f-762f2233dba3 afd02fb0-7461-ef47-630f-762f2233dba3}" +2025-02-25T23:59:24.347Z [ERROR] server-2cccd661-27a5-cfeb-d717-4dd39df40751: failed to heartbeat to: peer=afd02fb0-7461-ef47-630f-762f2233dba3 backoff time=10ms error="failed to connect to peer: afd02fb0-7461-ef47-630f-762f2233dba3" +2025-02-25T23:59:24.348Z [ERROR] server-2cccd661-27a5-cfeb-d717-4dd39df40751: failed to appendEntries to: peer="{Voter server-afd02fb0-7461-ef47-630f-762f2233dba3 afd02fb0-7461-ef47-630f-762f2233dba3}" error="failed to connect to peer: afd02fb0-7461-ef47-630f-762f2233dba3" +2025-02-25T23:59:24.370Z [WARN] server-2cccd661-27a5-cfeb-d717-4dd39df40751: failed to contact: server-id=server-afd02fb0-7461-ef47-630f-762f2233dba3 time=55.421633ms +2025-02-25T23:59:24.370Z [WARN] server-2cccd661-27a5-cfeb-d717-4dd39df40751: failed to contact quorum of nodes, stepping down +2025-02-25T23:59:24.370Z [INFO] server-2cccd661-27a5-cfeb-d717-4dd39df40751: entering follower state: follower="Node at 2cccd661-27a5-cfeb-d717-4dd39df40751 [Follower]" leader-address= leader-id= +2025-02-25T23:59:24.370Z [ERROR] server-2cccd661-27a5-cfeb-d717-4dd39df40751: failed to appendEntries to: peer="{Voter server-afd02fb0-7461-ef47-630f-762f2233dba3 afd02fb0-7461-ef47-630f-762f2233dba3}" error="failed to connect to peer: afd02fb0-7461-ef47-630f-762f2233dba3" +2025-02-25T23:59:24.370Z [ERROR] server-2cccd661-27a5-cfeb-d717-4dd39df40751: failed to heartbeat to: peer=afd02fb0-7461-ef47-630f-762f2233dba3 backoff time=10ms error="failed to connect to peer: afd02fb0-7461-ef47-630f-762f2233dba3" +2025-02-25T23:59:24.394Z [WARN] server-afd02fb0-7461-ef47-630f-762f2233dba3: heartbeat timeout reached, starting election: last-leader-addr=2cccd661-27a5-cfeb-d717-4dd39df40751 last-leader-id=server-2cccd661-27a5-cfeb-d717-4dd39df40751 +2025-02-25T23:59:24.394Z [INFO] server-afd02fb0-7461-ef47-630f-762f2233dba3: entering candidate state: node="Node at afd02fb0-7461-ef47-630f-762f2233dba3 [Candidate]" term=3 +2025-02-25T23:59:24.394Z [ERROR] server-afd02fb0-7461-ef47-630f-762f2233dba3: failed to make requestVote RPC: target="{Voter server-2cccd661-27a5-cfeb-d717-4dd39df40751 2cccd661-27a5-cfeb-d717-4dd39df40751}" error="failed to connect to peer: 2cccd661-27a5-cfeb-d717-4dd39df40751" term=3 +2025-02-25T23:59:24.426Z [WARN] server-2cccd661-27a5-cfeb-d717-4dd39df40751: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:24.426Z [INFO] server-2cccd661-27a5-cfeb-d717-4dd39df40751: entering candidate state: node="Node at 2cccd661-27a5-cfeb-d717-4dd39df40751 [Candidate]" term=3 +2025-02-25T23:59:24.426Z [ERROR] server-2cccd661-27a5-cfeb-d717-4dd39df40751: failed to make requestVote RPC: target="{Voter server-afd02fb0-7461-ef47-630f-762f2233dba3 afd02fb0-7461-ef47-630f-762f2233dba3}" error="failed to connect to peer: afd02fb0-7461-ef47-630f-762f2233dba3" term=3 +2025-02-25T23:59:24.437Z [WARN] server-afd02fb0-7461-ef47-630f-762f2233dba3: rejecting vote request since node is not in configuration: from=30786f19-75e0-f837-0654-eebcbfedee28 --- PASS: TestRaft_VoteNotGranted_WhenNodeNotInCluster (0.50s) === RUN TestRaft_ClusterCanRegainStability_WhenNonVoterWithHigherTermJoin -2026-03-31T06:17:59.959Z [INFO] server-1069ee2b-29b0-04ed-d3f8-9147982749e2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1069ee2b-29b0-04ed-d3f8-9147982749e2 Address:1069ee2b-29b0-04ed-d3f8-9147982749e2} {Suffrage:Voter ID:server-6b399796-cfe2-e9d2-fdb4-9813a1894569 Address:6b399796-cfe2-e9d2-fdb4-9813a1894569} {Suffrage:Voter ID:server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91 Address:d3dcd5fc-9f21-d4d4-397a-1f4073024c91}]" -2026-03-31T06:17:59.960Z [INFO] server-6b399796-cfe2-e9d2-fdb4-9813a1894569: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1069ee2b-29b0-04ed-d3f8-9147982749e2 Address:1069ee2b-29b0-04ed-d3f8-9147982749e2} {Suffrage:Voter ID:server-6b399796-cfe2-e9d2-fdb4-9813a1894569 Address:6b399796-cfe2-e9d2-fdb4-9813a1894569} {Suffrage:Voter ID:server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91 Address:d3dcd5fc-9f21-d4d4-397a-1f4073024c91}]" -2026-03-31T06:17:59.960Z [INFO] server-1069ee2b-29b0-04ed-d3f8-9147982749e2: entering follower state: follower="Node at 1069ee2b-29b0-04ed-d3f8-9147982749e2 [Follower]" leader-address= leader-id= -2026-03-31T06:17:59.960Z [INFO] server-6b399796-cfe2-e9d2-fdb4-9813a1894569: entering follower state: follower="Node at 6b399796-cfe2-e9d2-fdb4-9813a1894569 [Follower]" leader-address= leader-id= -2026-03-31T06:17:59.960Z [INFO] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1069ee2b-29b0-04ed-d3f8-9147982749e2 Address:1069ee2b-29b0-04ed-d3f8-9147982749e2} {Suffrage:Voter ID:server-6b399796-cfe2-e9d2-fdb4-9813a1894569 Address:6b399796-cfe2-e9d2-fdb4-9813a1894569} {Suffrage:Voter ID:server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91 Address:d3dcd5fc-9f21-d4d4-397a-1f4073024c91}]" -2026-03-31T06:17:59.960Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:17:59.960Z [INFO] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: entering follower state: follower="Node at d3dcd5fc-9f21-d4d4-397a-1f4073024c91 [Follower]" leader-address= leader-id= -2026-03-31T06:18:00.027Z [WARN] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:18:00.027Z [INFO] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: entering candidate state: node="Node at d3dcd5fc-9f21-d4d4-397a-1f4073024c91 [Candidate]" term=2 -2026-03-31T06:18:00.027Z [INFO] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: election won: term=2 tally=2 -2026-03-31T06:18:00.027Z [INFO] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: entering leader state: leader="Node at d3dcd5fc-9f21-d4d4-397a-1f4073024c91 [Leader]" -2026-03-31T06:18:00.027Z [INFO] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: added peer, starting replication: peer=server-1069ee2b-29b0-04ed-d3f8-9147982749e2 -2026-03-31T06:18:00.027Z [INFO] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: added peer, starting replication: peer=server-6b399796-cfe2-e9d2-fdb4-9813a1894569 -2026-03-31T06:18:00.027Z [INFO] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: pipelining replication: peer="{Voter server-6b399796-cfe2-e9d2-fdb4-9813a1894569 6b399796-cfe2-e9d2-fdb4-9813a1894569}" -2026-03-31T06:18:00.028Z [INFO] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: pipelining replication: peer="{Voter server-1069ee2b-29b0-04ed-d3f8-9147982749e2 1069ee2b-29b0-04ed-d3f8-9147982749e2}" -2026-03-31T06:18:00.133Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:18:00.02797118 +0000 UTC m=+108.305578601 (1 nodes), 67.7486ms from start of poll, 68.19228ms from cluster start. Timeout at 2026-03-31 06:18:00.13297418 +0000 UTC m=+108.410581561, 105.00296ms after stability -2026-03-31T06:18:00.139Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:18:00.244Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:18:00.13909882 +0000 UTC m=+108.416706201 (2 nodes), 1.92µs from start of poll, 179.31988ms from cluster start. Timeout at 2026-03-31 06:18:00.24412982 +0000 UTC m=+108.521737201, 105.031ms after stability -2026-03-31T06:18:00.245Z [ERROR] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: failed to heartbeat to: peer=1069ee2b-29b0-04ed-d3f8-9147982749e2 backoff time=10ms error="failed to connect to peer: 1069ee2b-29b0-04ed-d3f8-9147982749e2" -2026-03-31T06:18:00.251Z [ERROR] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: failed to pipeline appendEntries: peer="{Voter server-1069ee2b-29b0-04ed-d3f8-9147982749e2 1069ee2b-29b0-04ed-d3f8-9147982749e2}" error="append pipeline closed" -2026-03-31T06:18:00.251Z [INFO] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: aborting pipeline replication: peer="{Voter server-1069ee2b-29b0-04ed-d3f8-9147982749e2 1069ee2b-29b0-04ed-d3f8-9147982749e2}" -2026-03-31T06:18:00.261Z [ERROR] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: failed to appendEntries to: peer="{Voter server-1069ee2b-29b0-04ed-d3f8-9147982749e2 1069ee2b-29b0-04ed-d3f8-9147982749e2}" error="failed to connect to peer: 1069ee2b-29b0-04ed-d3f8-9147982749e2" -2026-03-31T06:18:00.264Z [ERROR] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: failed to heartbeat to: peer=1069ee2b-29b0-04ed-d3f8-9147982749e2 backoff time=10ms error="failed to connect to peer: 1069ee2b-29b0-04ed-d3f8-9147982749e2" -2026-03-31T06:18:00.281Z [ERROR] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: failed to heartbeat to: peer=1069ee2b-29b0-04ed-d3f8-9147982749e2 backoff time=10ms error="failed to connect to peer: 1069ee2b-29b0-04ed-d3f8-9147982749e2" -2026-03-31T06:18:00.282Z [ERROR] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: failed to appendEntries to: peer="{Voter server-1069ee2b-29b0-04ed-d3f8-9147982749e2 1069ee2b-29b0-04ed-d3f8-9147982749e2}" error="failed to connect to peer: 1069ee2b-29b0-04ed-d3f8-9147982749e2" -2026-03-31T06:18:00.293Z [WARN] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: failed to contact: server-id=server-1069ee2b-29b0-04ed-d3f8-9147982749e2 time=50.37372ms -2026-03-31T06:18:00.298Z [ERROR] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: failed to heartbeat to: peer=1069ee2b-29b0-04ed-d3f8-9147982749e2 backoff time=20ms error="failed to connect to peer: 1069ee2b-29b0-04ed-d3f8-9147982749e2" -2026-03-31T06:18:00.301Z [ERROR] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: failed to appendEntries to: peer="{Voter server-1069ee2b-29b0-04ed-d3f8-9147982749e2 1069ee2b-29b0-04ed-d3f8-9147982749e2}" error="failed to connect to peer: 1069ee2b-29b0-04ed-d3f8-9147982749e2" -2026-03-31T06:18:00.327Z [ERROR] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: failed to heartbeat to: peer=1069ee2b-29b0-04ed-d3f8-9147982749e2 backoff time=25ms error="failed to connect to peer: 1069ee2b-29b0-04ed-d3f8-9147982749e2" -2026-03-31T06:18:00.328Z [ERROR] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: failed to appendEntries to: peer="{Voter server-1069ee2b-29b0-04ed-d3f8-9147982749e2 1069ee2b-29b0-04ed-d3f8-9147982749e2}" error="failed to connect to peer: 1069ee2b-29b0-04ed-d3f8-9147982749e2" -2026-03-31T06:18:00.342Z [WARN] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: failed to contact: server-id=server-1069ee2b-29b0-04ed-d3f8-9147982749e2 time=99.50116ms -2026-03-31T06:18:00.349Z [INFO] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: updating configuration: command=RemoveServer server-id=server-1069ee2b-29b0-04ed-d3f8-9147982749e2 server-addr= servers="[{Suffrage:Voter ID:server-6b399796-cfe2-e9d2-fdb4-9813a1894569 Address:6b399796-cfe2-e9d2-fdb4-9813a1894569} {Suffrage:Voter ID:server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91 Address:d3dcd5fc-9f21-d4d4-397a-1f4073024c91}]" -2026-03-31T06:18:00.349Z [INFO] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: removed peer, stopping replication: peer=server-1069ee2b-29b0-04ed-d3f8-9147982749e2 last-index=3 -2026-03-31T06:18:00.350Z [INFO] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: updating configuration: command=AddNonvoter server-id=server-1069ee2b-29b0-04ed-d3f8-9147982749e2 server-addr=1069ee2b-29b0-04ed-d3f8-9147982749e2 servers="[{Suffrage:Voter ID:server-6b399796-cfe2-e9d2-fdb4-9813a1894569 Address:6b399796-cfe2-e9d2-fdb4-9813a1894569} {Suffrage:Voter ID:server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91 Address:d3dcd5fc-9f21-d4d4-397a-1f4073024c91} {Suffrage:Nonvoter ID:server-1069ee2b-29b0-04ed-d3f8-9147982749e2 Address:1069ee2b-29b0-04ed-d3f8-9147982749e2}]" -2026-03-31T06:18:00.350Z [INFO] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: added peer, starting replication: peer=server-1069ee2b-29b0-04ed-d3f8-9147982749e2 -2026-03-31T06:18:00.350Z [ERROR] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: failed to appendEntries to: peer="{Nonvoter server-1069ee2b-29b0-04ed-d3f8-9147982749e2 1069ee2b-29b0-04ed-d3f8-9147982749e2}" error="failed to connect to peer: 1069ee2b-29b0-04ed-d3f8-9147982749e2" -2026-03-31T06:18:00.364Z [WARN] server-1069ee2b-29b0-04ed-d3f8-9147982749e2: heartbeat timeout reached, starting election: last-leader-addr=d3dcd5fc-9f21-d4d4-397a-1f4073024c91 last-leader-id=server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91 -2026-03-31T06:18:00.364Z [INFO] server-1069ee2b-29b0-04ed-d3f8-9147982749e2: entering candidate state: node="Node at 1069ee2b-29b0-04ed-d3f8-9147982749e2 [Candidate]" term=23 -2026-03-31T06:18:00.364Z [WARN] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: rejecting vote request since we have a leader: from=1069ee2b-29b0-04ed-d3f8-9147982749e2 leader=d3dcd5fc-9f21-d4d4-397a-1f4073024c91 leader-id=server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91 -2026-03-31T06:18:00.364Z [WARN] server-6b399796-cfe2-e9d2-fdb4-9813a1894569: rejecting vote request since we have a leader: from=1069ee2b-29b0-04ed-d3f8-9147982749e2 leader=d3dcd5fc-9f21-d4d4-397a-1f4073024c91 leader-id=server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91 -2026-03-31T06:18:00.371Z [ERROR] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: peer has newer term, stopping replication: peer="{Nonvoter server-1069ee2b-29b0-04ed-d3f8-9147982749e2 1069ee2b-29b0-04ed-d3f8-9147982749e2}" -2026-03-31T06:18:00.371Z [INFO] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: entering follower state: follower="Node at d3dcd5fc-9f21-d4d4-397a-1f4073024c91 [Follower]" leader-address= leader-id= -2026-03-31T06:18:00.371Z [INFO] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: aborting pipeline replication: peer="{Voter server-6b399796-cfe2-e9d2-fdb4-9813a1894569 6b399796-cfe2-e9d2-fdb4-9813a1894569}" -2026-03-31T06:18:00.377Z [ERROR] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: peer has newer term, stopping replication: peer="{Voter server-1069ee2b-29b0-04ed-d3f8-9147982749e2 1069ee2b-29b0-04ed-d3f8-9147982749e2}" -2026-03-31T06:18:00.428Z [WARN] server-1069ee2b-29b0-04ed-d3f8-9147982749e2: Election timeout reached, restarting election -2026-03-31T06:18:00.428Z [INFO] server-1069ee2b-29b0-04ed-d3f8-9147982749e2: entering candidate state: node="Node at 1069ee2b-29b0-04ed-d3f8-9147982749e2 [Candidate]" term=24 -2026-03-31T06:18:00.428Z [WARN] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: rejecting vote request since node is not a voter: from=1069ee2b-29b0-04ed-d3f8-9147982749e2 -2026-03-31T06:18:00.428Z [WARN] server-6b399796-cfe2-e9d2-fdb4-9813a1894569: rejecting vote request since we have a leader: from=1069ee2b-29b0-04ed-d3f8-9147982749e2 leader=d3dcd5fc-9f21-d4d4-397a-1f4073024c91 leader-id=server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91 -2026-03-31T06:18:00.456Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:18:00.464Z [WARN] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:18:00.464Z [INFO] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: entering candidate state: node="Node at d3dcd5fc-9f21-d4d4-397a-1f4073024c91 [Candidate]" term=25 -2026-03-31T06:18:00.464Z [INFO] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: election won: term=25 tally=2 -2026-03-31T06:18:00.464Z [INFO] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: entering leader state: leader="Node at d3dcd5fc-9f21-d4d4-397a-1f4073024c91 [Leader]" -2026-03-31T06:18:00.464Z [INFO] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: added peer, starting replication: peer=server-6b399796-cfe2-e9d2-fdb4-9813a1894569 -2026-03-31T06:18:00.464Z [INFO] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: added peer, starting replication: peer=server-1069ee2b-29b0-04ed-d3f8-9147982749e2 -2026-03-31T06:18:00.464Z [WARN] server-1069ee2b-29b0-04ed-d3f8-9147982749e2: failed to get previous log: previous-index=4 last-index=2 error="log not found" -2026-03-31T06:18:00.464Z [INFO] server-1069ee2b-29b0-04ed-d3f8-9147982749e2: entering follower state: follower="Node at 1069ee2b-29b0-04ed-d3f8-9147982749e2 [Follower]" leader-address=d3dcd5fc-9f21-d4d4-397a-1f4073024c91 leader-id=server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91 -2026-03-31T06:18:00.464Z [INFO] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: pipelining replication: peer="{Voter server-6b399796-cfe2-e9d2-fdb4-9813a1894569 6b399796-cfe2-e9d2-fdb4-9813a1894569}" -2026-03-31T06:18:00.464Z [WARN] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: appendEntries rejected, sending older logs: peer="{Nonvoter server-1069ee2b-29b0-04ed-d3f8-9147982749e2 1069ee2b-29b0-04ed-d3f8-9147982749e2}" next=3 -2026-03-31T06:18:00.464Z [INFO] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: pipelining replication: peer="{Nonvoter server-1069ee2b-29b0-04ed-d3f8-9147982749e2 1069ee2b-29b0-04ed-d3f8-9147982749e2}" -2026-03-31T06:18:00.569Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:18:00.4644643 +0000 UTC m=+108.742071681 (1 nodes), 7.72128ms from start of poll, 504.68536ms from cluster start. Timeout at 2026-03-31 06:18:00.56946746 +0000 UTC m=+108.847074841, 105.00316ms after stability -2026-03-31T06:18:00.595Z [INFO] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: updating configuration: command=RemoveServer server-id=server-1069ee2b-29b0-04ed-d3f8-9147982749e2 server-addr= servers="[{Suffrage:Voter ID:server-6b399796-cfe2-e9d2-fdb4-9813a1894569 Address:6b399796-cfe2-e9d2-fdb4-9813a1894569} {Suffrage:Voter ID:server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91 Address:d3dcd5fc-9f21-d4d4-397a-1f4073024c91}]" -2026-03-31T06:18:00.595Z [INFO] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: removed peer, stopping replication: peer=server-1069ee2b-29b0-04ed-d3f8-9147982749e2 last-index=106 -2026-03-31T06:18:00.596Z [INFO] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: aborting pipeline replication: peer="{Nonvoter server-1069ee2b-29b0-04ed-d3f8-9147982749e2 1069ee2b-29b0-04ed-d3f8-9147982749e2}" -2026-03-31T06:18:00.596Z [INFO] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: updating configuration: command=AddVoter server-id=server-1069ee2b-29b0-04ed-d3f8-9147982749e2 server-addr=1069ee2b-29b0-04ed-d3f8-9147982749e2 servers="[{Suffrage:Voter ID:server-6b399796-cfe2-e9d2-fdb4-9813a1894569 Address:6b399796-cfe2-e9d2-fdb4-9813a1894569} {Suffrage:Voter ID:server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91 Address:d3dcd5fc-9f21-d4d4-397a-1f4073024c91} {Suffrage:Voter ID:server-1069ee2b-29b0-04ed-d3f8-9147982749e2 Address:1069ee2b-29b0-04ed-d3f8-9147982749e2}]" -2026-03-31T06:18:00.596Z [INFO] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: added peer, starting replication: peer=server-1069ee2b-29b0-04ed-d3f8-9147982749e2 -2026-03-31T06:18:00.596Z [WARN] server-1069ee2b-29b0-04ed-d3f8-9147982749e2: failed to get previous log: previous-index=107 last-index=106 error="log not found" -2026-03-31T06:18:00.596Z [WARN] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: appendEntries rejected, sending older logs: peer="{Voter server-1069ee2b-29b0-04ed-d3f8-9147982749e2 1069ee2b-29b0-04ed-d3f8-9147982749e2}" next=107 -2026-03-31T06:18:00.596Z [INFO] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: pipelining replication: peer="{Voter server-1069ee2b-29b0-04ed-d3f8-9147982749e2 1069ee2b-29b0-04ed-d3f8-9147982749e2}" -2026-03-31T06:18:01.669Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:18:01.775Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:18:01.66926138 +0000 UTC m=+109.946868761 (1 nodes), 1.12µs from start of poll, 1.70948244s from cluster start. Timeout at 2026-03-31 06:18:01.77429386 +0000 UTC m=+110.051901281, 105.03252ms after stability -2026-03-31T06:18:01.775Z [INFO] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: aborting pipeline replication: peer="{Voter server-1069ee2b-29b0-04ed-d3f8-9147982749e2 1069ee2b-29b0-04ed-d3f8-9147982749e2}" -2026-03-31T06:18:01.775Z [INFO] server-d3dcd5fc-9f21-d4d4-397a-1f4073024c91: aborting pipeline replication: peer="{Voter server-6b399796-cfe2-e9d2-fdb4-9813a1894569 6b399796-cfe2-e9d2-fdb4-9813a1894569}" ---- PASS: TestRaft_ClusterCanRegainStability_WhenNonVoterWithHigherTermJoin (1.82s) +2025-02-25T23:59:24.438Z [INFO] server-88e0084d-3e26-05db-b73a-265cca950214: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-88e0084d-3e26-05db-b73a-265cca950214 Address:88e0084d-3e26-05db-b73a-265cca950214} {Suffrage:Voter ID:server-3a6ecf89-c914-ea73-59b4-1c5d3849c08e Address:3a6ecf89-c914-ea73-59b4-1c5d3849c08e} {Suffrage:Voter ID:server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8 Address:7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8}]" +2025-02-25T23:59:24.438Z [INFO] server-3a6ecf89-c914-ea73-59b4-1c5d3849c08e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-88e0084d-3e26-05db-b73a-265cca950214 Address:88e0084d-3e26-05db-b73a-265cca950214} {Suffrage:Voter ID:server-3a6ecf89-c914-ea73-59b4-1c5d3849c08e Address:3a6ecf89-c914-ea73-59b4-1c5d3849c08e} {Suffrage:Voter ID:server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8 Address:7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8}]" +2025-02-25T23:59:24.438Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-88e0084d-3e26-05db-b73a-265cca950214 Address:88e0084d-3e26-05db-b73a-265cca950214} {Suffrage:Voter ID:server-3a6ecf89-c914-ea73-59b4-1c5d3849c08e Address:3a6ecf89-c914-ea73-59b4-1c5d3849c08e} {Suffrage:Voter ID:server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8 Address:7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8}]" +2025-02-25T23:59:24.438Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:24.438Z [INFO] server-88e0084d-3e26-05db-b73a-265cca950214: entering follower state: follower="Node at 88e0084d-3e26-05db-b73a-265cca950214 [Follower]" leader-address= leader-id= +2025-02-25T23:59:24.438Z [INFO] server-3a6ecf89-c914-ea73-59b4-1c5d3849c08e: entering follower state: follower="Node at 3a6ecf89-c914-ea73-59b4-1c5d3849c08e [Follower]" leader-address= leader-id= +2025-02-25T23:59:24.438Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: entering follower state: follower="Node at 7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8 [Follower]" leader-address= leader-id= +2025-02-25T23:59:24.489Z [WARN] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:24.489Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: entering candidate state: node="Node at 7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8 [Candidate]" term=2 +2025-02-25T23:59:24.490Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: election won: term=2 tally=2 +2025-02-25T23:59:24.490Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: entering leader state: leader="Node at 7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8 [Leader]" +2025-02-25T23:59:24.490Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: added peer, starting replication: peer=server-88e0084d-3e26-05db-b73a-265cca950214 +2025-02-25T23:59:24.490Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: added peer, starting replication: peer=server-3a6ecf89-c914-ea73-59b4-1c5d3849c08e +2025-02-25T23:59:24.490Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: pipelining replication: peer="{Voter server-3a6ecf89-c914-ea73-59b4-1c5d3849c08e 3a6ecf89-c914-ea73-59b4-1c5d3849c08e}" +2025-02-25T23:59:24.490Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: pipelining replication: peer="{Voter server-88e0084d-3e26-05db-b73a-265cca950214 88e0084d-3e26-05db-b73a-265cca950214}" +2025-02-25T23:59:24.597Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:24.490415246 +0000 UTC m=+112.099430653 (1 nodes), 51.81817ms from start of poll, 52.013371ms from cluster start. Timeout at 2025-02-25 23:59:24.595418806 +0000 UTC m=+112.204434213, 105.00356ms after stability +2025-02-25T23:59:24.610Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:59:24.718Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:59:24.610142208 +0000 UTC m=+112.219157615 (2 nodes), 1.04µs from start of poll, 171.740333ms from cluster start. Timeout at 2025-02-25 23:59:24.715170529 +0000 UTC m=+112.324185936, 105.028321ms after stability +2025-02-25T23:59:24.725Z [ERROR] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: failed to pipeline appendEntries: peer="{Voter server-88e0084d-3e26-05db-b73a-265cca950214 88e0084d-3e26-05db-b73a-265cca950214}" error="append pipeline closed" +2025-02-25T23:59:24.725Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: aborting pipeline replication: peer="{Voter server-88e0084d-3e26-05db-b73a-265cca950214 88e0084d-3e26-05db-b73a-265cca950214}" +2025-02-25T23:59:24.729Z [ERROR] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: failed to heartbeat to: peer=88e0084d-3e26-05db-b73a-265cca950214 backoff time=10ms error="failed to connect to peer: 88e0084d-3e26-05db-b73a-265cca950214" +2025-02-25T23:59:24.731Z [ERROR] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: failed to appendEntries to: peer="{Voter server-88e0084d-3e26-05db-b73a-265cca950214 88e0084d-3e26-05db-b73a-265cca950214}" error="failed to connect to peer: 88e0084d-3e26-05db-b73a-265cca950214" +2025-02-25T23:59:24.758Z [ERROR] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: failed to heartbeat to: peer=88e0084d-3e26-05db-b73a-265cca950214 backoff time=10ms error="failed to connect to peer: 88e0084d-3e26-05db-b73a-265cca950214" +2025-02-25T23:59:24.758Z [ERROR] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: failed to appendEntries to: peer="{Voter server-88e0084d-3e26-05db-b73a-265cca950214 88e0084d-3e26-05db-b73a-265cca950214}" error="failed to connect to peer: 88e0084d-3e26-05db-b73a-265cca950214" +2025-02-25T23:59:24.770Z [WARN] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: failed to contact: server-id=server-88e0084d-3e26-05db-b73a-265cca950214 time=51.80593ms +2025-02-25T23:59:24.778Z [ERROR] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: failed to heartbeat to: peer=88e0084d-3e26-05db-b73a-265cca950214 backoff time=10ms error="failed to connect to peer: 88e0084d-3e26-05db-b73a-265cca950214" +2025-02-25T23:59:24.778Z [ERROR] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: failed to appendEntries to: peer="{Voter server-88e0084d-3e26-05db-b73a-265cca950214 88e0084d-3e26-05db-b73a-265cca950214}" error="failed to connect to peer: 88e0084d-3e26-05db-b73a-265cca950214" +2025-02-25T23:59:24.797Z [ERROR] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: failed to heartbeat to: peer=88e0084d-3e26-05db-b73a-265cca950214 backoff time=20ms error="failed to connect to peer: 88e0084d-3e26-05db-b73a-265cca950214" +2025-02-25T23:59:24.810Z [ERROR] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: failed to appendEntries to: peer="{Voter server-88e0084d-3e26-05db-b73a-265cca950214 88e0084d-3e26-05db-b73a-265cca950214}" error="failed to connect to peer: 88e0084d-3e26-05db-b73a-265cca950214" +2025-02-25T23:59:24.822Z [WARN] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: failed to contact: server-id=server-88e0084d-3e26-05db-b73a-265cca950214 time=103.889981ms +2025-02-25T23:59:24.826Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: updating configuration: command=RemoveServer server-id=server-88e0084d-3e26-05db-b73a-265cca950214 server-addr= servers="[{Suffrage:Voter ID:server-3a6ecf89-c914-ea73-59b4-1c5d3849c08e Address:3a6ecf89-c914-ea73-59b4-1c5d3849c08e} {Suffrage:Voter ID:server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8 Address:7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8}]" +2025-02-25T23:59:24.826Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: removed peer, stopping replication: peer=server-88e0084d-3e26-05db-b73a-265cca950214 last-index=3 +2025-02-25T23:59:24.826Z [ERROR] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: failed to heartbeat to: peer=88e0084d-3e26-05db-b73a-265cca950214 backoff time=25ms error="failed to connect to peer: 88e0084d-3e26-05db-b73a-265cca950214" +2025-02-25T23:59:24.826Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: updating configuration: command=AddNonvoter server-id=server-88e0084d-3e26-05db-b73a-265cca950214 server-addr=88e0084d-3e26-05db-b73a-265cca950214 servers="[{Suffrage:Voter ID:server-3a6ecf89-c914-ea73-59b4-1c5d3849c08e Address:3a6ecf89-c914-ea73-59b4-1c5d3849c08e} {Suffrage:Voter ID:server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8 Address:7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8} {Suffrage:Nonvoter ID:server-88e0084d-3e26-05db-b73a-265cca950214 Address:88e0084d-3e26-05db-b73a-265cca950214}]" +2025-02-25T23:59:24.826Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: added peer, starting replication: peer=server-88e0084d-3e26-05db-b73a-265cca950214 +2025-02-25T23:59:24.826Z [ERROR] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: failed to appendEntries to: peer="{Nonvoter server-88e0084d-3e26-05db-b73a-265cca950214 88e0084d-3e26-05db-b73a-265cca950214}" error="failed to connect to peer: 88e0084d-3e26-05db-b73a-265cca950214" +2025-02-25T23:59:24.837Z [WARN] server-88e0084d-3e26-05db-b73a-265cca950214: heartbeat timeout reached, starting election: last-leader-addr=7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8 last-leader-id=server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8 +2025-02-25T23:59:24.837Z [INFO] server-88e0084d-3e26-05db-b73a-265cca950214: entering candidate state: node="Node at 88e0084d-3e26-05db-b73a-265cca950214 [Candidate]" term=23 +2025-02-25T23:59:24.838Z [WARN] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: rejecting vote request since we have a leader: from=88e0084d-3e26-05db-b73a-265cca950214 leader=7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8 leader-id=server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8 +2025-02-25T23:59:24.838Z [WARN] server-3a6ecf89-c914-ea73-59b4-1c5d3849c08e: rejecting vote request since we have a leader: from=88e0084d-3e26-05db-b73a-265cca950214 leader=7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8 leader-id=server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8 +2025-02-25T23:59:24.844Z [ERROR] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: peer has newer term, stopping replication: peer="{Nonvoter server-88e0084d-3e26-05db-b73a-265cca950214 88e0084d-3e26-05db-b73a-265cca950214}" +2025-02-25T23:59:24.844Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: entering follower state: follower="Node at 7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8 [Follower]" leader-address= leader-id= +2025-02-25T23:59:24.844Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: aborting pipeline replication: peer="{Voter server-3a6ecf89-c914-ea73-59b4-1c5d3849c08e 3a6ecf89-c914-ea73-59b4-1c5d3849c08e}" +2025-02-25T23:59:24.858Z [ERROR] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: peer has newer term, stopping replication: peer="{Voter server-88e0084d-3e26-05db-b73a-265cca950214 88e0084d-3e26-05db-b73a-265cca950214}" +2025-02-25T23:59:24.906Z [WARN] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:24.906Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: entering candidate state: node="Node at 7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8 [Candidate]" term=3 +2025-02-25T23:59:24.906Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: election won: term=3 tally=2 +2025-02-25T23:59:24.906Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: entering leader state: leader="Node at 7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8 [Leader]" +2025-02-25T23:59:24.906Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: added peer, starting replication: peer=server-3a6ecf89-c914-ea73-59b4-1c5d3849c08e +2025-02-25T23:59:24.906Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: added peer, starting replication: peer=server-88e0084d-3e26-05db-b73a-265cca950214 +2025-02-25T23:59:24.906Z [ERROR] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: peer has newer term, stopping replication: peer="{Nonvoter server-88e0084d-3e26-05db-b73a-265cca950214 88e0084d-3e26-05db-b73a-265cca950214}" +2025-02-25T23:59:24.906Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: entering follower state: follower="Node at 7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8 [Follower]" leader-address= leader-id= +2025-02-25T23:59:24.917Z [WARN] server-88e0084d-3e26-05db-b73a-265cca950214: Election timeout reached, restarting election +2025-02-25T23:59:24.917Z [INFO] server-88e0084d-3e26-05db-b73a-265cca950214: entering candidate state: node="Node at 88e0084d-3e26-05db-b73a-265cca950214 [Candidate]" term=24 +2025-02-25T23:59:24.917Z [WARN] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: rejecting vote request since node is not a voter: from=88e0084d-3e26-05db-b73a-265cca950214 +2025-02-25T23:59:24.918Z [WARN] server-3a6ecf89-c914-ea73-59b4-1c5d3849c08e: rejecting vote request since node is not a voter: from=88e0084d-3e26-05db-b73a-265cca950214 +2025-02-25T23:59:24.933Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:24.963Z [WARN] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:24.963Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: entering candidate state: node="Node at 7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8 [Candidate]" term=25 +2025-02-25T23:59:24.963Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: election won: term=25 tally=2 +2025-02-25T23:59:24.963Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: entering leader state: leader="Node at 7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8 [Leader]" +2025-02-25T23:59:24.963Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: added peer, starting replication: peer=server-3a6ecf89-c914-ea73-59b4-1c5d3849c08e +2025-02-25T23:59:24.963Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: added peer, starting replication: peer=server-88e0084d-3e26-05db-b73a-265cca950214 +2025-02-25T23:59:24.963Z [WARN] server-88e0084d-3e26-05db-b73a-265cca950214: failed to get previous log: previous-index=5 last-index=2 error="log not found" +2025-02-25T23:59:24.963Z [INFO] server-88e0084d-3e26-05db-b73a-265cca950214: entering follower state: follower="Node at 88e0084d-3e26-05db-b73a-265cca950214 [Follower]" leader-address=7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8 leader-id=server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8 +2025-02-25T23:59:24.963Z [WARN] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: appendEntries rejected, sending older logs: peer="{Nonvoter server-88e0084d-3e26-05db-b73a-265cca950214 88e0084d-3e26-05db-b73a-265cca950214}" next=3 +2025-02-25T23:59:24.963Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: pipelining replication: peer="{Nonvoter server-88e0084d-3e26-05db-b73a-265cca950214 88e0084d-3e26-05db-b73a-265cca950214}" +2025-02-25T23:59:24.963Z [WARN] server-3a6ecf89-c914-ea73-59b4-1c5d3849c08e: failed to get previous log: previous-index=5 last-index=4 error="log not found" +2025-02-25T23:59:24.963Z [WARN] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: appendEntries rejected, sending older logs: peer="{Voter server-3a6ecf89-c914-ea73-59b4-1c5d3849c08e 3a6ecf89-c914-ea73-59b4-1c5d3849c08e}" next=5 +2025-02-25T23:59:24.963Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: pipelining replication: peer="{Voter server-3a6ecf89-c914-ea73-59b4-1c5d3849c08e 3a6ecf89-c914-ea73-59b4-1c5d3849c08e}" +2025-02-25T23:59:25.070Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:24.963412417 +0000 UTC m=+112.572427824 (1 nodes), 29.417587ms from start of poll, 525.010542ms from cluster start. Timeout at 2025-02-25 23:59:25.068416297 +0000 UTC m=+112.677431704, 105.00388ms after stability +2025-02-25T23:59:25.105Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: updating configuration: command=RemoveServer server-id=server-88e0084d-3e26-05db-b73a-265cca950214 server-addr= servers="[{Suffrage:Voter ID:server-3a6ecf89-c914-ea73-59b4-1c5d3849c08e Address:3a6ecf89-c914-ea73-59b4-1c5d3849c08e} {Suffrage:Voter ID:server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8 Address:7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8}]" +2025-02-25T23:59:25.105Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: removed peer, stopping replication: peer=server-88e0084d-3e26-05db-b73a-265cca950214 last-index=107 +2025-02-25T23:59:25.106Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: aborting pipeline replication: peer="{Nonvoter server-88e0084d-3e26-05db-b73a-265cca950214 88e0084d-3e26-05db-b73a-265cca950214}" +2025-02-25T23:59:25.106Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: updating configuration: command=AddVoter server-id=server-88e0084d-3e26-05db-b73a-265cca950214 server-addr=88e0084d-3e26-05db-b73a-265cca950214 servers="[{Suffrage:Voter ID:server-3a6ecf89-c914-ea73-59b4-1c5d3849c08e Address:3a6ecf89-c914-ea73-59b4-1c5d3849c08e} {Suffrage:Voter ID:server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8 Address:7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8} {Suffrage:Voter ID:server-88e0084d-3e26-05db-b73a-265cca950214 Address:88e0084d-3e26-05db-b73a-265cca950214}]" +2025-02-25T23:59:25.106Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: added peer, starting replication: peer=server-88e0084d-3e26-05db-b73a-265cca950214 +2025-02-25T23:59:25.106Z [WARN] server-88e0084d-3e26-05db-b73a-265cca950214: failed to get previous log: previous-index=108 last-index=107 error="log not found" +2025-02-25T23:59:25.106Z [WARN] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: appendEntries rejected, sending older logs: peer="{Voter server-88e0084d-3e26-05db-b73a-265cca950214 88e0084d-3e26-05db-b73a-265cca950214}" next=108 +2025-02-25T23:59:25.106Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: pipelining replication: peer="{Voter server-88e0084d-3e26-05db-b73a-265cca950214 88e0084d-3e26-05db-b73a-265cca950214}" +2025-02-25T23:59:26.214Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:26.322Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:26.214061099 +0000 UTC m=+113.823076506 (1 nodes), 760ns from start of poll, 1.775659224s from cluster start. Timeout at 2025-02-25 23:59:26.319087419 +0000 UTC m=+113.928102826, 105.02632ms after stability +2025-02-25T23:59:26.322Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: aborting pipeline replication: peer="{Voter server-88e0084d-3e26-05db-b73a-265cca950214 88e0084d-3e26-05db-b73a-265cca950214}" +2025-02-25T23:59:26.322Z [INFO] server-7f96e1ef-a44d-9eb4-f854-92d3ae85d0a8: aborting pipeline replication: peer="{Voter server-3a6ecf89-c914-ea73-59b4-1c5d3849c08e 3a6ecf89-c914-ea73-59b4-1c5d3849c08e}" +--- PASS: TestRaft_ClusterCanRegainStability_WhenNonVoterWithHigherTermJoin (1.88s) === RUN TestRaft_FollowerRemovalNoElection -2026-03-31T06:18:01.776Z [INFO] server-1532fafc-d768-7ed9-e44e-da1908165de7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1532fafc-d768-7ed9-e44e-da1908165de7 Address:1532fafc-d768-7ed9-e44e-da1908165de7} {Suffrage:Voter ID:server-e4fdf3b0-1333-71be-5b89-134979621997 Address:e4fdf3b0-1333-71be-5b89-134979621997} {Suffrage:Voter ID:server-5a1b57c6-f45c-10f9-bacd-73d7bf5adfb5 Address:5a1b57c6-f45c-10f9-bacd-73d7bf5adfb5}]" -2026-03-31T06:18:01.776Z [INFO] server-e4fdf3b0-1333-71be-5b89-134979621997: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1532fafc-d768-7ed9-e44e-da1908165de7 Address:1532fafc-d768-7ed9-e44e-da1908165de7} {Suffrage:Voter ID:server-e4fdf3b0-1333-71be-5b89-134979621997 Address:e4fdf3b0-1333-71be-5b89-134979621997} {Suffrage:Voter ID:server-5a1b57c6-f45c-10f9-bacd-73d7bf5adfb5 Address:5a1b57c6-f45c-10f9-bacd-73d7bf5adfb5}]" -2026-03-31T06:18:01.776Z [INFO] server-1532fafc-d768-7ed9-e44e-da1908165de7: entering follower state: follower="Node at 1532fafc-d768-7ed9-e44e-da1908165de7 [Follower]" leader-address= leader-id= -2026-03-31T06:18:01.776Z [INFO] server-e4fdf3b0-1333-71be-5b89-134979621997: entering follower state: follower="Node at e4fdf3b0-1333-71be-5b89-134979621997 [Follower]" leader-address= leader-id= -2026-03-31T06:18:01.776Z [INFO] server-5a1b57c6-f45c-10f9-bacd-73d7bf5adfb5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1532fafc-d768-7ed9-e44e-da1908165de7 Address:1532fafc-d768-7ed9-e44e-da1908165de7} {Suffrage:Voter ID:server-e4fdf3b0-1333-71be-5b89-134979621997 Address:e4fdf3b0-1333-71be-5b89-134979621997} {Suffrage:Voter ID:server-5a1b57c6-f45c-10f9-bacd-73d7bf5adfb5 Address:5a1b57c6-f45c-10f9-bacd-73d7bf5adfb5}]" -2026-03-31T06:18:01.776Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:18:01.776Z [INFO] server-5a1b57c6-f45c-10f9-bacd-73d7bf5adfb5: entering follower state: follower="Node at 5a1b57c6-f45c-10f9-bacd-73d7bf5adfb5 [Follower]" leader-address= leader-id= -2026-03-31T06:18:01.934Z [WARN] server-1532fafc-d768-7ed9-e44e-da1908165de7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:18:01.934Z [INFO] server-1532fafc-d768-7ed9-e44e-da1908165de7: entering candidate state: node="Node at 1532fafc-d768-7ed9-e44e-da1908165de7 [Candidate]" term=2 -2026-03-31T06:18:01.934Z [WARN] server-e4fdf3b0-1333-71be-5b89-134979621997: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:18:01.934Z [INFO] server-e4fdf3b0-1333-71be-5b89-134979621997: entering candidate state: node="Node at e4fdf3b0-1333-71be-5b89-134979621997 [Candidate]" term=2 -2026-03-31T06:18:01.934Z [INFO] server-1532fafc-d768-7ed9-e44e-da1908165de7: election won: term=2 tally=2 -2026-03-31T06:18:01.934Z [INFO] server-e4fdf3b0-1333-71be-5b89-134979621997: duplicate requestVote for same term: term=2 -2026-03-31T06:18:01.934Z [INFO] server-1532fafc-d768-7ed9-e44e-da1908165de7: entering leader state: leader="Node at 1532fafc-d768-7ed9-e44e-da1908165de7 [Leader]" -2026-03-31T06:18:01.934Z [INFO] server-1532fafc-d768-7ed9-e44e-da1908165de7: added peer, starting replication: peer=server-e4fdf3b0-1333-71be-5b89-134979621997 -2026-03-31T06:18:01.934Z [INFO] server-1532fafc-d768-7ed9-e44e-da1908165de7: added peer, starting replication: peer=server-5a1b57c6-f45c-10f9-bacd-73d7bf5adfb5 -2026-03-31T06:18:01.934Z [INFO] server-5a1b57c6-f45c-10f9-bacd-73d7bf5adfb5: duplicate requestVote for same term: term=2 -2026-03-31T06:18:01.934Z [WARN] server-1532fafc-d768-7ed9-e44e-da1908165de7: rejecting vote request since we have a leader: from=e4fdf3b0-1333-71be-5b89-134979621997 leader=1532fafc-d768-7ed9-e44e-da1908165de7 leader-id=server-1532fafc-d768-7ed9-e44e-da1908165de7 -2026-03-31T06:18:01.935Z [INFO] server-1532fafc-d768-7ed9-e44e-da1908165de7: pipelining replication: peer="{Voter server-5a1b57c6-f45c-10f9-bacd-73d7bf5adfb5 5a1b57c6-f45c-10f9-bacd-73d7bf5adfb5}" -2026-03-31T06:18:01.935Z [INFO] server-1532fafc-d768-7ed9-e44e-da1908165de7: pipelining replication: peer="{Voter server-e4fdf3b0-1333-71be-5b89-134979621997 e4fdf3b0-1333-71be-5b89-134979621997}" -2026-03-31T06:18:01.935Z [INFO] server-e4fdf3b0-1333-71be-5b89-134979621997: entering follower state: follower="Node at e4fdf3b0-1333-71be-5b89-134979621997 [Follower]" leader-address=1532fafc-d768-7ed9-e44e-da1908165de7 leader-id=server-1532fafc-d768-7ed9-e44e-da1908165de7 -2026-03-31T06:18:02.140Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:18:01.93503534 +0000 UTC m=+110.212642761 (1 nodes), 158.52876ms from start of poll, 158.93716ms from cluster start. Timeout at 2026-03-31 06:18:02.14003662 +0000 UTC m=+110.417644001, 205.00124ms after stability -2026-03-31T06:18:02.140Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:18:02.346Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:18:02.1401679 +0000 UTC m=+110.417775281 (1 nodes), 2.4µs from start of poll, 364.06968ms from cluster start. Timeout at 2026-03-31 06:18:02.3452037 +0000 UTC m=+110.622811081, 205.0358ms after stability -2026-03-31T06:18:02.351Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:18:02.556Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:18:02.3514961 +0000 UTC m=+110.629103521 (2 nodes), 2.92µs from start of poll, 575.39792ms from cluster start. Timeout at 2026-03-31 06:18:02.55652598 +0000 UTC m=+110.834133361, 205.02984ms after stability -2026-03-31T06:18:02.556Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:18:02.762Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:18:02.5566793 +0000 UTC m=+110.834286721 (1 nodes), 2.48µs from start of poll, 780.58112ms from cluster start. Timeout at 2026-03-31 06:18:02.76171398 +0000 UTC m=+111.039321401, 205.03468ms after stability - raft_test.go:2961: [INFO] restarting Node at e4fdf3b0-1333-71be-5b89-134979621997 [Follower] -2026-03-31T06:18:02.762Z [INFO] server-e4fdf3b0-1333-71be-5b89-134979621997: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1532fafc-d768-7ed9-e44e-da1908165de7 Address:1532fafc-d768-7ed9-e44e-da1908165de7} {Suffrage:Voter ID:server-e4fdf3b0-1333-71be-5b89-134979621997 Address:e4fdf3b0-1333-71be-5b89-134979621997} {Suffrage:Voter ID:server-5a1b57c6-f45c-10f9-bacd-73d7bf5adfb5 Address:5a1b57c6-f45c-10f9-bacd-73d7bf5adfb5}]" -2026-03-31T06:18:02.762Z [INFO] server-e4fdf3b0-1333-71be-5b89-134979621997: entering follower state: follower="Node at e4fdf3b0-1333-71be-5b89-134979621997 [Follower]" leader-address= leader-id= -2026-03-31T06:18:03.012Z [INFO] server-1532fafc-d768-7ed9-e44e-da1908165de7: aborting pipeline replication: peer="{Voter server-5a1b57c6-f45c-10f9-bacd-73d7bf5adfb5 5a1b57c6-f45c-10f9-bacd-73d7bf5adfb5}" -2026-03-31T06:18:03.385Z [ERROR] server-1532fafc-d768-7ed9-e44e-da1908165de7: failed to pipeline appendEntries: peer="{Voter server-e4fdf3b0-1333-71be-5b89-134979621997 e4fdf3b0-1333-71be-5b89-134979621997}" error="command enqueue timeout" -2026-03-31T06:18:03.385Z [INFO] server-1532fafc-d768-7ed9-e44e-da1908165de7: aborting pipeline replication: peer="{Voter server-e4fdf3b0-1333-71be-5b89-134979621997 e4fdf3b0-1333-71be-5b89-134979621997}" -2026-03-31T06:18:03.518Z [ERROR] server-1532fafc-d768-7ed9-e44e-da1908165de7: failed to heartbeat to: peer=e4fdf3b0-1333-71be-5b89-134979621997 backoff time=10ms error="command timed out" ---- PASS: TestRaft_FollowerRemovalNoElection (1.74s) +2025-02-25T23:59:26.322Z [INFO] server-18273181-dedc-8f8c-c461-4c953526d655: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-18273181-dedc-8f8c-c461-4c953526d655 Address:18273181-dedc-8f8c-c461-4c953526d655} {Suffrage:Voter ID:server-e1cc7e35-f156-0095-80a0-ee7c39bf0fe2 Address:e1cc7e35-f156-0095-80a0-ee7c39bf0fe2} {Suffrage:Voter ID:server-daadc99d-b677-45e2-de81-7c53ef5bbc5d Address:daadc99d-b677-45e2-de81-7c53ef5bbc5d}]" +2025-02-25T23:59:26.322Z [INFO] server-e1cc7e35-f156-0095-80a0-ee7c39bf0fe2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-18273181-dedc-8f8c-c461-4c953526d655 Address:18273181-dedc-8f8c-c461-4c953526d655} {Suffrage:Voter ID:server-e1cc7e35-f156-0095-80a0-ee7c39bf0fe2 Address:e1cc7e35-f156-0095-80a0-ee7c39bf0fe2} {Suffrage:Voter ID:server-daadc99d-b677-45e2-de81-7c53ef5bbc5d Address:daadc99d-b677-45e2-de81-7c53ef5bbc5d}]" +2025-02-25T23:59:26.322Z [INFO] server-daadc99d-b677-45e2-de81-7c53ef5bbc5d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-18273181-dedc-8f8c-c461-4c953526d655 Address:18273181-dedc-8f8c-c461-4c953526d655} {Suffrage:Voter ID:server-e1cc7e35-f156-0095-80a0-ee7c39bf0fe2 Address:e1cc7e35-f156-0095-80a0-ee7c39bf0fe2} {Suffrage:Voter ID:server-daadc99d-b677-45e2-de81-7c53ef5bbc5d Address:daadc99d-b677-45e2-de81-7c53ef5bbc5d}]" +2025-02-25T23:59:26.322Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:26.322Z [INFO] server-18273181-dedc-8f8c-c461-4c953526d655: entering follower state: follower="Node at 18273181-dedc-8f8c-c461-4c953526d655 [Follower]" leader-address= leader-id= +2025-02-25T23:59:26.323Z [INFO] server-e1cc7e35-f156-0095-80a0-ee7c39bf0fe2: entering follower state: follower="Node at e1cc7e35-f156-0095-80a0-ee7c39bf0fe2 [Follower]" leader-address= leader-id= +2025-02-25T23:59:26.323Z [INFO] server-daadc99d-b677-45e2-de81-7c53ef5bbc5d: entering follower state: follower="Node at daadc99d-b677-45e2-de81-7c53ef5bbc5d [Follower]" leader-address= leader-id= +2025-02-25T23:59:26.438Z [WARN] server-e1cc7e35-f156-0095-80a0-ee7c39bf0fe2: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:26.438Z [INFO] server-e1cc7e35-f156-0095-80a0-ee7c39bf0fe2: entering candidate state: node="Node at e1cc7e35-f156-0095-80a0-ee7c39bf0fe2 [Candidate]" term=2 +2025-02-25T23:59:26.438Z [INFO] server-e1cc7e35-f156-0095-80a0-ee7c39bf0fe2: election won: term=2 tally=2 +2025-02-25T23:59:26.438Z [INFO] server-e1cc7e35-f156-0095-80a0-ee7c39bf0fe2: entering leader state: leader="Node at e1cc7e35-f156-0095-80a0-ee7c39bf0fe2 [Leader]" +2025-02-25T23:59:26.438Z [INFO] server-e1cc7e35-f156-0095-80a0-ee7c39bf0fe2: added peer, starting replication: peer=server-18273181-dedc-8f8c-c461-4c953526d655 +2025-02-25T23:59:26.438Z [INFO] server-e1cc7e35-f156-0095-80a0-ee7c39bf0fe2: added peer, starting replication: peer=server-daadc99d-b677-45e2-de81-7c53ef5bbc5d +2025-02-25T23:59:26.438Z [INFO] server-e1cc7e35-f156-0095-80a0-ee7c39bf0fe2: pipelining replication: peer="{Voter server-daadc99d-b677-45e2-de81-7c53ef5bbc5d daadc99d-b677-45e2-de81-7c53ef5bbc5d}" +2025-02-25T23:59:26.438Z [INFO] server-e1cc7e35-f156-0095-80a0-ee7c39bf0fe2: pipelining replication: peer="{Voter server-18273181-dedc-8f8c-c461-4c953526d655 18273181-dedc-8f8c-c461-4c953526d655}" +2025-02-25T23:59:26.646Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:26.438552968 +0000 UTC m=+114.047568375 (1 nodes), 115.595056ms from start of poll, 115.775857ms from cluster start. Timeout at 2025-02-25 23:59:26.643563048 +0000 UTC m=+114.252578455, 205.01008ms after stability +2025-02-25T23:59:26.646Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:26.854Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:26.646579532 +0000 UTC m=+114.255594939 (1 nodes), 480ns from start of poll, 323.802421ms from cluster start. Timeout at 2025-02-25 23:59:26.851611853 +0000 UTC m=+114.460627260, 205.032321ms after stability +2025-02-25T23:59:26.863Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:59:27.070Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:59:26.863473353 +0000 UTC m=+114.472488720 (2 nodes), 1.72µs from start of poll, 540.696202ms from cluster start. Timeout at 2025-02-25 23:59:27.068535593 +0000 UTC m=+114.677551000, 205.06228ms after stability +2025-02-25T23:59:27.070Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:27.278Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:27.07044279 +0000 UTC m=+114.679458197 (1 nodes), 1.48µs from start of poll, 747.665679ms from cluster start. Timeout at 2025-02-25 23:59:27.275496311 +0000 UTC m=+114.884511718, 205.053521ms after stability + raft_test.go:2961: [INFO] restarting Node at 18273181-dedc-8f8c-c461-4c953526d655 [Follower] +2025-02-25T23:59:27.278Z [INFO] server-18273181-dedc-8f8c-c461-4c953526d655: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-18273181-dedc-8f8c-c461-4c953526d655 Address:18273181-dedc-8f8c-c461-4c953526d655} {Suffrage:Voter ID:server-e1cc7e35-f156-0095-80a0-ee7c39bf0fe2 Address:e1cc7e35-f156-0095-80a0-ee7c39bf0fe2} {Suffrage:Voter ID:server-daadc99d-b677-45e2-de81-7c53ef5bbc5d Address:daadc99d-b677-45e2-de81-7c53ef5bbc5d}]" +2025-02-25T23:59:27.278Z [INFO] server-18273181-dedc-8f8c-c461-4c953526d655: entering follower state: follower="Node at 18273181-dedc-8f8c-c461-4c953526d655 [Follower]" leader-address= leader-id= +2025-02-25T23:59:27.329Z [WARN] server-e1cc7e35-f156-0095-80a0-ee7c39bf0fe2: failed to contact: server-id=server-18273181-dedc-8f8c-c461-4c953526d655 time=51.90041ms +2025-02-25T23:59:27.378Z [WARN] server-e1cc7e35-f156-0095-80a0-ee7c39bf0fe2: failed to contact: server-id=server-18273181-dedc-8f8c-c461-4c953526d655 time=100.5116ms +2025-02-25T23:59:27.426Z [WARN] server-e1cc7e35-f156-0095-80a0-ee7c39bf0fe2: failed to contact: server-id=server-18273181-dedc-8f8c-c461-4c953526d655 time=148.527786ms +2025-02-25T23:59:27.429Z [WARN] server-18273181-dedc-8f8c-c461-4c953526d655: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:27.429Z [INFO] server-18273181-dedc-8f8c-c461-4c953526d655: entering candidate state: node="Node at 18273181-dedc-8f8c-c461-4c953526d655 [Candidate]" term=3 +2025-02-25T23:59:27.429Z [WARN] server-daadc99d-b677-45e2-de81-7c53ef5bbc5d: rejecting vote request since we have a leader: from=18273181-dedc-8f8c-c461-4c953526d655 leader=e1cc7e35-f156-0095-80a0-ee7c39bf0fe2 leader-id=server-e1cc7e35-f156-0095-80a0-ee7c39bf0fe2 +2025-02-25T23:59:27.430Z [WARN] server-e1cc7e35-f156-0095-80a0-ee7c39bf0fe2: rejecting vote request since we have a leader: from=18273181-dedc-8f8c-c461-4c953526d655 leader=e1cc7e35-f156-0095-80a0-ee7c39bf0fe2 leader-id=server-e1cc7e35-f156-0095-80a0-ee7c39bf0fe2 +2025-02-25T23:59:27.530Z [ERROR] cluster: peer sees incorrect leader: peer="Node at 18273181-dedc-8f8c-c461-4c953526d655 [Candidate]" leader=[none] expected-leader=e1cc7e35-f156-0095-80a0-ee7c39bf0fe2 + testing.go:604: at least one peer has the wrong notion of leader +2025-02-25T23:59:27.530Z [INFO] server-e1cc7e35-f156-0095-80a0-ee7c39bf0fe2: aborting pipeline replication: peer="{Voter server-daadc99d-b677-45e2-de81-7c53ef5bbc5d daadc99d-b677-45e2-de81-7c53ef5bbc5d}" +2025-02-25T23:59:27.778Z [ERROR] server-e1cc7e35-f156-0095-80a0-ee7c39bf0fe2: failed to heartbeat to: peer=18273181-dedc-8f8c-c461-4c953526d655 backoff time=10ms error="command timed out" +2025-02-25T23:59:27.938Z [ERROR] server-e1cc7e35-f156-0095-80a0-ee7c39bf0fe2: failed to pipeline appendEntries: peer="{Voter server-18273181-dedc-8f8c-c461-4c953526d655 18273181-dedc-8f8c-c461-4c953526d655}" error="command enqueue timeout" +2025-02-25T23:59:27.938Z [INFO] server-e1cc7e35-f156-0095-80a0-ee7c39bf0fe2: aborting pipeline replication: peer="{Voter server-18273181-dedc-8f8c-c461-4c953526d655 18273181-dedc-8f8c-c461-4c953526d655}" +2025-02-25T23:59:28.318Z [ERROR] server-e1cc7e35-f156-0095-80a0-ee7c39bf0fe2: failed to heartbeat to: peer=18273181-dedc-8f8c-c461-4c953526d655 backoff time=10ms error="command timed out" +--- FAIL: TestRaft_FollowerRemovalNoElection (2.00s) === RUN TestRaft_VoteWithNoIDNoAddr -2026-03-31T06:18:03.518Z [INFO] server-0ad9bd12-e59f-1841-e4ea-49eb37419d45: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0ad9bd12-e59f-1841-e4ea-49eb37419d45 Address:0ad9bd12-e59f-1841-e4ea-49eb37419d45} {Suffrage:Voter ID:server-7944fac8-8013-88c0-c5fa-eff7f4f47452 Address:7944fac8-8013-88c0-c5fa-eff7f4f47452} {Suffrage:Voter ID:server-19c5b7ee-f229-f986-b138-7fb6144ed634 Address:19c5b7ee-f229-f986-b138-7fb6144ed634}]" -2026-03-31T06:18:03.518Z [INFO] server-0ad9bd12-e59f-1841-e4ea-49eb37419d45: entering follower state: follower="Node at 0ad9bd12-e59f-1841-e4ea-49eb37419d45 [Follower]" leader-address= leader-id= -2026-03-31T06:18:03.519Z [INFO] server-7944fac8-8013-88c0-c5fa-eff7f4f47452: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0ad9bd12-e59f-1841-e4ea-49eb37419d45 Address:0ad9bd12-e59f-1841-e4ea-49eb37419d45} {Suffrage:Voter ID:server-7944fac8-8013-88c0-c5fa-eff7f4f47452 Address:7944fac8-8013-88c0-c5fa-eff7f4f47452} {Suffrage:Voter ID:server-19c5b7ee-f229-f986-b138-7fb6144ed634 Address:19c5b7ee-f229-f986-b138-7fb6144ed634}]" -2026-03-31T06:18:03.519Z [INFO] server-7944fac8-8013-88c0-c5fa-eff7f4f47452: entering follower state: follower="Node at 7944fac8-8013-88c0-c5fa-eff7f4f47452 [Follower]" leader-address= leader-id= -2026-03-31T06:18:03.519Z [INFO] server-19c5b7ee-f229-f986-b138-7fb6144ed634: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0ad9bd12-e59f-1841-e4ea-49eb37419d45 Address:0ad9bd12-e59f-1841-e4ea-49eb37419d45} {Suffrage:Voter ID:server-7944fac8-8013-88c0-c5fa-eff7f4f47452 Address:7944fac8-8013-88c0-c5fa-eff7f4f47452} {Suffrage:Voter ID:server-19c5b7ee-f229-f986-b138-7fb6144ed634 Address:19c5b7ee-f229-f986-b138-7fb6144ed634}]" -2026-03-31T06:18:03.519Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:18:03.519Z [INFO] server-19c5b7ee-f229-f986-b138-7fb6144ed634: entering follower state: follower="Node at 19c5b7ee-f229-f986-b138-7fb6144ed634 [Follower]" leader-address= leader-id= -2026-03-31T06:18:03.584Z [WARN] server-7944fac8-8013-88c0-c5fa-eff7f4f47452: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2026-03-31T06:18:03.584Z [INFO] server-7944fac8-8013-88c0-c5fa-eff7f4f47452: entering candidate state: node="Node at 7944fac8-8013-88c0-c5fa-eff7f4f47452 [Candidate]" term=2 -2026-03-31T06:18:03.584Z [INFO] server-7944fac8-8013-88c0-c5fa-eff7f4f47452: election won: term=2 tally=2 -2026-03-31T06:18:03.584Z [INFO] server-7944fac8-8013-88c0-c5fa-eff7f4f47452: entering leader state: leader="Node at 7944fac8-8013-88c0-c5fa-eff7f4f47452 [Leader]" -2026-03-31T06:18:03.584Z [INFO] server-7944fac8-8013-88c0-c5fa-eff7f4f47452: added peer, starting replication: peer=server-0ad9bd12-e59f-1841-e4ea-49eb37419d45 -2026-03-31T06:18:03.584Z [INFO] server-7944fac8-8013-88c0-c5fa-eff7f4f47452: added peer, starting replication: peer=server-19c5b7ee-f229-f986-b138-7fb6144ed634 -2026-03-31T06:18:03.584Z [INFO] server-7944fac8-8013-88c0-c5fa-eff7f4f47452: pipelining replication: peer="{Voter server-0ad9bd12-e59f-1841-e4ea-49eb37419d45 0ad9bd12-e59f-1841-e4ea-49eb37419d45}" -2026-03-31T06:18:03.584Z [INFO] server-7944fac8-8013-88c0-c5fa-eff7f4f47452: pipelining replication: peer="{Voter server-19c5b7ee-f229-f986-b138-7fb6144ed634 19c5b7ee-f229-f986-b138-7fb6144ed634}" -2026-03-31T06:18:03.690Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:18:03.58453818 +0000 UTC m=+111.862145561 (1 nodes), 65.21984ms from start of poll, 65.7668ms from cluster start. Timeout at 2026-03-31 06:18:03.68954226 +0000 UTC m=+111.967149641, 105.00408ms after stability -2026-03-31T06:18:03.690Z [INFO] cluster: starting stability test: raft-state=Leader -2026-03-31T06:18:03.796Z [INFO] cluster: stable state for Leader reached at 2026-03-31 06:18:03.69077102 +0000 UTC m=+111.968378441 (1 nodes), 2.48µs from start of poll, 171.99968ms from cluster start. Timeout at 2026-03-31 06:18:03.79578802 +0000 UTC m=+112.073395401, 105.01696ms after stability -2026-03-31T06:18:03.801Z [INFO] cluster: starting stability test: raft-state=Follower -2026-03-31T06:18:03.906Z [INFO] cluster: stable state for Follower reached at 2026-03-31 06:18:03.80130458 +0000 UTC m=+112.078911961 (2 nodes), 3.08µs from start of poll, 282.5332ms from cluster start. Timeout at 2026-03-31 06:18:03.90634314 +0000 UTC m=+112.183950561, 105.0386ms after stability -2026-03-31T06:18:03.910Z [ERROR] server-7944fac8-8013-88c0-c5fa-eff7f4f47452: failed to heartbeat to: peer=0ad9bd12-e59f-1841-e4ea-49eb37419d45 backoff time=10ms error="failed to connect to peer: 0ad9bd12-e59f-1841-e4ea-49eb37419d45" -2026-03-31T06:18:03.910Z [ERROR] server-7944fac8-8013-88c0-c5fa-eff7f4f47452: failed to pipeline appendEntries: peer="{Voter server-0ad9bd12-e59f-1841-e4ea-49eb37419d45 0ad9bd12-e59f-1841-e4ea-49eb37419d45}" error="append pipeline closed" -2026-03-31T06:18:03.910Z [INFO] server-7944fac8-8013-88c0-c5fa-eff7f4f47452: aborting pipeline replication: peer="{Voter server-0ad9bd12-e59f-1841-e4ea-49eb37419d45 0ad9bd12-e59f-1841-e4ea-49eb37419d45}" -2026-03-31T06:18:03.910Z [ERROR] server-7944fac8-8013-88c0-c5fa-eff7f4f47452: failed to pipeline appendEntries: peer="{Voter server-19c5b7ee-f229-f986-b138-7fb6144ed634 19c5b7ee-f229-f986-b138-7fb6144ed634}" error="append pipeline closed" -2026-03-31T06:18:03.910Z [INFO] server-7944fac8-8013-88c0-c5fa-eff7f4f47452: aborting pipeline replication: peer="{Voter server-19c5b7ee-f229-f986-b138-7fb6144ed634 19c5b7ee-f229-f986-b138-7fb6144ed634}" -2026-03-31T06:18:03.911Z [ERROR] server-7944fac8-8013-88c0-c5fa-eff7f4f47452: failed to heartbeat to: peer=19c5b7ee-f229-f986-b138-7fb6144ed634 backoff time=10ms error="failed to connect to peer: 19c5b7ee-f229-f986-b138-7fb6144ed634" -2026-03-31T06:18:03.916Z [ERROR] server-7944fac8-8013-88c0-c5fa-eff7f4f47452: failed to appendEntries to: peer="{Voter server-0ad9bd12-e59f-1841-e4ea-49eb37419d45 0ad9bd12-e59f-1841-e4ea-49eb37419d45}" error="failed to connect to peer: 0ad9bd12-e59f-1841-e4ea-49eb37419d45" -2026-03-31T06:18:03.919Z [ERROR] server-7944fac8-8013-88c0-c5fa-eff7f4f47452: failed to appendEntries to: peer="{Voter server-19c5b7ee-f229-f986-b138-7fb6144ed634 19c5b7ee-f229-f986-b138-7fb6144ed634}" error="failed to connect to peer: 19c5b7ee-f229-f986-b138-7fb6144ed634" -2026-03-31T06:18:03.928Z [ERROR] server-7944fac8-8013-88c0-c5fa-eff7f4f47452: failed to heartbeat to: peer=19c5b7ee-f229-f986-b138-7fb6144ed634 backoff time=10ms error="failed to connect to peer: 19c5b7ee-f229-f986-b138-7fb6144ed634" -2026-03-31T06:18:03.928Z [ERROR] server-7944fac8-8013-88c0-c5fa-eff7f4f47452: failed to heartbeat to: peer=0ad9bd12-e59f-1841-e4ea-49eb37419d45 backoff time=10ms error="failed to connect to peer: 0ad9bd12-e59f-1841-e4ea-49eb37419d45" -2026-03-31T06:18:03.934Z [ERROR] server-7944fac8-8013-88c0-c5fa-eff7f4f47452: failed to appendEntries to: peer="{Voter server-0ad9bd12-e59f-1841-e4ea-49eb37419d45 0ad9bd12-e59f-1841-e4ea-49eb37419d45}" error="failed to connect to peer: 0ad9bd12-e59f-1841-e4ea-49eb37419d45" -2026-03-31T06:18:03.937Z [ERROR] server-7944fac8-8013-88c0-c5fa-eff7f4f47452: failed to appendEntries to: peer="{Voter server-19c5b7ee-f229-f986-b138-7fb6144ed634 19c5b7ee-f229-f986-b138-7fb6144ed634}" error="failed to connect to peer: 19c5b7ee-f229-f986-b138-7fb6144ed634" -2026-03-31T06:18:03.945Z [ERROR] server-7944fac8-8013-88c0-c5fa-eff7f4f47452: failed to heartbeat to: peer=19c5b7ee-f229-f986-b138-7fb6144ed634 backoff time=10ms error="failed to connect to peer: 19c5b7ee-f229-f986-b138-7fb6144ed634" -2026-03-31T06:18:03.946Z [ERROR] server-7944fac8-8013-88c0-c5fa-eff7f4f47452: failed to heartbeat to: peer=0ad9bd12-e59f-1841-e4ea-49eb37419d45 backoff time=10ms error="failed to connect to peer: 0ad9bd12-e59f-1841-e4ea-49eb37419d45" -2026-03-31T06:18:03.953Z [WARN] server-7944fac8-8013-88c0-c5fa-eff7f4f47452: failed to contact: server-id=server-0ad9bd12-e59f-1841-e4ea-49eb37419d45 time=50.17272ms -2026-03-31T06:18:03.954Z [ERROR] server-7944fac8-8013-88c0-c5fa-eff7f4f47452: failed to appendEntries to: peer="{Voter server-0ad9bd12-e59f-1841-e4ea-49eb37419d45 0ad9bd12-e59f-1841-e4ea-49eb37419d45}" error="failed to connect to peer: 0ad9bd12-e59f-1841-e4ea-49eb37419d45" -2026-03-31T06:18:03.954Z [ERROR] server-7944fac8-8013-88c0-c5fa-eff7f4f47452: failed to appendEntries to: peer="{Voter server-19c5b7ee-f229-f986-b138-7fb6144ed634 19c5b7ee-f229-f986-b138-7fb6144ed634}" error="failed to connect to peer: 19c5b7ee-f229-f986-b138-7fb6144ed634" -2026-03-31T06:18:03.963Z [WARN] server-7944fac8-8013-88c0-c5fa-eff7f4f47452: failed to contact: server-id=server-0ad9bd12-e59f-1841-e4ea-49eb37419d45 time=60.25508ms -2026-03-31T06:18:03.963Z [WARN] server-7944fac8-8013-88c0-c5fa-eff7f4f47452: failed to contact: server-id=server-19c5b7ee-f229-f986-b138-7fb6144ed634 time=57.7394ms -2026-03-31T06:18:03.963Z [WARN] server-7944fac8-8013-88c0-c5fa-eff7f4f47452: failed to contact quorum of nodes, stepping down -2026-03-31T06:18:03.963Z [ERROR] server-7944fac8-8013-88c0-c5fa-eff7f4f47452: failed to heartbeat to: peer=0ad9bd12-e59f-1841-e4ea-49eb37419d45 backoff time=20ms error="failed to connect to peer: 0ad9bd12-e59f-1841-e4ea-49eb37419d45" -2026-03-31T06:18:03.963Z [INFO] server-7944fac8-8013-88c0-c5fa-eff7f4f47452: entering follower state: follower="Node at 7944fac8-8013-88c0-c5fa-eff7f4f47452 [Follower]" leader-address= leader-id= -2026-03-31T06:18:03.973Z [WARN] server-19c5b7ee-f229-f986-b138-7fb6144ed634: heartbeat timeout reached, starting election: last-leader-addr=7944fac8-8013-88c0-c5fa-eff7f4f47452 last-leader-id=server-7944fac8-8013-88c0-c5fa-eff7f4f47452 -2026-03-31T06:18:03.973Z [INFO] server-19c5b7ee-f229-f986-b138-7fb6144ed634: entering candidate state: node="Node at 19c5b7ee-f229-f986-b138-7fb6144ed634 [Candidate]" term=3 -2026-03-31T06:18:03.973Z [ERROR] server-19c5b7ee-f229-f986-b138-7fb6144ed634: failed to make requestVote RPC: target="{Voter server-7944fac8-8013-88c0-c5fa-eff7f4f47452 7944fac8-8013-88c0-c5fa-eff7f4f47452}" error="failed to connect to peer: 7944fac8-8013-88c0-c5fa-eff7f4f47452" term=3 -2026-03-31T06:18:03.973Z [WARN] server-0ad9bd12-e59f-1841-e4ea-49eb37419d45: rejecting vote request since we have a leader: from=19c5b7ee-f229-f986-b138-7fb6144ed634 leader=7944fac8-8013-88c0-c5fa-eff7f4f47452 leader-id=server-7944fac8-8013-88c0-c5fa-eff7f4f47452 -2026-03-31T06:18:03.982Z [ERROR] server-7944fac8-8013-88c0-c5fa-eff7f4f47452: failed to appendEntries to: peer="{Voter server-0ad9bd12-e59f-1841-e4ea-49eb37419d45 0ad9bd12-e59f-1841-e4ea-49eb37419d45}" error="failed to connect to peer: 0ad9bd12-e59f-1841-e4ea-49eb37419d45" -2026-03-31T06:18:04.002Z [WARN] server-0ad9bd12-e59f-1841-e4ea-49eb37419d45: heartbeat timeout reached, starting election: last-leader-addr=7944fac8-8013-88c0-c5fa-eff7f4f47452 last-leader-id=server-7944fac8-8013-88c0-c5fa-eff7f4f47452 -2026-03-31T06:18:04.002Z [INFO] server-0ad9bd12-e59f-1841-e4ea-49eb37419d45: entering candidate state: node="Node at 0ad9bd12-e59f-1841-e4ea-49eb37419d45 [Candidate]" term=3 -2026-03-31T06:18:04.003Z [INFO] server-19c5b7ee-f229-f986-b138-7fb6144ed634: duplicate requestVote for same term: term=3 -2026-03-31T06:18:04.003Z [ERROR] server-0ad9bd12-e59f-1841-e4ea-49eb37419d45: failed to make requestVote RPC: target="{Voter server-7944fac8-8013-88c0-c5fa-eff7f4f47452 7944fac8-8013-88c0-c5fa-eff7f4f47452}" error="failed to connect to peer: 7944fac8-8013-88c0-c5fa-eff7f4f47452" term=3 -2026-03-31T06:18:04.012Z [INFO] server-0ad9bd12-e59f-1841-e4ea-49eb37419d45: entering follower state: follower="Node at 0ad9bd12-e59f-1841-e4ea-49eb37419d45 [Follower]" leader-address= leader-id= ---- PASS: TestRaft_VoteWithNoIDNoAddr (0.49s) +2025-02-25T23:59:28.318Z [INFO] server-297ce1f2-1426-777e-7ce2-0298aaa94d23: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-297ce1f2-1426-777e-7ce2-0298aaa94d23 Address:297ce1f2-1426-777e-7ce2-0298aaa94d23} {Suffrage:Voter ID:server-051c27cb-1e87-45a7-393c-98f65bc8986d Address:051c27cb-1e87-45a7-393c-98f65bc8986d} {Suffrage:Voter ID:server-693bd17e-c3fa-b11a-0c7f-1c433256b675 Address:693bd17e-c3fa-b11a-0c7f-1c433256b675}]" +2025-02-25T23:59:28.318Z [INFO] server-051c27cb-1e87-45a7-393c-98f65bc8986d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-297ce1f2-1426-777e-7ce2-0298aaa94d23 Address:297ce1f2-1426-777e-7ce2-0298aaa94d23} {Suffrage:Voter ID:server-051c27cb-1e87-45a7-393c-98f65bc8986d Address:051c27cb-1e87-45a7-393c-98f65bc8986d} {Suffrage:Voter ID:server-693bd17e-c3fa-b11a-0c7f-1c433256b675 Address:693bd17e-c3fa-b11a-0c7f-1c433256b675}]" +2025-02-25T23:59:28.318Z [INFO] server-693bd17e-c3fa-b11a-0c7f-1c433256b675: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-297ce1f2-1426-777e-7ce2-0298aaa94d23 Address:297ce1f2-1426-777e-7ce2-0298aaa94d23} {Suffrage:Voter ID:server-051c27cb-1e87-45a7-393c-98f65bc8986d Address:051c27cb-1e87-45a7-393c-98f65bc8986d} {Suffrage:Voter ID:server-693bd17e-c3fa-b11a-0c7f-1c433256b675 Address:693bd17e-c3fa-b11a-0c7f-1c433256b675}]" +2025-02-25T23:59:28.318Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:28.318Z [INFO] server-297ce1f2-1426-777e-7ce2-0298aaa94d23: entering follower state: follower="Node at 297ce1f2-1426-777e-7ce2-0298aaa94d23 [Follower]" leader-address= leader-id= +2025-02-25T23:59:28.318Z [INFO] server-051c27cb-1e87-45a7-393c-98f65bc8986d: entering follower state: follower="Node at 051c27cb-1e87-45a7-393c-98f65bc8986d [Follower]" leader-address= leader-id= +2025-02-25T23:59:28.318Z [INFO] server-693bd17e-c3fa-b11a-0c7f-1c433256b675: entering follower state: follower="Node at 693bd17e-c3fa-b11a-0c7f-1c433256b675 [Follower]" leader-address= leader-id= +2025-02-25T23:59:28.386Z [WARN] server-051c27cb-1e87-45a7-393c-98f65bc8986d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:28.386Z [INFO] server-051c27cb-1e87-45a7-393c-98f65bc8986d: entering candidate state: node="Node at 051c27cb-1e87-45a7-393c-98f65bc8986d [Candidate]" term=2 +2025-02-25T23:59:28.386Z [INFO] server-051c27cb-1e87-45a7-393c-98f65bc8986d: election won: term=2 tally=2 +2025-02-25T23:59:28.386Z [INFO] server-051c27cb-1e87-45a7-393c-98f65bc8986d: entering leader state: leader="Node at 051c27cb-1e87-45a7-393c-98f65bc8986d [Leader]" +2025-02-25T23:59:28.386Z [INFO] server-051c27cb-1e87-45a7-393c-98f65bc8986d: added peer, starting replication: peer=server-297ce1f2-1426-777e-7ce2-0298aaa94d23 +2025-02-25T23:59:28.386Z [INFO] server-051c27cb-1e87-45a7-393c-98f65bc8986d: added peer, starting replication: peer=server-693bd17e-c3fa-b11a-0c7f-1c433256b675 +2025-02-25T23:59:28.386Z [INFO] server-051c27cb-1e87-45a7-393c-98f65bc8986d: pipelining replication: peer="{Voter server-693bd17e-c3fa-b11a-0c7f-1c433256b675 693bd17e-c3fa-b11a-0c7f-1c433256b675}" +2025-02-25T23:59:28.386Z [WARN] server-693bd17e-c3fa-b11a-0c7f-1c433256b675: rejecting vote request since our last term is greater: candidate=051c27cb-1e87-45a7-393c-98f65bc8986d last-term=2 last-candidate-term=1 +2025-02-25T23:59:28.386Z [INFO] server-051c27cb-1e87-45a7-393c-98f65bc8986d: pipelining replication: peer="{Voter server-297ce1f2-1426-777e-7ce2-0298aaa94d23 297ce1f2-1426-777e-7ce2-0298aaa94d23}" +2025-02-25T23:59:28.494Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:28.386533528 +0000 UTC m=+115.995548935 (1 nodes), 67.784431ms from start of poll, 67.959632ms from cluster start. Timeout at 2025-02-25 23:59:28.491535088 +0000 UTC m=+116.100550495, 105.00156ms after stability +2025-02-25T23:59:28.494Z [INFO] cluster: starting stability test: raft-state=Leader +2025-02-25T23:59:28.602Z [INFO] cluster: stable state for Leader reached at 2025-02-25 23:59:28.494151094 +0000 UTC m=+116.103166501 (1 nodes), 1.92µs from start of poll, 175.577198ms from cluster start. Timeout at 2025-02-25 23:59:28.599189294 +0000 UTC m=+116.208204661, 105.03816ms after stability +2025-02-25T23:59:28.610Z [INFO] cluster: starting stability test: raft-state=Follower +2025-02-25T23:59:28.726Z [INFO] cluster: stable state for Follower reached at 2025-02-25 23:59:28.610186072 +0000 UTC m=+116.219201439 (2 nodes), 1.12µs from start of poll, 291.612136ms from cluster start. Timeout at 2025-02-25 23:59:28.715210672 +0000 UTC m=+116.324226079, 105.02464ms after stability +2025-02-25T23:59:28.726Z [ERROR] server-051c27cb-1e87-45a7-393c-98f65bc8986d: failed to pipeline appendEntries: peer="{Voter server-693bd17e-c3fa-b11a-0c7f-1c433256b675 693bd17e-c3fa-b11a-0c7f-1c433256b675}" error="append pipeline closed" +2025-02-25T23:59:28.726Z [INFO] server-051c27cb-1e87-45a7-393c-98f65bc8986d: aborting pipeline replication: peer="{Voter server-693bd17e-c3fa-b11a-0c7f-1c433256b675 693bd17e-c3fa-b11a-0c7f-1c433256b675}" +2025-02-25T23:59:28.735Z [ERROR] server-051c27cb-1e87-45a7-393c-98f65bc8986d: failed to pipeline appendEntries: peer="{Voter server-297ce1f2-1426-777e-7ce2-0298aaa94d23 297ce1f2-1426-777e-7ce2-0298aaa94d23}" error="append pipeline closed" +2025-02-25T23:59:28.735Z [INFO] server-051c27cb-1e87-45a7-393c-98f65bc8986d: aborting pipeline replication: peer="{Voter server-297ce1f2-1426-777e-7ce2-0298aaa94d23 297ce1f2-1426-777e-7ce2-0298aaa94d23}" +2025-02-25T23:59:28.735Z [ERROR] server-051c27cb-1e87-45a7-393c-98f65bc8986d: failed to heartbeat to: peer=297ce1f2-1426-777e-7ce2-0298aaa94d23 backoff time=10ms error="failed to connect to peer: 297ce1f2-1426-777e-7ce2-0298aaa94d23" +2025-02-25T23:59:28.735Z [ERROR] server-051c27cb-1e87-45a7-393c-98f65bc8986d: failed to appendEntries to: peer="{Voter server-693bd17e-c3fa-b11a-0c7f-1c433256b675 693bd17e-c3fa-b11a-0c7f-1c433256b675}" error="failed to connect to peer: 693bd17e-c3fa-b11a-0c7f-1c433256b675" +2025-02-25T23:59:28.735Z [ERROR] server-051c27cb-1e87-45a7-393c-98f65bc8986d: failed to heartbeat to: peer=693bd17e-c3fa-b11a-0c7f-1c433256b675 backoff time=10ms error="failed to connect to peer: 693bd17e-c3fa-b11a-0c7f-1c433256b675" +2025-02-25T23:59:28.741Z [ERROR] server-051c27cb-1e87-45a7-393c-98f65bc8986d: failed to appendEntries to: peer="{Voter server-297ce1f2-1426-777e-7ce2-0298aaa94d23 297ce1f2-1426-777e-7ce2-0298aaa94d23}" error="failed to connect to peer: 297ce1f2-1426-777e-7ce2-0298aaa94d23" +2025-02-25T23:59:28.754Z [ERROR] server-051c27cb-1e87-45a7-393c-98f65bc8986d: failed to heartbeat to: peer=693bd17e-c3fa-b11a-0c7f-1c433256b675 backoff time=10ms error="failed to connect to peer: 693bd17e-c3fa-b11a-0c7f-1c433256b675" +2025-02-25T23:59:28.754Z [ERROR] server-051c27cb-1e87-45a7-393c-98f65bc8986d: failed to appendEntries to: peer="{Voter server-693bd17e-c3fa-b11a-0c7f-1c433256b675 693bd17e-c3fa-b11a-0c7f-1c433256b675}" error="failed to connect to peer: 693bd17e-c3fa-b11a-0c7f-1c433256b675" +2025-02-25T23:59:28.758Z [ERROR] server-051c27cb-1e87-45a7-393c-98f65bc8986d: failed to heartbeat to: peer=297ce1f2-1426-777e-7ce2-0298aaa94d23 backoff time=10ms error="failed to connect to peer: 297ce1f2-1426-777e-7ce2-0298aaa94d23" +2025-02-25T23:59:28.765Z [ERROR] server-051c27cb-1e87-45a7-393c-98f65bc8986d: failed to appendEntries to: peer="{Voter server-297ce1f2-1426-777e-7ce2-0298aaa94d23 297ce1f2-1426-777e-7ce2-0298aaa94d23}" error="failed to connect to peer: 297ce1f2-1426-777e-7ce2-0298aaa94d23" +2025-02-25T23:59:28.773Z [ERROR] server-051c27cb-1e87-45a7-393c-98f65bc8986d: failed to heartbeat to: peer=693bd17e-c3fa-b11a-0c7f-1c433256b675 backoff time=10ms error="failed to connect to peer: 693bd17e-c3fa-b11a-0c7f-1c433256b675" +2025-02-25T23:59:28.778Z [ERROR] server-051c27cb-1e87-45a7-393c-98f65bc8986d: failed to appendEntries to: peer="{Voter server-693bd17e-c3fa-b11a-0c7f-1c433256b675 693bd17e-c3fa-b11a-0c7f-1c433256b675}" error="failed to connect to peer: 693bd17e-c3fa-b11a-0c7f-1c433256b675" +2025-02-25T23:59:28.778Z [ERROR] server-051c27cb-1e87-45a7-393c-98f65bc8986d: failed to heartbeat to: peer=297ce1f2-1426-777e-7ce2-0298aaa94d23 backoff time=10ms error="failed to connect to peer: 297ce1f2-1426-777e-7ce2-0298aaa94d23" +2025-02-25T23:59:28.781Z [WARN] server-693bd17e-c3fa-b11a-0c7f-1c433256b675: heartbeat timeout reached, starting election: last-leader-addr=051c27cb-1e87-45a7-393c-98f65bc8986d last-leader-id=server-051c27cb-1e87-45a7-393c-98f65bc8986d +2025-02-25T23:59:28.781Z [INFO] server-693bd17e-c3fa-b11a-0c7f-1c433256b675: entering candidate state: node="Node at 693bd17e-c3fa-b11a-0c7f-1c433256b675 [Candidate]" term=3 +2025-02-25T23:59:28.782Z [ERROR] server-693bd17e-c3fa-b11a-0c7f-1c433256b675: failed to make requestVote RPC: target="{Voter server-051c27cb-1e87-45a7-393c-98f65bc8986d 051c27cb-1e87-45a7-393c-98f65bc8986d}" error="failed to connect to peer: 051c27cb-1e87-45a7-393c-98f65bc8986d" term=3 +2025-02-25T23:59:28.782Z [WARN] server-297ce1f2-1426-777e-7ce2-0298aaa94d23: rejecting vote request since we have a leader: from=693bd17e-c3fa-b11a-0c7f-1c433256b675 leader=051c27cb-1e87-45a7-393c-98f65bc8986d leader-id=server-051c27cb-1e87-45a7-393c-98f65bc8986d +2025-02-25T23:59:28.785Z [ERROR] server-051c27cb-1e87-45a7-393c-98f65bc8986d: failed to appendEntries to: peer="{Voter server-297ce1f2-1426-777e-7ce2-0298aaa94d23 297ce1f2-1426-777e-7ce2-0298aaa94d23}" error="failed to connect to peer: 297ce1f2-1426-777e-7ce2-0298aaa94d23" +2025-02-25T23:59:28.786Z [WARN] server-051c27cb-1e87-45a7-393c-98f65bc8986d: failed to contact: server-id=server-297ce1f2-1426-777e-7ce2-0298aaa94d23 time=59.898022ms +2025-02-25T23:59:28.786Z [WARN] server-051c27cb-1e87-45a7-393c-98f65bc8986d: failed to contact: server-id=server-693bd17e-c3fa-b11a-0c7f-1c433256b675 time=60.089583ms +2025-02-25T23:59:28.786Z [WARN] server-051c27cb-1e87-45a7-393c-98f65bc8986d: failed to contact quorum of nodes, stepping down +2025-02-25T23:59:28.786Z [INFO] server-051c27cb-1e87-45a7-393c-98f65bc8986d: entering follower state: follower="Node at 051c27cb-1e87-45a7-393c-98f65bc8986d [Follower]" leader-address= leader-id= +2025-02-25T23:59:28.794Z [ERROR] server-051c27cb-1e87-45a7-393c-98f65bc8986d: failed to heartbeat to: peer=693bd17e-c3fa-b11a-0c7f-1c433256b675 backoff time=20ms error="failed to connect to peer: 693bd17e-c3fa-b11a-0c7f-1c433256b675" +2025-02-25T23:59:28.809Z [ERROR] server-051c27cb-1e87-45a7-393c-98f65bc8986d: failed to appendEntries to: peer="{Voter server-693bd17e-c3fa-b11a-0c7f-1c433256b675 693bd17e-c3fa-b11a-0c7f-1c433256b675}" error="failed to connect to peer: 693bd17e-c3fa-b11a-0c7f-1c433256b675" +2025-02-25T23:59:28.826Z [WARN] server-297ce1f2-1426-777e-7ce2-0298aaa94d23: heartbeat timeout reached, starting election: last-leader-addr=051c27cb-1e87-45a7-393c-98f65bc8986d last-leader-id=server-051c27cb-1e87-45a7-393c-98f65bc8986d +2025-02-25T23:59:28.826Z [INFO] server-297ce1f2-1426-777e-7ce2-0298aaa94d23: entering candidate state: node="Node at 297ce1f2-1426-777e-7ce2-0298aaa94d23 [Candidate]" term=3 +2025-02-25T23:59:28.826Z [INFO] server-693bd17e-c3fa-b11a-0c7f-1c433256b675: duplicate requestVote for same term: term=3 +2025-02-25T23:59:28.826Z [ERROR] server-297ce1f2-1426-777e-7ce2-0298aaa94d23: failed to make requestVote RPC: target="{Voter server-051c27cb-1e87-45a7-393c-98f65bc8986d 051c27cb-1e87-45a7-393c-98f65bc8986d}" error="failed to connect to peer: 051c27cb-1e87-45a7-393c-98f65bc8986d" term=3 +2025-02-25T23:59:28.834Z [INFO] server-297ce1f2-1426-777e-7ce2-0298aaa94d23: entering follower state: follower="Node at 297ce1f2-1426-777e-7ce2-0298aaa94d23 [Follower]" leader-address= leader-id= +--- PASS: TestRaft_VoteWithNoIDNoAddr (0.52s) === RUN TestRaft_runFollower_State_Transition === RUN TestRaft_runFollower_State_Transition/NonVoter -2026-03-31T06:18:04.013Z [INFO] 127.0.0.1:40615 :: starting node: addr=127.0.0.1:40615 -2026-03-31T06:18:04.013Z [INFO] 127.0.0.1:40615 :: initial configuration: index=0 servers=[] -2026-03-31T06:18:04.013Z [INFO] 127.0.0.1:40615 :: entering follower state: follower="Node at 127.0.0.1:40615 [Follower]" leader-address= leader-id= -2026-03-31T06:18:04.095Z [WARN] 127.0.0.1:40615 :: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +2025-02-25T23:59:28.834Z [INFO] 127.0.0.1:44693 :: starting node: addr=127.0.0.1:44693 +2025-02-25T23:59:28.834Z [INFO] 127.0.0.1:44693 :: initial configuration: index=0 servers=[] +2025-02-25T23:59:28.834Z [INFO] 127.0.0.1:44693 :: entering follower state: follower="Node at 127.0.0.1:44693 [Follower]" leader-address= leader-id= +2025-02-25T23:59:28.898Z [WARN] 127.0.0.1:44693 :: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election === RUN TestRaft_runFollower_State_Transition/Voter -2026-03-31T06:18:04.164Z [INFO] 127.0.0.1:40191 :: starting node: addr=127.0.0.1:40191 -2026-03-31T06:18:04.164Z [INFO] 127.0.0.1:40191 :: initial configuration: index=0 servers=[] -2026-03-31T06:18:04.164Z [INFO] 127.0.0.1:40191 :: entering follower state: follower="Node at 127.0.0.1:40191 [Follower]" leader-address= leader-id= -2026-03-31T06:18:04.254Z [WARN] 127.0.0.1:40191 :: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:28.986Z [INFO] 127.0.0.1:43075 :: starting node: addr=127.0.0.1:43075 +2025-02-25T23:59:28.986Z [INFO] 127.0.0.1:43075 :: initial configuration: index=0 servers=[] +2025-02-25T23:59:28.986Z [INFO] 127.0.0.1:43075 :: entering follower state: follower="Node at 127.0.0.1:43075 [Follower]" leader-address= leader-id= +2025-02-25T23:59:29.046Z [WARN] 127.0.0.1:43075 :: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= === RUN TestRaft_runFollower_State_Transition/Not_in_Config -2026-03-31T06:18:04.315Z [INFO] 127.0.0.1:34483 :: starting node: addr=127.0.0.1:34483 -2026-03-31T06:18:04.315Z [INFO] 127.0.0.1:34483 :: initial configuration: index=0 servers=[] -2026-03-31T06:18:04.315Z [INFO] 127.0.0.1:34483 :: entering follower state: follower="Node at 127.0.0.1:34483 [Follower]" leader-address= leader-id= -2026-03-31T06:18:04.389Z [WARN] 127.0.0.1:34483 :: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election ---- PASS: TestRaft_runFollower_State_Transition (0.45s) +2025-02-25T23:59:29.142Z [INFO] 127.0.0.1:35321 :: starting node: addr=127.0.0.1:35321 +2025-02-25T23:59:29.143Z [INFO] 127.0.0.1:35321 :: initial configuration: index=0 servers=[] +2025-02-25T23:59:29.143Z [INFO] 127.0.0.1:35321 :: entering follower state: follower="Node at 127.0.0.1:35321 [Follower]" leader-address= leader-id= +2025-02-25T23:59:29.245Z [WARN] 127.0.0.1:35321 :: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +--- PASS: TestRaft_runFollower_State_Transition (0.46s) --- PASS: TestRaft_runFollower_State_Transition/NonVoter (0.15s) - --- PASS: TestRaft_runFollower_State_Transition/Voter (0.15s) + --- PASS: TestRaft_runFollower_State_Transition/Voter (0.16s) --- PASS: TestRaft_runFollower_State_Transition/Not_in_Config (0.15s) === RUN TestRaft_runFollower_ReloadTimeoutConfigs -2026-03-31T06:18:04.467Z [INFO] 127.0.0.1:36609 :: starting node: addr=127.0.0.1:36609 -2026-03-31T06:18:04.467Z [INFO] 127.0.0.1:36609 :: initial configuration: index=0 servers=[] -2026-03-31T06:18:04.467Z [INFO] 127.0.0.1:36609 :: entering follower state: follower="Node at 127.0.0.1:36609 [Follower]" leader-address= leader-id= -2026-03-31T06:18:04.467Z [WARN] 127.0.0.1:36609 :: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-02-25T23:59:29.294Z [INFO] 127.0.0.1:37821 :: starting node: addr=127.0.0.1:37821 +2025-02-25T23:59:29.294Z [INFO] 127.0.0.1:37821 :: initial configuration: index=0 servers=[] +2025-02-25T23:59:29.294Z [INFO] 127.0.0.1:37821 :: entering follower state: follower="Node at 127.0.0.1:37821 [Follower]" leader-address= leader-id= +2025-02-25T23:59:29.294Z [WARN] 127.0.0.1:37821 :: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= --- PASS: TestRaft_runFollower_ReloadTimeoutConfigs (0.15s) === RUN TestSaturationMetric === RUN TestSaturationMetric/without_smoothing @@ -4241,7 +4331,7 @@ === RUN TestTransport_EncodeDecode --- PASS: TestTransport_EncodeDecode (0.00s) === RUN TestRandomTimeout ---- PASS: TestRandomTimeout (0.00s) +--- PASS: TestRandomTimeout (0.01s) === RUN TestNewSeed --- PASS: TestNewSeed (0.00s) === RUN TestRandomTimeout_NoTime @@ -4256,42 +4346,27 @@ --- PASS: TestBackoff (0.00s) === RUN TestOverrideNotifyBool --- PASS: TestOverrideNotifyBool (0.00s) -PASS -ok github.com/hashicorp/raft 112.915s +FAIL +FAIL github.com/hashicorp/raft 117.112s ? github.com/hashicorp/raft/bench [no test files] +FAIL +dh_auto_test: error: cd obj-aarch64-linux-gnu && go test -vet=off -v -p 12 github.com/hashicorp/raft github.com/hashicorp/raft/bench returned exit code 1 +make[1]: *** [debian/rules:17: override_dh_auto_test] Error 25 make[1]: Leaving directory '/build/reproducible-path/golang-github-hashicorp-raft-1.5.0' - create-stamp debian/debhelper-build-stamp - dh_testroot -O--buildsystem=golang - dh_prep -O--buildsystem=golang - dh_auto_install --destdir=debian/golang-github-hashicorp-raft-dev/ -O--buildsystem=golang - dh_installdocs -O--buildsystem=golang - dh_installchangelogs -O--buildsystem=golang - dh_installsystemduser -O--buildsystem=golang - dh_perl -O--buildsystem=golang - dh_link -O--buildsystem=golang - dh_strip_nondeterminism -O--buildsystem=golang - dh_compress -O--buildsystem=golang - dh_fixperms -O--buildsystem=golang - dh_missing -O--buildsystem=golang - dh_installdeb -O--buildsystem=golang - dh_golang -O--buildsystem=golang - dh_gencontrol -O--buildsystem=golang - dh_md5sums -O--buildsystem=golang - dh_builddeb -O--buildsystem=golang -dpkg-deb: building package 'golang-github-hashicorp-raft-dev' in '../golang-github-hashicorp-raft-dev_1.5.0-1_all.deb'. - dpkg-genbuildinfo --build=binary -O../golang-github-hashicorp-raft_1.5.0-1_arm64.buildinfo - dpkg-genchanges --build=binary -O../golang-github-hashicorp-raft_1.5.0-1_arm64.changes -dpkg-genchanges: info: binary-only upload (no source code included) - dpkg-source --after-build . -dpkg-buildpackage: info: binary-only upload (no source included) -dpkg-genchanges: info: including full source code in upload +make: *** [debian/rules:14: binary] Error 2 +dpkg-buildpackage: error: debian/rules binary subprocess returned exit status 2 I: copying local configuration +E: Failed autobuilding of package +I: user script /srv/workspace/pbuilder/4115557/tmp/hooks/C01_cleanup starting +debug output: disk usage on i-capture-the-hostname at Tue Feb 25 23:59:29 UTC 2025 +Filesystem Size Used Avail Use% Mounted on +tmpfs 30G 0 30G 0% /dev/shm + +I: user script /srv/workspace/pbuilder/4115557/tmp/hooks/C01_cleanup finished I: unmounting dev/ptmx filesystem I: unmounting dev/pts filesystem I: unmounting dev/shm filesystem I: unmounting proc filesystem I: unmounting sys filesystem I: cleaning the build env -I: removing directory /srv/workspace/pbuilder/674357 and its subdirectories -I: Current time: Mon Mar 30 18:18:10 -12 2026 -I: pbuilder-time-stamp: 1774937890 +I: removing directory /srv/workspace/pbuilder/4115557 and its subdirectories