Diff of the two buildlogs: -- --- b1/build.log 2025-01-25 14:30:20.827824999 +0000 +++ b2/build.log 2025-01-25 14:41:29.601778312 +0000 @@ -1,6 +1,6 @@ I: pbuilder: network access will be disabled during build -I: Current time: Sat Jan 25 02:24:48 -12 2025 -I: pbuilder-time-stamp: 1737815088 +I: Current time: Sun Jan 26 04:30:33 +14 2025 +I: pbuilder-time-stamp: 1737815433 I: Building the build Environment I: extracting base tarball [/var/cache/pbuilder/trixie-reproducible-base.tgz] I: copying local configuration @@ -24,52 +24,84 @@ dpkg-source: info: applying 0001-Skip-unreliable-tests.patch I: Not using root during the build. I: Installing the build-deps -I: user script /srv/workspace/pbuilder/22464/tmp/hooks/D02_print_environment starting +I: user script /srv/workspace/pbuilder/29299/tmp/hooks/D01_modify_environment starting +debug: Running on ff64a. +I: Changing host+domainname to test build reproducibility +I: Adding a custom variable just for the fun of it... +I: Changing /bin/sh to bash +'/bin/sh' -> '/bin/bash' +lrwxrwxrwx 1 root root 9 Jan 25 14:31 /bin/sh -> /bin/bash +I: Setting pbuilder2's login shell to /bin/bash +I: Setting pbuilder2's GECOS to second user,second room,second work-phone,second home-phone,second other +I: user script /srv/workspace/pbuilder/29299/tmp/hooks/D01_modify_environment finished +I: user script /srv/workspace/pbuilder/29299/tmp/hooks/D02_print_environment starting I: set - BUILDDIR='/build/reproducible-path' - BUILDUSERGECOS='first user,first room,first work-phone,first home-phone,first other' - BUILDUSERNAME='pbuilder1' - BUILD_ARCH='armhf' - DEBIAN_FRONTEND='noninteractive' - DEB_BUILD_OPTIONS='buildinfo=+all reproducible=+all parallel=3 ' - DISTRIBUTION='trixie' - HOME='/root' - HOST_ARCH='armhf' + BASH=/bin/sh + BASHOPTS=checkwinsize:cmdhist:complete_fullquote:extquote:force_fignore:globasciiranges:globskipdots:hostcomplete:interactive_comments:patsub_replacement:progcomp:promptvars:sourcepath + BASH_ALIASES=() + BASH_ARGC=() + BASH_ARGV=() + BASH_CMDS=() + BASH_LINENO=([0]="12" [1]="0") + BASH_LOADABLES_PATH=/usr/local/lib/bash:/usr/lib/bash:/opt/local/lib/bash:/usr/pkg/lib/bash:/opt/pkg/lib/bash:. + BASH_SOURCE=([0]="/tmp/hooks/D02_print_environment" [1]="/tmp/hooks/D02_print_environment") + BASH_VERSINFO=([0]="5" [1]="2" [2]="37" [3]="1" [4]="release" [5]="arm-unknown-linux-gnueabihf") + BASH_VERSION='5.2.37(1)-release' + BUILDDIR=/build/reproducible-path + BUILDUSERGECOS='second user,second room,second work-phone,second home-phone,second other' + BUILDUSERNAME=pbuilder2 + BUILD_ARCH=armhf + DEBIAN_FRONTEND=noninteractive + DEB_BUILD_OPTIONS='buildinfo=+all reproducible=+all parallel=6 ' + DIRSTACK=() + DISTRIBUTION=trixie + EUID=0 + FUNCNAME=([0]="Echo" [1]="main") + GROUPS=() + HOME=/root + HOSTNAME=i-capture-the-hostname + HOSTTYPE=arm + HOST_ARCH=armhf IFS=' ' - INVOCATION_ID='c8b9d59d771c4c03a0992e1db6f7a075' - LANG='C' - LANGUAGE='en_US:en' - LC_ALL='C' - MAIL='/var/mail/root' - OPTIND='1' - PATH='/usr/sbin:/usr/bin:/sbin:/bin:/usr/games' - PBCURRENTCOMMANDLINEOPERATION='build' - PBUILDER_OPERATION='build' - PBUILDER_PKGDATADIR='/usr/share/pbuilder' - PBUILDER_PKGLIBDIR='/usr/lib/pbuilder' - PBUILDER_SYSCONFDIR='/etc' - PPID='22464' - PS1='# ' - PS2='> ' + INVOCATION_ID=363c5261fab04829b485bd39388d6780 + LANG=C + LANGUAGE=it_CH:it + LC_ALL=C + MACHTYPE=arm-unknown-linux-gnueabihf + MAIL=/var/mail/root + OPTERR=1 + OPTIND=1 + OSTYPE=linux-gnueabihf + PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path + PBCURRENTCOMMANDLINEOPERATION=build + PBUILDER_OPERATION=build + PBUILDER_PKGDATADIR=/usr/share/pbuilder + PBUILDER_PKGLIBDIR=/usr/lib/pbuilder + PBUILDER_SYSCONFDIR=/etc + PIPESTATUS=([0]="0") + POSIXLY_CORRECT=y + PPID=29299 PS4='+ ' - PWD='/' - SHELL='/bin/bash' - SHLVL='2' - SUDO_COMMAND='/usr/bin/timeout -k 18.1h 18h /usr/bin/ionice -c 3 /usr/bin/nice /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/r-b-build.qVeudEaw/pbuilderrc_B4Jn --distribution trixie --hookdir /etc/pbuilder/first-build-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/trixie-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/r-b-build.qVeudEaw/b1 --logfile b1/build.log golang-github-hashicorp-raft_1.5.0-1.dsc' - SUDO_GID='110' - SUDO_UID='103' - SUDO_USER='jenkins' - TERM='unknown' - TZ='/usr/share/zoneinfo/Etc/GMT+12' - USER='root' - _='/usr/bin/systemd-run' - http_proxy='http://10.0.0.15:3142/' + PWD=/ + SHELL=/bin/bash + SHELLOPTS=braceexpand:errexit:hashall:interactive-comments:posix + SHLVL=3 + SUDO_COMMAND='/usr/bin/timeout -k 24.1h 24h /usr/bin/ionice -c 3 /usr/bin/nice -n 11 /usr/bin/unshare --uts -- /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/r-b-build.qVeudEaw/pbuilderrc_oaEC --distribution trixie --hookdir /etc/pbuilder/rebuild-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/trixie-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/r-b-build.qVeudEaw/b2 --logfile b2/build.log golang-github-hashicorp-raft_1.5.0-1.dsc' + SUDO_GID=114 + SUDO_UID=109 + SUDO_USER=jenkins + TERM=unknown + TZ=/usr/share/zoneinfo/Etc/GMT-14 + UID=0 + USER=root + _='I: set' + http_proxy=http://10.0.0.15:3142/ I: uname -a - Linux virt32z 6.1.0-30-armmp-lpae #1 SMP Debian 6.1.124-1 (2025-01-12) armv7l GNU/Linux + Linux i-capture-the-hostname 6.1.0-30-arm64 #1 SMP Debian 6.1.124-1 (2025-01-12) aarch64 GNU/Linux I: ls -l /bin lrwxrwxrwx 1 root root 7 Nov 22 14:40 /bin -> usr/bin -I: user script /srv/workspace/pbuilder/22464/tmp/hooks/D02_print_environment finished +I: user script /srv/workspace/pbuilder/29299/tmp/hooks/D02_print_environment finished -> Attempting to satisfy build-dependencies -> Creating pbuilder-satisfydepends-dummy package Package: pbuilder-satisfydepends-dummy @@ -223,7 +255,7 @@ Get: 104 http://deb.debian.org/debian trixie/main armhf golang-gopkg-check.v1-dev all 0.0+git20200902.038fdea-1 [32.1 kB] Get: 105 http://deb.debian.org/debian trixie/main armhf golang-gopkg-mgo.v2-dev all 2016.08.01-7 [316 kB] Get: 106 http://deb.debian.org/debian trixie/main armhf golang-github-hashicorp-go-msgpack-dev all 0.5.5-1 [43.3 kB] -Fetched 123 MB in 11s (11.0 MB/s) +Fetched 123 MB in 4s (34.0 MB/s) Preconfiguring packages ... Selecting previously unselected package sensible-utils. (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 19569 files and directories currently installed.) @@ -662,7 +694,11 @@ Building tag database... -> Finished parsing the build-deps I: Building the package -I: Running cd /build/reproducible-path/golang-github-hashicorp-raft-1.5.0/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" HOME="/nonexistent/first-build" dpkg-buildpackage -us -uc -b && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" HOME="/nonexistent/first-build" dpkg-genchanges -S > ../golang-github-hashicorp-raft_1.5.0-1_source.changes +I: user script /srv/workspace/pbuilder/29299/tmp/hooks/A99_set_merged_usr starting +Not re-configuring usrmerge for trixie +I: user script /srv/workspace/pbuilder/29299/tmp/hooks/A99_set_merged_usr finished +hostname: Name or service not known +I: Running cd /build/reproducible-path/golang-github-hashicorp-raft-1.5.0/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" HOME="/nonexistent/second-build" dpkg-buildpackage -us -uc -b && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" HOME="/nonexistent/second-build" dpkg-genchanges -S > ../golang-github-hashicorp-raft_1.5.0-1_source.changes dpkg-buildpackage: info: source package golang-github-hashicorp-raft dpkg-buildpackage: info: source version 1.5.0-1 dpkg-buildpackage: info: source distribution unstable @@ -680,36 +716,34 @@ dh_autoreconf -O--buildsystem=golang dh_auto_configure -O--buildsystem=golang dh_auto_build -O--buildsystem=golang - cd obj-arm-linux-gnueabihf && go install -trimpath -v -p 3 github.com/hashicorp/raft github.com/hashicorp/raft/bench + cd obj-arm-linux-gnueabihf && go install -trimpath -v -p 6 github.com/hashicorp/raft github.com/hashicorp/raft/bench +internal/byteorder +internal/godebugs +internal/coverage/rtcov internal/unsafeheader internal/goarch internal/cpu -internal/byteorder -internal/abi -internal/chacha8rand -internal/bytealg -internal/coverage/rtcov -internal/godebugs internal/goexperiment -internal/goos +internal/abi internal/profilerecord -internal/runtime/atomic +internal/goos internal/runtime/syscall -internal/stringslite runtime/internal/math -internal/runtime/exithook +internal/chacha8rand runtime/internal/sys internal/race +internal/bytealg sync/atomic +internal/runtime/atomic unicode -runtime unicode/utf8 +internal/stringslite container/list internal/asan internal/itoa internal/msan +internal/runtime/exithook math/bits -math crypto/internal/alias crypto/internal/boring/sig cmp @@ -719,146 +753,148 @@ internal/nettrace vendor/golang.org/x/crypto/cryptobyte/asn1 vendor/golang.org/x/crypto/internal/alias -internal/reflectlite +math +runtime crypto/subtle -sync +internal/reflectlite iter -slices internal/weak +sync maps -internal/bisect +slices internal/testlog internal/singleflight -errors +internal/bisect +runtime/cgo internal/godebug +errors +sort io internal/oserror +path +strconv syscall +math/rand bytes -runtime/cgo strings -time -bufio hash -strconv -context -crypto -crypto/cipher crypto/internal/randutil -internal/syscall/unix +github.com/hashicorp/golang-lru/simplelru +vendor/golang.org/x/net/dns/dnsmessage +crypto/cipher +crypto +bufio reflect crypto/internal/boring -path -io/fs -internal/filepathlite -internal/poll -internal/syscall/execenv -math/rand -os -github.com/hashicorp/golang-lru/simplelru -sort github.com/hashicorp/go-immutable-radix -vendor/golang.org/x/net/dns/dnsmessage math/rand/v2 -internal/fmtsort -fmt -encoding/binary internal/concurrent -unique -encoding/base64 hash/crc32 crypto/aes -net/netip -math/big -encoding/json -log -net -compress/flate -compress/gzip -crypto/rand +unique crypto/des crypto/internal/edwards25519/field +time +internal/syscall/execenv +internal/syscall/unix +net/netip crypto/internal/nistec/fiat -crypto/internal/bigmod -crypto/internal/boring/bbig crypto/sha512 -encoding/asn1 -vendor/golang.org/x/crypto/cryptobyte crypto/internal/edwards25519 -crypto/ed25519 -crypto/internal/nistec crypto/hmac +vendor/golang.org/x/crypto/hkdf +crypto/md5 +crypto/rc4 +crypto/sha1 +crypto/sha256 +net/http/internal/ascii +vendor/golang.org/x/text/transform +regexp/syntax +internal/saferio +html +hash/crc64 +context +io/fs +internal/poll +internal/fmtsort +encoding/binary +internal/filepathlite +regexp +os vendor/golang.org/x/crypto/chacha20 vendor/golang.org/x/crypto/internal/poly1305 +encoding/base64 +golang.org/x/sys/unix vendor/golang.org/x/crypto/chacha20poly1305 -vendor/golang.org/x/crypto/hkdf +encoding/pem +crypto/internal/nistec vendor/golang.org/x/sys/cpu +path/filepath +fmt +net crypto/ecdh -crypto/elliptic -crypto/internal/hpke vendor/golang.org/x/crypto/sha3 -crypto/ecdsa -crypto/internal/mlkem768 -crypto/md5 -crypto/rc4 -crypto/rsa -crypto/sha1 -crypto/sha256 -crypto/dsa +os/signal +io/ioutil +internal/sysinfo +log +encoding/json +compress/flate +math/big encoding/hex -encoding/pem -crypto/x509/pkix net/url -path/filepath -vendor/golang.org/x/text/transform +compress/gzip vendor/golang.org/x/text/unicode/bidi -vendor/golang.org/x/text/unicode/norm -vendor/golang.org/x/text/secure/bidirule vendor/golang.org/x/net/http2/hpack -crypto/x509 -net/textproto +vendor/golang.org/x/text/unicode/norm mime -vendor/golang.org/x/net/idna mime/quotedprintable -mime/multipart -vendor/golang.org/x/net/http/httpguts -crypto/tls -vendor/golang.org/x/net/http/httpproxy net/http/internal -net/http/internal/ascii -os/signal -golang.org/x/sys/unix -io/ioutil -regexp/syntax -regexp -internal/saferio -encoding/gob github.com/mattn/go-isatty +vendor/golang.org/x/text/secure/bidirule github.com/mattn/go-colorable +encoding/gob github.com/fatih/color -net/http/httptrace -github.com/hashicorp/go-hclog -net/http go/token -html +crypto/rand +crypto/elliptic +crypto/internal/bigmod +crypto/internal/boring/bbig +encoding/asn1 +crypto/ed25519 +crypto/internal/hpke +crypto/internal/mlkem768 +crypto/rsa +crypto/dsa +vendor/golang.org/x/net/idna +github.com/hashicorp/go-hclog text/template/parse -hash/crc64 +vendor/golang.org/x/crypto/cryptobyte +crypto/x509/pkix flag -internal/sysinfo runtime/debug -text/template +crypto/ecdsa runtime/trace testing +text/template html/template -net/rpc +net/textproto +vendor/golang.org/x/net/http/httpproxy +crypto/x509 +vendor/golang.org/x/net/http/httpguts +mime/multipart +crypto/tls +net/http/httptrace +net/http github.com/armon/go-metrics +net/rpc github.com/hashicorp/go-msgpack/codec github.com/hashicorp/raft github.com/hashicorp/raft/bench debian/rules override_dh_auto_test make[1]: Entering directory '/build/reproducible-path/golang-github-hashicorp-raft-1.5.0' dh_auto_test - cd obj-arm-linux-gnueabihf && go test -vet=off -v -p 3 github.com/hashicorp/raft github.com/hashicorp/raft/bench + cd obj-arm-linux-gnueabihf && go test -vet=off -v -p 6 github.com/hashicorp/raft github.com/hashicorp/raft/bench ? github.com/hashicorp/raft/bench [no test files] === RUN TestCommitment_setVoters --- PASS: TestCommitment_setVoters (0.00s) @@ -901,33 +937,33 @@ === RUN TestFileSnapshotSinkImpl --- PASS: TestFileSnapshotSinkImpl (0.00s) === RUN TestFileSS_CreateSnapshotMissingParentDir -2025-01-25T14:28:10.153Z [INFO] creating new snapshot: path=/tmp/raft986191241/raft904987765/snapshots/3-10-1737815290153.tmp ---- PASS: TestFileSS_CreateSnapshotMissingParentDir (0.01s) +2025-01-25T14:38:50.969Z [INFO] creating new snapshot: path=/tmp/raft3809586164/raft4160005269/snapshots/3-10-1737815930969.tmp +--- PASS: TestFileSS_CreateSnapshotMissingParentDir (0.02s) === RUN TestFileSS_CreateSnapshot -2025-01-25T14:28:10.162Z [INFO] creating new snapshot: path=/tmp/raft1671639067/snapshots/3-10-1737815290162.tmp -2025-01-25T14:28:10.167Z [WARN] found temporary snapshot: name=3-10-1737815290162.tmp ---- PASS: TestFileSS_CreateSnapshot (0.02s) +2025-01-25T14:38:50.984Z [INFO] creating new snapshot: path=/tmp/raft213862606/snapshots/3-10-1737815930983.tmp +2025-01-25T14:38:50.994Z [WARN] found temporary snapshot: name=3-10-1737815930983.tmp +--- PASS: TestFileSS_CreateSnapshot (0.04s) === RUN TestFileSS_CancelSnapshot -2025-01-25T14:28:10.179Z [INFO] creating new snapshot: path=/tmp/raft3437335716/snapshots/3-10-1737815290179.tmp +2025-01-25T14:38:51.022Z [INFO] creating new snapshot: path=/tmp/raft1257124753/snapshots/3-10-1737815931022.tmp --- PASS: TestFileSS_CancelSnapshot (0.01s) === RUN TestFileSS_Retention -2025-01-25T14:28:10.188Z [INFO] creating new snapshot: path=/tmp/raft3908741870/snapshots/3-10-1737815290188.tmp -2025-01-25T14:28:10.203Z [INFO] creating new snapshot: path=/tmp/raft3908741870/snapshots/3-11-1737815290203.tmp -2025-01-25T14:28:10.218Z [INFO] creating new snapshot: path=/tmp/raft3908741870/snapshots/3-12-1737815290218.tmp -2025-01-25T14:28:10.232Z [INFO] reaping snapshot: path=/tmp/raft3908741870/snapshots/3-10-1737815290188 -2025-01-25T14:28:10.232Z [INFO] creating new snapshot: path=/tmp/raft3908741870/snapshots/3-13-1737815290232.tmp -2025-01-25T14:28:10.248Z [INFO] reaping snapshot: path=/tmp/raft3908741870/snapshots/3-11-1737815290203 -2025-01-25T14:28:10.248Z [INFO] creating new snapshot: path=/tmp/raft3908741870/snapshots/3-14-1737815290248.tmp -2025-01-25T14:28:10.262Z [INFO] reaping snapshot: path=/tmp/raft3908741870/snapshots/3-12-1737815290218 ---- PASS: TestFileSS_Retention (0.08s) +2025-01-25T14:38:51.033Z [INFO] creating new snapshot: path=/tmp/raft1606631164/snapshots/3-10-1737815931033.tmp +2025-01-25T14:38:51.066Z [INFO] creating new snapshot: path=/tmp/raft1606631164/snapshots/3-11-1737815931066.tmp +2025-01-25T14:38:51.093Z [INFO] creating new snapshot: path=/tmp/raft1606631164/snapshots/3-12-1737815931093.tmp +2025-01-25T14:38:51.119Z [INFO] reaping snapshot: path=/tmp/raft1606631164/snapshots/3-10-1737815931033 +2025-01-25T14:38:51.120Z [INFO] creating new snapshot: path=/tmp/raft1606631164/snapshots/3-13-1737815931120.tmp +2025-01-25T14:38:51.149Z [INFO] reaping snapshot: path=/tmp/raft1606631164/snapshots/3-11-1737815931066 +2025-01-25T14:38:51.150Z [INFO] creating new snapshot: path=/tmp/raft1606631164/snapshots/3-14-1737815931150.tmp +2025-01-25T14:38:51.180Z [INFO] reaping snapshot: path=/tmp/raft1606631164/snapshots/3-12-1737815931093 +--- PASS: TestFileSS_Retention (0.15s) === RUN TestFileSS_BadPerm --- PASS: TestFileSS_BadPerm (0.00s) === RUN TestFileSS_MissingParentDir --- PASS: TestFileSS_MissingParentDir (0.00s) === RUN TestFileSS_Ordering -2025-01-25T14:28:10.265Z [INFO] creating new snapshot: path=/tmp/raft2509995759/snapshots/5-130350-1737815290265.tmp -2025-01-25T14:28:10.281Z [INFO] creating new snapshot: path=/tmp/raft2509995759/snapshots/36-204917-1737815290281.tmp ---- PASS: TestFileSS_Ordering (0.03s) +2025-01-25T14:38:51.193Z [INFO] creating new snapshot: path=/tmp/raft233169493/snapshots/5-130350-1737815931193.tmp +2025-01-25T14:38:51.215Z [INFO] creating new snapshot: path=/tmp/raft233169493/snapshots/36-204917-1737815931215.tmp +--- PASS: TestFileSS_Ordering (0.05s) === RUN TestDeferFutureSuccess --- PASS: TestDeferFutureSuccess (0.00s) === RUN TestDeferFutureError @@ -969,28 +1005,28 @@ === RUN TestNetworkTransport_Heartbeat_FastPath --- PASS: TestNetworkTransport_Heartbeat_FastPath (0.00s) === RUN TestNetworkTransport_AppendEntries ---- PASS: TestNetworkTransport_AppendEntries (0.00s) +--- PASS: TestNetworkTransport_AppendEntries (0.01s) === RUN TestNetworkTransport_AppendEntriesPipeline ---- PASS: TestNetworkTransport_AppendEntriesPipeline (0.01s) +--- PASS: TestNetworkTransport_AppendEntriesPipeline (0.02s) === RUN TestNetworkTransport_AppendEntriesPipeline_CloseStreams ---- PASS: TestNetworkTransport_AppendEntriesPipeline_CloseStreams (0.03s) +--- PASS: TestNetworkTransport_AppendEntriesPipeline_CloseStreams (0.06s) === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=0 -2025-01-25T14:28:10.420Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" +2025-01-25T14:38:51.426Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=1 === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=2 -2025-01-25T14:28:10.474Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" +2025-01-25T14:38:51.482Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=10 ---- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight (0.16s) +2025-01-25T14:38:51.539Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" +--- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight (0.17s) --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=0 (0.05s) --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=1 (0.00s) --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=2 (0.05s) - --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=10 (0.05s) + --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=10 (0.06s) === RUN TestNetworkTransport_RequestVote -2025-01-25T14:28:10.526Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" ---- PASS: TestNetworkTransport_RequestVote (0.00s) +--- PASS: TestNetworkTransport_RequestVote (0.01s) === RUN TestNetworkTransport_InstallSnapshot ---- PASS: TestNetworkTransport_InstallSnapshot (0.00s) +--- PASS: TestNetworkTransport_InstallSnapshot (0.01s) === RUN TestNetworkTransport_EncodeDecode --- PASS: TestNetworkTransport_EncodeDecode (0.00s) === RUN TestNetworkTransport_EncodeDecode_AddressProvider @@ -998,27 +1034,27 @@ === RUN TestNetworkTransport_PooledConn --- PASS: TestNetworkTransport_PooledConn (0.01s) === RUN TestNetworkTransport_ListenBackoff - net_transport_test.go:839: countingWriter: 2025-01-25T14:28:10.543Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-01-25T14:38:51.575Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-01-25T14:28:10.549Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-01-25T14:38:51.581Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-01-25T14:28:10.559Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-01-25T14:38:51.592Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-01-25T14:28:10.580Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-01-25T14:38:51.613Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-01-25T14:28:10.621Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-01-25T14:38:51.654Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-01-25T14:28:10.702Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-01-25T14:38:51.735Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-01-25T14:28:10.862Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-01-25T14:38:51.896Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-01-25T14:28:11.183Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-01-25T14:38:52.216Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-01-25T14:28:11.823Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-01-25T14:38:52.857Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-01-25T14:28:12.826Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-01-25T14:38:53.860Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-01-25T14:28:13.826Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-01-25T14:38:54.861Z [ERROR] test: failed to accept connection: error="intentional error in test" --- PASS: TestNetworkTransport_ListenBackoff (4.00s) === RUN TestPeersJSON_BadConfiguration @@ -1028,2544 +1064,2562 @@ === RUN TestPeersJSON_ReadConfigJSON --- PASS: TestPeersJSON_ReadConfigJSON (0.00s) === RUN TestRaft_StartStop -2025-01-25T14:28:14.546Z [INFO] server-122b83cd-22b5-4287-9bde-04a0b50e2f6a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-122b83cd-22b5-4287-9bde-04a0b50e2f6a Address:122b83cd-22b5-4287-9bde-04a0b50e2f6a}]" +2025-01-25T14:38:55.587Z [INFO] server-5096cecf-0bdd-fb50-d8c5-2f8d72d692f8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5096cecf-0bdd-fb50-d8c5-2f8d72d692f8 Address:5096cecf-0bdd-fb50-d8c5-2f8d72d692f8}]" --- PASS: TestRaft_StartStop (0.00s) === RUN TestRaft_AfterShutdown -2025-01-25T14:28:14.547Z [INFO] server-2c8c72a6-0931-e149-8e4b-641d0f405666: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2c8c72a6-0931-e149-8e4b-641d0f405666 Address:2c8c72a6-0931-e149-8e4b-641d0f405666}]" +2025-01-25T14:38:55.591Z [INFO] server-3f848fdd-f1a9-88f8-07df-1718b0629404: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3f848fdd-f1a9-88f8-07df-1718b0629404 Address:3f848fdd-f1a9-88f8-07df-1718b0629404}]" --- PASS: TestRaft_AfterShutdown (0.00s) === RUN TestRaft_LiveBootstrap -2025-01-25T14:28:14.549Z [INFO] server-f9c860e8-09cd-2778-edef-754deda31728: initial configuration: index=0 servers=[] -2025-01-25T14:28:14.549Z [INFO] server-3caa1664-b24f-f6c0-aa9c-895ee1f3c378: initial configuration: index=0 servers=[] -2025-01-25T14:28:14.549Z [INFO] server-16ab997b-d653-d66c-71ce-cacf38299658: initial configuration: index=0 servers=[] -2025-01-25T14:28:14.549Z [INFO] server-f9c860e8-09cd-2778-edef-754deda31728: entering follower state: follower="Node at f9c860e8-09cd-2778-edef-754deda31728 [Follower]" leader-address= leader-id= -2025-01-25T14:28:14.549Z [INFO] server-3caa1664-b24f-f6c0-aa9c-895ee1f3c378: entering follower state: follower="Node at 3caa1664-b24f-f6c0-aa9c-895ee1f3c378 [Follower]" leader-address= leader-id= -2025-01-25T14:28:14.549Z [INFO] server-16ab997b-d653-d66c-71ce-cacf38299658: entering follower state: follower="Node at 16ab997b-d653-d66c-71ce-cacf38299658 [Follower]" leader-address= leader-id= -2025-01-25T14:28:14.549Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:28:14.610Z [WARN] server-f9c860e8-09cd-2778-edef-754deda31728: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:14.610Z [INFO] server-f9c860e8-09cd-2778-edef-754deda31728: entering candidate state: node="Node at f9c860e8-09cd-2778-edef-754deda31728 [Candidate]" term=2 -2025-01-25T14:28:14.610Z [INFO] server-f9c860e8-09cd-2778-edef-754deda31728: election won: term=2 tally=2 -2025-01-25T14:28:14.610Z [INFO] server-f9c860e8-09cd-2778-edef-754deda31728: entering leader state: leader="Node at f9c860e8-09cd-2778-edef-754deda31728 [Leader]" -2025-01-25T14:28:14.610Z [INFO] server-f9c860e8-09cd-2778-edef-754deda31728: added peer, starting replication: peer=server-3caa1664-b24f-f6c0-aa9c-895ee1f3c378 -2025-01-25T14:28:14.610Z [INFO] server-f9c860e8-09cd-2778-edef-754deda31728: added peer, starting replication: peer=server-16ab997b-d653-d66c-71ce-cacf38299658 -2025-01-25T14:28:14.611Z [WARN] server-16ab997b-d653-d66c-71ce-cacf38299658: failed to get previous log: previous-index=1 last-index=0 error="log not found" -2025-01-25T14:28:14.611Z [WARN] server-f9c860e8-09cd-2778-edef-754deda31728: appendEntries rejected, sending older logs: peer="{Voter server-16ab997b-d653-d66c-71ce-cacf38299658 16ab997b-d653-d66c-71ce-cacf38299658}" next=1 -2025-01-25T14:28:14.611Z [INFO] server-f9c860e8-09cd-2778-edef-754deda31728: pipelining replication: peer="{Voter server-16ab997b-d653-d66c-71ce-cacf38299658 16ab997b-d653-d66c-71ce-cacf38299658}" -2025-01-25T14:28:14.611Z [WARN] server-3caa1664-b24f-f6c0-aa9c-895ee1f3c378: failed to get previous log: previous-index=1 last-index=0 error="log not found" -2025-01-25T14:28:14.611Z [WARN] server-f9c860e8-09cd-2778-edef-754deda31728: appendEntries rejected, sending older logs: peer="{Voter server-3caa1664-b24f-f6c0-aa9c-895ee1f3c378 3caa1664-b24f-f6c0-aa9c-895ee1f3c378}" next=1 -2025-01-25T14:28:14.611Z [INFO] server-f9c860e8-09cd-2778-edef-754deda31728: pipelining replication: peer="{Voter server-3caa1664-b24f-f6c0-aa9c-895ee1f3c378 3caa1664-b24f-f6c0-aa9c-895ee1f3c378}" -2025-01-25T14:28:14.717Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:28:14.611404565 +0000 UTC m=+4.465729547 (2 nodes), 61.441109ms from start of poll, 62.200002ms from cluster start. Timeout at 2025-01-25 14:28:14.716414405 +0000 UTC m=+4.570739347, 105.0098ms after stability -2025-01-25T14:28:14.717Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:14.823Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:14.717741138 +0000 UTC m=+4.572066080 (1 nodes), 1.84µs from start of poll, 168.536535ms from cluster start. Timeout at 2025-01-25 14:28:14.822773617 +0000 UTC m=+4.677098559, 105.032479ms after stability -2025-01-25T14:28:14.835Z [INFO] server-f9c860e8-09cd-2778-edef-754deda31728: aborting pipeline replication: peer="{Voter server-16ab997b-d653-d66c-71ce-cacf38299658 16ab997b-d653-d66c-71ce-cacf38299658}" -2025-01-25T14:28:14.835Z [INFO] server-f9c860e8-09cd-2778-edef-754deda31728: aborting pipeline replication: peer="{Voter server-3caa1664-b24f-f6c0-aa9c-895ee1f3c378 3caa1664-b24f-f6c0-aa9c-895ee1f3c378}" ---- PASS: TestRaft_LiveBootstrap (0.29s) +2025-01-25T14:38:55.597Z [INFO] server-10ee98ee-50ba-81d4-1cf3-d46d12459609: initial configuration: index=0 servers=[] +2025-01-25T14:38:55.597Z [INFO] server-aae0c362-ecdd-086a-d140-51c85a7fe06f: initial configuration: index=0 servers=[] +2025-01-25T14:38:55.597Z [INFO] server-10ee98ee-50ba-81d4-1cf3-d46d12459609: entering follower state: follower="Node at 10ee98ee-50ba-81d4-1cf3-d46d12459609 [Follower]" leader-address= leader-id= +2025-01-25T14:38:55.597Z [INFO] server-aae0c362-ecdd-086a-d140-51c85a7fe06f: entering follower state: follower="Node at aae0c362-ecdd-086a-d140-51c85a7fe06f [Follower]" leader-address= leader-id= +2025-01-25T14:38:55.597Z [INFO] server-9da07732-d5fa-7b25-2576-3c487219498d: initial configuration: index=0 servers=[] +2025-01-25T14:38:55.598Z [INFO] server-9da07732-d5fa-7b25-2576-3c487219498d: entering follower state: follower="Node at 9da07732-d5fa-7b25-2576-3c487219498d [Follower]" leader-address= leader-id= +2025-01-25T14:38:55.598Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:38:55.666Z [WARN] server-10ee98ee-50ba-81d4-1cf3-d46d12459609: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:38:55.667Z [INFO] server-10ee98ee-50ba-81d4-1cf3-d46d12459609: entering candidate state: node="Node at 10ee98ee-50ba-81d4-1cf3-d46d12459609 [Candidate]" term=2 +2025-01-25T14:38:55.667Z [INFO] server-10ee98ee-50ba-81d4-1cf3-d46d12459609: election won: term=2 tally=2 +2025-01-25T14:38:55.667Z [INFO] server-10ee98ee-50ba-81d4-1cf3-d46d12459609: entering leader state: leader="Node at 10ee98ee-50ba-81d4-1cf3-d46d12459609 [Leader]" +2025-01-25T14:38:55.667Z [INFO] server-10ee98ee-50ba-81d4-1cf3-d46d12459609: added peer, starting replication: peer=server-aae0c362-ecdd-086a-d140-51c85a7fe06f +2025-01-25T14:38:55.667Z [INFO] server-10ee98ee-50ba-81d4-1cf3-d46d12459609: added peer, starting replication: peer=server-9da07732-d5fa-7b25-2576-3c487219498d +2025-01-25T14:38:55.668Z [WARN] server-aae0c362-ecdd-086a-d140-51c85a7fe06f: failed to get previous log: previous-index=1 last-index=0 error="log not found" +2025-01-25T14:38:55.668Z [WARN] server-9da07732-d5fa-7b25-2576-3c487219498d: failed to get previous log: previous-index=1 last-index=0 error="log not found" +2025-01-25T14:38:55.669Z [WARN] server-10ee98ee-50ba-81d4-1cf3-d46d12459609: appendEntries rejected, sending older logs: peer="{Voter server-9da07732-d5fa-7b25-2576-3c487219498d 9da07732-d5fa-7b25-2576-3c487219498d}" next=1 +2025-01-25T14:38:55.669Z [WARN] server-10ee98ee-50ba-81d4-1cf3-d46d12459609: appendEntries rejected, sending older logs: peer="{Voter server-aae0c362-ecdd-086a-d140-51c85a7fe06f aae0c362-ecdd-086a-d140-51c85a7fe06f}" next=1 +2025-01-25T14:38:55.669Z [INFO] server-10ee98ee-50ba-81d4-1cf3-d46d12459609: pipelining replication: peer="{Voter server-9da07732-d5fa-7b25-2576-3c487219498d 9da07732-d5fa-7b25-2576-3c487219498d}" +2025-01-25T14:38:55.670Z [INFO] server-10ee98ee-50ba-81d4-1cf3-d46d12459609: pipelining replication: peer="{Voter server-aae0c362-ecdd-086a-d140-51c85a7fe06f aae0c362-ecdd-086a-d140-51c85a7fe06f}" +2025-01-25T14:38:55.773Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:38:55.667953533 +0000 UTC m=+4.713792850 (2 nodes), 69.33416ms from start of poll, 71.258166ms from cluster start. Timeout at 2025-01-25 14:38:55.772980657 +0000 UTC m=+4.818819975, 105.027125ms after stability +2025-01-25T14:38:55.773Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:38:55.879Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:38:55.774000633 +0000 UTC m=+4.819839951 (1 nodes), 6.125µs from start of poll, 177.305267ms from cluster start. Timeout at 2025-01-25 14:38:55.879035634 +0000 UTC m=+4.924874951, 105.035ms after stability +2025-01-25T14:38:55.892Z [INFO] server-10ee98ee-50ba-81d4-1cf3-d46d12459609: aborting pipeline replication: peer="{Voter server-9da07732-d5fa-7b25-2576-3c487219498d 9da07732-d5fa-7b25-2576-3c487219498d}" +2025-01-25T14:38:55.892Z [INFO] server-10ee98ee-50ba-81d4-1cf3-d46d12459609: aborting pipeline replication: peer="{Voter server-aae0c362-ecdd-086a-d140-51c85a7fe06f aae0c362-ecdd-086a-d140-51c85a7fe06f}" +2025-01-25T14:38:56.394Z [ERROR] server-10ee98ee-50ba-81d4-1cf3-d46d12459609: failed to heartbeat to: peer=aae0c362-ecdd-086a-d140-51c85a7fe06f backoff time=10ms error="command timed out" +--- PASS: TestRaft_LiveBootstrap (0.81s) === RUN TestRaft_LiveBootstrap_From_NonVoter -2025-01-25T14:28:14.837Z [INFO] server-3b47beb7-0922-d507-d8f3-c0117edab7ef: initial configuration: index=0 servers=[] -2025-01-25T14:28:14.837Z [INFO] server-4e3ad652-d8cf-a433-489f-c332805ebb18: initial configuration: index=0 servers=[] -2025-01-25T14:28:14.837Z [INFO] server-3b47beb7-0922-d507-d8f3-c0117edab7ef: entering follower state: follower="Node at 3b47beb7-0922-d507-d8f3-c0117edab7ef [Follower]" leader-address= leader-id= -2025-01-25T14:28:14.837Z [INFO] server-4e3ad652-d8cf-a433-489f-c332805ebb18: entering follower state: follower="Node at 4e3ad652-d8cf-a433-489f-c332805ebb18 [Follower]" leader-address= leader-id= ---- PASS: TestRaft_LiveBootstrap_From_NonVoter (0.00s) +2025-01-25T14:38:56.402Z [INFO] server-3ad84e97-8ba3-1bd7-6dd0-f1df6aaf4404: initial configuration: index=0 servers=[] +2025-01-25T14:38:56.402Z [INFO] server-2ac2c05b-e6ad-a20a-82e7-a3aba1a4f793: initial configuration: index=0 servers=[] +2025-01-25T14:38:56.402Z [INFO] server-3ad84e97-8ba3-1bd7-6dd0-f1df6aaf4404: entering follower state: follower="Node at 3ad84e97-8ba3-1bd7-6dd0-f1df6aaf4404 [Follower]" leader-address= leader-id= +2025-01-25T14:38:56.403Z [INFO] server-2ac2c05b-e6ad-a20a-82e7-a3aba1a4f793: entering follower state: follower="Node at 2ac2c05b-e6ad-a20a-82e7-a3aba1a4f793 [Follower]" leader-address= leader-id= +--- PASS: TestRaft_LiveBootstrap_From_NonVoter (0.01s) === RUN TestRaft_RecoverCluster_NoState -2025-01-25T14:28:14.839Z [INFO] server-12290731-d721-ab25-5ce7-121a65f5de79: initial configuration: index=0 servers=[] +2025-01-25T14:38:56.408Z [INFO] server-1c899e18-7fa4-4123-5e67-36e020369ee7: initial configuration: index=0 servers=[] +2025-01-25T14:38:56.408Z [INFO] server-1c899e18-7fa4-4123-5e67-36e020369ee7: entering follower state: follower="Node at 1c899e18-7fa4-4123-5e67-36e020369ee7 [Shutdown]" leader-address= leader-id= --- PASS: TestRaft_RecoverCluster_NoState (0.00s) === RUN TestRaft_RecoverCluster === RUN TestRaft_RecoverCluster/no_snapshot,_no_trailing_logs -2025-01-25T14:28:14.841Z [INFO] server-72e4e832-f6c8-eefc-918e-38def62c0f95: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-72e4e832-f6c8-eefc-918e-38def62c0f95 Address:72e4e832-f6c8-eefc-918e-38def62c0f95} {Suffrage:Voter ID:server-36934e59-7602-1f6e-1619-13735feb8156 Address:36934e59-7602-1f6e-1619-13735feb8156} {Suffrage:Voter ID:server-9b6bcc83-9a5a-f8f8-1dcd-2472348e6924 Address:9b6bcc83-9a5a-f8f8-1dcd-2472348e6924}]" -2025-01-25T14:28:14.841Z [INFO] server-36934e59-7602-1f6e-1619-13735feb8156: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-72e4e832-f6c8-eefc-918e-38def62c0f95 Address:72e4e832-f6c8-eefc-918e-38def62c0f95} {Suffrage:Voter ID:server-36934e59-7602-1f6e-1619-13735feb8156 Address:36934e59-7602-1f6e-1619-13735feb8156} {Suffrage:Voter ID:server-9b6bcc83-9a5a-f8f8-1dcd-2472348e6924 Address:9b6bcc83-9a5a-f8f8-1dcd-2472348e6924}]" -2025-01-25T14:28:14.841Z [INFO] server-72e4e832-f6c8-eefc-918e-38def62c0f95: entering follower state: follower="Node at 72e4e832-f6c8-eefc-918e-38def62c0f95 [Follower]" leader-address= leader-id= -2025-01-25T14:28:14.841Z [INFO] server-36934e59-7602-1f6e-1619-13735feb8156: entering follower state: follower="Node at 36934e59-7602-1f6e-1619-13735feb8156 [Follower]" leader-address= leader-id= -2025-01-25T14:28:14.841Z [INFO] server-9b6bcc83-9a5a-f8f8-1dcd-2472348e6924: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-72e4e832-f6c8-eefc-918e-38def62c0f95 Address:72e4e832-f6c8-eefc-918e-38def62c0f95} {Suffrage:Voter ID:server-36934e59-7602-1f6e-1619-13735feb8156 Address:36934e59-7602-1f6e-1619-13735feb8156} {Suffrage:Voter ID:server-9b6bcc83-9a5a-f8f8-1dcd-2472348e6924 Address:9b6bcc83-9a5a-f8f8-1dcd-2472348e6924}]" -2025-01-25T14:28:14.841Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:14.841Z [INFO] server-9b6bcc83-9a5a-f8f8-1dcd-2472348e6924: entering follower state: follower="Node at 9b6bcc83-9a5a-f8f8-1dcd-2472348e6924 [Follower]" leader-address= leader-id= -2025-01-25T14:28:14.899Z [WARN] server-72e4e832-f6c8-eefc-918e-38def62c0f95: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:14.899Z [INFO] server-72e4e832-f6c8-eefc-918e-38def62c0f95: entering candidate state: node="Node at 72e4e832-f6c8-eefc-918e-38def62c0f95 [Candidate]" term=2 -2025-01-25T14:28:14.900Z [INFO] server-72e4e832-f6c8-eefc-918e-38def62c0f95: election won: term=2 tally=2 -2025-01-25T14:28:14.900Z [INFO] server-72e4e832-f6c8-eefc-918e-38def62c0f95: entering leader state: leader="Node at 72e4e832-f6c8-eefc-918e-38def62c0f95 [Leader]" -2025-01-25T14:28:14.900Z [INFO] server-72e4e832-f6c8-eefc-918e-38def62c0f95: added peer, starting replication: peer=server-36934e59-7602-1f6e-1619-13735feb8156 -2025-01-25T14:28:14.900Z [INFO] server-72e4e832-f6c8-eefc-918e-38def62c0f95: added peer, starting replication: peer=server-9b6bcc83-9a5a-f8f8-1dcd-2472348e6924 -2025-01-25T14:28:14.900Z [INFO] server-72e4e832-f6c8-eefc-918e-38def62c0f95: pipelining replication: peer="{Voter server-9b6bcc83-9a5a-f8f8-1dcd-2472348e6924 9b6bcc83-9a5a-f8f8-1dcd-2472348e6924}" -2025-01-25T14:28:14.900Z [INFO] server-72e4e832-f6c8-eefc-918e-38def62c0f95: pipelining replication: peer="{Voter server-36934e59-7602-1f6e-1619-13735feb8156 36934e59-7602-1f6e-1619-13735feb8156}" -2025-01-25T14:28:15.006Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:14.900309729 +0000 UTC m=+4.754634711 (1 nodes), 58.626531ms from start of poll, 59.495939ms from cluster start. Timeout at 2025-01-25 14:28:15.005314449 +0000 UTC m=+4.859639431, 105.00472ms after stability -2025-01-25T14:28:15.006Z [INFO] server-72e4e832-f6c8-eefc-918e-38def62c0f95: aborting pipeline replication: peer="{Voter server-9b6bcc83-9a5a-f8f8-1dcd-2472348e6924 9b6bcc83-9a5a-f8f8-1dcd-2472348e6924}" -2025-01-25T14:28:15.006Z [INFO] server-72e4e832-f6c8-eefc-918e-38def62c0f95: aborting pipeline replication: peer="{Voter server-36934e59-7602-1f6e-1619-13735feb8156 36934e59-7602-1f6e-1619-13735feb8156}" -2025-01-25T14:28:15.006Z [INFO] creating new snapshot: path=/tmp/raft911086418/snapshots/2-2-1737815295006.tmp -2025-01-25T14:28:15.008Z [INFO] server-72e4e832-f6c8-eefc-918e-38def62c0f95: starting restore from snapshot: id=2-2-1737815295006 last-index=2 last-term=2 size-in-bytes=1 -2025-01-25T14:28:15.008Z [INFO] server-72e4e832-f6c8-eefc-918e-38def62c0f95: snapshot restore progress: id=2-2-1737815295006 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% -2025-01-25T14:28:15.008Z [INFO] server-72e4e832-f6c8-eefc-918e-38def62c0f95: restored from snapshot: id=2-2-1737815295006 last-index=2 last-term=2 size-in-bytes=1 -2025-01-25T14:28:15.008Z [INFO] server-72e4e832-f6c8-eefc-918e-38def62c0f95: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-72e4e832-f6c8-eefc-918e-38def62c0f95 Address:72e4e832-f6c8-eefc-918e-38def62c0f95} {Suffrage:Voter ID:server-36934e59-7602-1f6e-1619-13735feb8156 Address:36934e59-7602-1f6e-1619-13735feb8156} {Suffrage:Voter ID:server-9b6bcc83-9a5a-f8f8-1dcd-2472348e6924 Address:9b6bcc83-9a5a-f8f8-1dcd-2472348e6924}]" -2025-01-25T14:28:15.009Z [INFO] creating new snapshot: path=/tmp/raft1592049412/snapshots/2-2-1737815295009.tmp -2025-01-25T14:28:15.009Z [INFO] server-72e4e832-f6c8-eefc-918e-38def62c0f95: entering follower state: follower="Node at 72e4e832-f6c8-eefc-918e-38def62c0f95 [Follower]" leader-address= leader-id= -2025-01-25T14:28:15.010Z [INFO] server-36934e59-7602-1f6e-1619-13735feb8156: starting restore from snapshot: id=2-2-1737815295009 last-index=2 last-term=2 size-in-bytes=1 -2025-01-25T14:28:15.010Z [INFO] server-36934e59-7602-1f6e-1619-13735feb8156: snapshot restore progress: id=2-2-1737815295009 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% -2025-01-25T14:28:15.011Z [INFO] server-36934e59-7602-1f6e-1619-13735feb8156: restored from snapshot: id=2-2-1737815295009 last-index=2 last-term=2 size-in-bytes=1 -2025-01-25T14:28:15.011Z [INFO] server-36934e59-7602-1f6e-1619-13735feb8156: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-72e4e832-f6c8-eefc-918e-38def62c0f95 Address:72e4e832-f6c8-eefc-918e-38def62c0f95} {Suffrage:Voter ID:server-36934e59-7602-1f6e-1619-13735feb8156 Address:36934e59-7602-1f6e-1619-13735feb8156} {Suffrage:Voter ID:server-9b6bcc83-9a5a-f8f8-1dcd-2472348e6924 Address:9b6bcc83-9a5a-f8f8-1dcd-2472348e6924}]" -2025-01-25T14:28:15.011Z [INFO] creating new snapshot: path=/tmp/raft539846833/snapshots/2-2-1737815295011.tmp -2025-01-25T14:28:15.011Z [INFO] server-36934e59-7602-1f6e-1619-13735feb8156: entering follower state: follower="Node at 36934e59-7602-1f6e-1619-13735feb8156 [Follower]" leader-address= leader-id= -2025-01-25T14:28:15.012Z [INFO] server-9b6bcc83-9a5a-f8f8-1dcd-2472348e6924: starting restore from snapshot: id=2-2-1737815295011 last-index=2 last-term=2 size-in-bytes=1 -2025-01-25T14:28:15.013Z [INFO] server-9b6bcc83-9a5a-f8f8-1dcd-2472348e6924: snapshot restore progress: id=2-2-1737815295011 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% -2025-01-25T14:28:15.013Z [INFO] server-9b6bcc83-9a5a-f8f8-1dcd-2472348e6924: restored from snapshot: id=2-2-1737815295011 last-index=2 last-term=2 size-in-bytes=1 -2025-01-25T14:28:15.013Z [INFO] server-9b6bcc83-9a5a-f8f8-1dcd-2472348e6924: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-72e4e832-f6c8-eefc-918e-38def62c0f95 Address:72e4e832-f6c8-eefc-918e-38def62c0f95} {Suffrage:Voter ID:server-36934e59-7602-1f6e-1619-13735feb8156 Address:36934e59-7602-1f6e-1619-13735feb8156} {Suffrage:Voter ID:server-9b6bcc83-9a5a-f8f8-1dcd-2472348e6924 Address:9b6bcc83-9a5a-f8f8-1dcd-2472348e6924}]" -2025-01-25T14:28:15.013Z [INFO] server-9b6bcc83-9a5a-f8f8-1dcd-2472348e6924: entering follower state: follower="Node at 9b6bcc83-9a5a-f8f8-1dcd-2472348e6924 [Follower]" leader-address= leader-id= -2025-01-25T14:28:15.072Z [WARN] server-36934e59-7602-1f6e-1619-13735feb8156: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:15.072Z [INFO] server-36934e59-7602-1f6e-1619-13735feb8156: entering candidate state: node="Node at 36934e59-7602-1f6e-1619-13735feb8156 [Candidate]" term=3 -2025-01-25T14:28:15.072Z [INFO] server-36934e59-7602-1f6e-1619-13735feb8156: election won: term=3 tally=2 -2025-01-25T14:28:15.072Z [INFO] server-36934e59-7602-1f6e-1619-13735feb8156: entering leader state: leader="Node at 36934e59-7602-1f6e-1619-13735feb8156 [Leader]" -2025-01-25T14:28:15.072Z [INFO] server-36934e59-7602-1f6e-1619-13735feb8156: added peer, starting replication: peer=server-72e4e832-f6c8-eefc-918e-38def62c0f95 -2025-01-25T14:28:15.072Z [INFO] server-36934e59-7602-1f6e-1619-13735feb8156: added peer, starting replication: peer=server-9b6bcc83-9a5a-f8f8-1dcd-2472348e6924 -2025-01-25T14:28:15.072Z [INFO] server-36934e59-7602-1f6e-1619-13735feb8156: pipelining replication: peer="{Voter server-9b6bcc83-9a5a-f8f8-1dcd-2472348e6924 9b6bcc83-9a5a-f8f8-1dcd-2472348e6924}" -2025-01-25T14:28:15.073Z [INFO] server-36934e59-7602-1f6e-1619-13735feb8156: pipelining replication: peer="{Voter server-72e4e832-f6c8-eefc-918e-38def62c0f95 72e4e832-f6c8-eefc-918e-38def62c0f95}" -2025-01-25T14:28:15.328Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:15.433Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:15.328399604 +0000 UTC m=+5.182724586 (1 nodes), 48.318µs from start of poll, 487.585814ms from cluster start. Timeout at 2025-01-25 14:28:15.433401204 +0000 UTC m=+5.287726186, 105.0016ms after stability -2025-01-25T14:28:15.433Z [INFO] server-36934e59-7602-1f6e-1619-13735feb8156: aborting pipeline replication: peer="{Voter server-9b6bcc83-9a5a-f8f8-1dcd-2472348e6924 9b6bcc83-9a5a-f8f8-1dcd-2472348e6924}" -2025-01-25T14:28:15.433Z [INFO] server-36934e59-7602-1f6e-1619-13735feb8156: aborting pipeline replication: peer="{Voter server-72e4e832-f6c8-eefc-918e-38def62c0f95 72e4e832-f6c8-eefc-918e-38def62c0f95}" +2025-01-25T14:38:56.415Z [INFO] server-5aee53fc-e526-1170-a03d-5e16a4963bb4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5aee53fc-e526-1170-a03d-5e16a4963bb4 Address:5aee53fc-e526-1170-a03d-5e16a4963bb4} {Suffrage:Voter ID:server-2203e000-8725-5b50-6c7a-c3fb1439af7c Address:2203e000-8725-5b50-6c7a-c3fb1439af7c} {Suffrage:Voter ID:server-dff00595-e9b6-d60d-1c22-74dad11551d7 Address:dff00595-e9b6-d60d-1c22-74dad11551d7}]" +2025-01-25T14:38:56.415Z [INFO] server-5aee53fc-e526-1170-a03d-5e16a4963bb4: entering follower state: follower="Node at 5aee53fc-e526-1170-a03d-5e16a4963bb4 [Follower]" leader-address= leader-id= +2025-01-25T14:38:56.416Z [INFO] server-2203e000-8725-5b50-6c7a-c3fb1439af7c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5aee53fc-e526-1170-a03d-5e16a4963bb4 Address:5aee53fc-e526-1170-a03d-5e16a4963bb4} {Suffrage:Voter ID:server-2203e000-8725-5b50-6c7a-c3fb1439af7c Address:2203e000-8725-5b50-6c7a-c3fb1439af7c} {Suffrage:Voter ID:server-dff00595-e9b6-d60d-1c22-74dad11551d7 Address:dff00595-e9b6-d60d-1c22-74dad11551d7}]" +2025-01-25T14:38:56.416Z [INFO] server-2203e000-8725-5b50-6c7a-c3fb1439af7c: entering follower state: follower="Node at 2203e000-8725-5b50-6c7a-c3fb1439af7c [Follower]" leader-address= leader-id= +2025-01-25T14:38:56.416Z [INFO] server-dff00595-e9b6-d60d-1c22-74dad11551d7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5aee53fc-e526-1170-a03d-5e16a4963bb4 Address:5aee53fc-e526-1170-a03d-5e16a4963bb4} {Suffrage:Voter ID:server-2203e000-8725-5b50-6c7a-c3fb1439af7c Address:2203e000-8725-5b50-6c7a-c3fb1439af7c} {Suffrage:Voter ID:server-dff00595-e9b6-d60d-1c22-74dad11551d7 Address:dff00595-e9b6-d60d-1c22-74dad11551d7}]" +2025-01-25T14:38:56.417Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:38:56.417Z [INFO] server-dff00595-e9b6-d60d-1c22-74dad11551d7: entering follower state: follower="Node at dff00595-e9b6-d60d-1c22-74dad11551d7 [Follower]" leader-address= leader-id= +2025-01-25T14:38:56.477Z [WARN] server-5aee53fc-e526-1170-a03d-5e16a4963bb4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:38:56.477Z [INFO] server-5aee53fc-e526-1170-a03d-5e16a4963bb4: entering candidate state: node="Node at 5aee53fc-e526-1170-a03d-5e16a4963bb4 [Candidate]" term=2 +2025-01-25T14:38:56.478Z [INFO] server-5aee53fc-e526-1170-a03d-5e16a4963bb4: election won: term=2 tally=2 +2025-01-25T14:38:56.478Z [INFO] server-5aee53fc-e526-1170-a03d-5e16a4963bb4: entering leader state: leader="Node at 5aee53fc-e526-1170-a03d-5e16a4963bb4 [Leader]" +2025-01-25T14:38:56.478Z [INFO] server-5aee53fc-e526-1170-a03d-5e16a4963bb4: added peer, starting replication: peer=server-2203e000-8725-5b50-6c7a-c3fb1439af7c +2025-01-25T14:38:56.478Z [INFO] server-5aee53fc-e526-1170-a03d-5e16a4963bb4: added peer, starting replication: peer=server-dff00595-e9b6-d60d-1c22-74dad11551d7 +2025-01-25T14:38:56.479Z [INFO] server-5aee53fc-e526-1170-a03d-5e16a4963bb4: pipelining replication: peer="{Voter server-2203e000-8725-5b50-6c7a-c3fb1439af7c 2203e000-8725-5b50-6c7a-c3fb1439af7c}" +2025-01-25T14:38:56.479Z [INFO] server-5aee53fc-e526-1170-a03d-5e16a4963bb4: pipelining replication: peer="{Voter server-dff00595-e9b6-d60d-1c22-74dad11551d7 dff00595-e9b6-d60d-1c22-74dad11551d7}" +2025-01-25T14:38:56.584Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:38:56.478540851 +0000 UTC m=+5.524380169 (1 nodes), 61.24147ms from start of poll, 64.150562ms from cluster start. Timeout at 2025-01-25 14:38:56.583571476 +0000 UTC m=+5.629410794, 105.030625ms after stability +2025-01-25T14:38:56.585Z [INFO] server-5aee53fc-e526-1170-a03d-5e16a4963bb4: aborting pipeline replication: peer="{Voter server-dff00595-e9b6-d60d-1c22-74dad11551d7 dff00595-e9b6-d60d-1c22-74dad11551d7}" +2025-01-25T14:38:56.585Z [INFO] server-5aee53fc-e526-1170-a03d-5e16a4963bb4: aborting pipeline replication: peer="{Voter server-2203e000-8725-5b50-6c7a-c3fb1439af7c 2203e000-8725-5b50-6c7a-c3fb1439af7c}" +2025-01-25T14:38:56.586Z [INFO] creating new snapshot: path=/tmp/raft2905586913/snapshots/2-2-1737815936586.tmp +2025-01-25T14:38:56.591Z [INFO] server-5aee53fc-e526-1170-a03d-5e16a4963bb4: starting restore from snapshot: id=2-2-1737815936586 last-index=2 last-term=2 size-in-bytes=1 +2025-01-25T14:38:56.593Z [INFO] server-5aee53fc-e526-1170-a03d-5e16a4963bb4: snapshot restore progress: id=2-2-1737815936586 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% +2025-01-25T14:38:56.593Z [INFO] server-5aee53fc-e526-1170-a03d-5e16a4963bb4: restored from snapshot: id=2-2-1737815936586 last-index=2 last-term=2 size-in-bytes=1 +2025-01-25T14:38:56.593Z [INFO] server-5aee53fc-e526-1170-a03d-5e16a4963bb4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5aee53fc-e526-1170-a03d-5e16a4963bb4 Address:5aee53fc-e526-1170-a03d-5e16a4963bb4} {Suffrage:Voter ID:server-2203e000-8725-5b50-6c7a-c3fb1439af7c Address:2203e000-8725-5b50-6c7a-c3fb1439af7c} {Suffrage:Voter ID:server-dff00595-e9b6-d60d-1c22-74dad11551d7 Address:dff00595-e9b6-d60d-1c22-74dad11551d7}]" +2025-01-25T14:38:56.593Z [INFO] server-5aee53fc-e526-1170-a03d-5e16a4963bb4: entering follower state: follower="Node at 5aee53fc-e526-1170-a03d-5e16a4963bb4 [Follower]" leader-address= leader-id= +2025-01-25T14:38:56.594Z [INFO] creating new snapshot: path=/tmp/raft1387624059/snapshots/2-2-1737815936594.tmp +2025-01-25T14:38:56.599Z [INFO] server-2203e000-8725-5b50-6c7a-c3fb1439af7c: starting restore from snapshot: id=2-2-1737815936594 last-index=2 last-term=2 size-in-bytes=1 +2025-01-25T14:38:56.600Z [INFO] server-2203e000-8725-5b50-6c7a-c3fb1439af7c: snapshot restore progress: id=2-2-1737815936594 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% +2025-01-25T14:38:56.600Z [INFO] server-2203e000-8725-5b50-6c7a-c3fb1439af7c: restored from snapshot: id=2-2-1737815936594 last-index=2 last-term=2 size-in-bytes=1 +2025-01-25T14:38:56.600Z [INFO] server-2203e000-8725-5b50-6c7a-c3fb1439af7c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5aee53fc-e526-1170-a03d-5e16a4963bb4 Address:5aee53fc-e526-1170-a03d-5e16a4963bb4} {Suffrage:Voter ID:server-2203e000-8725-5b50-6c7a-c3fb1439af7c Address:2203e000-8725-5b50-6c7a-c3fb1439af7c} {Suffrage:Voter ID:server-dff00595-e9b6-d60d-1c22-74dad11551d7 Address:dff00595-e9b6-d60d-1c22-74dad11551d7}]" +2025-01-25T14:38:56.601Z [INFO] server-2203e000-8725-5b50-6c7a-c3fb1439af7c: entering follower state: follower="Node at 2203e000-8725-5b50-6c7a-c3fb1439af7c [Follower]" leader-address= leader-id= +2025-01-25T14:38:56.601Z [INFO] creating new snapshot: path=/tmp/raft1648158186/snapshots/2-2-1737815936601.tmp +2025-01-25T14:38:56.606Z [INFO] server-dff00595-e9b6-d60d-1c22-74dad11551d7: starting restore from snapshot: id=2-2-1737815936601 last-index=2 last-term=2 size-in-bytes=1 +2025-01-25T14:38:56.607Z [INFO] server-dff00595-e9b6-d60d-1c22-74dad11551d7: snapshot restore progress: id=2-2-1737815936601 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% +2025-01-25T14:38:56.608Z [INFO] server-dff00595-e9b6-d60d-1c22-74dad11551d7: restored from snapshot: id=2-2-1737815936601 last-index=2 last-term=2 size-in-bytes=1 +2025-01-25T14:38:56.608Z [INFO] server-dff00595-e9b6-d60d-1c22-74dad11551d7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5aee53fc-e526-1170-a03d-5e16a4963bb4 Address:5aee53fc-e526-1170-a03d-5e16a4963bb4} {Suffrage:Voter ID:server-2203e000-8725-5b50-6c7a-c3fb1439af7c Address:2203e000-8725-5b50-6c7a-c3fb1439af7c} {Suffrage:Voter ID:server-dff00595-e9b6-d60d-1c22-74dad11551d7 Address:dff00595-e9b6-d60d-1c22-74dad11551d7}]" +2025-01-25T14:38:56.608Z [INFO] server-dff00595-e9b6-d60d-1c22-74dad11551d7: entering follower state: follower="Node at dff00595-e9b6-d60d-1c22-74dad11551d7 [Follower]" leader-address= leader-id= +2025-01-25T14:38:56.662Z [WARN] server-2203e000-8725-5b50-6c7a-c3fb1439af7c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:38:56.662Z [INFO] server-2203e000-8725-5b50-6c7a-c3fb1439af7c: entering candidate state: node="Node at 2203e000-8725-5b50-6c7a-c3fb1439af7c [Candidate]" term=3 +2025-01-25T14:38:56.663Z [INFO] server-2203e000-8725-5b50-6c7a-c3fb1439af7c: election won: term=3 tally=2 +2025-01-25T14:38:56.663Z [INFO] server-2203e000-8725-5b50-6c7a-c3fb1439af7c: entering leader state: leader="Node at 2203e000-8725-5b50-6c7a-c3fb1439af7c [Leader]" +2025-01-25T14:38:56.663Z [INFO] server-2203e000-8725-5b50-6c7a-c3fb1439af7c: added peer, starting replication: peer=server-5aee53fc-e526-1170-a03d-5e16a4963bb4 +2025-01-25T14:38:56.663Z [INFO] server-2203e000-8725-5b50-6c7a-c3fb1439af7c: added peer, starting replication: peer=server-dff00595-e9b6-d60d-1c22-74dad11551d7 +2025-01-25T14:38:56.664Z [INFO] server-2203e000-8725-5b50-6c7a-c3fb1439af7c: pipelining replication: peer="{Voter server-5aee53fc-e526-1170-a03d-5e16a4963bb4 5aee53fc-e526-1170-a03d-5e16a4963bb4}" +2025-01-25T14:38:56.664Z [INFO] server-2203e000-8725-5b50-6c7a-c3fb1439af7c: pipelining replication: peer="{Voter server-dff00595-e9b6-d60d-1c22-74dad11551d7 dff00595-e9b6-d60d-1c22-74dad11551d7}" +2025-01-25T14:38:56.923Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:38:57.030Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:38:56.924410691 +0000 UTC m=+5.970250008 (1 nodes), 182.292µs from start of poll, 510.020401ms from cluster start. Timeout at 2025-01-25 14:38:57.029473691 +0000 UTC m=+6.075313300, 105.063292ms after stability +2025-01-25T14:38:57.030Z [INFO] server-2203e000-8725-5b50-6c7a-c3fb1439af7c: aborting pipeline replication: peer="{Voter server-dff00595-e9b6-d60d-1c22-74dad11551d7 dff00595-e9b6-d60d-1c22-74dad11551d7}" +2025-01-25T14:38:57.030Z [INFO] server-2203e000-8725-5b50-6c7a-c3fb1439af7c: aborting pipeline replication: peer="{Voter server-5aee53fc-e526-1170-a03d-5e16a4963bb4 5aee53fc-e526-1170-a03d-5e16a4963bb4}" === RUN TestRaft_RecoverCluster/no_snapshot,_some_trailing_logs -2025-01-25T14:28:15.437Z [INFO] server-117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3 Address:117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3} {Suffrage:Voter ID:server-1b894bae-bd68-2120-0a73-49314a23d461 Address:1b894bae-bd68-2120-0a73-49314a23d461} {Suffrage:Voter ID:server-17c065a1-4cd4-c1ee-1880-5d926ce79e57 Address:17c065a1-4cd4-c1ee-1880-5d926ce79e57}]" -2025-01-25T14:28:15.438Z [INFO] server-117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3: entering follower state: follower="Node at 117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3 [Follower]" leader-address= leader-id= -2025-01-25T14:28:15.438Z [INFO] server-1b894bae-bd68-2120-0a73-49314a23d461: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3 Address:117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3} {Suffrage:Voter ID:server-1b894bae-bd68-2120-0a73-49314a23d461 Address:1b894bae-bd68-2120-0a73-49314a23d461} {Suffrage:Voter ID:server-17c065a1-4cd4-c1ee-1880-5d926ce79e57 Address:17c065a1-4cd4-c1ee-1880-5d926ce79e57}]" -2025-01-25T14:28:15.438Z [INFO] server-1b894bae-bd68-2120-0a73-49314a23d461: entering follower state: follower="Node at 1b894bae-bd68-2120-0a73-49314a23d461 [Follower]" leader-address= leader-id= -2025-01-25T14:28:15.438Z [INFO] server-17c065a1-4cd4-c1ee-1880-5d926ce79e57: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3 Address:117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3} {Suffrage:Voter ID:server-1b894bae-bd68-2120-0a73-49314a23d461 Address:1b894bae-bd68-2120-0a73-49314a23d461} {Suffrage:Voter ID:server-17c065a1-4cd4-c1ee-1880-5d926ce79e57 Address:17c065a1-4cd4-c1ee-1880-5d926ce79e57}]" -2025-01-25T14:28:15.438Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:15.438Z [INFO] server-17c065a1-4cd4-c1ee-1880-5d926ce79e57: entering follower state: follower="Node at 17c065a1-4cd4-c1ee-1880-5d926ce79e57 [Follower]" leader-address= leader-id= -2025-01-25T14:28:15.503Z [WARN] server-1b894bae-bd68-2120-0a73-49314a23d461: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:15.503Z [INFO] server-1b894bae-bd68-2120-0a73-49314a23d461: entering candidate state: node="Node at 1b894bae-bd68-2120-0a73-49314a23d461 [Candidate]" term=2 -2025-01-25T14:28:15.503Z [INFO] server-1b894bae-bd68-2120-0a73-49314a23d461: election won: term=2 tally=2 -2025-01-25T14:28:15.503Z [INFO] server-1b894bae-bd68-2120-0a73-49314a23d461: entering leader state: leader="Node at 1b894bae-bd68-2120-0a73-49314a23d461 [Leader]" -2025-01-25T14:28:15.504Z [INFO] server-1b894bae-bd68-2120-0a73-49314a23d461: added peer, starting replication: peer=server-117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3 -2025-01-25T14:28:15.504Z [INFO] server-1b894bae-bd68-2120-0a73-49314a23d461: added peer, starting replication: peer=server-17c065a1-4cd4-c1ee-1880-5d926ce79e57 -2025-01-25T14:28:15.504Z [WARN] server-117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:15.504Z [INFO] server-117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3: entering candidate state: node="Node at 117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3 [Candidate]" term=2 -2025-01-25T14:28:15.504Z [INFO] server-117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3: duplicate requestVote for same term: term=2 -2025-01-25T14:28:15.504Z [WARN] server-1b894bae-bd68-2120-0a73-49314a23d461: rejecting vote request since we have a leader: from=117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3 leader=1b894bae-bd68-2120-0a73-49314a23d461 leader-id=server-1b894bae-bd68-2120-0a73-49314a23d461 -2025-01-25T14:28:15.504Z [INFO] server-117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3: entering follower state: follower="Node at 117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3 [Follower]" leader-address=1b894bae-bd68-2120-0a73-49314a23d461 leader-id=server-1b894bae-bd68-2120-0a73-49314a23d461 -2025-01-25T14:28:15.504Z [INFO] server-1b894bae-bd68-2120-0a73-49314a23d461: pipelining replication: peer="{Voter server-17c065a1-4cd4-c1ee-1880-5d926ce79e57 17c065a1-4cd4-c1ee-1880-5d926ce79e57}" -2025-01-25T14:28:15.504Z [WARN] server-17c065a1-4cd4-c1ee-1880-5d926ce79e57: rejecting vote request since we have a leader: from=117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3 leader=1b894bae-bd68-2120-0a73-49314a23d461 leader-id=server-1b894bae-bd68-2120-0a73-49314a23d461 -2025-01-25T14:28:15.504Z [INFO] server-1b894bae-bd68-2120-0a73-49314a23d461: pipelining replication: peer="{Voter server-117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3 117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3}" -2025-01-25T14:28:15.610Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:15.504386912 +0000 UTC m=+5.358711894 (1 nodes), 66.018625ms from start of poll, 66.662242ms from cluster start. Timeout at 2025-01-25 14:28:15.609402152 +0000 UTC m=+5.463727134, 105.01524ms after stability -2025-01-25T14:28:15.611Z [INFO] server-1b894bae-bd68-2120-0a73-49314a23d461: aborting pipeline replication: peer="{Voter server-17c065a1-4cd4-c1ee-1880-5d926ce79e57 17c065a1-4cd4-c1ee-1880-5d926ce79e57}" -2025-01-25T14:28:15.611Z [INFO] server-1b894bae-bd68-2120-0a73-49314a23d461: aborting pipeline replication: peer="{Voter server-117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3 117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3}" -2025-01-25T14:28:15.611Z [INFO] creating new snapshot: path=/tmp/raft1685918046/snapshots/2-6-1737815295611.tmp -2025-01-25T14:28:15.613Z [INFO] server-117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3: starting restore from snapshot: id=2-6-1737815295611 last-index=6 last-term=2 size-in-bytes=25 -2025-01-25T14:28:15.613Z [INFO] server-117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3: snapshot restore progress: id=2-6-1737815295611 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% -2025-01-25T14:28:15.613Z [INFO] server-117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3: restored from snapshot: id=2-6-1737815295611 last-index=6 last-term=2 size-in-bytes=25 -2025-01-25T14:28:15.613Z [INFO] server-117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3 Address:117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3} {Suffrage:Voter ID:server-1b894bae-bd68-2120-0a73-49314a23d461 Address:1b894bae-bd68-2120-0a73-49314a23d461} {Suffrage:Voter ID:server-17c065a1-4cd4-c1ee-1880-5d926ce79e57 Address:17c065a1-4cd4-c1ee-1880-5d926ce79e57}]" -2025-01-25T14:28:15.614Z [INFO] server-117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3: entering follower state: follower="Node at 117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3 [Follower]" leader-address= leader-id= -2025-01-25T14:28:15.614Z [INFO] creating new snapshot: path=/tmp/raft1303537295/snapshots/2-6-1737815295614.tmp -2025-01-25T14:28:15.615Z [INFO] server-1b894bae-bd68-2120-0a73-49314a23d461: starting restore from snapshot: id=2-6-1737815295614 last-index=6 last-term=2 size-in-bytes=25 -2025-01-25T14:28:15.615Z [INFO] server-1b894bae-bd68-2120-0a73-49314a23d461: snapshot restore progress: id=2-6-1737815295614 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% -2025-01-25T14:28:15.615Z [INFO] server-1b894bae-bd68-2120-0a73-49314a23d461: restored from snapshot: id=2-6-1737815295614 last-index=6 last-term=2 size-in-bytes=25 -2025-01-25T14:28:15.615Z [INFO] server-1b894bae-bd68-2120-0a73-49314a23d461: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3 Address:117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3} {Suffrage:Voter ID:server-1b894bae-bd68-2120-0a73-49314a23d461 Address:1b894bae-bd68-2120-0a73-49314a23d461} {Suffrage:Voter ID:server-17c065a1-4cd4-c1ee-1880-5d926ce79e57 Address:17c065a1-4cd4-c1ee-1880-5d926ce79e57}]" -2025-01-25T14:28:15.615Z [INFO] server-1b894bae-bd68-2120-0a73-49314a23d461: entering follower state: follower="Node at 1b894bae-bd68-2120-0a73-49314a23d461 [Follower]" leader-address= leader-id= -2025-01-25T14:28:15.615Z [INFO] creating new snapshot: path=/tmp/raft2329099858/snapshots/2-6-1737815295615.tmp -2025-01-25T14:28:15.616Z [INFO] server-17c065a1-4cd4-c1ee-1880-5d926ce79e57: starting restore from snapshot: id=2-6-1737815295615 last-index=6 last-term=2 size-in-bytes=25 -2025-01-25T14:28:15.617Z [INFO] server-17c065a1-4cd4-c1ee-1880-5d926ce79e57: snapshot restore progress: id=2-6-1737815295615 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% -2025-01-25T14:28:15.617Z [INFO] server-17c065a1-4cd4-c1ee-1880-5d926ce79e57: restored from snapshot: id=2-6-1737815295615 last-index=6 last-term=2 size-in-bytes=25 -2025-01-25T14:28:15.617Z [INFO] server-17c065a1-4cd4-c1ee-1880-5d926ce79e57: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3 Address:117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3} {Suffrage:Voter ID:server-1b894bae-bd68-2120-0a73-49314a23d461 Address:1b894bae-bd68-2120-0a73-49314a23d461} {Suffrage:Voter ID:server-17c065a1-4cd4-c1ee-1880-5d926ce79e57 Address:17c065a1-4cd4-c1ee-1880-5d926ce79e57}]" -2025-01-25T14:28:15.617Z [INFO] server-17c065a1-4cd4-c1ee-1880-5d926ce79e57: entering follower state: follower="Node at 17c065a1-4cd4-c1ee-1880-5d926ce79e57 [Follower]" leader-address= leader-id= -2025-01-25T14:28:15.699Z [WARN] server-17c065a1-4cd4-c1ee-1880-5d926ce79e57: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:15.699Z [INFO] server-17c065a1-4cd4-c1ee-1880-5d926ce79e57: entering candidate state: node="Node at 17c065a1-4cd4-c1ee-1880-5d926ce79e57 [Candidate]" term=3 -2025-01-25T14:28:15.699Z [INFO] server-17c065a1-4cd4-c1ee-1880-5d926ce79e57: election won: term=3 tally=2 -2025-01-25T14:28:15.699Z [INFO] server-17c065a1-4cd4-c1ee-1880-5d926ce79e57: entering leader state: leader="Node at 17c065a1-4cd4-c1ee-1880-5d926ce79e57 [Leader]" -2025-01-25T14:28:15.699Z [INFO] server-17c065a1-4cd4-c1ee-1880-5d926ce79e57: added peer, starting replication: peer=server-117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3 -2025-01-25T14:28:15.699Z [INFO] server-17c065a1-4cd4-c1ee-1880-5d926ce79e57: added peer, starting replication: peer=server-1b894bae-bd68-2120-0a73-49314a23d461 -2025-01-25T14:28:15.699Z [INFO] server-17c065a1-4cd4-c1ee-1880-5d926ce79e57: pipelining replication: peer="{Voter server-1b894bae-bd68-2120-0a73-49314a23d461 1b894bae-bd68-2120-0a73-49314a23d461}" -2025-01-25T14:28:15.699Z [INFO] server-17c065a1-4cd4-c1ee-1880-5d926ce79e57: pipelining replication: peer="{Voter server-117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3 117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3}" -2025-01-25T14:28:15.932Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:16.038Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:15.933077446 +0000 UTC m=+5.787402388 (1 nodes), 69.758µs from start of poll, 495.352736ms from cluster start. Timeout at 2025-01-25 14:28:16.038105525 +0000 UTC m=+5.892430507, 105.028119ms after stability -2025-01-25T14:28:16.038Z [INFO] server-17c065a1-4cd4-c1ee-1880-5d926ce79e57: aborting pipeline replication: peer="{Voter server-1b894bae-bd68-2120-0a73-49314a23d461 1b894bae-bd68-2120-0a73-49314a23d461}" -2025-01-25T14:28:16.038Z [INFO] server-17c065a1-4cd4-c1ee-1880-5d926ce79e57: aborting pipeline replication: peer="{Voter server-117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3 117a8b7d-8b5c-1c6d-285f-1f6f74a0bbb3}" +2025-01-25T14:38:57.041Z [INFO] server-56cb941a-0ed8-e08b-8473-9d714008d547: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-56cb941a-0ed8-e08b-8473-9d714008d547 Address:56cb941a-0ed8-e08b-8473-9d714008d547} {Suffrage:Voter ID:server-6496bf14-6428-76d1-e795-31ea0d5dbce8 Address:6496bf14-6428-76d1-e795-31ea0d5dbce8} {Suffrage:Voter ID:server-1cbef813-d685-1e1c-f734-6d1a363e6796 Address:1cbef813-d685-1e1c-f734-6d1a363e6796}]" +2025-01-25T14:38:57.041Z [INFO] server-56cb941a-0ed8-e08b-8473-9d714008d547: entering follower state: follower="Node at 56cb941a-0ed8-e08b-8473-9d714008d547 [Follower]" leader-address= leader-id= +2025-01-25T14:38:57.042Z [INFO] server-6496bf14-6428-76d1-e795-31ea0d5dbce8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-56cb941a-0ed8-e08b-8473-9d714008d547 Address:56cb941a-0ed8-e08b-8473-9d714008d547} {Suffrage:Voter ID:server-6496bf14-6428-76d1-e795-31ea0d5dbce8 Address:6496bf14-6428-76d1-e795-31ea0d5dbce8} {Suffrage:Voter ID:server-1cbef813-d685-1e1c-f734-6d1a363e6796 Address:1cbef813-d685-1e1c-f734-6d1a363e6796}]" +2025-01-25T14:38:57.042Z [INFO] server-6496bf14-6428-76d1-e795-31ea0d5dbce8: entering follower state: follower="Node at 6496bf14-6428-76d1-e795-31ea0d5dbce8 [Follower]" leader-address= leader-id= +2025-01-25T14:38:57.043Z [INFO] server-1cbef813-d685-1e1c-f734-6d1a363e6796: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-56cb941a-0ed8-e08b-8473-9d714008d547 Address:56cb941a-0ed8-e08b-8473-9d714008d547} {Suffrage:Voter ID:server-6496bf14-6428-76d1-e795-31ea0d5dbce8 Address:6496bf14-6428-76d1-e795-31ea0d5dbce8} {Suffrage:Voter ID:server-1cbef813-d685-1e1c-f734-6d1a363e6796 Address:1cbef813-d685-1e1c-f734-6d1a363e6796}]" +2025-01-25T14:38:57.043Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:38:57.043Z [INFO] server-1cbef813-d685-1e1c-f734-6d1a363e6796: entering follower state: follower="Node at 1cbef813-d685-1e1c-f734-6d1a363e6796 [Follower]" leader-address= leader-id= +2025-01-25T14:38:57.101Z [WARN] server-1cbef813-d685-1e1c-f734-6d1a363e6796: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:38:57.101Z [INFO] server-1cbef813-d685-1e1c-f734-6d1a363e6796: entering candidate state: node="Node at 1cbef813-d685-1e1c-f734-6d1a363e6796 [Candidate]" term=2 +2025-01-25T14:38:57.102Z [INFO] server-1cbef813-d685-1e1c-f734-6d1a363e6796: election won: term=2 tally=2 +2025-01-25T14:38:57.102Z [INFO] server-1cbef813-d685-1e1c-f734-6d1a363e6796: entering leader state: leader="Node at 1cbef813-d685-1e1c-f734-6d1a363e6796 [Leader]" +2025-01-25T14:38:57.102Z [INFO] server-1cbef813-d685-1e1c-f734-6d1a363e6796: added peer, starting replication: peer=server-56cb941a-0ed8-e08b-8473-9d714008d547 +2025-01-25T14:38:57.102Z [INFO] server-1cbef813-d685-1e1c-f734-6d1a363e6796: added peer, starting replication: peer=server-6496bf14-6428-76d1-e795-31ea0d5dbce8 +2025-01-25T14:38:57.103Z [INFO] server-1cbef813-d685-1e1c-f734-6d1a363e6796: pipelining replication: peer="{Voter server-56cb941a-0ed8-e08b-8473-9d714008d547 56cb941a-0ed8-e08b-8473-9d714008d547}" +2025-01-25T14:38:57.103Z [INFO] server-1cbef813-d685-1e1c-f734-6d1a363e6796: pipelining replication: peer="{Voter server-6496bf14-6428-76d1-e795-31ea0d5dbce8 6496bf14-6428-76d1-e795-31ea0d5dbce8}" +2025-01-25T14:38:57.208Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:38:57.102247917 +0000 UTC m=+6.148087235 (1 nodes), 58.485171ms from start of poll, 61.806389ms from cluster start. Timeout at 2025-01-25 14:38:57.207278543 +0000 UTC m=+6.253117860, 105.030625ms after stability +2025-01-25T14:38:57.211Z [INFO] server-1cbef813-d685-1e1c-f734-6d1a363e6796: aborting pipeline replication: peer="{Voter server-56cb941a-0ed8-e08b-8473-9d714008d547 56cb941a-0ed8-e08b-8473-9d714008d547}" +2025-01-25T14:38:57.211Z [INFO] server-1cbef813-d685-1e1c-f734-6d1a363e6796: aborting pipeline replication: peer="{Voter server-6496bf14-6428-76d1-e795-31ea0d5dbce8 6496bf14-6428-76d1-e795-31ea0d5dbce8}" +2025-01-25T14:38:57.213Z [INFO] creating new snapshot: path=/tmp/raft2622305668/snapshots/2-6-1737815937213.tmp +2025-01-25T14:38:57.218Z [INFO] server-56cb941a-0ed8-e08b-8473-9d714008d547: starting restore from snapshot: id=2-6-1737815937213 last-index=6 last-term=2 size-in-bytes=25 +2025-01-25T14:38:57.219Z [INFO] server-56cb941a-0ed8-e08b-8473-9d714008d547: snapshot restore progress: id=2-6-1737815937213 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% +2025-01-25T14:38:57.220Z [INFO] server-56cb941a-0ed8-e08b-8473-9d714008d547: restored from snapshot: id=2-6-1737815937213 last-index=6 last-term=2 size-in-bytes=25 +2025-01-25T14:38:57.220Z [INFO] server-56cb941a-0ed8-e08b-8473-9d714008d547: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-56cb941a-0ed8-e08b-8473-9d714008d547 Address:56cb941a-0ed8-e08b-8473-9d714008d547} {Suffrage:Voter ID:server-6496bf14-6428-76d1-e795-31ea0d5dbce8 Address:6496bf14-6428-76d1-e795-31ea0d5dbce8} {Suffrage:Voter ID:server-1cbef813-d685-1e1c-f734-6d1a363e6796 Address:1cbef813-d685-1e1c-f734-6d1a363e6796}]" +2025-01-25T14:38:57.220Z [INFO] server-56cb941a-0ed8-e08b-8473-9d714008d547: entering follower state: follower="Node at 56cb941a-0ed8-e08b-8473-9d714008d547 [Follower]" leader-address= leader-id= +2025-01-25T14:38:57.220Z [INFO] creating new snapshot: path=/tmp/raft469013126/snapshots/2-6-1737815937220.tmp +2025-01-25T14:38:57.225Z [INFO] server-6496bf14-6428-76d1-e795-31ea0d5dbce8: starting restore from snapshot: id=2-6-1737815937220 last-index=6 last-term=2 size-in-bytes=25 +2025-01-25T14:38:57.226Z [INFO] server-6496bf14-6428-76d1-e795-31ea0d5dbce8: snapshot restore progress: id=2-6-1737815937220 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% +2025-01-25T14:38:57.226Z [INFO] server-6496bf14-6428-76d1-e795-31ea0d5dbce8: restored from snapshot: id=2-6-1737815937220 last-index=6 last-term=2 size-in-bytes=25 +2025-01-25T14:38:57.226Z [INFO] server-6496bf14-6428-76d1-e795-31ea0d5dbce8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-56cb941a-0ed8-e08b-8473-9d714008d547 Address:56cb941a-0ed8-e08b-8473-9d714008d547} {Suffrage:Voter ID:server-6496bf14-6428-76d1-e795-31ea0d5dbce8 Address:6496bf14-6428-76d1-e795-31ea0d5dbce8} {Suffrage:Voter ID:server-1cbef813-d685-1e1c-f734-6d1a363e6796 Address:1cbef813-d685-1e1c-f734-6d1a363e6796}]" +2025-01-25T14:38:57.226Z [INFO] server-6496bf14-6428-76d1-e795-31ea0d5dbce8: entering follower state: follower="Node at 6496bf14-6428-76d1-e795-31ea0d5dbce8 [Follower]" leader-address= leader-id= +2025-01-25T14:38:57.227Z [INFO] creating new snapshot: path=/tmp/raft3395270853/snapshots/2-6-1737815937227.tmp +2025-01-25T14:38:57.231Z [INFO] server-1cbef813-d685-1e1c-f734-6d1a363e6796: starting restore from snapshot: id=2-6-1737815937227 last-index=6 last-term=2 size-in-bytes=25 +2025-01-25T14:38:57.232Z [INFO] server-1cbef813-d685-1e1c-f734-6d1a363e6796: snapshot restore progress: id=2-6-1737815937227 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% +2025-01-25T14:38:57.233Z [INFO] server-1cbef813-d685-1e1c-f734-6d1a363e6796: restored from snapshot: id=2-6-1737815937227 last-index=6 last-term=2 size-in-bytes=25 +2025-01-25T14:38:57.233Z [INFO] server-1cbef813-d685-1e1c-f734-6d1a363e6796: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-56cb941a-0ed8-e08b-8473-9d714008d547 Address:56cb941a-0ed8-e08b-8473-9d714008d547} {Suffrage:Voter ID:server-6496bf14-6428-76d1-e795-31ea0d5dbce8 Address:6496bf14-6428-76d1-e795-31ea0d5dbce8} {Suffrage:Voter ID:server-1cbef813-d685-1e1c-f734-6d1a363e6796 Address:1cbef813-d685-1e1c-f734-6d1a363e6796}]" +2025-01-25T14:38:57.233Z [INFO] server-1cbef813-d685-1e1c-f734-6d1a363e6796: entering follower state: follower="Node at 1cbef813-d685-1e1c-f734-6d1a363e6796 [Follower]" leader-address= leader-id= +2025-01-25T14:38:57.275Z [WARN] server-56cb941a-0ed8-e08b-8473-9d714008d547: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:38:57.275Z [INFO] server-56cb941a-0ed8-e08b-8473-9d714008d547: entering candidate state: node="Node at 56cb941a-0ed8-e08b-8473-9d714008d547 [Candidate]" term=3 +2025-01-25T14:38:57.276Z [INFO] server-56cb941a-0ed8-e08b-8473-9d714008d547: election won: term=3 tally=2 +2025-01-25T14:38:57.276Z [INFO] server-56cb941a-0ed8-e08b-8473-9d714008d547: entering leader state: leader="Node at 56cb941a-0ed8-e08b-8473-9d714008d547 [Leader]" +2025-01-25T14:38:57.276Z [INFO] server-56cb941a-0ed8-e08b-8473-9d714008d547: added peer, starting replication: peer=server-6496bf14-6428-76d1-e795-31ea0d5dbce8 +2025-01-25T14:38:57.276Z [INFO] server-56cb941a-0ed8-e08b-8473-9d714008d547: added peer, starting replication: peer=server-1cbef813-d685-1e1c-f734-6d1a363e6796 +2025-01-25T14:38:57.277Z [INFO] server-56cb941a-0ed8-e08b-8473-9d714008d547: pipelining replication: peer="{Voter server-6496bf14-6428-76d1-e795-31ea0d5dbce8 6496bf14-6428-76d1-e795-31ea0d5dbce8}" +2025-01-25T14:38:57.278Z [INFO] server-56cb941a-0ed8-e08b-8473-9d714008d547: pipelining replication: peer="{Voter server-1cbef813-d685-1e1c-f734-6d1a363e6796 1cbef813-d685-1e1c-f734-6d1a363e6796}" +2025-01-25T14:38:57.549Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:38:57.655Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:38:57.54965422 +0000 UTC m=+6.595493537 (1 nodes), 203µs from start of poll, 509.212691ms from cluster start. Timeout at 2025-01-25 14:38:57.654769428 +0000 UTC m=+6.700608746, 105.115209ms after stability +2025-01-25T14:38:57.655Z [INFO] server-56cb941a-0ed8-e08b-8473-9d714008d547: aborting pipeline replication: peer="{Voter server-1cbef813-d685-1e1c-f734-6d1a363e6796 1cbef813-d685-1e1c-f734-6d1a363e6796}" +2025-01-25T14:38:57.655Z [INFO] server-56cb941a-0ed8-e08b-8473-9d714008d547: aborting pipeline replication: peer="{Voter server-6496bf14-6428-76d1-e795-31ea0d5dbce8 6496bf14-6428-76d1-e795-31ea0d5dbce8}" === RUN TestRaft_RecoverCluster/snapshot,_with_trailing_logs -2025-01-25T14:28:16.041Z [INFO] server-da29ad0a-06b9-b157-b461-4e63f2707990: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-da29ad0a-06b9-b157-b461-4e63f2707990 Address:da29ad0a-06b9-b157-b461-4e63f2707990} {Suffrage:Voter ID:server-24cd48bf-6e20-3d67-dc4a-e767e1e7dd30 Address:24cd48bf-6e20-3d67-dc4a-e767e1e7dd30} {Suffrage:Voter ID:server-1480d5ab-4df9-17e1-303b-8c4a9ce584a4 Address:1480d5ab-4df9-17e1-303b-8c4a9ce584a4}]" -2025-01-25T14:28:16.042Z [INFO] server-24cd48bf-6e20-3d67-dc4a-e767e1e7dd30: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-da29ad0a-06b9-b157-b461-4e63f2707990 Address:da29ad0a-06b9-b157-b461-4e63f2707990} {Suffrage:Voter ID:server-24cd48bf-6e20-3d67-dc4a-e767e1e7dd30 Address:24cd48bf-6e20-3d67-dc4a-e767e1e7dd30} {Suffrage:Voter ID:server-1480d5ab-4df9-17e1-303b-8c4a9ce584a4 Address:1480d5ab-4df9-17e1-303b-8c4a9ce584a4}]" -2025-01-25T14:28:16.042Z [INFO] server-da29ad0a-06b9-b157-b461-4e63f2707990: entering follower state: follower="Node at da29ad0a-06b9-b157-b461-4e63f2707990 [Follower]" leader-address= leader-id= -2025-01-25T14:28:16.042Z [INFO] server-24cd48bf-6e20-3d67-dc4a-e767e1e7dd30: entering follower state: follower="Node at 24cd48bf-6e20-3d67-dc4a-e767e1e7dd30 [Follower]" leader-address= leader-id= -2025-01-25T14:28:16.042Z [INFO] server-1480d5ab-4df9-17e1-303b-8c4a9ce584a4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-da29ad0a-06b9-b157-b461-4e63f2707990 Address:da29ad0a-06b9-b157-b461-4e63f2707990} {Suffrage:Voter ID:server-24cd48bf-6e20-3d67-dc4a-e767e1e7dd30 Address:24cd48bf-6e20-3d67-dc4a-e767e1e7dd30} {Suffrage:Voter ID:server-1480d5ab-4df9-17e1-303b-8c4a9ce584a4 Address:1480d5ab-4df9-17e1-303b-8c4a9ce584a4}]" -2025-01-25T14:28:16.042Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:16.042Z [INFO] server-1480d5ab-4df9-17e1-303b-8c4a9ce584a4: entering follower state: follower="Node at 1480d5ab-4df9-17e1-303b-8c4a9ce584a4 [Follower]" leader-address= leader-id= -2025-01-25T14:28:16.107Z [WARN] server-da29ad0a-06b9-b157-b461-4e63f2707990: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:16.107Z [INFO] server-da29ad0a-06b9-b157-b461-4e63f2707990: entering candidate state: node="Node at da29ad0a-06b9-b157-b461-4e63f2707990 [Candidate]" term=2 -2025-01-25T14:28:16.107Z [INFO] server-da29ad0a-06b9-b157-b461-4e63f2707990: election won: term=2 tally=2 -2025-01-25T14:28:16.107Z [INFO] server-da29ad0a-06b9-b157-b461-4e63f2707990: entering leader state: leader="Node at da29ad0a-06b9-b157-b461-4e63f2707990 [Leader]" -2025-01-25T14:28:16.107Z [INFO] server-da29ad0a-06b9-b157-b461-4e63f2707990: added peer, starting replication: peer=server-24cd48bf-6e20-3d67-dc4a-e767e1e7dd30 -2025-01-25T14:28:16.107Z [INFO] server-da29ad0a-06b9-b157-b461-4e63f2707990: added peer, starting replication: peer=server-1480d5ab-4df9-17e1-303b-8c4a9ce584a4 -2025-01-25T14:28:16.107Z [INFO] server-da29ad0a-06b9-b157-b461-4e63f2707990: pipelining replication: peer="{Voter server-24cd48bf-6e20-3d67-dc4a-e767e1e7dd30 24cd48bf-6e20-3d67-dc4a-e767e1e7dd30}" -2025-01-25T14:28:16.107Z [INFO] server-da29ad0a-06b9-b157-b461-4e63f2707990: pipelining replication: peer="{Voter server-1480d5ab-4df9-17e1-303b-8c4a9ce584a4 1480d5ab-4df9-17e1-303b-8c4a9ce584a4}" -2025-01-25T14:28:16.213Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:16.107409573 +0000 UTC m=+5.961734515 (1 nodes), 65.141536ms from start of poll, 65.783073ms from cluster start. Timeout at 2025-01-25 14:28:16.212411652 +0000 UTC m=+6.066736594, 105.002079ms after stability -2025-01-25T14:28:16.219Z [INFO] server-da29ad0a-06b9-b157-b461-4e63f2707990: aborting pipeline replication: peer="{Voter server-24cd48bf-6e20-3d67-dc4a-e767e1e7dd30 24cd48bf-6e20-3d67-dc4a-e767e1e7dd30}" -2025-01-25T14:28:16.219Z [INFO] server-da29ad0a-06b9-b157-b461-4e63f2707990: aborting pipeline replication: peer="{Voter server-1480d5ab-4df9-17e1-303b-8c4a9ce584a4 1480d5ab-4df9-17e1-303b-8c4a9ce584a4}" -2025-01-25T14:28:16.219Z [INFO] creating new snapshot: path=/tmp/raft807531108/snapshots/2-27-1737815296219.tmp -2025-01-25T14:28:16.221Z [INFO] server-da29ad0a-06b9-b157-b461-4e63f2707990: starting restore from snapshot: id=2-27-1737815296219 last-index=27 last-term=2 size-in-bytes=168 -2025-01-25T14:28:16.221Z [INFO] server-da29ad0a-06b9-b157-b461-4e63f2707990: snapshot restore progress: id=2-27-1737815296219 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% -2025-01-25T14:28:16.221Z [INFO] server-da29ad0a-06b9-b157-b461-4e63f2707990: restored from snapshot: id=2-27-1737815296219 last-index=27 last-term=2 size-in-bytes=168 -2025-01-25T14:28:16.221Z [INFO] server-da29ad0a-06b9-b157-b461-4e63f2707990: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-da29ad0a-06b9-b157-b461-4e63f2707990 Address:da29ad0a-06b9-b157-b461-4e63f2707990} {Suffrage:Voter ID:server-24cd48bf-6e20-3d67-dc4a-e767e1e7dd30 Address:24cd48bf-6e20-3d67-dc4a-e767e1e7dd30} {Suffrage:Voter ID:server-1480d5ab-4df9-17e1-303b-8c4a9ce584a4 Address:1480d5ab-4df9-17e1-303b-8c4a9ce584a4}]" -2025-01-25T14:28:16.221Z [INFO] creating new snapshot: path=/tmp/raft3444269178/snapshots/2-27-1737815296221.tmp -2025-01-25T14:28:16.221Z [INFO] server-da29ad0a-06b9-b157-b461-4e63f2707990: entering follower state: follower="Node at da29ad0a-06b9-b157-b461-4e63f2707990 [Follower]" leader-address= leader-id= -2025-01-25T14:28:16.222Z [INFO] server-24cd48bf-6e20-3d67-dc4a-e767e1e7dd30: starting restore from snapshot: id=2-27-1737815296221 last-index=27 last-term=2 size-in-bytes=168 -2025-01-25T14:28:16.223Z [INFO] server-24cd48bf-6e20-3d67-dc4a-e767e1e7dd30: snapshot restore progress: id=2-27-1737815296221 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% -2025-01-25T14:28:16.223Z [INFO] server-24cd48bf-6e20-3d67-dc4a-e767e1e7dd30: restored from snapshot: id=2-27-1737815296221 last-index=27 last-term=2 size-in-bytes=168 -2025-01-25T14:28:16.223Z [INFO] server-24cd48bf-6e20-3d67-dc4a-e767e1e7dd30: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-da29ad0a-06b9-b157-b461-4e63f2707990 Address:da29ad0a-06b9-b157-b461-4e63f2707990} {Suffrage:Voter ID:server-24cd48bf-6e20-3d67-dc4a-e767e1e7dd30 Address:24cd48bf-6e20-3d67-dc4a-e767e1e7dd30} {Suffrage:Voter ID:server-1480d5ab-4df9-17e1-303b-8c4a9ce584a4 Address:1480d5ab-4df9-17e1-303b-8c4a9ce584a4}]" -2025-01-25T14:28:16.223Z [INFO] server-24cd48bf-6e20-3d67-dc4a-e767e1e7dd30: entering follower state: follower="Node at 24cd48bf-6e20-3d67-dc4a-e767e1e7dd30 [Follower]" leader-address= leader-id= -2025-01-25T14:28:16.223Z [INFO] creating new snapshot: path=/tmp/raft4161443190/snapshots/2-27-1737815296223.tmp -2025-01-25T14:28:16.224Z [INFO] server-1480d5ab-4df9-17e1-303b-8c4a9ce584a4: starting restore from snapshot: id=2-27-1737815296223 last-index=27 last-term=2 size-in-bytes=168 -2025-01-25T14:28:16.224Z [INFO] server-1480d5ab-4df9-17e1-303b-8c4a9ce584a4: snapshot restore progress: id=2-27-1737815296223 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% -2025-01-25T14:28:16.224Z [INFO] server-1480d5ab-4df9-17e1-303b-8c4a9ce584a4: restored from snapshot: id=2-27-1737815296223 last-index=27 last-term=2 size-in-bytes=168 -2025-01-25T14:28:16.224Z [INFO] server-1480d5ab-4df9-17e1-303b-8c4a9ce584a4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-da29ad0a-06b9-b157-b461-4e63f2707990 Address:da29ad0a-06b9-b157-b461-4e63f2707990} {Suffrage:Voter ID:server-24cd48bf-6e20-3d67-dc4a-e767e1e7dd30 Address:24cd48bf-6e20-3d67-dc4a-e767e1e7dd30} {Suffrage:Voter ID:server-1480d5ab-4df9-17e1-303b-8c4a9ce584a4 Address:1480d5ab-4df9-17e1-303b-8c4a9ce584a4}]" -2025-01-25T14:28:16.224Z [INFO] server-1480d5ab-4df9-17e1-303b-8c4a9ce584a4: entering follower state: follower="Node at 1480d5ab-4df9-17e1-303b-8c4a9ce584a4 [Follower]" leader-address= leader-id= -2025-01-25T14:28:16.295Z [WARN] server-24cd48bf-6e20-3d67-dc4a-e767e1e7dd30: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:16.295Z [INFO] server-24cd48bf-6e20-3d67-dc4a-e767e1e7dd30: entering candidate state: node="Node at 24cd48bf-6e20-3d67-dc4a-e767e1e7dd30 [Candidate]" term=3 -2025-01-25T14:28:16.295Z [INFO] server-24cd48bf-6e20-3d67-dc4a-e767e1e7dd30: election won: term=3 tally=2 -2025-01-25T14:28:16.295Z [INFO] server-24cd48bf-6e20-3d67-dc4a-e767e1e7dd30: entering leader state: leader="Node at 24cd48bf-6e20-3d67-dc4a-e767e1e7dd30 [Leader]" -2025-01-25T14:28:16.295Z [INFO] server-24cd48bf-6e20-3d67-dc4a-e767e1e7dd30: added peer, starting replication: peer=server-da29ad0a-06b9-b157-b461-4e63f2707990 -2025-01-25T14:28:16.295Z [INFO] server-24cd48bf-6e20-3d67-dc4a-e767e1e7dd30: added peer, starting replication: peer=server-1480d5ab-4df9-17e1-303b-8c4a9ce584a4 -2025-01-25T14:28:16.295Z [INFO] server-24cd48bf-6e20-3d67-dc4a-e767e1e7dd30: pipelining replication: peer="{Voter server-1480d5ab-4df9-17e1-303b-8c4a9ce584a4 1480d5ab-4df9-17e1-303b-8c4a9ce584a4}" -2025-01-25T14:28:16.295Z [INFO] server-24cd48bf-6e20-3d67-dc4a-e767e1e7dd30: pipelining replication: peer="{Voter server-da29ad0a-06b9-b157-b461-4e63f2707990 da29ad0a-06b9-b157-b461-4e63f2707990}" -2025-01-25T14:28:16.540Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:16.646Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:16.54044623 +0000 UTC m=+6.394771212 (1 nodes), 59.918µs from start of poll, 498.81977ms from cluster start. Timeout at 2025-01-25 14:28:16.645489149 +0000 UTC m=+6.499814091, 105.042879ms after stability -2025-01-25T14:28:16.646Z [INFO] server-24cd48bf-6e20-3d67-dc4a-e767e1e7dd30: aborting pipeline replication: peer="{Voter server-1480d5ab-4df9-17e1-303b-8c4a9ce584a4 1480d5ab-4df9-17e1-303b-8c4a9ce584a4}" -2025-01-25T14:28:16.646Z [INFO] server-24cd48bf-6e20-3d67-dc4a-e767e1e7dd30: aborting pipeline replication: peer="{Voter server-da29ad0a-06b9-b157-b461-4e63f2707990 da29ad0a-06b9-b157-b461-4e63f2707990}" ---- PASS: TestRaft_RecoverCluster (1.81s) - --- PASS: TestRaft_RecoverCluster/no_snapshot,_no_trailing_logs (0.60s) - --- PASS: TestRaft_RecoverCluster/no_snapshot,_some_trailing_logs (0.60s) - --- PASS: TestRaft_RecoverCluster/snapshot,_with_trailing_logs (0.61s) +2025-01-25T14:38:57.667Z [INFO] server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e Address:0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e} {Suffrage:Voter ID:server-712ceb46-72bf-b51a-ba92-3802c0d1ca78 Address:712ceb46-72bf-b51a-ba92-3802c0d1ca78} {Suffrage:Voter ID:server-c78f82c2-e27e-a5e7-dfa4-6595d6121e68 Address:c78f82c2-e27e-a5e7-dfa4-6595d6121e68}]" +2025-01-25T14:38:57.668Z [INFO] server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e: entering follower state: follower="Node at 0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e [Follower]" leader-address= leader-id= +2025-01-25T14:38:57.669Z [INFO] server-712ceb46-72bf-b51a-ba92-3802c0d1ca78: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e Address:0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e} {Suffrage:Voter ID:server-712ceb46-72bf-b51a-ba92-3802c0d1ca78 Address:712ceb46-72bf-b51a-ba92-3802c0d1ca78} {Suffrage:Voter ID:server-c78f82c2-e27e-a5e7-dfa4-6595d6121e68 Address:c78f82c2-e27e-a5e7-dfa4-6595d6121e68}]" +2025-01-25T14:38:57.669Z [INFO] server-712ceb46-72bf-b51a-ba92-3802c0d1ca78: entering follower state: follower="Node at 712ceb46-72bf-b51a-ba92-3802c0d1ca78 [Follower]" leader-address= leader-id= +2025-01-25T14:38:57.670Z [INFO] server-c78f82c2-e27e-a5e7-dfa4-6595d6121e68: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e Address:0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e} {Suffrage:Voter ID:server-712ceb46-72bf-b51a-ba92-3802c0d1ca78 Address:712ceb46-72bf-b51a-ba92-3802c0d1ca78} {Suffrage:Voter ID:server-c78f82c2-e27e-a5e7-dfa4-6595d6121e68 Address:c78f82c2-e27e-a5e7-dfa4-6595d6121e68}]" +2025-01-25T14:38:57.670Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:38:57.670Z [INFO] server-c78f82c2-e27e-a5e7-dfa4-6595d6121e68: entering follower state: follower="Node at c78f82c2-e27e-a5e7-dfa4-6595d6121e68 [Follower]" leader-address= leader-id= +2025-01-25T14:38:57.754Z [WARN] server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:38:57.755Z [WARN] server-712ceb46-72bf-b51a-ba92-3802c0d1ca78: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:38:57.755Z [INFO] server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e: entering candidate state: node="Node at 0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e [Candidate]" term=2 +2025-01-25T14:38:57.755Z [INFO] server-712ceb46-72bf-b51a-ba92-3802c0d1ca78: entering candidate state: node="Node at 712ceb46-72bf-b51a-ba92-3802c0d1ca78 [Candidate]" term=2 +2025-01-25T14:38:57.755Z [INFO] server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e: duplicate requestVote for same term: term=2 +2025-01-25T14:38:57.755Z [INFO] server-c78f82c2-e27e-a5e7-dfa4-6595d6121e68: duplicate requestVote for same term: term=2 +2025-01-25T14:38:57.755Z [INFO] server-712ceb46-72bf-b51a-ba92-3802c0d1ca78: duplicate requestVote for same term: term=2 +2025-01-25T14:38:57.755Z [INFO] server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e: election won: term=2 tally=2 +2025-01-25T14:38:57.756Z [INFO] server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e: entering leader state: leader="Node at 0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e [Leader]" +2025-01-25T14:38:57.756Z [INFO] server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e: added peer, starting replication: peer=server-712ceb46-72bf-b51a-ba92-3802c0d1ca78 +2025-01-25T14:38:57.756Z [INFO] server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e: added peer, starting replication: peer=server-c78f82c2-e27e-a5e7-dfa4-6595d6121e68 +2025-01-25T14:38:57.763Z [INFO] server-712ceb46-72bf-b51a-ba92-3802c0d1ca78: entering follower state: follower="Node at 712ceb46-72bf-b51a-ba92-3802c0d1ca78 [Follower]" leader-address=0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e leader-id=server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e +2025-01-25T14:38:57.762Z [INFO] server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e: pipelining replication: peer="{Voter server-c78f82c2-e27e-a5e7-dfa4-6595d6121e68 c78f82c2-e27e-a5e7-dfa4-6595d6121e68}" +2025-01-25T14:38:57.764Z [INFO] server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e: pipelining replication: peer="{Voter server-712ceb46-72bf-b51a-ba92-3802c0d1ca78 712ceb46-72bf-b51a-ba92-3802c0d1ca78}" +2025-01-25T14:38:57.870Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:38:57.763986094 +0000 UTC m=+6.809825411 (1 nodes), 93.275855ms from start of poll, 96.783448ms from cluster start. Timeout at 2025-01-25 14:38:57.869012344 +0000 UTC m=+6.914851370, 105.025959ms after stability +2025-01-25T14:38:57.894Z [INFO] server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e: aborting pipeline replication: peer="{Voter server-c78f82c2-e27e-a5e7-dfa4-6595d6121e68 c78f82c2-e27e-a5e7-dfa4-6595d6121e68}" +2025-01-25T14:38:57.894Z [INFO] server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e: aborting pipeline replication: peer="{Voter server-712ceb46-72bf-b51a-ba92-3802c0d1ca78 712ceb46-72bf-b51a-ba92-3802c0d1ca78}" +2025-01-25T14:38:57.895Z [INFO] creating new snapshot: path=/tmp/raft4128753388/snapshots/2-27-1737815937895.tmp +2025-01-25T14:38:57.900Z [INFO] server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e: starting restore from snapshot: id=2-27-1737815937895 last-index=27 last-term=2 size-in-bytes=168 +2025-01-25T14:38:57.901Z [INFO] server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e: snapshot restore progress: id=2-27-1737815937895 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% +2025-01-25T14:38:57.901Z [INFO] server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e: restored from snapshot: id=2-27-1737815937895 last-index=27 last-term=2 size-in-bytes=168 +2025-01-25T14:38:57.901Z [INFO] server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e Address:0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e} {Suffrage:Voter ID:server-712ceb46-72bf-b51a-ba92-3802c0d1ca78 Address:712ceb46-72bf-b51a-ba92-3802c0d1ca78} {Suffrage:Voter ID:server-c78f82c2-e27e-a5e7-dfa4-6595d6121e68 Address:c78f82c2-e27e-a5e7-dfa4-6595d6121e68}]" +2025-01-25T14:38:57.902Z [INFO] server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e: entering follower state: follower="Node at 0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e [Follower]" leader-address= leader-id= +2025-01-25T14:38:57.902Z [INFO] creating new snapshot: path=/tmp/raft3793361956/snapshots/2-27-1737815937902.tmp +2025-01-25T14:38:57.907Z [INFO] server-712ceb46-72bf-b51a-ba92-3802c0d1ca78: starting restore from snapshot: id=2-27-1737815937902 last-index=27 last-term=2 size-in-bytes=168 +2025-01-25T14:38:57.908Z [INFO] server-712ceb46-72bf-b51a-ba92-3802c0d1ca78: snapshot restore progress: id=2-27-1737815937902 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% +2025-01-25T14:38:57.909Z [INFO] server-712ceb46-72bf-b51a-ba92-3802c0d1ca78: restored from snapshot: id=2-27-1737815937902 last-index=27 last-term=2 size-in-bytes=168 +2025-01-25T14:38:57.909Z [INFO] server-712ceb46-72bf-b51a-ba92-3802c0d1ca78: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e Address:0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e} {Suffrage:Voter ID:server-712ceb46-72bf-b51a-ba92-3802c0d1ca78 Address:712ceb46-72bf-b51a-ba92-3802c0d1ca78} {Suffrage:Voter ID:server-c78f82c2-e27e-a5e7-dfa4-6595d6121e68 Address:c78f82c2-e27e-a5e7-dfa4-6595d6121e68}]" +2025-01-25T14:38:57.909Z [INFO] server-712ceb46-72bf-b51a-ba92-3802c0d1ca78: entering follower state: follower="Node at 712ceb46-72bf-b51a-ba92-3802c0d1ca78 [Follower]" leader-address= leader-id= +2025-01-25T14:38:57.909Z [INFO] creating new snapshot: path=/tmp/raft1185916075/snapshots/2-27-1737815937909.tmp +2025-01-25T14:38:57.914Z [INFO] server-c78f82c2-e27e-a5e7-dfa4-6595d6121e68: starting restore from snapshot: id=2-27-1737815937909 last-index=27 last-term=2 size-in-bytes=168 +2025-01-25T14:38:57.915Z [INFO] server-c78f82c2-e27e-a5e7-dfa4-6595d6121e68: snapshot restore progress: id=2-27-1737815937909 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% +2025-01-25T14:38:57.915Z [INFO] server-c78f82c2-e27e-a5e7-dfa4-6595d6121e68: restored from snapshot: id=2-27-1737815937909 last-index=27 last-term=2 size-in-bytes=168 +2025-01-25T14:38:57.915Z [INFO] server-c78f82c2-e27e-a5e7-dfa4-6595d6121e68: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e Address:0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e} {Suffrage:Voter ID:server-712ceb46-72bf-b51a-ba92-3802c0d1ca78 Address:712ceb46-72bf-b51a-ba92-3802c0d1ca78} {Suffrage:Voter ID:server-c78f82c2-e27e-a5e7-dfa4-6595d6121e68 Address:c78f82c2-e27e-a5e7-dfa4-6595d6121e68}]" +2025-01-25T14:38:57.916Z [INFO] server-c78f82c2-e27e-a5e7-dfa4-6595d6121e68: entering follower state: follower="Node at c78f82c2-e27e-a5e7-dfa4-6595d6121e68 [Follower]" leader-address= leader-id= +2025-01-25T14:38:57.960Z [WARN] server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:38:57.961Z [INFO] server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e: entering candidate state: node="Node at 0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e [Candidate]" term=3 +2025-01-25T14:38:57.961Z [INFO] server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e: election won: term=3 tally=2 +2025-01-25T14:38:57.961Z [INFO] server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e: entering leader state: leader="Node at 0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e [Leader]" +2025-01-25T14:38:57.961Z [INFO] server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e: added peer, starting replication: peer=server-712ceb46-72bf-b51a-ba92-3802c0d1ca78 +2025-01-25T14:38:57.961Z [INFO] server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e: added peer, starting replication: peer=server-c78f82c2-e27e-a5e7-dfa4-6595d6121e68 +2025-01-25T14:38:57.962Z [INFO] server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e: pipelining replication: peer="{Voter server-712ceb46-72bf-b51a-ba92-3802c0d1ca78 712ceb46-72bf-b51a-ba92-3802c0d1ca78}" +2025-01-25T14:38:57.962Z [INFO] server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e: pipelining replication: peer="{Voter server-c78f82c2-e27e-a5e7-dfa4-6595d6121e68 c78f82c2-e27e-a5e7-dfa4-6595d6121e68}" +2025-01-25T14:38:58.232Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:38:58.338Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:38:58.232736625 +0000 UTC m=+7.278575943 (1 nodes), 234.501µs from start of poll, 565.53398ms from cluster start. Timeout at 2025-01-25 14:38:58.337816249 +0000 UTC m=+7.383655567, 105.079624ms after stability +2025-01-25T14:38:58.338Z [INFO] server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e: aborting pipeline replication: peer="{Voter server-c78f82c2-e27e-a5e7-dfa4-6595d6121e68 c78f82c2-e27e-a5e7-dfa4-6595d6121e68}" +2025-01-25T14:38:58.338Z [INFO] server-0a1fbb61-a781-e6e7-9b5b-38dcc28e6c1e: aborting pipeline replication: peer="{Voter server-712ceb46-72bf-b51a-ba92-3802c0d1ca78 712ceb46-72bf-b51a-ba92-3802c0d1ca78}" +--- PASS: TestRaft_RecoverCluster (1.93s) + --- PASS: TestRaft_RecoverCluster/no_snapshot,_no_trailing_logs (0.63s) + --- PASS: TestRaft_RecoverCluster/no_snapshot,_some_trailing_logs (0.63s) + --- PASS: TestRaft_RecoverCluster/snapshot,_with_trailing_logs (0.68s) === RUN TestRaft_HasExistingState raft_test.go:258: Skipping unreliable test --- SKIP: TestRaft_HasExistingState (0.00s) === RUN TestRaft_SingleNode -2025-01-25T14:28:16.649Z [INFO] server-a10f318f-3368-47b9-ecfd-e399dbb3480f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a10f318f-3368-47b9-ecfd-e399dbb3480f Address:a10f318f-3368-47b9-ecfd-e399dbb3480f}]" -2025-01-25T14:28:16.649Z [INFO] server-a10f318f-3368-47b9-ecfd-e399dbb3480f: entering follower state: follower="Node at a10f318f-3368-47b9-ecfd-e399dbb3480f [Follower]" leader-address= leader-id= -2025-01-25T14:28:16.737Z [WARN] server-a10f318f-3368-47b9-ecfd-e399dbb3480f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:16.737Z [INFO] server-a10f318f-3368-47b9-ecfd-e399dbb3480f: entering candidate state: node="Node at a10f318f-3368-47b9-ecfd-e399dbb3480f [Candidate]" term=2 -2025-01-25T14:28:16.737Z [INFO] server-a10f318f-3368-47b9-ecfd-e399dbb3480f: election won: term=2 tally=1 -2025-01-25T14:28:16.737Z [INFO] server-a10f318f-3368-47b9-ecfd-e399dbb3480f: entering leader state: leader="Node at a10f318f-3368-47b9-ecfd-e399dbb3480f [Leader]" ---- PASS: TestRaft_SingleNode (0.09s) +2025-01-25T14:38:58.347Z [INFO] server-003c33a6-305a-fe62-6ba3-0b4395c8a466: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-003c33a6-305a-fe62-6ba3-0b4395c8a466 Address:003c33a6-305a-fe62-6ba3-0b4395c8a466}]" +2025-01-25T14:38:58.347Z [INFO] server-003c33a6-305a-fe62-6ba3-0b4395c8a466: entering follower state: follower="Node at 003c33a6-305a-fe62-6ba3-0b4395c8a466 [Follower]" leader-address= leader-id= +2025-01-25T14:38:58.404Z [WARN] server-003c33a6-305a-fe62-6ba3-0b4395c8a466: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:38:58.404Z [INFO] server-003c33a6-305a-fe62-6ba3-0b4395c8a466: entering candidate state: node="Node at 003c33a6-305a-fe62-6ba3-0b4395c8a466 [Candidate]" term=2 +2025-01-25T14:38:58.405Z [INFO] server-003c33a6-305a-fe62-6ba3-0b4395c8a466: election won: term=2 tally=1 +2025-01-25T14:38:58.405Z [INFO] server-003c33a6-305a-fe62-6ba3-0b4395c8a466: entering leader state: leader="Node at 003c33a6-305a-fe62-6ba3-0b4395c8a466 [Leader]" +--- PASS: TestRaft_SingleNode (0.06s) === RUN TestRaft_TripleNode -2025-01-25T14:28:16.740Z [INFO] server-c57e67b9-345e-2411-7165-ca2cbc39ed65: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c57e67b9-345e-2411-7165-ca2cbc39ed65 Address:c57e67b9-345e-2411-7165-ca2cbc39ed65} {Suffrage:Voter ID:server-66814d4c-919b-4586-6941-30a133915fd8 Address:66814d4c-919b-4586-6941-30a133915fd8} {Suffrage:Voter ID:server-54f9a64c-2f91-e6ab-d9a7-d8732d9a7868 Address:54f9a64c-2f91-e6ab-d9a7-d8732d9a7868}]" -2025-01-25T14:28:16.740Z [INFO] server-66814d4c-919b-4586-6941-30a133915fd8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c57e67b9-345e-2411-7165-ca2cbc39ed65 Address:c57e67b9-345e-2411-7165-ca2cbc39ed65} {Suffrage:Voter ID:server-66814d4c-919b-4586-6941-30a133915fd8 Address:66814d4c-919b-4586-6941-30a133915fd8} {Suffrage:Voter ID:server-54f9a64c-2f91-e6ab-d9a7-d8732d9a7868 Address:54f9a64c-2f91-e6ab-d9a7-d8732d9a7868}]" -2025-01-25T14:28:16.740Z [INFO] server-c57e67b9-345e-2411-7165-ca2cbc39ed65: entering follower state: follower="Node at c57e67b9-345e-2411-7165-ca2cbc39ed65 [Follower]" leader-address= leader-id= -2025-01-25T14:28:16.740Z [INFO] server-66814d4c-919b-4586-6941-30a133915fd8: entering follower state: follower="Node at 66814d4c-919b-4586-6941-30a133915fd8 [Follower]" leader-address= leader-id= -2025-01-25T14:28:16.740Z [INFO] server-54f9a64c-2f91-e6ab-d9a7-d8732d9a7868: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c57e67b9-345e-2411-7165-ca2cbc39ed65 Address:c57e67b9-345e-2411-7165-ca2cbc39ed65} {Suffrage:Voter ID:server-66814d4c-919b-4586-6941-30a133915fd8 Address:66814d4c-919b-4586-6941-30a133915fd8} {Suffrage:Voter ID:server-54f9a64c-2f91-e6ab-d9a7-d8732d9a7868 Address:54f9a64c-2f91-e6ab-d9a7-d8732d9a7868}]" -2025-01-25T14:28:16.740Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:28:16.740Z [INFO] server-54f9a64c-2f91-e6ab-d9a7-d8732d9a7868: entering follower state: follower="Node at 54f9a64c-2f91-e6ab-d9a7-d8732d9a7868 [Follower]" leader-address= leader-id= -2025-01-25T14:28:16.807Z [WARN] server-54f9a64c-2f91-e6ab-d9a7-d8732d9a7868: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:16.807Z [INFO] server-54f9a64c-2f91-e6ab-d9a7-d8732d9a7868: entering candidate state: node="Node at 54f9a64c-2f91-e6ab-d9a7-d8732d9a7868 [Candidate]" term=2 -2025-01-25T14:28:16.807Z [INFO] server-54f9a64c-2f91-e6ab-d9a7-d8732d9a7868: election won: term=2 tally=2 -2025-01-25T14:28:16.807Z [INFO] server-54f9a64c-2f91-e6ab-d9a7-d8732d9a7868: entering leader state: leader="Node at 54f9a64c-2f91-e6ab-d9a7-d8732d9a7868 [Leader]" -2025-01-25T14:28:16.807Z [INFO] server-54f9a64c-2f91-e6ab-d9a7-d8732d9a7868: added peer, starting replication: peer=server-c57e67b9-345e-2411-7165-ca2cbc39ed65 -2025-01-25T14:28:16.807Z [INFO] server-54f9a64c-2f91-e6ab-d9a7-d8732d9a7868: added peer, starting replication: peer=server-66814d4c-919b-4586-6941-30a133915fd8 -2025-01-25T14:28:16.807Z [INFO] server-54f9a64c-2f91-e6ab-d9a7-d8732d9a7868: pipelining replication: peer="{Voter server-66814d4c-919b-4586-6941-30a133915fd8 66814d4c-919b-4586-6941-30a133915fd8}" -2025-01-25T14:28:16.808Z [INFO] server-54f9a64c-2f91-e6ab-d9a7-d8732d9a7868: pipelining replication: peer="{Voter server-c57e67b9-345e-2411-7165-ca2cbc39ed65 c57e67b9-345e-2411-7165-ca2cbc39ed65}" -2025-01-25T14:28:16.913Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:28:16.808130398 +0000 UTC m=+6.662455340 (2 nodes), 67.592808ms from start of poll, 68.245225ms from cluster start. Timeout at 2025-01-25 14:28:16.913141438 +0000 UTC m=+6.767466420, 105.01108ms after stability -2025-01-25T14:28:16.913Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:17.019Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:16.913979909 +0000 UTC m=+6.768304851 (1 nodes), 2.32µs from start of poll, 174.094736ms from cluster start. Timeout at 2025-01-25 14:28:17.018991349 +0000 UTC m=+6.873316331, 105.01148ms after stability -2025-01-25T14:28:17.030Z [INFO] server-54f9a64c-2f91-e6ab-d9a7-d8732d9a7868: aborting pipeline replication: peer="{Voter server-66814d4c-919b-4586-6941-30a133915fd8 66814d4c-919b-4586-6941-30a133915fd8}" -2025-01-25T14:28:17.030Z [INFO] server-54f9a64c-2f91-e6ab-d9a7-d8732d9a7868: aborting pipeline replication: peer="{Voter server-c57e67b9-345e-2411-7165-ca2cbc39ed65 c57e67b9-345e-2411-7165-ca2cbc39ed65}" ---- PASS: TestRaft_TripleNode (0.29s) +2025-01-25T14:38:58.410Z [INFO] server-106b8867-22de-3719-f3ba-978b048d3eb7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-106b8867-22de-3719-f3ba-978b048d3eb7 Address:106b8867-22de-3719-f3ba-978b048d3eb7} {Suffrage:Voter ID:server-6a5a73c6-8173-abf6-a747-85499ce9de73 Address:6a5a73c6-8173-abf6-a747-85499ce9de73} {Suffrage:Voter ID:server-a53ff053-a6a3-b706-bf96-0e042d36f6f4 Address:a53ff053-a6a3-b706-bf96-0e042d36f6f4}]" +2025-01-25T14:38:58.410Z [INFO] server-106b8867-22de-3719-f3ba-978b048d3eb7: entering follower state: follower="Node at 106b8867-22de-3719-f3ba-978b048d3eb7 [Follower]" leader-address= leader-id= +2025-01-25T14:38:58.410Z [INFO] server-6a5a73c6-8173-abf6-a747-85499ce9de73: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-106b8867-22de-3719-f3ba-978b048d3eb7 Address:106b8867-22de-3719-f3ba-978b048d3eb7} {Suffrage:Voter ID:server-6a5a73c6-8173-abf6-a747-85499ce9de73 Address:6a5a73c6-8173-abf6-a747-85499ce9de73} {Suffrage:Voter ID:server-a53ff053-a6a3-b706-bf96-0e042d36f6f4 Address:a53ff053-a6a3-b706-bf96-0e042d36f6f4}]" +2025-01-25T14:38:58.410Z [INFO] server-6a5a73c6-8173-abf6-a747-85499ce9de73: entering follower state: follower="Node at 6a5a73c6-8173-abf6-a747-85499ce9de73 [Follower]" leader-address= leader-id= +2025-01-25T14:38:58.411Z [INFO] server-a53ff053-a6a3-b706-bf96-0e042d36f6f4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-106b8867-22de-3719-f3ba-978b048d3eb7 Address:106b8867-22de-3719-f3ba-978b048d3eb7} {Suffrage:Voter ID:server-6a5a73c6-8173-abf6-a747-85499ce9de73 Address:6a5a73c6-8173-abf6-a747-85499ce9de73} {Suffrage:Voter ID:server-a53ff053-a6a3-b706-bf96-0e042d36f6f4 Address:a53ff053-a6a3-b706-bf96-0e042d36f6f4}]" +2025-01-25T14:38:58.411Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:38:58.411Z [INFO] server-a53ff053-a6a3-b706-bf96-0e042d36f6f4: entering follower state: follower="Node at a53ff053-a6a3-b706-bf96-0e042d36f6f4 [Follower]" leader-address= leader-id= +2025-01-25T14:38:58.483Z [WARN] server-106b8867-22de-3719-f3ba-978b048d3eb7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:38:58.484Z [INFO] server-106b8867-22de-3719-f3ba-978b048d3eb7: entering candidate state: node="Node at 106b8867-22de-3719-f3ba-978b048d3eb7 [Candidate]" term=2 +2025-01-25T14:38:58.484Z [INFO] server-106b8867-22de-3719-f3ba-978b048d3eb7: election won: term=2 tally=2 +2025-01-25T14:38:58.484Z [INFO] server-106b8867-22de-3719-f3ba-978b048d3eb7: entering leader state: leader="Node at 106b8867-22de-3719-f3ba-978b048d3eb7 [Leader]" +2025-01-25T14:38:58.484Z [INFO] server-106b8867-22de-3719-f3ba-978b048d3eb7: added peer, starting replication: peer=server-6a5a73c6-8173-abf6-a747-85499ce9de73 +2025-01-25T14:38:58.484Z [INFO] server-106b8867-22de-3719-f3ba-978b048d3eb7: added peer, starting replication: peer=server-a53ff053-a6a3-b706-bf96-0e042d36f6f4 +2025-01-25T14:38:58.485Z [INFO] server-106b8867-22de-3719-f3ba-978b048d3eb7: pipelining replication: peer="{Voter server-a53ff053-a6a3-b706-bf96-0e042d36f6f4 a53ff053-a6a3-b706-bf96-0e042d36f6f4}" +2025-01-25T14:38:58.485Z [INFO] server-106b8867-22de-3719-f3ba-978b048d3eb7: pipelining replication: peer="{Voter server-6a5a73c6-8173-abf6-a747-85499ce9de73 6a5a73c6-8173-abf6-a747-85499ce9de73}" +2025-01-25T14:38:58.590Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:38:58.484746817 +0000 UTC m=+7.530586135 (2 nodes), 73.237547ms from start of poll, 74.920177ms from cluster start. Timeout at 2025-01-25 14:38:58.589762859 +0000 UTC m=+7.635602176, 105.016041ms after stability +2025-01-25T14:38:58.590Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:38:58.695Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:38:58.590479625 +0000 UTC m=+7.636318943 (1 nodes), 6.125µs from start of poll, 180.652985ms from cluster start. Timeout at 2025-01-25 14:38:58.695527459 +0000 UTC m=+7.741366776, 105.047833ms after stability +2025-01-25T14:38:58.708Z [INFO] server-106b8867-22de-3719-f3ba-978b048d3eb7: aborting pipeline replication: peer="{Voter server-a53ff053-a6a3-b706-bf96-0e042d36f6f4 a53ff053-a6a3-b706-bf96-0e042d36f6f4}" +2025-01-25T14:38:58.708Z [INFO] server-106b8867-22de-3719-f3ba-978b048d3eb7: aborting pipeline replication: peer="{Voter server-6a5a73c6-8173-abf6-a747-85499ce9de73 6a5a73c6-8173-abf6-a747-85499ce9de73}" +--- PASS: TestRaft_TripleNode (0.30s) === RUN TestRaft_LeaderFail -2025-01-25T14:28:17.032Z [INFO] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 Address:4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041} {Suffrage:Voter ID:server-8967c740-002b-c919-8b54-d6f78ad1c486 Address:8967c740-002b-c919-8b54-d6f78ad1c486} {Suffrage:Voter ID:server-e4cec951-a9ef-47d9-077c-6c2670d776c0 Address:e4cec951-a9ef-47d9-077c-6c2670d776c0}]" -2025-01-25T14:28:17.033Z [INFO] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: entering follower state: follower="Node at 4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 [Follower]" leader-address= leader-id= -2025-01-25T14:28:17.033Z [INFO] server-8967c740-002b-c919-8b54-d6f78ad1c486: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 Address:4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041} {Suffrage:Voter ID:server-8967c740-002b-c919-8b54-d6f78ad1c486 Address:8967c740-002b-c919-8b54-d6f78ad1c486} {Suffrage:Voter ID:server-e4cec951-a9ef-47d9-077c-6c2670d776c0 Address:e4cec951-a9ef-47d9-077c-6c2670d776c0}]" -2025-01-25T14:28:17.033Z [INFO] server-8967c740-002b-c919-8b54-d6f78ad1c486: entering follower state: follower="Node at 8967c740-002b-c919-8b54-d6f78ad1c486 [Follower]" leader-address= leader-id= -2025-01-25T14:28:17.033Z [INFO] server-e4cec951-a9ef-47d9-077c-6c2670d776c0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 Address:4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041} {Suffrage:Voter ID:server-8967c740-002b-c919-8b54-d6f78ad1c486 Address:8967c740-002b-c919-8b54-d6f78ad1c486} {Suffrage:Voter ID:server-e4cec951-a9ef-47d9-077c-6c2670d776c0 Address:e4cec951-a9ef-47d9-077c-6c2670d776c0}]" -2025-01-25T14:28:17.033Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:28:17.033Z [INFO] server-e4cec951-a9ef-47d9-077c-6c2670d776c0: entering follower state: follower="Node at e4cec951-a9ef-47d9-077c-6c2670d776c0 [Follower]" leader-address= leader-id= -2025-01-25T14:28:17.085Z [WARN] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:17.085Z [INFO] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: entering candidate state: node="Node at 4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 [Candidate]" term=2 -2025-01-25T14:28:17.085Z [INFO] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: election won: term=2 tally=2 -2025-01-25T14:28:17.085Z [INFO] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: entering leader state: leader="Node at 4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 [Leader]" -2025-01-25T14:28:17.085Z [INFO] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: added peer, starting replication: peer=server-8967c740-002b-c919-8b54-d6f78ad1c486 -2025-01-25T14:28:17.085Z [INFO] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: added peer, starting replication: peer=server-e4cec951-a9ef-47d9-077c-6c2670d776c0 -2025-01-25T14:28:17.085Z [INFO] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: pipelining replication: peer="{Voter server-e4cec951-a9ef-47d9-077c-6c2670d776c0 e4cec951-a9ef-47d9-077c-6c2670d776c0}" -2025-01-25T14:28:17.086Z [INFO] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: pipelining replication: peer="{Voter server-8967c740-002b-c919-8b54-d6f78ad1c486 8967c740-002b-c919-8b54-d6f78ad1c486}" -2025-01-25T14:28:17.191Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:28:17.086133834 +0000 UTC m=+6.940458816 (2 nodes), 52.814179ms from start of poll, 53.436317ms from cluster start. Timeout at 2025-01-25 14:28:17.191143314 +0000 UTC m=+7.045468296, 105.00948ms after stability -2025-01-25T14:28:17.191Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:17.297Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:17.192015864 +0000 UTC m=+7.046340846 (1 nodes), 2.36µs from start of poll, 159.318347ms from cluster start. Timeout at 2025-01-25 14:28:17.297047703 +0000 UTC m=+7.151372725, 105.031879ms after stability - raft_test.go:378: [INFO] Disconnecting Node at 4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 [Leader] -2025-01-25T14:28:17.312Z [ERROR] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: failed to heartbeat to: peer=e4cec951-a9ef-47d9-077c-6c2670d776c0 backoff time=10ms error="failed to connect to peer: e4cec951-a9ef-47d9-077c-6c2670d776c0" -2025-01-25T14:28:17.312Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:17.312Z [ERROR] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: failed to pipeline appendEntries: peer="{Voter server-8967c740-002b-c919-8b54-d6f78ad1c486 8967c740-002b-c919-8b54-d6f78ad1c486}" error="append pipeline closed" -2025-01-25T14:28:17.313Z [INFO] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: aborting pipeline replication: peer="{Voter server-8967c740-002b-c919-8b54-d6f78ad1c486 8967c740-002b-c919-8b54-d6f78ad1c486}" -2025-01-25T14:28:17.315Z [ERROR] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: failed to pipeline appendEntries: peer="{Voter server-e4cec951-a9ef-47d9-077c-6c2670d776c0 e4cec951-a9ef-47d9-077c-6c2670d776c0}" error="append pipeline closed" -2025-01-25T14:28:17.315Z [INFO] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: aborting pipeline replication: peer="{Voter server-e4cec951-a9ef-47d9-077c-6c2670d776c0 e4cec951-a9ef-47d9-077c-6c2670d776c0}" -2025-01-25T14:28:17.315Z [ERROR] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: failed to heartbeat to: peer=8967c740-002b-c919-8b54-d6f78ad1c486 backoff time=10ms error="failed to connect to peer: 8967c740-002b-c919-8b54-d6f78ad1c486" -2025-01-25T14:28:17.321Z [ERROR] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: failed to appendEntries to: peer="{Voter server-8967c740-002b-c919-8b54-d6f78ad1c486 8967c740-002b-c919-8b54-d6f78ad1c486}" error="failed to connect to peer: 8967c740-002b-c919-8b54-d6f78ad1c486" -2025-01-25T14:28:17.326Z [ERROR] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: failed to appendEntries to: peer="{Voter server-e4cec951-a9ef-47d9-077c-6c2670d776c0 e4cec951-a9ef-47d9-077c-6c2670d776c0}" error="failed to connect to peer: e4cec951-a9ef-47d9-077c-6c2670d776c0" -2025-01-25T14:28:17.333Z [ERROR] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: failed to heartbeat to: peer=e4cec951-a9ef-47d9-077c-6c2670d776c0 backoff time=10ms error="failed to connect to peer: e4cec951-a9ef-47d9-077c-6c2670d776c0" -2025-01-25T14:28:17.335Z [ERROR] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: failed to heartbeat to: peer=8967c740-002b-c919-8b54-d6f78ad1c486 backoff time=10ms error="failed to connect to peer: 8967c740-002b-c919-8b54-d6f78ad1c486" -2025-01-25T14:28:17.339Z [ERROR] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: failed to appendEntries to: peer="{Voter server-8967c740-002b-c919-8b54-d6f78ad1c486 8967c740-002b-c919-8b54-d6f78ad1c486}" error="failed to connect to peer: 8967c740-002b-c919-8b54-d6f78ad1c486" -2025-01-25T14:28:17.344Z [ERROR] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: failed to appendEntries to: peer="{Voter server-e4cec951-a9ef-47d9-077c-6c2670d776c0 e4cec951-a9ef-47d9-077c-6c2670d776c0}" error="failed to connect to peer: e4cec951-a9ef-47d9-077c-6c2670d776c0" -2025-01-25T14:28:17.351Z [ERROR] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: failed to heartbeat to: peer=e4cec951-a9ef-47d9-077c-6c2670d776c0 backoff time=10ms error="failed to connect to peer: e4cec951-a9ef-47d9-077c-6c2670d776c0" -2025-01-25T14:28:17.354Z [ERROR] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: failed to heartbeat to: peer=8967c740-002b-c919-8b54-d6f78ad1c486 backoff time=10ms error="failed to connect to peer: 8967c740-002b-c919-8b54-d6f78ad1c486" -2025-01-25T14:28:17.356Z [WARN] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: failed to contact: server-id=server-8967c740-002b-c919-8b54-d6f78ad1c486 time=50.108197ms -2025-01-25T14:28:17.357Z [ERROR] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: failed to appendEntries to: peer="{Voter server-8967c740-002b-c919-8b54-d6f78ad1c486 8967c740-002b-c919-8b54-d6f78ad1c486}" error="failed to connect to peer: 8967c740-002b-c919-8b54-d6f78ad1c486" -2025-01-25T14:28:17.363Z [ERROR] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: failed to appendEntries to: peer="{Voter server-e4cec951-a9ef-47d9-077c-6c2670d776c0 e4cec951-a9ef-47d9-077c-6c2670d776c0}" error="failed to connect to peer: e4cec951-a9ef-47d9-077c-6c2670d776c0" -2025-01-25T14:28:17.367Z [WARN] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: failed to contact: server-id=server-8967c740-002b-c919-8b54-d6f78ad1c486 time=61.278435ms -2025-01-25T14:28:17.367Z [WARN] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: failed to contact: server-id=server-e4cec951-a9ef-47d9-077c-6c2670d776c0 time=60.084677ms -2025-01-25T14:28:17.367Z [WARN] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: failed to contact quorum of nodes, stepping down -2025-01-25T14:28:17.367Z [INFO] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: entering follower state: follower="Node at 4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 [Follower]" leader-address= leader-id= -2025-01-25T14:28:17.373Z [ERROR] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: failed to heartbeat to: peer=8967c740-002b-c919-8b54-d6f78ad1c486 backoff time=20ms error="failed to connect to peer: 8967c740-002b-c919-8b54-d6f78ad1c486" -2025-01-25T14:28:17.381Z [WARN] server-8967c740-002b-c919-8b54-d6f78ad1c486: heartbeat timeout reached, starting election: last-leader-addr=4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 last-leader-id=server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 -2025-01-25T14:28:17.382Z [INFO] server-8967c740-002b-c919-8b54-d6f78ad1c486: entering candidate state: node="Node at 8967c740-002b-c919-8b54-d6f78ad1c486 [Candidate]" term=3 -2025-01-25T14:28:17.382Z [WARN] server-e4cec951-a9ef-47d9-077c-6c2670d776c0: rejecting vote request since we have a leader: from=8967c740-002b-c919-8b54-d6f78ad1c486 leader=4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 leader-id=server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 -2025-01-25T14:28:17.382Z [ERROR] server-8967c740-002b-c919-8b54-d6f78ad1c486: failed to make requestVote RPC: target="{Voter server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041}" error="failed to connect to peer: 4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041" term=3 -2025-01-25T14:28:17.388Z [WARN] server-e4cec951-a9ef-47d9-077c-6c2670d776c0: heartbeat timeout reached, starting election: last-leader-addr=4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 last-leader-id=server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 -2025-01-25T14:28:17.388Z [INFO] server-e4cec951-a9ef-47d9-077c-6c2670d776c0: entering candidate state: node="Node at e4cec951-a9ef-47d9-077c-6c2670d776c0 [Candidate]" term=3 -2025-01-25T14:28:17.388Z [ERROR] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: failed to appendEntries to: peer="{Voter server-8967c740-002b-c919-8b54-d6f78ad1c486 8967c740-002b-c919-8b54-d6f78ad1c486}" error="failed to connect to peer: 8967c740-002b-c919-8b54-d6f78ad1c486" -2025-01-25T14:28:17.388Z [INFO] server-8967c740-002b-c919-8b54-d6f78ad1c486: duplicate requestVote for same term: term=3 -2025-01-25T14:28:17.388Z [ERROR] server-e4cec951-a9ef-47d9-077c-6c2670d776c0: failed to make requestVote RPC: target="{Voter server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041}" error="failed to connect to peer: 4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041" term=3 -2025-01-25T14:28:17.447Z [WARN] server-8967c740-002b-c919-8b54-d6f78ad1c486: Election timeout reached, restarting election -2025-01-25T14:28:17.447Z [INFO] server-8967c740-002b-c919-8b54-d6f78ad1c486: entering candidate state: node="Node at 8967c740-002b-c919-8b54-d6f78ad1c486 [Candidate]" term=4 -2025-01-25T14:28:17.447Z [INFO] server-e4cec951-a9ef-47d9-077c-6c2670d776c0: entering follower state: follower="Node at e4cec951-a9ef-47d9-077c-6c2670d776c0 [Follower]" leader-address= leader-id= -2025-01-25T14:28:17.447Z [INFO] server-8967c740-002b-c919-8b54-d6f78ad1c486: election won: term=4 tally=2 -2025-01-25T14:28:17.447Z [INFO] server-8967c740-002b-c919-8b54-d6f78ad1c486: entering leader state: leader="Node at 8967c740-002b-c919-8b54-d6f78ad1c486 [Leader]" -2025-01-25T14:28:17.447Z [INFO] server-8967c740-002b-c919-8b54-d6f78ad1c486: added peer, starting replication: peer=server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 -2025-01-25T14:28:17.447Z [INFO] server-8967c740-002b-c919-8b54-d6f78ad1c486: added peer, starting replication: peer=server-e4cec951-a9ef-47d9-077c-6c2670d776c0 -2025-01-25T14:28:17.447Z [ERROR] server-8967c740-002b-c919-8b54-d6f78ad1c486: failed to make requestVote RPC: target="{Voter server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041}" error="failed to connect to peer: 4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041" term=4 -2025-01-25T14:28:17.448Z [ERROR] server-8967c740-002b-c919-8b54-d6f78ad1c486: failed to appendEntries to: peer="{Voter server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041}" error="failed to connect to peer: 4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041" -2025-01-25T14:28:17.448Z [INFO] server-8967c740-002b-c919-8b54-d6f78ad1c486: pipelining replication: peer="{Voter server-e4cec951-a9ef-47d9-077c-6c2670d776c0 e4cec951-a9ef-47d9-077c-6c2670d776c0}" -2025-01-25T14:28:17.455Z [ERROR] server-8967c740-002b-c919-8b54-d6f78ad1c486: failed to heartbeat to: peer=4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 backoff time=10ms error="failed to connect to peer: 4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041" -2025-01-25T14:28:17.460Z [WARN] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:17.460Z [INFO] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: entering candidate state: node="Node at 4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 [Candidate]" term=3 -2025-01-25T14:28:17.460Z [ERROR] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: failed to make requestVote RPC: target="{Voter server-e4cec951-a9ef-47d9-077c-6c2670d776c0 e4cec951-a9ef-47d9-077c-6c2670d776c0}" error="failed to connect to peer: e4cec951-a9ef-47d9-077c-6c2670d776c0" term=3 -2025-01-25T14:28:17.460Z [ERROR] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: failed to make requestVote RPC: target="{Voter server-8967c740-002b-c919-8b54-d6f78ad1c486 8967c740-002b-c919-8b54-d6f78ad1c486}" error="failed to connect to peer: 8967c740-002b-c919-8b54-d6f78ad1c486" term=3 -2025-01-25T14:28:17.467Z [ERROR] server-8967c740-002b-c919-8b54-d6f78ad1c486: failed to appendEntries to: peer="{Voter server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041}" error="failed to connect to peer: 4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041" -2025-01-25T14:28:17.474Z [ERROR] server-8967c740-002b-c919-8b54-d6f78ad1c486: failed to heartbeat to: peer=4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 backoff time=10ms error="failed to connect to peer: 4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041" -2025-01-25T14:28:17.483Z [ERROR] server-8967c740-002b-c919-8b54-d6f78ad1c486: failed to appendEntries to: peer="{Voter server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041}" error="failed to connect to peer: 4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041" -2025-01-25T14:28:17.492Z [ERROR] server-8967c740-002b-c919-8b54-d6f78ad1c486: failed to heartbeat to: peer=4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 backoff time=10ms error="failed to connect to peer: 4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041" -2025-01-25T14:28:17.498Z [WARN] server-8967c740-002b-c919-8b54-d6f78ad1c486: failed to contact: server-id=server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 time=50.31803ms -2025-01-25T14:28:17.509Z [ERROR] server-8967c740-002b-c919-8b54-d6f78ad1c486: failed to heartbeat to: peer=4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 backoff time=20ms error="failed to connect to peer: 4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041" -2025-01-25T14:28:17.510Z [ERROR] server-8967c740-002b-c919-8b54-d6f78ad1c486: failed to appendEntries to: peer="{Voter server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041}" error="failed to connect to peer: 4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041" -2025-01-25T14:28:17.536Z [ERROR] server-8967c740-002b-c919-8b54-d6f78ad1c486: failed to heartbeat to: peer=4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 backoff time=25ms error="failed to connect to peer: 4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041" -2025-01-25T14:28:17.546Z [WARN] server-8967c740-002b-c919-8b54-d6f78ad1c486: failed to contact: server-id=server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 time=98.11543ms -2025-01-25T14:28:17.557Z [ERROR] server-8967c740-002b-c919-8b54-d6f78ad1c486: failed to appendEntries to: peer="{Voter server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041}" error="failed to connect to peer: 4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041" -2025-01-25T14:28:17.560Z [WARN] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: Election timeout reached, restarting election -2025-01-25T14:28:17.560Z [INFO] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: entering candidate state: node="Node at 4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 [Candidate]" term=4 -2025-01-25T14:28:17.560Z [ERROR] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: failed to make requestVote RPC: target="{Voter server-e4cec951-a9ef-47d9-077c-6c2670d776c0 e4cec951-a9ef-47d9-077c-6c2670d776c0}" error="failed to connect to peer: e4cec951-a9ef-47d9-077c-6c2670d776c0" term=4 -2025-01-25T14:28:17.560Z [ERROR] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: failed to make requestVote RPC: target="{Voter server-8967c740-002b-c919-8b54-d6f78ad1c486 8967c740-002b-c919-8b54-d6f78ad1c486}" error="failed to connect to peer: 8967c740-002b-c919-8b54-d6f78ad1c486" term=4 -2025-01-25T14:28:17.565Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:17.460462724 +0000 UTC m=+7.314787706 (1 nodes), 147.580689ms from start of poll, 427.765207ms from cluster start. Timeout at 2025-01-25 14:28:17.565472003 +0000 UTC m=+7.419797065, 105.009359ms after stability - raft_test.go:413: [INFO] Reconnecting Node at 4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 [Candidate] -2025-01-25T14:28:17.568Z [INFO] server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041: entering follower state: follower="Node at 4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 [Follower]" leader-address=8967c740-002b-c919-8b54-d6f78ad1c486 leader-id=server-8967c740-002b-c919-8b54-d6f78ad1c486 -2025-01-25T14:28:17.646Z [INFO] server-8967c740-002b-c919-8b54-d6f78ad1c486: pipelining replication: peer="{Voter server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041}" -2025-01-25T14:28:17.649Z [INFO] server-8967c740-002b-c919-8b54-d6f78ad1c486: aborting pipeline replication: peer="{Voter server-e4cec951-a9ef-47d9-077c-6c2670d776c0 e4cec951-a9ef-47d9-077c-6c2670d776c0}" -2025-01-25T14:28:17.649Z [INFO] server-8967c740-002b-c919-8b54-d6f78ad1c486: aborting pipeline replication: peer="{Voter server-4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041 4bf52fbd-cc7e-d3c3-1e19-0dfd3cb65041}" ---- PASS: TestRaft_LeaderFail (0.62s) +2025-01-25T14:38:58.716Z [INFO] server-894600a5-4747-d0a6-c350-5365268cd2c3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-894600a5-4747-d0a6-c350-5365268cd2c3 Address:894600a5-4747-d0a6-c350-5365268cd2c3} {Suffrage:Voter ID:server-46833bc1-7132-46ed-f0c3-1f35c343c4dc Address:46833bc1-7132-46ed-f0c3-1f35c343c4dc} {Suffrage:Voter ID:server-e723e141-007e-e436-2e33-0b5a2fd0e38c Address:e723e141-007e-e436-2e33-0b5a2fd0e38c}]" +2025-01-25T14:38:58.716Z [INFO] server-894600a5-4747-d0a6-c350-5365268cd2c3: entering follower state: follower="Node at 894600a5-4747-d0a6-c350-5365268cd2c3 [Follower]" leader-address= leader-id= +2025-01-25T14:38:58.717Z [INFO] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-894600a5-4747-d0a6-c350-5365268cd2c3 Address:894600a5-4747-d0a6-c350-5365268cd2c3} {Suffrage:Voter ID:server-46833bc1-7132-46ed-f0c3-1f35c343c4dc Address:46833bc1-7132-46ed-f0c3-1f35c343c4dc} {Suffrage:Voter ID:server-e723e141-007e-e436-2e33-0b5a2fd0e38c Address:e723e141-007e-e436-2e33-0b5a2fd0e38c}]" +2025-01-25T14:38:58.717Z [INFO] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: entering follower state: follower="Node at 46833bc1-7132-46ed-f0c3-1f35c343c4dc [Follower]" leader-address= leader-id= +2025-01-25T14:38:58.718Z [INFO] server-e723e141-007e-e436-2e33-0b5a2fd0e38c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-894600a5-4747-d0a6-c350-5365268cd2c3 Address:894600a5-4747-d0a6-c350-5365268cd2c3} {Suffrage:Voter ID:server-46833bc1-7132-46ed-f0c3-1f35c343c4dc Address:46833bc1-7132-46ed-f0c3-1f35c343c4dc} {Suffrage:Voter ID:server-e723e141-007e-e436-2e33-0b5a2fd0e38c Address:e723e141-007e-e436-2e33-0b5a2fd0e38c}]" +2025-01-25T14:38:58.718Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:38:58.718Z [INFO] server-e723e141-007e-e436-2e33-0b5a2fd0e38c: entering follower state: follower="Node at e723e141-007e-e436-2e33-0b5a2fd0e38c [Follower]" leader-address= leader-id= +2025-01-25T14:38:58.785Z [WARN] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:38:58.785Z [WARN] server-894600a5-4747-d0a6-c350-5365268cd2c3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:38:58.785Z [INFO] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: entering candidate state: node="Node at 46833bc1-7132-46ed-f0c3-1f35c343c4dc [Candidate]" term=2 +2025-01-25T14:38:58.785Z [INFO] server-894600a5-4747-d0a6-c350-5365268cd2c3: entering candidate state: node="Node at 894600a5-4747-d0a6-c350-5365268cd2c3 [Candidate]" term=2 +2025-01-25T14:38:58.785Z [INFO] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: election won: term=2 tally=2 +2025-01-25T14:38:58.786Z [INFO] server-894600a5-4747-d0a6-c350-5365268cd2c3: duplicate requestVote for same term: term=2 +2025-01-25T14:38:58.786Z [INFO] server-e723e141-007e-e436-2e33-0b5a2fd0e38c: duplicate requestVote for same term: term=2 +2025-01-25T14:38:58.786Z [INFO] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: entering leader state: leader="Node at 46833bc1-7132-46ed-f0c3-1f35c343c4dc [Leader]" +2025-01-25T14:38:58.786Z [INFO] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: added peer, starting replication: peer=server-894600a5-4747-d0a6-c350-5365268cd2c3 +2025-01-25T14:38:58.786Z [INFO] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: added peer, starting replication: peer=server-e723e141-007e-e436-2e33-0b5a2fd0e38c +2025-01-25T14:38:58.786Z [WARN] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: rejecting vote request since we have a leader: from=894600a5-4747-d0a6-c350-5365268cd2c3 leader=46833bc1-7132-46ed-f0c3-1f35c343c4dc leader-id=server-46833bc1-7132-46ed-f0c3-1f35c343c4dc +2025-01-25T14:38:58.786Z [INFO] server-894600a5-4747-d0a6-c350-5365268cd2c3: entering follower state: follower="Node at 894600a5-4747-d0a6-c350-5365268cd2c3 [Follower]" leader-address=46833bc1-7132-46ed-f0c3-1f35c343c4dc leader-id=server-46833bc1-7132-46ed-f0c3-1f35c343c4dc +2025-01-25T14:38:58.787Z [INFO] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: pipelining replication: peer="{Voter server-e723e141-007e-e436-2e33-0b5a2fd0e38c e723e141-007e-e436-2e33-0b5a2fd0e38c}" +2025-01-25T14:38:58.787Z [INFO] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: pipelining replication: peer="{Voter server-894600a5-4747-d0a6-c350-5365268cd2c3 894600a5-4747-d0a6-c350-5365268cd2c3}" +2025-01-25T14:38:58.893Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:38:58.786943905 +0000 UTC m=+7.832783223 (2 nodes), 68.453074ms from start of poll, 71.122124ms from cluster start. Timeout at 2025-01-25 14:38:58.891965198 +0000 UTC m=+7.937804515, 105.021292ms after stability +2025-01-25T14:38:58.893Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:38:58.999Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:38:58.893334215 +0000 UTC m=+7.939173533 (1 nodes), 5.542µs from start of poll, 177.512434ms from cluster start. Timeout at 2025-01-25 14:38:58.998390799 +0000 UTC m=+8.044230409, 105.056876ms after stability + raft_test.go:378: [INFO] Disconnecting Node at 46833bc1-7132-46ed-f0c3-1f35c343c4dc [Leader] +2025-01-25T14:38:59.021Z [ERROR] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: failed to pipeline appendEntries: peer="{Voter server-894600a5-4747-d0a6-c350-5365268cd2c3 894600a5-4747-d0a6-c350-5365268cd2c3}" error="append pipeline closed" +2025-01-25T14:38:59.022Z [INFO] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: aborting pipeline replication: peer="{Voter server-894600a5-4747-d0a6-c350-5365268cd2c3 894600a5-4747-d0a6-c350-5365268cd2c3}" +2025-01-25T14:38:59.023Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:38:59.024Z [ERROR] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: failed to pipeline appendEntries: peer="{Voter server-e723e141-007e-e436-2e33-0b5a2fd0e38c e723e141-007e-e436-2e33-0b5a2fd0e38c}" error="append pipeline closed" +2025-01-25T14:38:59.024Z [INFO] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: aborting pipeline replication: peer="{Voter server-e723e141-007e-e436-2e33-0b5a2fd0e38c e723e141-007e-e436-2e33-0b5a2fd0e38c}" +2025-01-25T14:38:59.024Z [ERROR] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: failed to heartbeat to: peer=e723e141-007e-e436-2e33-0b5a2fd0e38c backoff time=10ms error="failed to connect to peer: e723e141-007e-e436-2e33-0b5a2fd0e38c" +2025-01-25T14:38:59.026Z [ERROR] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: failed to heartbeat to: peer=894600a5-4747-d0a6-c350-5365268cd2c3 backoff time=10ms error="failed to connect to peer: 894600a5-4747-d0a6-c350-5365268cd2c3" +2025-01-25T14:38:59.029Z [ERROR] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: failed to appendEntries to: peer="{Voter server-894600a5-4747-d0a6-c350-5365268cd2c3 894600a5-4747-d0a6-c350-5365268cd2c3}" error="failed to connect to peer: 894600a5-4747-d0a6-c350-5365268cd2c3" +2025-01-25T14:38:59.034Z [ERROR] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: failed to appendEntries to: peer="{Voter server-e723e141-007e-e436-2e33-0b5a2fd0e38c e723e141-007e-e436-2e33-0b5a2fd0e38c}" error="failed to connect to peer: e723e141-007e-e436-2e33-0b5a2fd0e38c" +2025-01-25T14:38:59.045Z [ERROR] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: failed to heartbeat to: peer=e723e141-007e-e436-2e33-0b5a2fd0e38c backoff time=10ms error="failed to connect to peer: e723e141-007e-e436-2e33-0b5a2fd0e38c" +2025-01-25T14:38:59.046Z [ERROR] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: failed to heartbeat to: peer=894600a5-4747-d0a6-c350-5365268cd2c3 backoff time=10ms error="failed to connect to peer: 894600a5-4747-d0a6-c350-5365268cd2c3" +2025-01-25T14:38:59.050Z [ERROR] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: failed to appendEntries to: peer="{Voter server-894600a5-4747-d0a6-c350-5365268cd2c3 894600a5-4747-d0a6-c350-5365268cd2c3}" error="failed to connect to peer: 894600a5-4747-d0a6-c350-5365268cd2c3" +2025-01-25T14:38:59.053Z [ERROR] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: failed to appendEntries to: peer="{Voter server-e723e141-007e-e436-2e33-0b5a2fd0e38c e723e141-007e-e436-2e33-0b5a2fd0e38c}" error="failed to connect to peer: e723e141-007e-e436-2e33-0b5a2fd0e38c" +2025-01-25T14:38:59.061Z [ERROR] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: failed to heartbeat to: peer=e723e141-007e-e436-2e33-0b5a2fd0e38c backoff time=10ms error="failed to connect to peer: e723e141-007e-e436-2e33-0b5a2fd0e38c" +2025-01-25T14:38:59.066Z [WARN] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: failed to contact: server-id=server-894600a5-4747-d0a6-c350-5365268cd2c3 time=50.098562ms +2025-01-25T14:38:59.067Z [WARN] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: failed to contact: server-id=server-e723e141-007e-e436-2e33-0b5a2fd0e38c time=51.13019ms +2025-01-25T14:38:59.067Z [WARN] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: failed to contact quorum of nodes, stepping down +2025-01-25T14:38:59.067Z [ERROR] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: failed to appendEntries to: peer="{Voter server-894600a5-4747-d0a6-c350-5365268cd2c3 894600a5-4747-d0a6-c350-5365268cd2c3}" error="failed to connect to peer: 894600a5-4747-d0a6-c350-5365268cd2c3" +2025-01-25T14:38:59.067Z [ERROR] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: failed to heartbeat to: peer=894600a5-4747-d0a6-c350-5365268cd2c3 backoff time=10ms error="failed to connect to peer: 894600a5-4747-d0a6-c350-5365268cd2c3" +2025-01-25T14:38:59.067Z [INFO] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: entering follower state: follower="Node at 46833bc1-7132-46ed-f0c3-1f35c343c4dc [Follower]" leader-address= leader-id= +2025-01-25T14:38:59.068Z [ERROR] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: failed to appendEntries to: peer="{Voter server-e723e141-007e-e436-2e33-0b5a2fd0e38c e723e141-007e-e436-2e33-0b5a2fd0e38c}" error="failed to connect to peer: e723e141-007e-e436-2e33-0b5a2fd0e38c" +2025-01-25T14:38:59.107Z [WARN] server-e723e141-007e-e436-2e33-0b5a2fd0e38c: heartbeat timeout reached, starting election: last-leader-addr=46833bc1-7132-46ed-f0c3-1f35c343c4dc last-leader-id=server-46833bc1-7132-46ed-f0c3-1f35c343c4dc +2025-01-25T14:38:59.107Z [INFO] server-e723e141-007e-e436-2e33-0b5a2fd0e38c: entering candidate state: node="Node at e723e141-007e-e436-2e33-0b5a2fd0e38c [Candidate]" term=3 +2025-01-25T14:38:59.108Z [ERROR] server-e723e141-007e-e436-2e33-0b5a2fd0e38c: failed to make requestVote RPC: target="{Voter server-46833bc1-7132-46ed-f0c3-1f35c343c4dc 46833bc1-7132-46ed-f0c3-1f35c343c4dc}" error="failed to connect to peer: 46833bc1-7132-46ed-f0c3-1f35c343c4dc" term=3 +2025-01-25T14:38:59.108Z [WARN] server-894600a5-4747-d0a6-c350-5365268cd2c3: rejecting vote request since we have a leader: from=e723e141-007e-e436-2e33-0b5a2fd0e38c leader=46833bc1-7132-46ed-f0c3-1f35c343c4dc leader-id=server-46833bc1-7132-46ed-f0c3-1f35c343c4dc +2025-01-25T14:38:59.138Z [WARN] server-894600a5-4747-d0a6-c350-5365268cd2c3: heartbeat timeout reached, starting election: last-leader-addr=46833bc1-7132-46ed-f0c3-1f35c343c4dc last-leader-id=server-46833bc1-7132-46ed-f0c3-1f35c343c4dc +2025-01-25T14:38:59.138Z [INFO] server-894600a5-4747-d0a6-c350-5365268cd2c3: entering candidate state: node="Node at 894600a5-4747-d0a6-c350-5365268cd2c3 [Candidate]" term=3 +2025-01-25T14:38:59.139Z [INFO] server-e723e141-007e-e436-2e33-0b5a2fd0e38c: duplicate requestVote for same term: term=3 +2025-01-25T14:38:59.139Z [ERROR] server-894600a5-4747-d0a6-c350-5365268cd2c3: failed to make requestVote RPC: target="{Voter server-46833bc1-7132-46ed-f0c3-1f35c343c4dc 46833bc1-7132-46ed-f0c3-1f35c343c4dc}" error="failed to connect to peer: 46833bc1-7132-46ed-f0c3-1f35c343c4dc" term=3 +2025-01-25T14:38:59.160Z [WARN] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:38:59.160Z [INFO] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: entering candidate state: node="Node at 46833bc1-7132-46ed-f0c3-1f35c343c4dc [Candidate]" term=3 +2025-01-25T14:38:59.160Z [ERROR] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: failed to make requestVote RPC: target="{Voter server-e723e141-007e-e436-2e33-0b5a2fd0e38c e723e141-007e-e436-2e33-0b5a2fd0e38c}" error="failed to connect to peer: e723e141-007e-e436-2e33-0b5a2fd0e38c" term=3 +2025-01-25T14:38:59.161Z [ERROR] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: failed to make requestVote RPC: target="{Voter server-894600a5-4747-d0a6-c350-5365268cd2c3 894600a5-4747-d0a6-c350-5365268cd2c3}" error="failed to connect to peer: 894600a5-4747-d0a6-c350-5365268cd2c3" term=3 +2025-01-25T14:38:59.161Z [WARN] server-e723e141-007e-e436-2e33-0b5a2fd0e38c: Election timeout reached, restarting election +2025-01-25T14:38:59.161Z [INFO] server-e723e141-007e-e436-2e33-0b5a2fd0e38c: entering candidate state: node="Node at e723e141-007e-e436-2e33-0b5a2fd0e38c [Candidate]" term=4 +2025-01-25T14:38:59.161Z [ERROR] server-e723e141-007e-e436-2e33-0b5a2fd0e38c: failed to make requestVote RPC: target="{Voter server-46833bc1-7132-46ed-f0c3-1f35c343c4dc 46833bc1-7132-46ed-f0c3-1f35c343c4dc}" error="failed to connect to peer: 46833bc1-7132-46ed-f0c3-1f35c343c4dc" term=4 +2025-01-25T14:38:59.161Z [INFO] server-894600a5-4747-d0a6-c350-5365268cd2c3: entering follower state: follower="Node at 894600a5-4747-d0a6-c350-5365268cd2c3 [Follower]" leader-address= leader-id= +2025-01-25T14:38:59.162Z [INFO] server-e723e141-007e-e436-2e33-0b5a2fd0e38c: election won: term=4 tally=2 +2025-01-25T14:38:59.162Z [INFO] server-e723e141-007e-e436-2e33-0b5a2fd0e38c: entering leader state: leader="Node at e723e141-007e-e436-2e33-0b5a2fd0e38c [Leader]" +2025-01-25T14:38:59.162Z [INFO] server-e723e141-007e-e436-2e33-0b5a2fd0e38c: added peer, starting replication: peer=server-894600a5-4747-d0a6-c350-5365268cd2c3 +2025-01-25T14:38:59.162Z [INFO] server-e723e141-007e-e436-2e33-0b5a2fd0e38c: added peer, starting replication: peer=server-46833bc1-7132-46ed-f0c3-1f35c343c4dc +2025-01-25T14:38:59.163Z [INFO] server-e723e141-007e-e436-2e33-0b5a2fd0e38c: pipelining replication: peer="{Voter server-894600a5-4747-d0a6-c350-5365268cd2c3 894600a5-4747-d0a6-c350-5365268cd2c3}" +2025-01-25T14:38:59.163Z [ERROR] server-e723e141-007e-e436-2e33-0b5a2fd0e38c: failed to appendEntries to: peer="{Voter server-46833bc1-7132-46ed-f0c3-1f35c343c4dc 46833bc1-7132-46ed-f0c3-1f35c343c4dc}" error="failed to connect to peer: 46833bc1-7132-46ed-f0c3-1f35c343c4dc" +2025-01-25T14:38:59.170Z [ERROR] server-e723e141-007e-e436-2e33-0b5a2fd0e38c: failed to heartbeat to: peer=46833bc1-7132-46ed-f0c3-1f35c343c4dc backoff time=10ms error="failed to connect to peer: 46833bc1-7132-46ed-f0c3-1f35c343c4dc" +2025-01-25T14:38:59.180Z [ERROR] server-e723e141-007e-e436-2e33-0b5a2fd0e38c: failed to appendEntries to: peer="{Voter server-46833bc1-7132-46ed-f0c3-1f35c343c4dc 46833bc1-7132-46ed-f0c3-1f35c343c4dc}" error="failed to connect to peer: 46833bc1-7132-46ed-f0c3-1f35c343c4dc" +2025-01-25T14:38:59.189Z [ERROR] server-e723e141-007e-e436-2e33-0b5a2fd0e38c: failed to heartbeat to: peer=46833bc1-7132-46ed-f0c3-1f35c343c4dc backoff time=10ms error="failed to connect to peer: 46833bc1-7132-46ed-f0c3-1f35c343c4dc" +2025-01-25T14:38:59.200Z [ERROR] server-e723e141-007e-e436-2e33-0b5a2fd0e38c: failed to appendEntries to: peer="{Voter server-46833bc1-7132-46ed-f0c3-1f35c343c4dc 46833bc1-7132-46ed-f0c3-1f35c343c4dc}" error="failed to connect to peer: 46833bc1-7132-46ed-f0c3-1f35c343c4dc" +2025-01-25T14:38:59.208Z [ERROR] server-e723e141-007e-e436-2e33-0b5a2fd0e38c: failed to heartbeat to: peer=46833bc1-7132-46ed-f0c3-1f35c343c4dc backoff time=10ms error="failed to connect to peer: 46833bc1-7132-46ed-f0c3-1f35c343c4dc" +2025-01-25T14:38:59.213Z [WARN] server-e723e141-007e-e436-2e33-0b5a2fd0e38c: failed to contact: server-id=server-46833bc1-7132-46ed-f0c3-1f35c343c4dc time=50.382397ms +2025-01-25T14:38:59.220Z [WARN] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: Election timeout reached, restarting election +2025-01-25T14:38:59.220Z [INFO] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: entering candidate state: node="Node at 46833bc1-7132-46ed-f0c3-1f35c343c4dc [Candidate]" term=4 +2025-01-25T14:38:59.221Z [ERROR] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: failed to make requestVote RPC: target="{Voter server-894600a5-4747-d0a6-c350-5365268cd2c3 894600a5-4747-d0a6-c350-5365268cd2c3}" error="failed to connect to peer: 894600a5-4747-d0a6-c350-5365268cd2c3" term=4 +2025-01-25T14:38:59.221Z [ERROR] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: failed to make requestVote RPC: target="{Voter server-e723e141-007e-e436-2e33-0b5a2fd0e38c e723e141-007e-e436-2e33-0b5a2fd0e38c}" error="failed to connect to peer: e723e141-007e-e436-2e33-0b5a2fd0e38c" term=4 +2025-01-25T14:38:59.226Z [ERROR] server-e723e141-007e-e436-2e33-0b5a2fd0e38c: failed to heartbeat to: peer=46833bc1-7132-46ed-f0c3-1f35c343c4dc backoff time=20ms error="failed to connect to peer: 46833bc1-7132-46ed-f0c3-1f35c343c4dc" +2025-01-25T14:38:59.232Z [ERROR] server-e723e141-007e-e436-2e33-0b5a2fd0e38c: failed to appendEntries to: peer="{Voter server-46833bc1-7132-46ed-f0c3-1f35c343c4dc 46833bc1-7132-46ed-f0c3-1f35c343c4dc}" error="failed to connect to peer: 46833bc1-7132-46ed-f0c3-1f35c343c4dc" +2025-01-25T14:38:59.255Z [ERROR] server-e723e141-007e-e436-2e33-0b5a2fd0e38c: failed to heartbeat to: peer=46833bc1-7132-46ed-f0c3-1f35c343c4dc backoff time=25ms error="failed to connect to peer: 46833bc1-7132-46ed-f0c3-1f35c343c4dc" +2025-01-25T14:38:59.260Z [WARN] server-e723e141-007e-e436-2e33-0b5a2fd0e38c: failed to contact: server-id=server-46833bc1-7132-46ed-f0c3-1f35c343c4dc time=97.408951ms +2025-01-25T14:38:59.268Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:38:59.162303748 +0000 UTC m=+8.208143358 (1 nodes), 139.177031ms from start of poll, 446.482259ms from cluster start. Timeout at 2025-01-25 14:38:59.267327957 +0000 UTC m=+8.313167275, 105.023917ms after stability + raft_test.go:413: [INFO] Reconnecting Node at 46833bc1-7132-46ed-f0c3-1f35c343c4dc [Candidate] +2025-01-25T14:38:59.282Z [INFO] server-46833bc1-7132-46ed-f0c3-1f35c343c4dc: entering follower state: follower="Node at 46833bc1-7132-46ed-f0c3-1f35c343c4dc [Follower]" leader-address=e723e141-007e-e436-2e33-0b5a2fd0e38c leader-id=server-e723e141-007e-e436-2e33-0b5a2fd0e38c +2025-01-25T14:38:59.282Z [INFO] server-e723e141-007e-e436-2e33-0b5a2fd0e38c: pipelining replication: peer="{Voter server-46833bc1-7132-46ed-f0c3-1f35c343c4dc 46833bc1-7132-46ed-f0c3-1f35c343c4dc}" +2025-01-25T14:38:59.287Z [INFO] server-e723e141-007e-e436-2e33-0b5a2fd0e38c: aborting pipeline replication: peer="{Voter server-894600a5-4747-d0a6-c350-5365268cd2c3 894600a5-4747-d0a6-c350-5365268cd2c3}" +2025-01-25T14:38:59.287Z [INFO] server-e723e141-007e-e436-2e33-0b5a2fd0e38c: aborting pipeline replication: peer="{Voter server-46833bc1-7132-46ed-f0c3-1f35c343c4dc 46833bc1-7132-46ed-f0c3-1f35c343c4dc}" +2025-01-25T14:38:59.789Z [ERROR] server-e723e141-007e-e436-2e33-0b5a2fd0e38c: failed to heartbeat to: peer=46833bc1-7132-46ed-f0c3-1f35c343c4dc backoff time=25ms error="command timed out" +--- PASS: TestRaft_LeaderFail (1.08s) === RUN TestRaft_BehindFollower -2025-01-25T14:28:17.651Z [INFO] server-e5108ba7-450f-e118-c1b0-c48038aef9e4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e5108ba7-450f-e118-c1b0-c48038aef9e4 Address:e5108ba7-450f-e118-c1b0-c48038aef9e4} {Suffrage:Voter ID:server-5734fc47-f636-8665-780c-90a00ec2398a Address:5734fc47-f636-8665-780c-90a00ec2398a} {Suffrage:Voter ID:server-404920e1-e475-3b28-01bd-ec1e4c8213ac Address:404920e1-e475-3b28-01bd-ec1e4c8213ac}]" -2025-01-25T14:28:17.651Z [INFO] server-e5108ba7-450f-e118-c1b0-c48038aef9e4: entering follower state: follower="Node at e5108ba7-450f-e118-c1b0-c48038aef9e4 [Follower]" leader-address= leader-id= -2025-01-25T14:28:17.652Z [INFO] server-5734fc47-f636-8665-780c-90a00ec2398a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e5108ba7-450f-e118-c1b0-c48038aef9e4 Address:e5108ba7-450f-e118-c1b0-c48038aef9e4} {Suffrage:Voter ID:server-5734fc47-f636-8665-780c-90a00ec2398a Address:5734fc47-f636-8665-780c-90a00ec2398a} {Suffrage:Voter ID:server-404920e1-e475-3b28-01bd-ec1e4c8213ac Address:404920e1-e475-3b28-01bd-ec1e4c8213ac}]" -2025-01-25T14:28:17.652Z [INFO] server-5734fc47-f636-8665-780c-90a00ec2398a: entering follower state: follower="Node at 5734fc47-f636-8665-780c-90a00ec2398a [Follower]" leader-address= leader-id= -2025-01-25T14:28:17.652Z [INFO] server-404920e1-e475-3b28-01bd-ec1e4c8213ac: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e5108ba7-450f-e118-c1b0-c48038aef9e4 Address:e5108ba7-450f-e118-c1b0-c48038aef9e4} {Suffrage:Voter ID:server-5734fc47-f636-8665-780c-90a00ec2398a Address:5734fc47-f636-8665-780c-90a00ec2398a} {Suffrage:Voter ID:server-404920e1-e475-3b28-01bd-ec1e4c8213ac Address:404920e1-e475-3b28-01bd-ec1e4c8213ac}]" -2025-01-25T14:28:17.652Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:17.652Z [INFO] server-404920e1-e475-3b28-01bd-ec1e4c8213ac: entering follower state: follower="Node at 404920e1-e475-3b28-01bd-ec1e4c8213ac [Follower]" leader-address= leader-id= -2025-01-25T14:28:17.747Z [WARN] server-e5108ba7-450f-e118-c1b0-c48038aef9e4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:17.747Z [WARN] server-404920e1-e475-3b28-01bd-ec1e4c8213ac: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:17.747Z [INFO] server-404920e1-e475-3b28-01bd-ec1e4c8213ac: entering candidate state: node="Node at 404920e1-e475-3b28-01bd-ec1e4c8213ac [Candidate]" term=2 -2025-01-25T14:28:17.747Z [WARN] server-5734fc47-f636-8665-780c-90a00ec2398a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:17.747Z [INFO] server-5734fc47-f636-8665-780c-90a00ec2398a: entering candidate state: node="Node at 5734fc47-f636-8665-780c-90a00ec2398a [Candidate]" term=2 -2025-01-25T14:28:17.747Z [INFO] server-e5108ba7-450f-e118-c1b0-c48038aef9e4: entering candidate state: node="Node at e5108ba7-450f-e118-c1b0-c48038aef9e4 [Candidate]" term=2 -2025-01-25T14:28:17.747Z [INFO] server-5734fc47-f636-8665-780c-90a00ec2398a: duplicate requestVote for same term: term=2 -2025-01-25T14:28:17.747Z [INFO] server-404920e1-e475-3b28-01bd-ec1e4c8213ac: duplicate requestVote for same term: term=2 -2025-01-25T14:28:17.747Z [INFO] server-e5108ba7-450f-e118-c1b0-c48038aef9e4: duplicate requestVote for same term: term=2 -2025-01-25T14:28:17.747Z [INFO] server-404920e1-e475-3b28-01bd-ec1e4c8213ac: duplicate requestVote for same term: term=2 -2025-01-25T14:28:17.747Z [INFO] server-e5108ba7-450f-e118-c1b0-c48038aef9e4: duplicate requestVote for same term: term=2 -2025-01-25T14:28:17.747Z [INFO] server-5734fc47-f636-8665-780c-90a00ec2398a: duplicate requestVote for same term: term=2 -2025-01-25T14:28:17.824Z [WARN] server-5734fc47-f636-8665-780c-90a00ec2398a: Election timeout reached, restarting election -2025-01-25T14:28:17.824Z [INFO] server-5734fc47-f636-8665-780c-90a00ec2398a: entering candidate state: node="Node at 5734fc47-f636-8665-780c-90a00ec2398a [Candidate]" term=3 -2025-01-25T14:28:17.824Z [INFO] server-404920e1-e475-3b28-01bd-ec1e4c8213ac: entering follower state: follower="Node at 404920e1-e475-3b28-01bd-ec1e4c8213ac [Follower]" leader-address= leader-id= -2025-01-25T14:28:17.824Z [INFO] server-5734fc47-f636-8665-780c-90a00ec2398a: election won: term=3 tally=2 -2025-01-25T14:28:17.824Z [INFO] server-5734fc47-f636-8665-780c-90a00ec2398a: entering leader state: leader="Node at 5734fc47-f636-8665-780c-90a00ec2398a [Leader]" -2025-01-25T14:28:17.824Z [INFO] server-5734fc47-f636-8665-780c-90a00ec2398a: added peer, starting replication: peer=server-e5108ba7-450f-e118-c1b0-c48038aef9e4 -2025-01-25T14:28:17.824Z [INFO] server-5734fc47-f636-8665-780c-90a00ec2398a: added peer, starting replication: peer=server-404920e1-e475-3b28-01bd-ec1e4c8213ac -2025-01-25T14:28:17.825Z [INFO] server-5734fc47-f636-8665-780c-90a00ec2398a: pipelining replication: peer="{Voter server-404920e1-e475-3b28-01bd-ec1e4c8213ac 404920e1-e475-3b28-01bd-ec1e4c8213ac}" -2025-01-25T14:28:17.825Z [INFO] server-e5108ba7-450f-e118-c1b0-c48038aef9e4: entering follower state: follower="Node at e5108ba7-450f-e118-c1b0-c48038aef9e4 [Follower]" leader-address= leader-id= -2025-01-25T14:28:17.825Z [INFO] server-5734fc47-f636-8665-780c-90a00ec2398a: pipelining replication: peer="{Voter server-e5108ba7-450f-e118-c1b0-c48038aef9e4 e5108ba7-450f-e118-c1b0-c48038aef9e4}" -2025-01-25T14:28:17.930Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:17.825252678 +0000 UTC m=+7.679577660 (1 nodes), 172.969656ms from start of poll, 173.629673ms from cluster start. Timeout at 2025-01-25 14:28:17.930275637 +0000 UTC m=+7.784600619, 105.022959ms after stability -2025-01-25T14:28:17.930Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:28:18.035Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:28:17.930699523 +0000 UTC m=+7.785024505 (2 nodes), 2.8µs from start of poll, 279.076518ms from cluster start. Timeout at 2025-01-25 14:28:18.035735921 +0000 UTC m=+7.890060903, 105.036398ms after stability -2025-01-25T14:28:18.036Z [ERROR] server-5734fc47-f636-8665-780c-90a00ec2398a: failed to pipeline appendEntries: peer="{Voter server-e5108ba7-450f-e118-c1b0-c48038aef9e4 e5108ba7-450f-e118-c1b0-c48038aef9e4}" error="append pipeline closed" -2025-01-25T14:28:18.036Z [INFO] server-5734fc47-f636-8665-780c-90a00ec2398a: aborting pipeline replication: peer="{Voter server-e5108ba7-450f-e118-c1b0-c48038aef9e4 e5108ba7-450f-e118-c1b0-c48038aef9e4}" -2025-01-25T14:28:18.036Z [ERROR] server-5734fc47-f636-8665-780c-90a00ec2398a: failed to appendEntries to: peer="{Voter server-e5108ba7-450f-e118-c1b0-c48038aef9e4 e5108ba7-450f-e118-c1b0-c48038aef9e4}" error="failed to connect to peer: e5108ba7-450f-e118-c1b0-c48038aef9e4" -2025-01-25T14:28:18.043Z [ERROR] server-5734fc47-f636-8665-780c-90a00ec2398a: failed to heartbeat to: peer=e5108ba7-450f-e118-c1b0-c48038aef9e4 backoff time=10ms error="failed to connect to peer: e5108ba7-450f-e118-c1b0-c48038aef9e4" +2025-01-25T14:38:59.797Z [INFO] server-3b42df68-ccbd-88ee-93b1-d2b5292e6e4d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3b42df68-ccbd-88ee-93b1-d2b5292e6e4d Address:3b42df68-ccbd-88ee-93b1-d2b5292e6e4d} {Suffrage:Voter ID:server-e79eb5cd-803a-7fa6-5f41-5c0a15bd8ef0 Address:e79eb5cd-803a-7fa6-5f41-5c0a15bd8ef0} {Suffrage:Voter ID:server-74ef5429-11ab-b753-5e16-0f529093c362 Address:74ef5429-11ab-b753-5e16-0f529093c362}]" +2025-01-25T14:38:59.797Z [INFO] server-3b42df68-ccbd-88ee-93b1-d2b5292e6e4d: entering follower state: follower="Node at 3b42df68-ccbd-88ee-93b1-d2b5292e6e4d [Follower]" leader-address= leader-id= +2025-01-25T14:38:59.798Z [INFO] server-e79eb5cd-803a-7fa6-5f41-5c0a15bd8ef0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3b42df68-ccbd-88ee-93b1-d2b5292e6e4d Address:3b42df68-ccbd-88ee-93b1-d2b5292e6e4d} {Suffrage:Voter ID:server-e79eb5cd-803a-7fa6-5f41-5c0a15bd8ef0 Address:e79eb5cd-803a-7fa6-5f41-5c0a15bd8ef0} {Suffrage:Voter ID:server-74ef5429-11ab-b753-5e16-0f529093c362 Address:74ef5429-11ab-b753-5e16-0f529093c362}]" +2025-01-25T14:38:59.798Z [INFO] server-e79eb5cd-803a-7fa6-5f41-5c0a15bd8ef0: entering follower state: follower="Node at e79eb5cd-803a-7fa6-5f41-5c0a15bd8ef0 [Follower]" leader-address= leader-id= +2025-01-25T14:38:59.799Z [INFO] server-74ef5429-11ab-b753-5e16-0f529093c362: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3b42df68-ccbd-88ee-93b1-d2b5292e6e4d Address:3b42df68-ccbd-88ee-93b1-d2b5292e6e4d} {Suffrage:Voter ID:server-e79eb5cd-803a-7fa6-5f41-5c0a15bd8ef0 Address:e79eb5cd-803a-7fa6-5f41-5c0a15bd8ef0} {Suffrage:Voter ID:server-74ef5429-11ab-b753-5e16-0f529093c362 Address:74ef5429-11ab-b753-5e16-0f529093c362}]" +2025-01-25T14:38:59.799Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:38:59.799Z [INFO] server-74ef5429-11ab-b753-5e16-0f529093c362: entering follower state: follower="Node at 74ef5429-11ab-b753-5e16-0f529093c362 [Follower]" leader-address= leader-id= +2025-01-25T14:38:59.871Z [WARN] server-74ef5429-11ab-b753-5e16-0f529093c362: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:38:59.871Z [INFO] server-74ef5429-11ab-b753-5e16-0f529093c362: entering candidate state: node="Node at 74ef5429-11ab-b753-5e16-0f529093c362 [Candidate]" term=2 +2025-01-25T14:38:59.872Z [INFO] server-74ef5429-11ab-b753-5e16-0f529093c362: election won: term=2 tally=2 +2025-01-25T14:38:59.872Z [INFO] server-74ef5429-11ab-b753-5e16-0f529093c362: entering leader state: leader="Node at 74ef5429-11ab-b753-5e16-0f529093c362 [Leader]" +2025-01-25T14:38:59.872Z [INFO] server-74ef5429-11ab-b753-5e16-0f529093c362: added peer, starting replication: peer=server-3b42df68-ccbd-88ee-93b1-d2b5292e6e4d +2025-01-25T14:38:59.872Z [INFO] server-74ef5429-11ab-b753-5e16-0f529093c362: added peer, starting replication: peer=server-e79eb5cd-803a-7fa6-5f41-5c0a15bd8ef0 +2025-01-25T14:38:59.873Z [INFO] server-74ef5429-11ab-b753-5e16-0f529093c362: pipelining replication: peer="{Voter server-3b42df68-ccbd-88ee-93b1-d2b5292e6e4d 3b42df68-ccbd-88ee-93b1-d2b5292e6e4d}" +2025-01-25T14:38:59.873Z [INFO] server-74ef5429-11ab-b753-5e16-0f529093c362: pipelining replication: peer="{Voter server-e79eb5cd-803a-7fa6-5f41-5c0a15bd8ef0 e79eb5cd-803a-7fa6-5f41-5c0a15bd8ef0}" +2025-01-25T14:38:59.977Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:38:59.872428399 +0000 UTC m=+8.918267717 (1 nodes), 72.809754ms from start of poll, 75.901721ms from cluster start. Timeout at 2025-01-25 14:38:59.97744794 +0000 UTC m=+9.023287258, 105.019541ms after stability +2025-01-25T14:38:59.977Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:39:00.083Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:38:59.97796429 +0000 UTC m=+9.023803607 (2 nodes), 7.875µs from start of poll, 181.437611ms from cluster start. Timeout at 2025-01-25 14:39:00.082991999 +0000 UTC m=+9.128831316, 105.027709ms after stability +2025-01-25T14:39:00.084Z [ERROR] server-74ef5429-11ab-b753-5e16-0f529093c362: failed to pipeline appendEntries: peer="{Voter server-3b42df68-ccbd-88ee-93b1-d2b5292e6e4d 3b42df68-ccbd-88ee-93b1-d2b5292e6e4d}" error="append pipeline closed" +2025-01-25T14:39:00.084Z [INFO] server-74ef5429-11ab-b753-5e16-0f529093c362: aborting pipeline replication: peer="{Voter server-3b42df68-ccbd-88ee-93b1-d2b5292e6e4d 3b42df68-ccbd-88ee-93b1-d2b5292e6e4d}" +2025-01-25T14:39:00.084Z [ERROR] server-74ef5429-11ab-b753-5e16-0f529093c362: failed to appendEntries to: peer="{Voter server-3b42df68-ccbd-88ee-93b1-d2b5292e6e4d 3b42df68-ccbd-88ee-93b1-d2b5292e6e4d}" error="failed to connect to peer: 3b42df68-ccbd-88ee-93b1-d2b5292e6e4d" +2025-01-25T14:39:00.089Z [ERROR] server-74ef5429-11ab-b753-5e16-0f529093c362: failed to heartbeat to: peer=3b42df68-ccbd-88ee-93b1-d2b5292e6e4d backoff time=10ms error="failed to connect to peer: 3b42df68-ccbd-88ee-93b1-d2b5292e6e4d" +2025-01-25T14:39:00.094Z [ERROR] server-74ef5429-11ab-b753-5e16-0f529093c362: failed to appendEntries to: peer="{Voter server-3b42df68-ccbd-88ee-93b1-d2b5292e6e4d 3b42df68-ccbd-88ee-93b1-d2b5292e6e4d}" error="failed to connect to peer: 3b42df68-ccbd-88ee-93b1-d2b5292e6e4d" +2025-01-25T14:39:00.105Z [ERROR] server-74ef5429-11ab-b753-5e16-0f529093c362: failed to appendEntries to: peer="{Voter server-3b42df68-ccbd-88ee-93b1-d2b5292e6e4d 3b42df68-ccbd-88ee-93b1-d2b5292e6e4d}" error="failed to connect to peer: 3b42df68-ccbd-88ee-93b1-d2b5292e6e4d" +2025-01-25T14:39:00.107Z [ERROR] server-74ef5429-11ab-b753-5e16-0f529093c362: failed to heartbeat to: peer=3b42df68-ccbd-88ee-93b1-d2b5292e6e4d backoff time=10ms error="failed to connect to peer: 3b42df68-ccbd-88ee-93b1-d2b5292e6e4d" raft_test.go:459: [INFO] Finished apply without behind follower -2025-01-25T14:28:18.046Z [INFO] server-5734fc47-f636-8665-780c-90a00ec2398a: pipelining replication: peer="{Voter server-e5108ba7-450f-e118-c1b0-c48038aef9e4 e5108ba7-450f-e118-c1b0-c48038aef9e4}" -2025-01-25T14:28:18.054Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:18.160Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:18.054920973 +0000 UTC m=+7.909245955 (1 nodes), 1.68µs from start of poll, 403.297968ms from cluster start. Timeout at 2025-01-25 14:28:18.159960572 +0000 UTC m=+8.014285674, 105.039719ms after stability -2025-01-25T14:28:18.160Z [INFO] server-5734fc47-f636-8665-780c-90a00ec2398a: aborting pipeline replication: peer="{Voter server-404920e1-e475-3b28-01bd-ec1e4c8213ac 404920e1-e475-3b28-01bd-ec1e4c8213ac}" -2025-01-25T14:28:18.160Z [INFO] server-5734fc47-f636-8665-780c-90a00ec2398a: aborting pipeline replication: peer="{Voter server-e5108ba7-450f-e118-c1b0-c48038aef9e4 e5108ba7-450f-e118-c1b0-c48038aef9e4}" ---- PASS: TestRaft_BehindFollower (0.51s) +2025-01-25T14:39:00.128Z [INFO] server-74ef5429-11ab-b753-5e16-0f529093c362: pipelining replication: peer="{Voter server-3b42df68-ccbd-88ee-93b1-d2b5292e6e4d 3b42df68-ccbd-88ee-93b1-d2b5292e6e4d}" +2025-01-25T14:39:00.130Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:00.236Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:00.130719651 +0000 UTC m=+9.176558969 (1 nodes), 6.125µs from start of poll, 334.192973ms from cluster start. Timeout at 2025-01-25 14:39:00.235769527 +0000 UTC m=+9.281608845, 105.049876ms after stability +2025-01-25T14:39:00.236Z [INFO] server-74ef5429-11ab-b753-5e16-0f529093c362: aborting pipeline replication: peer="{Voter server-e79eb5cd-803a-7fa6-5f41-5c0a15bd8ef0 e79eb5cd-803a-7fa6-5f41-5c0a15bd8ef0}" +2025-01-25T14:39:00.237Z [INFO] server-74ef5429-11ab-b753-5e16-0f529093c362: aborting pipeline replication: peer="{Voter server-3b42df68-ccbd-88ee-93b1-d2b5292e6e4d 3b42df68-ccbd-88ee-93b1-d2b5292e6e4d}" +2025-01-25T14:39:00.738Z [ERROR] server-74ef5429-11ab-b753-5e16-0f529093c362: failed to heartbeat to: peer=3b42df68-ccbd-88ee-93b1-d2b5292e6e4d backoff time=10ms error="command timed out" +--- PASS: TestRaft_BehindFollower (0.95s) === RUN TestRaft_ApplyNonLeader -2025-01-25T14:28:18.164Z [INFO] server-07a949fa-857e-58dc-6852-8e2424b37ceb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-07a949fa-857e-58dc-6852-8e2424b37ceb Address:07a949fa-857e-58dc-6852-8e2424b37ceb} {Suffrage:Voter ID:server-9e32af90-93ed-156e-02b6-5267982b6d22 Address:9e32af90-93ed-156e-02b6-5267982b6d22} {Suffrage:Voter ID:server-b9d6c7d3-7840-3f7d-50ba-a0ff69e55a71 Address:b9d6c7d3-7840-3f7d-50ba-a0ff69e55a71}]" -2025-01-25T14:28:18.164Z [INFO] server-07a949fa-857e-58dc-6852-8e2424b37ceb: entering follower state: follower="Node at 07a949fa-857e-58dc-6852-8e2424b37ceb [Follower]" leader-address= leader-id= -2025-01-25T14:28:18.165Z [INFO] server-9e32af90-93ed-156e-02b6-5267982b6d22: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-07a949fa-857e-58dc-6852-8e2424b37ceb Address:07a949fa-857e-58dc-6852-8e2424b37ceb} {Suffrage:Voter ID:server-9e32af90-93ed-156e-02b6-5267982b6d22 Address:9e32af90-93ed-156e-02b6-5267982b6d22} {Suffrage:Voter ID:server-b9d6c7d3-7840-3f7d-50ba-a0ff69e55a71 Address:b9d6c7d3-7840-3f7d-50ba-a0ff69e55a71}]" -2025-01-25T14:28:18.165Z [INFO] server-b9d6c7d3-7840-3f7d-50ba-a0ff69e55a71: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-07a949fa-857e-58dc-6852-8e2424b37ceb Address:07a949fa-857e-58dc-6852-8e2424b37ceb} {Suffrage:Voter ID:server-9e32af90-93ed-156e-02b6-5267982b6d22 Address:9e32af90-93ed-156e-02b6-5267982b6d22} {Suffrage:Voter ID:server-b9d6c7d3-7840-3f7d-50ba-a0ff69e55a71 Address:b9d6c7d3-7840-3f7d-50ba-a0ff69e55a71}]" -2025-01-25T14:28:18.165Z [INFO] server-9e32af90-93ed-156e-02b6-5267982b6d22: entering follower state: follower="Node at 9e32af90-93ed-156e-02b6-5267982b6d22 [Follower]" leader-address= leader-id= -2025-01-25T14:28:18.165Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:18.165Z [INFO] server-b9d6c7d3-7840-3f7d-50ba-a0ff69e55a71: entering follower state: follower="Node at b9d6c7d3-7840-3f7d-50ba-a0ff69e55a71 [Follower]" leader-address= leader-id= -2025-01-25T14:28:18.223Z [WARN] server-b9d6c7d3-7840-3f7d-50ba-a0ff69e55a71: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:18.223Z [INFO] server-b9d6c7d3-7840-3f7d-50ba-a0ff69e55a71: entering candidate state: node="Node at b9d6c7d3-7840-3f7d-50ba-a0ff69e55a71 [Candidate]" term=2 -2025-01-25T14:28:18.223Z [INFO] server-b9d6c7d3-7840-3f7d-50ba-a0ff69e55a71: election won: term=2 tally=2 -2025-01-25T14:28:18.223Z [INFO] server-b9d6c7d3-7840-3f7d-50ba-a0ff69e55a71: entering leader state: leader="Node at b9d6c7d3-7840-3f7d-50ba-a0ff69e55a71 [Leader]" -2025-01-25T14:28:18.223Z [INFO] server-b9d6c7d3-7840-3f7d-50ba-a0ff69e55a71: added peer, starting replication: peer=server-07a949fa-857e-58dc-6852-8e2424b37ceb -2025-01-25T14:28:18.223Z [INFO] server-b9d6c7d3-7840-3f7d-50ba-a0ff69e55a71: added peer, starting replication: peer=server-9e32af90-93ed-156e-02b6-5267982b6d22 -2025-01-25T14:28:18.223Z [INFO] server-b9d6c7d3-7840-3f7d-50ba-a0ff69e55a71: pipelining replication: peer="{Voter server-9e32af90-93ed-156e-02b6-5267982b6d22 9e32af90-93ed-156e-02b6-5267982b6d22}" -2025-01-25T14:28:18.223Z [INFO] server-b9d6c7d3-7840-3f7d-50ba-a0ff69e55a71: pipelining replication: peer="{Voter server-07a949fa-857e-58dc-6852-8e2424b37ceb 07a949fa-857e-58dc-6852-8e2424b37ceb}" -2025-01-25T14:28:18.329Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:18.223870374 +0000 UTC m=+8.078195356 (1 nodes), 57.872278ms from start of poll, 60.021761ms from cluster start. Timeout at 2025-01-25 14:28:18.328872814 +0000 UTC m=+8.183197756, 105.0024ms after stability -2025-01-25T14:28:18.330Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:28:18.435Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:28:18.330051873 +0000 UTC m=+8.184376855 (2 nodes), 2.84µs from start of poll, 166.20326ms from cluster start. Timeout at 2025-01-25 14:28:18.435085432 +0000 UTC m=+8.289410414, 105.033559ms after stability -2025-01-25T14:28:18.436Z [INFO] server-b9d6c7d3-7840-3f7d-50ba-a0ff69e55a71: aborting pipeline replication: peer="{Voter server-9e32af90-93ed-156e-02b6-5267982b6d22 9e32af90-93ed-156e-02b6-5267982b6d22}" -2025-01-25T14:28:18.436Z [INFO] server-b9d6c7d3-7840-3f7d-50ba-a0ff69e55a71: aborting pipeline replication: peer="{Voter server-07a949fa-857e-58dc-6852-8e2424b37ceb 07a949fa-857e-58dc-6852-8e2424b37ceb}" ---- PASS: TestRaft_ApplyNonLeader (0.28s) +2025-01-25T14:39:00.744Z [INFO] server-e1765230-c7b0-e429-8bf9-b9bac595c2f9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e1765230-c7b0-e429-8bf9-b9bac595c2f9 Address:e1765230-c7b0-e429-8bf9-b9bac595c2f9} {Suffrage:Voter ID:server-bb1df234-6c52-6eaf-2b26-79f32b4b7d7d Address:bb1df234-6c52-6eaf-2b26-79f32b4b7d7d} {Suffrage:Voter ID:server-6a7f5812-2931-8f9c-1170-cf2aff8dd565 Address:6a7f5812-2931-8f9c-1170-cf2aff8dd565}]" +2025-01-25T14:39:00.745Z [INFO] server-e1765230-c7b0-e429-8bf9-b9bac595c2f9: entering follower state: follower="Node at e1765230-c7b0-e429-8bf9-b9bac595c2f9 [Follower]" leader-address= leader-id= +2025-01-25T14:39:00.745Z [INFO] server-bb1df234-6c52-6eaf-2b26-79f32b4b7d7d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e1765230-c7b0-e429-8bf9-b9bac595c2f9 Address:e1765230-c7b0-e429-8bf9-b9bac595c2f9} {Suffrage:Voter ID:server-bb1df234-6c52-6eaf-2b26-79f32b4b7d7d Address:bb1df234-6c52-6eaf-2b26-79f32b4b7d7d} {Suffrage:Voter ID:server-6a7f5812-2931-8f9c-1170-cf2aff8dd565 Address:6a7f5812-2931-8f9c-1170-cf2aff8dd565}]" +2025-01-25T14:39:00.745Z [INFO] server-bb1df234-6c52-6eaf-2b26-79f32b4b7d7d: entering follower state: follower="Node at bb1df234-6c52-6eaf-2b26-79f32b4b7d7d [Follower]" leader-address= leader-id= +2025-01-25T14:39:00.746Z [INFO] server-6a7f5812-2931-8f9c-1170-cf2aff8dd565: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e1765230-c7b0-e429-8bf9-b9bac595c2f9 Address:e1765230-c7b0-e429-8bf9-b9bac595c2f9} {Suffrage:Voter ID:server-bb1df234-6c52-6eaf-2b26-79f32b4b7d7d Address:bb1df234-6c52-6eaf-2b26-79f32b4b7d7d} {Suffrage:Voter ID:server-6a7f5812-2931-8f9c-1170-cf2aff8dd565 Address:6a7f5812-2931-8f9c-1170-cf2aff8dd565}]" +2025-01-25T14:39:00.746Z [INFO] server-6a7f5812-2931-8f9c-1170-cf2aff8dd565: entering follower state: follower="Node at 6a7f5812-2931-8f9c-1170-cf2aff8dd565 [Follower]" leader-address= leader-id= +2025-01-25T14:39:00.746Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:00.810Z [WARN] server-bb1df234-6c52-6eaf-2b26-79f32b4b7d7d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:00.810Z [INFO] server-bb1df234-6c52-6eaf-2b26-79f32b4b7d7d: entering candidate state: node="Node at bb1df234-6c52-6eaf-2b26-79f32b4b7d7d [Candidate]" term=2 +2025-01-25T14:39:00.810Z [INFO] server-bb1df234-6c52-6eaf-2b26-79f32b4b7d7d: election won: term=2 tally=2 +2025-01-25T14:39:00.810Z [INFO] server-bb1df234-6c52-6eaf-2b26-79f32b4b7d7d: entering leader state: leader="Node at bb1df234-6c52-6eaf-2b26-79f32b4b7d7d [Leader]" +2025-01-25T14:39:00.811Z [INFO] server-bb1df234-6c52-6eaf-2b26-79f32b4b7d7d: added peer, starting replication: peer=server-e1765230-c7b0-e429-8bf9-b9bac595c2f9 +2025-01-25T14:39:00.811Z [INFO] server-bb1df234-6c52-6eaf-2b26-79f32b4b7d7d: added peer, starting replication: peer=server-6a7f5812-2931-8f9c-1170-cf2aff8dd565 +2025-01-25T14:39:00.811Z [INFO] server-bb1df234-6c52-6eaf-2b26-79f32b4b7d7d: pipelining replication: peer="{Voter server-e1765230-c7b0-e429-8bf9-b9bac595c2f9 e1765230-c7b0-e429-8bf9-b9bac595c2f9}" +2025-01-25T14:39:00.812Z [INFO] server-bb1df234-6c52-6eaf-2b26-79f32b4b7d7d: pipelining replication: peer="{Voter server-6a7f5812-2931-8f9c-1170-cf2aff8dd565 6a7f5812-2931-8f9c-1170-cf2aff8dd565}" +2025-01-25T14:39:00.916Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:00.811036132 +0000 UTC m=+9.856875450 (1 nodes), 64.051187ms from start of poll, 67.00457ms from cluster start. Timeout at 2025-01-25 14:39:00.916055383 +0000 UTC m=+9.961894700, 105.01925ms after stability +2025-01-25T14:39:00.916Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:39:01.022Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:39:00.916573689 +0000 UTC m=+9.962413007 (2 nodes), 7µs from start of poll, 172.542127ms from cluster start. Timeout at 2025-01-25 14:39:01.021625023 +0000 UTC m=+10.067464340, 105.051333ms after stability +2025-01-25T14:39:01.022Z [INFO] server-bb1df234-6c52-6eaf-2b26-79f32b4b7d7d: aborting pipeline replication: peer="{Voter server-6a7f5812-2931-8f9c-1170-cf2aff8dd565 6a7f5812-2931-8f9c-1170-cf2aff8dd565}" +2025-01-25T14:39:01.022Z [INFO] server-bb1df234-6c52-6eaf-2b26-79f32b4b7d7d: aborting pipeline replication: peer="{Voter server-e1765230-c7b0-e429-8bf9-b9bac595c2f9 e1765230-c7b0-e429-8bf9-b9bac595c2f9}" +2025-01-25T14:39:01.523Z [ERROR] server-bb1df234-6c52-6eaf-2b26-79f32b4b7d7d: failed to heartbeat to: peer=6a7f5812-2931-8f9c-1170-cf2aff8dd565 backoff time=10ms error="command timed out" +--- PASS: TestRaft_ApplyNonLeader (0.79s) === RUN TestRaft_ApplyConcurrent -2025-01-25T14:28:18.438Z [INFO] server-6af0f32e-f7a7-263c-a0bc-632980e4ab2b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6af0f32e-f7a7-263c-a0bc-632980e4ab2b Address:6af0f32e-f7a7-263c-a0bc-632980e4ab2b} {Suffrage:Voter ID:server-67949996-9e51-fe6a-f647-52f1a2f95191 Address:67949996-9e51-fe6a-f647-52f1a2f95191} {Suffrage:Voter ID:server-2d8e96d3-58c2-81f2-c49a-8240ce5df7d8 Address:2d8e96d3-58c2-81f2-c49a-8240ce5df7d8}]" -2025-01-25T14:28:18.438Z [INFO] server-67949996-9e51-fe6a-f647-52f1a2f95191: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6af0f32e-f7a7-263c-a0bc-632980e4ab2b Address:6af0f32e-f7a7-263c-a0bc-632980e4ab2b} {Suffrage:Voter ID:server-67949996-9e51-fe6a-f647-52f1a2f95191 Address:67949996-9e51-fe6a-f647-52f1a2f95191} {Suffrage:Voter ID:server-2d8e96d3-58c2-81f2-c49a-8240ce5df7d8 Address:2d8e96d3-58c2-81f2-c49a-8240ce5df7d8}]" -2025-01-25T14:28:18.438Z [INFO] server-6af0f32e-f7a7-263c-a0bc-632980e4ab2b: entering follower state: follower="Node at 6af0f32e-f7a7-263c-a0bc-632980e4ab2b [Follower]" leader-address= leader-id= -2025-01-25T14:28:18.438Z [INFO] server-67949996-9e51-fe6a-f647-52f1a2f95191: entering follower state: follower="Node at 67949996-9e51-fe6a-f647-52f1a2f95191 [Follower]" leader-address= leader-id= -2025-01-25T14:28:18.438Z [INFO] server-2d8e96d3-58c2-81f2-c49a-8240ce5df7d8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6af0f32e-f7a7-263c-a0bc-632980e4ab2b Address:6af0f32e-f7a7-263c-a0bc-632980e4ab2b} {Suffrage:Voter ID:server-67949996-9e51-fe6a-f647-52f1a2f95191 Address:67949996-9e51-fe6a-f647-52f1a2f95191} {Suffrage:Voter ID:server-2d8e96d3-58c2-81f2-c49a-8240ce5df7d8 Address:2d8e96d3-58c2-81f2-c49a-8240ce5df7d8}]" -2025-01-25T14:28:18.438Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:18.438Z [INFO] server-2d8e96d3-58c2-81f2-c49a-8240ce5df7d8: entering follower state: follower="Node at 2d8e96d3-58c2-81f2-c49a-8240ce5df7d8 [Follower]" leader-address= leader-id= -2025-01-25T14:28:18.597Z [WARN] server-6af0f32e-f7a7-263c-a0bc-632980e4ab2b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:18.597Z [INFO] server-6af0f32e-f7a7-263c-a0bc-632980e4ab2b: entering candidate state: node="Node at 6af0f32e-f7a7-263c-a0bc-632980e4ab2b [Candidate]" term=2 -2025-01-25T14:28:18.597Z [INFO] server-6af0f32e-f7a7-263c-a0bc-632980e4ab2b: election won: term=2 tally=2 -2025-01-25T14:28:18.597Z [INFO] server-6af0f32e-f7a7-263c-a0bc-632980e4ab2b: entering leader state: leader="Node at 6af0f32e-f7a7-263c-a0bc-632980e4ab2b [Leader]" -2025-01-25T14:28:18.597Z [INFO] server-6af0f32e-f7a7-263c-a0bc-632980e4ab2b: added peer, starting replication: peer=server-67949996-9e51-fe6a-f647-52f1a2f95191 -2025-01-25T14:28:18.597Z [INFO] server-6af0f32e-f7a7-263c-a0bc-632980e4ab2b: added peer, starting replication: peer=server-2d8e96d3-58c2-81f2-c49a-8240ce5df7d8 -2025-01-25T14:28:18.597Z [INFO] server-6af0f32e-f7a7-263c-a0bc-632980e4ab2b: pipelining replication: peer="{Voter server-67949996-9e51-fe6a-f647-52f1a2f95191 67949996-9e51-fe6a-f647-52f1a2f95191}" -2025-01-25T14:28:18.597Z [INFO] server-6af0f32e-f7a7-263c-a0bc-632980e4ab2b: pipelining replication: peer="{Voter server-2d8e96d3-58c2-81f2-c49a-8240ce5df7d8 2d8e96d3-58c2-81f2-c49a-8240ce5df7d8}" -2025-01-25T14:28:18.803Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:18.597715561 +0000 UTC m=+8.452040543 (1 nodes), 158.852324ms from start of poll, 159.52102ms from cluster start. Timeout at 2025-01-25 14:28:18.802719001 +0000 UTC m=+8.657043943, 205.0034ms after stability -2025-01-25T14:28:18.818Z [INFO] server-6af0f32e-f7a7-263c-a0bc-632980e4ab2b: aborting pipeline replication: peer="{Voter server-2d8e96d3-58c2-81f2-c49a-8240ce5df7d8 2d8e96d3-58c2-81f2-c49a-8240ce5df7d8}" -2025-01-25T14:28:18.818Z [INFO] server-6af0f32e-f7a7-263c-a0bc-632980e4ab2b: aborting pipeline replication: peer="{Voter server-67949996-9e51-fe6a-f647-52f1a2f95191 67949996-9e51-fe6a-f647-52f1a2f95191}" ---- PASS: TestRaft_ApplyConcurrent (0.38s) +2025-01-25T14:39:01.533Z [INFO] server-d59f7b24-3c6c-909b-327c-fd1e82669865: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d59f7b24-3c6c-909b-327c-fd1e82669865 Address:d59f7b24-3c6c-909b-327c-fd1e82669865} {Suffrage:Voter ID:server-95bba6d2-1ae7-2c95-0378-ba5a8ffa273e Address:95bba6d2-1ae7-2c95-0378-ba5a8ffa273e} {Suffrage:Voter ID:server-3c5ed3ca-0d80-5357-010d-9e00e3b732a6 Address:3c5ed3ca-0d80-5357-010d-9e00e3b732a6}]" +2025-01-25T14:39:01.533Z [INFO] server-d59f7b24-3c6c-909b-327c-fd1e82669865: entering follower state: follower="Node at d59f7b24-3c6c-909b-327c-fd1e82669865 [Follower]" leader-address= leader-id= +2025-01-25T14:39:01.534Z [INFO] server-95bba6d2-1ae7-2c95-0378-ba5a8ffa273e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d59f7b24-3c6c-909b-327c-fd1e82669865 Address:d59f7b24-3c6c-909b-327c-fd1e82669865} {Suffrage:Voter ID:server-95bba6d2-1ae7-2c95-0378-ba5a8ffa273e Address:95bba6d2-1ae7-2c95-0378-ba5a8ffa273e} {Suffrage:Voter ID:server-3c5ed3ca-0d80-5357-010d-9e00e3b732a6 Address:3c5ed3ca-0d80-5357-010d-9e00e3b732a6}]" +2025-01-25T14:39:01.534Z [INFO] server-95bba6d2-1ae7-2c95-0378-ba5a8ffa273e: entering follower state: follower="Node at 95bba6d2-1ae7-2c95-0378-ba5a8ffa273e [Follower]" leader-address= leader-id= +2025-01-25T14:39:01.535Z [INFO] server-3c5ed3ca-0d80-5357-010d-9e00e3b732a6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d59f7b24-3c6c-909b-327c-fd1e82669865 Address:d59f7b24-3c6c-909b-327c-fd1e82669865} {Suffrage:Voter ID:server-95bba6d2-1ae7-2c95-0378-ba5a8ffa273e Address:95bba6d2-1ae7-2c95-0378-ba5a8ffa273e} {Suffrage:Voter ID:server-3c5ed3ca-0d80-5357-010d-9e00e3b732a6 Address:3c5ed3ca-0d80-5357-010d-9e00e3b732a6}]" +2025-01-25T14:39:01.535Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:01.535Z [INFO] server-3c5ed3ca-0d80-5357-010d-9e00e3b732a6: entering follower state: follower="Node at 3c5ed3ca-0d80-5357-010d-9e00e3b732a6 [Follower]" leader-address= leader-id= +2025-01-25T14:39:01.716Z [WARN] server-95bba6d2-1ae7-2c95-0378-ba5a8ffa273e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:01.716Z [INFO] server-95bba6d2-1ae7-2c95-0378-ba5a8ffa273e: entering candidate state: node="Node at 95bba6d2-1ae7-2c95-0378-ba5a8ffa273e [Candidate]" term=2 +2025-01-25T14:39:01.716Z [INFO] server-95bba6d2-1ae7-2c95-0378-ba5a8ffa273e: election won: term=2 tally=2 +2025-01-25T14:39:01.717Z [INFO] server-95bba6d2-1ae7-2c95-0378-ba5a8ffa273e: entering leader state: leader="Node at 95bba6d2-1ae7-2c95-0378-ba5a8ffa273e [Leader]" +2025-01-25T14:39:01.717Z [INFO] server-95bba6d2-1ae7-2c95-0378-ba5a8ffa273e: added peer, starting replication: peer=server-d59f7b24-3c6c-909b-327c-fd1e82669865 +2025-01-25T14:39:01.717Z [INFO] server-95bba6d2-1ae7-2c95-0378-ba5a8ffa273e: added peer, starting replication: peer=server-3c5ed3ca-0d80-5357-010d-9e00e3b732a6 +2025-01-25T14:39:01.717Z [INFO] server-95bba6d2-1ae7-2c95-0378-ba5a8ffa273e: pipelining replication: peer="{Voter server-d59f7b24-3c6c-909b-327c-fd1e82669865 d59f7b24-3c6c-909b-327c-fd1e82669865}" +2025-01-25T14:39:01.717Z [INFO] server-95bba6d2-1ae7-2c95-0378-ba5a8ffa273e: pipelining replication: peer="{Voter server-3c5ed3ca-0d80-5357-010d-9e00e3b732a6 3c5ed3ca-0d80-5357-010d-9e00e3b732a6}" +2025-01-25T14:39:01.922Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:01.717044311 +0000 UTC m=+10.762883629 (1 nodes), 181.578487ms from start of poll, 184.127953ms from cluster start. Timeout at 2025-01-25 14:39:01.922059478 +0000 UTC m=+10.967898504, 205.014875ms after stability +2025-01-25T14:39:01.939Z [INFO] server-95bba6d2-1ae7-2c95-0378-ba5a8ffa273e: aborting pipeline replication: peer="{Voter server-d59f7b24-3c6c-909b-327c-fd1e82669865 d59f7b24-3c6c-909b-327c-fd1e82669865}" +2025-01-25T14:39:01.939Z [INFO] server-95bba6d2-1ae7-2c95-0378-ba5a8ffa273e: aborting pipeline replication: peer="{Voter server-3c5ed3ca-0d80-5357-010d-9e00e3b732a6 3c5ed3ca-0d80-5357-010d-9e00e3b732a6}" +--- PASS: TestRaft_ApplyConcurrent (0.41s) === RUN TestRaft_ApplyConcurrent_Timeout -2025-01-25T14:28:18.819Z [INFO] server-0754073f-4eb0-1938-6b44-664ad4d119cf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0754073f-4eb0-1938-6b44-664ad4d119cf Address:0754073f-4eb0-1938-6b44-664ad4d119cf}]" -2025-01-25T14:28:18.820Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:18.820Z [INFO] server-0754073f-4eb0-1938-6b44-664ad4d119cf: entering follower state: follower="Node at 0754073f-4eb0-1938-6b44-664ad4d119cf [Follower]" leader-address= leader-id= -2025-01-25T14:28:18.934Z [WARN] server-0754073f-4eb0-1938-6b44-664ad4d119cf: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:18.934Z [INFO] server-0754073f-4eb0-1938-6b44-664ad4d119cf: entering candidate state: node="Node at 0754073f-4eb0-1938-6b44-664ad4d119cf [Candidate]" term=2 -2025-01-25T14:28:18.934Z [INFO] server-0754073f-4eb0-1938-6b44-664ad4d119cf: election won: term=2 tally=1 -2025-01-25T14:28:18.934Z [INFO] server-0754073f-4eb0-1938-6b44-664ad4d119cf: entering leader state: leader="Node at 0754073f-4eb0-1938-6b44-664ad4d119cf [Leader]" -2025-01-25T14:28:19.136Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:18.934531001 +0000 UTC m=+8.788855983 (1 nodes), 114.474841ms from start of poll, 114.77503ms from cluster start. Timeout at 2025-01-25 14:28:19.135536201 +0000 UTC m=+8.989861143, 201.00516ms after stability +2025-01-25T14:39:01.945Z [INFO] server-521190dc-567b-fbef-33af-12960fac815c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-521190dc-567b-fbef-33af-12960fac815c Address:521190dc-567b-fbef-33af-12960fac815c}]" +2025-01-25T14:39:01.945Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:01.945Z [INFO] server-521190dc-567b-fbef-33af-12960fac815c: entering follower state: follower="Node at 521190dc-567b-fbef-33af-12960fac815c [Follower]" leader-address= leader-id= +2025-01-25T14:39:02.058Z [WARN] server-521190dc-567b-fbef-33af-12960fac815c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:02.058Z [INFO] server-521190dc-567b-fbef-33af-12960fac815c: entering candidate state: node="Node at 521190dc-567b-fbef-33af-12960fac815c [Candidate]" term=2 +2025-01-25T14:39:02.059Z [INFO] server-521190dc-567b-fbef-33af-12960fac815c: election won: term=2 tally=1 +2025-01-25T14:39:02.059Z [INFO] server-521190dc-567b-fbef-33af-12960fac815c: entering leader state: leader="Node at 521190dc-567b-fbef-33af-12960fac815c [Leader]" +2025-01-25T14:39:02.261Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:02.059257891 +0000 UTC m=+11.105097209 (1 nodes), 113.796206ms from start of poll, 114.758709ms from cluster start. Timeout at 2025-01-25 14:39:02.260289391 +0000 UTC m=+11.306128709, 201.0315ms after stability --- PASS: TestRaft_ApplyConcurrent_Timeout (0.32s) === RUN TestRaft_JoinNode -2025-01-25T14:28:19.138Z [INFO] server-0c9b1edb-7cf4-7768-93b3-f988d8cdb637: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0c9b1edb-7cf4-7768-93b3-f988d8cdb637 Address:0c9b1edb-7cf4-7768-93b3-f988d8cdb637} {Suffrage:Voter ID:server-0b1e82db-67f2-3ee1-2ebd-8a9e60a63702 Address:0b1e82db-67f2-3ee1-2ebd-8a9e60a63702}]" -2025-01-25T14:28:19.139Z [INFO] server-0c9b1edb-7cf4-7768-93b3-f988d8cdb637: entering follower state: follower="Node at 0c9b1edb-7cf4-7768-93b3-f988d8cdb637 [Follower]" leader-address= leader-id= -2025-01-25T14:28:19.139Z [INFO] server-0b1e82db-67f2-3ee1-2ebd-8a9e60a63702: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0c9b1edb-7cf4-7768-93b3-f988d8cdb637 Address:0c9b1edb-7cf4-7768-93b3-f988d8cdb637} {Suffrage:Voter ID:server-0b1e82db-67f2-3ee1-2ebd-8a9e60a63702 Address:0b1e82db-67f2-3ee1-2ebd-8a9e60a63702}]" -2025-01-25T14:28:19.139Z [INFO] server-0b1e82db-67f2-3ee1-2ebd-8a9e60a63702: entering follower state: follower="Node at 0b1e82db-67f2-3ee1-2ebd-8a9e60a63702 [Follower]" leader-address= leader-id= -2025-01-25T14:28:19.139Z [INFO] server-37ee8ec6-3b94-752c-20f1-4b78d967bdb6: initial configuration: index=0 servers=[] -2025-01-25T14:28:19.139Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:19.140Z [INFO] server-37ee8ec6-3b94-752c-20f1-4b78d967bdb6: entering follower state: follower="Node at 37ee8ec6-3b94-752c-20f1-4b78d967bdb6 [Follower]" leader-address= leader-id= -2025-01-25T14:28:19.198Z [WARN] server-37ee8ec6-3b94-752c-20f1-4b78d967bdb6: no known peers, aborting election -2025-01-25T14:28:19.205Z [WARN] server-0c9b1edb-7cf4-7768-93b3-f988d8cdb637: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:19.205Z [INFO] server-0c9b1edb-7cf4-7768-93b3-f988d8cdb637: entering candidate state: node="Node at 0c9b1edb-7cf4-7768-93b3-f988d8cdb637 [Candidate]" term=2 -2025-01-25T14:28:19.205Z [INFO] server-0c9b1edb-7cf4-7768-93b3-f988d8cdb637: election won: term=2 tally=2 -2025-01-25T14:28:19.205Z [INFO] server-0c9b1edb-7cf4-7768-93b3-f988d8cdb637: entering leader state: leader="Node at 0c9b1edb-7cf4-7768-93b3-f988d8cdb637 [Leader]" -2025-01-25T14:28:19.205Z [INFO] server-0c9b1edb-7cf4-7768-93b3-f988d8cdb637: added peer, starting replication: peer=server-0b1e82db-67f2-3ee1-2ebd-8a9e60a63702 -2025-01-25T14:28:19.205Z [INFO] server-0c9b1edb-7cf4-7768-93b3-f988d8cdb637: pipelining replication: peer="{Voter server-0b1e82db-67f2-3ee1-2ebd-8a9e60a63702 0b1e82db-67f2-3ee1-2ebd-8a9e60a63702}" -2025-01-25T14:28:19.311Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:19.206024352 +0000 UTC m=+9.060349334 (1 nodes), 66.010225ms from start of poll, 67.235261ms from cluster start. Timeout at 2025-01-25 14:28:19.311028432 +0000 UTC m=+9.165353414, 105.00408ms after stability -2025-01-25T14:28:19.311Z [INFO] server-0c9b1edb-7cf4-7768-93b3-f988d8cdb637: updating configuration: command=AddVoter server-id=server-37ee8ec6-3b94-752c-20f1-4b78d967bdb6 server-addr=37ee8ec6-3b94-752c-20f1-4b78d967bdb6 servers="[{Suffrage:Voter ID:server-0c9b1edb-7cf4-7768-93b3-f988d8cdb637 Address:0c9b1edb-7cf4-7768-93b3-f988d8cdb637} {Suffrage:Voter ID:server-0b1e82db-67f2-3ee1-2ebd-8a9e60a63702 Address:0b1e82db-67f2-3ee1-2ebd-8a9e60a63702} {Suffrage:Voter ID:server-37ee8ec6-3b94-752c-20f1-4b78d967bdb6 Address:37ee8ec6-3b94-752c-20f1-4b78d967bdb6}]" -2025-01-25T14:28:19.311Z [INFO] server-0c9b1edb-7cf4-7768-93b3-f988d8cdb637: added peer, starting replication: peer=server-37ee8ec6-3b94-752c-20f1-4b78d967bdb6 -2025-01-25T14:28:19.311Z [WARN] server-37ee8ec6-3b94-752c-20f1-4b78d967bdb6: failed to get previous log: previous-index=3 last-index=0 error="log not found" -2025-01-25T14:28:19.311Z [WARN] server-0c9b1edb-7cf4-7768-93b3-f988d8cdb637: appendEntries rejected, sending older logs: peer="{Voter server-37ee8ec6-3b94-752c-20f1-4b78d967bdb6 37ee8ec6-3b94-752c-20f1-4b78d967bdb6}" next=1 -2025-01-25T14:28:19.312Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:19.312Z [INFO] server-0c9b1edb-7cf4-7768-93b3-f988d8cdb637: pipelining replication: peer="{Voter server-37ee8ec6-3b94-752c-20f1-4b78d967bdb6 37ee8ec6-3b94-752c-20f1-4b78d967bdb6}" -2025-01-25T14:28:19.417Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:19.312204811 +0000 UTC m=+9.166529793 (1 nodes), 1.32µs from start of poll, 173.41572ms from cluster start. Timeout at 2025-01-25 14:28:19.41722325 +0000 UTC m=+9.271548232, 105.018439ms after stability -2025-01-25T14:28:19.417Z [INFO] server-0c9b1edb-7cf4-7768-93b3-f988d8cdb637: aborting pipeline replication: peer="{Voter server-37ee8ec6-3b94-752c-20f1-4b78d967bdb6 37ee8ec6-3b94-752c-20f1-4b78d967bdb6}" -2025-01-25T14:28:19.417Z [INFO] server-0c9b1edb-7cf4-7768-93b3-f988d8cdb637: aborting pipeline replication: peer="{Voter server-0b1e82db-67f2-3ee1-2ebd-8a9e60a63702 0b1e82db-67f2-3ee1-2ebd-8a9e60a63702}" ---- PASS: TestRaft_JoinNode (0.28s) +2025-01-25T14:39:02.266Z [INFO] server-6affbeef-5d75-884b-4d90-1a662c507a7d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6affbeef-5d75-884b-4d90-1a662c507a7d Address:6affbeef-5d75-884b-4d90-1a662c507a7d} {Suffrage:Voter ID:server-571c0b5c-2e94-a242-26b1-849556b8c04c Address:571c0b5c-2e94-a242-26b1-849556b8c04c}]" +2025-01-25T14:39:02.266Z [INFO] server-6affbeef-5d75-884b-4d90-1a662c507a7d: entering follower state: follower="Node at 6affbeef-5d75-884b-4d90-1a662c507a7d [Follower]" leader-address= leader-id= +2025-01-25T14:39:02.266Z [INFO] server-571c0b5c-2e94-a242-26b1-849556b8c04c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6affbeef-5d75-884b-4d90-1a662c507a7d Address:6affbeef-5d75-884b-4d90-1a662c507a7d} {Suffrage:Voter ID:server-571c0b5c-2e94-a242-26b1-849556b8c04c Address:571c0b5c-2e94-a242-26b1-849556b8c04c}]" +2025-01-25T14:39:02.266Z [INFO] server-571c0b5c-2e94-a242-26b1-849556b8c04c: entering follower state: follower="Node at 571c0b5c-2e94-a242-26b1-849556b8c04c [Follower]" leader-address= leader-id= +2025-01-25T14:39:02.267Z [INFO] server-6b689fff-e4f8-e792-098f-ba7d8e33ea32: initial configuration: index=0 servers=[] +2025-01-25T14:39:02.268Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:02.268Z [INFO] server-6b689fff-e4f8-e792-098f-ba7d8e33ea32: entering follower state: follower="Node at 6b689fff-e4f8-e792-098f-ba7d8e33ea32 [Follower]" leader-address= leader-id= +2025-01-25T14:39:02.330Z [WARN] server-6affbeef-5d75-884b-4d90-1a662c507a7d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:02.331Z [INFO] server-6affbeef-5d75-884b-4d90-1a662c507a7d: entering candidate state: node="Node at 6affbeef-5d75-884b-4d90-1a662c507a7d [Candidate]" term=2 +2025-01-25T14:39:02.331Z [INFO] server-6affbeef-5d75-884b-4d90-1a662c507a7d: election won: term=2 tally=2 +2025-01-25T14:39:02.331Z [INFO] server-6affbeef-5d75-884b-4d90-1a662c507a7d: entering leader state: leader="Node at 6affbeef-5d75-884b-4d90-1a662c507a7d [Leader]" +2025-01-25T14:39:02.331Z [INFO] server-6affbeef-5d75-884b-4d90-1a662c507a7d: added peer, starting replication: peer=server-571c0b5c-2e94-a242-26b1-849556b8c04c +2025-01-25T14:39:02.332Z [INFO] server-6affbeef-5d75-884b-4d90-1a662c507a7d: pipelining replication: peer="{Voter server-571c0b5c-2e94-a242-26b1-849556b8c04c 571c0b5c-2e94-a242-26b1-849556b8c04c}" +2025-01-25T14:39:02.345Z [WARN] server-6b689fff-e4f8-e792-098f-ba7d8e33ea32: no known peers, aborting election +2025-01-25T14:39:02.437Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:02.331902852 +0000 UTC m=+11.377742169 (1 nodes), 63.76206ms from start of poll, 66.038234ms from cluster start. Timeout at 2025-01-25 14:39:02.436915977 +0000 UTC m=+11.482755295, 105.013126ms after stability +2025-01-25T14:39:02.437Z [INFO] server-6affbeef-5d75-884b-4d90-1a662c507a7d: updating configuration: command=AddVoter server-id=server-6b689fff-e4f8-e792-098f-ba7d8e33ea32 server-addr=6b689fff-e4f8-e792-098f-ba7d8e33ea32 servers="[{Suffrage:Voter ID:server-6affbeef-5d75-884b-4d90-1a662c507a7d Address:6affbeef-5d75-884b-4d90-1a662c507a7d} {Suffrage:Voter ID:server-571c0b5c-2e94-a242-26b1-849556b8c04c Address:571c0b5c-2e94-a242-26b1-849556b8c04c} {Suffrage:Voter ID:server-6b689fff-e4f8-e792-098f-ba7d8e33ea32 Address:6b689fff-e4f8-e792-098f-ba7d8e33ea32}]" +2025-01-25T14:39:02.438Z [INFO] server-6affbeef-5d75-884b-4d90-1a662c507a7d: added peer, starting replication: peer=server-6b689fff-e4f8-e792-098f-ba7d8e33ea32 +2025-01-25T14:39:02.438Z [WARN] server-6b689fff-e4f8-e792-098f-ba7d8e33ea32: failed to get previous log: previous-index=3 last-index=0 error="log not found" +2025-01-25T14:39:02.438Z [WARN] server-6affbeef-5d75-884b-4d90-1a662c507a7d: appendEntries rejected, sending older logs: peer="{Voter server-6b689fff-e4f8-e792-098f-ba7d8e33ea32 6b689fff-e4f8-e792-098f-ba7d8e33ea32}" next=1 +2025-01-25T14:39:02.439Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:02.439Z [INFO] server-6affbeef-5d75-884b-4d90-1a662c507a7d: pipelining replication: peer="{Voter server-6b689fff-e4f8-e792-098f-ba7d8e33ea32 6b689fff-e4f8-e792-098f-ba7d8e33ea32}" +2025-01-25T14:39:02.545Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:02.439233289 +0000 UTC m=+11.485072607 (1 nodes), 4.959µs from start of poll, 173.368672ms from cluster start. Timeout at 2025-01-25 14:39:02.544274414 +0000 UTC m=+11.590113732, 105.041125ms after stability +2025-01-25T14:39:02.545Z [INFO] server-6affbeef-5d75-884b-4d90-1a662c507a7d: aborting pipeline replication: peer="{Voter server-6b689fff-e4f8-e792-098f-ba7d8e33ea32 6b689fff-e4f8-e792-098f-ba7d8e33ea32}" +2025-01-25T14:39:02.545Z [INFO] server-6affbeef-5d75-884b-4d90-1a662c507a7d: aborting pipeline replication: peer="{Voter server-571c0b5c-2e94-a242-26b1-849556b8c04c 571c0b5c-2e94-a242-26b1-849556b8c04c}" +--- PASS: TestRaft_JoinNode (0.29s) === RUN TestRaft_JoinNode_ConfigStore -2025-01-25T14:28:19.419Z [INFO] server-9faaba19-5697-0983-daae-aedf305f130d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9faaba19-5697-0983-daae-aedf305f130d Address:9faaba19-5697-0983-daae-aedf305f130d}]" -2025-01-25T14:28:19.419Z [INFO] server-9faaba19-5697-0983-daae-aedf305f130d: entering follower state: follower="Node at 9faaba19-5697-0983-daae-aedf305f130d [Follower]" leader-address= leader-id= -2025-01-25T14:28:19.420Z [INFO] server-7ece0a12-22fc-e71e-3ff5-2fb2f56d1aef: initial configuration: index=0 servers=[] -2025-01-25T14:28:19.420Z [INFO] server-7ece0a12-22fc-e71e-3ff5-2fb2f56d1aef: entering follower state: follower="Node at 7ece0a12-22fc-e71e-3ff5-2fb2f56d1aef [Follower]" leader-address= leader-id= -2025-01-25T14:28:19.420Z [INFO] server-4d9f8495-f09e-c8e0-a12d-9f809ef17a48: initial configuration: index=0 servers=[] -2025-01-25T14:28:19.420Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:19.420Z [INFO] server-4d9f8495-f09e-c8e0-a12d-9f809ef17a48: entering follower state: follower="Node at 4d9f8495-f09e-c8e0-a12d-9f809ef17a48 [Follower]" leader-address= leader-id= -2025-01-25T14:28:19.493Z [WARN] server-4d9f8495-f09e-c8e0-a12d-9f809ef17a48: no known peers, aborting election -2025-01-25T14:28:19.501Z [WARN] server-9faaba19-5697-0983-daae-aedf305f130d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:19.501Z [INFO] server-9faaba19-5697-0983-daae-aedf305f130d: entering candidate state: node="Node at 9faaba19-5697-0983-daae-aedf305f130d [Candidate]" term=2 -2025-01-25T14:28:19.501Z [INFO] server-9faaba19-5697-0983-daae-aedf305f130d: election won: term=2 tally=1 -2025-01-25T14:28:19.501Z [INFO] server-9faaba19-5697-0983-daae-aedf305f130d: entering leader state: leader="Node at 9faaba19-5697-0983-daae-aedf305f130d [Leader]" -2025-01-25T14:28:19.515Z [WARN] server-7ece0a12-22fc-e71e-3ff5-2fb2f56d1aef: no known peers, aborting election -2025-01-25T14:28:19.607Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:19.501948703 +0000 UTC m=+9.356273685 (1 nodes), 81.196278ms from start of poll, 82.348677ms from cluster start. Timeout at 2025-01-25 14:28:19.606952943 +0000 UTC m=+9.461277885, 105.0042ms after stability -2025-01-25T14:28:19.607Z [INFO] server-9faaba19-5697-0983-daae-aedf305f130d: updating configuration: command=AddVoter server-id=server-7ece0a12-22fc-e71e-3ff5-2fb2f56d1aef server-addr=7ece0a12-22fc-e71e-3ff5-2fb2f56d1aef servers="[{Suffrage:Voter ID:server-9faaba19-5697-0983-daae-aedf305f130d Address:9faaba19-5697-0983-daae-aedf305f130d} {Suffrage:Voter ID:server-7ece0a12-22fc-e71e-3ff5-2fb2f56d1aef Address:7ece0a12-22fc-e71e-3ff5-2fb2f56d1aef}]" -2025-01-25T14:28:19.607Z [INFO] server-9faaba19-5697-0983-daae-aedf305f130d: added peer, starting replication: peer=server-7ece0a12-22fc-e71e-3ff5-2fb2f56d1aef -2025-01-25T14:28:19.607Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:19.607Z [WARN] server-7ece0a12-22fc-e71e-3ff5-2fb2f56d1aef: failed to get previous log: previous-index=3 last-index=0 error="log not found" -2025-01-25T14:28:19.608Z [WARN] server-9faaba19-5697-0983-daae-aedf305f130d: appendEntries rejected, sending older logs: peer="{Voter server-7ece0a12-22fc-e71e-3ff5-2fb2f56d1aef 7ece0a12-22fc-e71e-3ff5-2fb2f56d1aef}" next=1 -2025-01-25T14:28:19.608Z [INFO] server-9faaba19-5697-0983-daae-aedf305f130d: pipelining replication: peer="{Voter server-7ece0a12-22fc-e71e-3ff5-2fb2f56d1aef 7ece0a12-22fc-e71e-3ff5-2fb2f56d1aef}" -2025-01-25T14:28:19.713Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:19.607864252 +0000 UTC m=+9.462189234 (1 nodes), 1.36µs from start of poll, 188.264226ms from cluster start. Timeout at 2025-01-25 14:28:19.712880772 +0000 UTC m=+9.567205834, 105.0166ms after stability -2025-01-25T14:28:19.713Z [INFO] server-9faaba19-5697-0983-daae-aedf305f130d: updating configuration: command=AddVoter server-id=server-4d9f8495-f09e-c8e0-a12d-9f809ef17a48 server-addr=4d9f8495-f09e-c8e0-a12d-9f809ef17a48 servers="[{Suffrage:Voter ID:server-9faaba19-5697-0983-daae-aedf305f130d Address:9faaba19-5697-0983-daae-aedf305f130d} {Suffrage:Voter ID:server-7ece0a12-22fc-e71e-3ff5-2fb2f56d1aef Address:7ece0a12-22fc-e71e-3ff5-2fb2f56d1aef} {Suffrage:Voter ID:server-4d9f8495-f09e-c8e0-a12d-9f809ef17a48 Address:4d9f8495-f09e-c8e0-a12d-9f809ef17a48}]" -2025-01-25T14:28:19.714Z [INFO] server-9faaba19-5697-0983-daae-aedf305f130d: added peer, starting replication: peer=server-4d9f8495-f09e-c8e0-a12d-9f809ef17a48 -2025-01-25T14:28:19.714Z [WARN] server-4d9f8495-f09e-c8e0-a12d-9f809ef17a48: failed to get previous log: previous-index=4 last-index=0 error="log not found" -2025-01-25T14:28:19.714Z [WARN] server-9faaba19-5697-0983-daae-aedf305f130d: appendEntries rejected, sending older logs: peer="{Voter server-4d9f8495-f09e-c8e0-a12d-9f809ef17a48 4d9f8495-f09e-c8e0-a12d-9f809ef17a48}" next=1 -2025-01-25T14:28:19.714Z [INFO] server-9faaba19-5697-0983-daae-aedf305f130d: pipelining replication: peer="{Voter server-4d9f8495-f09e-c8e0-a12d-9f809ef17a48 4d9f8495-f09e-c8e0-a12d-9f809ef17a48}" -2025-01-25T14:28:19.714Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:19.820Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:19.714872801 +0000 UTC m=+9.569197783 (1 nodes), 1.24µs from start of poll, 295.272775ms from cluster start. Timeout at 2025-01-25 14:28:19.819915759 +0000 UTC m=+9.674240741, 105.042958ms after stability -2025-01-25T14:28:19.820Z [INFO] server-9faaba19-5697-0983-daae-aedf305f130d: aborting pipeline replication: peer="{Voter server-7ece0a12-22fc-e71e-3ff5-2fb2f56d1aef 7ece0a12-22fc-e71e-3ff5-2fb2f56d1aef}" -2025-01-25T14:28:19.820Z [INFO] server-9faaba19-5697-0983-daae-aedf305f130d: aborting pipeline replication: peer="{Voter server-4d9f8495-f09e-c8e0-a12d-9f809ef17a48 4d9f8495-f09e-c8e0-a12d-9f809ef17a48}" ---- PASS: TestRaft_JoinNode_ConfigStore (0.40s) +2025-01-25T14:39:02.552Z [INFO] server-a8195adc-c85f-90e7-6f0a-377f7514be78: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a8195adc-c85f-90e7-6f0a-377f7514be78 Address:a8195adc-c85f-90e7-6f0a-377f7514be78}]" +2025-01-25T14:39:02.552Z [INFO] server-a8195adc-c85f-90e7-6f0a-377f7514be78: entering follower state: follower="Node at a8195adc-c85f-90e7-6f0a-377f7514be78 [Follower]" leader-address= leader-id= +2025-01-25T14:39:02.554Z [INFO] server-9fd14bd8-7a5c-70ee-5ffe-f6e1ce844aa9: initial configuration: index=0 servers=[] +2025-01-25T14:39:02.554Z [INFO] server-9fd14bd8-7a5c-70ee-5ffe-f6e1ce844aa9: entering follower state: follower="Node at 9fd14bd8-7a5c-70ee-5ffe-f6e1ce844aa9 [Follower]" leader-address= leader-id= +2025-01-25T14:39:02.556Z [INFO] server-459ac2c7-fc28-a807-9e04-7f4d98b2bbb6: initial configuration: index=0 servers=[] +2025-01-25T14:39:02.556Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:02.556Z [INFO] server-459ac2c7-fc28-a807-9e04-7f4d98b2bbb6: entering follower state: follower="Node at 459ac2c7-fc28-a807-9e04-7f4d98b2bbb6 [Follower]" leader-address= leader-id= +2025-01-25T14:39:02.621Z [WARN] server-9fd14bd8-7a5c-70ee-5ffe-f6e1ce844aa9: no known peers, aborting election +2025-01-25T14:39:02.624Z [WARN] server-a8195adc-c85f-90e7-6f0a-377f7514be78: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:02.624Z [INFO] server-a8195adc-c85f-90e7-6f0a-377f7514be78: entering candidate state: node="Node at a8195adc-c85f-90e7-6f0a-377f7514be78 [Candidate]" term=2 +2025-01-25T14:39:02.624Z [INFO] server-a8195adc-c85f-90e7-6f0a-377f7514be78: election won: term=2 tally=1 +2025-01-25T14:39:02.624Z [INFO] server-a8195adc-c85f-90e7-6f0a-377f7514be78: entering leader state: leader="Node at a8195adc-c85f-90e7-6f0a-377f7514be78 [Leader]" +2025-01-25T14:39:02.628Z [WARN] server-459ac2c7-fc28-a807-9e04-7f4d98b2bbb6: no known peers, aborting election +2025-01-25T14:39:02.730Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:02.624996663 +0000 UTC m=+11.670835981 (1 nodes), 68.457324ms from start of poll, 73.177088ms from cluster start. Timeout at 2025-01-25 14:39:02.730019413 +0000 UTC m=+11.775858439, 105.022458ms after stability +2025-01-25T14:39:02.730Z [INFO] server-a8195adc-c85f-90e7-6f0a-377f7514be78: updating configuration: command=AddVoter server-id=server-9fd14bd8-7a5c-70ee-5ffe-f6e1ce844aa9 server-addr=9fd14bd8-7a5c-70ee-5ffe-f6e1ce844aa9 servers="[{Suffrage:Voter ID:server-a8195adc-c85f-90e7-6f0a-377f7514be78 Address:a8195adc-c85f-90e7-6f0a-377f7514be78} {Suffrage:Voter ID:server-9fd14bd8-7a5c-70ee-5ffe-f6e1ce844aa9 Address:9fd14bd8-7a5c-70ee-5ffe-f6e1ce844aa9}]" +2025-01-25T14:39:02.730Z [INFO] server-a8195adc-c85f-90e7-6f0a-377f7514be78: added peer, starting replication: peer=server-9fd14bd8-7a5c-70ee-5ffe-f6e1ce844aa9 +2025-01-25T14:39:02.731Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:02.731Z [WARN] server-9fd14bd8-7a5c-70ee-5ffe-f6e1ce844aa9: failed to get previous log: previous-index=3 last-index=0 error="log not found" +2025-01-25T14:39:02.731Z [WARN] server-a8195adc-c85f-90e7-6f0a-377f7514be78: appendEntries rejected, sending older logs: peer="{Voter server-9fd14bd8-7a5c-70ee-5ffe-f6e1ce844aa9 9fd14bd8-7a5c-70ee-5ffe-f6e1ce844aa9}" next=1 +2025-01-25T14:39:02.731Z [INFO] server-a8195adc-c85f-90e7-6f0a-377f7514be78: pipelining replication: peer="{Voter server-9fd14bd8-7a5c-70ee-5ffe-f6e1ce844aa9 9fd14bd8-7a5c-70ee-5ffe-f6e1ce844aa9}" +2025-01-25T14:39:02.837Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:02.731202681 +0000 UTC m=+11.777041999 (1 nodes), 3.5µs from start of poll, 179.383106ms from cluster start. Timeout at 2025-01-25 14:39:02.836248181 +0000 UTC m=+11.882087498, 105.045499ms after stability +2025-01-25T14:39:02.837Z [INFO] server-a8195adc-c85f-90e7-6f0a-377f7514be78: updating configuration: command=AddVoter server-id=server-459ac2c7-fc28-a807-9e04-7f4d98b2bbb6 server-addr=459ac2c7-fc28-a807-9e04-7f4d98b2bbb6 servers="[{Suffrage:Voter ID:server-a8195adc-c85f-90e7-6f0a-377f7514be78 Address:a8195adc-c85f-90e7-6f0a-377f7514be78} {Suffrage:Voter ID:server-9fd14bd8-7a5c-70ee-5ffe-f6e1ce844aa9 Address:9fd14bd8-7a5c-70ee-5ffe-f6e1ce844aa9} {Suffrage:Voter ID:server-459ac2c7-fc28-a807-9e04-7f4d98b2bbb6 Address:459ac2c7-fc28-a807-9e04-7f4d98b2bbb6}]" +2025-01-25T14:39:02.838Z [INFO] server-a8195adc-c85f-90e7-6f0a-377f7514be78: added peer, starting replication: peer=server-459ac2c7-fc28-a807-9e04-7f4d98b2bbb6 +2025-01-25T14:39:02.838Z [WARN] server-459ac2c7-fc28-a807-9e04-7f4d98b2bbb6: failed to get previous log: previous-index=4 last-index=0 error="log not found" +2025-01-25T14:39:02.839Z [WARN] server-a8195adc-c85f-90e7-6f0a-377f7514be78: appendEntries rejected, sending older logs: peer="{Voter server-459ac2c7-fc28-a807-9e04-7f4d98b2bbb6 459ac2c7-fc28-a807-9e04-7f4d98b2bbb6}" next=1 +2025-01-25T14:39:02.839Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:02.840Z [INFO] server-a8195adc-c85f-90e7-6f0a-377f7514be78: pipelining replication: peer="{Voter server-459ac2c7-fc28-a807-9e04-7f4d98b2bbb6 459ac2c7-fc28-a807-9e04-7f4d98b2bbb6}" +2025-01-25T14:39:02.945Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:02.83947383 +0000 UTC m=+11.885313147 (1 nodes), 3.5µs from start of poll, 287.654254ms from cluster start. Timeout at 2025-01-25 14:39:02.94452458 +0000 UTC m=+11.990363897, 105.05075ms after stability +2025-01-25T14:39:02.946Z [INFO] server-a8195adc-c85f-90e7-6f0a-377f7514be78: aborting pipeline replication: peer="{Voter server-9fd14bd8-7a5c-70ee-5ffe-f6e1ce844aa9 9fd14bd8-7a5c-70ee-5ffe-f6e1ce844aa9}" +2025-01-25T14:39:02.946Z [INFO] server-a8195adc-c85f-90e7-6f0a-377f7514be78: aborting pipeline replication: peer="{Voter server-459ac2c7-fc28-a807-9e04-7f4d98b2bbb6 459ac2c7-fc28-a807-9e04-7f4d98b2bbb6}" +2025-01-25T14:39:03.447Z [ERROR] server-a8195adc-c85f-90e7-6f0a-377f7514be78: failed to heartbeat to: peer=459ac2c7-fc28-a807-9e04-7f4d98b2bbb6 backoff time=10ms error="command timed out" +--- PASS: TestRaft_JoinNode_ConfigStore (0.90s) === RUN TestRaft_RemoveFollower -2025-01-25T14:28:19.823Z [INFO] server-63bd4c4e-e9a3-464e-93df-d626cdcef492: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-63bd4c4e-e9a3-464e-93df-d626cdcef492 Address:63bd4c4e-e9a3-464e-93df-d626cdcef492} {Suffrage:Voter ID:server-e3a7e5b7-62c1-7793-de55-2969fd575b18 Address:e3a7e5b7-62c1-7793-de55-2969fd575b18} {Suffrage:Voter ID:server-65520df9-2c38-c96a-22f2-65d49ef3852d Address:65520df9-2c38-c96a-22f2-65d49ef3852d}]" -2025-01-25T14:28:19.824Z [INFO] server-e3a7e5b7-62c1-7793-de55-2969fd575b18: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-63bd4c4e-e9a3-464e-93df-d626cdcef492 Address:63bd4c4e-e9a3-464e-93df-d626cdcef492} {Suffrage:Voter ID:server-e3a7e5b7-62c1-7793-de55-2969fd575b18 Address:e3a7e5b7-62c1-7793-de55-2969fd575b18} {Suffrage:Voter ID:server-65520df9-2c38-c96a-22f2-65d49ef3852d Address:65520df9-2c38-c96a-22f2-65d49ef3852d}]" -2025-01-25T14:28:19.824Z [INFO] server-65520df9-2c38-c96a-22f2-65d49ef3852d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-63bd4c4e-e9a3-464e-93df-d626cdcef492 Address:63bd4c4e-e9a3-464e-93df-d626cdcef492} {Suffrage:Voter ID:server-e3a7e5b7-62c1-7793-de55-2969fd575b18 Address:e3a7e5b7-62c1-7793-de55-2969fd575b18} {Suffrage:Voter ID:server-65520df9-2c38-c96a-22f2-65d49ef3852d Address:65520df9-2c38-c96a-22f2-65d49ef3852d}]" -2025-01-25T14:28:19.824Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:19.824Z [INFO] server-63bd4c4e-e9a3-464e-93df-d626cdcef492: entering follower state: follower="Node at 63bd4c4e-e9a3-464e-93df-d626cdcef492 [Follower]" leader-address= leader-id= -2025-01-25T14:28:19.824Z [INFO] server-e3a7e5b7-62c1-7793-de55-2969fd575b18: entering follower state: follower="Node at e3a7e5b7-62c1-7793-de55-2969fd575b18 [Follower]" leader-address= leader-id= -2025-01-25T14:28:19.824Z [INFO] server-65520df9-2c38-c96a-22f2-65d49ef3852d: entering follower state: follower="Node at 65520df9-2c38-c96a-22f2-65d49ef3852d [Follower]" leader-address= leader-id= -2025-01-25T14:28:19.895Z [WARN] server-63bd4c4e-e9a3-464e-93df-d626cdcef492: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:19.895Z [WARN] server-65520df9-2c38-c96a-22f2-65d49ef3852d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:19.895Z [INFO] server-63bd4c4e-e9a3-464e-93df-d626cdcef492: entering candidate state: node="Node at 63bd4c4e-e9a3-464e-93df-d626cdcef492 [Candidate]" term=2 -2025-01-25T14:28:19.895Z [INFO] server-65520df9-2c38-c96a-22f2-65d49ef3852d: entering candidate state: node="Node at 65520df9-2c38-c96a-22f2-65d49ef3852d [Candidate]" term=2 -2025-01-25T14:28:19.895Z [INFO] server-65520df9-2c38-c96a-22f2-65d49ef3852d: duplicate requestVote for same term: term=2 -2025-01-25T14:28:19.895Z [INFO] server-63bd4c4e-e9a3-464e-93df-d626cdcef492: duplicate requestVote for same term: term=2 -2025-01-25T14:28:19.895Z [INFO] server-63bd4c4e-e9a3-464e-93df-d626cdcef492: election won: term=2 tally=2 -2025-01-25T14:28:19.895Z [INFO] server-63bd4c4e-e9a3-464e-93df-d626cdcef492: entering leader state: leader="Node at 63bd4c4e-e9a3-464e-93df-d626cdcef492 [Leader]" -2025-01-25T14:28:19.895Z [INFO] server-e3a7e5b7-62c1-7793-de55-2969fd575b18: duplicate requestVote for same term: term=2 -2025-01-25T14:28:19.895Z [INFO] server-63bd4c4e-e9a3-464e-93df-d626cdcef492: added peer, starting replication: peer=server-e3a7e5b7-62c1-7793-de55-2969fd575b18 -2025-01-25T14:28:19.895Z [INFO] server-63bd4c4e-e9a3-464e-93df-d626cdcef492: added peer, starting replication: peer=server-65520df9-2c38-c96a-22f2-65d49ef3852d -2025-01-25T14:28:19.895Z [INFO] server-65520df9-2c38-c96a-22f2-65d49ef3852d: entering follower state: follower="Node at 65520df9-2c38-c96a-22f2-65d49ef3852d [Follower]" leader-address=63bd4c4e-e9a3-464e-93df-d626cdcef492 leader-id=server-63bd4c4e-e9a3-464e-93df-d626cdcef492 -2025-01-25T14:28:19.895Z [INFO] server-63bd4c4e-e9a3-464e-93df-d626cdcef492: pipelining replication: peer="{Voter server-e3a7e5b7-62c1-7793-de55-2969fd575b18 e3a7e5b7-62c1-7793-de55-2969fd575b18}" -2025-01-25T14:28:19.896Z [INFO] server-63bd4c4e-e9a3-464e-93df-d626cdcef492: pipelining replication: peer="{Voter server-65520df9-2c38-c96a-22f2-65d49ef3852d 65520df9-2c38-c96a-22f2-65d49ef3852d}" -2025-01-25T14:28:20.002Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:19.896077361 +0000 UTC m=+9.750402343 (1 nodes), 71.552066ms from start of poll, 72.390435ms from cluster start. Timeout at 2025-01-25 14:28:20.001080801 +0000 UTC m=+9.855405943, 105.0036ms after stability -2025-01-25T14:28:20.007Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:28:20.113Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:28:20.007846099 +0000 UTC m=+9.862171081 (2 nodes), 4.04µs from start of poll, 184.159173ms from cluster start. Timeout at 2025-01-25 14:28:20.112877138 +0000 UTC m=+9.967202240, 105.031159ms after stability -2025-01-25T14:28:20.114Z [INFO] server-63bd4c4e-e9a3-464e-93df-d626cdcef492: updating configuration: command=RemoveServer server-id=server-e3a7e5b7-62c1-7793-de55-2969fd575b18 server-addr= servers="[{Suffrage:Voter ID:server-63bd4c4e-e9a3-464e-93df-d626cdcef492 Address:63bd4c4e-e9a3-464e-93df-d626cdcef492} {Suffrage:Voter ID:server-65520df9-2c38-c96a-22f2-65d49ef3852d Address:65520df9-2c38-c96a-22f2-65d49ef3852d}]" -2025-01-25T14:28:20.114Z [INFO] server-63bd4c4e-e9a3-464e-93df-d626cdcef492: removed peer, stopping replication: peer=server-e3a7e5b7-62c1-7793-de55-2969fd575b18 last-index=3 -2025-01-25T14:28:20.114Z [INFO] server-63bd4c4e-e9a3-464e-93df-d626cdcef492: aborting pipeline replication: peer="{Voter server-e3a7e5b7-62c1-7793-de55-2969fd575b18 e3a7e5b7-62c1-7793-de55-2969fd575b18}" -2025-01-25T14:28:20.212Z [WARN] server-e3a7e5b7-62c1-7793-de55-2969fd575b18: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election -2025-01-25T14:28:20.220Z [INFO] server-63bd4c4e-e9a3-464e-93df-d626cdcef492: aborting pipeline replication: peer="{Voter server-65520df9-2c38-c96a-22f2-65d49ef3852d 65520df9-2c38-c96a-22f2-65d49ef3852d}" ---- PASS: TestRaft_RemoveFollower (0.40s) +2025-01-25T14:39:03.453Z [INFO] server-39f9a787-946c-a798-330e-f7d2cd685933: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-39f9a787-946c-a798-330e-f7d2cd685933 Address:39f9a787-946c-a798-330e-f7d2cd685933} {Suffrage:Voter ID:server-a24e0e1d-94ee-3fea-93d4-022be7f248e1 Address:a24e0e1d-94ee-3fea-93d4-022be7f248e1} {Suffrage:Voter ID:server-3c0dbdc9-bf06-5c47-bba0-1e4ebdf827af Address:3c0dbdc9-bf06-5c47-bba0-1e4ebdf827af}]" +2025-01-25T14:39:03.453Z [INFO] server-39f9a787-946c-a798-330e-f7d2cd685933: entering follower state: follower="Node at 39f9a787-946c-a798-330e-f7d2cd685933 [Follower]" leader-address= leader-id= +2025-01-25T14:39:03.453Z [INFO] server-a24e0e1d-94ee-3fea-93d4-022be7f248e1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-39f9a787-946c-a798-330e-f7d2cd685933 Address:39f9a787-946c-a798-330e-f7d2cd685933} {Suffrage:Voter ID:server-a24e0e1d-94ee-3fea-93d4-022be7f248e1 Address:a24e0e1d-94ee-3fea-93d4-022be7f248e1} {Suffrage:Voter ID:server-3c0dbdc9-bf06-5c47-bba0-1e4ebdf827af Address:3c0dbdc9-bf06-5c47-bba0-1e4ebdf827af}]" +2025-01-25T14:39:03.454Z [INFO] server-a24e0e1d-94ee-3fea-93d4-022be7f248e1: entering follower state: follower="Node at a24e0e1d-94ee-3fea-93d4-022be7f248e1 [Follower]" leader-address= leader-id= +2025-01-25T14:39:03.454Z [INFO] server-3c0dbdc9-bf06-5c47-bba0-1e4ebdf827af: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-39f9a787-946c-a798-330e-f7d2cd685933 Address:39f9a787-946c-a798-330e-f7d2cd685933} {Suffrage:Voter ID:server-a24e0e1d-94ee-3fea-93d4-022be7f248e1 Address:a24e0e1d-94ee-3fea-93d4-022be7f248e1} {Suffrage:Voter ID:server-3c0dbdc9-bf06-5c47-bba0-1e4ebdf827af Address:3c0dbdc9-bf06-5c47-bba0-1e4ebdf827af}]" +2025-01-25T14:39:03.454Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:03.454Z [INFO] server-3c0dbdc9-bf06-5c47-bba0-1e4ebdf827af: entering follower state: follower="Node at 3c0dbdc9-bf06-5c47-bba0-1e4ebdf827af [Follower]" leader-address= leader-id= +2025-01-25T14:39:03.525Z [WARN] server-a24e0e1d-94ee-3fea-93d4-022be7f248e1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:03.525Z [INFO] server-a24e0e1d-94ee-3fea-93d4-022be7f248e1: entering candidate state: node="Node at a24e0e1d-94ee-3fea-93d4-022be7f248e1 [Candidate]" term=2 +2025-01-25T14:39:03.525Z [INFO] server-a24e0e1d-94ee-3fea-93d4-022be7f248e1: election won: term=2 tally=2 +2025-01-25T14:39:03.526Z [INFO] server-a24e0e1d-94ee-3fea-93d4-022be7f248e1: entering leader state: leader="Node at a24e0e1d-94ee-3fea-93d4-022be7f248e1 [Leader]" +2025-01-25T14:39:03.526Z [INFO] server-a24e0e1d-94ee-3fea-93d4-022be7f248e1: added peer, starting replication: peer=server-39f9a787-946c-a798-330e-f7d2cd685933 +2025-01-25T14:39:03.526Z [INFO] server-a24e0e1d-94ee-3fea-93d4-022be7f248e1: added peer, starting replication: peer=server-3c0dbdc9-bf06-5c47-bba0-1e4ebdf827af +2025-01-25T14:39:03.526Z [INFO] server-a24e0e1d-94ee-3fea-93d4-022be7f248e1: pipelining replication: peer="{Voter server-39f9a787-946c-a798-330e-f7d2cd685933 39f9a787-946c-a798-330e-f7d2cd685933}" +2025-01-25T14:39:03.527Z [INFO] server-a24e0e1d-94ee-3fea-93d4-022be7f248e1: pipelining replication: peer="{Voter server-3c0dbdc9-bf06-5c47-bba0-1e4ebdf827af 3c0dbdc9-bf06-5c47-bba0-1e4ebdf827af}" +2025-01-25T14:39:03.632Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:03.526081787 +0000 UTC m=+12.571921105 (1 nodes), 71.353833ms from start of poll, 73.198339ms from cluster start. Timeout at 2025-01-25 14:39:03.631095204 +0000 UTC m=+12.676934521, 105.013416ms after stability +2025-01-25T14:39:03.637Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:39:03.743Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:39:03.63778757 +0000 UTC m=+12.683626888 (2 nodes), 7.292µs from start of poll, 184.904122ms from cluster start. Timeout at 2025-01-25 14:39:03.74284182 +0000 UTC m=+12.788681138, 105.05425ms after stability +2025-01-25T14:39:03.743Z [INFO] server-a24e0e1d-94ee-3fea-93d4-022be7f248e1: updating configuration: command=RemoveServer server-id=server-39f9a787-946c-a798-330e-f7d2cd685933 server-addr= servers="[{Suffrage:Voter ID:server-a24e0e1d-94ee-3fea-93d4-022be7f248e1 Address:a24e0e1d-94ee-3fea-93d4-022be7f248e1} {Suffrage:Voter ID:server-3c0dbdc9-bf06-5c47-bba0-1e4ebdf827af Address:3c0dbdc9-bf06-5c47-bba0-1e4ebdf827af}]" +2025-01-25T14:39:03.743Z [INFO] server-a24e0e1d-94ee-3fea-93d4-022be7f248e1: removed peer, stopping replication: peer=server-39f9a787-946c-a798-330e-f7d2cd685933 last-index=3 +2025-01-25T14:39:03.744Z [INFO] server-a24e0e1d-94ee-3fea-93d4-022be7f248e1: aborting pipeline replication: peer="{Voter server-39f9a787-946c-a798-330e-f7d2cd685933 39f9a787-946c-a798-330e-f7d2cd685933}" +2025-01-25T14:39:03.813Z [WARN] server-39f9a787-946c-a798-330e-f7d2cd685933: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +2025-01-25T14:39:03.850Z [INFO] server-a24e0e1d-94ee-3fea-93d4-022be7f248e1: aborting pipeline replication: peer="{Voter server-3c0dbdc9-bf06-5c47-bba0-1e4ebdf827af 3c0dbdc9-bf06-5c47-bba0-1e4ebdf827af}" +2025-01-25T14:39:04.351Z [ERROR] server-a24e0e1d-94ee-3fea-93d4-022be7f248e1: failed to heartbeat to: peer=3c0dbdc9-bf06-5c47-bba0-1e4ebdf827af backoff time=10ms error="command timed out" +--- PASS: TestRaft_RemoveFollower (0.91s) === RUN TestRaft_RemoveLeader -2025-01-25T14:28:20.223Z [INFO] server-7ca8c4ed-46f5-f629-d8ea-3bc9f04a0365: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7ca8c4ed-46f5-f629-d8ea-3bc9f04a0365 Address:7ca8c4ed-46f5-f629-d8ea-3bc9f04a0365} {Suffrage:Voter ID:server-e4d9ce19-7e7e-5748-7c0d-00213f306dee Address:e4d9ce19-7e7e-5748-7c0d-00213f306dee} {Suffrage:Voter ID:server-f7b71b2a-f14d-846b-0f6c-095d91c0fd8e Address:f7b71b2a-f14d-846b-0f6c-095d91c0fd8e}]" -2025-01-25T14:28:20.223Z [INFO] server-e4d9ce19-7e7e-5748-7c0d-00213f306dee: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7ca8c4ed-46f5-f629-d8ea-3bc9f04a0365 Address:7ca8c4ed-46f5-f629-d8ea-3bc9f04a0365} {Suffrage:Voter ID:server-e4d9ce19-7e7e-5748-7c0d-00213f306dee Address:e4d9ce19-7e7e-5748-7c0d-00213f306dee} {Suffrage:Voter ID:server-f7b71b2a-f14d-846b-0f6c-095d91c0fd8e Address:f7b71b2a-f14d-846b-0f6c-095d91c0fd8e}]" -2025-01-25T14:28:20.223Z [INFO] server-7ca8c4ed-46f5-f629-d8ea-3bc9f04a0365: entering follower state: follower="Node at 7ca8c4ed-46f5-f629-d8ea-3bc9f04a0365 [Follower]" leader-address= leader-id= -2025-01-25T14:28:20.223Z [INFO] server-e4d9ce19-7e7e-5748-7c0d-00213f306dee: entering follower state: follower="Node at e4d9ce19-7e7e-5748-7c0d-00213f306dee [Follower]" leader-address= leader-id= -2025-01-25T14:28:20.224Z [INFO] server-f7b71b2a-f14d-846b-0f6c-095d91c0fd8e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7ca8c4ed-46f5-f629-d8ea-3bc9f04a0365 Address:7ca8c4ed-46f5-f629-d8ea-3bc9f04a0365} {Suffrage:Voter ID:server-e4d9ce19-7e7e-5748-7c0d-00213f306dee Address:e4d9ce19-7e7e-5748-7c0d-00213f306dee} {Suffrage:Voter ID:server-f7b71b2a-f14d-846b-0f6c-095d91c0fd8e Address:f7b71b2a-f14d-846b-0f6c-095d91c0fd8e}]" -2025-01-25T14:28:20.224Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:20.224Z [INFO] server-f7b71b2a-f14d-846b-0f6c-095d91c0fd8e: entering follower state: follower="Node at f7b71b2a-f14d-846b-0f6c-095d91c0fd8e [Follower]" leader-address= leader-id= -2025-01-25T14:28:20.300Z [WARN] server-e4d9ce19-7e7e-5748-7c0d-00213f306dee: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:20.300Z [INFO] server-e4d9ce19-7e7e-5748-7c0d-00213f306dee: entering candidate state: node="Node at e4d9ce19-7e7e-5748-7c0d-00213f306dee [Candidate]" term=2 -2025-01-25T14:28:20.300Z [INFO] server-e4d9ce19-7e7e-5748-7c0d-00213f306dee: election won: term=2 tally=2 -2025-01-25T14:28:20.300Z [INFO] server-e4d9ce19-7e7e-5748-7c0d-00213f306dee: entering leader state: leader="Node at e4d9ce19-7e7e-5748-7c0d-00213f306dee [Leader]" -2025-01-25T14:28:20.300Z [INFO] server-e4d9ce19-7e7e-5748-7c0d-00213f306dee: added peer, starting replication: peer=server-7ca8c4ed-46f5-f629-d8ea-3bc9f04a0365 -2025-01-25T14:28:20.300Z [INFO] server-e4d9ce19-7e7e-5748-7c0d-00213f306dee: added peer, starting replication: peer=server-f7b71b2a-f14d-846b-0f6c-095d91c0fd8e -2025-01-25T14:28:20.301Z [INFO] server-e4d9ce19-7e7e-5748-7c0d-00213f306dee: pipelining replication: peer="{Voter server-f7b71b2a-f14d-846b-0f6c-095d91c0fd8e f7b71b2a-f14d-846b-0f6c-095d91c0fd8e}" -2025-01-25T14:28:20.301Z [INFO] server-e4d9ce19-7e7e-5748-7c0d-00213f306dee: pipelining replication: peer="{Voter server-7ca8c4ed-46f5-f629-d8ea-3bc9f04a0365 7ca8c4ed-46f5-f629-d8ea-3bc9f04a0365}" -2025-01-25T14:28:20.407Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:20.301046348 +0000 UTC m=+10.155371330 (1 nodes), 76.881873ms from start of poll, 77.668085ms from cluster start. Timeout at 2025-01-25 14:28:20.406050508 +0000 UTC m=+10.260375490, 105.00416ms after stability -2025-01-25T14:28:20.413Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:28:20.518Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:28:20.413086597 +0000 UTC m=+10.267411579 (2 nodes), 2.24µs from start of poll, 189.708334ms from cluster start. Timeout at 2025-01-25 14:28:20.518138756 +0000 UTC m=+10.372463738, 105.052159ms after stability -2025-01-25T14:28:20.518Z [INFO] server-e4d9ce19-7e7e-5748-7c0d-00213f306dee: updating configuration: command=RemoveServer server-id=server-e4d9ce19-7e7e-5748-7c0d-00213f306dee server-addr= servers="[{Suffrage:Voter ID:server-7ca8c4ed-46f5-f629-d8ea-3bc9f04a0365 Address:7ca8c4ed-46f5-f629-d8ea-3bc9f04a0365} {Suffrage:Voter ID:server-f7b71b2a-f14d-846b-0f6c-095d91c0fd8e Address:f7b71b2a-f14d-846b-0f6c-095d91c0fd8e}]" -2025-01-25T14:28:20.518Z [INFO] server-e4d9ce19-7e7e-5748-7c0d-00213f306dee: removed ourself, shutting down -2025-01-25T14:28:20.518Z [INFO] server-e4d9ce19-7e7e-5748-7c0d-00213f306dee: aborting pipeline replication: peer="{Voter server-f7b71b2a-f14d-846b-0f6c-095d91c0fd8e f7b71b2a-f14d-846b-0f6c-095d91c0fd8e}" -2025-01-25T14:28:20.518Z [INFO] server-e4d9ce19-7e7e-5748-7c0d-00213f306dee: aborting pipeline replication: peer="{Voter server-7ca8c4ed-46f5-f629-d8ea-3bc9f04a0365 7ca8c4ed-46f5-f629-d8ea-3bc9f04a0365}" -2025-01-25T14:28:20.569Z [WARN] server-f7b71b2a-f14d-846b-0f6c-095d91c0fd8e: heartbeat timeout reached, starting election: last-leader-addr=e4d9ce19-7e7e-5748-7c0d-00213f306dee last-leader-id=server-e4d9ce19-7e7e-5748-7c0d-00213f306dee -2025-01-25T14:28:20.569Z [INFO] server-f7b71b2a-f14d-846b-0f6c-095d91c0fd8e: entering candidate state: node="Node at f7b71b2a-f14d-846b-0f6c-095d91c0fd8e [Candidate]" term=3 -2025-01-25T14:28:20.569Z [WARN] server-7ca8c4ed-46f5-f629-d8ea-3bc9f04a0365: rejecting vote request since we have a leader: from=f7b71b2a-f14d-846b-0f6c-095d91c0fd8e leader=e4d9ce19-7e7e-5748-7c0d-00213f306dee leader-id=server-e4d9ce19-7e7e-5748-7c0d-00213f306dee -2025-01-25T14:28:20.623Z [WARN] server-7ca8c4ed-46f5-f629-d8ea-3bc9f04a0365: heartbeat timeout reached, starting election: last-leader-addr=e4d9ce19-7e7e-5748-7c0d-00213f306dee last-leader-id=server-e4d9ce19-7e7e-5748-7c0d-00213f306dee -2025-01-25T14:28:20.623Z [INFO] server-7ca8c4ed-46f5-f629-d8ea-3bc9f04a0365: entering candidate state: node="Node at 7ca8c4ed-46f5-f629-d8ea-3bc9f04a0365 [Candidate]" term=3 -2025-01-25T14:28:20.623Z [INFO] server-f7b71b2a-f14d-846b-0f6c-095d91c0fd8e: duplicate requestVote for same term: term=3 -2025-01-25T14:28:20.630Z [WARN] server-f7b71b2a-f14d-846b-0f6c-095d91c0fd8e: Election timeout reached, restarting election -2025-01-25T14:28:20.630Z [INFO] server-f7b71b2a-f14d-846b-0f6c-095d91c0fd8e: entering candidate state: node="Node at f7b71b2a-f14d-846b-0f6c-095d91c0fd8e [Candidate]" term=4 -2025-01-25T14:28:20.630Z [INFO] server-7ca8c4ed-46f5-f629-d8ea-3bc9f04a0365: entering follower state: follower="Node at 7ca8c4ed-46f5-f629-d8ea-3bc9f04a0365 [Follower]" leader-address= leader-id= -2025-01-25T14:28:20.630Z [INFO] server-f7b71b2a-f14d-846b-0f6c-095d91c0fd8e: election won: term=4 tally=2 -2025-01-25T14:28:20.630Z [INFO] server-f7b71b2a-f14d-846b-0f6c-095d91c0fd8e: entering leader state: leader="Node at f7b71b2a-f14d-846b-0f6c-095d91c0fd8e [Leader]" -2025-01-25T14:28:20.630Z [INFO] server-f7b71b2a-f14d-846b-0f6c-095d91c0fd8e: added peer, starting replication: peer=server-7ca8c4ed-46f5-f629-d8ea-3bc9f04a0365 -2025-01-25T14:28:20.630Z [INFO] server-f7b71b2a-f14d-846b-0f6c-095d91c0fd8e: pipelining replication: peer="{Voter server-7ca8c4ed-46f5-f629-d8ea-3bc9f04a0365 7ca8c4ed-46f5-f629-d8ea-3bc9f04a0365}" -2025-01-25T14:28:20.729Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:20.836Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:20.729958714 +0000 UTC m=+10.584283656 (1 nodes), 126.195µs from start of poll, 506.580411ms from cluster start. Timeout at 2025-01-25 14:28:20.834994193 +0000 UTC m=+10.689319215, 105.035559ms after stability -2025-01-25T14:28:20.836Z [INFO] server-f7b71b2a-f14d-846b-0f6c-095d91c0fd8e: aborting pipeline replication: peer="{Voter server-7ca8c4ed-46f5-f629-d8ea-3bc9f04a0365 7ca8c4ed-46f5-f629-d8ea-3bc9f04a0365}" ---- PASS: TestRaft_RemoveLeader (0.62s) +2025-01-25T14:39:04.360Z [INFO] server-6cb82776-a8bc-1bdb-d0b8-370bbbe684ec: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6cb82776-a8bc-1bdb-d0b8-370bbbe684ec Address:6cb82776-a8bc-1bdb-d0b8-370bbbe684ec} {Suffrage:Voter ID:server-1946ffeb-a637-9300-3eb5-c5a651fd5d6c Address:1946ffeb-a637-9300-3eb5-c5a651fd5d6c} {Suffrage:Voter ID:server-de98de25-c748-8dbf-70c4-4bc692c295e6 Address:de98de25-c748-8dbf-70c4-4bc692c295e6}]" +2025-01-25T14:39:04.361Z [INFO] server-6cb82776-a8bc-1bdb-d0b8-370bbbe684ec: entering follower state: follower="Node at 6cb82776-a8bc-1bdb-d0b8-370bbbe684ec [Follower]" leader-address= leader-id= +2025-01-25T14:39:04.362Z [INFO] server-1946ffeb-a637-9300-3eb5-c5a651fd5d6c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6cb82776-a8bc-1bdb-d0b8-370bbbe684ec Address:6cb82776-a8bc-1bdb-d0b8-370bbbe684ec} {Suffrage:Voter ID:server-1946ffeb-a637-9300-3eb5-c5a651fd5d6c Address:1946ffeb-a637-9300-3eb5-c5a651fd5d6c} {Suffrage:Voter ID:server-de98de25-c748-8dbf-70c4-4bc692c295e6 Address:de98de25-c748-8dbf-70c4-4bc692c295e6}]" +2025-01-25T14:39:04.362Z [INFO] server-1946ffeb-a637-9300-3eb5-c5a651fd5d6c: entering follower state: follower="Node at 1946ffeb-a637-9300-3eb5-c5a651fd5d6c [Follower]" leader-address= leader-id= +2025-01-25T14:39:04.363Z [INFO] server-de98de25-c748-8dbf-70c4-4bc692c295e6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6cb82776-a8bc-1bdb-d0b8-370bbbe684ec Address:6cb82776-a8bc-1bdb-d0b8-370bbbe684ec} {Suffrage:Voter ID:server-1946ffeb-a637-9300-3eb5-c5a651fd5d6c Address:1946ffeb-a637-9300-3eb5-c5a651fd5d6c} {Suffrage:Voter ID:server-de98de25-c748-8dbf-70c4-4bc692c295e6 Address:de98de25-c748-8dbf-70c4-4bc692c295e6}]" +2025-01-25T14:39:04.363Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:04.363Z [INFO] server-de98de25-c748-8dbf-70c4-4bc692c295e6: entering follower state: follower="Node at de98de25-c748-8dbf-70c4-4bc692c295e6 [Follower]" leader-address= leader-id= +2025-01-25T14:39:04.420Z [WARN] server-6cb82776-a8bc-1bdb-d0b8-370bbbe684ec: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:04.421Z [INFO] server-6cb82776-a8bc-1bdb-d0b8-370bbbe684ec: entering candidate state: node="Node at 6cb82776-a8bc-1bdb-d0b8-370bbbe684ec [Candidate]" term=2 +2025-01-25T14:39:04.421Z [INFO] server-6cb82776-a8bc-1bdb-d0b8-370bbbe684ec: election won: term=2 tally=2 +2025-01-25T14:39:04.421Z [INFO] server-6cb82776-a8bc-1bdb-d0b8-370bbbe684ec: entering leader state: leader="Node at 6cb82776-a8bc-1bdb-d0b8-370bbbe684ec [Leader]" +2025-01-25T14:39:04.421Z [INFO] server-6cb82776-a8bc-1bdb-d0b8-370bbbe684ec: added peer, starting replication: peer=server-1946ffeb-a637-9300-3eb5-c5a651fd5d6c +2025-01-25T14:39:04.422Z [INFO] server-6cb82776-a8bc-1bdb-d0b8-370bbbe684ec: added peer, starting replication: peer=server-de98de25-c748-8dbf-70c4-4bc692c295e6 +2025-01-25T14:39:04.422Z [INFO] server-6cb82776-a8bc-1bdb-d0b8-370bbbe684ec: pipelining replication: peer="{Voter server-1946ffeb-a637-9300-3eb5-c5a651fd5d6c 1946ffeb-a637-9300-3eb5-c5a651fd5d6c}" +2025-01-25T14:39:04.422Z [INFO] server-6cb82776-a8bc-1bdb-d0b8-370bbbe684ec: pipelining replication: peer="{Voter server-de98de25-c748-8dbf-70c4-4bc692c295e6 de98de25-c748-8dbf-70c4-4bc692c295e6}" +2025-01-25T14:39:04.528Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:04.421913645 +0000 UTC m=+13.467752963 (1 nodes), 58.228211ms from start of poll, 61.782472ms from cluster start. Timeout at 2025-01-25 14:39:04.526935228 +0000 UTC m=+13.572774546, 105.021583ms after stability +2025-01-25T14:39:04.534Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:39:04.640Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:39:04.534528515 +0000 UTC m=+13.580367832 (2 nodes), 7.291µs from start of poll, 174.397341ms from cluster start. Timeout at 2025-01-25 14:39:04.639571098 +0000 UTC m=+13.685410415, 105.042583ms after stability +2025-01-25T14:39:04.641Z [INFO] server-6cb82776-a8bc-1bdb-d0b8-370bbbe684ec: updating configuration: command=RemoveServer server-id=server-6cb82776-a8bc-1bdb-d0b8-370bbbe684ec server-addr= servers="[{Suffrage:Voter ID:server-1946ffeb-a637-9300-3eb5-c5a651fd5d6c Address:1946ffeb-a637-9300-3eb5-c5a651fd5d6c} {Suffrage:Voter ID:server-de98de25-c748-8dbf-70c4-4bc692c295e6 Address:de98de25-c748-8dbf-70c4-4bc692c295e6}]" +2025-01-25T14:39:04.642Z [INFO] server-6cb82776-a8bc-1bdb-d0b8-370bbbe684ec: removed ourself, shutting down +2025-01-25T14:39:04.642Z [INFO] server-6cb82776-a8bc-1bdb-d0b8-370bbbe684ec: aborting pipeline replication: peer="{Voter server-de98de25-c748-8dbf-70c4-4bc692c295e6 de98de25-c748-8dbf-70c4-4bc692c295e6}" +2025-01-25T14:39:04.642Z [INFO] server-6cb82776-a8bc-1bdb-d0b8-370bbbe684ec: aborting pipeline replication: peer="{Voter server-1946ffeb-a637-9300-3eb5-c5a651fd5d6c 1946ffeb-a637-9300-3eb5-c5a651fd5d6c}" +2025-01-25T14:39:04.693Z [WARN] server-1946ffeb-a637-9300-3eb5-c5a651fd5d6c: heartbeat timeout reached, starting election: last-leader-addr=6cb82776-a8bc-1bdb-d0b8-370bbbe684ec last-leader-id=server-6cb82776-a8bc-1bdb-d0b8-370bbbe684ec +2025-01-25T14:39:04.693Z [INFO] server-1946ffeb-a637-9300-3eb5-c5a651fd5d6c: entering candidate state: node="Node at 1946ffeb-a637-9300-3eb5-c5a651fd5d6c [Candidate]" term=3 +2025-01-25T14:39:04.694Z [WARN] server-de98de25-c748-8dbf-70c4-4bc692c295e6: rejecting vote request since we have a leader: from=1946ffeb-a637-9300-3eb5-c5a651fd5d6c leader=6cb82776-a8bc-1bdb-d0b8-370bbbe684ec leader-id=server-6cb82776-a8bc-1bdb-d0b8-370bbbe684ec +2025-01-25T14:39:04.700Z [WARN] server-de98de25-c748-8dbf-70c4-4bc692c295e6: heartbeat timeout reached, starting election: last-leader-addr=6cb82776-a8bc-1bdb-d0b8-370bbbe684ec last-leader-id=server-6cb82776-a8bc-1bdb-d0b8-370bbbe684ec +2025-01-25T14:39:04.701Z [INFO] server-de98de25-c748-8dbf-70c4-4bc692c295e6: entering candidate state: node="Node at de98de25-c748-8dbf-70c4-4bc692c295e6 [Candidate]" term=3 +2025-01-25T14:39:04.701Z [INFO] server-1946ffeb-a637-9300-3eb5-c5a651fd5d6c: duplicate requestVote for same term: term=3 +2025-01-25T14:39:04.786Z [WARN] server-1946ffeb-a637-9300-3eb5-c5a651fd5d6c: Election timeout reached, restarting election +2025-01-25T14:39:04.786Z [INFO] server-1946ffeb-a637-9300-3eb5-c5a651fd5d6c: entering candidate state: node="Node at 1946ffeb-a637-9300-3eb5-c5a651fd5d6c [Candidate]" term=4 +2025-01-25T14:39:04.787Z [INFO] server-de98de25-c748-8dbf-70c4-4bc692c295e6: entering follower state: follower="Node at de98de25-c748-8dbf-70c4-4bc692c295e6 [Follower]" leader-address= leader-id= +2025-01-25T14:39:04.787Z [INFO] server-1946ffeb-a637-9300-3eb5-c5a651fd5d6c: election won: term=4 tally=2 +2025-01-25T14:39:04.787Z [INFO] server-1946ffeb-a637-9300-3eb5-c5a651fd5d6c: entering leader state: leader="Node at 1946ffeb-a637-9300-3eb5-c5a651fd5d6c [Leader]" +2025-01-25T14:39:04.788Z [INFO] server-1946ffeb-a637-9300-3eb5-c5a651fd5d6c: added peer, starting replication: peer=server-de98de25-c748-8dbf-70c4-4bc692c295e6 +2025-01-25T14:39:04.788Z [INFO] server-1946ffeb-a637-9300-3eb5-c5a651fd5d6c: pipelining replication: peer="{Voter server-de98de25-c748-8dbf-70c4-4bc692c295e6 de98de25-c748-8dbf-70c4-4bc692c295e6}" +2025-01-25T14:39:04.853Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:04.960Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:04.853920278 +0000 UTC m=+13.899759596 (1 nodes), 381.21µs from start of poll, 493.789105ms from cluster start. Timeout at 2025-01-25 14:39:04.958970737 +0000 UTC m=+14.004810054, 105.050458ms after stability +2025-01-25T14:39:04.960Z [INFO] server-1946ffeb-a637-9300-3eb5-c5a651fd5d6c: aborting pipeline replication: peer="{Voter server-de98de25-c748-8dbf-70c4-4bc692c295e6 de98de25-c748-8dbf-70c4-4bc692c295e6}" +--- PASS: TestRaft_RemoveLeader (0.61s) === RUN TestRaft_RemoveLeader_NoShutdown -2025-01-25T14:28:20.838Z [INFO] server-a0ef52de-67d2-e87f-ecf0-3a477bbdc7c4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a0ef52de-67d2-e87f-ecf0-3a477bbdc7c4 Address:a0ef52de-67d2-e87f-ecf0-3a477bbdc7c4} {Suffrage:Voter ID:server-ac52c234-61a7-d637-6af5-5c84a6d32a15 Address:ac52c234-61a7-d637-6af5-5c84a6d32a15} {Suffrage:Voter ID:server-17c25c52-0283-6ee2-e7cd-1bcd7ba2a995 Address:17c25c52-0283-6ee2-e7cd-1bcd7ba2a995}]" -2025-01-25T14:28:20.838Z [INFO] server-ac52c234-61a7-d637-6af5-5c84a6d32a15: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a0ef52de-67d2-e87f-ecf0-3a477bbdc7c4 Address:a0ef52de-67d2-e87f-ecf0-3a477bbdc7c4} {Suffrage:Voter ID:server-ac52c234-61a7-d637-6af5-5c84a6d32a15 Address:ac52c234-61a7-d637-6af5-5c84a6d32a15} {Suffrage:Voter ID:server-17c25c52-0283-6ee2-e7cd-1bcd7ba2a995 Address:17c25c52-0283-6ee2-e7cd-1bcd7ba2a995}]" -2025-01-25T14:28:20.839Z [INFO] server-a0ef52de-67d2-e87f-ecf0-3a477bbdc7c4: entering follower state: follower="Node at a0ef52de-67d2-e87f-ecf0-3a477bbdc7c4 [Follower]" leader-address= leader-id= -2025-01-25T14:28:20.839Z [INFO] server-ac52c234-61a7-d637-6af5-5c84a6d32a15: entering follower state: follower="Node at ac52c234-61a7-d637-6af5-5c84a6d32a15 [Follower]" leader-address= leader-id= -2025-01-25T14:28:20.839Z [INFO] server-17c25c52-0283-6ee2-e7cd-1bcd7ba2a995: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a0ef52de-67d2-e87f-ecf0-3a477bbdc7c4 Address:a0ef52de-67d2-e87f-ecf0-3a477bbdc7c4} {Suffrage:Voter ID:server-ac52c234-61a7-d637-6af5-5c84a6d32a15 Address:ac52c234-61a7-d637-6af5-5c84a6d32a15} {Suffrage:Voter ID:server-17c25c52-0283-6ee2-e7cd-1bcd7ba2a995 Address:17c25c52-0283-6ee2-e7cd-1bcd7ba2a995}]" -2025-01-25T14:28:20.839Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:28:20.839Z [INFO] server-17c25c52-0283-6ee2-e7cd-1bcd7ba2a995: entering follower state: follower="Node at 17c25c52-0283-6ee2-e7cd-1bcd7ba2a995 [Follower]" leader-address= leader-id= -2025-01-25T14:28:20.897Z [WARN] server-ac52c234-61a7-d637-6af5-5c84a6d32a15: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:20.897Z [INFO] server-ac52c234-61a7-d637-6af5-5c84a6d32a15: entering candidate state: node="Node at ac52c234-61a7-d637-6af5-5c84a6d32a15 [Candidate]" term=2 -2025-01-25T14:28:20.897Z [INFO] server-ac52c234-61a7-d637-6af5-5c84a6d32a15: election won: term=2 tally=2 -2025-01-25T14:28:20.897Z [INFO] server-ac52c234-61a7-d637-6af5-5c84a6d32a15: entering leader state: leader="Node at ac52c234-61a7-d637-6af5-5c84a6d32a15 [Leader]" -2025-01-25T14:28:20.897Z [INFO] server-ac52c234-61a7-d637-6af5-5c84a6d32a15: added peer, starting replication: peer=server-a0ef52de-67d2-e87f-ecf0-3a477bbdc7c4 -2025-01-25T14:28:20.897Z [INFO] server-ac52c234-61a7-d637-6af5-5c84a6d32a15: added peer, starting replication: peer=server-17c25c52-0283-6ee2-e7cd-1bcd7ba2a995 -2025-01-25T14:28:20.897Z [INFO] server-ac52c234-61a7-d637-6af5-5c84a6d32a15: pipelining replication: peer="{Voter server-a0ef52de-67d2-e87f-ecf0-3a477bbdc7c4 a0ef52de-67d2-e87f-ecf0-3a477bbdc7c4}" -2025-01-25T14:28:20.897Z [INFO] server-ac52c234-61a7-d637-6af5-5c84a6d32a15: pipelining replication: peer="{Voter server-17c25c52-0283-6ee2-e7cd-1bcd7ba2a995 17c25c52-0283-6ee2-e7cd-1bcd7ba2a995}" -2025-01-25T14:28:21.002Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:28:20.897561523 +0000 UTC m=+10.751886505 (2 nodes), 58.139428ms from start of poll, 59.117513ms from cluster start. Timeout at 2025-01-25 14:28:21.002574322 +0000 UTC m=+10.856899264, 105.012759ms after stability -2025-01-25T14:28:21.003Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:21.108Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:21.003051887 +0000 UTC m=+10.857376869 (1 nodes), 1.52µs from start of poll, 164.607877ms from cluster start. Timeout at 2025-01-25 14:28:21.108079926 +0000 UTC m=+10.962404868, 105.027999ms after stability -2025-01-25T14:28:21.117Z [INFO] server-ac52c234-61a7-d637-6af5-5c84a6d32a15: updating configuration: command=RemoveServer server-id=server-ac52c234-61a7-d637-6af5-5c84a6d32a15 server-addr= servers="[{Suffrage:Voter ID:server-a0ef52de-67d2-e87f-ecf0-3a477bbdc7c4 Address:a0ef52de-67d2-e87f-ecf0-3a477bbdc7c4} {Suffrage:Voter ID:server-17c25c52-0283-6ee2-e7cd-1bcd7ba2a995 Address:17c25c52-0283-6ee2-e7cd-1bcd7ba2a995}]" -2025-01-25T14:28:21.117Z [INFO] server-ac52c234-61a7-d637-6af5-5c84a6d32a15: removed ourself, transitioning to follower -2025-01-25T14:28:21.117Z [INFO] server-ac52c234-61a7-d637-6af5-5c84a6d32a15: entering follower state: follower="Node at ac52c234-61a7-d637-6af5-5c84a6d32a15 [Follower]" leader-address= leader-id= -2025-01-25T14:28:21.117Z [INFO] server-ac52c234-61a7-d637-6af5-5c84a6d32a15: aborting pipeline replication: peer="{Voter server-a0ef52de-67d2-e87f-ecf0-3a477bbdc7c4 a0ef52de-67d2-e87f-ecf0-3a477bbdc7c4}" -2025-01-25T14:28:21.117Z [INFO] server-ac52c234-61a7-d637-6af5-5c84a6d32a15: aborting pipeline replication: peer="{Voter server-17c25c52-0283-6ee2-e7cd-1bcd7ba2a995 17c25c52-0283-6ee2-e7cd-1bcd7ba2a995}" -2025-01-25T14:28:21.202Z [WARN] server-ac52c234-61a7-d637-6af5-5c84a6d32a15: not part of stable configuration, aborting election -2025-01-25T14:28:21.212Z [WARN] server-17c25c52-0283-6ee2-e7cd-1bcd7ba2a995: heartbeat timeout reached, starting election: last-leader-addr=ac52c234-61a7-d637-6af5-5c84a6d32a15 last-leader-id=server-ac52c234-61a7-d637-6af5-5c84a6d32a15 -2025-01-25T14:28:21.212Z [INFO] server-17c25c52-0283-6ee2-e7cd-1bcd7ba2a995: entering candidate state: node="Node at 17c25c52-0283-6ee2-e7cd-1bcd7ba2a995 [Candidate]" term=3 -2025-01-25T14:28:21.212Z [WARN] server-a0ef52de-67d2-e87f-ecf0-3a477bbdc7c4: rejecting vote request since we have a leader: from=17c25c52-0283-6ee2-e7cd-1bcd7ba2a995 leader=ac52c234-61a7-d637-6af5-5c84a6d32a15 leader-id=server-ac52c234-61a7-d637-6af5-5c84a6d32a15 -2025-01-25T14:28:21.215Z [WARN] server-a0ef52de-67d2-e87f-ecf0-3a477bbdc7c4: heartbeat timeout reached, starting election: last-leader-addr=ac52c234-61a7-d637-6af5-5c84a6d32a15 last-leader-id=server-ac52c234-61a7-d637-6af5-5c84a6d32a15 -2025-01-25T14:28:21.215Z [INFO] server-a0ef52de-67d2-e87f-ecf0-3a477bbdc7c4: entering candidate state: node="Node at a0ef52de-67d2-e87f-ecf0-3a477bbdc7c4 [Candidate]" term=3 -2025-01-25T14:28:21.215Z [INFO] server-17c25c52-0283-6ee2-e7cd-1bcd7ba2a995: duplicate requestVote for same term: term=3 -2025-01-25T14:28:21.224Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:21.280Z [WARN] server-17c25c52-0283-6ee2-e7cd-1bcd7ba2a995: Election timeout reached, restarting election -2025-01-25T14:28:21.280Z [INFO] server-17c25c52-0283-6ee2-e7cd-1bcd7ba2a995: entering candidate state: node="Node at 17c25c52-0283-6ee2-e7cd-1bcd7ba2a995 [Candidate]" term=4 -2025-01-25T14:28:21.280Z [INFO] server-a0ef52de-67d2-e87f-ecf0-3a477bbdc7c4: entering follower state: follower="Node at a0ef52de-67d2-e87f-ecf0-3a477bbdc7c4 [Follower]" leader-address= leader-id= -2025-01-25T14:28:21.280Z [INFO] server-17c25c52-0283-6ee2-e7cd-1bcd7ba2a995: election won: term=4 tally=2 -2025-01-25T14:28:21.280Z [INFO] server-17c25c52-0283-6ee2-e7cd-1bcd7ba2a995: entering leader state: leader="Node at 17c25c52-0283-6ee2-e7cd-1bcd7ba2a995 [Leader]" -2025-01-25T14:28:21.280Z [INFO] server-17c25c52-0283-6ee2-e7cd-1bcd7ba2a995: added peer, starting replication: peer=server-a0ef52de-67d2-e87f-ecf0-3a477bbdc7c4 -2025-01-25T14:28:21.280Z [INFO] server-17c25c52-0283-6ee2-e7cd-1bcd7ba2a995: pipelining replication: peer="{Voter server-a0ef52de-67d2-e87f-ecf0-3a477bbdc7c4 a0ef52de-67d2-e87f-ecf0-3a477bbdc7c4}" -2025-01-25T14:28:21.386Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:21.281004485 +0000 UTC m=+11.135329467 (1 nodes), 56.923871ms from start of poll, 442.560475ms from cluster start. Timeout at 2025-01-25 14:28:21.386008325 +0000 UTC m=+11.240333307, 105.00384ms after stability -2025-01-25T14:28:21.492Z [INFO] server-17c25c52-0283-6ee2-e7cd-1bcd7ba2a995: aborting pipeline replication: peer="{Voter server-a0ef52de-67d2-e87f-ecf0-3a477bbdc7c4 a0ef52de-67d2-e87f-ecf0-3a477bbdc7c4}" +2025-01-25T14:39:04.971Z [INFO] server-22001eb9-d111-1e35-295f-55884e365dec: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-22001eb9-d111-1e35-295f-55884e365dec Address:22001eb9-d111-1e35-295f-55884e365dec} {Suffrage:Voter ID:server-ba7af99c-c45b-859d-738f-df611325eccc Address:ba7af99c-c45b-859d-738f-df611325eccc} {Suffrage:Voter ID:server-bf77e5b8-ccd8-f3ad-e5bb-7745ded97481 Address:bf77e5b8-ccd8-f3ad-e5bb-7745ded97481}]" +2025-01-25T14:39:04.972Z [INFO] server-22001eb9-d111-1e35-295f-55884e365dec: entering follower state: follower="Node at 22001eb9-d111-1e35-295f-55884e365dec [Follower]" leader-address= leader-id= +2025-01-25T14:39:04.972Z [INFO] server-ba7af99c-c45b-859d-738f-df611325eccc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-22001eb9-d111-1e35-295f-55884e365dec Address:22001eb9-d111-1e35-295f-55884e365dec} {Suffrage:Voter ID:server-ba7af99c-c45b-859d-738f-df611325eccc Address:ba7af99c-c45b-859d-738f-df611325eccc} {Suffrage:Voter ID:server-bf77e5b8-ccd8-f3ad-e5bb-7745ded97481 Address:bf77e5b8-ccd8-f3ad-e5bb-7745ded97481}]" +2025-01-25T14:39:04.973Z [INFO] server-ba7af99c-c45b-859d-738f-df611325eccc: entering follower state: follower="Node at ba7af99c-c45b-859d-738f-df611325eccc [Follower]" leader-address= leader-id= +2025-01-25T14:39:04.973Z [INFO] server-bf77e5b8-ccd8-f3ad-e5bb-7745ded97481: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-22001eb9-d111-1e35-295f-55884e365dec Address:22001eb9-d111-1e35-295f-55884e365dec} {Suffrage:Voter ID:server-ba7af99c-c45b-859d-738f-df611325eccc Address:ba7af99c-c45b-859d-738f-df611325eccc} {Suffrage:Voter ID:server-bf77e5b8-ccd8-f3ad-e5bb-7745ded97481 Address:bf77e5b8-ccd8-f3ad-e5bb-7745ded97481}]" +2025-01-25T14:39:04.973Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:39:04.974Z [INFO] server-bf77e5b8-ccd8-f3ad-e5bb-7745ded97481: entering follower state: follower="Node at bf77e5b8-ccd8-f3ad-e5bb-7745ded97481 [Follower]" leader-address= leader-id= +2025-01-25T14:39:05.057Z [WARN] server-bf77e5b8-ccd8-f3ad-e5bb-7745ded97481: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:05.057Z [INFO] server-bf77e5b8-ccd8-f3ad-e5bb-7745ded97481: entering candidate state: node="Node at bf77e5b8-ccd8-f3ad-e5bb-7745ded97481 [Candidate]" term=2 +2025-01-25T14:39:05.057Z [INFO] server-bf77e5b8-ccd8-f3ad-e5bb-7745ded97481: election won: term=2 tally=2 +2025-01-25T14:39:05.058Z [INFO] server-bf77e5b8-ccd8-f3ad-e5bb-7745ded97481: entering leader state: leader="Node at bf77e5b8-ccd8-f3ad-e5bb-7745ded97481 [Leader]" +2025-01-25T14:39:05.058Z [INFO] server-bf77e5b8-ccd8-f3ad-e5bb-7745ded97481: added peer, starting replication: peer=server-22001eb9-d111-1e35-295f-55884e365dec +2025-01-25T14:39:05.058Z [INFO] server-bf77e5b8-ccd8-f3ad-e5bb-7745ded97481: added peer, starting replication: peer=server-ba7af99c-c45b-859d-738f-df611325eccc +2025-01-25T14:39:05.058Z [INFO] server-bf77e5b8-ccd8-f3ad-e5bb-7745ded97481: pipelining replication: peer="{Voter server-22001eb9-d111-1e35-295f-55884e365dec 22001eb9-d111-1e35-295f-55884e365dec}" +2025-01-25T14:39:05.058Z [INFO] server-bf77e5b8-ccd8-f3ad-e5bb-7745ded97481: pipelining replication: peer="{Voter server-ba7af99c-c45b-859d-738f-df611325eccc ba7af99c-c45b-859d-738f-df611325eccc}" +2025-01-25T14:39:05.164Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:39:05.058258456 +0000 UTC m=+14.104097774 (2 nodes), 84.257495ms from start of poll, 86.940837ms from cluster start. Timeout at 2025-01-25 14:39:05.163268373 +0000 UTC m=+14.209107691, 105.009917ms after stability +2025-01-25T14:39:05.164Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:05.270Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:05.164840642 +0000 UTC m=+14.210679959 (1 nodes), 5.25µs from start of poll, 193.523022ms from cluster start. Timeout at 2025-01-25 14:39:05.269879141 +0000 UTC m=+14.315718459, 105.0385ms after stability +2025-01-25T14:39:05.293Z [INFO] server-bf77e5b8-ccd8-f3ad-e5bb-7745ded97481: updating configuration: command=RemoveServer server-id=server-bf77e5b8-ccd8-f3ad-e5bb-7745ded97481 server-addr= servers="[{Suffrage:Voter ID:server-22001eb9-d111-1e35-295f-55884e365dec Address:22001eb9-d111-1e35-295f-55884e365dec} {Suffrage:Voter ID:server-ba7af99c-c45b-859d-738f-df611325eccc Address:ba7af99c-c45b-859d-738f-df611325eccc}]" +2025-01-25T14:39:05.294Z [INFO] server-bf77e5b8-ccd8-f3ad-e5bb-7745ded97481: removed ourself, transitioning to follower +2025-01-25T14:39:05.294Z [INFO] server-bf77e5b8-ccd8-f3ad-e5bb-7745ded97481: entering follower state: follower="Node at bf77e5b8-ccd8-f3ad-e5bb-7745ded97481 [Follower]" leader-address= leader-id= +2025-01-25T14:39:05.294Z [INFO] server-bf77e5b8-ccd8-f3ad-e5bb-7745ded97481: aborting pipeline replication: peer="{Voter server-22001eb9-d111-1e35-295f-55884e365dec 22001eb9-d111-1e35-295f-55884e365dec}" +2025-01-25T14:39:05.294Z [INFO] server-bf77e5b8-ccd8-f3ad-e5bb-7745ded97481: aborting pipeline replication: peer="{Voter server-ba7af99c-c45b-859d-738f-df611325eccc ba7af99c-c45b-859d-738f-df611325eccc}" +2025-01-25T14:39:05.352Z [WARN] server-22001eb9-d111-1e35-295f-55884e365dec: heartbeat timeout reached, starting election: last-leader-addr=bf77e5b8-ccd8-f3ad-e5bb-7745ded97481 last-leader-id=server-bf77e5b8-ccd8-f3ad-e5bb-7745ded97481 +2025-01-25T14:39:05.352Z [INFO] server-22001eb9-d111-1e35-295f-55884e365dec: entering candidate state: node="Node at 22001eb9-d111-1e35-295f-55884e365dec [Candidate]" term=3 +2025-01-25T14:39:05.352Z [WARN] server-ba7af99c-c45b-859d-738f-df611325eccc: rejecting vote request since we have a leader: from=22001eb9-d111-1e35-295f-55884e365dec leader=bf77e5b8-ccd8-f3ad-e5bb-7745ded97481 leader-id=server-bf77e5b8-ccd8-f3ad-e5bb-7745ded97481 +2025-01-25T14:39:05.360Z [WARN] server-bf77e5b8-ccd8-f3ad-e5bb-7745ded97481: not part of stable configuration, aborting election +2025-01-25T14:39:05.383Z [WARN] server-ba7af99c-c45b-859d-738f-df611325eccc: heartbeat timeout reached, starting election: last-leader-addr=bf77e5b8-ccd8-f3ad-e5bb-7745ded97481 last-leader-id=server-bf77e5b8-ccd8-f3ad-e5bb-7745ded97481 +2025-01-25T14:39:05.383Z [INFO] server-ba7af99c-c45b-859d-738f-df611325eccc: entering candidate state: node="Node at ba7af99c-c45b-859d-738f-df611325eccc [Candidate]" term=3 +2025-01-25T14:39:05.383Z [INFO] server-22001eb9-d111-1e35-295f-55884e365dec: duplicate requestVote for same term: term=3 +2025-01-25T14:39:05.401Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:05.407Z [WARN] server-22001eb9-d111-1e35-295f-55884e365dec: Election timeout reached, restarting election +2025-01-25T14:39:05.407Z [INFO] server-22001eb9-d111-1e35-295f-55884e365dec: entering candidate state: node="Node at 22001eb9-d111-1e35-295f-55884e365dec [Candidate]" term=4 +2025-01-25T14:39:05.407Z [INFO] server-ba7af99c-c45b-859d-738f-df611325eccc: entering follower state: follower="Node at ba7af99c-c45b-859d-738f-df611325eccc [Follower]" leader-address= leader-id= +2025-01-25T14:39:05.408Z [INFO] server-22001eb9-d111-1e35-295f-55884e365dec: election won: term=4 tally=2 +2025-01-25T14:39:05.408Z [INFO] server-22001eb9-d111-1e35-295f-55884e365dec: entering leader state: leader="Node at 22001eb9-d111-1e35-295f-55884e365dec [Leader]" +2025-01-25T14:39:05.408Z [INFO] server-22001eb9-d111-1e35-295f-55884e365dec: added peer, starting replication: peer=server-ba7af99c-c45b-859d-738f-df611325eccc +2025-01-25T14:39:05.408Z [INFO] server-22001eb9-d111-1e35-295f-55884e365dec: pipelining replication: peer="{Voter server-ba7af99c-c45b-859d-738f-df611325eccc ba7af99c-c45b-859d-738f-df611325eccc}" +2025-01-25T14:39:05.514Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:05.408305184 +0000 UTC m=+14.454144501 (1 nodes), 6.236476ms from start of poll, 436.987564ms from cluster start. Timeout at 2025-01-25 14:39:05.513334933 +0000 UTC m=+14.559174251, 105.02975ms after stability +2025-01-25T14:39:05.620Z [INFO] server-22001eb9-d111-1e35-295f-55884e365dec: aborting pipeline replication: peer="{Voter server-ba7af99c-c45b-859d-738f-df611325eccc ba7af99c-c45b-859d-738f-df611325eccc}" --- PASS: TestRaft_RemoveLeader_NoShutdown (0.66s) === RUN TestRaft_RemoveFollower_SplitCluster -2025-01-25T14:28:21.495Z [INFO] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b6c58e15-48df-3152-f01b-b31b19b7aa67 Address:b6c58e15-48df-3152-f01b-b31b19b7aa67} {Suffrage:Voter ID:server-fb7430f8-6ee3-1406-dbe8-c946d0093ef2 Address:fb7430f8-6ee3-1406-dbe8-c946d0093ef2} {Suffrage:Voter ID:server-d8b4de19-dab6-d624-44b2-3032a71b6029 Address:d8b4de19-dab6-d624-44b2-3032a71b6029} {Suffrage:Voter ID:server-3ea6bb86-68ac-01f0-b227-2995e4ea1456 Address:3ea6bb86-68ac-01f0-b227-2995e4ea1456}]" -2025-01-25T14:28:21.495Z [INFO] server-fb7430f8-6ee3-1406-dbe8-c946d0093ef2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b6c58e15-48df-3152-f01b-b31b19b7aa67 Address:b6c58e15-48df-3152-f01b-b31b19b7aa67} {Suffrage:Voter ID:server-fb7430f8-6ee3-1406-dbe8-c946d0093ef2 Address:fb7430f8-6ee3-1406-dbe8-c946d0093ef2} {Suffrage:Voter ID:server-d8b4de19-dab6-d624-44b2-3032a71b6029 Address:d8b4de19-dab6-d624-44b2-3032a71b6029} {Suffrage:Voter ID:server-3ea6bb86-68ac-01f0-b227-2995e4ea1456 Address:3ea6bb86-68ac-01f0-b227-2995e4ea1456}]" -2025-01-25T14:28:21.495Z [INFO] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: entering follower state: follower="Node at b6c58e15-48df-3152-f01b-b31b19b7aa67 [Follower]" leader-address= leader-id= -2025-01-25T14:28:21.495Z [INFO] server-d8b4de19-dab6-d624-44b2-3032a71b6029: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b6c58e15-48df-3152-f01b-b31b19b7aa67 Address:b6c58e15-48df-3152-f01b-b31b19b7aa67} {Suffrage:Voter ID:server-fb7430f8-6ee3-1406-dbe8-c946d0093ef2 Address:fb7430f8-6ee3-1406-dbe8-c946d0093ef2} {Suffrage:Voter ID:server-d8b4de19-dab6-d624-44b2-3032a71b6029 Address:d8b4de19-dab6-d624-44b2-3032a71b6029} {Suffrage:Voter ID:server-3ea6bb86-68ac-01f0-b227-2995e4ea1456 Address:3ea6bb86-68ac-01f0-b227-2995e4ea1456}]" -2025-01-25T14:28:21.495Z [INFO] server-fb7430f8-6ee3-1406-dbe8-c946d0093ef2: entering follower state: follower="Node at fb7430f8-6ee3-1406-dbe8-c946d0093ef2 [Follower]" leader-address= leader-id= -2025-01-25T14:28:21.495Z [INFO] server-d8b4de19-dab6-d624-44b2-3032a71b6029: entering follower state: follower="Node at d8b4de19-dab6-d624-44b2-3032a71b6029 [Follower]" leader-address= leader-id= -2025-01-25T14:28:21.495Z [INFO] server-3ea6bb86-68ac-01f0-b227-2995e4ea1456: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b6c58e15-48df-3152-f01b-b31b19b7aa67 Address:b6c58e15-48df-3152-f01b-b31b19b7aa67} {Suffrage:Voter ID:server-fb7430f8-6ee3-1406-dbe8-c946d0093ef2 Address:fb7430f8-6ee3-1406-dbe8-c946d0093ef2} {Suffrage:Voter ID:server-d8b4de19-dab6-d624-44b2-3032a71b6029 Address:d8b4de19-dab6-d624-44b2-3032a71b6029} {Suffrage:Voter ID:server-3ea6bb86-68ac-01f0-b227-2995e4ea1456 Address:3ea6bb86-68ac-01f0-b227-2995e4ea1456}]" -2025-01-25T14:28:21.495Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:21.495Z [INFO] server-3ea6bb86-68ac-01f0-b227-2995e4ea1456: entering follower state: follower="Node at 3ea6bb86-68ac-01f0-b227-2995e4ea1456 [Follower]" leader-address= leader-id= -2025-01-25T14:28:21.548Z [WARN] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:21.548Z [INFO] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: entering candidate state: node="Node at b6c58e15-48df-3152-f01b-b31b19b7aa67 [Candidate]" term=2 -2025-01-25T14:28:21.548Z [INFO] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: election won: term=2 tally=3 -2025-01-25T14:28:21.548Z [INFO] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: entering leader state: leader="Node at b6c58e15-48df-3152-f01b-b31b19b7aa67 [Leader]" -2025-01-25T14:28:21.548Z [INFO] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: added peer, starting replication: peer=server-fb7430f8-6ee3-1406-dbe8-c946d0093ef2 -2025-01-25T14:28:21.548Z [INFO] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: added peer, starting replication: peer=server-d8b4de19-dab6-d624-44b2-3032a71b6029 -2025-01-25T14:28:21.548Z [INFO] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: added peer, starting replication: peer=server-3ea6bb86-68ac-01f0-b227-2995e4ea1456 -2025-01-25T14:28:21.549Z [INFO] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: pipelining replication: peer="{Voter server-3ea6bb86-68ac-01f0-b227-2995e4ea1456 3ea6bb86-68ac-01f0-b227-2995e4ea1456}" -2025-01-25T14:28:21.549Z [INFO] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: pipelining replication: peer="{Voter server-fb7430f8-6ee3-1406-dbe8-c946d0093ef2 fb7430f8-6ee3-1406-dbe8-c946d0093ef2}" -2025-01-25T14:28:21.549Z [INFO] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: pipelining replication: peer="{Voter server-d8b4de19-dab6-d624-44b2-3032a71b6029 d8b4de19-dab6-d624-44b2-3032a71b6029}" -2025-01-25T14:28:21.654Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:21.548977522 +0000 UTC m=+11.403302504 (1 nodes), 53.006212ms from start of poll, 53.963298ms from cluster start. Timeout at 2025-01-25 14:28:21.653982401 +0000 UTC m=+11.508307383, 105.004879ms after stability -2025-01-25T14:28:21.761Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:28:21.866Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:28:21.76108341 +0000 UTC m=+11.615408392 (3 nodes), 1.84µs from start of poll, 266.069186ms from cluster start. Timeout at 2025-01-25 14:28:21.866121649 +0000 UTC m=+11.720446591, 105.038199ms after stability -2025-01-25T14:28:21.866Z [INFO] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: updating configuration: command=RemoveServer server-id=server-3ea6bb86-68ac-01f0-b227-2995e4ea1456 server-addr= servers="[{Suffrage:Voter ID:server-b6c58e15-48df-3152-f01b-b31b19b7aa67 Address:b6c58e15-48df-3152-f01b-b31b19b7aa67} {Suffrage:Voter ID:server-fb7430f8-6ee3-1406-dbe8-c946d0093ef2 Address:fb7430f8-6ee3-1406-dbe8-c946d0093ef2} {Suffrage:Voter ID:server-d8b4de19-dab6-d624-44b2-3032a71b6029 Address:d8b4de19-dab6-d624-44b2-3032a71b6029}]" -2025-01-25T14:28:21.866Z [ERROR] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: failed to heartbeat to: peer=fb7430f8-6ee3-1406-dbe8-c946d0093ef2 backoff time=10ms error="failed to connect to peer: fb7430f8-6ee3-1406-dbe8-c946d0093ef2" -2025-01-25T14:28:21.866Z [ERROR] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: failed to pipeline appendEntries: peer="{Voter server-fb7430f8-6ee3-1406-dbe8-c946d0093ef2 fb7430f8-6ee3-1406-dbe8-c946d0093ef2}" error="append pipeline closed" -2025-01-25T14:28:21.866Z [INFO] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: aborting pipeline replication: peer="{Voter server-fb7430f8-6ee3-1406-dbe8-c946d0093ef2 fb7430f8-6ee3-1406-dbe8-c946d0093ef2}" -2025-01-25T14:28:21.866Z [INFO] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: removed peer, stopping replication: peer=server-3ea6bb86-68ac-01f0-b227-2995e4ea1456 last-index=3 -2025-01-25T14:28:21.866Z [ERROR] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: failed to pipeline appendEntries: peer="{Voter server-d8b4de19-dab6-d624-44b2-3032a71b6029 d8b4de19-dab6-d624-44b2-3032a71b6029}" error="append pipeline closed" -2025-01-25T14:28:21.866Z [INFO] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: aborting pipeline replication: peer="{Voter server-d8b4de19-dab6-d624-44b2-3032a71b6029 d8b4de19-dab6-d624-44b2-3032a71b6029}" -2025-01-25T14:28:21.866Z [INFO] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: aborting pipeline replication: peer="{Voter server-3ea6bb86-68ac-01f0-b227-2995e4ea1456 3ea6bb86-68ac-01f0-b227-2995e4ea1456}" -2025-01-25T14:28:21.872Z [ERROR] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: failed to appendEntries to: peer="{Voter server-fb7430f8-6ee3-1406-dbe8-c946d0093ef2 fb7430f8-6ee3-1406-dbe8-c946d0093ef2}" error="failed to connect to peer: fb7430f8-6ee3-1406-dbe8-c946d0093ef2" -2025-01-25T14:28:21.879Z [ERROR] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: failed to heartbeat to: peer=d8b4de19-dab6-d624-44b2-3032a71b6029 backoff time=10ms error="failed to connect to peer: d8b4de19-dab6-d624-44b2-3032a71b6029" -2025-01-25T14:28:21.879Z [ERROR] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: failed to appendEntries to: peer="{Voter server-d8b4de19-dab6-d624-44b2-3032a71b6029 d8b4de19-dab6-d624-44b2-3032a71b6029}" error="failed to connect to peer: d8b4de19-dab6-d624-44b2-3032a71b6029" -2025-01-25T14:28:21.885Z [ERROR] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: failed to heartbeat to: peer=fb7430f8-6ee3-1406-dbe8-c946d0093ef2 backoff time=10ms error="failed to connect to peer: fb7430f8-6ee3-1406-dbe8-c946d0093ef2" -2025-01-25T14:28:21.893Z [ERROR] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: failed to appendEntries to: peer="{Voter server-fb7430f8-6ee3-1406-dbe8-c946d0093ef2 fb7430f8-6ee3-1406-dbe8-c946d0093ef2}" error="failed to connect to peer: fb7430f8-6ee3-1406-dbe8-c946d0093ef2" -2025-01-25T14:28:21.897Z [ERROR] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: failed to heartbeat to: peer=d8b4de19-dab6-d624-44b2-3032a71b6029 backoff time=10ms error="failed to connect to peer: d8b4de19-dab6-d624-44b2-3032a71b6029" -2025-01-25T14:28:21.897Z [ERROR] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: failed to appendEntries to: peer="{Voter server-d8b4de19-dab6-d624-44b2-3032a71b6029 d8b4de19-dab6-d624-44b2-3032a71b6029}" error="failed to connect to peer: d8b4de19-dab6-d624-44b2-3032a71b6029" -2025-01-25T14:28:21.904Z [ERROR] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: failed to heartbeat to: peer=fb7430f8-6ee3-1406-dbe8-c946d0093ef2 backoff time=10ms error="failed to connect to peer: fb7430f8-6ee3-1406-dbe8-c946d0093ef2" -2025-01-25T14:28:21.908Z [WARN] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: failed to contact: server-id=server-fb7430f8-6ee3-1406-dbe8-c946d0093ef2 time=50.109157ms -2025-01-25T14:28:21.912Z [ERROR] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: failed to appendEntries to: peer="{Voter server-fb7430f8-6ee3-1406-dbe8-c946d0093ef2 fb7430f8-6ee3-1406-dbe8-c946d0093ef2}" error="failed to connect to peer: fb7430f8-6ee3-1406-dbe8-c946d0093ef2" -2025-01-25T14:28:21.915Z [ERROR] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: failed to heartbeat to: peer=d8b4de19-dab6-d624-44b2-3032a71b6029 backoff time=10ms error="failed to connect to peer: d8b4de19-dab6-d624-44b2-3032a71b6029" -2025-01-25T14:28:21.916Z [ERROR] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: failed to appendEntries to: peer="{Voter server-d8b4de19-dab6-d624-44b2-3032a71b6029 d8b4de19-dab6-d624-44b2-3032a71b6029}" error="failed to connect to peer: d8b4de19-dab6-d624-44b2-3032a71b6029" -2025-01-25T14:28:21.918Z [WARN] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: failed to contact: server-id=server-fb7430f8-6ee3-1406-dbe8-c946d0093ef2 time=60.376027ms -2025-01-25T14:28:21.918Z [WARN] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: failed to contact: server-id=server-d8b4de19-dab6-d624-44b2-3032a71b6029 time=52.440472ms -2025-01-25T14:28:21.918Z [WARN] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: failed to contact quorum of nodes, stepping down -2025-01-25T14:28:21.918Z [INFO] server-b6c58e15-48df-3152-f01b-b31b19b7aa67: entering follower state: follower="Node at b6c58e15-48df-3152-f01b-b31b19b7aa67 [Follower]" leader-address= leader-id= ---- PASS: TestRaft_RemoveFollower_SplitCluster (0.43s) +2025-01-25T14:39:05.630Z [INFO] server-3e957dfb-d374-81ba-cb33-f1421a3224b7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3e957dfb-d374-81ba-cb33-f1421a3224b7 Address:3e957dfb-d374-81ba-cb33-f1421a3224b7} {Suffrage:Voter ID:server-b0ca88ca-cf01-3bd6-c985-9274d678b422 Address:b0ca88ca-cf01-3bd6-c985-9274d678b422} {Suffrage:Voter ID:server-e04a3e3a-ca83-98be-b615-8ed42844b671 Address:e04a3e3a-ca83-98be-b615-8ed42844b671} {Suffrage:Voter ID:server-75dbab02-265b-4f98-096a-3f0ab1d99e27 Address:75dbab02-265b-4f98-096a-3f0ab1d99e27}]" +2025-01-25T14:39:05.631Z [INFO] server-3e957dfb-d374-81ba-cb33-f1421a3224b7: entering follower state: follower="Node at 3e957dfb-d374-81ba-cb33-f1421a3224b7 [Follower]" leader-address= leader-id= +2025-01-25T14:39:05.631Z [INFO] server-b0ca88ca-cf01-3bd6-c985-9274d678b422: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3e957dfb-d374-81ba-cb33-f1421a3224b7 Address:3e957dfb-d374-81ba-cb33-f1421a3224b7} {Suffrage:Voter ID:server-b0ca88ca-cf01-3bd6-c985-9274d678b422 Address:b0ca88ca-cf01-3bd6-c985-9274d678b422} {Suffrage:Voter ID:server-e04a3e3a-ca83-98be-b615-8ed42844b671 Address:e04a3e3a-ca83-98be-b615-8ed42844b671} {Suffrage:Voter ID:server-75dbab02-265b-4f98-096a-3f0ab1d99e27 Address:75dbab02-265b-4f98-096a-3f0ab1d99e27}]" +2025-01-25T14:39:05.631Z [INFO] server-b0ca88ca-cf01-3bd6-c985-9274d678b422: entering follower state: follower="Node at b0ca88ca-cf01-3bd6-c985-9274d678b422 [Follower]" leader-address= leader-id= +2025-01-25T14:39:05.632Z [INFO] server-e04a3e3a-ca83-98be-b615-8ed42844b671: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3e957dfb-d374-81ba-cb33-f1421a3224b7 Address:3e957dfb-d374-81ba-cb33-f1421a3224b7} {Suffrage:Voter ID:server-b0ca88ca-cf01-3bd6-c985-9274d678b422 Address:b0ca88ca-cf01-3bd6-c985-9274d678b422} {Suffrage:Voter ID:server-e04a3e3a-ca83-98be-b615-8ed42844b671 Address:e04a3e3a-ca83-98be-b615-8ed42844b671} {Suffrage:Voter ID:server-75dbab02-265b-4f98-096a-3f0ab1d99e27 Address:75dbab02-265b-4f98-096a-3f0ab1d99e27}]" +2025-01-25T14:39:05.632Z [INFO] server-e04a3e3a-ca83-98be-b615-8ed42844b671: entering follower state: follower="Node at e04a3e3a-ca83-98be-b615-8ed42844b671 [Follower]" leader-address= leader-id= +2025-01-25T14:39:05.633Z [INFO] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3e957dfb-d374-81ba-cb33-f1421a3224b7 Address:3e957dfb-d374-81ba-cb33-f1421a3224b7} {Suffrage:Voter ID:server-b0ca88ca-cf01-3bd6-c985-9274d678b422 Address:b0ca88ca-cf01-3bd6-c985-9274d678b422} {Suffrage:Voter ID:server-e04a3e3a-ca83-98be-b615-8ed42844b671 Address:e04a3e3a-ca83-98be-b615-8ed42844b671} {Suffrage:Voter ID:server-75dbab02-265b-4f98-096a-3f0ab1d99e27 Address:75dbab02-265b-4f98-096a-3f0ab1d99e27}]" +2025-01-25T14:39:05.634Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:05.635Z [INFO] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: entering follower state: follower="Node at 75dbab02-265b-4f98-096a-3f0ab1d99e27 [Follower]" leader-address= leader-id= +2025-01-25T14:39:05.708Z [WARN] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:05.709Z [INFO] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: entering candidate state: node="Node at 75dbab02-265b-4f98-096a-3f0ab1d99e27 [Candidate]" term=2 +2025-01-25T14:39:05.709Z [INFO] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: election won: term=2 tally=3 +2025-01-25T14:39:05.709Z [INFO] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: entering leader state: leader="Node at 75dbab02-265b-4f98-096a-3f0ab1d99e27 [Leader]" +2025-01-25T14:39:05.709Z [INFO] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: added peer, starting replication: peer=server-3e957dfb-d374-81ba-cb33-f1421a3224b7 +2025-01-25T14:39:05.710Z [INFO] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: added peer, starting replication: peer=server-b0ca88ca-cf01-3bd6-c985-9274d678b422 +2025-01-25T14:39:05.710Z [INFO] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: added peer, starting replication: peer=server-e04a3e3a-ca83-98be-b615-8ed42844b671 +2025-01-25T14:39:05.710Z [INFO] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: pipelining replication: peer="{Voter server-b0ca88ca-cf01-3bd6-c985-9274d678b422 b0ca88ca-cf01-3bd6-c985-9274d678b422}" +2025-01-25T14:39:05.710Z [INFO] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: pipelining replication: peer="{Voter server-3e957dfb-d374-81ba-cb33-f1421a3224b7 3e957dfb-d374-81ba-cb33-f1421a3224b7}" +2025-01-25T14:39:05.710Z [INFO] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: pipelining replication: peer="{Voter server-e04a3e3a-ca83-98be-b615-8ed42844b671 e04a3e3a-ca83-98be-b615-8ed42844b671}" +2025-01-25T14:39:05.815Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:05.709939479 +0000 UTC m=+14.755778796 (1 nodes), 75.12076ms from start of poll, 79.954816ms from cluster start. Timeout at 2025-01-25 14:39:05.814963103 +0000 UTC m=+14.860802713, 105.023917ms after stability +2025-01-25T14:39:05.921Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:39:06.027Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:39:05.921645762 +0000 UTC m=+14.967485079 (3 nodes), 8.166µs from start of poll, 291.661099ms from cluster start. Timeout at 2025-01-25 14:39:06.026732095 +0000 UTC m=+15.072571413, 105.086334ms after stability +2025-01-25T14:39:06.027Z [INFO] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: updating configuration: command=RemoveServer server-id=server-e04a3e3a-ca83-98be-b615-8ed42844b671 server-addr= servers="[{Suffrage:Voter ID:server-3e957dfb-d374-81ba-cb33-f1421a3224b7 Address:3e957dfb-d374-81ba-cb33-f1421a3224b7} {Suffrage:Voter ID:server-b0ca88ca-cf01-3bd6-c985-9274d678b422 Address:b0ca88ca-cf01-3bd6-c985-9274d678b422} {Suffrage:Voter ID:server-75dbab02-265b-4f98-096a-3f0ab1d99e27 Address:75dbab02-265b-4f98-096a-3f0ab1d99e27}]" +2025-01-25T14:39:06.028Z [INFO] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: removed peer, stopping replication: peer=server-e04a3e3a-ca83-98be-b615-8ed42844b671 last-index=3 +2025-01-25T14:39:06.028Z [ERROR] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: failed to pipeline appendEntries: peer="{Voter server-3e957dfb-d374-81ba-cb33-f1421a3224b7 3e957dfb-d374-81ba-cb33-f1421a3224b7}" error="append pipeline closed" +2025-01-25T14:39:06.028Z [ERROR] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: failed to pipeline appendEntries: peer="{Voter server-b0ca88ca-cf01-3bd6-c985-9274d678b422 b0ca88ca-cf01-3bd6-c985-9274d678b422}" error="append pipeline closed" +2025-01-25T14:39:06.028Z [INFO] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: aborting pipeline replication: peer="{Voter server-3e957dfb-d374-81ba-cb33-f1421a3224b7 3e957dfb-d374-81ba-cb33-f1421a3224b7}" +2025-01-25T14:39:06.028Z [INFO] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: aborting pipeline replication: peer="{Voter server-e04a3e3a-ca83-98be-b615-8ed42844b671 e04a3e3a-ca83-98be-b615-8ed42844b671}" +2025-01-25T14:39:06.028Z [INFO] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: aborting pipeline replication: peer="{Voter server-b0ca88ca-cf01-3bd6-c985-9274d678b422 b0ca88ca-cf01-3bd6-c985-9274d678b422}" +2025-01-25T14:39:06.033Z [ERROR] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: failed to heartbeat to: peer=b0ca88ca-cf01-3bd6-c985-9274d678b422 backoff time=10ms error="failed to connect to peer: b0ca88ca-cf01-3bd6-c985-9274d678b422" +2025-01-25T14:39:06.035Z [ERROR] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: failed to heartbeat to: peer=3e957dfb-d374-81ba-cb33-f1421a3224b7 backoff time=10ms error="failed to connect to peer: 3e957dfb-d374-81ba-cb33-f1421a3224b7" +2025-01-25T14:39:06.038Z [ERROR] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: failed to appendEntries to: peer="{Voter server-3e957dfb-d374-81ba-cb33-f1421a3224b7 3e957dfb-d374-81ba-cb33-f1421a3224b7}" error="failed to connect to peer: 3e957dfb-d374-81ba-cb33-f1421a3224b7" +2025-01-25T14:39:06.039Z [ERROR] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: failed to appendEntries to: peer="{Voter server-b0ca88ca-cf01-3bd6-c985-9274d678b422 b0ca88ca-cf01-3bd6-c985-9274d678b422}" error="failed to connect to peer: b0ca88ca-cf01-3bd6-c985-9274d678b422" +2025-01-25T14:39:06.052Z [ERROR] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: failed to heartbeat to: peer=b0ca88ca-cf01-3bd6-c985-9274d678b422 backoff time=10ms error="failed to connect to peer: b0ca88ca-cf01-3bd6-c985-9274d678b422" +2025-01-25T14:39:06.052Z [ERROR] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: failed to heartbeat to: peer=3e957dfb-d374-81ba-cb33-f1421a3224b7 backoff time=10ms error="failed to connect to peer: 3e957dfb-d374-81ba-cb33-f1421a3224b7" +2025-01-25T14:39:06.059Z [ERROR] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: failed to appendEntries to: peer="{Voter server-3e957dfb-d374-81ba-cb33-f1421a3224b7 3e957dfb-d374-81ba-cb33-f1421a3224b7}" error="failed to connect to peer: 3e957dfb-d374-81ba-cb33-f1421a3224b7" +2025-01-25T14:39:06.059Z [ERROR] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: failed to appendEntries to: peer="{Voter server-b0ca88ca-cf01-3bd6-c985-9274d678b422 b0ca88ca-cf01-3bd6-c985-9274d678b422}" error="failed to connect to peer: b0ca88ca-cf01-3bd6-c985-9274d678b422" +2025-01-25T14:39:06.069Z [ERROR] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: failed to heartbeat to: peer=b0ca88ca-cf01-3bd6-c985-9274d678b422 backoff time=10ms error="failed to connect to peer: b0ca88ca-cf01-3bd6-c985-9274d678b422" +2025-01-25T14:39:06.070Z [ERROR] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: failed to heartbeat to: peer=3e957dfb-d374-81ba-cb33-f1421a3224b7 backoff time=10ms error="failed to connect to peer: 3e957dfb-d374-81ba-cb33-f1421a3224b7" +2025-01-25T14:39:06.076Z [ERROR] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: failed to appendEntries to: peer="{Voter server-b0ca88ca-cf01-3bd6-c985-9274d678b422 b0ca88ca-cf01-3bd6-c985-9274d678b422}" error="failed to connect to peer: b0ca88ca-cf01-3bd6-c985-9274d678b422" +2025-01-25T14:39:06.078Z [WARN] server-b0ca88ca-cf01-3bd6-c985-9274d678b422: heartbeat timeout reached, starting election: last-leader-addr=75dbab02-265b-4f98-096a-3f0ab1d99e27 last-leader-id=server-75dbab02-265b-4f98-096a-3f0ab1d99e27 +2025-01-25T14:39:06.078Z [WARN] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: failed to contact: server-id=server-3e957dfb-d374-81ba-cb33-f1421a3224b7 time=51.190274ms +2025-01-25T14:39:06.079Z [WARN] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: failed to contact: server-id=server-b0ca88ca-cf01-3bd6-c985-9274d678b422 time=51.276607ms +2025-01-25T14:39:06.079Z [WARN] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: failed to contact quorum of nodes, stepping down +2025-01-25T14:39:06.078Z [INFO] server-b0ca88ca-cf01-3bd6-c985-9274d678b422: entering candidate state: node="Node at b0ca88ca-cf01-3bd6-c985-9274d678b422 [Candidate]" term=3 +2025-01-25T14:39:06.079Z [INFO] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: entering follower state: follower="Node at 75dbab02-265b-4f98-096a-3f0ab1d99e27 [Follower]" leader-address= leader-id= +2025-01-25T14:39:06.079Z [ERROR] server-b0ca88ca-cf01-3bd6-c985-9274d678b422: failed to make requestVote RPC: target="{Voter server-e04a3e3a-ca83-98be-b615-8ed42844b671 e04a3e3a-ca83-98be-b615-8ed42844b671}" error="failed to connect to peer: e04a3e3a-ca83-98be-b615-8ed42844b671" term=3 +2025-01-25T14:39:06.079Z [ERROR] server-75dbab02-265b-4f98-096a-3f0ab1d99e27: failed to appendEntries to: peer="{Voter server-3e957dfb-d374-81ba-cb33-f1421a3224b7 3e957dfb-d374-81ba-cb33-f1421a3224b7}" error="failed to connect to peer: 3e957dfb-d374-81ba-cb33-f1421a3224b7" +2025-01-25T14:39:06.079Z [ERROR] server-b0ca88ca-cf01-3bd6-c985-9274d678b422: failed to make requestVote RPC: target="{Voter server-75dbab02-265b-4f98-096a-3f0ab1d99e27 75dbab02-265b-4f98-096a-3f0ab1d99e27}" error="failed to connect to peer: 75dbab02-265b-4f98-096a-3f0ab1d99e27" term=3 +--- PASS: TestRaft_RemoveFollower_SplitCluster (0.46s) === RUN TestRaft_AddKnownPeer -2025-01-25T14:28:21.921Z [INFO] server-90b306c5-c416-2a48-4f30-74063a43b43e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-90b306c5-c416-2a48-4f30-74063a43b43e Address:90b306c5-c416-2a48-4f30-74063a43b43e} {Suffrage:Voter ID:server-8c5280e2-4471-9752-12ba-76c0ba9daf7f Address:8c5280e2-4471-9752-12ba-76c0ba9daf7f} {Suffrage:Voter ID:server-7b6b3118-69a9-ec91-912a-adcd732057cd Address:7b6b3118-69a9-ec91-912a-adcd732057cd}]" -2025-01-25T14:28:21.921Z [INFO] server-90b306c5-c416-2a48-4f30-74063a43b43e: entering follower state: follower="Node at 90b306c5-c416-2a48-4f30-74063a43b43e [Follower]" leader-address= leader-id= -2025-01-25T14:28:21.921Z [INFO] server-8c5280e2-4471-9752-12ba-76c0ba9daf7f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-90b306c5-c416-2a48-4f30-74063a43b43e Address:90b306c5-c416-2a48-4f30-74063a43b43e} {Suffrage:Voter ID:server-8c5280e2-4471-9752-12ba-76c0ba9daf7f Address:8c5280e2-4471-9752-12ba-76c0ba9daf7f} {Suffrage:Voter ID:server-7b6b3118-69a9-ec91-912a-adcd732057cd Address:7b6b3118-69a9-ec91-912a-adcd732057cd}]" -2025-01-25T14:28:21.922Z [INFO] server-8c5280e2-4471-9752-12ba-76c0ba9daf7f: entering follower state: follower="Node at 8c5280e2-4471-9752-12ba-76c0ba9daf7f [Follower]" leader-address= leader-id= -2025-01-25T14:28:21.922Z [INFO] server-7b6b3118-69a9-ec91-912a-adcd732057cd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-90b306c5-c416-2a48-4f30-74063a43b43e Address:90b306c5-c416-2a48-4f30-74063a43b43e} {Suffrage:Voter ID:server-8c5280e2-4471-9752-12ba-76c0ba9daf7f Address:8c5280e2-4471-9752-12ba-76c0ba9daf7f} {Suffrage:Voter ID:server-7b6b3118-69a9-ec91-912a-adcd732057cd Address:7b6b3118-69a9-ec91-912a-adcd732057cd}]" -2025-01-25T14:28:21.922Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:21.922Z [INFO] server-7b6b3118-69a9-ec91-912a-adcd732057cd: entering follower state: follower="Node at 7b6b3118-69a9-ec91-912a-adcd732057cd [Follower]" leader-address= leader-id= -2025-01-25T14:28:21.989Z [WARN] server-8c5280e2-4471-9752-12ba-76c0ba9daf7f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:21.990Z [INFO] server-8c5280e2-4471-9752-12ba-76c0ba9daf7f: entering candidate state: node="Node at 8c5280e2-4471-9752-12ba-76c0ba9daf7f [Candidate]" term=2 -2025-01-25T14:28:21.990Z [INFO] server-8c5280e2-4471-9752-12ba-76c0ba9daf7f: election won: term=2 tally=2 -2025-01-25T14:28:21.990Z [INFO] server-8c5280e2-4471-9752-12ba-76c0ba9daf7f: entering leader state: leader="Node at 8c5280e2-4471-9752-12ba-76c0ba9daf7f [Leader]" -2025-01-25T14:28:21.990Z [INFO] server-8c5280e2-4471-9752-12ba-76c0ba9daf7f: added peer, starting replication: peer=server-90b306c5-c416-2a48-4f30-74063a43b43e -2025-01-25T14:28:21.990Z [INFO] server-8c5280e2-4471-9752-12ba-76c0ba9daf7f: added peer, starting replication: peer=server-7b6b3118-69a9-ec91-912a-adcd732057cd -2025-01-25T14:28:21.990Z [INFO] server-8c5280e2-4471-9752-12ba-76c0ba9daf7f: pipelining replication: peer="{Voter server-90b306c5-c416-2a48-4f30-74063a43b43e 90b306c5-c416-2a48-4f30-74063a43b43e}" -2025-01-25T14:28:21.990Z [INFO] server-8c5280e2-4471-9752-12ba-76c0ba9daf7f: pipelining replication: peer="{Voter server-7b6b3118-69a9-ec91-912a-adcd732057cd 7b6b3118-69a9-ec91-912a-adcd732057cd}" -2025-01-25T14:28:22.095Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:21.990438077 +0000 UTC m=+11.844763059 (1 nodes), 68.186186ms from start of poll, 68.91756ms from cluster start. Timeout at 2025-01-25 14:28:22.095443277 +0000 UTC m=+11.949768259, 105.0052ms after stability -2025-01-25T14:28:22.096Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:28:22.201Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:28:22.096095435 +0000 UTC m=+11.950420417 (2 nodes), 2.08µs from start of poll, 174.574918ms from cluster start. Timeout at 2025-01-25 14:28:22.201149273 +0000 UTC m=+12.055474215, 105.053798ms after stability -2025-01-25T14:28:22.201Z [INFO] server-8c5280e2-4471-9752-12ba-76c0ba9daf7f: updating configuration: command=AddVoter server-id=server-90b306c5-c416-2a48-4f30-74063a43b43e server-addr=90b306c5-c416-2a48-4f30-74063a43b43e servers="[{Suffrage:Voter ID:server-90b306c5-c416-2a48-4f30-74063a43b43e Address:90b306c5-c416-2a48-4f30-74063a43b43e} {Suffrage:Voter ID:server-8c5280e2-4471-9752-12ba-76c0ba9daf7f Address:8c5280e2-4471-9752-12ba-76c0ba9daf7f} {Suffrage:Voter ID:server-7b6b3118-69a9-ec91-912a-adcd732057cd Address:7b6b3118-69a9-ec91-912a-adcd732057cd}]" -2025-01-25T14:28:22.202Z [INFO] server-8c5280e2-4471-9752-12ba-76c0ba9daf7f: aborting pipeline replication: peer="{Voter server-7b6b3118-69a9-ec91-912a-adcd732057cd 7b6b3118-69a9-ec91-912a-adcd732057cd}" -2025-01-25T14:28:22.202Z [INFO] server-8c5280e2-4471-9752-12ba-76c0ba9daf7f: aborting pipeline replication: peer="{Voter server-90b306c5-c416-2a48-4f30-74063a43b43e 90b306c5-c416-2a48-4f30-74063a43b43e}" +2025-01-25T14:39:06.088Z [INFO] server-983ff90d-0907-c381-5355-1add497d80cc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-983ff90d-0907-c381-5355-1add497d80cc Address:983ff90d-0907-c381-5355-1add497d80cc} {Suffrage:Voter ID:server-5f67d65d-8132-2c67-a0d6-7341883326ff Address:5f67d65d-8132-2c67-a0d6-7341883326ff} {Suffrage:Voter ID:server-aca92b9d-3a74-5116-f8bd-50a236a60bb6 Address:aca92b9d-3a74-5116-f8bd-50a236a60bb6}]" +2025-01-25T14:39:06.089Z [INFO] server-983ff90d-0907-c381-5355-1add497d80cc: entering follower state: follower="Node at 983ff90d-0907-c381-5355-1add497d80cc [Follower]" leader-address= leader-id= +2025-01-25T14:39:06.089Z [INFO] server-5f67d65d-8132-2c67-a0d6-7341883326ff: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-983ff90d-0907-c381-5355-1add497d80cc Address:983ff90d-0907-c381-5355-1add497d80cc} {Suffrage:Voter ID:server-5f67d65d-8132-2c67-a0d6-7341883326ff Address:5f67d65d-8132-2c67-a0d6-7341883326ff} {Suffrage:Voter ID:server-aca92b9d-3a74-5116-f8bd-50a236a60bb6 Address:aca92b9d-3a74-5116-f8bd-50a236a60bb6}]" +2025-01-25T14:39:06.089Z [INFO] server-5f67d65d-8132-2c67-a0d6-7341883326ff: entering follower state: follower="Node at 5f67d65d-8132-2c67-a0d6-7341883326ff [Follower]" leader-address= leader-id= +2025-01-25T14:39:06.090Z [INFO] server-aca92b9d-3a74-5116-f8bd-50a236a60bb6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-983ff90d-0907-c381-5355-1add497d80cc Address:983ff90d-0907-c381-5355-1add497d80cc} {Suffrage:Voter ID:server-5f67d65d-8132-2c67-a0d6-7341883326ff Address:5f67d65d-8132-2c67-a0d6-7341883326ff} {Suffrage:Voter ID:server-aca92b9d-3a74-5116-f8bd-50a236a60bb6 Address:aca92b9d-3a74-5116-f8bd-50a236a60bb6}]" +2025-01-25T14:39:06.090Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:06.090Z [INFO] server-aca92b9d-3a74-5116-f8bd-50a236a60bb6: entering follower state: follower="Node at aca92b9d-3a74-5116-f8bd-50a236a60bb6 [Follower]" leader-address= leader-id= +2025-01-25T14:39:06.145Z [WARN] server-5f67d65d-8132-2c67-a0d6-7341883326ff: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:06.146Z [INFO] server-5f67d65d-8132-2c67-a0d6-7341883326ff: entering candidate state: node="Node at 5f67d65d-8132-2c67-a0d6-7341883326ff [Candidate]" term=2 +2025-01-25T14:39:06.146Z [INFO] server-5f67d65d-8132-2c67-a0d6-7341883326ff: election won: term=2 tally=2 +2025-01-25T14:39:06.146Z [INFO] server-5f67d65d-8132-2c67-a0d6-7341883326ff: entering leader state: leader="Node at 5f67d65d-8132-2c67-a0d6-7341883326ff [Leader]" +2025-01-25T14:39:06.146Z [INFO] server-5f67d65d-8132-2c67-a0d6-7341883326ff: added peer, starting replication: peer=server-983ff90d-0907-c381-5355-1add497d80cc +2025-01-25T14:39:06.146Z [INFO] server-5f67d65d-8132-2c67-a0d6-7341883326ff: added peer, starting replication: peer=server-aca92b9d-3a74-5116-f8bd-50a236a60bb6 +2025-01-25T14:39:06.147Z [INFO] server-5f67d65d-8132-2c67-a0d6-7341883326ff: pipelining replication: peer="{Voter server-983ff90d-0907-c381-5355-1add497d80cc 983ff90d-0907-c381-5355-1add497d80cc}" +2025-01-25T14:39:06.147Z [INFO] server-5f67d65d-8132-2c67-a0d6-7341883326ff: pipelining replication: peer="{Voter server-aca92b9d-3a74-5116-f8bd-50a236a60bb6 aca92b9d-3a74-5116-f8bd-50a236a60bb6}" +2025-01-25T14:39:06.252Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:06.146920251 +0000 UTC m=+15.192759569 (1 nodes), 56.250122ms from start of poll, 59.089506ms from cluster start. Timeout at 2025-01-25 14:39:06.251935418 +0000 UTC m=+15.297774735, 105.015166ms after stability +2025-01-25T14:39:06.252Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:39:06.358Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:39:06.253006851 +0000 UTC m=+15.298846169 (2 nodes), 4.667µs from start of poll, 165.176106ms from cluster start. Timeout at 2025-01-25 14:39:06.358080059 +0000 UTC m=+15.403919377, 105.073208ms after stability +2025-01-25T14:39:06.358Z [INFO] server-5f67d65d-8132-2c67-a0d6-7341883326ff: updating configuration: command=AddVoter server-id=server-983ff90d-0907-c381-5355-1add497d80cc server-addr=983ff90d-0907-c381-5355-1add497d80cc servers="[{Suffrage:Voter ID:server-983ff90d-0907-c381-5355-1add497d80cc Address:983ff90d-0907-c381-5355-1add497d80cc} {Suffrage:Voter ID:server-5f67d65d-8132-2c67-a0d6-7341883326ff Address:5f67d65d-8132-2c67-a0d6-7341883326ff} {Suffrage:Voter ID:server-aca92b9d-3a74-5116-f8bd-50a236a60bb6 Address:aca92b9d-3a74-5116-f8bd-50a236a60bb6}]" +2025-01-25T14:39:06.359Z [INFO] server-5f67d65d-8132-2c67-a0d6-7341883326ff: aborting pipeline replication: peer="{Voter server-aca92b9d-3a74-5116-f8bd-50a236a60bb6 aca92b9d-3a74-5116-f8bd-50a236a60bb6}" +2025-01-25T14:39:06.359Z [INFO] server-5f67d65d-8132-2c67-a0d6-7341883326ff: aborting pipeline replication: peer="{Voter server-983ff90d-0907-c381-5355-1add497d80cc 983ff90d-0907-c381-5355-1add497d80cc}" --- PASS: TestRaft_AddKnownPeer (0.28s) === RUN TestRaft_RemoveUnknownPeer -2025-01-25T14:28:22.204Z [INFO] server-60b2d22b-7c2c-3bb6-dbcc-d226b2a0b5ef: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-60b2d22b-7c2c-3bb6-dbcc-d226b2a0b5ef Address:60b2d22b-7c2c-3bb6-dbcc-d226b2a0b5ef} {Suffrage:Voter ID:server-a4938ee5-c214-7fd7-8fa7-4334d89e43f7 Address:a4938ee5-c214-7fd7-8fa7-4334d89e43f7} {Suffrage:Voter ID:server-6e2ff882-ae08-3db9-39ea-342981b18dd3 Address:6e2ff882-ae08-3db9-39ea-342981b18dd3}]" -2025-01-25T14:28:22.204Z [INFO] server-a4938ee5-c214-7fd7-8fa7-4334d89e43f7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-60b2d22b-7c2c-3bb6-dbcc-d226b2a0b5ef Address:60b2d22b-7c2c-3bb6-dbcc-d226b2a0b5ef} {Suffrage:Voter ID:server-a4938ee5-c214-7fd7-8fa7-4334d89e43f7 Address:a4938ee5-c214-7fd7-8fa7-4334d89e43f7} {Suffrage:Voter ID:server-6e2ff882-ae08-3db9-39ea-342981b18dd3 Address:6e2ff882-ae08-3db9-39ea-342981b18dd3}]" -2025-01-25T14:28:22.204Z [INFO] server-6e2ff882-ae08-3db9-39ea-342981b18dd3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-60b2d22b-7c2c-3bb6-dbcc-d226b2a0b5ef Address:60b2d22b-7c2c-3bb6-dbcc-d226b2a0b5ef} {Suffrage:Voter ID:server-a4938ee5-c214-7fd7-8fa7-4334d89e43f7 Address:a4938ee5-c214-7fd7-8fa7-4334d89e43f7} {Suffrage:Voter ID:server-6e2ff882-ae08-3db9-39ea-342981b18dd3 Address:6e2ff882-ae08-3db9-39ea-342981b18dd3}]" -2025-01-25T14:28:22.204Z [INFO] server-60b2d22b-7c2c-3bb6-dbcc-d226b2a0b5ef: entering follower state: follower="Node at 60b2d22b-7c2c-3bb6-dbcc-d226b2a0b5ef [Follower]" leader-address= leader-id= -2025-01-25T14:28:22.204Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:22.204Z [INFO] server-a4938ee5-c214-7fd7-8fa7-4334d89e43f7: entering follower state: follower="Node at a4938ee5-c214-7fd7-8fa7-4334d89e43f7 [Follower]" leader-address= leader-id= -2025-01-25T14:28:22.204Z [INFO] server-6e2ff882-ae08-3db9-39ea-342981b18dd3: entering follower state: follower="Node at 6e2ff882-ae08-3db9-39ea-342981b18dd3 [Follower]" leader-address= leader-id= -2025-01-25T14:28:22.257Z [WARN] server-a4938ee5-c214-7fd7-8fa7-4334d89e43f7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:22.257Z [INFO] server-a4938ee5-c214-7fd7-8fa7-4334d89e43f7: entering candidate state: node="Node at a4938ee5-c214-7fd7-8fa7-4334d89e43f7 [Candidate]" term=2 -2025-01-25T14:28:22.257Z [INFO] server-a4938ee5-c214-7fd7-8fa7-4334d89e43f7: election won: term=2 tally=2 -2025-01-25T14:28:22.257Z [INFO] server-a4938ee5-c214-7fd7-8fa7-4334d89e43f7: entering leader state: leader="Node at a4938ee5-c214-7fd7-8fa7-4334d89e43f7 [Leader]" -2025-01-25T14:28:22.257Z [INFO] server-a4938ee5-c214-7fd7-8fa7-4334d89e43f7: added peer, starting replication: peer=server-60b2d22b-7c2c-3bb6-dbcc-d226b2a0b5ef -2025-01-25T14:28:22.257Z [INFO] server-a4938ee5-c214-7fd7-8fa7-4334d89e43f7: added peer, starting replication: peer=server-6e2ff882-ae08-3db9-39ea-342981b18dd3 -2025-01-25T14:28:22.257Z [INFO] server-a4938ee5-c214-7fd7-8fa7-4334d89e43f7: pipelining replication: peer="{Voter server-6e2ff882-ae08-3db9-39ea-342981b18dd3 6e2ff882-ae08-3db9-39ea-342981b18dd3}" -2025-01-25T14:28:22.257Z [INFO] server-a4938ee5-c214-7fd7-8fa7-4334d89e43f7: pipelining replication: peer="{Voter server-60b2d22b-7c2c-3bb6-dbcc-d226b2a0b5ef 60b2d22b-7c2c-3bb6-dbcc-d226b2a0b5ef}" -2025-01-25T14:28:22.363Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:22.257724139 +0000 UTC m=+12.112049121 (1 nodes), 52.912296ms from start of poll, 53.599311ms from cluster start. Timeout at 2025-01-25 14:28:22.362729099 +0000 UTC m=+12.217054201, 105.00508ms after stability -2025-01-25T14:28:22.364Z [INFO] server-a4938ee5-c214-7fd7-8fa7-4334d89e43f7: updating configuration: command=RemoveServer server-id=984de5d9-5e76-c32a-ea13-66e7225d0626 server-addr= servers="[{Suffrage:Voter ID:server-60b2d22b-7c2c-3bb6-dbcc-d226b2a0b5ef Address:60b2d22b-7c2c-3bb6-dbcc-d226b2a0b5ef} {Suffrage:Voter ID:server-a4938ee5-c214-7fd7-8fa7-4334d89e43f7 Address:a4938ee5-c214-7fd7-8fa7-4334d89e43f7} {Suffrage:Voter ID:server-6e2ff882-ae08-3db9-39ea-342981b18dd3 Address:6e2ff882-ae08-3db9-39ea-342981b18dd3}]" -2025-01-25T14:28:22.364Z [INFO] server-a4938ee5-c214-7fd7-8fa7-4334d89e43f7: aborting pipeline replication: peer="{Voter server-60b2d22b-7c2c-3bb6-dbcc-d226b2a0b5ef 60b2d22b-7c2c-3bb6-dbcc-d226b2a0b5ef}" -2025-01-25T14:28:22.364Z [INFO] server-a4938ee5-c214-7fd7-8fa7-4334d89e43f7: aborting pipeline replication: peer="{Voter server-6e2ff882-ae08-3db9-39ea-342981b18dd3 6e2ff882-ae08-3db9-39ea-342981b18dd3}" ---- PASS: TestRaft_RemoveUnknownPeer (0.16s) +2025-01-25T14:39:06.368Z [INFO] server-fe8180fc-ff9b-52a4-000c-9baef4169b74: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fe8180fc-ff9b-52a4-000c-9baef4169b74 Address:fe8180fc-ff9b-52a4-000c-9baef4169b74} {Suffrage:Voter ID:server-ae7b4143-3d31-2292-b399-a796b2f80d87 Address:ae7b4143-3d31-2292-b399-a796b2f80d87} {Suffrage:Voter ID:server-6a14982b-4317-b695-7d01-aba5b403c1e0 Address:6a14982b-4317-b695-7d01-aba5b403c1e0}]" +2025-01-25T14:39:06.368Z [INFO] server-fe8180fc-ff9b-52a4-000c-9baef4169b74: entering follower state: follower="Node at fe8180fc-ff9b-52a4-000c-9baef4169b74 [Follower]" leader-address= leader-id= +2025-01-25T14:39:06.368Z [INFO] server-ae7b4143-3d31-2292-b399-a796b2f80d87: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fe8180fc-ff9b-52a4-000c-9baef4169b74 Address:fe8180fc-ff9b-52a4-000c-9baef4169b74} {Suffrage:Voter ID:server-ae7b4143-3d31-2292-b399-a796b2f80d87 Address:ae7b4143-3d31-2292-b399-a796b2f80d87} {Suffrage:Voter ID:server-6a14982b-4317-b695-7d01-aba5b403c1e0 Address:6a14982b-4317-b695-7d01-aba5b403c1e0}]" +2025-01-25T14:39:06.369Z [INFO] server-ae7b4143-3d31-2292-b399-a796b2f80d87: entering follower state: follower="Node at ae7b4143-3d31-2292-b399-a796b2f80d87 [Follower]" leader-address= leader-id= +2025-01-25T14:39:06.369Z [INFO] server-6a14982b-4317-b695-7d01-aba5b403c1e0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fe8180fc-ff9b-52a4-000c-9baef4169b74 Address:fe8180fc-ff9b-52a4-000c-9baef4169b74} {Suffrage:Voter ID:server-ae7b4143-3d31-2292-b399-a796b2f80d87 Address:ae7b4143-3d31-2292-b399-a796b2f80d87} {Suffrage:Voter ID:server-6a14982b-4317-b695-7d01-aba5b403c1e0 Address:6a14982b-4317-b695-7d01-aba5b403c1e0}]" +2025-01-25T14:39:06.370Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:06.370Z [INFO] server-6a14982b-4317-b695-7d01-aba5b403c1e0: entering follower state: follower="Node at 6a14982b-4317-b695-7d01-aba5b403c1e0 [Follower]" leader-address= leader-id= +2025-01-25T14:39:06.427Z [WARN] server-6a14982b-4317-b695-7d01-aba5b403c1e0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:06.427Z [INFO] server-6a14982b-4317-b695-7d01-aba5b403c1e0: entering candidate state: node="Node at 6a14982b-4317-b695-7d01-aba5b403c1e0 [Candidate]" term=2 +2025-01-25T14:39:06.428Z [INFO] server-6a14982b-4317-b695-7d01-aba5b403c1e0: election won: term=2 tally=2 +2025-01-25T14:39:06.428Z [INFO] server-6a14982b-4317-b695-7d01-aba5b403c1e0: entering leader state: leader="Node at 6a14982b-4317-b695-7d01-aba5b403c1e0 [Leader]" +2025-01-25T14:39:06.428Z [INFO] server-6a14982b-4317-b695-7d01-aba5b403c1e0: added peer, starting replication: peer=server-fe8180fc-ff9b-52a4-000c-9baef4169b74 +2025-01-25T14:39:06.428Z [INFO] server-6a14982b-4317-b695-7d01-aba5b403c1e0: added peer, starting replication: peer=server-ae7b4143-3d31-2292-b399-a796b2f80d87 +2025-01-25T14:39:06.429Z [INFO] server-6a14982b-4317-b695-7d01-aba5b403c1e0: pipelining replication: peer="{Voter server-fe8180fc-ff9b-52a4-000c-9baef4169b74 fe8180fc-ff9b-52a4-000c-9baef4169b74}" +2025-01-25T14:39:06.429Z [INFO] server-6a14982b-4317-b695-7d01-aba5b403c1e0: pipelining replication: peer="{Voter server-ae7b4143-3d31-2292-b399-a796b2f80d87 ae7b4143-3d31-2292-b399-a796b2f80d87}" +2025-01-25T14:39:06.534Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:06.428442779 +0000 UTC m=+15.474282097 (1 nodes), 58.269337ms from start of poll, 61.141678ms from cluster start. Timeout at 2025-01-25 14:39:06.53346728 +0000 UTC m=+15.579306597, 105.0245ms after stability +2025-01-25T14:39:06.534Z [INFO] server-6a14982b-4317-b695-7d01-aba5b403c1e0: updating configuration: command=RemoveServer server-id=463c67b9-863c-567f-4fad-d5a8261ebb12 server-addr= servers="[{Suffrage:Voter ID:server-fe8180fc-ff9b-52a4-000c-9baef4169b74 Address:fe8180fc-ff9b-52a4-000c-9baef4169b74} {Suffrage:Voter ID:server-ae7b4143-3d31-2292-b399-a796b2f80d87 Address:ae7b4143-3d31-2292-b399-a796b2f80d87} {Suffrage:Voter ID:server-6a14982b-4317-b695-7d01-aba5b403c1e0 Address:6a14982b-4317-b695-7d01-aba5b403c1e0}]" +2025-01-25T14:39:06.536Z [INFO] server-6a14982b-4317-b695-7d01-aba5b403c1e0: aborting pipeline replication: peer="{Voter server-ae7b4143-3d31-2292-b399-a796b2f80d87 ae7b4143-3d31-2292-b399-a796b2f80d87}" +2025-01-25T14:39:06.536Z [INFO] server-6a14982b-4317-b695-7d01-aba5b403c1e0: aborting pipeline replication: peer="{Voter server-fe8180fc-ff9b-52a4-000c-9baef4169b74 fe8180fc-ff9b-52a4-000c-9baef4169b74}" +--- PASS: TestRaft_RemoveUnknownPeer (0.18s) === RUN TestRaft_SnapshotRestore -2025-01-25T14:28:22.366Z [INFO] server-39d61487-4038-aaf2-7a39-1c894b4ca10e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-39d61487-4038-aaf2-7a39-1c894b4ca10e Address:39d61487-4038-aaf2-7a39-1c894b4ca10e}]" -2025-01-25T14:28:22.366Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:22.366Z [INFO] server-39d61487-4038-aaf2-7a39-1c894b4ca10e: entering follower state: follower="Node at 39d61487-4038-aaf2-7a39-1c894b4ca10e [Follower]" leader-address= leader-id= -2025-01-25T14:28:22.452Z [WARN] server-39d61487-4038-aaf2-7a39-1c894b4ca10e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:22.452Z [INFO] server-39d61487-4038-aaf2-7a39-1c894b4ca10e: entering candidate state: node="Node at 39d61487-4038-aaf2-7a39-1c894b4ca10e [Candidate]" term=2 -2025-01-25T14:28:22.452Z [INFO] server-39d61487-4038-aaf2-7a39-1c894b4ca10e: election won: term=2 tally=1 -2025-01-25T14:28:22.452Z [INFO] server-39d61487-4038-aaf2-7a39-1c894b4ca10e: entering leader state: leader="Node at 39d61487-4038-aaf2-7a39-1c894b4ca10e [Leader]" -2025-01-25T14:28:22.558Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:22.452888836 +0000 UTC m=+12.307213818 (1 nodes), 86.600524ms from start of poll, 86.858955ms from cluster start. Timeout at 2025-01-25 14:28:22.557892716 +0000 UTC m=+12.412217698, 105.00388ms after stability -2025-01-25T14:28:22.564Z [INFO] server-39d61487-4038-aaf2-7a39-1c894b4ca10e: starting snapshot up to: index=102 -2025-01-25T14:28:22.564Z [INFO] creating new snapshot: path=/tmp/raft3455121960/snapshots/2-102-1737815302564.tmp -2025-01-25T14:28:22.565Z [INFO] server-39d61487-4038-aaf2-7a39-1c894b4ca10e: compacting logs: from=1 to=92 -2025-01-25T14:28:22.565Z [INFO] server-39d61487-4038-aaf2-7a39-1c894b4ca10e: snapshot complete up to: index=102 -2025-01-25T14:28:22.566Z [INFO] server-39d61487-4038-aaf2-7a39-1c894b4ca10e: starting restore from snapshot: id=2-102-1737815302564 last-index=102 last-term=2 size-in-bytes=693 -2025-01-25T14:28:22.566Z [INFO] server-39d61487-4038-aaf2-7a39-1c894b4ca10e: snapshot restore progress: id=2-102-1737815302564 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% -2025-01-25T14:28:22.566Z [INFO] server-39d61487-4038-aaf2-7a39-1c894b4ca10e: restored from snapshot: id=2-102-1737815302564 last-index=102 last-term=2 size-in-bytes=693 -2025-01-25T14:28:22.566Z [INFO] server-39d61487-4038-aaf2-7a39-1c894b4ca10e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-39d61487-4038-aaf2-7a39-1c894b4ca10e Address:39d61487-4038-aaf2-7a39-1c894b4ca10e}]" -2025-01-25T14:28:22.566Z [INFO] server-39d61487-4038-aaf2-7a39-1c894b4ca10e: entering follower state: follower="Node at 39d61487-4038-aaf2-7a39-1c894b4ca10e [Shutdown]" leader-address= leader-id= ---- PASS: TestRaft_SnapshotRestore (0.21s) +2025-01-25T14:39:06.542Z [INFO] server-73acae1e-59ee-a328-83ca-324aba2c4486: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-73acae1e-59ee-a328-83ca-324aba2c4486 Address:73acae1e-59ee-a328-83ca-324aba2c4486}]" +2025-01-25T14:39:06.542Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:06.542Z [INFO] server-73acae1e-59ee-a328-83ca-324aba2c4486: entering follower state: follower="Node at 73acae1e-59ee-a328-83ca-324aba2c4486 [Follower]" leader-address= leader-id= +2025-01-25T14:39:06.594Z [WARN] server-73acae1e-59ee-a328-83ca-324aba2c4486: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:06.594Z [INFO] server-73acae1e-59ee-a328-83ca-324aba2c4486: entering candidate state: node="Node at 73acae1e-59ee-a328-83ca-324aba2c4486 [Candidate]" term=2 +2025-01-25T14:39:06.595Z [INFO] server-73acae1e-59ee-a328-83ca-324aba2c4486: election won: term=2 tally=1 +2025-01-25T14:39:06.595Z [INFO] server-73acae1e-59ee-a328-83ca-324aba2c4486: entering leader state: leader="Node at 73acae1e-59ee-a328-83ca-324aba2c4486 [Leader]" +2025-01-25T14:39:06.700Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:06.595341057 +0000 UTC m=+15.641180374 (1 nodes), 52.536069ms from start of poll, 53.408155ms from cluster start. Timeout at 2025-01-25 14:39:06.700346598 +0000 UTC m=+15.746185916, 105.005542ms after stability +2025-01-25T14:39:06.724Z [INFO] server-73acae1e-59ee-a328-83ca-324aba2c4486: starting snapshot up to: index=102 +2025-01-25T14:39:06.724Z [INFO] creating new snapshot: path=/tmp/raft4109291258/snapshots/2-102-1737815946724.tmp +2025-01-25T14:39:06.728Z [INFO] server-73acae1e-59ee-a328-83ca-324aba2c4486: compacting logs: from=1 to=92 +2025-01-25T14:39:06.729Z [INFO] server-73acae1e-59ee-a328-83ca-324aba2c4486: snapshot complete up to: index=102 +2025-01-25T14:39:06.730Z [INFO] server-73acae1e-59ee-a328-83ca-324aba2c4486: starting restore from snapshot: id=2-102-1737815946724 last-index=102 last-term=2 size-in-bytes=693 +2025-01-25T14:39:06.732Z [INFO] server-73acae1e-59ee-a328-83ca-324aba2c4486: snapshot restore progress: id=2-102-1737815946724 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% +2025-01-25T14:39:06.732Z [INFO] server-73acae1e-59ee-a328-83ca-324aba2c4486: restored from snapshot: id=2-102-1737815946724 last-index=102 last-term=2 size-in-bytes=693 +2025-01-25T14:39:06.732Z [INFO] server-73acae1e-59ee-a328-83ca-324aba2c4486: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-73acae1e-59ee-a328-83ca-324aba2c4486 Address:73acae1e-59ee-a328-83ca-324aba2c4486}]" +--- PASS: TestRaft_SnapshotRestore (0.19s) === RUN TestRaft_RestoreSnapshotOnStartup_Monotonic -2025-01-25T14:28:22.573Z [INFO] server-41e82818-87ba-968e-4a42-ddaf71a2b4d8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-41e82818-87ba-968e-4a42-ddaf71a2b4d8 Address:41e82818-87ba-968e-4a42-ddaf71a2b4d8}]" -2025-01-25T14:28:22.573Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:22.573Z [INFO] server-41e82818-87ba-968e-4a42-ddaf71a2b4d8: entering follower state: follower="Node at 41e82818-87ba-968e-4a42-ddaf71a2b4d8 [Follower]" leader-address= leader-id= -2025-01-25T14:28:22.660Z [WARN] server-41e82818-87ba-968e-4a42-ddaf71a2b4d8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:22.660Z [INFO] server-41e82818-87ba-968e-4a42-ddaf71a2b4d8: entering candidate state: node="Node at 41e82818-87ba-968e-4a42-ddaf71a2b4d8 [Candidate]" term=2 -2025-01-25T14:28:22.660Z [INFO] server-41e82818-87ba-968e-4a42-ddaf71a2b4d8: election won: term=2 tally=1 -2025-01-25T14:28:22.660Z [INFO] server-41e82818-87ba-968e-4a42-ddaf71a2b4d8: entering leader state: leader="Node at 41e82818-87ba-968e-4a42-ddaf71a2b4d8 [Leader]" -2025-01-25T14:28:22.766Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:22.661093504 +0000 UTC m=+12.515418486 (1 nodes), 87.700244ms from start of poll, 88.018993ms from cluster start. Timeout at 2025-01-25 14:28:22.766097344 +0000 UTC m=+12.620422326, 105.00384ms after stability -2025-01-25T14:28:22.772Z [INFO] server-41e82818-87ba-968e-4a42-ddaf71a2b4d8: starting snapshot up to: index=102 -2025-01-25T14:28:22.772Z [INFO] creating new snapshot: path=/tmp/raft1921148889/snapshots/2-102-1737815302772.tmp -2025-01-25T14:28:22.773Z [INFO] server-41e82818-87ba-968e-4a42-ddaf71a2b4d8: compacting logs: from=1 to=92 -2025-01-25T14:28:22.773Z [INFO] server-41e82818-87ba-968e-4a42-ddaf71a2b4d8: snapshot complete up to: index=102 -2025-01-25T14:28:22.774Z [INFO] server-41e82818-87ba-968e-4a42-ddaf71a2b4d8: starting restore from snapshot: id=2-102-1737815302772 last-index=102 last-term=2 size-in-bytes=693 -2025-01-25T14:28:22.774Z [INFO] server-41e82818-87ba-968e-4a42-ddaf71a2b4d8: snapshot restore progress: id=2-102-1737815302772 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% -2025-01-25T14:28:22.774Z [INFO] server-41e82818-87ba-968e-4a42-ddaf71a2b4d8: restored from snapshot: id=2-102-1737815302772 last-index=102 last-term=2 size-in-bytes=693 -2025-01-25T14:28:22.774Z [INFO] server-41e82818-87ba-968e-4a42-ddaf71a2b4d8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-41e82818-87ba-968e-4a42-ddaf71a2b4d8 Address:41e82818-87ba-968e-4a42-ddaf71a2b4d8}]" ---- PASS: TestRaft_RestoreSnapshotOnStartup_Monotonic (0.20s) +2025-01-25T14:39:06.737Z [INFO] server-0f4aea09-a16d-ef9d-86be-3cd4f3632faa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0f4aea09-a16d-ef9d-86be-3cd4f3632faa Address:0f4aea09-a16d-ef9d-86be-3cd4f3632faa}]" +2025-01-25T14:39:06.737Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:06.737Z [INFO] server-0f4aea09-a16d-ef9d-86be-3cd4f3632faa: entering follower state: follower="Node at 0f4aea09-a16d-ef9d-86be-3cd4f3632faa [Follower]" leader-address= leader-id= +2025-01-25T14:39:06.831Z [WARN] server-0f4aea09-a16d-ef9d-86be-3cd4f3632faa: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:06.831Z [INFO] server-0f4aea09-a16d-ef9d-86be-3cd4f3632faa: entering candidate state: node="Node at 0f4aea09-a16d-ef9d-86be-3cd4f3632faa [Candidate]" term=2 +2025-01-25T14:39:06.831Z [INFO] server-0f4aea09-a16d-ef9d-86be-3cd4f3632faa: election won: term=2 tally=1 +2025-01-25T14:39:06.831Z [INFO] server-0f4aea09-a16d-ef9d-86be-3cd4f3632faa: entering leader state: leader="Node at 0f4aea09-a16d-ef9d-86be-3cd4f3632faa [Leader]" +2025-01-25T14:39:06.937Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:06.831859287 +0000 UTC m=+15.877698605 (1 nodes), 94.100691ms from start of poll, 95.033152ms from cluster start. Timeout at 2025-01-25 14:39:06.936905954 +0000 UTC m=+15.982745272, 105.046667ms after stability +2025-01-25T14:39:06.957Z [INFO] server-0f4aea09-a16d-ef9d-86be-3cd4f3632faa: starting snapshot up to: index=102 +2025-01-25T14:39:06.957Z [INFO] creating new snapshot: path=/tmp/raft3410611394/snapshots/2-102-1737815946957.tmp +2025-01-25T14:39:06.962Z [INFO] server-0f4aea09-a16d-ef9d-86be-3cd4f3632faa: compacting logs: from=1 to=92 +2025-01-25T14:39:06.962Z [INFO] server-0f4aea09-a16d-ef9d-86be-3cd4f3632faa: snapshot complete up to: index=102 +2025-01-25T14:39:06.964Z [INFO] server-0f4aea09-a16d-ef9d-86be-3cd4f3632faa: starting restore from snapshot: id=2-102-1737815946957 last-index=102 last-term=2 size-in-bytes=693 +2025-01-25T14:39:06.966Z [INFO] server-0f4aea09-a16d-ef9d-86be-3cd4f3632faa: snapshot restore progress: id=2-102-1737815946957 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% +2025-01-25T14:39:06.966Z [INFO] server-0f4aea09-a16d-ef9d-86be-3cd4f3632faa: restored from snapshot: id=2-102-1737815946957 last-index=102 last-term=2 size-in-bytes=693 +2025-01-25T14:39:06.966Z [INFO] server-0f4aea09-a16d-ef9d-86be-3cd4f3632faa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0f4aea09-a16d-ef9d-86be-3cd4f3632faa Address:0f4aea09-a16d-ef9d-86be-3cd4f3632faa}]" +2025-01-25T14:39:06.966Z [INFO] server-0f4aea09-a16d-ef9d-86be-3cd4f3632faa: entering follower state: follower="Node at 0f4aea09-a16d-ef9d-86be-3cd4f3632faa [Shutdown]" leader-address= leader-id= +--- PASS: TestRaft_RestoreSnapshotOnStartup_Monotonic (0.23s) === RUN TestRaft_SnapshotRestore_Progress -2025-01-25T14:28:22.776Z [INFO] server-a0be593a-bac6-86af-d40a-24fc9f66e6bf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a0be593a-bac6-86af-d40a-24fc9f66e6bf Address:a0be593a-bac6-86af-d40a-24fc9f66e6bf}]" -2025-01-25T14:28:22.776Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:22.776Z [INFO] server-a0be593a-bac6-86af-d40a-24fc9f66e6bf: entering follower state: follower="Node at a0be593a-bac6-86af-d40a-24fc9f66e6bf [Follower]" leader-address= leader-id= -2025-01-25T14:28:22.846Z [WARN] server-a0be593a-bac6-86af-d40a-24fc9f66e6bf: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:22.846Z [INFO] server-a0be593a-bac6-86af-d40a-24fc9f66e6bf: entering candidate state: node="Node at a0be593a-bac6-86af-d40a-24fc9f66e6bf [Candidate]" term=2 -2025-01-25T14:28:22.846Z [INFO] server-a0be593a-bac6-86af-d40a-24fc9f66e6bf: election won: term=2 tally=1 -2025-01-25T14:28:22.846Z [INFO] server-a0be593a-bac6-86af-d40a-24fc9f66e6bf: entering leader state: leader="Node at a0be593a-bac6-86af-d40a-24fc9f66e6bf [Leader]" -2025-01-25T14:28:22.952Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:22.847166848 +0000 UTC m=+12.701491830 (1 nodes), 70.798572ms from start of poll, 71.056163ms from cluster start. Timeout at 2025-01-25 14:28:22.952175647 +0000 UTC m=+12.806500749, 105.008919ms after stability -2025-01-25T14:28:22.957Z [INFO] server-a0be593a-bac6-86af-d40a-24fc9f66e6bf: starting snapshot up to: index=102 -2025-01-25T14:28:22.957Z [INFO] creating new snapshot: path=/tmp/raft1795339862/snapshots/2-102-1737815302957.tmp -2025-01-25T14:28:22.958Z [INFO] server-a0be593a-bac6-86af-d40a-24fc9f66e6bf: compacting logs: from=1 to=92 -2025-01-25T14:28:22.958Z [INFO] server-a0be593a-bac6-86af-d40a-24fc9f66e6bf: snapshot complete up to: index=102 ---- PASS: TestRaft_SnapshotRestore_Progress (0.19s) +2025-01-25T14:39:06.971Z [INFO] server-89061918-5259-b5cb-100f-73759d803237: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-89061918-5259-b5cb-100f-73759d803237 Address:89061918-5259-b5cb-100f-73759d803237}]" +2025-01-25T14:39:06.971Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:06.971Z [INFO] server-89061918-5259-b5cb-100f-73759d803237: entering follower state: follower="Node at 89061918-5259-b5cb-100f-73759d803237 [Follower]" leader-address= leader-id= +2025-01-25T14:39:07.041Z [WARN] server-89061918-5259-b5cb-100f-73759d803237: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:07.041Z [INFO] server-89061918-5259-b5cb-100f-73759d803237: entering candidate state: node="Node at 89061918-5259-b5cb-100f-73759d803237 [Candidate]" term=2 +2025-01-25T14:39:07.041Z [INFO] server-89061918-5259-b5cb-100f-73759d803237: election won: term=2 tally=1 +2025-01-25T14:39:07.041Z [INFO] server-89061918-5259-b5cb-100f-73759d803237: entering leader state: leader="Node at 89061918-5259-b5cb-100f-73759d803237 [Leader]" +2025-01-25T14:39:07.147Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:07.041861315 +0000 UTC m=+16.087700633 (1 nodes), 69.976871ms from start of poll, 70.936165ms from cluster start. Timeout at 2025-01-25 14:39:07.146883191 +0000 UTC m=+16.192722508, 105.021875ms after stability +2025-01-25T14:39:07.165Z [INFO] server-89061918-5259-b5cb-100f-73759d803237: starting snapshot up to: index=102 +2025-01-25T14:39:07.165Z [INFO] creating new snapshot: path=/tmp/raft3986295102/snapshots/2-102-1737815947165.tmp +2025-01-25T14:39:07.167Z [INFO] server-89061918-5259-b5cb-100f-73759d803237: compacting logs: from=1 to=92 +2025-01-25T14:39:07.167Z [INFO] server-89061918-5259-b5cb-100f-73759d803237: snapshot complete up to: index=102 +--- PASS: TestRaft_SnapshotRestore_Progress (0.20s) === RUN TestRaft_NoRestoreOnStart -2025-01-25T14:28:22.963Z [INFO] server-2d5b6e6e-5630-52bf-b163-99cccce47836: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2d5b6e6e-5630-52bf-b163-99cccce47836 Address:2d5b6e6e-5630-52bf-b163-99cccce47836}]" -2025-01-25T14:28:22.963Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:22.963Z [INFO] server-2d5b6e6e-5630-52bf-b163-99cccce47836: entering follower state: follower="Node at 2d5b6e6e-5630-52bf-b163-99cccce47836 [Follower]" leader-address= leader-id= -2025-01-25T14:28:23.028Z [WARN] server-2d5b6e6e-5630-52bf-b163-99cccce47836: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:23.028Z [INFO] server-2d5b6e6e-5630-52bf-b163-99cccce47836: entering candidate state: node="Node at 2d5b6e6e-5630-52bf-b163-99cccce47836 [Candidate]" term=2 -2025-01-25T14:28:23.028Z [INFO] server-2d5b6e6e-5630-52bf-b163-99cccce47836: election won: term=2 tally=1 -2025-01-25T14:28:23.028Z [INFO] server-2d5b6e6e-5630-52bf-b163-99cccce47836: entering leader state: leader="Node at 2d5b6e6e-5630-52bf-b163-99cccce47836 [Leader]" -2025-01-25T14:28:23.135Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:23.029070422 +0000 UTC m=+12.883395364 (1 nodes), 65.146135ms from start of poll, 65.446604ms from cluster start. Timeout at 2025-01-25 14:28:23.134072582 +0000 UTC m=+12.988397564, 105.0022ms after stability -2025-01-25T14:28:23.141Z [INFO] server-2d5b6e6e-5630-52bf-b163-99cccce47836: starting snapshot up to: index=102 -2025-01-25T14:28:23.141Z [INFO] creating new snapshot: path=/tmp/raft1170098183/snapshots/2-102-1737815303141.tmp -2025-01-25T14:28:23.142Z [INFO] server-2d5b6e6e-5630-52bf-b163-99cccce47836: compacting logs: from=1 to=92 -2025-01-25T14:28:23.142Z [INFO] server-2d5b6e6e-5630-52bf-b163-99cccce47836: snapshot complete up to: index=102 -2025-01-25T14:28:23.143Z [INFO] server-2d5b6e6e-5630-52bf-b163-99cccce47836: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2d5b6e6e-5630-52bf-b163-99cccce47836 Address:2d5b6e6e-5630-52bf-b163-99cccce47836}]" ---- PASS: TestRaft_NoRestoreOnStart (0.18s) +2025-01-25T14:39:07.176Z [INFO] server-4d7be0c5-1926-393f-c9e4-6821701919b7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4d7be0c5-1926-393f-c9e4-6821701919b7 Address:4d7be0c5-1926-393f-c9e4-6821701919b7}]" +2025-01-25T14:39:07.176Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:07.176Z [INFO] server-4d7be0c5-1926-393f-c9e4-6821701919b7: entering follower state: follower="Node at 4d7be0c5-1926-393f-c9e4-6821701919b7 [Follower]" leader-address= leader-id= +2025-01-25T14:39:07.267Z [WARN] server-4d7be0c5-1926-393f-c9e4-6821701919b7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:07.267Z [INFO] server-4d7be0c5-1926-393f-c9e4-6821701919b7: entering candidate state: node="Node at 4d7be0c5-1926-393f-c9e4-6821701919b7 [Candidate]" term=2 +2025-01-25T14:39:07.268Z [INFO] server-4d7be0c5-1926-393f-c9e4-6821701919b7: election won: term=2 tally=1 +2025-01-25T14:39:07.268Z [INFO] server-4d7be0c5-1926-393f-c9e4-6821701919b7: entering leader state: leader="Node at 4d7be0c5-1926-393f-c9e4-6821701919b7 [Leader]" +2025-01-25T14:39:07.374Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:07.268386391 +0000 UTC m=+16.314225709 (1 nodes), 91.807017ms from start of poll, 93.066146ms from cluster start. Timeout at 2025-01-25 14:39:07.373414099 +0000 UTC m=+16.419253416, 105.027707ms after stability +2025-01-25T14:39:07.396Z [INFO] server-4d7be0c5-1926-393f-c9e4-6821701919b7: starting snapshot up to: index=102 +2025-01-25T14:39:07.396Z [INFO] creating new snapshot: path=/tmp/raft557472696/snapshots/2-102-1737815947396.tmp +2025-01-25T14:39:07.399Z [INFO] server-4d7be0c5-1926-393f-c9e4-6821701919b7: compacting logs: from=1 to=92 +2025-01-25T14:39:07.400Z [INFO] server-4d7be0c5-1926-393f-c9e4-6821701919b7: snapshot complete up to: index=102 +2025-01-25T14:39:07.401Z [INFO] server-4d7be0c5-1926-393f-c9e4-6821701919b7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4d7be0c5-1926-393f-c9e4-6821701919b7 Address:4d7be0c5-1926-393f-c9e4-6821701919b7}]" +--- PASS: TestRaft_NoRestoreOnStart (0.23s) +2025-01-25T14:39:07.401Z [INFO] server-4d7be0c5-1926-393f-c9e4-6821701919b7: entering follower state: follower="Node at 4d7be0c5-1926-393f-c9e4-6821701919b7 [Follower]" leader-address= leader-id= === RUN TestRaft_SnapshotRestore_PeerChange raft_test.go:1266: Skipping unreliable test --- SKIP: TestRaft_SnapshotRestore_PeerChange (0.00s) === RUN TestRaft_AutoSnapshot -2025-01-25T14:28:23.143Z [INFO] server-2d5b6e6e-5630-52bf-b163-99cccce47836: entering follower state: follower="Node at 2d5b6e6e-5630-52bf-b163-99cccce47836 [Follower]" leader-address= leader-id= -2025-01-25T14:28:23.144Z [INFO] server-d8221df0-c784-f02f-9292-ab5bb8771427: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d8221df0-c784-f02f-9292-ab5bb8771427 Address:d8221df0-c784-f02f-9292-ab5bb8771427}]" -2025-01-25T14:28:23.144Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:23.144Z [INFO] server-d8221df0-c784-f02f-9292-ab5bb8771427: entering follower state: follower="Node at d8221df0-c784-f02f-9292-ab5bb8771427 [Follower]" leader-address= leader-id= -2025-01-25T14:28:23.202Z [WARN] server-d8221df0-c784-f02f-9292-ab5bb8771427: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:23.202Z [INFO] server-d8221df0-c784-f02f-9292-ab5bb8771427: entering candidate state: node="Node at d8221df0-c784-f02f-9292-ab5bb8771427 [Candidate]" term=2 -2025-01-25T14:28:23.202Z [INFO] server-d8221df0-c784-f02f-9292-ab5bb8771427: election won: term=2 tally=1 -2025-01-25T14:28:23.202Z [INFO] server-d8221df0-c784-f02f-9292-ab5bb8771427: entering leader state: leader="Node at d8221df0-c784-f02f-9292-ab5bb8771427 [Leader]" -2025-01-25T14:28:23.224Z [WARN] server-2d5b6e6e-5630-52bf-b163-99cccce47836: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:23.224Z [INFO] server-2d5b6e6e-5630-52bf-b163-99cccce47836: entering candidate state: node="Node at 2d5b6e6e-5630-52bf-b163-99cccce47836 [Candidate]" term=3 -2025-01-25T14:28:23.224Z [INFO] server-2d5b6e6e-5630-52bf-b163-99cccce47836: election won: term=3 tally=1 -2025-01-25T14:28:23.224Z [INFO] server-2d5b6e6e-5630-52bf-b163-99cccce47836: entering leader state: leader="Node at 2d5b6e6e-5630-52bf-b163-99cccce47836 [Leader]" -2025-01-25T14:28:23.308Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:23.202428304 +0000 UTC m=+13.056753246 (1 nodes), 58.031991ms from start of poll, 58.268623ms from cluster start. Timeout at 2025-01-25 14:28:23.307439904 +0000 UTC m=+13.161764846, 105.0116ms after stability -2025-01-25T14:28:23.320Z [INFO] server-d8221df0-c784-f02f-9292-ab5bb8771427: starting snapshot up to: index=102 -2025-01-25T14:28:23.320Z [INFO] creating new snapshot: path=/tmp/raft1349550150/snapshots/2-102-1737815303320.tmp -2025-01-25T14:28:23.322Z [INFO] server-d8221df0-c784-f02f-9292-ab5bb8771427: compacting logs: from=1 to=92 -2025-01-25T14:28:23.323Z [INFO] server-d8221df0-c784-f02f-9292-ab5bb8771427: snapshot complete up to: index=102 ---- PASS: TestRaft_AutoSnapshot (0.28s) +2025-01-25T14:39:07.405Z [INFO] server-9c48ef1d-93c9-98aa-fb98-0194c0954e87: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9c48ef1d-93c9-98aa-fb98-0194c0954e87 Address:9c48ef1d-93c9-98aa-fb98-0194c0954e87}]" +2025-01-25T14:39:07.405Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:07.406Z [INFO] server-9c48ef1d-93c9-98aa-fb98-0194c0954e87: entering follower state: follower="Node at 9c48ef1d-93c9-98aa-fb98-0194c0954e87 [Follower]" leader-address= leader-id= +2025-01-25T14:39:07.488Z [WARN] server-9c48ef1d-93c9-98aa-fb98-0194c0954e87: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:07.488Z [INFO] server-9c48ef1d-93c9-98aa-fb98-0194c0954e87: entering candidate state: node="Node at 9c48ef1d-93c9-98aa-fb98-0194c0954e87 [Candidate]" term=2 +2025-01-25T14:39:07.489Z [INFO] server-9c48ef1d-93c9-98aa-fb98-0194c0954e87: election won: term=2 tally=1 +2025-01-25T14:39:07.489Z [INFO] server-9c48ef1d-93c9-98aa-fb98-0194c0954e87: entering leader state: leader="Node at 9c48ef1d-93c9-98aa-fb98-0194c0954e87 [Leader]" +2025-01-25T14:39:07.491Z [WARN] server-4d7be0c5-1926-393f-c9e4-6821701919b7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:07.491Z [INFO] server-4d7be0c5-1926-393f-c9e4-6821701919b7: entering candidate state: node="Node at 4d7be0c5-1926-393f-c9e4-6821701919b7 [Candidate]" term=3 +2025-01-25T14:39:07.491Z [INFO] server-4d7be0c5-1926-393f-c9e4-6821701919b7: election won: term=3 tally=1 +2025-01-25T14:39:07.491Z [INFO] server-4d7be0c5-1926-393f-c9e4-6821701919b7: entering leader state: leader="Node at 4d7be0c5-1926-393f-c9e4-6821701919b7 [Leader]" +2025-01-25T14:39:07.595Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:07.489311326 +0000 UTC m=+16.535150644 (1 nodes), 83.33716ms from start of poll, 84.327662ms from cluster start. Timeout at 2025-01-25 14:39:07.594332035 +0000 UTC m=+16.640171353, 105.020709ms after stability +2025-01-25T14:39:07.616Z [INFO] server-9c48ef1d-93c9-98aa-fb98-0194c0954e87: starting snapshot up to: index=102 +2025-01-25T14:39:07.617Z [INFO] creating new snapshot: path=/tmp/raft3654313699/snapshots/2-102-1737815947617.tmp +2025-01-25T14:39:07.619Z [INFO] server-9c48ef1d-93c9-98aa-fb98-0194c0954e87: compacting logs: from=1 to=92 +2025-01-25T14:39:07.619Z [INFO] server-9c48ef1d-93c9-98aa-fb98-0194c0954e87: snapshot complete up to: index=102 +--- PASS: TestRaft_AutoSnapshot (0.32s) === RUN TestRaft_UserSnapshot -2025-01-25T14:28:23.421Z [INFO] server-2e478ced-bfbd-ab8f-40e4-f18b73616d3f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2e478ced-bfbd-ab8f-40e4-f18b73616d3f Address:2e478ced-bfbd-ab8f-40e4-f18b73616d3f}]" -2025-01-25T14:28:23.421Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:23.421Z [INFO] server-2e478ced-bfbd-ab8f-40e4-f18b73616d3f: entering follower state: follower="Node at 2e478ced-bfbd-ab8f-40e4-f18b73616d3f [Follower]" leader-address= leader-id= -2025-01-25T14:28:23.474Z [WARN] server-2e478ced-bfbd-ab8f-40e4-f18b73616d3f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:23.474Z [INFO] server-2e478ced-bfbd-ab8f-40e4-f18b73616d3f: entering candidate state: node="Node at 2e478ced-bfbd-ab8f-40e4-f18b73616d3f [Candidate]" term=2 -2025-01-25T14:28:23.474Z [INFO] server-2e478ced-bfbd-ab8f-40e4-f18b73616d3f: election won: term=2 tally=1 -2025-01-25T14:28:23.474Z [INFO] server-2e478ced-bfbd-ab8f-40e4-f18b73616d3f: entering leader state: leader="Node at 2e478ced-bfbd-ab8f-40e4-f18b73616d3f [Leader]" -2025-01-25T14:28:23.580Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:23.4748913 +0000 UTC m=+13.329216322 (1 nodes), 53.220965ms from start of poll, 53.498395ms from cluster start. Timeout at 2025-01-25 14:28:23.57990298 +0000 UTC m=+13.434228082, 105.01176ms after stability -2025-01-25T14:28:23.580Z [ERROR] server-2e478ced-bfbd-ab8f-40e4-f18b73616d3f: failed to take snapshot: error="nothing new to snapshot" -2025-01-25T14:28:23.581Z [INFO] server-2e478ced-bfbd-ab8f-40e4-f18b73616d3f: starting snapshot up to: index=12 -2025-01-25T14:28:23.581Z [INFO] creating new snapshot: path=/tmp/raft2371625306/snapshots/2-12-1737815303581.tmp -2025-01-25T14:28:23.582Z [INFO] server-2e478ced-bfbd-ab8f-40e4-f18b73616d3f: compacting logs: from=1 to=2 -2025-01-25T14:28:23.582Z [INFO] server-2e478ced-bfbd-ab8f-40e4-f18b73616d3f: snapshot complete up to: index=12 ---- PASS: TestRaft_UserSnapshot (0.16s) +2025-01-25T14:39:07.726Z [INFO] server-d0cfba64-fd20-833a-296c-22464866121d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d0cfba64-fd20-833a-296c-22464866121d Address:d0cfba64-fd20-833a-296c-22464866121d}]" +2025-01-25T14:39:07.726Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:07.726Z [INFO] server-d0cfba64-fd20-833a-296c-22464866121d: entering follower state: follower="Node at d0cfba64-fd20-833a-296c-22464866121d [Follower]" leader-address= leader-id= +2025-01-25T14:39:07.820Z [WARN] server-d0cfba64-fd20-833a-296c-22464866121d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:07.821Z [INFO] server-d0cfba64-fd20-833a-296c-22464866121d: entering candidate state: node="Node at d0cfba64-fd20-833a-296c-22464866121d [Candidate]" term=2 +2025-01-25T14:39:07.821Z [INFO] server-d0cfba64-fd20-833a-296c-22464866121d: election won: term=2 tally=1 +2025-01-25T14:39:07.821Z [INFO] server-d0cfba64-fd20-833a-296c-22464866121d: entering leader state: leader="Node at d0cfba64-fd20-833a-296c-22464866121d [Leader]" +2025-01-25T14:39:07.927Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:07.821455293 +0000 UTC m=+16.867294611 (1 nodes), 94.576984ms from start of poll, 95.456945ms from cluster start. Timeout at 2025-01-25 14:39:07.92646521 +0000 UTC m=+16.972304528, 105.009917ms after stability +2025-01-25T14:39:07.927Z [ERROR] server-d0cfba64-fd20-833a-296c-22464866121d: failed to take snapshot: error="nothing new to snapshot" +2025-01-25T14:39:07.930Z [INFO] server-d0cfba64-fd20-833a-296c-22464866121d: starting snapshot up to: index=12 +2025-01-25T14:39:07.930Z [INFO] creating new snapshot: path=/tmp/raft802961427/snapshots/2-12-1737815947930.tmp +2025-01-25T14:39:07.934Z [INFO] server-d0cfba64-fd20-833a-296c-22464866121d: compacting logs: from=1 to=2 +2025-01-25T14:39:07.934Z [INFO] server-d0cfba64-fd20-833a-296c-22464866121d: snapshot complete up to: index=12 +--- PASS: TestRaft_UserSnapshot (0.22s) === RUN TestRaft_UserRestore === RUN TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_false -2025-01-25T14:28:23.585Z [INFO] server-f5f038c3-0c91-3db2-f9a9-7b07b30e36b8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f5f038c3-0c91-3db2-f9a9-7b07b30e36b8 Address:f5f038c3-0c91-3db2-f9a9-7b07b30e36b8} {Suffrage:Voter ID:server-da7a359e-d96a-ff91-fd39-ec315a69c951 Address:da7a359e-d96a-ff91-fd39-ec315a69c951} {Suffrage:Voter ID:server-3e88375d-24de-1f19-4836-cbee105fa881 Address:3e88375d-24de-1f19-4836-cbee105fa881}]" -2025-01-25T14:28:23.585Z [INFO] server-f5f038c3-0c91-3db2-f9a9-7b07b30e36b8: entering follower state: follower="Node at f5f038c3-0c91-3db2-f9a9-7b07b30e36b8 [Follower]" leader-address= leader-id= -2025-01-25T14:28:23.585Z [INFO] server-da7a359e-d96a-ff91-fd39-ec315a69c951: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f5f038c3-0c91-3db2-f9a9-7b07b30e36b8 Address:f5f038c3-0c91-3db2-f9a9-7b07b30e36b8} {Suffrage:Voter ID:server-da7a359e-d96a-ff91-fd39-ec315a69c951 Address:da7a359e-d96a-ff91-fd39-ec315a69c951} {Suffrage:Voter ID:server-3e88375d-24de-1f19-4836-cbee105fa881 Address:3e88375d-24de-1f19-4836-cbee105fa881}]" -2025-01-25T14:28:23.585Z [INFO] server-da7a359e-d96a-ff91-fd39-ec315a69c951: entering follower state: follower="Node at da7a359e-d96a-ff91-fd39-ec315a69c951 [Follower]" leader-address= leader-id= -2025-01-25T14:28:23.585Z [INFO] server-3e88375d-24de-1f19-4836-cbee105fa881: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f5f038c3-0c91-3db2-f9a9-7b07b30e36b8 Address:f5f038c3-0c91-3db2-f9a9-7b07b30e36b8} {Suffrage:Voter ID:server-da7a359e-d96a-ff91-fd39-ec315a69c951 Address:da7a359e-d96a-ff91-fd39-ec315a69c951} {Suffrage:Voter ID:server-3e88375d-24de-1f19-4836-cbee105fa881 Address:3e88375d-24de-1f19-4836-cbee105fa881}]" -2025-01-25T14:28:23.586Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:23.586Z [INFO] server-3e88375d-24de-1f19-4836-cbee105fa881: entering follower state: follower="Node at 3e88375d-24de-1f19-4836-cbee105fa881 [Follower]" leader-address= leader-id= -2025-01-25T14:28:24.254Z [WARN] server-3e88375d-24de-1f19-4836-cbee105fa881: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:24.254Z [INFO] server-3e88375d-24de-1f19-4836-cbee105fa881: entering candidate state: node="Node at 3e88375d-24de-1f19-4836-cbee105fa881 [Candidate]" term=2 -2025-01-25T14:28:24.254Z [INFO] server-3e88375d-24de-1f19-4836-cbee105fa881: election won: term=2 tally=2 -2025-01-25T14:28:24.254Z [INFO] server-3e88375d-24de-1f19-4836-cbee105fa881: entering leader state: leader="Node at 3e88375d-24de-1f19-4836-cbee105fa881 [Leader]" -2025-01-25T14:28:24.254Z [INFO] server-3e88375d-24de-1f19-4836-cbee105fa881: added peer, starting replication: peer=server-f5f038c3-0c91-3db2-f9a9-7b07b30e36b8 -2025-01-25T14:28:24.254Z [INFO] server-3e88375d-24de-1f19-4836-cbee105fa881: added peer, starting replication: peer=server-da7a359e-d96a-ff91-fd39-ec315a69c951 -2025-01-25T14:28:24.254Z [INFO] server-3e88375d-24de-1f19-4836-cbee105fa881: pipelining replication: peer="{Voter server-da7a359e-d96a-ff91-fd39-ec315a69c951 da7a359e-d96a-ff91-fd39-ec315a69c951}" -2025-01-25T14:28:24.255Z [INFO] server-3e88375d-24de-1f19-4836-cbee105fa881: pipelining replication: peer="{Voter server-f5f038c3-0c91-3db2-f9a9-7b07b30e36b8 f5f038c3-0c91-3db2-f9a9-7b07b30e36b8}" -2025-01-25T14:28:25.260Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:24.255166942 +0000 UTC m=+14.109491924 (1 nodes), 669.082763ms from start of poll, 669.950852ms from cluster start. Timeout at 2025-01-25 14:28:25.260169942 +0000 UTC m=+15.114494924, 1.005003s after stability -2025-01-25T14:28:25.261Z [INFO] server-3e88375d-24de-1f19-4836-cbee105fa881: starting snapshot up to: index=12 -2025-01-25T14:28:25.261Z [INFO] creating new snapshot: path=/tmp/raft3249874888/snapshots/2-12-1737815305261.tmp -2025-01-25T14:28:25.263Z [INFO] server-3e88375d-24de-1f19-4836-cbee105fa881: snapshot complete up to: index=12 -2025-01-25T14:28:25.264Z [INFO] creating new snapshot: path=/tmp/raft3249874888/snapshots/2-23-1737815305264.tmp -2025-01-25T14:28:25.265Z [INFO] server-3e88375d-24de-1f19-4836-cbee105fa881: copied to local snapshot: bytes=71 -2025-01-25T14:28:25.266Z [INFO] server-3e88375d-24de-1f19-4836-cbee105fa881: snapshot restore progress: id=2-23-1737815305264 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:25.266Z [INFO] server-3e88375d-24de-1f19-4836-cbee105fa881: restored user snapshot: index=23 -2025-01-25T14:28:25.266Z [ERROR] server-3e88375d-24de-1f19-4836-cbee105fa881: failed to get log: index=23 error="log not found" -2025-01-25T14:28:25.266Z [INFO] server-3e88375d-24de-1f19-4836-cbee105fa881: aborting pipeline replication: peer="{Voter server-da7a359e-d96a-ff91-fd39-ec315a69c951 da7a359e-d96a-ff91-fd39-ec315a69c951}" -2025-01-25T14:28:25.266Z [ERROR] server-3e88375d-24de-1f19-4836-cbee105fa881: failed to get log: index=23 error="log not found" -2025-01-25T14:28:25.266Z [INFO] server-3e88375d-24de-1f19-4836-cbee105fa881: aborting pipeline replication: peer="{Voter server-f5f038c3-0c91-3db2-f9a9-7b07b30e36b8 f5f038c3-0c91-3db2-f9a9-7b07b30e36b8}" -2025-01-25T14:28:25.273Z [ERROR] server-3e88375d-24de-1f19-4836-cbee105fa881: failed to get log: index=23 error="log not found" -2025-01-25T14:28:25.273Z [INFO] creating new snapshot: path=/tmp/raft1302280046/snapshots/2-23-1737815305273.tmp -2025-01-25T14:28:25.274Z [INFO] server-f5f038c3-0c91-3db2-f9a9-7b07b30e36b8: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:25.275Z [INFO] server-f5f038c3-0c91-3db2-f9a9-7b07b30e36b8: copied to local snapshot: bytes=71 -2025-01-25T14:28:25.275Z [ERROR] server-3e88375d-24de-1f19-4836-cbee105fa881: failed to get log: index=23 error="log not found" -2025-01-25T14:28:25.275Z [INFO] server-f5f038c3-0c91-3db2-f9a9-7b07b30e36b8: snapshot restore progress: id=2-23-1737815305273 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:25.275Z [INFO] server-f5f038c3-0c91-3db2-f9a9-7b07b30e36b8: Installed remote snapshot -2025-01-25T14:28:25.275Z [WARN] server-f5f038c3-0c91-3db2-f9a9-7b07b30e36b8: skipping application of old log: index=22 -2025-01-25T14:28:25.275Z [INFO] server-3e88375d-24de-1f19-4836-cbee105fa881: pipelining replication: peer="{Voter server-f5f038c3-0c91-3db2-f9a9-7b07b30e36b8 f5f038c3-0c91-3db2-f9a9-7b07b30e36b8}" -2025-01-25T14:28:25.276Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:28:25.276Z [INFO] creating new snapshot: path=/tmp/raft2831107949/snapshots/2-23-1737815305276.tmp -2025-01-25T14:28:25.276Z [INFO] server-da7a359e-d96a-ff91-fd39-ec315a69c951: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:25.277Z [INFO] server-da7a359e-d96a-ff91-fd39-ec315a69c951: copied to local snapshot: bytes=71 -2025-01-25T14:28:25.278Z [INFO] server-da7a359e-d96a-ff91-fd39-ec315a69c951: snapshot restore progress: id=2-23-1737815305276 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:25.278Z [INFO] server-da7a359e-d96a-ff91-fd39-ec315a69c951: Installed remote snapshot -2025-01-25T14:28:25.278Z [INFO] server-3e88375d-24de-1f19-4836-cbee105fa881: pipelining replication: peer="{Voter server-da7a359e-d96a-ff91-fd39-ec315a69c951 da7a359e-d96a-ff91-fd39-ec315a69c951}" -2025-01-25T14:28:26.282Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:28:25.276075046 +0000 UTC m=+15.130400028 (2 nodes), 2.4µs from start of poll, 1.690858956s from cluster start. Timeout at 2025-01-25 14:28:26.281093005 +0000 UTC m=+16.135417987, 1.005017959s after stability -2025-01-25T14:28:26.294Z [INFO] server-3e88375d-24de-1f19-4836-cbee105fa881: aborting pipeline replication: peer="{Voter server-f5f038c3-0c91-3db2-f9a9-7b07b30e36b8 f5f038c3-0c91-3db2-f9a9-7b07b30e36b8}" -2025-01-25T14:28:26.294Z [INFO] server-3e88375d-24de-1f19-4836-cbee105fa881: aborting pipeline replication: peer="{Voter server-da7a359e-d96a-ff91-fd39-ec315a69c951 da7a359e-d96a-ff91-fd39-ec315a69c951}" +2025-01-25T14:39:07.945Z [INFO] server-6837fa3a-9c93-76cb-f443-20adb132e005: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6837fa3a-9c93-76cb-f443-20adb132e005 Address:6837fa3a-9c93-76cb-f443-20adb132e005} {Suffrage:Voter ID:server-b582c81a-fad7-6310-07a1-2e2528545f11 Address:b582c81a-fad7-6310-07a1-2e2528545f11} {Suffrage:Voter ID:server-77fa750b-bab7-9341-ce7d-f0c2890cc640 Address:77fa750b-bab7-9341-ce7d-f0c2890cc640}]" +2025-01-25T14:39:07.946Z [INFO] server-6837fa3a-9c93-76cb-f443-20adb132e005: entering follower state: follower="Node at 6837fa3a-9c93-76cb-f443-20adb132e005 [Follower]" leader-address= leader-id= +2025-01-25T14:39:07.946Z [INFO] server-b582c81a-fad7-6310-07a1-2e2528545f11: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6837fa3a-9c93-76cb-f443-20adb132e005 Address:6837fa3a-9c93-76cb-f443-20adb132e005} {Suffrage:Voter ID:server-b582c81a-fad7-6310-07a1-2e2528545f11 Address:b582c81a-fad7-6310-07a1-2e2528545f11} {Suffrage:Voter ID:server-77fa750b-bab7-9341-ce7d-f0c2890cc640 Address:77fa750b-bab7-9341-ce7d-f0c2890cc640}]" +2025-01-25T14:39:07.947Z [INFO] server-b582c81a-fad7-6310-07a1-2e2528545f11: entering follower state: follower="Node at b582c81a-fad7-6310-07a1-2e2528545f11 [Follower]" leader-address= leader-id= +2025-01-25T14:39:07.947Z [INFO] server-77fa750b-bab7-9341-ce7d-f0c2890cc640: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6837fa3a-9c93-76cb-f443-20adb132e005 Address:6837fa3a-9c93-76cb-f443-20adb132e005} {Suffrage:Voter ID:server-b582c81a-fad7-6310-07a1-2e2528545f11 Address:b582c81a-fad7-6310-07a1-2e2528545f11} {Suffrage:Voter ID:server-77fa750b-bab7-9341-ce7d-f0c2890cc640 Address:77fa750b-bab7-9341-ce7d-f0c2890cc640}]" +2025-01-25T14:39:07.947Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:07.947Z [INFO] server-77fa750b-bab7-9341-ce7d-f0c2890cc640: entering follower state: follower="Node at 77fa750b-bab7-9341-ce7d-f0c2890cc640 [Follower]" leader-address= leader-id= +2025-01-25T14:39:08.729Z [WARN] server-6837fa3a-9c93-76cb-f443-20adb132e005: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:08.730Z [INFO] server-6837fa3a-9c93-76cb-f443-20adb132e005: entering candidate state: node="Node at 6837fa3a-9c93-76cb-f443-20adb132e005 [Candidate]" term=2 +2025-01-25T14:39:08.730Z [INFO] server-6837fa3a-9c93-76cb-f443-20adb132e005: election won: term=2 tally=2 +2025-01-25T14:39:08.730Z [INFO] server-6837fa3a-9c93-76cb-f443-20adb132e005: entering leader state: leader="Node at 6837fa3a-9c93-76cb-f443-20adb132e005 [Leader]" +2025-01-25T14:39:08.730Z [INFO] server-6837fa3a-9c93-76cb-f443-20adb132e005: added peer, starting replication: peer=server-b582c81a-fad7-6310-07a1-2e2528545f11 +2025-01-25T14:39:08.731Z [INFO] server-6837fa3a-9c93-76cb-f443-20adb132e005: added peer, starting replication: peer=server-77fa750b-bab7-9341-ce7d-f0c2890cc640 +2025-01-25T14:39:08.731Z [INFO] server-6837fa3a-9c93-76cb-f443-20adb132e005: pipelining replication: peer="{Voter server-77fa750b-bab7-9341-ce7d-f0c2890cc640 77fa750b-bab7-9341-ce7d-f0c2890cc640}" +2025-01-25T14:39:08.731Z [INFO] server-6837fa3a-9c93-76cb-f443-20adb132e005: pipelining replication: peer="{Voter server-b582c81a-fad7-6310-07a1-2e2528545f11 b582c81a-fad7-6310-07a1-2e2528545f11}" +2025-01-25T14:39:09.736Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:08.730763149 +0000 UTC m=+17.776602467 (1 nodes), 782.870113ms from start of poll, 785.617038ms from cluster start. Timeout at 2025-01-25 14:39:09.735790565 +0000 UTC m=+18.781629883, 1.005027416s after stability +2025-01-25T14:39:09.739Z [INFO] server-6837fa3a-9c93-76cb-f443-20adb132e005: starting snapshot up to: index=12 +2025-01-25T14:39:09.739Z [INFO] creating new snapshot: path=/tmp/raft2781283595/snapshots/2-12-1737815949739.tmp +2025-01-25T14:39:09.743Z [INFO] server-6837fa3a-9c93-76cb-f443-20adb132e005: snapshot complete up to: index=12 +2025-01-25T14:39:09.747Z [INFO] creating new snapshot: path=/tmp/raft2781283595/snapshots/2-23-1737815949747.tmp +2025-01-25T14:39:09.752Z [INFO] server-6837fa3a-9c93-76cb-f443-20adb132e005: copied to local snapshot: bytes=71 +2025-01-25T14:39:09.753Z [INFO] server-6837fa3a-9c93-76cb-f443-20adb132e005: snapshot restore progress: id=2-23-1737815949747 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:09.753Z [INFO] server-6837fa3a-9c93-76cb-f443-20adb132e005: restored user snapshot: index=23 +2025-01-25T14:39:09.754Z [ERROR] server-6837fa3a-9c93-76cb-f443-20adb132e005: failed to get log: index=23 error="log not found" +2025-01-25T14:39:09.754Z [ERROR] server-6837fa3a-9c93-76cb-f443-20adb132e005: failed to get log: index=23 error="log not found" +2025-01-25T14:39:09.754Z [INFO] server-6837fa3a-9c93-76cb-f443-20adb132e005: aborting pipeline replication: peer="{Voter server-77fa750b-bab7-9341-ce7d-f0c2890cc640 77fa750b-bab7-9341-ce7d-f0c2890cc640}" +2025-01-25T14:39:09.754Z [INFO] server-6837fa3a-9c93-76cb-f443-20adb132e005: aborting pipeline replication: peer="{Voter server-b582c81a-fad7-6310-07a1-2e2528545f11 b582c81a-fad7-6310-07a1-2e2528545f11}" +2025-01-25T14:39:09.763Z [ERROR] server-6837fa3a-9c93-76cb-f443-20adb132e005: failed to get log: index=23 error="log not found" +2025-01-25T14:39:09.764Z [ERROR] server-6837fa3a-9c93-76cb-f443-20adb132e005: failed to get log: index=23 error="log not found" +2025-01-25T14:39:09.765Z [INFO] creating new snapshot: path=/tmp/raft3036258483/snapshots/2-23-1737815949765.tmp +2025-01-25T14:39:09.766Z [INFO] creating new snapshot: path=/tmp/raft3634529025/snapshots/2-23-1737815949766.tmp +2025-01-25T14:39:09.766Z [INFO] server-b582c81a-fad7-6310-07a1-2e2528545f11: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:09.768Z [INFO] server-b582c81a-fad7-6310-07a1-2e2528545f11: copied to local snapshot: bytes=71 +2025-01-25T14:39:09.768Z [INFO] server-77fa750b-bab7-9341-ce7d-f0c2890cc640: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:09.769Z [INFO] server-b582c81a-fad7-6310-07a1-2e2528545f11: snapshot restore progress: id=2-23-1737815949765 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:09.769Z [INFO] server-b582c81a-fad7-6310-07a1-2e2528545f11: Installed remote snapshot +2025-01-25T14:39:09.770Z [INFO] server-6837fa3a-9c93-76cb-f443-20adb132e005: pipelining replication: peer="{Voter server-b582c81a-fad7-6310-07a1-2e2528545f11 b582c81a-fad7-6310-07a1-2e2528545f11}" +2025-01-25T14:39:09.770Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:39:09.771Z [INFO] server-77fa750b-bab7-9341-ce7d-f0c2890cc640: copied to local snapshot: bytes=71 +2025-01-25T14:39:09.772Z [INFO] server-77fa750b-bab7-9341-ce7d-f0c2890cc640: snapshot restore progress: id=2-23-1737815949766 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:09.772Z [INFO] server-77fa750b-bab7-9341-ce7d-f0c2890cc640: Installed remote snapshot +2025-01-25T14:39:09.773Z [INFO] server-6837fa3a-9c93-76cb-f443-20adb132e005: pipelining replication: peer="{Voter server-77fa750b-bab7-9341-ce7d-f0c2890cc640 77fa750b-bab7-9341-ce7d-f0c2890cc640}" +2025-01-25T14:39:10.776Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:39:09.770430093 +0000 UTC m=+18.816269410 (2 nodes), 4.375µs from start of poll, 1.825283981s from cluster start. Timeout at 2025-01-25 14:39:10.775485802 +0000 UTC m=+19.821325120, 1.00505571s after stability +2025-01-25T14:39:10.790Z [INFO] server-6837fa3a-9c93-76cb-f443-20adb132e005: aborting pipeline replication: peer="{Voter server-b582c81a-fad7-6310-07a1-2e2528545f11 b582c81a-fad7-6310-07a1-2e2528545f11}" +2025-01-25T14:39:10.790Z [INFO] server-6837fa3a-9c93-76cb-f443-20adb132e005: aborting pipeline replication: peer="{Voter server-77fa750b-bab7-9341-ce7d-f0c2890cc640 77fa750b-bab7-9341-ce7d-f0c2890cc640}" === RUN TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_false -2025-01-25T14:28:26.300Z [INFO] server-2a0d79cf-9cfb-0500-3f63-44160acf1b11: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2a0d79cf-9cfb-0500-3f63-44160acf1b11 Address:2a0d79cf-9cfb-0500-3f63-44160acf1b11} {Suffrage:Voter ID:server-32597975-50ca-6368-9b67-126543c1b1eb Address:32597975-50ca-6368-9b67-126543c1b1eb} {Suffrage:Voter ID:server-164f35b8-acee-1a04-a74d-8841a8c45ed6 Address:164f35b8-acee-1a04-a74d-8841a8c45ed6}]" -2025-01-25T14:28:26.300Z [INFO] server-32597975-50ca-6368-9b67-126543c1b1eb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2a0d79cf-9cfb-0500-3f63-44160acf1b11 Address:2a0d79cf-9cfb-0500-3f63-44160acf1b11} {Suffrage:Voter ID:server-32597975-50ca-6368-9b67-126543c1b1eb Address:32597975-50ca-6368-9b67-126543c1b1eb} {Suffrage:Voter ID:server-164f35b8-acee-1a04-a74d-8841a8c45ed6 Address:164f35b8-acee-1a04-a74d-8841a8c45ed6}]" -2025-01-25T14:28:26.300Z [INFO] server-2a0d79cf-9cfb-0500-3f63-44160acf1b11: entering follower state: follower="Node at 2a0d79cf-9cfb-0500-3f63-44160acf1b11 [Follower]" leader-address= leader-id= -2025-01-25T14:28:26.300Z [INFO] server-32597975-50ca-6368-9b67-126543c1b1eb: entering follower state: follower="Node at 32597975-50ca-6368-9b67-126543c1b1eb [Follower]" leader-address= leader-id= -2025-01-25T14:28:26.300Z [INFO] server-164f35b8-acee-1a04-a74d-8841a8c45ed6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2a0d79cf-9cfb-0500-3f63-44160acf1b11 Address:2a0d79cf-9cfb-0500-3f63-44160acf1b11} {Suffrage:Voter ID:server-32597975-50ca-6368-9b67-126543c1b1eb Address:32597975-50ca-6368-9b67-126543c1b1eb} {Suffrage:Voter ID:server-164f35b8-acee-1a04-a74d-8841a8c45ed6 Address:164f35b8-acee-1a04-a74d-8841a8c45ed6}]" -2025-01-25T14:28:26.300Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:26.300Z [INFO] server-164f35b8-acee-1a04-a74d-8841a8c45ed6: entering follower state: follower="Node at 164f35b8-acee-1a04-a74d-8841a8c45ed6 [Follower]" leader-address= leader-id= -2025-01-25T14:28:26.833Z [WARN] server-164f35b8-acee-1a04-a74d-8841a8c45ed6: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:26.833Z [INFO] server-164f35b8-acee-1a04-a74d-8841a8c45ed6: entering candidate state: node="Node at 164f35b8-acee-1a04-a74d-8841a8c45ed6 [Candidate]" term=2 -2025-01-25T14:28:26.833Z [INFO] server-164f35b8-acee-1a04-a74d-8841a8c45ed6: election won: term=2 tally=2 -2025-01-25T14:28:26.833Z [INFO] server-164f35b8-acee-1a04-a74d-8841a8c45ed6: entering leader state: leader="Node at 164f35b8-acee-1a04-a74d-8841a8c45ed6 [Leader]" -2025-01-25T14:28:26.833Z [INFO] server-164f35b8-acee-1a04-a74d-8841a8c45ed6: added peer, starting replication: peer=server-2a0d79cf-9cfb-0500-3f63-44160acf1b11 -2025-01-25T14:28:26.833Z [INFO] server-164f35b8-acee-1a04-a74d-8841a8c45ed6: added peer, starting replication: peer=server-32597975-50ca-6368-9b67-126543c1b1eb -2025-01-25T14:28:26.833Z [INFO] server-164f35b8-acee-1a04-a74d-8841a8c45ed6: pipelining replication: peer="{Voter server-32597975-50ca-6368-9b67-126543c1b1eb 32597975-50ca-6368-9b67-126543c1b1eb}" -2025-01-25T14:28:26.833Z [INFO] server-164f35b8-acee-1a04-a74d-8841a8c45ed6: pipelining replication: peer="{Voter server-2a0d79cf-9cfb-0500-3f63-44160acf1b11 2a0d79cf-9cfb-0500-3f63-44160acf1b11}" -2025-01-25T14:28:27.839Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:26.833754913 +0000 UTC m=+16.688079895 (1 nodes), 533.055978ms from start of poll, 533.84227ms from cluster start. Timeout at 2025-01-25 14:28:27.838757513 +0000 UTC m=+17.693082495, 1.0050026s after stability -2025-01-25T14:28:27.840Z [INFO] server-164f35b8-acee-1a04-a74d-8841a8c45ed6: starting snapshot up to: index=12 -2025-01-25T14:28:27.840Z [INFO] creating new snapshot: path=/tmp/raft1058140866/snapshots/2-12-1737815307840.tmp -2025-01-25T14:28:27.841Z [INFO] server-164f35b8-acee-1a04-a74d-8841a8c45ed6: snapshot complete up to: index=12 -2025-01-25T14:28:27.844Z [INFO] creating new snapshot: path=/tmp/raft1058140866/snapshots/2-23-1737815307844.tmp -2025-01-25T14:28:27.845Z [INFO] server-164f35b8-acee-1a04-a74d-8841a8c45ed6: copied to local snapshot: bytes=71 -2025-01-25T14:28:27.845Z [INFO] server-164f35b8-acee-1a04-a74d-8841a8c45ed6: snapshot restore progress: id=2-23-1737815307844 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:27.845Z [INFO] server-164f35b8-acee-1a04-a74d-8841a8c45ed6: removing all old logs from log store -2025-01-25T14:28:27.845Z [INFO] server-164f35b8-acee-1a04-a74d-8841a8c45ed6: compacting logs: from=1 to=22 -2025-01-25T14:28:27.846Z [INFO] server-164f35b8-acee-1a04-a74d-8841a8c45ed6: restored user snapshot: index=23 -2025-01-25T14:28:27.846Z [ERROR] server-164f35b8-acee-1a04-a74d-8841a8c45ed6: failed to get log: index=22 error="log not found" -2025-01-25T14:28:27.846Z [INFO] server-164f35b8-acee-1a04-a74d-8841a8c45ed6: aborting pipeline replication: peer="{Voter server-32597975-50ca-6368-9b67-126543c1b1eb 32597975-50ca-6368-9b67-126543c1b1eb}" -2025-01-25T14:28:27.846Z [ERROR] server-164f35b8-acee-1a04-a74d-8841a8c45ed6: failed to get log: index=22 error="log not found" -2025-01-25T14:28:27.846Z [INFO] server-164f35b8-acee-1a04-a74d-8841a8c45ed6: aborting pipeline replication: peer="{Voter server-2a0d79cf-9cfb-0500-3f63-44160acf1b11 2a0d79cf-9cfb-0500-3f63-44160acf1b11}" -2025-01-25T14:28:27.853Z [ERROR] server-164f35b8-acee-1a04-a74d-8841a8c45ed6: failed to get log: index=22 error="log not found" -2025-01-25T14:28:27.853Z [ERROR] server-164f35b8-acee-1a04-a74d-8841a8c45ed6: failed to get log: index=22 error="log not found" -2025-01-25T14:28:27.854Z [INFO] creating new snapshot: path=/tmp/raft204205405/snapshots/2-23-1737815307854.tmp -2025-01-25T14:28:27.854Z [INFO] creating new snapshot: path=/tmp/raft15249334/snapshots/2-23-1737815307854.tmp -2025-01-25T14:28:27.854Z [INFO] server-32597975-50ca-6368-9b67-126543c1b1eb: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:27.855Z [INFO] server-2a0d79cf-9cfb-0500-3f63-44160acf1b11: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:27.855Z [INFO] server-32597975-50ca-6368-9b67-126543c1b1eb: copied to local snapshot: bytes=71 -2025-01-25T14:28:27.855Z [INFO] server-32597975-50ca-6368-9b67-126543c1b1eb: snapshot restore progress: id=2-23-1737815307854 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:27.855Z [INFO] server-2a0d79cf-9cfb-0500-3f63-44160acf1b11: copied to local snapshot: bytes=71 -2025-01-25T14:28:27.855Z [INFO] server-32597975-50ca-6368-9b67-126543c1b1eb: removing all old logs from log store -2025-01-25T14:28:27.856Z [INFO] server-32597975-50ca-6368-9b67-126543c1b1eb: compacting logs: from=1 to=22 -2025-01-25T14:28:27.856Z [INFO] server-32597975-50ca-6368-9b67-126543c1b1eb: Installed remote snapshot -2025-01-25T14:28:27.856Z [WARN] server-32597975-50ca-6368-9b67-126543c1b1eb: skipping application of old log: index=22 -2025-01-25T14:28:27.856Z [INFO] server-164f35b8-acee-1a04-a74d-8841a8c45ed6: pipelining replication: peer="{Voter server-32597975-50ca-6368-9b67-126543c1b1eb 32597975-50ca-6368-9b67-126543c1b1eb}" -2025-01-25T14:28:27.856Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:28:27.856Z [INFO] server-2a0d79cf-9cfb-0500-3f63-44160acf1b11: snapshot restore progress: id=2-23-1737815307854 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:27.856Z [INFO] server-2a0d79cf-9cfb-0500-3f63-44160acf1b11: removing all old logs from log store -2025-01-25T14:28:27.856Z [INFO] server-2a0d79cf-9cfb-0500-3f63-44160acf1b11: compacting logs: from=1 to=22 -2025-01-25T14:28:27.856Z [INFO] server-2a0d79cf-9cfb-0500-3f63-44160acf1b11: Installed remote snapshot -2025-01-25T14:28:27.856Z [INFO] server-164f35b8-acee-1a04-a74d-8841a8c45ed6: pipelining replication: peer="{Voter server-2a0d79cf-9cfb-0500-3f63-44160acf1b11 2a0d79cf-9cfb-0500-3f63-44160acf1b11}" -2025-01-25T14:28:28.862Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:28:27.856358676 +0000 UTC m=+17.710683658 (2 nodes), 1.92µs from start of poll, 1.556446033s from cluster start. Timeout at 2025-01-25 14:28:28.861381835 +0000 UTC m=+18.715706937, 1.005023279s after stability -2025-01-25T14:28:28.875Z [INFO] server-164f35b8-acee-1a04-a74d-8841a8c45ed6: aborting pipeline replication: peer="{Voter server-32597975-50ca-6368-9b67-126543c1b1eb 32597975-50ca-6368-9b67-126543c1b1eb}" -2025-01-25T14:28:28.876Z [INFO] server-164f35b8-acee-1a04-a74d-8841a8c45ed6: aborting pipeline replication: peer="{Voter server-2a0d79cf-9cfb-0500-3f63-44160acf1b11 2a0d79cf-9cfb-0500-3f63-44160acf1b11}" +2025-01-25T14:39:10.802Z [INFO] server-a0d21218-ac29-019f-7935-a7fc83fb2e7c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a0d21218-ac29-019f-7935-a7fc83fb2e7c Address:a0d21218-ac29-019f-7935-a7fc83fb2e7c} {Suffrage:Voter ID:server-62701990-6efc-f83f-a56b-03b634322837 Address:62701990-6efc-f83f-a56b-03b634322837} {Suffrage:Voter ID:server-90fc1ea2-72de-677a-a019-f4723e291b29 Address:90fc1ea2-72de-677a-a019-f4723e291b29}]" +2025-01-25T14:39:10.802Z [INFO] server-a0d21218-ac29-019f-7935-a7fc83fb2e7c: entering follower state: follower="Node at a0d21218-ac29-019f-7935-a7fc83fb2e7c [Follower]" leader-address= leader-id= +2025-01-25T14:39:10.803Z [INFO] server-62701990-6efc-f83f-a56b-03b634322837: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a0d21218-ac29-019f-7935-a7fc83fb2e7c Address:a0d21218-ac29-019f-7935-a7fc83fb2e7c} {Suffrage:Voter ID:server-62701990-6efc-f83f-a56b-03b634322837 Address:62701990-6efc-f83f-a56b-03b634322837} {Suffrage:Voter ID:server-90fc1ea2-72de-677a-a019-f4723e291b29 Address:90fc1ea2-72de-677a-a019-f4723e291b29}]" +2025-01-25T14:39:10.803Z [INFO] server-62701990-6efc-f83f-a56b-03b634322837: entering follower state: follower="Node at 62701990-6efc-f83f-a56b-03b634322837 [Follower]" leader-address= leader-id= +2025-01-25T14:39:10.804Z [INFO] server-90fc1ea2-72de-677a-a019-f4723e291b29: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a0d21218-ac29-019f-7935-a7fc83fb2e7c Address:a0d21218-ac29-019f-7935-a7fc83fb2e7c} {Suffrage:Voter ID:server-62701990-6efc-f83f-a56b-03b634322837 Address:62701990-6efc-f83f-a56b-03b634322837} {Suffrage:Voter ID:server-90fc1ea2-72de-677a-a019-f4723e291b29 Address:90fc1ea2-72de-677a-a019-f4723e291b29}]" +2025-01-25T14:39:10.804Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:10.804Z [INFO] server-90fc1ea2-72de-677a-a019-f4723e291b29: entering follower state: follower="Node at 90fc1ea2-72de-677a-a019-f4723e291b29 [Follower]" leader-address= leader-id= +2025-01-25T14:39:11.313Z [WARN] server-a0d21218-ac29-019f-7935-a7fc83fb2e7c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:11.313Z [INFO] server-a0d21218-ac29-019f-7935-a7fc83fb2e7c: entering candidate state: node="Node at a0d21218-ac29-019f-7935-a7fc83fb2e7c [Candidate]" term=2 +2025-01-25T14:39:11.313Z [INFO] server-a0d21218-ac29-019f-7935-a7fc83fb2e7c: election won: term=2 tally=2 +2025-01-25T14:39:11.314Z [INFO] server-a0d21218-ac29-019f-7935-a7fc83fb2e7c: entering leader state: leader="Node at a0d21218-ac29-019f-7935-a7fc83fb2e7c [Leader]" +2025-01-25T14:39:11.314Z [INFO] server-a0d21218-ac29-019f-7935-a7fc83fb2e7c: added peer, starting replication: peer=server-62701990-6efc-f83f-a56b-03b634322837 +2025-01-25T14:39:11.314Z [INFO] server-a0d21218-ac29-019f-7935-a7fc83fb2e7c: added peer, starting replication: peer=server-90fc1ea2-72de-677a-a019-f4723e291b29 +2025-01-25T14:39:11.315Z [INFO] server-a0d21218-ac29-019f-7935-a7fc83fb2e7c: pipelining replication: peer="{Voter server-62701990-6efc-f83f-a56b-03b634322837 62701990-6efc-f83f-a56b-03b634322837}" +2025-01-25T14:39:11.315Z [INFO] server-a0d21218-ac29-019f-7935-a7fc83fb2e7c: pipelining replication: peer="{Voter server-90fc1ea2-72de-677a-a019-f4723e291b29 90fc1ea2-72de-677a-a019-f4723e291b29}" +2025-01-25T14:39:12.319Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:11.31405442 +0000 UTC m=+20.359893738 (1 nodes), 509.029107ms from start of poll, 512.178241ms from cluster start. Timeout at 2025-01-25 14:39:12.319066962 +0000 UTC m=+21.364906280, 1.005012542s after stability +2025-01-25T14:39:12.323Z [INFO] server-a0d21218-ac29-019f-7935-a7fc83fb2e7c: starting snapshot up to: index=12 +2025-01-25T14:39:12.323Z [INFO] creating new snapshot: path=/tmp/raft3513126006/snapshots/2-12-1737815952323.tmp +2025-01-25T14:39:12.327Z [INFO] server-a0d21218-ac29-019f-7935-a7fc83fb2e7c: snapshot complete up to: index=12 +2025-01-25T14:39:12.331Z [INFO] creating new snapshot: path=/tmp/raft3513126006/snapshots/2-23-1737815952331.tmp +2025-01-25T14:39:12.336Z [INFO] server-a0d21218-ac29-019f-7935-a7fc83fb2e7c: copied to local snapshot: bytes=71 +2025-01-25T14:39:12.337Z [INFO] server-a0d21218-ac29-019f-7935-a7fc83fb2e7c: snapshot restore progress: id=2-23-1737815952331 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:12.338Z [INFO] server-a0d21218-ac29-019f-7935-a7fc83fb2e7c: removing all old logs from log store +2025-01-25T14:39:12.338Z [INFO] server-a0d21218-ac29-019f-7935-a7fc83fb2e7c: compacting logs: from=1 to=22 +2025-01-25T14:39:12.338Z [INFO] server-a0d21218-ac29-019f-7935-a7fc83fb2e7c: restored user snapshot: index=23 +2025-01-25T14:39:12.338Z [ERROR] server-a0d21218-ac29-019f-7935-a7fc83fb2e7c: failed to get log: index=22 error="log not found" +2025-01-25T14:39:12.338Z [INFO] server-a0d21218-ac29-019f-7935-a7fc83fb2e7c: aborting pipeline replication: peer="{Voter server-90fc1ea2-72de-677a-a019-f4723e291b29 90fc1ea2-72de-677a-a019-f4723e291b29}" +2025-01-25T14:39:12.338Z [ERROR] server-a0d21218-ac29-019f-7935-a7fc83fb2e7c: failed to get log: index=22 error="log not found" +2025-01-25T14:39:12.338Z [INFO] server-a0d21218-ac29-019f-7935-a7fc83fb2e7c: aborting pipeline replication: peer="{Voter server-62701990-6efc-f83f-a56b-03b634322837 62701990-6efc-f83f-a56b-03b634322837}" +2025-01-25T14:39:12.344Z [ERROR] server-a0d21218-ac29-019f-7935-a7fc83fb2e7c: failed to get log: index=22 error="log not found" +2025-01-25T14:39:12.346Z [INFO] creating new snapshot: path=/tmp/raft4045979254/snapshots/2-23-1737815952346.tmp +2025-01-25T14:39:12.347Z [INFO] server-90fc1ea2-72de-677a-a019-f4723e291b29: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:12.349Z [ERROR] server-a0d21218-ac29-019f-7935-a7fc83fb2e7c: failed to get log: index=22 error="log not found" +2025-01-25T14:39:12.350Z [INFO] server-90fc1ea2-72de-677a-a019-f4723e291b29: copied to local snapshot: bytes=71 +2025-01-25T14:39:12.351Z [INFO] creating new snapshot: path=/tmp/raft910465838/snapshots/2-23-1737815952351.tmp +2025-01-25T14:39:12.351Z [INFO] server-90fc1ea2-72de-677a-a019-f4723e291b29: snapshot restore progress: id=2-23-1737815952346 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:12.352Z [INFO] server-90fc1ea2-72de-677a-a019-f4723e291b29: removing all old logs from log store +2025-01-25T14:39:12.352Z [INFO] server-90fc1ea2-72de-677a-a019-f4723e291b29: compacting logs: from=1 to=22 +2025-01-25T14:39:12.352Z [INFO] server-90fc1ea2-72de-677a-a019-f4723e291b29: Installed remote snapshot +2025-01-25T14:39:12.353Z [WARN] server-90fc1ea2-72de-677a-a019-f4723e291b29: skipping application of old log: index=22 +2025-01-25T14:39:12.353Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:39:12.353Z [INFO] server-a0d21218-ac29-019f-7935-a7fc83fb2e7c: pipelining replication: peer="{Voter server-90fc1ea2-72de-677a-a019-f4723e291b29 90fc1ea2-72de-677a-a019-f4723e291b29}" +2025-01-25T14:39:12.353Z [INFO] server-62701990-6efc-f83f-a56b-03b634322837: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:12.356Z [INFO] server-62701990-6efc-f83f-a56b-03b634322837: copied to local snapshot: bytes=71 +2025-01-25T14:39:12.357Z [INFO] server-62701990-6efc-f83f-a56b-03b634322837: snapshot restore progress: id=2-23-1737815952351 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:12.358Z [INFO] server-62701990-6efc-f83f-a56b-03b634322837: removing all old logs from log store +2025-01-25T14:39:12.358Z [INFO] server-62701990-6efc-f83f-a56b-03b634322837: compacting logs: from=1 to=22 +2025-01-25T14:39:12.358Z [INFO] server-62701990-6efc-f83f-a56b-03b634322837: Installed remote snapshot +2025-01-25T14:39:12.359Z [INFO] server-a0d21218-ac29-019f-7935-a7fc83fb2e7c: pipelining replication: peer="{Voter server-62701990-6efc-f83f-a56b-03b634322837 62701990-6efc-f83f-a56b-03b634322837}" +2025-01-25T14:39:13.359Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:39:12.353870448 +0000 UTC m=+21.399709765 (2 nodes), 4.666µs from start of poll, 1.551994268s from cluster start. Timeout at 2025-01-25 14:39:13.358989157 +0000 UTC m=+22.404828474, 1.005118709s after stability +2025-01-25T14:39:13.373Z [INFO] server-a0d21218-ac29-019f-7935-a7fc83fb2e7c: aborting pipeline replication: peer="{Voter server-90fc1ea2-72de-677a-a019-f4723e291b29 90fc1ea2-72de-677a-a019-f4723e291b29}" +2025-01-25T14:39:13.373Z [INFO] server-a0d21218-ac29-019f-7935-a7fc83fb2e7c: aborting pipeline replication: peer="{Voter server-62701990-6efc-f83f-a56b-03b634322837 62701990-6efc-f83f-a56b-03b634322837}" === RUN TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_true -2025-01-25T14:28:28.878Z [INFO] server-dd0b3047-2bc0-c9c9-a469-a4a4a4d223f0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dd0b3047-2bc0-c9c9-a469-a4a4a4d223f0 Address:dd0b3047-2bc0-c9c9-a469-a4a4a4d223f0} {Suffrage:Voter ID:server-a640e938-62dd-b60f-a2a4-83e2d644b629 Address:a640e938-62dd-b60f-a2a4-83e2d644b629} {Suffrage:Voter ID:server-95b0d786-5552-3df0-7e2a-df0bbd4203df Address:95b0d786-5552-3df0-7e2a-df0bbd4203df}]" -2025-01-25T14:28:28.879Z [INFO] server-dd0b3047-2bc0-c9c9-a469-a4a4a4d223f0: entering follower state: follower="Node at dd0b3047-2bc0-c9c9-a469-a4a4a4d223f0 [Follower]" leader-address= leader-id= -2025-01-25T14:28:28.879Z [INFO] server-a640e938-62dd-b60f-a2a4-83e2d644b629: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dd0b3047-2bc0-c9c9-a469-a4a4a4d223f0 Address:dd0b3047-2bc0-c9c9-a469-a4a4a4d223f0} {Suffrage:Voter ID:server-a640e938-62dd-b60f-a2a4-83e2d644b629 Address:a640e938-62dd-b60f-a2a4-83e2d644b629} {Suffrage:Voter ID:server-95b0d786-5552-3df0-7e2a-df0bbd4203df Address:95b0d786-5552-3df0-7e2a-df0bbd4203df}]" -2025-01-25T14:28:28.879Z [INFO] server-a640e938-62dd-b60f-a2a4-83e2d644b629: entering follower state: follower="Node at a640e938-62dd-b60f-a2a4-83e2d644b629 [Follower]" leader-address= leader-id= -2025-01-25T14:28:28.879Z [INFO] server-95b0d786-5552-3df0-7e2a-df0bbd4203df: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dd0b3047-2bc0-c9c9-a469-a4a4a4d223f0 Address:dd0b3047-2bc0-c9c9-a469-a4a4a4d223f0} {Suffrage:Voter ID:server-a640e938-62dd-b60f-a2a4-83e2d644b629 Address:a640e938-62dd-b60f-a2a4-83e2d644b629} {Suffrage:Voter ID:server-95b0d786-5552-3df0-7e2a-df0bbd4203df Address:95b0d786-5552-3df0-7e2a-df0bbd4203df}]" -2025-01-25T14:28:28.879Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:28.879Z [INFO] server-95b0d786-5552-3df0-7e2a-df0bbd4203df: entering follower state: follower="Node at 95b0d786-5552-3df0-7e2a-df0bbd4203df [Follower]" leader-address= leader-id= -2025-01-25T14:28:29.390Z [WARN] server-95b0d786-5552-3df0-7e2a-df0bbd4203df: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:29.390Z [INFO] server-95b0d786-5552-3df0-7e2a-df0bbd4203df: entering candidate state: node="Node at 95b0d786-5552-3df0-7e2a-df0bbd4203df [Candidate]" term=2 -2025-01-25T14:28:29.390Z [INFO] server-95b0d786-5552-3df0-7e2a-df0bbd4203df: election won: term=2 tally=2 -2025-01-25T14:28:29.390Z [INFO] server-95b0d786-5552-3df0-7e2a-df0bbd4203df: entering leader state: leader="Node at 95b0d786-5552-3df0-7e2a-df0bbd4203df [Leader]" -2025-01-25T14:28:29.390Z [INFO] server-95b0d786-5552-3df0-7e2a-df0bbd4203df: added peer, starting replication: peer=server-dd0b3047-2bc0-c9c9-a469-a4a4a4d223f0 -2025-01-25T14:28:29.390Z [INFO] server-95b0d786-5552-3df0-7e2a-df0bbd4203df: added peer, starting replication: peer=server-a640e938-62dd-b60f-a2a4-83e2d644b629 -2025-01-25T14:28:29.390Z [INFO] server-95b0d786-5552-3df0-7e2a-df0bbd4203df: pipelining replication: peer="{Voter server-dd0b3047-2bc0-c9c9-a469-a4a4a4d223f0 dd0b3047-2bc0-c9c9-a469-a4a4a4d223f0}" -2025-01-25T14:28:29.390Z [INFO] server-95b0d786-5552-3df0-7e2a-df0bbd4203df: pipelining replication: peer="{Voter server-a640e938-62dd-b60f-a2a4-83e2d644b629 a640e938-62dd-b60f-a2a4-83e2d644b629}" -2025-01-25T14:28:30.396Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:29.390756741 +0000 UTC m=+19.245081723 (1 nodes), 511.150566ms from start of poll, 512.079973ms from cluster start. Timeout at 2025-01-25 14:28:30.39576994 +0000 UTC m=+20.250094882, 1.005013159s after stability -2025-01-25T14:28:30.398Z [INFO] server-95b0d786-5552-3df0-7e2a-df0bbd4203df: starting snapshot up to: index=12 -2025-01-25T14:28:30.398Z [INFO] creating new snapshot: path=/tmp/raft77379406/snapshots/2-12-1737815310398.tmp -2025-01-25T14:28:30.399Z [INFO] server-95b0d786-5552-3df0-7e2a-df0bbd4203df: snapshot complete up to: index=12 -2025-01-25T14:28:30.402Z [INFO] server-eef17de3-64ee-8f3e-7579-3c1c746e4acc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eef17de3-64ee-8f3e-7579-3c1c746e4acc Address:eef17de3-64ee-8f3e-7579-3c1c746e4acc} {Suffrage:Voter ID:server-66ece0d9-ebca-02c7-a98d-14a1ce7a503b Address:66ece0d9-ebca-02c7-a98d-14a1ce7a503b} {Suffrage:Voter ID:server-972da02a-ccc4-5e53-1f90-1452016ead72 Address:972da02a-ccc4-5e53-1f90-1452016ead72}]" -2025-01-25T14:28:30.402Z [INFO] server-66ece0d9-ebca-02c7-a98d-14a1ce7a503b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eef17de3-64ee-8f3e-7579-3c1c746e4acc Address:eef17de3-64ee-8f3e-7579-3c1c746e4acc} {Suffrage:Voter ID:server-66ece0d9-ebca-02c7-a98d-14a1ce7a503b Address:66ece0d9-ebca-02c7-a98d-14a1ce7a503b} {Suffrage:Voter ID:server-972da02a-ccc4-5e53-1f90-1452016ead72 Address:972da02a-ccc4-5e53-1f90-1452016ead72}]" -2025-01-25T14:28:30.402Z [INFO] server-eef17de3-64ee-8f3e-7579-3c1c746e4acc: entering follower state: follower="Node at eef17de3-64ee-8f3e-7579-3c1c746e4acc [Follower]" leader-address= leader-id= -2025-01-25T14:28:30.402Z [INFO] server-66ece0d9-ebca-02c7-a98d-14a1ce7a503b: entering follower state: follower="Node at 66ece0d9-ebca-02c7-a98d-14a1ce7a503b [Follower]" leader-address= leader-id= -2025-01-25T14:28:30.402Z [INFO] server-972da02a-ccc4-5e53-1f90-1452016ead72: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eef17de3-64ee-8f3e-7579-3c1c746e4acc Address:eef17de3-64ee-8f3e-7579-3c1c746e4acc} {Suffrage:Voter ID:server-66ece0d9-ebca-02c7-a98d-14a1ce7a503b Address:66ece0d9-ebca-02c7-a98d-14a1ce7a503b} {Suffrage:Voter ID:server-972da02a-ccc4-5e53-1f90-1452016ead72 Address:972da02a-ccc4-5e53-1f90-1452016ead72}]" -2025-01-25T14:28:30.402Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:30.402Z [INFO] server-972da02a-ccc4-5e53-1f90-1452016ead72: entering follower state: follower="Node at 972da02a-ccc4-5e53-1f90-1452016ead72 [Follower]" leader-address= leader-id= -2025-01-25T14:28:30.916Z [WARN] server-972da02a-ccc4-5e53-1f90-1452016ead72: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:30.916Z [INFO] server-972da02a-ccc4-5e53-1f90-1452016ead72: entering candidate state: node="Node at 972da02a-ccc4-5e53-1f90-1452016ead72 [Candidate]" term=2 -2025-01-25T14:28:30.916Z [INFO] server-972da02a-ccc4-5e53-1f90-1452016ead72: election won: term=2 tally=2 -2025-01-25T14:28:30.916Z [INFO] server-972da02a-ccc4-5e53-1f90-1452016ead72: entering leader state: leader="Node at 972da02a-ccc4-5e53-1f90-1452016ead72 [Leader]" -2025-01-25T14:28:30.916Z [INFO] server-972da02a-ccc4-5e53-1f90-1452016ead72: added peer, starting replication: peer=server-eef17de3-64ee-8f3e-7579-3c1c746e4acc -2025-01-25T14:28:30.917Z [INFO] server-972da02a-ccc4-5e53-1f90-1452016ead72: added peer, starting replication: peer=server-66ece0d9-ebca-02c7-a98d-14a1ce7a503b -2025-01-25T14:28:30.917Z [INFO] server-972da02a-ccc4-5e53-1f90-1452016ead72: pipelining replication: peer="{Voter server-66ece0d9-ebca-02c7-a98d-14a1ce7a503b 66ece0d9-ebca-02c7-a98d-14a1ce7a503b}" -2025-01-25T14:28:30.917Z [INFO] server-972da02a-ccc4-5e53-1f90-1452016ead72: pipelining replication: peer="{Voter server-eef17de3-64ee-8f3e-7579-3c1c746e4acc eef17de3-64ee-8f3e-7579-3c1c746e4acc}" -2025-01-25T14:28:31.922Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:30.917218332 +0000 UTC m=+20.771543314 (1 nodes), 514.605922ms from start of poll, 515.461651ms from cluster start. Timeout at 2025-01-25 14:28:31.922233251 +0000 UTC m=+21.776558233, 1.005014919s after stability -2025-01-25T14:28:31.922Z [INFO] creating new snapshot: path=/tmp/raft830560830/snapshots/2-13-1737815311922.tmp -2025-01-25T14:28:31.924Z [INFO] server-972da02a-ccc4-5e53-1f90-1452016ead72: copied to local snapshot: bytes=71 -2025-01-25T14:28:31.924Z [INFO] server-972da02a-ccc4-5e53-1f90-1452016ead72: snapshot restore progress: id=2-13-1737815311922 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:31.924Z [INFO] server-972da02a-ccc4-5e53-1f90-1452016ead72: restored user snapshot: index=13 -2025-01-25T14:28:31.924Z [ERROR] server-972da02a-ccc4-5e53-1f90-1452016ead72: failed to get log: index=3 error="log not found" -2025-01-25T14:28:31.924Z [ERROR] server-972da02a-ccc4-5e53-1f90-1452016ead72: failed to get log: index=3 error="log not found" -2025-01-25T14:28:31.924Z [INFO] server-972da02a-ccc4-5e53-1f90-1452016ead72: aborting pipeline replication: peer="{Voter server-eef17de3-64ee-8f3e-7579-3c1c746e4acc eef17de3-64ee-8f3e-7579-3c1c746e4acc}" -2025-01-25T14:28:31.924Z [INFO] server-972da02a-ccc4-5e53-1f90-1452016ead72: aborting pipeline replication: peer="{Voter server-66ece0d9-ebca-02c7-a98d-14a1ce7a503b 66ece0d9-ebca-02c7-a98d-14a1ce7a503b}" -2025-01-25T14:28:31.931Z [ERROR] server-972da02a-ccc4-5e53-1f90-1452016ead72: failed to get log: index=3 error="log not found" -2025-01-25T14:28:31.932Z [INFO] creating new snapshot: path=/tmp/raft1915430452/snapshots/2-13-1737815311932.tmp -2025-01-25T14:28:31.933Z [ERROR] server-972da02a-ccc4-5e53-1f90-1452016ead72: failed to get log: index=3 error="log not found" -2025-01-25T14:28:31.934Z [INFO] server-66ece0d9-ebca-02c7-a98d-14a1ce7a503b: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:31.935Z [INFO] creating new snapshot: path=/tmp/raft1345354572/snapshots/2-13-1737815311935.tmp -2025-01-25T14:28:31.935Z [INFO] server-66ece0d9-ebca-02c7-a98d-14a1ce7a503b: copied to local snapshot: bytes=71 -2025-01-25T14:28:31.935Z [INFO] server-eef17de3-64ee-8f3e-7579-3c1c746e4acc: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:31.936Z [INFO] server-66ece0d9-ebca-02c7-a98d-14a1ce7a503b: snapshot restore progress: id=2-13-1737815311932 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:31.936Z [INFO] server-66ece0d9-ebca-02c7-a98d-14a1ce7a503b: Installed remote snapshot -2025-01-25T14:28:31.936Z [INFO] server-eef17de3-64ee-8f3e-7579-3c1c746e4acc: copied to local snapshot: bytes=71 -2025-01-25T14:28:31.936Z [INFO] server-972da02a-ccc4-5e53-1f90-1452016ead72: pipelining replication: peer="{Voter server-66ece0d9-ebca-02c7-a98d-14a1ce7a503b 66ece0d9-ebca-02c7-a98d-14a1ce7a503b}" -2025-01-25T14:28:31.936Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:28:31.936Z [INFO] server-eef17de3-64ee-8f3e-7579-3c1c746e4acc: snapshot restore progress: id=2-13-1737815311935 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:31.936Z [INFO] server-eef17de3-64ee-8f3e-7579-3c1c746e4acc: Installed remote snapshot -2025-01-25T14:28:31.936Z [INFO] server-972da02a-ccc4-5e53-1f90-1452016ead72: pipelining replication: peer="{Voter server-eef17de3-64ee-8f3e-7579-3c1c746e4acc eef17de3-64ee-8f3e-7579-3c1c746e4acc}" -2025-01-25T14:28:32.941Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:28:31.936556932 +0000 UTC m=+21.790881914 (2 nodes), 1.8µs from start of poll, 1.534800251s from cluster start. Timeout at 2025-01-25 14:28:32.941595051 +0000 UTC m=+22.795920033, 1.005038119s after stability -2025-01-25T14:28:32.954Z [INFO] server-95b0d786-5552-3df0-7e2a-df0bbd4203df: aborting pipeline replication: peer="{Voter server-a640e938-62dd-b60f-a2a4-83e2d644b629 a640e938-62dd-b60f-a2a4-83e2d644b629}" -2025-01-25T14:28:32.954Z [INFO] server-95b0d786-5552-3df0-7e2a-df0bbd4203df: aborting pipeline replication: peer="{Voter server-dd0b3047-2bc0-c9c9-a469-a4a4a4d223f0 dd0b3047-2bc0-c9c9-a469-a4a4a4d223f0}" +2025-01-25T14:39:13.385Z [INFO] server-c8afbd38-b716-a4e7-67f7-427b37380fef: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c8afbd38-b716-a4e7-67f7-427b37380fef Address:c8afbd38-b716-a4e7-67f7-427b37380fef} {Suffrage:Voter ID:server-f61f03f6-0a21-9f97-f451-791f2adf0792 Address:f61f03f6-0a21-9f97-f451-791f2adf0792} {Suffrage:Voter ID:server-ed370f7b-8d02-fab0-add8-0401ad051891 Address:ed370f7b-8d02-fab0-add8-0401ad051891}]" +2025-01-25T14:39:13.385Z [INFO] server-c8afbd38-b716-a4e7-67f7-427b37380fef: entering follower state: follower="Node at c8afbd38-b716-a4e7-67f7-427b37380fef [Follower]" leader-address= leader-id= +2025-01-25T14:39:13.385Z [INFO] server-f61f03f6-0a21-9f97-f451-791f2adf0792: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c8afbd38-b716-a4e7-67f7-427b37380fef Address:c8afbd38-b716-a4e7-67f7-427b37380fef} {Suffrage:Voter ID:server-f61f03f6-0a21-9f97-f451-791f2adf0792 Address:f61f03f6-0a21-9f97-f451-791f2adf0792} {Suffrage:Voter ID:server-ed370f7b-8d02-fab0-add8-0401ad051891 Address:ed370f7b-8d02-fab0-add8-0401ad051891}]" +2025-01-25T14:39:13.386Z [INFO] server-f61f03f6-0a21-9f97-f451-791f2adf0792: entering follower state: follower="Node at f61f03f6-0a21-9f97-f451-791f2adf0792 [Follower]" leader-address= leader-id= +2025-01-25T14:39:13.386Z [INFO] server-ed370f7b-8d02-fab0-add8-0401ad051891: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c8afbd38-b716-a4e7-67f7-427b37380fef Address:c8afbd38-b716-a4e7-67f7-427b37380fef} {Suffrage:Voter ID:server-f61f03f6-0a21-9f97-f451-791f2adf0792 Address:f61f03f6-0a21-9f97-f451-791f2adf0792} {Suffrage:Voter ID:server-ed370f7b-8d02-fab0-add8-0401ad051891 Address:ed370f7b-8d02-fab0-add8-0401ad051891}]" +2025-01-25T14:39:13.387Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:13.387Z [INFO] server-ed370f7b-8d02-fab0-add8-0401ad051891: entering follower state: follower="Node at ed370f7b-8d02-fab0-add8-0401ad051891 [Follower]" leader-address= leader-id= +2025-01-25T14:39:13.896Z [WARN] server-c8afbd38-b716-a4e7-67f7-427b37380fef: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:13.896Z [INFO] server-c8afbd38-b716-a4e7-67f7-427b37380fef: entering candidate state: node="Node at c8afbd38-b716-a4e7-67f7-427b37380fef [Candidate]" term=2 +2025-01-25T14:39:13.896Z [INFO] server-c8afbd38-b716-a4e7-67f7-427b37380fef: election won: term=2 tally=2 +2025-01-25T14:39:13.896Z [INFO] server-c8afbd38-b716-a4e7-67f7-427b37380fef: entering leader state: leader="Node at c8afbd38-b716-a4e7-67f7-427b37380fef [Leader]" +2025-01-25T14:39:13.897Z [INFO] server-c8afbd38-b716-a4e7-67f7-427b37380fef: added peer, starting replication: peer=server-f61f03f6-0a21-9f97-f451-791f2adf0792 +2025-01-25T14:39:13.897Z [INFO] server-c8afbd38-b716-a4e7-67f7-427b37380fef: added peer, starting replication: peer=server-ed370f7b-8d02-fab0-add8-0401ad051891 +2025-01-25T14:39:13.897Z [INFO] server-c8afbd38-b716-a4e7-67f7-427b37380fef: pipelining replication: peer="{Voter server-f61f03f6-0a21-9f97-f451-791f2adf0792 f61f03f6-0a21-9f97-f451-791f2adf0792}" +2025-01-25T14:39:13.897Z [INFO] server-c8afbd38-b716-a4e7-67f7-427b37380fef: pipelining replication: peer="{Voter server-ed370f7b-8d02-fab0-add8-0401ad051891 ed370f7b-8d02-fab0-add8-0401ad051891}" +2025-01-25T14:39:14.902Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:13.897100482 +0000 UTC m=+22.942939800 (1 nodes), 509.928235ms from start of poll, 512.706368ms from cluster start. Timeout at 2025-01-25 14:39:14.902138399 +0000 UTC m=+23.947977716, 1.005037916s after stability +2025-01-25T14:39:14.906Z [INFO] server-c8afbd38-b716-a4e7-67f7-427b37380fef: starting snapshot up to: index=12 +2025-01-25T14:39:14.906Z [INFO] creating new snapshot: path=/tmp/raft2287532075/snapshots/2-12-1737815954906.tmp +2025-01-25T14:39:14.914Z [INFO] server-c8afbd38-b716-a4e7-67f7-427b37380fef: snapshot complete up to: index=12 +2025-01-25T14:39:14.923Z [INFO] server-0b4d8721-abbe-7fde-6320-3a0f2c1357dd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0b4d8721-abbe-7fde-6320-3a0f2c1357dd Address:0b4d8721-abbe-7fde-6320-3a0f2c1357dd} {Suffrage:Voter ID:server-db207482-df02-042e-bc9f-20f1fee7e240 Address:db207482-df02-042e-bc9f-20f1fee7e240} {Suffrage:Voter ID:server-df950d79-258b-ff65-0af4-8c9e8c594672 Address:df950d79-258b-ff65-0af4-8c9e8c594672}]" +2025-01-25T14:39:14.923Z [INFO] server-0b4d8721-abbe-7fde-6320-3a0f2c1357dd: entering follower state: follower="Node at 0b4d8721-abbe-7fde-6320-3a0f2c1357dd [Follower]" leader-address= leader-id= +2025-01-25T14:39:14.924Z [INFO] server-db207482-df02-042e-bc9f-20f1fee7e240: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0b4d8721-abbe-7fde-6320-3a0f2c1357dd Address:0b4d8721-abbe-7fde-6320-3a0f2c1357dd} {Suffrage:Voter ID:server-db207482-df02-042e-bc9f-20f1fee7e240 Address:db207482-df02-042e-bc9f-20f1fee7e240} {Suffrage:Voter ID:server-df950d79-258b-ff65-0af4-8c9e8c594672 Address:df950d79-258b-ff65-0af4-8c9e8c594672}]" +2025-01-25T14:39:14.925Z [INFO] server-db207482-df02-042e-bc9f-20f1fee7e240: entering follower state: follower="Node at db207482-df02-042e-bc9f-20f1fee7e240 [Follower]" leader-address= leader-id= +2025-01-25T14:39:14.925Z [INFO] server-df950d79-258b-ff65-0af4-8c9e8c594672: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0b4d8721-abbe-7fde-6320-3a0f2c1357dd Address:0b4d8721-abbe-7fde-6320-3a0f2c1357dd} {Suffrage:Voter ID:server-db207482-df02-042e-bc9f-20f1fee7e240 Address:db207482-df02-042e-bc9f-20f1fee7e240} {Suffrage:Voter ID:server-df950d79-258b-ff65-0af4-8c9e8c594672 Address:df950d79-258b-ff65-0af4-8c9e8c594672}]" +2025-01-25T14:39:14.925Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:14.925Z [INFO] server-df950d79-258b-ff65-0af4-8c9e8c594672: entering follower state: follower="Node at df950d79-258b-ff65-0af4-8c9e8c594672 [Follower]" leader-address= leader-id= +2025-01-25T14:39:15.576Z [WARN] server-df950d79-258b-ff65-0af4-8c9e8c594672: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:15.577Z [INFO] server-df950d79-258b-ff65-0af4-8c9e8c594672: entering candidate state: node="Node at df950d79-258b-ff65-0af4-8c9e8c594672 [Candidate]" term=2 +2025-01-25T14:39:15.577Z [INFO] server-df950d79-258b-ff65-0af4-8c9e8c594672: election won: term=2 tally=2 +2025-01-25T14:39:15.577Z [INFO] server-df950d79-258b-ff65-0af4-8c9e8c594672: entering leader state: leader="Node at df950d79-258b-ff65-0af4-8c9e8c594672 [Leader]" +2025-01-25T14:39:15.577Z [INFO] server-df950d79-258b-ff65-0af4-8c9e8c594672: added peer, starting replication: peer=server-0b4d8721-abbe-7fde-6320-3a0f2c1357dd +2025-01-25T14:39:15.577Z [INFO] server-df950d79-258b-ff65-0af4-8c9e8c594672: added peer, starting replication: peer=server-db207482-df02-042e-bc9f-20f1fee7e240 +2025-01-25T14:39:15.578Z [INFO] server-df950d79-258b-ff65-0af4-8c9e8c594672: pipelining replication: peer="{Voter server-0b4d8721-abbe-7fde-6320-3a0f2c1357dd 0b4d8721-abbe-7fde-6320-3a0f2c1357dd}" +2025-01-25T14:39:15.578Z [INFO] server-df950d79-258b-ff65-0af4-8c9e8c594672: pipelining replication: peer="{Voter server-db207482-df02-042e-bc9f-20f1fee7e240 db207482-df02-042e-bc9f-20f1fee7e240}" +2025-01-25T14:39:16.583Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:15.577800626 +0000 UTC m=+24.623639943 (1 nodes), 651.810272ms from start of poll, 655.236491ms from cluster start. Timeout at 2025-01-25 14:39:16.582831834 +0000 UTC m=+25.628671152, 1.005031209s after stability +2025-01-25T14:39:16.585Z [INFO] creating new snapshot: path=/tmp/raft2449166165/snapshots/2-13-1737815956585.tmp +2025-01-25T14:39:16.589Z [INFO] server-df950d79-258b-ff65-0af4-8c9e8c594672: copied to local snapshot: bytes=71 +2025-01-25T14:39:16.590Z [INFO] server-df950d79-258b-ff65-0af4-8c9e8c594672: snapshot restore progress: id=2-13-1737815956585 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:16.590Z [INFO] server-df950d79-258b-ff65-0af4-8c9e8c594672: restored user snapshot: index=13 +2025-01-25T14:39:16.591Z [ERROR] server-df950d79-258b-ff65-0af4-8c9e8c594672: failed to get log: index=3 error="log not found" +2025-01-25T14:39:16.591Z [INFO] server-df950d79-258b-ff65-0af4-8c9e8c594672: aborting pipeline replication: peer="{Voter server-db207482-df02-042e-bc9f-20f1fee7e240 db207482-df02-042e-bc9f-20f1fee7e240}" +2025-01-25T14:39:16.591Z [ERROR] server-df950d79-258b-ff65-0af4-8c9e8c594672: failed to get log: index=3 error="log not found" +2025-01-25T14:39:16.591Z [INFO] server-df950d79-258b-ff65-0af4-8c9e8c594672: aborting pipeline replication: peer="{Voter server-0b4d8721-abbe-7fde-6320-3a0f2c1357dd 0b4d8721-abbe-7fde-6320-3a0f2c1357dd}" +2025-01-25T14:39:16.599Z [ERROR] server-df950d79-258b-ff65-0af4-8c9e8c594672: failed to get log: index=3 error="log not found" +2025-01-25T14:39:16.599Z [ERROR] server-df950d79-258b-ff65-0af4-8c9e8c594672: failed to get log: index=3 error="log not found" +2025-01-25T14:39:16.601Z [INFO] creating new snapshot: path=/tmp/raft1454479376/snapshots/2-13-1737815956601.tmp +2025-01-25T14:39:16.601Z [INFO] creating new snapshot: path=/tmp/raft3376880077/snapshots/2-13-1737815956601.tmp +2025-01-25T14:39:16.602Z [INFO] server-0b4d8721-abbe-7fde-6320-3a0f2c1357dd: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:16.603Z [INFO] server-0b4d8721-abbe-7fde-6320-3a0f2c1357dd: copied to local snapshot: bytes=71 +2025-01-25T14:39:16.603Z [INFO] server-db207482-df02-042e-bc9f-20f1fee7e240: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:16.604Z [INFO] server-0b4d8721-abbe-7fde-6320-3a0f2c1357dd: snapshot restore progress: id=2-13-1737815956601 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:16.605Z [INFO] server-0b4d8721-abbe-7fde-6320-3a0f2c1357dd: Installed remote snapshot +2025-01-25T14:39:16.605Z [INFO] server-df950d79-258b-ff65-0af4-8c9e8c594672: pipelining replication: peer="{Voter server-0b4d8721-abbe-7fde-6320-3a0f2c1357dd 0b4d8721-abbe-7fde-6320-3a0f2c1357dd}" +2025-01-25T14:39:16.606Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:39:16.607Z [INFO] server-db207482-df02-042e-bc9f-20f1fee7e240: copied to local snapshot: bytes=71 +2025-01-25T14:39:16.609Z [INFO] server-db207482-df02-042e-bc9f-20f1fee7e240: snapshot restore progress: id=2-13-1737815956601 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:16.610Z [INFO] server-db207482-df02-042e-bc9f-20f1fee7e240: Installed remote snapshot +2025-01-25T14:39:16.611Z [INFO] server-df950d79-258b-ff65-0af4-8c9e8c594672: pipelining replication: peer="{Voter server-db207482-df02-042e-bc9f-20f1fee7e240 db207482-df02-042e-bc9f-20f1fee7e240}" +2025-01-25T14:39:17.611Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:39:16.606314329 +0000 UTC m=+25.652153646 (2 nodes), 7µs from start of poll, 1.683750194s from cluster start. Timeout at 2025-01-25 14:39:17.611364788 +0000 UTC m=+26.657204105, 1.005050459s after stability +2025-01-25T14:39:17.626Z [INFO] server-c8afbd38-b716-a4e7-67f7-427b37380fef: aborting pipeline replication: peer="{Voter server-ed370f7b-8d02-fab0-add8-0401ad051891 ed370f7b-8d02-fab0-add8-0401ad051891}" +2025-01-25T14:39:17.626Z [INFO] server-c8afbd38-b716-a4e7-67f7-427b37380fef: aborting pipeline replication: peer="{Voter server-f61f03f6-0a21-9f97-f451-791f2adf0792 f61f03f6-0a21-9f97-f451-791f2adf0792}" === RUN TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_true -2025-01-25T14:28:32.957Z [INFO] server-79269506-73d6-fa20-351f-cbab4553a4b1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-79269506-73d6-fa20-351f-cbab4553a4b1 Address:79269506-73d6-fa20-351f-cbab4553a4b1} {Suffrage:Voter ID:server-2d05640c-3080-5f64-fa40-3f7a458d415d Address:2d05640c-3080-5f64-fa40-3f7a458d415d} {Suffrage:Voter ID:server-5550ac9f-b445-20c7-7776-71e81e3228c3 Address:5550ac9f-b445-20c7-7776-71e81e3228c3}]" -2025-01-25T14:28:32.957Z [INFO] server-79269506-73d6-fa20-351f-cbab4553a4b1: entering follower state: follower="Node at 79269506-73d6-fa20-351f-cbab4553a4b1 [Follower]" leader-address= leader-id= -2025-01-25T14:28:32.957Z [INFO] server-2d05640c-3080-5f64-fa40-3f7a458d415d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-79269506-73d6-fa20-351f-cbab4553a4b1 Address:79269506-73d6-fa20-351f-cbab4553a4b1} {Suffrage:Voter ID:server-2d05640c-3080-5f64-fa40-3f7a458d415d Address:2d05640c-3080-5f64-fa40-3f7a458d415d} {Suffrage:Voter ID:server-5550ac9f-b445-20c7-7776-71e81e3228c3 Address:5550ac9f-b445-20c7-7776-71e81e3228c3}]" -2025-01-25T14:28:32.957Z [INFO] server-2d05640c-3080-5f64-fa40-3f7a458d415d: entering follower state: follower="Node at 2d05640c-3080-5f64-fa40-3f7a458d415d [Follower]" leader-address= leader-id= -2025-01-25T14:28:32.957Z [INFO] server-5550ac9f-b445-20c7-7776-71e81e3228c3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-79269506-73d6-fa20-351f-cbab4553a4b1 Address:79269506-73d6-fa20-351f-cbab4553a4b1} {Suffrage:Voter ID:server-2d05640c-3080-5f64-fa40-3f7a458d415d Address:2d05640c-3080-5f64-fa40-3f7a458d415d} {Suffrage:Voter ID:server-5550ac9f-b445-20c7-7776-71e81e3228c3 Address:5550ac9f-b445-20c7-7776-71e81e3228c3}]" -2025-01-25T14:28:32.957Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:32.957Z [INFO] server-5550ac9f-b445-20c7-7776-71e81e3228c3: entering follower state: follower="Node at 5550ac9f-b445-20c7-7776-71e81e3228c3 [Follower]" leader-address= leader-id= -2025-01-25T14:28:33.657Z [WARN] server-79269506-73d6-fa20-351f-cbab4553a4b1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:33.657Z [INFO] server-79269506-73d6-fa20-351f-cbab4553a4b1: entering candidate state: node="Node at 79269506-73d6-fa20-351f-cbab4553a4b1 [Candidate]" term=2 -2025-01-25T14:28:33.657Z [INFO] server-79269506-73d6-fa20-351f-cbab4553a4b1: election won: term=2 tally=2 -2025-01-25T14:28:33.657Z [INFO] server-79269506-73d6-fa20-351f-cbab4553a4b1: entering leader state: leader="Node at 79269506-73d6-fa20-351f-cbab4553a4b1 [Leader]" -2025-01-25T14:28:33.657Z [INFO] server-79269506-73d6-fa20-351f-cbab4553a4b1: added peer, starting replication: peer=server-2d05640c-3080-5f64-fa40-3f7a458d415d -2025-01-25T14:28:33.657Z [INFO] server-79269506-73d6-fa20-351f-cbab4553a4b1: added peer, starting replication: peer=server-5550ac9f-b445-20c7-7776-71e81e3228c3 -2025-01-25T14:28:33.658Z [INFO] server-79269506-73d6-fa20-351f-cbab4553a4b1: pipelining replication: peer="{Voter server-5550ac9f-b445-20c7-7776-71e81e3228c3 5550ac9f-b445-20c7-7776-71e81e3228c3}" -2025-01-25T14:28:33.658Z [INFO] server-79269506-73d6-fa20-351f-cbab4553a4b1: pipelining replication: peer="{Voter server-2d05640c-3080-5f64-fa40-3f7a458d415d 2d05640c-3080-5f64-fa40-3f7a458d415d}" -2025-01-25T14:28:34.663Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:33.657867511 +0000 UTC m=+23.512192493 (1 nodes), 700.121886ms from start of poll, 700.824861ms from cluster start. Timeout at 2025-01-25 14:28:34.662876271 +0000 UTC m=+24.517201253, 1.00500876s after stability -2025-01-25T14:28:34.665Z [INFO] server-79269506-73d6-fa20-351f-cbab4553a4b1: starting snapshot up to: index=12 -2025-01-25T14:28:34.665Z [INFO] creating new snapshot: path=/tmp/raft1583965139/snapshots/2-12-1737815314665.tmp -2025-01-25T14:28:34.666Z [INFO] server-79269506-73d6-fa20-351f-cbab4553a4b1: snapshot complete up to: index=12 -2025-01-25T14:28:34.668Z [INFO] server-1e302c46-c6a3-3d89-52e8-d43528206fe0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1e302c46-c6a3-3d89-52e8-d43528206fe0 Address:1e302c46-c6a3-3d89-52e8-d43528206fe0} {Suffrage:Voter ID:server-af0e5466-4af7-3e6e-0d9f-d32f2ffeddb2 Address:af0e5466-4af7-3e6e-0d9f-d32f2ffeddb2} {Suffrage:Voter ID:server-23ad6e03-93c7-f0a6-af79-eec31ce61721 Address:23ad6e03-93c7-f0a6-af79-eec31ce61721}]" -2025-01-25T14:28:34.669Z [INFO] server-af0e5466-4af7-3e6e-0d9f-d32f2ffeddb2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1e302c46-c6a3-3d89-52e8-d43528206fe0 Address:1e302c46-c6a3-3d89-52e8-d43528206fe0} {Suffrage:Voter ID:server-af0e5466-4af7-3e6e-0d9f-d32f2ffeddb2 Address:af0e5466-4af7-3e6e-0d9f-d32f2ffeddb2} {Suffrage:Voter ID:server-23ad6e03-93c7-f0a6-af79-eec31ce61721 Address:23ad6e03-93c7-f0a6-af79-eec31ce61721}]" -2025-01-25T14:28:34.669Z [INFO] server-1e302c46-c6a3-3d89-52e8-d43528206fe0: entering follower state: follower="Node at 1e302c46-c6a3-3d89-52e8-d43528206fe0 [Follower]" leader-address= leader-id= -2025-01-25T14:28:34.669Z [INFO] server-af0e5466-4af7-3e6e-0d9f-d32f2ffeddb2: entering follower state: follower="Node at af0e5466-4af7-3e6e-0d9f-d32f2ffeddb2 [Follower]" leader-address= leader-id= -2025-01-25T14:28:34.669Z [INFO] server-23ad6e03-93c7-f0a6-af79-eec31ce61721: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1e302c46-c6a3-3d89-52e8-d43528206fe0 Address:1e302c46-c6a3-3d89-52e8-d43528206fe0} {Suffrage:Voter ID:server-af0e5466-4af7-3e6e-0d9f-d32f2ffeddb2 Address:af0e5466-4af7-3e6e-0d9f-d32f2ffeddb2} {Suffrage:Voter ID:server-23ad6e03-93c7-f0a6-af79-eec31ce61721 Address:23ad6e03-93c7-f0a6-af79-eec31ce61721}]" -2025-01-25T14:28:34.669Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:34.669Z [INFO] server-23ad6e03-93c7-f0a6-af79-eec31ce61721: entering follower state: follower="Node at 23ad6e03-93c7-f0a6-af79-eec31ce61721 [Follower]" leader-address= leader-id= -2025-01-25T14:28:35.171Z [WARN] server-1e302c46-c6a3-3d89-52e8-d43528206fe0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:35.171Z [INFO] server-1e302c46-c6a3-3d89-52e8-d43528206fe0: entering candidate state: node="Node at 1e302c46-c6a3-3d89-52e8-d43528206fe0 [Candidate]" term=2 -2025-01-25T14:28:35.171Z [INFO] server-1e302c46-c6a3-3d89-52e8-d43528206fe0: election won: term=2 tally=2 -2025-01-25T14:28:35.171Z [INFO] server-1e302c46-c6a3-3d89-52e8-d43528206fe0: entering leader state: leader="Node at 1e302c46-c6a3-3d89-52e8-d43528206fe0 [Leader]" -2025-01-25T14:28:35.171Z [INFO] server-1e302c46-c6a3-3d89-52e8-d43528206fe0: added peer, starting replication: peer=server-af0e5466-4af7-3e6e-0d9f-d32f2ffeddb2 -2025-01-25T14:28:35.172Z [INFO] server-1e302c46-c6a3-3d89-52e8-d43528206fe0: added peer, starting replication: peer=server-23ad6e03-93c7-f0a6-af79-eec31ce61721 -2025-01-25T14:28:35.172Z [INFO] server-1e302c46-c6a3-3d89-52e8-d43528206fe0: pipelining replication: peer="{Voter server-af0e5466-4af7-3e6e-0d9f-d32f2ffeddb2 af0e5466-4af7-3e6e-0d9f-d32f2ffeddb2}" -2025-01-25T14:28:35.172Z [INFO] server-1e302c46-c6a3-3d89-52e8-d43528206fe0: pipelining replication: peer="{Voter server-23ad6e03-93c7-f0a6-af79-eec31ce61721 23ad6e03-93c7-f0a6-af79-eec31ce61721}" -2025-01-25T14:28:36.177Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:35.172051464 +0000 UTC m=+25.026376446 (1 nodes), 502.632713ms from start of poll, 503.361287ms from cluster start. Timeout at 2025-01-25 14:28:36.177059824 +0000 UTC m=+26.031384806, 1.00500836s after stability -2025-01-25T14:28:36.178Z [INFO] creating new snapshot: path=/tmp/raft1132033211/snapshots/2-13-1737815316178.tmp -2025-01-25T14:28:36.179Z [INFO] server-1e302c46-c6a3-3d89-52e8-d43528206fe0: copied to local snapshot: bytes=71 -2025-01-25T14:28:36.180Z [INFO] server-1e302c46-c6a3-3d89-52e8-d43528206fe0: snapshot restore progress: id=2-13-1737815316178 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:36.180Z [INFO] server-1e302c46-c6a3-3d89-52e8-d43528206fe0: removing all old logs from log store -2025-01-25T14:28:36.180Z [INFO] server-1e302c46-c6a3-3d89-52e8-d43528206fe0: compacting logs: from=1 to=2 -2025-01-25T14:28:36.180Z [INFO] server-1e302c46-c6a3-3d89-52e8-d43528206fe0: restored user snapshot: index=13 -2025-01-25T14:28:36.180Z [ERROR] server-1e302c46-c6a3-3d89-52e8-d43528206fe0: failed to get log: index=2 error="log not found" -2025-01-25T14:28:36.180Z [INFO] server-1e302c46-c6a3-3d89-52e8-d43528206fe0: aborting pipeline replication: peer="{Voter server-23ad6e03-93c7-f0a6-af79-eec31ce61721 23ad6e03-93c7-f0a6-af79-eec31ce61721}" -2025-01-25T14:28:36.180Z [ERROR] server-1e302c46-c6a3-3d89-52e8-d43528206fe0: failed to get log: index=2 error="log not found" -2025-01-25T14:28:36.180Z [INFO] server-1e302c46-c6a3-3d89-52e8-d43528206fe0: aborting pipeline replication: peer="{Voter server-af0e5466-4af7-3e6e-0d9f-d32f2ffeddb2 af0e5466-4af7-3e6e-0d9f-d32f2ffeddb2}" -2025-01-25T14:28:36.187Z [ERROR] server-1e302c46-c6a3-3d89-52e8-d43528206fe0: failed to get log: index=2 error="log not found" -2025-01-25T14:28:36.187Z [INFO] creating new snapshot: path=/tmp/raft1872886137/snapshots/2-13-1737815316187.tmp -2025-01-25T14:28:36.188Z [INFO] server-23ad6e03-93c7-f0a6-af79-eec31ce61721: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:36.189Z [INFO] server-23ad6e03-93c7-f0a6-af79-eec31ce61721: copied to local snapshot: bytes=71 -2025-01-25T14:28:36.189Z [ERROR] server-1e302c46-c6a3-3d89-52e8-d43528206fe0: failed to get log: index=2 error="log not found" -2025-01-25T14:28:36.189Z [INFO] server-23ad6e03-93c7-f0a6-af79-eec31ce61721: snapshot restore progress: id=2-13-1737815316187 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:36.189Z [INFO] server-23ad6e03-93c7-f0a6-af79-eec31ce61721: removing all old logs from log store -2025-01-25T14:28:36.189Z [INFO] server-23ad6e03-93c7-f0a6-af79-eec31ce61721: compacting logs: from=1 to=2 -2025-01-25T14:28:36.189Z [INFO] server-23ad6e03-93c7-f0a6-af79-eec31ce61721: Installed remote snapshot -2025-01-25T14:28:36.190Z [INFO] creating new snapshot: path=/tmp/raft1346027855/snapshots/2-13-1737815316190.tmp -2025-01-25T14:28:36.190Z [INFO] server-1e302c46-c6a3-3d89-52e8-d43528206fe0: pipelining replication: peer="{Voter server-23ad6e03-93c7-f0a6-af79-eec31ce61721 23ad6e03-93c7-f0a6-af79-eec31ce61721}" -2025-01-25T14:28:36.190Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:28:36.190Z [INFO] server-af0e5466-4af7-3e6e-0d9f-d32f2ffeddb2: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:36.191Z [INFO] server-af0e5466-4af7-3e6e-0d9f-d32f2ffeddb2: copied to local snapshot: bytes=71 -2025-01-25T14:28:36.191Z [INFO] server-af0e5466-4af7-3e6e-0d9f-d32f2ffeddb2: snapshot restore progress: id=2-13-1737815316190 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:36.191Z [INFO] server-af0e5466-4af7-3e6e-0d9f-d32f2ffeddb2: removing all old logs from log store -2025-01-25T14:28:36.191Z [INFO] server-af0e5466-4af7-3e6e-0d9f-d32f2ffeddb2: compacting logs: from=1 to=2 -2025-01-25T14:28:36.191Z [INFO] server-af0e5466-4af7-3e6e-0d9f-d32f2ffeddb2: Installed remote snapshot -2025-01-25T14:28:36.192Z [INFO] server-1e302c46-c6a3-3d89-52e8-d43528206fe0: pipelining replication: peer="{Voter server-af0e5466-4af7-3e6e-0d9f-d32f2ffeddb2 af0e5466-4af7-3e6e-0d9f-d32f2ffeddb2}" -2025-01-25T14:28:37.196Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:28:36.190309302 +0000 UTC m=+26.044634284 (2 nodes), 1.84µs from start of poll, 1.521619125s from cluster start. Timeout at 2025-01-25 14:28:37.1953529 +0000 UTC m=+27.049677882, 1.005043598s after stability -2025-01-25T14:28:37.208Z [INFO] server-79269506-73d6-fa20-351f-cbab4553a4b1: aborting pipeline replication: peer="{Voter server-5550ac9f-b445-20c7-7776-71e81e3228c3 5550ac9f-b445-20c7-7776-71e81e3228c3}" -2025-01-25T14:28:37.208Z [INFO] server-79269506-73d6-fa20-351f-cbab4553a4b1: aborting pipeline replication: peer="{Voter server-2d05640c-3080-5f64-fa40-3f7a458d415d 2d05640c-3080-5f64-fa40-3f7a458d415d}" +2025-01-25T14:39:17.635Z [INFO] server-52bc4aa7-898c-fced-a792-6a2c59674982: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-52bc4aa7-898c-fced-a792-6a2c59674982 Address:52bc4aa7-898c-fced-a792-6a2c59674982} {Suffrage:Voter ID:server-e155d96c-eeb6-7588-2051-d4aca8799fc1 Address:e155d96c-eeb6-7588-2051-d4aca8799fc1} {Suffrage:Voter ID:server-225b9642-b380-0d4a-7546-73310af152ca Address:225b9642-b380-0d4a-7546-73310af152ca}]" +2025-01-25T14:39:17.636Z [INFO] server-52bc4aa7-898c-fced-a792-6a2c59674982: entering follower state: follower="Node at 52bc4aa7-898c-fced-a792-6a2c59674982 [Follower]" leader-address= leader-id= +2025-01-25T14:39:17.636Z [INFO] server-e155d96c-eeb6-7588-2051-d4aca8799fc1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-52bc4aa7-898c-fced-a792-6a2c59674982 Address:52bc4aa7-898c-fced-a792-6a2c59674982} {Suffrage:Voter ID:server-e155d96c-eeb6-7588-2051-d4aca8799fc1 Address:e155d96c-eeb6-7588-2051-d4aca8799fc1} {Suffrage:Voter ID:server-225b9642-b380-0d4a-7546-73310af152ca Address:225b9642-b380-0d4a-7546-73310af152ca}]" +2025-01-25T14:39:17.636Z [INFO] server-e155d96c-eeb6-7588-2051-d4aca8799fc1: entering follower state: follower="Node at e155d96c-eeb6-7588-2051-d4aca8799fc1 [Follower]" leader-address= leader-id= +2025-01-25T14:39:17.637Z [INFO] server-225b9642-b380-0d4a-7546-73310af152ca: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-52bc4aa7-898c-fced-a792-6a2c59674982 Address:52bc4aa7-898c-fced-a792-6a2c59674982} {Suffrage:Voter ID:server-e155d96c-eeb6-7588-2051-d4aca8799fc1 Address:e155d96c-eeb6-7588-2051-d4aca8799fc1} {Suffrage:Voter ID:server-225b9642-b380-0d4a-7546-73310af152ca Address:225b9642-b380-0d4a-7546-73310af152ca}]" +2025-01-25T14:39:17.637Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:17.637Z [INFO] server-225b9642-b380-0d4a-7546-73310af152ca: entering follower state: follower="Node at 225b9642-b380-0d4a-7546-73310af152ca [Follower]" leader-address= leader-id= +2025-01-25T14:39:18.192Z [WARN] server-e155d96c-eeb6-7588-2051-d4aca8799fc1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:18.193Z [INFO] server-e155d96c-eeb6-7588-2051-d4aca8799fc1: entering candidate state: node="Node at e155d96c-eeb6-7588-2051-d4aca8799fc1 [Candidate]" term=2 +2025-01-25T14:39:18.193Z [INFO] server-e155d96c-eeb6-7588-2051-d4aca8799fc1: election won: term=2 tally=2 +2025-01-25T14:39:18.193Z [INFO] server-e155d96c-eeb6-7588-2051-d4aca8799fc1: entering leader state: leader="Node at e155d96c-eeb6-7588-2051-d4aca8799fc1 [Leader]" +2025-01-25T14:39:18.193Z [INFO] server-e155d96c-eeb6-7588-2051-d4aca8799fc1: added peer, starting replication: peer=server-52bc4aa7-898c-fced-a792-6a2c59674982 +2025-01-25T14:39:18.193Z [INFO] server-e155d96c-eeb6-7588-2051-d4aca8799fc1: added peer, starting replication: peer=server-225b9642-b380-0d4a-7546-73310af152ca +2025-01-25T14:39:18.194Z [INFO] server-e155d96c-eeb6-7588-2051-d4aca8799fc1: pipelining replication: peer="{Voter server-52bc4aa7-898c-fced-a792-6a2c59674982 52bc4aa7-898c-fced-a792-6a2c59674982}" +2025-01-25T14:39:18.194Z [INFO] server-e155d96c-eeb6-7588-2051-d4aca8799fc1: pipelining replication: peer="{Voter server-225b9642-b380-0d4a-7546-73310af152ca 225b9642-b380-0d4a-7546-73310af152ca}" +2025-01-25T14:39:19.199Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:18.193941743 +0000 UTC m=+27.239781060 (1 nodes), 556.128327ms from start of poll, 558.851627ms from cluster start. Timeout at 2025-01-25 14:39:19.198953701 +0000 UTC m=+28.244793019, 1.005011959s after stability +2025-01-25T14:39:19.202Z [INFO] server-e155d96c-eeb6-7588-2051-d4aca8799fc1: starting snapshot up to: index=12 +2025-01-25T14:39:19.203Z [INFO] creating new snapshot: path=/tmp/raft3083935716/snapshots/2-12-1737815959203.tmp +2025-01-25T14:39:19.206Z [INFO] server-e155d96c-eeb6-7588-2051-d4aca8799fc1: snapshot complete up to: index=12 +2025-01-25T14:39:19.214Z [INFO] server-a32d553a-021e-beed-d147-b1ebb99c7456: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a32d553a-021e-beed-d147-b1ebb99c7456 Address:a32d553a-021e-beed-d147-b1ebb99c7456} {Suffrage:Voter ID:server-ffca4dc7-d1b4-1779-1bff-69fd790966fd Address:ffca4dc7-d1b4-1779-1bff-69fd790966fd} {Suffrage:Voter ID:server-25c1f79f-4733-bdc1-aba0-d1d7e4f2f21c Address:25c1f79f-4733-bdc1-aba0-d1d7e4f2f21c}]" +2025-01-25T14:39:19.215Z [INFO] server-ffca4dc7-d1b4-1779-1bff-69fd790966fd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a32d553a-021e-beed-d147-b1ebb99c7456 Address:a32d553a-021e-beed-d147-b1ebb99c7456} {Suffrage:Voter ID:server-ffca4dc7-d1b4-1779-1bff-69fd790966fd Address:ffca4dc7-d1b4-1779-1bff-69fd790966fd} {Suffrage:Voter ID:server-25c1f79f-4733-bdc1-aba0-d1d7e4f2f21c Address:25c1f79f-4733-bdc1-aba0-d1d7e4f2f21c}]" +2025-01-25T14:39:19.217Z [INFO] server-a32d553a-021e-beed-d147-b1ebb99c7456: entering follower state: follower="Node at a32d553a-021e-beed-d147-b1ebb99c7456 [Follower]" leader-address= leader-id= +2025-01-25T14:39:19.217Z [INFO] server-ffca4dc7-d1b4-1779-1bff-69fd790966fd: entering follower state: follower="Node at ffca4dc7-d1b4-1779-1bff-69fd790966fd [Follower]" leader-address= leader-id= +2025-01-25T14:39:19.219Z [INFO] server-25c1f79f-4733-bdc1-aba0-d1d7e4f2f21c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a32d553a-021e-beed-d147-b1ebb99c7456 Address:a32d553a-021e-beed-d147-b1ebb99c7456} {Suffrage:Voter ID:server-ffca4dc7-d1b4-1779-1bff-69fd790966fd Address:ffca4dc7-d1b4-1779-1bff-69fd790966fd} {Suffrage:Voter ID:server-25c1f79f-4733-bdc1-aba0-d1d7e4f2f21c Address:25c1f79f-4733-bdc1-aba0-d1d7e4f2f21c}]" +2025-01-25T14:39:19.220Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:19.220Z [INFO] server-25c1f79f-4733-bdc1-aba0-d1d7e4f2f21c: entering follower state: follower="Node at 25c1f79f-4733-bdc1-aba0-d1d7e4f2f21c [Follower]" leader-address= leader-id= +2025-01-25T14:39:19.830Z [WARN] server-a32d553a-021e-beed-d147-b1ebb99c7456: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:19.831Z [INFO] server-a32d553a-021e-beed-d147-b1ebb99c7456: entering candidate state: node="Node at a32d553a-021e-beed-d147-b1ebb99c7456 [Candidate]" term=2 +2025-01-25T14:39:19.831Z [INFO] server-a32d553a-021e-beed-d147-b1ebb99c7456: election won: term=2 tally=2 +2025-01-25T14:39:19.831Z [INFO] server-a32d553a-021e-beed-d147-b1ebb99c7456: entering leader state: leader="Node at a32d553a-021e-beed-d147-b1ebb99c7456 [Leader]" +2025-01-25T14:39:19.831Z [INFO] server-a32d553a-021e-beed-d147-b1ebb99c7456: added peer, starting replication: peer=server-ffca4dc7-d1b4-1779-1bff-69fd790966fd +2025-01-25T14:39:19.831Z [INFO] server-a32d553a-021e-beed-d147-b1ebb99c7456: added peer, starting replication: peer=server-25c1f79f-4733-bdc1-aba0-d1d7e4f2f21c +2025-01-25T14:39:19.832Z [INFO] server-a32d553a-021e-beed-d147-b1ebb99c7456: pipelining replication: peer="{Voter server-25c1f79f-4733-bdc1-aba0-d1d7e4f2f21c 25c1f79f-4733-bdc1-aba0-d1d7e4f2f21c}" +2025-01-25T14:39:19.832Z [INFO] server-a32d553a-021e-beed-d147-b1ebb99c7456: pipelining replication: peer="{Voter server-ffca4dc7-d1b4-1779-1bff-69fd790966fd ffca4dc7-d1b4-1779-1bff-69fd790966fd}" +2025-01-25T14:39:20.837Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:19.831754178 +0000 UTC m=+28.877593496 (1 nodes), 611.320447ms from start of poll, 619.389387ms from cluster start. Timeout at 2025-01-25 14:39:20.836759719 +0000 UTC m=+29.882599037, 1.005005541s after stability +2025-01-25T14:39:20.838Z [INFO] creating new snapshot: path=/tmp/raft386760217/snapshots/2-13-1737815960838.tmp +2025-01-25T14:39:20.841Z [INFO] server-a32d553a-021e-beed-d147-b1ebb99c7456: copied to local snapshot: bytes=71 +2025-01-25T14:39:20.842Z [INFO] server-a32d553a-021e-beed-d147-b1ebb99c7456: snapshot restore progress: id=2-13-1737815960838 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:20.842Z [INFO] server-a32d553a-021e-beed-d147-b1ebb99c7456: removing all old logs from log store +2025-01-25T14:39:20.843Z [INFO] server-a32d553a-021e-beed-d147-b1ebb99c7456: compacting logs: from=1 to=2 +2025-01-25T14:39:20.843Z [INFO] server-a32d553a-021e-beed-d147-b1ebb99c7456: restored user snapshot: index=13 +2025-01-25T14:39:20.843Z [ERROR] server-a32d553a-021e-beed-d147-b1ebb99c7456: failed to get log: index=2 error="log not found" +2025-01-25T14:39:20.843Z [INFO] server-a32d553a-021e-beed-d147-b1ebb99c7456: aborting pipeline replication: peer="{Voter server-25c1f79f-4733-bdc1-aba0-d1d7e4f2f21c 25c1f79f-4733-bdc1-aba0-d1d7e4f2f21c}" +2025-01-25T14:39:20.843Z [ERROR] server-a32d553a-021e-beed-d147-b1ebb99c7456: failed to get log: index=2 error="log not found" +2025-01-25T14:39:20.843Z [INFO] server-a32d553a-021e-beed-d147-b1ebb99c7456: aborting pipeline replication: peer="{Voter server-ffca4dc7-d1b4-1779-1bff-69fd790966fd ffca4dc7-d1b4-1779-1bff-69fd790966fd}" +2025-01-25T14:39:20.850Z [ERROR] server-a32d553a-021e-beed-d147-b1ebb99c7456: failed to get log: index=2 error="log not found" +2025-01-25T14:39:20.853Z [INFO] creating new snapshot: path=/tmp/raft3837388808/snapshots/2-13-1737815960853.tmp +2025-01-25T14:39:20.854Z [ERROR] server-a32d553a-021e-beed-d147-b1ebb99c7456: failed to get log: index=2 error="log not found" +2025-01-25T14:39:20.855Z [INFO] server-25c1f79f-4733-bdc1-aba0-d1d7e4f2f21c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:20.856Z [INFO] creating new snapshot: path=/tmp/raft3425120002/snapshots/2-13-1737815960856.tmp +2025-01-25T14:39:20.857Z [INFO] server-25c1f79f-4733-bdc1-aba0-d1d7e4f2f21c: copied to local snapshot: bytes=71 +2025-01-25T14:39:20.857Z [INFO] server-ffca4dc7-d1b4-1779-1bff-69fd790966fd: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:20.859Z [INFO] server-25c1f79f-4733-bdc1-aba0-d1d7e4f2f21c: snapshot restore progress: id=2-13-1737815960853 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:20.859Z [INFO] server-25c1f79f-4733-bdc1-aba0-d1d7e4f2f21c: removing all old logs from log store +2025-01-25T14:39:20.859Z [INFO] server-25c1f79f-4733-bdc1-aba0-d1d7e4f2f21c: compacting logs: from=1 to=2 +2025-01-25T14:39:20.859Z [INFO] server-25c1f79f-4733-bdc1-aba0-d1d7e4f2f21c: Installed remote snapshot +2025-01-25T14:39:20.860Z [INFO] server-ffca4dc7-d1b4-1779-1bff-69fd790966fd: copied to local snapshot: bytes=71 +2025-01-25T14:39:20.861Z [INFO] server-a32d553a-021e-beed-d147-b1ebb99c7456: pipelining replication: peer="{Voter server-25c1f79f-4733-bdc1-aba0-d1d7e4f2f21c 25c1f79f-4733-bdc1-aba0-d1d7e4f2f21c}" +2025-01-25T14:39:20.864Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:39:20.867Z [INFO] server-ffca4dc7-d1b4-1779-1bff-69fd790966fd: snapshot restore progress: id=2-13-1737815960856 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:20.868Z [INFO] server-ffca4dc7-d1b4-1779-1bff-69fd790966fd: removing all old logs from log store +2025-01-25T14:39:20.868Z [INFO] server-ffca4dc7-d1b4-1779-1bff-69fd790966fd: compacting logs: from=1 to=2 +2025-01-25T14:39:20.868Z [INFO] server-ffca4dc7-d1b4-1779-1bff-69fd790966fd: Installed remote snapshot +2025-01-25T14:39:20.869Z [INFO] server-a32d553a-021e-beed-d147-b1ebb99c7456: pipelining replication: peer="{Voter server-ffca4dc7-d1b4-1779-1bff-69fd790966fd ffca4dc7-d1b4-1779-1bff-69fd790966fd}" +2025-01-25T14:39:21.871Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:39:20.864930978 +0000 UTC m=+29.910770295 (2 nodes), 9.041µs from start of poll, 1.652566186s from cluster start. Timeout at 2025-01-25 14:39:21.870028103 +0000 UTC m=+30.915867420, 1.005097125s after stability +2025-01-25T14:39:21.890Z [INFO] server-e155d96c-eeb6-7588-2051-d4aca8799fc1: aborting pipeline replication: peer="{Voter server-225b9642-b380-0d4a-7546-73310af152ca 225b9642-b380-0d4a-7546-73310af152ca}" +2025-01-25T14:39:21.890Z [INFO] server-e155d96c-eeb6-7588-2051-d4aca8799fc1: aborting pipeline replication: peer="{Voter server-52bc4aa7-898c-fced-a792-6a2c59674982 52bc4aa7-898c-fced-a792-6a2c59674982}" === RUN TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_false -2025-01-25T14:28:37.217Z [INFO] server-68d235ef-4029-115e-d7fc-8ba5151dcce7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-68d235ef-4029-115e-d7fc-8ba5151dcce7 Address:68d235ef-4029-115e-d7fc-8ba5151dcce7} {Suffrage:Voter ID:server-46503a29-1cb2-fd58-9484-a783c709e240 Address:46503a29-1cb2-fd58-9484-a783c709e240} {Suffrage:Voter ID:server-4fb5308e-41f8-8c84-e79b-dba926c1efdf Address:4fb5308e-41f8-8c84-e79b-dba926c1efdf}]" -2025-01-25T14:28:37.217Z [INFO] server-68d235ef-4029-115e-d7fc-8ba5151dcce7: entering follower state: follower="Node at 68d235ef-4029-115e-d7fc-8ba5151dcce7 [Follower]" leader-address= leader-id= -2025-01-25T14:28:37.217Z [INFO] server-46503a29-1cb2-fd58-9484-a783c709e240: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-68d235ef-4029-115e-d7fc-8ba5151dcce7 Address:68d235ef-4029-115e-d7fc-8ba5151dcce7} {Suffrage:Voter ID:server-46503a29-1cb2-fd58-9484-a783c709e240 Address:46503a29-1cb2-fd58-9484-a783c709e240} {Suffrage:Voter ID:server-4fb5308e-41f8-8c84-e79b-dba926c1efdf Address:4fb5308e-41f8-8c84-e79b-dba926c1efdf}]" -2025-01-25T14:28:37.217Z [INFO] server-46503a29-1cb2-fd58-9484-a783c709e240: entering follower state: follower="Node at 46503a29-1cb2-fd58-9484-a783c709e240 [Follower]" leader-address= leader-id= -2025-01-25T14:28:37.217Z [INFO] server-4fb5308e-41f8-8c84-e79b-dba926c1efdf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-68d235ef-4029-115e-d7fc-8ba5151dcce7 Address:68d235ef-4029-115e-d7fc-8ba5151dcce7} {Suffrage:Voter ID:server-46503a29-1cb2-fd58-9484-a783c709e240 Address:46503a29-1cb2-fd58-9484-a783c709e240} {Suffrage:Voter ID:server-4fb5308e-41f8-8c84-e79b-dba926c1efdf Address:4fb5308e-41f8-8c84-e79b-dba926c1efdf}]" -2025-01-25T14:28:37.217Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:37.217Z [INFO] server-4fb5308e-41f8-8c84-e79b-dba926c1efdf: entering follower state: follower="Node at 4fb5308e-41f8-8c84-e79b-dba926c1efdf [Follower]" leader-address= leader-id= -2025-01-25T14:28:38.072Z [WARN] server-46503a29-1cb2-fd58-9484-a783c709e240: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:38.072Z [INFO] server-46503a29-1cb2-fd58-9484-a783c709e240: entering candidate state: node="Node at 46503a29-1cb2-fd58-9484-a783c709e240 [Candidate]" term=2 -2025-01-25T14:28:38.072Z [INFO] server-46503a29-1cb2-fd58-9484-a783c709e240: election won: term=2 tally=2 -2025-01-25T14:28:38.072Z [INFO] server-46503a29-1cb2-fd58-9484-a783c709e240: entering leader state: leader="Node at 46503a29-1cb2-fd58-9484-a783c709e240 [Leader]" -2025-01-25T14:28:38.072Z [INFO] server-46503a29-1cb2-fd58-9484-a783c709e240: added peer, starting replication: peer=server-68d235ef-4029-115e-d7fc-8ba5151dcce7 -2025-01-25T14:28:38.072Z [INFO] server-46503a29-1cb2-fd58-9484-a783c709e240: added peer, starting replication: peer=server-4fb5308e-41f8-8c84-e79b-dba926c1efdf -2025-01-25T14:28:38.072Z [INFO] server-46503a29-1cb2-fd58-9484-a783c709e240: pipelining replication: peer="{Voter server-68d235ef-4029-115e-d7fc-8ba5151dcce7 68d235ef-4029-115e-d7fc-8ba5151dcce7}" -2025-01-25T14:28:38.073Z [INFO] server-46503a29-1cb2-fd58-9484-a783c709e240: pipelining replication: peer="{Voter server-4fb5308e-41f8-8c84-e79b-dba926c1efdf 4fb5308e-41f8-8c84-e79b-dba926c1efdf}" -2025-01-25T14:28:39.078Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:38.072863039 +0000 UTC m=+27.927188021 (1 nodes), 855.042472ms from start of poll, 855.893721ms from cluster start. Timeout at 2025-01-25 14:28:39.077869359 +0000 UTC m=+28.932194341, 1.00500632s after stability -2025-01-25T14:28:39.079Z [INFO] server-46503a29-1cb2-fd58-9484-a783c709e240: starting snapshot up to: index=12 -2025-01-25T14:28:39.079Z [INFO] creating new snapshot: path=/tmp/raft4141175798/snapshots/2-12-1737815319079.tmp -2025-01-25T14:28:39.080Z [INFO] server-46503a29-1cb2-fd58-9484-a783c709e240: snapshot complete up to: index=12 -2025-01-25T14:28:39.082Z [INFO] creating new snapshot: path=/tmp/raft4141175798/snapshots/2-23-1737815319082.tmp -2025-01-25T14:28:39.083Z [INFO] server-46503a29-1cb2-fd58-9484-a783c709e240: copied to local snapshot: bytes=71 -2025-01-25T14:28:39.083Z [INFO] server-46503a29-1cb2-fd58-9484-a783c709e240: snapshot restore progress: id=2-23-1737815319082 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:39.083Z [INFO] server-46503a29-1cb2-fd58-9484-a783c709e240: restored user snapshot: index=23 -2025-01-25T14:28:39.083Z [ERROR] server-46503a29-1cb2-fd58-9484-a783c709e240: failed to get log: index=23 error="log not found" -2025-01-25T14:28:39.083Z [INFO] server-46503a29-1cb2-fd58-9484-a783c709e240: aborting pipeline replication: peer="{Voter server-4fb5308e-41f8-8c84-e79b-dba926c1efdf 4fb5308e-41f8-8c84-e79b-dba926c1efdf}" -2025-01-25T14:28:39.083Z [ERROR] server-46503a29-1cb2-fd58-9484-a783c709e240: failed to get log: index=23 error="log not found" -2025-01-25T14:28:39.083Z [INFO] server-46503a29-1cb2-fd58-9484-a783c709e240: aborting pipeline replication: peer="{Voter server-68d235ef-4029-115e-d7fc-8ba5151dcce7 68d235ef-4029-115e-d7fc-8ba5151dcce7}" -2025-01-25T14:28:39.092Z [ERROR] server-46503a29-1cb2-fd58-9484-a783c709e240: failed to get log: index=23 error="log not found" -2025-01-25T14:28:39.093Z [INFO] creating new snapshot: path=/tmp/raft2560695296/snapshots/2-23-1737815319093.tmp -2025-01-25T14:28:39.093Z [ERROR] server-46503a29-1cb2-fd58-9484-a783c709e240: failed to get log: index=23 error="log not found" -2025-01-25T14:28:39.094Z [INFO] server-4fb5308e-41f8-8c84-e79b-dba926c1efdf: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:39.094Z [INFO] creating new snapshot: path=/tmp/raft4041373679/snapshots/2-23-1737815319094.tmp -2025-01-25T14:28:39.094Z [INFO] server-4fb5308e-41f8-8c84-e79b-dba926c1efdf: copied to local snapshot: bytes=71 -2025-01-25T14:28:39.094Z [INFO] server-68d235ef-4029-115e-d7fc-8ba5151dcce7: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:39.095Z [INFO] server-4fb5308e-41f8-8c84-e79b-dba926c1efdf: snapshot restore progress: id=2-23-1737815319093 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:39.095Z [INFO] server-68d235ef-4029-115e-d7fc-8ba5151dcce7: copied to local snapshot: bytes=71 -2025-01-25T14:28:39.095Z [INFO] server-4fb5308e-41f8-8c84-e79b-dba926c1efdf: Installed remote snapshot -2025-01-25T14:28:39.095Z [WARN] server-4fb5308e-41f8-8c84-e79b-dba926c1efdf: skipping application of old log: index=22 -2025-01-25T14:28:39.095Z [INFO] server-46503a29-1cb2-fd58-9484-a783c709e240: pipelining replication: peer="{Voter server-4fb5308e-41f8-8c84-e79b-dba926c1efdf 4fb5308e-41f8-8c84-e79b-dba926c1efdf}" -2025-01-25T14:28:39.095Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:28:39.095Z [INFO] server-68d235ef-4029-115e-d7fc-8ba5151dcce7: snapshot restore progress: id=2-23-1737815319094 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:39.095Z [INFO] server-68d235ef-4029-115e-d7fc-8ba5151dcce7: Installed remote snapshot -2025-01-25T14:28:39.096Z [INFO] server-46503a29-1cb2-fd58-9484-a783c709e240: pipelining replication: peer="{Voter server-68d235ef-4029-115e-d7fc-8ba5151dcce7 68d235ef-4029-115e-d7fc-8ba5151dcce7}" -2025-01-25T14:28:40.101Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:28:39.095867306 +0000 UTC m=+28.950192288 (2 nodes), 1.64µs from start of poll, 1.878897988s from cluster start. Timeout at 2025-01-25 14:28:40.100938263 +0000 UTC m=+29.955263205, 1.005070917s after stability -2025-01-25T14:28:40.117Z [INFO] server-46503a29-1cb2-fd58-9484-a783c709e240: aborting pipeline replication: peer="{Voter server-4fb5308e-41f8-8c84-e79b-dba926c1efdf 4fb5308e-41f8-8c84-e79b-dba926c1efdf}" -2025-01-25T14:28:40.117Z [INFO] server-46503a29-1cb2-fd58-9484-a783c709e240: aborting pipeline replication: peer="{Voter server-68d235ef-4029-115e-d7fc-8ba5151dcce7 68d235ef-4029-115e-d7fc-8ba5151dcce7}" +2025-01-25T14:39:21.896Z [INFO] server-0182e887-7376-4b87-8310-de28fed32a1b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0182e887-7376-4b87-8310-de28fed32a1b Address:0182e887-7376-4b87-8310-de28fed32a1b} {Suffrage:Voter ID:server-59a63400-aff4-87d6-1e50-920ac659261c Address:59a63400-aff4-87d6-1e50-920ac659261c} {Suffrage:Voter ID:server-8ef5b44b-8a08-45db-5033-5a871eec8cd9 Address:8ef5b44b-8a08-45db-5033-5a871eec8cd9}]" +2025-01-25T14:39:21.896Z [INFO] server-0182e887-7376-4b87-8310-de28fed32a1b: entering follower state: follower="Node at 0182e887-7376-4b87-8310-de28fed32a1b [Follower]" leader-address= leader-id= +2025-01-25T14:39:21.897Z [INFO] server-59a63400-aff4-87d6-1e50-920ac659261c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0182e887-7376-4b87-8310-de28fed32a1b Address:0182e887-7376-4b87-8310-de28fed32a1b} {Suffrage:Voter ID:server-59a63400-aff4-87d6-1e50-920ac659261c Address:59a63400-aff4-87d6-1e50-920ac659261c} {Suffrage:Voter ID:server-8ef5b44b-8a08-45db-5033-5a871eec8cd9 Address:8ef5b44b-8a08-45db-5033-5a871eec8cd9}]" +2025-01-25T14:39:21.897Z [INFO] server-59a63400-aff4-87d6-1e50-920ac659261c: entering follower state: follower="Node at 59a63400-aff4-87d6-1e50-920ac659261c [Follower]" leader-address= leader-id= +2025-01-25T14:39:21.897Z [INFO] server-8ef5b44b-8a08-45db-5033-5a871eec8cd9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0182e887-7376-4b87-8310-de28fed32a1b Address:0182e887-7376-4b87-8310-de28fed32a1b} {Suffrage:Voter ID:server-59a63400-aff4-87d6-1e50-920ac659261c Address:59a63400-aff4-87d6-1e50-920ac659261c} {Suffrage:Voter ID:server-8ef5b44b-8a08-45db-5033-5a871eec8cd9 Address:8ef5b44b-8a08-45db-5033-5a871eec8cd9}]" +2025-01-25T14:39:21.897Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:21.897Z [INFO] server-8ef5b44b-8a08-45db-5033-5a871eec8cd9: entering follower state: follower="Node at 8ef5b44b-8a08-45db-5033-5a871eec8cd9 [Follower]" leader-address= leader-id= +2025-01-25T14:39:22.603Z [WARN] server-59a63400-aff4-87d6-1e50-920ac659261c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:22.603Z [INFO] server-59a63400-aff4-87d6-1e50-920ac659261c: entering candidate state: node="Node at 59a63400-aff4-87d6-1e50-920ac659261c [Candidate]" term=2 +2025-01-25T14:39:22.604Z [INFO] server-59a63400-aff4-87d6-1e50-920ac659261c: election won: term=2 tally=2 +2025-01-25T14:39:22.604Z [INFO] server-59a63400-aff4-87d6-1e50-920ac659261c: entering leader state: leader="Node at 59a63400-aff4-87d6-1e50-920ac659261c [Leader]" +2025-01-25T14:39:22.604Z [INFO] server-59a63400-aff4-87d6-1e50-920ac659261c: added peer, starting replication: peer=server-0182e887-7376-4b87-8310-de28fed32a1b +2025-01-25T14:39:22.604Z [INFO] server-59a63400-aff4-87d6-1e50-920ac659261c: added peer, starting replication: peer=server-8ef5b44b-8a08-45db-5033-5a871eec8cd9 +2025-01-25T14:39:22.605Z [INFO] server-59a63400-aff4-87d6-1e50-920ac659261c: pipelining replication: peer="{Voter server-0182e887-7376-4b87-8310-de28fed32a1b 0182e887-7376-4b87-8310-de28fed32a1b}" +2025-01-25T14:39:22.605Z [INFO] server-59a63400-aff4-87d6-1e50-920ac659261c: pipelining replication: peer="{Voter server-8ef5b44b-8a08-45db-5033-5a871eec8cd9 8ef5b44b-8a08-45db-5033-5a871eec8cd9}" +2025-01-25T14:39:23.610Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:22.604607626 +0000 UTC m=+31.650446944 (1 nodes), 706.676766ms from start of poll, 708.473438ms from cluster start. Timeout at 2025-01-25 14:39:23.609653126 +0000 UTC m=+32.655492444, 1.0050455s after stability +2025-01-25T14:39:23.614Z [INFO] server-59a63400-aff4-87d6-1e50-920ac659261c: starting snapshot up to: index=12 +2025-01-25T14:39:23.614Z [INFO] creating new snapshot: path=/tmp/raft945284263/snapshots/2-12-1737815963614.tmp +2025-01-25T14:39:23.618Z [INFO] server-59a63400-aff4-87d6-1e50-920ac659261c: snapshot complete up to: index=12 +2025-01-25T14:39:23.630Z [INFO] creating new snapshot: path=/tmp/raft945284263/snapshots/2-23-1737815963630.tmp +2025-01-25T14:39:23.635Z [INFO] server-59a63400-aff4-87d6-1e50-920ac659261c: copied to local snapshot: bytes=71 +2025-01-25T14:39:23.637Z [INFO] server-59a63400-aff4-87d6-1e50-920ac659261c: snapshot restore progress: id=2-23-1737815963630 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:23.637Z [INFO] server-59a63400-aff4-87d6-1e50-920ac659261c: restored user snapshot: index=23 +2025-01-25T14:39:23.637Z [ERROR] server-59a63400-aff4-87d6-1e50-920ac659261c: failed to get log: index=23 error="log not found" +2025-01-25T14:39:23.637Z [ERROR] server-59a63400-aff4-87d6-1e50-920ac659261c: failed to get log: index=23 error="log not found" +2025-01-25T14:39:23.637Z [INFO] server-59a63400-aff4-87d6-1e50-920ac659261c: aborting pipeline replication: peer="{Voter server-8ef5b44b-8a08-45db-5033-5a871eec8cd9 8ef5b44b-8a08-45db-5033-5a871eec8cd9}" +2025-01-25T14:39:23.637Z [INFO] server-59a63400-aff4-87d6-1e50-920ac659261c: aborting pipeline replication: peer="{Voter server-0182e887-7376-4b87-8310-de28fed32a1b 0182e887-7376-4b87-8310-de28fed32a1b}" +2025-01-25T14:39:23.648Z [ERROR] server-59a63400-aff4-87d6-1e50-920ac659261c: failed to get log: index=23 error="log not found" +2025-01-25T14:39:23.648Z [ERROR] server-59a63400-aff4-87d6-1e50-920ac659261c: failed to get log: index=23 error="log not found" +2025-01-25T14:39:23.650Z [INFO] creating new snapshot: path=/tmp/raft1616812099/snapshots/2-23-1737815963650.tmp +2025-01-25T14:39:23.651Z [INFO] creating new snapshot: path=/tmp/raft1732292619/snapshots/2-23-1737815963651.tmp +2025-01-25T14:39:23.652Z [INFO] server-0182e887-7376-4b87-8310-de28fed32a1b: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:23.653Z [INFO] server-8ef5b44b-8a08-45db-5033-5a871eec8cd9: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:23.654Z [INFO] server-0182e887-7376-4b87-8310-de28fed32a1b: copied to local snapshot: bytes=71 +2025-01-25T14:39:23.655Z [INFO] server-8ef5b44b-8a08-45db-5033-5a871eec8cd9: copied to local snapshot: bytes=71 +2025-01-25T14:39:23.656Z [INFO] server-0182e887-7376-4b87-8310-de28fed32a1b: snapshot restore progress: id=2-23-1737815963650 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:23.656Z [INFO] server-0182e887-7376-4b87-8310-de28fed32a1b: Installed remote snapshot +2025-01-25T14:39:23.656Z [WARN] server-0182e887-7376-4b87-8310-de28fed32a1b: skipping application of old log: index=22 +2025-01-25T14:39:23.657Z [INFO] server-8ef5b44b-8a08-45db-5033-5a871eec8cd9: snapshot restore progress: id=2-23-1737815963651 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:23.657Z [INFO] server-8ef5b44b-8a08-45db-5033-5a871eec8cd9: Installed remote snapshot +2025-01-25T14:39:23.657Z [INFO] server-59a63400-aff4-87d6-1e50-920ac659261c: pipelining replication: peer="{Voter server-0182e887-7376-4b87-8310-de28fed32a1b 0182e887-7376-4b87-8310-de28fed32a1b}" +2025-01-25T14:39:23.657Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:39:23.658Z [INFO] server-59a63400-aff4-87d6-1e50-920ac659261c: pipelining replication: peer="{Voter server-8ef5b44b-8a08-45db-5033-5a871eec8cd9 8ef5b44b-8a08-45db-5033-5a871eec8cd9}" +2025-01-25T14:39:24.663Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:39:23.657688692 +0000 UTC m=+32.703528010 (2 nodes), 4.959µs from start of poll, 1.761554504s from cluster start. Timeout at 2025-01-25 14:39:24.662788151 +0000 UTC m=+33.708627468, 1.005099458s after stability +2025-01-25T14:39:24.678Z [INFO] server-59a63400-aff4-87d6-1e50-920ac659261c: aborting pipeline replication: peer="{Voter server-8ef5b44b-8a08-45db-5033-5a871eec8cd9 8ef5b44b-8a08-45db-5033-5a871eec8cd9}" +2025-01-25T14:39:24.678Z [INFO] server-59a63400-aff4-87d6-1e50-920ac659261c: aborting pipeline replication: peer="{Voter server-0182e887-7376-4b87-8310-de28fed32a1b 0182e887-7376-4b87-8310-de28fed32a1b}" === RUN TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_false -2025-01-25T14:28:40.120Z [INFO] server-6f11e85b-dd51-5a6d-b735-f79e3596b986: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6f11e85b-dd51-5a6d-b735-f79e3596b986 Address:6f11e85b-dd51-5a6d-b735-f79e3596b986} {Suffrage:Voter ID:server-e3ff4c4f-117d-646c-624b-67a40bc7b260 Address:e3ff4c4f-117d-646c-624b-67a40bc7b260} {Suffrage:Voter ID:server-27903de3-84e1-954a-c9ab-0f496cc530ed Address:27903de3-84e1-954a-c9ab-0f496cc530ed}]" -2025-01-25T14:28:40.120Z [INFO] server-e3ff4c4f-117d-646c-624b-67a40bc7b260: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6f11e85b-dd51-5a6d-b735-f79e3596b986 Address:6f11e85b-dd51-5a6d-b735-f79e3596b986} {Suffrage:Voter ID:server-e3ff4c4f-117d-646c-624b-67a40bc7b260 Address:e3ff4c4f-117d-646c-624b-67a40bc7b260} {Suffrage:Voter ID:server-27903de3-84e1-954a-c9ab-0f496cc530ed Address:27903de3-84e1-954a-c9ab-0f496cc530ed}]" -2025-01-25T14:28:40.120Z [INFO] server-6f11e85b-dd51-5a6d-b735-f79e3596b986: entering follower state: follower="Node at 6f11e85b-dd51-5a6d-b735-f79e3596b986 [Follower]" leader-address= leader-id= -2025-01-25T14:28:40.121Z [INFO] server-27903de3-84e1-954a-c9ab-0f496cc530ed: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6f11e85b-dd51-5a6d-b735-f79e3596b986 Address:6f11e85b-dd51-5a6d-b735-f79e3596b986} {Suffrage:Voter ID:server-e3ff4c4f-117d-646c-624b-67a40bc7b260 Address:e3ff4c4f-117d-646c-624b-67a40bc7b260} {Suffrage:Voter ID:server-27903de3-84e1-954a-c9ab-0f496cc530ed Address:27903de3-84e1-954a-c9ab-0f496cc530ed}]" -2025-01-25T14:28:40.121Z [INFO] server-e3ff4c4f-117d-646c-624b-67a40bc7b260: entering follower state: follower="Node at e3ff4c4f-117d-646c-624b-67a40bc7b260 [Follower]" leader-address= leader-id= -2025-01-25T14:28:40.121Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:40.121Z [INFO] server-27903de3-84e1-954a-c9ab-0f496cc530ed: entering follower state: follower="Node at 27903de3-84e1-954a-c9ab-0f496cc530ed [Follower]" leader-address= leader-id= -2025-01-25T14:28:40.678Z [WARN] server-e3ff4c4f-117d-646c-624b-67a40bc7b260: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:40.678Z [INFO] server-e3ff4c4f-117d-646c-624b-67a40bc7b260: entering candidate state: node="Node at e3ff4c4f-117d-646c-624b-67a40bc7b260 [Candidate]" term=2 -2025-01-25T14:28:40.678Z [INFO] server-e3ff4c4f-117d-646c-624b-67a40bc7b260: election won: term=2 tally=2 -2025-01-25T14:28:40.678Z [INFO] server-e3ff4c4f-117d-646c-624b-67a40bc7b260: entering leader state: leader="Node at e3ff4c4f-117d-646c-624b-67a40bc7b260 [Leader]" -2025-01-25T14:28:40.678Z [INFO] server-e3ff4c4f-117d-646c-624b-67a40bc7b260: added peer, starting replication: peer=server-6f11e85b-dd51-5a6d-b735-f79e3596b986 -2025-01-25T14:28:40.678Z [INFO] server-e3ff4c4f-117d-646c-624b-67a40bc7b260: added peer, starting replication: peer=server-27903de3-84e1-954a-c9ab-0f496cc530ed -2025-01-25T14:28:40.678Z [INFO] server-e3ff4c4f-117d-646c-624b-67a40bc7b260: pipelining replication: peer="{Voter server-6f11e85b-dd51-5a6d-b735-f79e3596b986 6f11e85b-dd51-5a6d-b735-f79e3596b986}" -2025-01-25T14:28:40.678Z [INFO] server-e3ff4c4f-117d-646c-624b-67a40bc7b260: pipelining replication: peer="{Voter server-27903de3-84e1-954a-c9ab-0f496cc530ed 27903de3-84e1-954a-c9ab-0f496cc530ed}" -2025-01-25T14:28:41.684Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:40.678730267 +0000 UTC m=+30.533055249 (1 nodes), 557.632414ms from start of poll, 558.266191ms from cluster start. Timeout at 2025-01-25 14:28:41.683734867 +0000 UTC m=+31.538059809, 1.00500456s after stability -2025-01-25T14:28:41.685Z [INFO] server-e3ff4c4f-117d-646c-624b-67a40bc7b260: starting snapshot up to: index=12 -2025-01-25T14:28:41.685Z [INFO] creating new snapshot: path=/tmp/raft2218033496/snapshots/2-12-1737815321685.tmp -2025-01-25T14:28:41.686Z [INFO] server-e3ff4c4f-117d-646c-624b-67a40bc7b260: snapshot complete up to: index=12 -2025-01-25T14:28:41.687Z [INFO] creating new snapshot: path=/tmp/raft2218033496/snapshots/2-23-1737815321687.tmp -2025-01-25T14:28:41.688Z [INFO] server-e3ff4c4f-117d-646c-624b-67a40bc7b260: copied to local snapshot: bytes=71 -2025-01-25T14:28:41.689Z [INFO] server-e3ff4c4f-117d-646c-624b-67a40bc7b260: snapshot restore progress: id=2-23-1737815321687 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:41.689Z [INFO] server-e3ff4c4f-117d-646c-624b-67a40bc7b260: removing all old logs from log store -2025-01-25T14:28:41.689Z [INFO] server-e3ff4c4f-117d-646c-624b-67a40bc7b260: compacting logs: from=1 to=22 -2025-01-25T14:28:41.689Z [INFO] server-e3ff4c4f-117d-646c-624b-67a40bc7b260: restored user snapshot: index=23 -2025-01-25T14:28:41.689Z [ERROR] server-e3ff4c4f-117d-646c-624b-67a40bc7b260: failed to get log: index=22 error="log not found" -2025-01-25T14:28:41.689Z [INFO] server-e3ff4c4f-117d-646c-624b-67a40bc7b260: aborting pipeline replication: peer="{Voter server-27903de3-84e1-954a-c9ab-0f496cc530ed 27903de3-84e1-954a-c9ab-0f496cc530ed}" -2025-01-25T14:28:41.689Z [ERROR] server-e3ff4c4f-117d-646c-624b-67a40bc7b260: failed to get log: index=22 error="log not found" -2025-01-25T14:28:41.689Z [INFO] server-e3ff4c4f-117d-646c-624b-67a40bc7b260: aborting pipeline replication: peer="{Voter server-6f11e85b-dd51-5a6d-b735-f79e3596b986 6f11e85b-dd51-5a6d-b735-f79e3596b986}" -2025-01-25T14:28:41.696Z [ERROR] server-e3ff4c4f-117d-646c-624b-67a40bc7b260: failed to get log: index=22 error="log not found" -2025-01-25T14:28:41.697Z [INFO] creating new snapshot: path=/tmp/raft1304655243/snapshots/2-23-1737815321697.tmp -2025-01-25T14:28:41.697Z [ERROR] server-e3ff4c4f-117d-646c-624b-67a40bc7b260: failed to get log: index=22 error="log not found" -2025-01-25T14:28:41.697Z [INFO] server-6f11e85b-dd51-5a6d-b735-f79e3596b986: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:41.698Z [INFO] creating new snapshot: path=/tmp/raft1978040639/snapshots/2-23-1737815321698.tmp -2025-01-25T14:28:41.698Z [INFO] server-6f11e85b-dd51-5a6d-b735-f79e3596b986: copied to local snapshot: bytes=71 -2025-01-25T14:28:41.698Z [INFO] server-27903de3-84e1-954a-c9ab-0f496cc530ed: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:41.699Z [INFO] server-6f11e85b-dd51-5a6d-b735-f79e3596b986: snapshot restore progress: id=2-23-1737815321697 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:41.699Z [INFO] server-6f11e85b-dd51-5a6d-b735-f79e3596b986: removing all old logs from log store -2025-01-25T14:28:41.699Z [INFO] server-6f11e85b-dd51-5a6d-b735-f79e3596b986: compacting logs: from=1 to=22 -2025-01-25T14:28:41.699Z [INFO] server-6f11e85b-dd51-5a6d-b735-f79e3596b986: Installed remote snapshot -2025-01-25T14:28:41.700Z [INFO] server-27903de3-84e1-954a-c9ab-0f496cc530ed: copied to local snapshot: bytes=71 -2025-01-25T14:28:41.700Z [WARN] server-6f11e85b-dd51-5a6d-b735-f79e3596b986: skipping application of old log: index=22 -2025-01-25T14:28:41.700Z [INFO] server-e3ff4c4f-117d-646c-624b-67a40bc7b260: pipelining replication: peer="{Voter server-6f11e85b-dd51-5a6d-b735-f79e3596b986 6f11e85b-dd51-5a6d-b735-f79e3596b986}" -2025-01-25T14:28:41.700Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:28:41.701Z [INFO] server-27903de3-84e1-954a-c9ab-0f496cc530ed: snapshot restore progress: id=2-23-1737815321698 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:41.701Z [INFO] server-27903de3-84e1-954a-c9ab-0f496cc530ed: removing all old logs from log store -2025-01-25T14:28:41.701Z [INFO] server-27903de3-84e1-954a-c9ab-0f496cc530ed: compacting logs: from=1 to=22 -2025-01-25T14:28:41.701Z [INFO] server-27903de3-84e1-954a-c9ab-0f496cc530ed: Installed remote snapshot -2025-01-25T14:28:41.702Z [INFO] server-e3ff4c4f-117d-646c-624b-67a40bc7b260: pipelining replication: peer="{Voter server-27903de3-84e1-954a-c9ab-0f496cc530ed 27903de3-84e1-954a-c9ab-0f496cc530ed}" -2025-01-25T14:28:42.706Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:28:41.701034199 +0000 UTC m=+31.555359141 (2 nodes), 3.2µs from start of poll, 1.580570083s from cluster start. Timeout at 2025-01-25 14:28:42.706082197 +0000 UTC m=+32.560407259, 1.005048118s after stability -2025-01-25T14:28:42.723Z [INFO] server-e3ff4c4f-117d-646c-624b-67a40bc7b260: aborting pipeline replication: peer="{Voter server-27903de3-84e1-954a-c9ab-0f496cc530ed 27903de3-84e1-954a-c9ab-0f496cc530ed}" -2025-01-25T14:28:42.723Z [INFO] server-e3ff4c4f-117d-646c-624b-67a40bc7b260: aborting pipeline replication: peer="{Voter server-6f11e85b-dd51-5a6d-b735-f79e3596b986 6f11e85b-dd51-5a6d-b735-f79e3596b986}" +2025-01-25T14:39:24.687Z [INFO] server-aecdbd3a-e8b0-5e82-6fe6-fdb93b662758: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-aecdbd3a-e8b0-5e82-6fe6-fdb93b662758 Address:aecdbd3a-e8b0-5e82-6fe6-fdb93b662758} {Suffrage:Voter ID:server-8bc981a1-4cb8-2a9c-c730-cec180af62da Address:8bc981a1-4cb8-2a9c-c730-cec180af62da} {Suffrage:Voter ID:server-4cfa20f4-a670-d436-8744-df673388436e Address:4cfa20f4-a670-d436-8744-df673388436e}]" +2025-01-25T14:39:24.687Z [INFO] server-aecdbd3a-e8b0-5e82-6fe6-fdb93b662758: entering follower state: follower="Node at aecdbd3a-e8b0-5e82-6fe6-fdb93b662758 [Follower]" leader-address= leader-id= +2025-01-25T14:39:24.687Z [INFO] server-8bc981a1-4cb8-2a9c-c730-cec180af62da: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-aecdbd3a-e8b0-5e82-6fe6-fdb93b662758 Address:aecdbd3a-e8b0-5e82-6fe6-fdb93b662758} {Suffrage:Voter ID:server-8bc981a1-4cb8-2a9c-c730-cec180af62da Address:8bc981a1-4cb8-2a9c-c730-cec180af62da} {Suffrage:Voter ID:server-4cfa20f4-a670-d436-8744-df673388436e Address:4cfa20f4-a670-d436-8744-df673388436e}]" +2025-01-25T14:39:24.687Z [INFO] server-8bc981a1-4cb8-2a9c-c730-cec180af62da: entering follower state: follower="Node at 8bc981a1-4cb8-2a9c-c730-cec180af62da [Follower]" leader-address= leader-id= +2025-01-25T14:39:24.688Z [INFO] server-4cfa20f4-a670-d436-8744-df673388436e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-aecdbd3a-e8b0-5e82-6fe6-fdb93b662758 Address:aecdbd3a-e8b0-5e82-6fe6-fdb93b662758} {Suffrage:Voter ID:server-8bc981a1-4cb8-2a9c-c730-cec180af62da Address:8bc981a1-4cb8-2a9c-c730-cec180af62da} {Suffrage:Voter ID:server-4cfa20f4-a670-d436-8744-df673388436e Address:4cfa20f4-a670-d436-8744-df673388436e}]" +2025-01-25T14:39:24.688Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:24.688Z [INFO] server-4cfa20f4-a670-d436-8744-df673388436e: entering follower state: follower="Node at 4cfa20f4-a670-d436-8744-df673388436e [Follower]" leader-address= leader-id= +2025-01-25T14:39:25.429Z [WARN] server-4cfa20f4-a670-d436-8744-df673388436e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:25.429Z [INFO] server-4cfa20f4-a670-d436-8744-df673388436e: entering candidate state: node="Node at 4cfa20f4-a670-d436-8744-df673388436e [Candidate]" term=2 +2025-01-25T14:39:25.430Z [INFO] server-4cfa20f4-a670-d436-8744-df673388436e: election won: term=2 tally=2 +2025-01-25T14:39:25.430Z [INFO] server-4cfa20f4-a670-d436-8744-df673388436e: entering leader state: leader="Node at 4cfa20f4-a670-d436-8744-df673388436e [Leader]" +2025-01-25T14:39:25.430Z [INFO] server-4cfa20f4-a670-d436-8744-df673388436e: added peer, starting replication: peer=server-aecdbd3a-e8b0-5e82-6fe6-fdb93b662758 +2025-01-25T14:39:25.430Z [INFO] server-4cfa20f4-a670-d436-8744-df673388436e: added peer, starting replication: peer=server-8bc981a1-4cb8-2a9c-c730-cec180af62da +2025-01-25T14:39:25.430Z [INFO] server-4cfa20f4-a670-d436-8744-df673388436e: pipelining replication: peer="{Voter server-aecdbd3a-e8b0-5e82-6fe6-fdb93b662758 aecdbd3a-e8b0-5e82-6fe6-fdb93b662758}" +2025-01-25T14:39:25.431Z [INFO] server-4cfa20f4-a670-d436-8744-df673388436e: pipelining replication: peer="{Voter server-8bc981a1-4cb8-2a9c-c730-cec180af62da 8bc981a1-4cb8-2a9c-c730-cec180af62da}" +2025-01-25T14:39:26.435Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:25.430298728 +0000 UTC m=+34.476138046 (1 nodes), 741.826202ms from start of poll, 743.675082ms from cluster start. Timeout at 2025-01-25 14:39:26.435350353 +0000 UTC m=+35.481189671, 1.005051625s after stability +2025-01-25T14:39:26.445Z [INFO] server-4cfa20f4-a670-d436-8744-df673388436e: starting snapshot up to: index=12 +2025-01-25T14:39:26.445Z [INFO] creating new snapshot: path=/tmp/raft1346733103/snapshots/2-12-1737815966445.tmp +2025-01-25T14:39:26.448Z [INFO] server-4cfa20f4-a670-d436-8744-df673388436e: snapshot complete up to: index=12 +2025-01-25T14:39:26.452Z [INFO] creating new snapshot: path=/tmp/raft1346733103/snapshots/2-23-1737815966452.tmp +2025-01-25T14:39:26.454Z [INFO] server-4cfa20f4-a670-d436-8744-df673388436e: copied to local snapshot: bytes=71 +2025-01-25T14:39:26.455Z [INFO] server-4cfa20f4-a670-d436-8744-df673388436e: snapshot restore progress: id=2-23-1737815966452 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:26.455Z [INFO] server-4cfa20f4-a670-d436-8744-df673388436e: removing all old logs from log store +2025-01-25T14:39:26.455Z [INFO] server-4cfa20f4-a670-d436-8744-df673388436e: compacting logs: from=1 to=22 +2025-01-25T14:39:26.455Z [INFO] server-4cfa20f4-a670-d436-8744-df673388436e: restored user snapshot: index=23 +2025-01-25T14:39:26.456Z [ERROR] server-4cfa20f4-a670-d436-8744-df673388436e: failed to get log: index=22 error="log not found" +2025-01-25T14:39:26.456Z [INFO] server-4cfa20f4-a670-d436-8744-df673388436e: aborting pipeline replication: peer="{Voter server-8bc981a1-4cb8-2a9c-c730-cec180af62da 8bc981a1-4cb8-2a9c-c730-cec180af62da}" +2025-01-25T14:39:26.456Z [ERROR] server-4cfa20f4-a670-d436-8744-df673388436e: failed to get log: index=22 error="log not found" +2025-01-25T14:39:26.456Z [INFO] server-4cfa20f4-a670-d436-8744-df673388436e: aborting pipeline replication: peer="{Voter server-aecdbd3a-e8b0-5e82-6fe6-fdb93b662758 aecdbd3a-e8b0-5e82-6fe6-fdb93b662758}" +2025-01-25T14:39:26.463Z [ERROR] server-4cfa20f4-a670-d436-8744-df673388436e: failed to get log: index=22 error="log not found" +2025-01-25T14:39:26.465Z [ERROR] server-4cfa20f4-a670-d436-8744-df673388436e: failed to get log: index=22 error="log not found" +2025-01-25T14:39:26.465Z [INFO] creating new snapshot: path=/tmp/raft2101691283/snapshots/2-23-1737815966465.tmp +2025-01-25T14:39:26.467Z [INFO] server-aecdbd3a-e8b0-5e82-6fe6-fdb93b662758: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:26.468Z [INFO] creating new snapshot: path=/tmp/raft2394630325/snapshots/2-23-1737815966468.tmp +2025-01-25T14:39:26.470Z [INFO] server-aecdbd3a-e8b0-5e82-6fe6-fdb93b662758: copied to local snapshot: bytes=71 +2025-01-25T14:39:26.470Z [INFO] server-8bc981a1-4cb8-2a9c-c730-cec180af62da: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:26.471Z [INFO] server-aecdbd3a-e8b0-5e82-6fe6-fdb93b662758: snapshot restore progress: id=2-23-1737815966465 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:26.471Z [INFO] server-aecdbd3a-e8b0-5e82-6fe6-fdb93b662758: removing all old logs from log store +2025-01-25T14:39:26.471Z [INFO] server-aecdbd3a-e8b0-5e82-6fe6-fdb93b662758: compacting logs: from=1 to=22 +2025-01-25T14:39:26.471Z [INFO] server-aecdbd3a-e8b0-5e82-6fe6-fdb93b662758: Installed remote snapshot +2025-01-25T14:39:26.472Z [WARN] server-aecdbd3a-e8b0-5e82-6fe6-fdb93b662758: skipping application of old log: index=22 +2025-01-25T14:39:26.472Z [INFO] server-4cfa20f4-a670-d436-8744-df673388436e: pipelining replication: peer="{Voter server-aecdbd3a-e8b0-5e82-6fe6-fdb93b662758 aecdbd3a-e8b0-5e82-6fe6-fdb93b662758}" +2025-01-25T14:39:26.472Z [INFO] server-8bc981a1-4cb8-2a9c-c730-cec180af62da: copied to local snapshot: bytes=71 +2025-01-25T14:39:26.473Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:39:26.474Z [INFO] server-8bc981a1-4cb8-2a9c-c730-cec180af62da: snapshot restore progress: id=2-23-1737815966468 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:26.474Z [INFO] server-8bc981a1-4cb8-2a9c-c730-cec180af62da: removing all old logs from log store +2025-01-25T14:39:26.474Z [INFO] server-8bc981a1-4cb8-2a9c-c730-cec180af62da: compacting logs: from=1 to=22 +2025-01-25T14:39:26.474Z [INFO] server-8bc981a1-4cb8-2a9c-c730-cec180af62da: Installed remote snapshot +2025-01-25T14:39:26.475Z [INFO] server-4cfa20f4-a670-d436-8744-df673388436e: pipelining replication: peer="{Voter server-8bc981a1-4cb8-2a9c-c730-cec180af62da 8bc981a1-4cb8-2a9c-c730-cec180af62da}" +2025-01-25T14:39:27.479Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:39:26.473070639 +0000 UTC m=+35.518909957 (2 nodes), 4.375µs from start of poll, 1.786446993s from cluster start. Timeout at 2025-01-25 14:39:27.478111472 +0000 UTC m=+36.523950790, 1.005040833s after stability +2025-01-25T14:39:27.494Z [INFO] server-4cfa20f4-a670-d436-8744-df673388436e: aborting pipeline replication: peer="{Voter server-8bc981a1-4cb8-2a9c-c730-cec180af62da 8bc981a1-4cb8-2a9c-c730-cec180af62da}" +2025-01-25T14:39:27.494Z [INFO] server-4cfa20f4-a670-d436-8744-df673388436e: aborting pipeline replication: peer="{Voter server-aecdbd3a-e8b0-5e82-6fe6-fdb93b662758 aecdbd3a-e8b0-5e82-6fe6-fdb93b662758}" === RUN TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_true -2025-01-25T14:28:42.726Z [INFO] server-4a9d605f-d0a6-dbff-0ad6-70e49d73585c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4a9d605f-d0a6-dbff-0ad6-70e49d73585c Address:4a9d605f-d0a6-dbff-0ad6-70e49d73585c} {Suffrage:Voter ID:server-b854593a-a3cd-cb49-f242-46ef9085ebe8 Address:b854593a-a3cd-cb49-f242-46ef9085ebe8} {Suffrage:Voter ID:server-a367d3ea-d4f2-66de-82fd-f95fe55af871 Address:a367d3ea-d4f2-66de-82fd-f95fe55af871}]" -2025-01-25T14:28:42.727Z [INFO] server-b854593a-a3cd-cb49-f242-46ef9085ebe8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4a9d605f-d0a6-dbff-0ad6-70e49d73585c Address:4a9d605f-d0a6-dbff-0ad6-70e49d73585c} {Suffrage:Voter ID:server-b854593a-a3cd-cb49-f242-46ef9085ebe8 Address:b854593a-a3cd-cb49-f242-46ef9085ebe8} {Suffrage:Voter ID:server-a367d3ea-d4f2-66de-82fd-f95fe55af871 Address:a367d3ea-d4f2-66de-82fd-f95fe55af871}]" -2025-01-25T14:28:42.727Z [INFO] server-4a9d605f-d0a6-dbff-0ad6-70e49d73585c: entering follower state: follower="Node at 4a9d605f-d0a6-dbff-0ad6-70e49d73585c [Follower]" leader-address= leader-id= -2025-01-25T14:28:42.727Z [INFO] server-b854593a-a3cd-cb49-f242-46ef9085ebe8: entering follower state: follower="Node at b854593a-a3cd-cb49-f242-46ef9085ebe8 [Follower]" leader-address= leader-id= -2025-01-25T14:28:42.727Z [INFO] server-a367d3ea-d4f2-66de-82fd-f95fe55af871: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4a9d605f-d0a6-dbff-0ad6-70e49d73585c Address:4a9d605f-d0a6-dbff-0ad6-70e49d73585c} {Suffrage:Voter ID:server-b854593a-a3cd-cb49-f242-46ef9085ebe8 Address:b854593a-a3cd-cb49-f242-46ef9085ebe8} {Suffrage:Voter ID:server-a367d3ea-d4f2-66de-82fd-f95fe55af871 Address:a367d3ea-d4f2-66de-82fd-f95fe55af871}]" -2025-01-25T14:28:42.727Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:42.727Z [INFO] server-a367d3ea-d4f2-66de-82fd-f95fe55af871: entering follower state: follower="Node at a367d3ea-d4f2-66de-82fd-f95fe55af871 [Follower]" leader-address= leader-id= -2025-01-25T14:28:43.615Z [WARN] server-4a9d605f-d0a6-dbff-0ad6-70e49d73585c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:43.615Z [INFO] server-4a9d605f-d0a6-dbff-0ad6-70e49d73585c: entering candidate state: node="Node at 4a9d605f-d0a6-dbff-0ad6-70e49d73585c [Candidate]" term=2 -2025-01-25T14:28:43.615Z [INFO] server-4a9d605f-d0a6-dbff-0ad6-70e49d73585c: election won: term=2 tally=2 -2025-01-25T14:28:43.615Z [INFO] server-4a9d605f-d0a6-dbff-0ad6-70e49d73585c: entering leader state: leader="Node at 4a9d605f-d0a6-dbff-0ad6-70e49d73585c [Leader]" -2025-01-25T14:28:43.615Z [INFO] server-4a9d605f-d0a6-dbff-0ad6-70e49d73585c: added peer, starting replication: peer=server-b854593a-a3cd-cb49-f242-46ef9085ebe8 -2025-01-25T14:28:43.615Z [INFO] server-4a9d605f-d0a6-dbff-0ad6-70e49d73585c: added peer, starting replication: peer=server-a367d3ea-d4f2-66de-82fd-f95fe55af871 -2025-01-25T14:28:43.616Z [INFO] server-4a9d605f-d0a6-dbff-0ad6-70e49d73585c: pipelining replication: peer="{Voter server-b854593a-a3cd-cb49-f242-46ef9085ebe8 b854593a-a3cd-cb49-f242-46ef9085ebe8}" -2025-01-25T14:28:43.616Z [INFO] server-4a9d605f-d0a6-dbff-0ad6-70e49d73585c: pipelining replication: peer="{Voter server-a367d3ea-d4f2-66de-82fd-f95fe55af871 a367d3ea-d4f2-66de-82fd-f95fe55af871}" -2025-01-25T14:28:44.621Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:43.61598481 +0000 UTC m=+33.470309752 (1 nodes), 888.42259ms from start of poll, 889.26304ms from cluster start. Timeout at 2025-01-25 14:28:44.62098765 +0000 UTC m=+34.475312752, 1.005003s after stability -2025-01-25T14:28:44.623Z [INFO] server-4a9d605f-d0a6-dbff-0ad6-70e49d73585c: starting snapshot up to: index=12 -2025-01-25T14:28:44.623Z [INFO] creating new snapshot: path=/tmp/raft3100709629/snapshots/2-12-1737815324623.tmp -2025-01-25T14:28:44.624Z [INFO] server-4a9d605f-d0a6-dbff-0ad6-70e49d73585c: snapshot complete up to: index=12 -2025-01-25T14:28:44.627Z [INFO] server-dc031cf1-d77b-1ffc-db35-1bf7abed0a49: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dc031cf1-d77b-1ffc-db35-1bf7abed0a49 Address:dc031cf1-d77b-1ffc-db35-1bf7abed0a49} {Suffrage:Voter ID:server-caf2aa5b-052a-bfa5-7114-4d7ab5885b59 Address:caf2aa5b-052a-bfa5-7114-4d7ab5885b59} {Suffrage:Voter ID:server-14c2fa7d-c8fc-a947-d9ed-6001de72e3f1 Address:14c2fa7d-c8fc-a947-d9ed-6001de72e3f1}]" -2025-01-25T14:28:44.627Z [INFO] server-dc031cf1-d77b-1ffc-db35-1bf7abed0a49: entering follower state: follower="Node at dc031cf1-d77b-1ffc-db35-1bf7abed0a49 [Follower]" leader-address= leader-id= -2025-01-25T14:28:44.627Z [INFO] server-caf2aa5b-052a-bfa5-7114-4d7ab5885b59: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dc031cf1-d77b-1ffc-db35-1bf7abed0a49 Address:dc031cf1-d77b-1ffc-db35-1bf7abed0a49} {Suffrage:Voter ID:server-caf2aa5b-052a-bfa5-7114-4d7ab5885b59 Address:caf2aa5b-052a-bfa5-7114-4d7ab5885b59} {Suffrage:Voter ID:server-14c2fa7d-c8fc-a947-d9ed-6001de72e3f1 Address:14c2fa7d-c8fc-a947-d9ed-6001de72e3f1}]" -2025-01-25T14:28:44.627Z [INFO] server-caf2aa5b-052a-bfa5-7114-4d7ab5885b59: entering follower state: follower="Node at caf2aa5b-052a-bfa5-7114-4d7ab5885b59 [Follower]" leader-address= leader-id= -2025-01-25T14:28:44.627Z [INFO] server-14c2fa7d-c8fc-a947-d9ed-6001de72e3f1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dc031cf1-d77b-1ffc-db35-1bf7abed0a49 Address:dc031cf1-d77b-1ffc-db35-1bf7abed0a49} {Suffrage:Voter ID:server-caf2aa5b-052a-bfa5-7114-4d7ab5885b59 Address:caf2aa5b-052a-bfa5-7114-4d7ab5885b59} {Suffrage:Voter ID:server-14c2fa7d-c8fc-a947-d9ed-6001de72e3f1 Address:14c2fa7d-c8fc-a947-d9ed-6001de72e3f1}]" -2025-01-25T14:28:44.627Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:44.627Z [INFO] server-14c2fa7d-c8fc-a947-d9ed-6001de72e3f1: entering follower state: follower="Node at 14c2fa7d-c8fc-a947-d9ed-6001de72e3f1 [Follower]" leader-address= leader-id= -2025-01-25T14:28:45.131Z [WARN] server-caf2aa5b-052a-bfa5-7114-4d7ab5885b59: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:45.131Z [INFO] server-caf2aa5b-052a-bfa5-7114-4d7ab5885b59: entering candidate state: node="Node at caf2aa5b-052a-bfa5-7114-4d7ab5885b59 [Candidate]" term=2 -2025-01-25T14:28:45.131Z [INFO] server-caf2aa5b-052a-bfa5-7114-4d7ab5885b59: election won: term=2 tally=2 -2025-01-25T14:28:45.131Z [INFO] server-caf2aa5b-052a-bfa5-7114-4d7ab5885b59: entering leader state: leader="Node at caf2aa5b-052a-bfa5-7114-4d7ab5885b59 [Leader]" -2025-01-25T14:28:45.131Z [INFO] server-caf2aa5b-052a-bfa5-7114-4d7ab5885b59: added peer, starting replication: peer=server-dc031cf1-d77b-1ffc-db35-1bf7abed0a49 -2025-01-25T14:28:45.131Z [INFO] server-caf2aa5b-052a-bfa5-7114-4d7ab5885b59: added peer, starting replication: peer=server-14c2fa7d-c8fc-a947-d9ed-6001de72e3f1 -2025-01-25T14:28:45.131Z [INFO] server-caf2aa5b-052a-bfa5-7114-4d7ab5885b59: pipelining replication: peer="{Voter server-dc031cf1-d77b-1ffc-db35-1bf7abed0a49 dc031cf1-d77b-1ffc-db35-1bf7abed0a49}" -2025-01-25T14:28:45.131Z [INFO] server-caf2aa5b-052a-bfa5-7114-4d7ab5885b59: pipelining replication: peer="{Voter server-14c2fa7d-c8fc-a947-d9ed-6001de72e3f1 14c2fa7d-c8fc-a947-d9ed-6001de72e3f1}" -2025-01-25T14:28:46.136Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:45.131540473 +0000 UTC m=+34.985865455 (1 nodes), 503.742313ms from start of poll, 504.39169ms from cluster start. Timeout at 2025-01-25 14:28:46.136551192 +0000 UTC m=+35.990876174, 1.005010719s after stability -2025-01-25T14:28:46.137Z [INFO] creating new snapshot: path=/tmp/raft934595688/snapshots/2-14-1737815326137.tmp -2025-01-25T14:28:46.138Z [INFO] server-caf2aa5b-052a-bfa5-7114-4d7ab5885b59: copied to local snapshot: bytes=71 -2025-01-25T14:28:46.138Z [INFO] server-caf2aa5b-052a-bfa5-7114-4d7ab5885b59: snapshot restore progress: id=2-14-1737815326137 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:46.139Z [INFO] server-caf2aa5b-052a-bfa5-7114-4d7ab5885b59: restored user snapshot: index=14 -2025-01-25T14:28:46.139Z [ERROR] server-caf2aa5b-052a-bfa5-7114-4d7ab5885b59: failed to get log: index=3 error="log not found" -2025-01-25T14:28:46.139Z [INFO] server-caf2aa5b-052a-bfa5-7114-4d7ab5885b59: aborting pipeline replication: peer="{Voter server-14c2fa7d-c8fc-a947-d9ed-6001de72e3f1 14c2fa7d-c8fc-a947-d9ed-6001de72e3f1}" -2025-01-25T14:28:46.139Z [ERROR] server-caf2aa5b-052a-bfa5-7114-4d7ab5885b59: failed to get log: index=3 error="log not found" -2025-01-25T14:28:46.139Z [INFO] server-caf2aa5b-052a-bfa5-7114-4d7ab5885b59: aborting pipeline replication: peer="{Voter server-dc031cf1-d77b-1ffc-db35-1bf7abed0a49 dc031cf1-d77b-1ffc-db35-1bf7abed0a49}" -2025-01-25T14:28:46.139Z [ERROR] server-caf2aa5b-052a-bfa5-7114-4d7ab5885b59: failed to get log: index=3 error="log not found" -2025-01-25T14:28:46.139Z [INFO] creating new snapshot: path=/tmp/raft2186777723/snapshots/2-14-1737815326139.tmp -2025-01-25T14:28:46.140Z [INFO] server-14c2fa7d-c8fc-a947-d9ed-6001de72e3f1: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:46.140Z [INFO] server-14c2fa7d-c8fc-a947-d9ed-6001de72e3f1: copied to local snapshot: bytes=71 -2025-01-25T14:28:46.141Z [INFO] server-14c2fa7d-c8fc-a947-d9ed-6001de72e3f1: snapshot restore progress: id=2-14-1737815326139 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:46.141Z [INFO] server-14c2fa7d-c8fc-a947-d9ed-6001de72e3f1: Installed remote snapshot -2025-01-25T14:28:46.141Z [INFO] server-caf2aa5b-052a-bfa5-7114-4d7ab5885b59: pipelining replication: peer="{Voter server-14c2fa7d-c8fc-a947-d9ed-6001de72e3f1 14c2fa7d-c8fc-a947-d9ed-6001de72e3f1}" -2025-01-25T14:28:46.141Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:28:46.148Z [ERROR] server-caf2aa5b-052a-bfa5-7114-4d7ab5885b59: failed to get log: index=3 error="log not found" -2025-01-25T14:28:46.149Z [INFO] creating new snapshot: path=/tmp/raft1490453568/snapshots/2-14-1737815326149.tmp -2025-01-25T14:28:46.150Z [INFO] server-dc031cf1-d77b-1ffc-db35-1bf7abed0a49: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:46.150Z [INFO] server-dc031cf1-d77b-1ffc-db35-1bf7abed0a49: copied to local snapshot: bytes=71 -2025-01-25T14:28:46.151Z [INFO] server-dc031cf1-d77b-1ffc-db35-1bf7abed0a49: snapshot restore progress: id=2-14-1737815326149 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:46.151Z [INFO] server-dc031cf1-d77b-1ffc-db35-1bf7abed0a49: Installed remote snapshot -2025-01-25T14:28:46.151Z [INFO] server-caf2aa5b-052a-bfa5-7114-4d7ab5885b59: pipelining replication: peer="{Voter server-dc031cf1-d77b-1ffc-db35-1bf7abed0a49 dc031cf1-d77b-1ffc-db35-1bf7abed0a49}" -2025-01-25T14:28:47.147Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:28:46.141900955 +0000 UTC m=+35.996225937 (2 nodes), 1.96µs from start of poll, 1.514752172s from cluster start. Timeout at 2025-01-25 14:28:47.146975033 +0000 UTC m=+37.001299975, 1.005074038s after stability -2025-01-25T14:28:47.158Z [INFO] server-4a9d605f-d0a6-dbff-0ad6-70e49d73585c: aborting pipeline replication: peer="{Voter server-b854593a-a3cd-cb49-f242-46ef9085ebe8 b854593a-a3cd-cb49-f242-46ef9085ebe8}" -2025-01-25T14:28:47.158Z [INFO] server-4a9d605f-d0a6-dbff-0ad6-70e49d73585c: aborting pipeline replication: peer="{Voter server-a367d3ea-d4f2-66de-82fd-f95fe55af871 a367d3ea-d4f2-66de-82fd-f95fe55af871}" +2025-01-25T14:39:27.506Z [INFO] server-231c3d18-6526-0d0f-fd09-a85a42e337a6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-231c3d18-6526-0d0f-fd09-a85a42e337a6 Address:231c3d18-6526-0d0f-fd09-a85a42e337a6} {Suffrage:Voter ID:server-545dad61-6561-6a87-f6b1-1083f778866b Address:545dad61-6561-6a87-f6b1-1083f778866b} {Suffrage:Voter ID:server-b45a77e6-508c-49df-21d1-4bdbc7678a3a Address:b45a77e6-508c-49df-21d1-4bdbc7678a3a}]" +2025-01-25T14:39:27.507Z [INFO] server-231c3d18-6526-0d0f-fd09-a85a42e337a6: entering follower state: follower="Node at 231c3d18-6526-0d0f-fd09-a85a42e337a6 [Follower]" leader-address= leader-id= +2025-01-25T14:39:27.507Z [INFO] server-545dad61-6561-6a87-f6b1-1083f778866b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-231c3d18-6526-0d0f-fd09-a85a42e337a6 Address:231c3d18-6526-0d0f-fd09-a85a42e337a6} {Suffrage:Voter ID:server-545dad61-6561-6a87-f6b1-1083f778866b Address:545dad61-6561-6a87-f6b1-1083f778866b} {Suffrage:Voter ID:server-b45a77e6-508c-49df-21d1-4bdbc7678a3a Address:b45a77e6-508c-49df-21d1-4bdbc7678a3a}]" +2025-01-25T14:39:27.507Z [INFO] server-545dad61-6561-6a87-f6b1-1083f778866b: entering follower state: follower="Node at 545dad61-6561-6a87-f6b1-1083f778866b [Follower]" leader-address= leader-id= +2025-01-25T14:39:27.508Z [INFO] server-b45a77e6-508c-49df-21d1-4bdbc7678a3a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-231c3d18-6526-0d0f-fd09-a85a42e337a6 Address:231c3d18-6526-0d0f-fd09-a85a42e337a6} {Suffrage:Voter ID:server-545dad61-6561-6a87-f6b1-1083f778866b Address:545dad61-6561-6a87-f6b1-1083f778866b} {Suffrage:Voter ID:server-b45a77e6-508c-49df-21d1-4bdbc7678a3a Address:b45a77e6-508c-49df-21d1-4bdbc7678a3a}]" +2025-01-25T14:39:27.508Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:27.508Z [INFO] server-b45a77e6-508c-49df-21d1-4bdbc7678a3a: entering follower state: follower="Node at b45a77e6-508c-49df-21d1-4bdbc7678a3a [Follower]" leader-address= leader-id= +2025-01-25T14:39:28.180Z [WARN] server-b45a77e6-508c-49df-21d1-4bdbc7678a3a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:28.180Z [INFO] server-b45a77e6-508c-49df-21d1-4bdbc7678a3a: entering candidate state: node="Node at b45a77e6-508c-49df-21d1-4bdbc7678a3a [Candidate]" term=2 +2025-01-25T14:39:28.181Z [INFO] server-b45a77e6-508c-49df-21d1-4bdbc7678a3a: election won: term=2 tally=2 +2025-01-25T14:39:28.181Z [INFO] server-b45a77e6-508c-49df-21d1-4bdbc7678a3a: entering leader state: leader="Node at b45a77e6-508c-49df-21d1-4bdbc7678a3a [Leader]" +2025-01-25T14:39:28.181Z [INFO] server-b45a77e6-508c-49df-21d1-4bdbc7678a3a: added peer, starting replication: peer=server-231c3d18-6526-0d0f-fd09-a85a42e337a6 +2025-01-25T14:39:28.181Z [INFO] server-b45a77e6-508c-49df-21d1-4bdbc7678a3a: added peer, starting replication: peer=server-545dad61-6561-6a87-f6b1-1083f778866b +2025-01-25T14:39:28.182Z [INFO] server-b45a77e6-508c-49df-21d1-4bdbc7678a3a: pipelining replication: peer="{Voter server-545dad61-6561-6a87-f6b1-1083f778866b 545dad61-6561-6a87-f6b1-1083f778866b}" +2025-01-25T14:39:28.182Z [INFO] server-b45a77e6-508c-49df-21d1-4bdbc7678a3a: pipelining replication: peer="{Voter server-231c3d18-6526-0d0f-fd09-a85a42e337a6 231c3d18-6526-0d0f-fd09-a85a42e337a6}" +2025-01-25T14:39:29.187Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:28.181817364 +0000 UTC m=+37.227656682 (1 nodes), 673.122585ms from start of poll, 675.822551ms from cluster start. Timeout at 2025-01-25 14:39:29.186835739 +0000 UTC m=+38.232675057, 1.005018375s after stability +2025-01-25T14:39:29.191Z [INFO] server-b45a77e6-508c-49df-21d1-4bdbc7678a3a: starting snapshot up to: index=12 +2025-01-25T14:39:29.191Z [INFO] creating new snapshot: path=/tmp/raft2947311661/snapshots/2-12-1737815969191.tmp +2025-01-25T14:39:29.198Z [INFO] server-b45a77e6-508c-49df-21d1-4bdbc7678a3a: snapshot complete up to: index=12 +2025-01-25T14:39:29.205Z [INFO] server-37629d18-9ef1-6866-2cd8-3e5e2c4caaf6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-37629d18-9ef1-6866-2cd8-3e5e2c4caaf6 Address:37629d18-9ef1-6866-2cd8-3e5e2c4caaf6} {Suffrage:Voter ID:server-45b39ffc-fd8c-83ee-2910-0f25848c553a Address:45b39ffc-fd8c-83ee-2910-0f25848c553a} {Suffrage:Voter ID:server-b5763578-4fb8-4db6-bb5a-20dabfa2cf71 Address:b5763578-4fb8-4db6-bb5a-20dabfa2cf71}]" +2025-01-25T14:39:29.205Z [INFO] server-37629d18-9ef1-6866-2cd8-3e5e2c4caaf6: entering follower state: follower="Node at 37629d18-9ef1-6866-2cd8-3e5e2c4caaf6 [Follower]" leader-address= leader-id= +2025-01-25T14:39:29.205Z [INFO] server-45b39ffc-fd8c-83ee-2910-0f25848c553a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-37629d18-9ef1-6866-2cd8-3e5e2c4caaf6 Address:37629d18-9ef1-6866-2cd8-3e5e2c4caaf6} {Suffrage:Voter ID:server-45b39ffc-fd8c-83ee-2910-0f25848c553a Address:45b39ffc-fd8c-83ee-2910-0f25848c553a} {Suffrage:Voter ID:server-b5763578-4fb8-4db6-bb5a-20dabfa2cf71 Address:b5763578-4fb8-4db6-bb5a-20dabfa2cf71}]" +2025-01-25T14:39:29.206Z [INFO] server-45b39ffc-fd8c-83ee-2910-0f25848c553a: entering follower state: follower="Node at 45b39ffc-fd8c-83ee-2910-0f25848c553a [Follower]" leader-address= leader-id= +2025-01-25T14:39:29.207Z [INFO] server-b5763578-4fb8-4db6-bb5a-20dabfa2cf71: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-37629d18-9ef1-6866-2cd8-3e5e2c4caaf6 Address:37629d18-9ef1-6866-2cd8-3e5e2c4caaf6} {Suffrage:Voter ID:server-45b39ffc-fd8c-83ee-2910-0f25848c553a Address:45b39ffc-fd8c-83ee-2910-0f25848c553a} {Suffrage:Voter ID:server-b5763578-4fb8-4db6-bb5a-20dabfa2cf71 Address:b5763578-4fb8-4db6-bb5a-20dabfa2cf71}]" +2025-01-25T14:39:29.207Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:29.207Z [INFO] server-b5763578-4fb8-4db6-bb5a-20dabfa2cf71: entering follower state: follower="Node at b5763578-4fb8-4db6-bb5a-20dabfa2cf71 [Follower]" leader-address= leader-id= +2025-01-25T14:39:29.873Z [WARN] server-b5763578-4fb8-4db6-bb5a-20dabfa2cf71: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:29.873Z [INFO] server-b5763578-4fb8-4db6-bb5a-20dabfa2cf71: entering candidate state: node="Node at b5763578-4fb8-4db6-bb5a-20dabfa2cf71 [Candidate]" term=2 +2025-01-25T14:39:29.874Z [INFO] server-b5763578-4fb8-4db6-bb5a-20dabfa2cf71: election won: term=2 tally=2 +2025-01-25T14:39:29.874Z [INFO] server-b5763578-4fb8-4db6-bb5a-20dabfa2cf71: entering leader state: leader="Node at b5763578-4fb8-4db6-bb5a-20dabfa2cf71 [Leader]" +2025-01-25T14:39:29.874Z [INFO] server-b5763578-4fb8-4db6-bb5a-20dabfa2cf71: added peer, starting replication: peer=server-37629d18-9ef1-6866-2cd8-3e5e2c4caaf6 +2025-01-25T14:39:29.874Z [INFO] server-b5763578-4fb8-4db6-bb5a-20dabfa2cf71: added peer, starting replication: peer=server-45b39ffc-fd8c-83ee-2910-0f25848c553a +2025-01-25T14:39:29.874Z [INFO] server-b5763578-4fb8-4db6-bb5a-20dabfa2cf71: pipelining replication: peer="{Voter server-37629d18-9ef1-6866-2cd8-3e5e2c4caaf6 37629d18-9ef1-6866-2cd8-3e5e2c4caaf6}" +2025-01-25T14:39:29.875Z [INFO] server-b5763578-4fb8-4db6-bb5a-20dabfa2cf71: pipelining replication: peer="{Voter server-45b39ffc-fd8c-83ee-2910-0f25848c553a 45b39ffc-fd8c-83ee-2910-0f25848c553a}" +2025-01-25T14:39:30.879Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:29.874353708 +0000 UTC m=+38.920192734 (1 nodes), 666.737358ms from start of poll, 669.42974ms from cluster start. Timeout at 2025-01-25 14:39:30.879394541 +0000 UTC m=+39.925234151, 1.005041417s after stability +2025-01-25T14:39:30.881Z [INFO] creating new snapshot: path=/tmp/raft3817988840/snapshots/2-14-1737815970881.tmp +2025-01-25T14:39:30.884Z [INFO] server-b5763578-4fb8-4db6-bb5a-20dabfa2cf71: copied to local snapshot: bytes=71 +2025-01-25T14:39:30.886Z [INFO] server-b5763578-4fb8-4db6-bb5a-20dabfa2cf71: snapshot restore progress: id=2-14-1737815970881 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:30.886Z [INFO] server-b5763578-4fb8-4db6-bb5a-20dabfa2cf71: restored user snapshot: index=14 +2025-01-25T14:39:30.887Z [ERROR] server-b5763578-4fb8-4db6-bb5a-20dabfa2cf71: failed to get log: index=3 error="log not found" +2025-01-25T14:39:30.887Z [ERROR] server-b5763578-4fb8-4db6-bb5a-20dabfa2cf71: failed to get log: index=3 error="log not found" +2025-01-25T14:39:30.887Z [INFO] server-b5763578-4fb8-4db6-bb5a-20dabfa2cf71: aborting pipeline replication: peer="{Voter server-37629d18-9ef1-6866-2cd8-3e5e2c4caaf6 37629d18-9ef1-6866-2cd8-3e5e2c4caaf6}" +2025-01-25T14:39:30.887Z [INFO] server-b5763578-4fb8-4db6-bb5a-20dabfa2cf71: aborting pipeline replication: peer="{Voter server-45b39ffc-fd8c-83ee-2910-0f25848c553a 45b39ffc-fd8c-83ee-2910-0f25848c553a}" +2025-01-25T14:39:30.896Z [ERROR] server-b5763578-4fb8-4db6-bb5a-20dabfa2cf71: failed to get log: index=3 error="log not found" +2025-01-25T14:39:30.896Z [ERROR] server-b5763578-4fb8-4db6-bb5a-20dabfa2cf71: failed to get log: index=3 error="log not found" +2025-01-25T14:39:30.901Z [INFO] creating new snapshot: path=/tmp/raft3622216385/snapshots/2-14-1737815970901.tmp +2025-01-25T14:39:30.901Z [INFO] creating new snapshot: path=/tmp/raft2637224043/snapshots/2-14-1737815970901.tmp +2025-01-25T14:39:30.905Z [INFO] server-45b39ffc-fd8c-83ee-2910-0f25848c553a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:30.905Z [INFO] server-37629d18-9ef1-6866-2cd8-3e5e2c4caaf6: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:30.906Z [INFO] server-45b39ffc-fd8c-83ee-2910-0f25848c553a: copied to local snapshot: bytes=71 +2025-01-25T14:39:30.907Z [INFO] server-45b39ffc-fd8c-83ee-2910-0f25848c553a: snapshot restore progress: id=2-14-1737815970901 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:30.907Z [INFO] server-45b39ffc-fd8c-83ee-2910-0f25848c553a: Installed remote snapshot +2025-01-25T14:39:30.908Z [INFO] server-37629d18-9ef1-6866-2cd8-3e5e2c4caaf6: copied to local snapshot: bytes=71 +2025-01-25T14:39:30.908Z [INFO] server-b5763578-4fb8-4db6-bb5a-20dabfa2cf71: pipelining replication: peer="{Voter server-45b39ffc-fd8c-83ee-2910-0f25848c553a 45b39ffc-fd8c-83ee-2910-0f25848c553a}" +2025-01-25T14:39:30.908Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:39:30.909Z [INFO] server-37629d18-9ef1-6866-2cd8-3e5e2c4caaf6: snapshot restore progress: id=2-14-1737815970901 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:30.910Z [INFO] server-37629d18-9ef1-6866-2cd8-3e5e2c4caaf6: Installed remote snapshot +2025-01-25T14:39:30.910Z [INFO] server-b5763578-4fb8-4db6-bb5a-20dabfa2cf71: pipelining replication: peer="{Voter server-37629d18-9ef1-6866-2cd8-3e5e2c4caaf6 37629d18-9ef1-6866-2cd8-3e5e2c4caaf6}" +2025-01-25T14:39:31.915Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:39:30.90886047 +0000 UTC m=+39.954699788 (2 nodes), 4.667µs from start of poll, 1.703936794s from cluster start. Timeout at 2025-01-25 14:39:31.913929595 +0000 UTC m=+40.959768912, 1.005069124s after stability +2025-01-25T14:39:31.930Z [INFO] server-b45a77e6-508c-49df-21d1-4bdbc7678a3a: aborting pipeline replication: peer="{Voter server-545dad61-6561-6a87-f6b1-1083f778866b 545dad61-6561-6a87-f6b1-1083f778866b}" +2025-01-25T14:39:31.930Z [INFO] server-b45a77e6-508c-49df-21d1-4bdbc7678a3a: aborting pipeline replication: peer="{Voter server-231c3d18-6526-0d0f-fd09-a85a42e337a6 231c3d18-6526-0d0f-fd09-a85a42e337a6}" === RUN TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_true -2025-01-25T14:28:47.161Z [INFO] server-ad3dd07a-2371-1554-368e-4c8027140051: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ad3dd07a-2371-1554-368e-4c8027140051 Address:ad3dd07a-2371-1554-368e-4c8027140051} {Suffrage:Voter ID:server-5c29a50a-eb0a-ac2f-4837-7066db5b1f4f Address:5c29a50a-eb0a-ac2f-4837-7066db5b1f4f} {Suffrage:Voter ID:server-fb073403-dd4f-7a68-d87b-25cf16a08e4a Address:fb073403-dd4f-7a68-d87b-25cf16a08e4a}]" -2025-01-25T14:28:47.161Z [INFO] server-5c29a50a-eb0a-ac2f-4837-7066db5b1f4f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ad3dd07a-2371-1554-368e-4c8027140051 Address:ad3dd07a-2371-1554-368e-4c8027140051} {Suffrage:Voter ID:server-5c29a50a-eb0a-ac2f-4837-7066db5b1f4f Address:5c29a50a-eb0a-ac2f-4837-7066db5b1f4f} {Suffrage:Voter ID:server-fb073403-dd4f-7a68-d87b-25cf16a08e4a Address:fb073403-dd4f-7a68-d87b-25cf16a08e4a}]" -2025-01-25T14:28:47.161Z [INFO] server-ad3dd07a-2371-1554-368e-4c8027140051: entering follower state: follower="Node at ad3dd07a-2371-1554-368e-4c8027140051 [Follower]" leader-address= leader-id= -2025-01-25T14:28:47.161Z [INFO] server-5c29a50a-eb0a-ac2f-4837-7066db5b1f4f: entering follower state: follower="Node at 5c29a50a-eb0a-ac2f-4837-7066db5b1f4f [Follower]" leader-address= leader-id= -2025-01-25T14:28:47.161Z [INFO] server-fb073403-dd4f-7a68-d87b-25cf16a08e4a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ad3dd07a-2371-1554-368e-4c8027140051 Address:ad3dd07a-2371-1554-368e-4c8027140051} {Suffrage:Voter ID:server-5c29a50a-eb0a-ac2f-4837-7066db5b1f4f Address:5c29a50a-eb0a-ac2f-4837-7066db5b1f4f} {Suffrage:Voter ID:server-fb073403-dd4f-7a68-d87b-25cf16a08e4a Address:fb073403-dd4f-7a68-d87b-25cf16a08e4a}]" -2025-01-25T14:28:47.161Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:47.161Z [INFO] server-fb073403-dd4f-7a68-d87b-25cf16a08e4a: entering follower state: follower="Node at fb073403-dd4f-7a68-d87b-25cf16a08e4a [Follower]" leader-address= leader-id= -2025-01-25T14:28:47.749Z [WARN] server-5c29a50a-eb0a-ac2f-4837-7066db5b1f4f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:47.749Z [INFO] server-5c29a50a-eb0a-ac2f-4837-7066db5b1f4f: entering candidate state: node="Node at 5c29a50a-eb0a-ac2f-4837-7066db5b1f4f [Candidate]" term=2 -2025-01-25T14:28:47.750Z [INFO] server-5c29a50a-eb0a-ac2f-4837-7066db5b1f4f: election won: term=2 tally=2 -2025-01-25T14:28:47.750Z [INFO] server-5c29a50a-eb0a-ac2f-4837-7066db5b1f4f: entering leader state: leader="Node at 5c29a50a-eb0a-ac2f-4837-7066db5b1f4f [Leader]" -2025-01-25T14:28:47.750Z [INFO] server-5c29a50a-eb0a-ac2f-4837-7066db5b1f4f: added peer, starting replication: peer=server-ad3dd07a-2371-1554-368e-4c8027140051 -2025-01-25T14:28:47.750Z [INFO] server-5c29a50a-eb0a-ac2f-4837-7066db5b1f4f: added peer, starting replication: peer=server-fb073403-dd4f-7a68-d87b-25cf16a08e4a -2025-01-25T14:28:47.750Z [INFO] server-5c29a50a-eb0a-ac2f-4837-7066db5b1f4f: pipelining replication: peer="{Voter server-fb073403-dd4f-7a68-d87b-25cf16a08e4a fb073403-dd4f-7a68-d87b-25cf16a08e4a}" -2025-01-25T14:28:47.750Z [INFO] server-5c29a50a-eb0a-ac2f-4837-7066db5b1f4f: pipelining replication: peer="{Voter server-ad3dd07a-2371-1554-368e-4c8027140051 ad3dd07a-2371-1554-368e-4c8027140051}" -2025-01-25T14:28:48.756Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:47.750277837 +0000 UTC m=+37.604602819 (1 nodes), 588.30427ms from start of poll, 588.970286ms from cluster start. Timeout at 2025-01-25 14:28:48.755283357 +0000 UTC m=+38.609608339, 1.00500552s after stability -2025-01-25T14:28:48.757Z [INFO] server-5c29a50a-eb0a-ac2f-4837-7066db5b1f4f: starting snapshot up to: index=12 -2025-01-25T14:28:48.757Z [INFO] creating new snapshot: path=/tmp/raft1153494203/snapshots/2-12-1737815328757.tmp -2025-01-25T14:28:48.758Z [INFO] server-5c29a50a-eb0a-ac2f-4837-7066db5b1f4f: snapshot complete up to: index=12 -2025-01-25T14:28:48.760Z [INFO] server-f73d587f-e8b3-a88d-bb5a-b5b1658d93b8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f73d587f-e8b3-a88d-bb5a-b5b1658d93b8 Address:f73d587f-e8b3-a88d-bb5a-b5b1658d93b8} {Suffrage:Voter ID:server-ae0659f9-1b2d-87f0-d6ea-7f158bcbea49 Address:ae0659f9-1b2d-87f0-d6ea-7f158bcbea49} {Suffrage:Voter ID:server-47a2e944-0095-2788-ca16-cfe24d926a1e Address:47a2e944-0095-2788-ca16-cfe24d926a1e}]" -2025-01-25T14:28:48.760Z [INFO] server-f73d587f-e8b3-a88d-bb5a-b5b1658d93b8: entering follower state: follower="Node at f73d587f-e8b3-a88d-bb5a-b5b1658d93b8 [Follower]" leader-address= leader-id= -2025-01-25T14:28:48.760Z [INFO] server-ae0659f9-1b2d-87f0-d6ea-7f158bcbea49: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f73d587f-e8b3-a88d-bb5a-b5b1658d93b8 Address:f73d587f-e8b3-a88d-bb5a-b5b1658d93b8} {Suffrage:Voter ID:server-ae0659f9-1b2d-87f0-d6ea-7f158bcbea49 Address:ae0659f9-1b2d-87f0-d6ea-7f158bcbea49} {Suffrage:Voter ID:server-47a2e944-0095-2788-ca16-cfe24d926a1e Address:47a2e944-0095-2788-ca16-cfe24d926a1e}]" -2025-01-25T14:28:48.761Z [INFO] server-ae0659f9-1b2d-87f0-d6ea-7f158bcbea49: entering follower state: follower="Node at ae0659f9-1b2d-87f0-d6ea-7f158bcbea49 [Follower]" leader-address= leader-id= -2025-01-25T14:28:48.761Z [INFO] server-47a2e944-0095-2788-ca16-cfe24d926a1e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f73d587f-e8b3-a88d-bb5a-b5b1658d93b8 Address:f73d587f-e8b3-a88d-bb5a-b5b1658d93b8} {Suffrage:Voter ID:server-ae0659f9-1b2d-87f0-d6ea-7f158bcbea49 Address:ae0659f9-1b2d-87f0-d6ea-7f158bcbea49} {Suffrage:Voter ID:server-47a2e944-0095-2788-ca16-cfe24d926a1e Address:47a2e944-0095-2788-ca16-cfe24d926a1e}]" -2025-01-25T14:28:48.761Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:48.761Z [INFO] server-47a2e944-0095-2788-ca16-cfe24d926a1e: entering follower state: follower="Node at 47a2e944-0095-2788-ca16-cfe24d926a1e [Follower]" leader-address= leader-id= -2025-01-25T14:28:49.675Z [WARN] server-f73d587f-e8b3-a88d-bb5a-b5b1658d93b8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:49.675Z [INFO] server-f73d587f-e8b3-a88d-bb5a-b5b1658d93b8: entering candidate state: node="Node at f73d587f-e8b3-a88d-bb5a-b5b1658d93b8 [Candidate]" term=2 -2025-01-25T14:28:49.675Z [INFO] server-f73d587f-e8b3-a88d-bb5a-b5b1658d93b8: election won: term=2 tally=2 -2025-01-25T14:28:49.675Z [INFO] server-f73d587f-e8b3-a88d-bb5a-b5b1658d93b8: entering leader state: leader="Node at f73d587f-e8b3-a88d-bb5a-b5b1658d93b8 [Leader]" -2025-01-25T14:28:49.675Z [INFO] server-f73d587f-e8b3-a88d-bb5a-b5b1658d93b8: added peer, starting replication: peer=server-ae0659f9-1b2d-87f0-d6ea-7f158bcbea49 -2025-01-25T14:28:49.675Z [INFO] server-f73d587f-e8b3-a88d-bb5a-b5b1658d93b8: added peer, starting replication: peer=server-47a2e944-0095-2788-ca16-cfe24d926a1e -2025-01-25T14:28:49.676Z [INFO] server-f73d587f-e8b3-a88d-bb5a-b5b1658d93b8: pipelining replication: peer="{Voter server-47a2e944-0095-2788-ca16-cfe24d926a1e 47a2e944-0095-2788-ca16-cfe24d926a1e}" -2025-01-25T14:28:49.676Z [INFO] server-f73d587f-e8b3-a88d-bb5a-b5b1658d93b8: pipelining replication: peer="{Voter server-ae0659f9-1b2d-87f0-d6ea-7f158bcbea49 ae0659f9-1b2d-87f0-d6ea-7f158bcbea49}" -2025-01-25T14:28:50.680Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:49.675808627 +0000 UTC m=+39.530133609 (1 nodes), 914.631327ms from start of poll, 915.268384ms from cluster start. Timeout at 2025-01-25 14:28:50.680810227 +0000 UTC m=+40.535135169, 1.00500156s after stability -2025-01-25T14:28:50.682Z [INFO] creating new snapshot: path=/tmp/raft3488167176/snapshots/2-14-1737815330682.tmp -2025-01-25T14:28:50.684Z [INFO] server-f73d587f-e8b3-a88d-bb5a-b5b1658d93b8: copied to local snapshot: bytes=71 -2025-01-25T14:28:50.685Z [INFO] server-f73d587f-e8b3-a88d-bb5a-b5b1658d93b8: snapshot restore progress: id=2-14-1737815330682 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:50.685Z [INFO] server-f73d587f-e8b3-a88d-bb5a-b5b1658d93b8: removing all old logs from log store -2025-01-25T14:28:50.685Z [INFO] server-f73d587f-e8b3-a88d-bb5a-b5b1658d93b8: compacting logs: from=1 to=2 -2025-01-25T14:28:50.685Z [INFO] server-f73d587f-e8b3-a88d-bb5a-b5b1658d93b8: restored user snapshot: index=14 -2025-01-25T14:28:50.685Z [ERROR] server-f73d587f-e8b3-a88d-bb5a-b5b1658d93b8: failed to get log: index=2 error="log not found" -2025-01-25T14:28:50.685Z [INFO] server-f73d587f-e8b3-a88d-bb5a-b5b1658d93b8: aborting pipeline replication: peer="{Voter server-47a2e944-0095-2788-ca16-cfe24d926a1e 47a2e944-0095-2788-ca16-cfe24d926a1e}" -2025-01-25T14:28:50.685Z [ERROR] server-f73d587f-e8b3-a88d-bb5a-b5b1658d93b8: failed to get log: index=2 error="log not found" -2025-01-25T14:28:50.685Z [INFO] server-f73d587f-e8b3-a88d-bb5a-b5b1658d93b8: aborting pipeline replication: peer="{Voter server-ae0659f9-1b2d-87f0-d6ea-7f158bcbea49 ae0659f9-1b2d-87f0-d6ea-7f158bcbea49}" -2025-01-25T14:28:50.693Z [ERROR] server-f73d587f-e8b3-a88d-bb5a-b5b1658d93b8: failed to get log: index=2 error="log not found" -2025-01-25T14:28:50.693Z [ERROR] server-f73d587f-e8b3-a88d-bb5a-b5b1658d93b8: failed to get log: index=2 error="log not found" -2025-01-25T14:28:50.694Z [INFO] creating new snapshot: path=/tmp/raft739605567/snapshots/2-14-1737815330694.tmp -2025-01-25T14:28:50.694Z [INFO] creating new snapshot: path=/tmp/raft743504276/snapshots/2-14-1737815330694.tmp -2025-01-25T14:28:50.695Z [INFO] server-ae0659f9-1b2d-87f0-d6ea-7f158bcbea49: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:50.695Z [INFO] server-47a2e944-0095-2788-ca16-cfe24d926a1e: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:50.695Z [INFO] server-47a2e944-0095-2788-ca16-cfe24d926a1e: copied to local snapshot: bytes=71 -2025-01-25T14:28:50.696Z [INFO] server-ae0659f9-1b2d-87f0-d6ea-7f158bcbea49: copied to local snapshot: bytes=71 -2025-01-25T14:28:50.696Z [INFO] server-47a2e944-0095-2788-ca16-cfe24d926a1e: snapshot restore progress: id=2-14-1737815330694 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:50.696Z [INFO] server-47a2e944-0095-2788-ca16-cfe24d926a1e: removing all old logs from log store -2025-01-25T14:28:50.696Z [INFO] server-47a2e944-0095-2788-ca16-cfe24d926a1e: compacting logs: from=1 to=2 -2025-01-25T14:28:50.696Z [INFO] server-47a2e944-0095-2788-ca16-cfe24d926a1e: Installed remote snapshot -2025-01-25T14:28:50.696Z [INFO] server-ae0659f9-1b2d-87f0-d6ea-7f158bcbea49: snapshot restore progress: id=2-14-1737815330694 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:50.696Z [INFO] server-ae0659f9-1b2d-87f0-d6ea-7f158bcbea49: removing all old logs from log store -2025-01-25T14:28:50.696Z [INFO] server-ae0659f9-1b2d-87f0-d6ea-7f158bcbea49: compacting logs: from=1 to=2 -2025-01-25T14:28:50.696Z [INFO] server-ae0659f9-1b2d-87f0-d6ea-7f158bcbea49: Installed remote snapshot -2025-01-25T14:28:50.696Z [INFO] server-f73d587f-e8b3-a88d-bb5a-b5b1658d93b8: pipelining replication: peer="{Voter server-47a2e944-0095-2788-ca16-cfe24d926a1e 47a2e944-0095-2788-ca16-cfe24d926a1e}" -2025-01-25T14:28:50.696Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:28:50.696Z [WARN] server-ae0659f9-1b2d-87f0-d6ea-7f158bcbea49: skipping application of old log: index=14 -2025-01-25T14:28:50.697Z [INFO] server-f73d587f-e8b3-a88d-bb5a-b5b1658d93b8: pipelining replication: peer="{Voter server-ae0659f9-1b2d-87f0-d6ea-7f158bcbea49 ae0659f9-1b2d-87f0-d6ea-7f158bcbea49}" -2025-01-25T14:28:51.703Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:28:50.696902722 +0000 UTC m=+40.551227704 (2 nodes), 1.52µs from start of poll, 1.936362479s from cluster start. Timeout at 2025-01-25 14:28:51.702006879 +0000 UTC m=+41.556331821, 1.005104117s after stability -2025-01-25T14:28:51.716Z [INFO] server-5c29a50a-eb0a-ac2f-4837-7066db5b1f4f: aborting pipeline replication: peer="{Voter server-fb073403-dd4f-7a68-d87b-25cf16a08e4a fb073403-dd4f-7a68-d87b-25cf16a08e4a}" -2025-01-25T14:28:51.716Z [INFO] server-5c29a50a-eb0a-ac2f-4837-7066db5b1f4f: aborting pipeline replication: peer="{Voter server-ad3dd07a-2371-1554-368e-4c8027140051 ad3dd07a-2371-1554-368e-4c8027140051}" +2025-01-25T14:39:31.946Z [INFO] server-2faf455e-5c59-18bb-850a-7a1014001070: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2faf455e-5c59-18bb-850a-7a1014001070 Address:2faf455e-5c59-18bb-850a-7a1014001070} {Suffrage:Voter ID:server-da8909c4-ee74-c388-5f38-eaba6b8b90df Address:da8909c4-ee74-c388-5f38-eaba6b8b90df} {Suffrage:Voter ID:server-a3f9abdc-f62b-5e0a-fcfb-76ad301b0a11 Address:a3f9abdc-f62b-5e0a-fcfb-76ad301b0a11}]" +2025-01-25T14:39:31.946Z [INFO] server-2faf455e-5c59-18bb-850a-7a1014001070: entering follower state: follower="Node at 2faf455e-5c59-18bb-850a-7a1014001070 [Follower]" leader-address= leader-id= +2025-01-25T14:39:31.947Z [INFO] server-da8909c4-ee74-c388-5f38-eaba6b8b90df: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2faf455e-5c59-18bb-850a-7a1014001070 Address:2faf455e-5c59-18bb-850a-7a1014001070} {Suffrage:Voter ID:server-da8909c4-ee74-c388-5f38-eaba6b8b90df Address:da8909c4-ee74-c388-5f38-eaba6b8b90df} {Suffrage:Voter ID:server-a3f9abdc-f62b-5e0a-fcfb-76ad301b0a11 Address:a3f9abdc-f62b-5e0a-fcfb-76ad301b0a11}]" +2025-01-25T14:39:31.947Z [INFO] server-da8909c4-ee74-c388-5f38-eaba6b8b90df: entering follower state: follower="Node at da8909c4-ee74-c388-5f38-eaba6b8b90df [Follower]" leader-address= leader-id= +2025-01-25T14:39:31.948Z [INFO] server-a3f9abdc-f62b-5e0a-fcfb-76ad301b0a11: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2faf455e-5c59-18bb-850a-7a1014001070 Address:2faf455e-5c59-18bb-850a-7a1014001070} {Suffrage:Voter ID:server-da8909c4-ee74-c388-5f38-eaba6b8b90df Address:da8909c4-ee74-c388-5f38-eaba6b8b90df} {Suffrage:Voter ID:server-a3f9abdc-f62b-5e0a-fcfb-76ad301b0a11 Address:a3f9abdc-f62b-5e0a-fcfb-76ad301b0a11}]" +2025-01-25T14:39:31.948Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:31.948Z [INFO] server-a3f9abdc-f62b-5e0a-fcfb-76ad301b0a11: entering follower state: follower="Node at a3f9abdc-f62b-5e0a-fcfb-76ad301b0a11 [Follower]" leader-address= leader-id= +2025-01-25T14:39:32.475Z [WARN] server-a3f9abdc-f62b-5e0a-fcfb-76ad301b0a11: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:32.475Z [INFO] server-a3f9abdc-f62b-5e0a-fcfb-76ad301b0a11: entering candidate state: node="Node at a3f9abdc-f62b-5e0a-fcfb-76ad301b0a11 [Candidate]" term=2 +2025-01-25T14:39:32.476Z [INFO] server-a3f9abdc-f62b-5e0a-fcfb-76ad301b0a11: election won: term=2 tally=2 +2025-01-25T14:39:32.476Z [INFO] server-a3f9abdc-f62b-5e0a-fcfb-76ad301b0a11: entering leader state: leader="Node at a3f9abdc-f62b-5e0a-fcfb-76ad301b0a11 [Leader]" +2025-01-25T14:39:32.476Z [INFO] server-a3f9abdc-f62b-5e0a-fcfb-76ad301b0a11: added peer, starting replication: peer=server-2faf455e-5c59-18bb-850a-7a1014001070 +2025-01-25T14:39:32.476Z [INFO] server-a3f9abdc-f62b-5e0a-fcfb-76ad301b0a11: added peer, starting replication: peer=server-da8909c4-ee74-c388-5f38-eaba6b8b90df +2025-01-25T14:39:32.477Z [INFO] server-a3f9abdc-f62b-5e0a-fcfb-76ad301b0a11: pipelining replication: peer="{Voter server-2faf455e-5c59-18bb-850a-7a1014001070 2faf455e-5c59-18bb-850a-7a1014001070}" +2025-01-25T14:39:32.477Z [INFO] server-a3f9abdc-f62b-5e0a-fcfb-76ad301b0a11: pipelining replication: peer="{Voter server-da8909c4-ee74-c388-5f38-eaba6b8b90df da8909c4-ee74-c388-5f38-eaba6b8b90df}" +2025-01-25T14:39:33.482Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:32.476868952 +0000 UTC m=+41.522708270 (1 nodes), 528.128788ms from start of poll, 531.489673ms from cluster start. Timeout at 2025-01-25 14:39:33.481888202 +0000 UTC m=+42.527727519, 1.005019249s after stability +2025-01-25T14:39:33.485Z [INFO] server-a3f9abdc-f62b-5e0a-fcfb-76ad301b0a11: starting snapshot up to: index=12 +2025-01-25T14:39:33.485Z [INFO] creating new snapshot: path=/tmp/raft3487799559/snapshots/2-12-1737815973485.tmp +2025-01-25T14:39:33.489Z [INFO] server-a3f9abdc-f62b-5e0a-fcfb-76ad301b0a11: snapshot complete up to: index=12 +2025-01-25T14:39:33.502Z [INFO] server-daadad67-e7c1-1e23-89af-bec07ca67b98: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-daadad67-e7c1-1e23-89af-bec07ca67b98 Address:daadad67-e7c1-1e23-89af-bec07ca67b98} {Suffrage:Voter ID:server-20c1b02a-9c1c-c49a-7600-a3c706ec42d3 Address:20c1b02a-9c1c-c49a-7600-a3c706ec42d3} {Suffrage:Voter ID:server-1dcd06fa-c2a4-f9dc-a959-d7de3cb8771c Address:1dcd06fa-c2a4-f9dc-a959-d7de3cb8771c}]" +2025-01-25T14:39:33.503Z [INFO] server-daadad67-e7c1-1e23-89af-bec07ca67b98: entering follower state: follower="Node at daadad67-e7c1-1e23-89af-bec07ca67b98 [Follower]" leader-address= leader-id= +2025-01-25T14:39:33.503Z [INFO] server-20c1b02a-9c1c-c49a-7600-a3c706ec42d3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-daadad67-e7c1-1e23-89af-bec07ca67b98 Address:daadad67-e7c1-1e23-89af-bec07ca67b98} {Suffrage:Voter ID:server-20c1b02a-9c1c-c49a-7600-a3c706ec42d3 Address:20c1b02a-9c1c-c49a-7600-a3c706ec42d3} {Suffrage:Voter ID:server-1dcd06fa-c2a4-f9dc-a959-d7de3cb8771c Address:1dcd06fa-c2a4-f9dc-a959-d7de3cb8771c}]" +2025-01-25T14:39:33.503Z [INFO] server-20c1b02a-9c1c-c49a-7600-a3c706ec42d3: entering follower state: follower="Node at 20c1b02a-9c1c-c49a-7600-a3c706ec42d3 [Follower]" leader-address= leader-id= +2025-01-25T14:39:33.503Z [INFO] server-1dcd06fa-c2a4-f9dc-a959-d7de3cb8771c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-daadad67-e7c1-1e23-89af-bec07ca67b98 Address:daadad67-e7c1-1e23-89af-bec07ca67b98} {Suffrage:Voter ID:server-20c1b02a-9c1c-c49a-7600-a3c706ec42d3 Address:20c1b02a-9c1c-c49a-7600-a3c706ec42d3} {Suffrage:Voter ID:server-1dcd06fa-c2a4-f9dc-a959-d7de3cb8771c Address:1dcd06fa-c2a4-f9dc-a959-d7de3cb8771c}]" +2025-01-25T14:39:33.504Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:33.504Z [INFO] server-1dcd06fa-c2a4-f9dc-a959-d7de3cb8771c: entering follower state: follower="Node at 1dcd06fa-c2a4-f9dc-a959-d7de3cb8771c [Follower]" leader-address= leader-id= +2025-01-25T14:39:34.079Z [WARN] server-daadad67-e7c1-1e23-89af-bec07ca67b98: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:34.079Z [INFO] server-daadad67-e7c1-1e23-89af-bec07ca67b98: entering candidate state: node="Node at daadad67-e7c1-1e23-89af-bec07ca67b98 [Candidate]" term=2 +2025-01-25T14:39:34.080Z [INFO] server-daadad67-e7c1-1e23-89af-bec07ca67b98: election won: term=2 tally=2 +2025-01-25T14:39:34.080Z [INFO] server-daadad67-e7c1-1e23-89af-bec07ca67b98: entering leader state: leader="Node at daadad67-e7c1-1e23-89af-bec07ca67b98 [Leader]" +2025-01-25T14:39:34.080Z [INFO] server-daadad67-e7c1-1e23-89af-bec07ca67b98: added peer, starting replication: peer=server-20c1b02a-9c1c-c49a-7600-a3c706ec42d3 +2025-01-25T14:39:34.080Z [INFO] server-daadad67-e7c1-1e23-89af-bec07ca67b98: added peer, starting replication: peer=server-1dcd06fa-c2a4-f9dc-a959-d7de3cb8771c +2025-01-25T14:39:34.081Z [INFO] server-daadad67-e7c1-1e23-89af-bec07ca67b98: pipelining replication: peer="{Voter server-20c1b02a-9c1c-c49a-7600-a3c706ec42d3 20c1b02a-9c1c-c49a-7600-a3c706ec42d3}" +2025-01-25T14:39:34.081Z [INFO] server-daadad67-e7c1-1e23-89af-bec07ca67b98: pipelining replication: peer="{Voter server-1dcd06fa-c2a4-f9dc-a959-d7de3cb8771c 1dcd06fa-c2a4-f9dc-a959-d7de3cb8771c}" +2025-01-25T14:39:35.085Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:34.080448037 +0000 UTC m=+43.126287355 (1 nodes), 576.315594ms from start of poll, 578.247308ms from cluster start. Timeout at 2025-01-25 14:39:35.085464079 +0000 UTC m=+44.131303396, 1.005016041s after stability +2025-01-25T14:39:35.086Z [INFO] creating new snapshot: path=/tmp/raft351743757/snapshots/2-14-1737815975086.tmp +2025-01-25T14:39:35.089Z [INFO] server-daadad67-e7c1-1e23-89af-bec07ca67b98: copied to local snapshot: bytes=71 +2025-01-25T14:39:35.097Z [INFO] server-daadad67-e7c1-1e23-89af-bec07ca67b98: snapshot restore progress: id=2-14-1737815975086 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:35.097Z [INFO] server-daadad67-e7c1-1e23-89af-bec07ca67b98: removing all old logs from log store +2025-01-25T14:39:35.099Z [INFO] server-daadad67-e7c1-1e23-89af-bec07ca67b98: compacting logs: from=1 to=2 +2025-01-25T14:39:35.099Z [INFO] server-daadad67-e7c1-1e23-89af-bec07ca67b98: restored user snapshot: index=14 +2025-01-25T14:39:35.099Z [ERROR] server-daadad67-e7c1-1e23-89af-bec07ca67b98: failed to get log: index=2 error="log not found" +2025-01-25T14:39:35.099Z [ERROR] server-daadad67-e7c1-1e23-89af-bec07ca67b98: failed to get log: index=2 error="log not found" +2025-01-25T14:39:35.099Z [INFO] server-daadad67-e7c1-1e23-89af-bec07ca67b98: aborting pipeline replication: peer="{Voter server-1dcd06fa-c2a4-f9dc-a959-d7de3cb8771c 1dcd06fa-c2a4-f9dc-a959-d7de3cb8771c}" +2025-01-25T14:39:35.100Z [ERROR] server-daadad67-e7c1-1e23-89af-bec07ca67b98: failed to get log: index=2 error="log not found" +2025-01-25T14:39:35.100Z [INFO] server-daadad67-e7c1-1e23-89af-bec07ca67b98: aborting pipeline replication: peer="{Voter server-20c1b02a-9c1c-c49a-7600-a3c706ec42d3 20c1b02a-9c1c-c49a-7600-a3c706ec42d3}" +2025-01-25T14:39:35.102Z [INFO] creating new snapshot: path=/tmp/raft204531820/snapshots/2-14-1737815975102.tmp +2025-01-25T14:39:35.104Z [INFO] server-1dcd06fa-c2a4-f9dc-a959-d7de3cb8771c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:35.106Z [ERROR] server-daadad67-e7c1-1e23-89af-bec07ca67b98: failed to get log: index=2 error="log not found" +2025-01-25T14:39:35.106Z [INFO] server-1dcd06fa-c2a4-f9dc-a959-d7de3cb8771c: copied to local snapshot: bytes=71 +2025-01-25T14:39:35.108Z [INFO] server-1dcd06fa-c2a4-f9dc-a959-d7de3cb8771c: snapshot restore progress: id=2-14-1737815975102 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:35.108Z [INFO] server-1dcd06fa-c2a4-f9dc-a959-d7de3cb8771c: removing all old logs from log store +2025-01-25T14:39:35.108Z [INFO] server-1dcd06fa-c2a4-f9dc-a959-d7de3cb8771c: compacting logs: from=1 to=2 +2025-01-25T14:39:35.108Z [INFO] server-1dcd06fa-c2a4-f9dc-a959-d7de3cb8771c: Installed remote snapshot +2025-01-25T14:39:35.108Z [INFO] creating new snapshot: path=/tmp/raft2525322150/snapshots/2-14-1737815975108.tmp +2025-01-25T14:39:35.109Z [INFO] server-daadad67-e7c1-1e23-89af-bec07ca67b98: pipelining replication: peer="{Voter server-1dcd06fa-c2a4-f9dc-a959-d7de3cb8771c 1dcd06fa-c2a4-f9dc-a959-d7de3cb8771c}" +2025-01-25T14:39:35.109Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:39:35.110Z [INFO] server-20c1b02a-9c1c-c49a-7600-a3c706ec42d3: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:35.113Z [INFO] server-20c1b02a-9c1c-c49a-7600-a3c706ec42d3: copied to local snapshot: bytes=71 +2025-01-25T14:39:35.114Z [INFO] server-20c1b02a-9c1c-c49a-7600-a3c706ec42d3: snapshot restore progress: id=2-14-1737815975108 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:35.114Z [INFO] server-20c1b02a-9c1c-c49a-7600-a3c706ec42d3: removing all old logs from log store +2025-01-25T14:39:35.115Z [INFO] server-20c1b02a-9c1c-c49a-7600-a3c706ec42d3: compacting logs: from=1 to=2 +2025-01-25T14:39:35.115Z [INFO] server-20c1b02a-9c1c-c49a-7600-a3c706ec42d3: Installed remote snapshot +2025-01-25T14:39:35.115Z [INFO] server-daadad67-e7c1-1e23-89af-bec07ca67b98: pipelining replication: peer="{Voter server-20c1b02a-9c1c-c49a-7600-a3c706ec42d3 20c1b02a-9c1c-c49a-7600-a3c706ec42d3}" +2025-01-25T14:39:36.116Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:39:35.109927368 +0000 UTC m=+44.155766686 (2 nodes), 7µs from start of poll, 1.607726639s from cluster start. Timeout at 2025-01-25 14:39:36.115107327 +0000 UTC m=+45.160946645, 1.005179959s after stability +2025-01-25T14:39:36.130Z [INFO] server-a3f9abdc-f62b-5e0a-fcfb-76ad301b0a11: aborting pipeline replication: peer="{Voter server-da8909c4-ee74-c388-5f38-eaba6b8b90df da8909c4-ee74-c388-5f38-eaba6b8b90df}" +2025-01-25T14:39:36.130Z [INFO] server-a3f9abdc-f62b-5e0a-fcfb-76ad301b0a11: aborting pipeline replication: peer="{Voter server-2faf455e-5c59-18bb-850a-7a1014001070 2faf455e-5c59-18bb-850a-7a1014001070}" === RUN TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_false -2025-01-25T14:28:51.718Z [INFO] server-908fbfd9-b228-2fec-6428-268069742cda: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-908fbfd9-b228-2fec-6428-268069742cda Address:908fbfd9-b228-2fec-6428-268069742cda} {Suffrage:Voter ID:server-3dd21ff8-240f-f532-b9f2-77487d6b4b82 Address:3dd21ff8-240f-f532-b9f2-77487d6b4b82} {Suffrage:Voter ID:server-24dd8be8-8623-ee91-0673-2a99a935066a Address:24dd8be8-8623-ee91-0673-2a99a935066a}]" -2025-01-25T14:28:51.718Z [INFO] server-908fbfd9-b228-2fec-6428-268069742cda: entering follower state: follower="Node at 908fbfd9-b228-2fec-6428-268069742cda [Follower]" leader-address= leader-id= -2025-01-25T14:28:51.719Z [INFO] server-3dd21ff8-240f-f532-b9f2-77487d6b4b82: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-908fbfd9-b228-2fec-6428-268069742cda Address:908fbfd9-b228-2fec-6428-268069742cda} {Suffrage:Voter ID:server-3dd21ff8-240f-f532-b9f2-77487d6b4b82 Address:3dd21ff8-240f-f532-b9f2-77487d6b4b82} {Suffrage:Voter ID:server-24dd8be8-8623-ee91-0673-2a99a935066a Address:24dd8be8-8623-ee91-0673-2a99a935066a}]" -2025-01-25T14:28:51.719Z [INFO] server-3dd21ff8-240f-f532-b9f2-77487d6b4b82: entering follower state: follower="Node at 3dd21ff8-240f-f532-b9f2-77487d6b4b82 [Follower]" leader-address= leader-id= -2025-01-25T14:28:51.719Z [INFO] server-24dd8be8-8623-ee91-0673-2a99a935066a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-908fbfd9-b228-2fec-6428-268069742cda Address:908fbfd9-b228-2fec-6428-268069742cda} {Suffrage:Voter ID:server-3dd21ff8-240f-f532-b9f2-77487d6b4b82 Address:3dd21ff8-240f-f532-b9f2-77487d6b4b82} {Suffrage:Voter ID:server-24dd8be8-8623-ee91-0673-2a99a935066a Address:24dd8be8-8623-ee91-0673-2a99a935066a}]" -2025-01-25T14:28:51.719Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:51.719Z [INFO] server-24dd8be8-8623-ee91-0673-2a99a935066a: entering follower state: follower="Node at 24dd8be8-8623-ee91-0673-2a99a935066a [Follower]" leader-address= leader-id= -2025-01-25T14:28:52.221Z [WARN] server-3dd21ff8-240f-f532-b9f2-77487d6b4b82: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:52.221Z [INFO] server-3dd21ff8-240f-f532-b9f2-77487d6b4b82: entering candidate state: node="Node at 3dd21ff8-240f-f532-b9f2-77487d6b4b82 [Candidate]" term=2 -2025-01-25T14:28:52.221Z [INFO] server-3dd21ff8-240f-f532-b9f2-77487d6b4b82: election won: term=2 tally=2 -2025-01-25T14:28:52.221Z [INFO] server-3dd21ff8-240f-f532-b9f2-77487d6b4b82: entering leader state: leader="Node at 3dd21ff8-240f-f532-b9f2-77487d6b4b82 [Leader]" -2025-01-25T14:28:52.221Z [INFO] server-3dd21ff8-240f-f532-b9f2-77487d6b4b82: added peer, starting replication: peer=server-908fbfd9-b228-2fec-6428-268069742cda -2025-01-25T14:28:52.221Z [INFO] server-3dd21ff8-240f-f532-b9f2-77487d6b4b82: added peer, starting replication: peer=server-24dd8be8-8623-ee91-0673-2a99a935066a -2025-01-25T14:28:52.221Z [INFO] server-3dd21ff8-240f-f532-b9f2-77487d6b4b82: pipelining replication: peer="{Voter server-24dd8be8-8623-ee91-0673-2a99a935066a 24dd8be8-8623-ee91-0673-2a99a935066a}" -2025-01-25T14:28:52.221Z [INFO] server-3dd21ff8-240f-f532-b9f2-77487d6b4b82: pipelining replication: peer="{Voter server-908fbfd9-b228-2fec-6428-268069742cda 908fbfd9-b228-2fec-6428-268069742cda}" -2025-01-25T14:28:53.226Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:52.2214919 +0000 UTC m=+42.075816842 (1 nodes), 502.076053ms from start of poll, 502.819506ms from cluster start. Timeout at 2025-01-25 14:28:53.226494379 +0000 UTC m=+43.080819361, 1.005002519s after stability -2025-01-25T14:28:53.227Z [INFO] server-3dd21ff8-240f-f532-b9f2-77487d6b4b82: starting snapshot up to: index=12 -2025-01-25T14:28:53.227Z [INFO] creating new snapshot: path=/tmp/raft1488112060/snapshots/2-12-1737815333227.tmp -2025-01-25T14:28:53.229Z [INFO] server-3dd21ff8-240f-f532-b9f2-77487d6b4b82: snapshot complete up to: index=12 -2025-01-25T14:28:53.230Z [INFO] creating new snapshot: path=/tmp/raft1488112060/snapshots/2-23-1737815333230.tmp -2025-01-25T14:28:53.232Z [INFO] server-3dd21ff8-240f-f532-b9f2-77487d6b4b82: copied to local snapshot: bytes=71 -2025-01-25T14:28:53.232Z [INFO] server-3dd21ff8-240f-f532-b9f2-77487d6b4b82: snapshot restore progress: id=2-23-1737815333230 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:53.232Z [INFO] server-3dd21ff8-240f-f532-b9f2-77487d6b4b82: restored user snapshot: index=23 -2025-01-25T14:28:53.232Z [ERROR] server-3dd21ff8-240f-f532-b9f2-77487d6b4b82: failed to get log: index=23 error="log not found" -2025-01-25T14:28:53.232Z [ERROR] server-3dd21ff8-240f-f532-b9f2-77487d6b4b82: failed to get log: index=23 error="log not found" -2025-01-25T14:28:53.232Z [INFO] server-3dd21ff8-240f-f532-b9f2-77487d6b4b82: aborting pipeline replication: peer="{Voter server-908fbfd9-b228-2fec-6428-268069742cda 908fbfd9-b228-2fec-6428-268069742cda}" -2025-01-25T14:28:53.232Z [INFO] server-3dd21ff8-240f-f532-b9f2-77487d6b4b82: aborting pipeline replication: peer="{Voter server-24dd8be8-8623-ee91-0673-2a99a935066a 24dd8be8-8623-ee91-0673-2a99a935066a}" -2025-01-25T14:28:53.238Z [ERROR] server-3dd21ff8-240f-f532-b9f2-77487d6b4b82: failed to get log: index=23 error="log not found" -2025-01-25T14:28:53.239Z [INFO] creating new snapshot: path=/tmp/raft1728878160/snapshots/2-23-1737815333239.tmp -2025-01-25T14:28:53.240Z [INFO] server-24dd8be8-8623-ee91-0673-2a99a935066a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:53.240Z [INFO] server-24dd8be8-8623-ee91-0673-2a99a935066a: copied to local snapshot: bytes=71 -2025-01-25T14:28:53.241Z [INFO] server-24dd8be8-8623-ee91-0673-2a99a935066a: snapshot restore progress: id=2-23-1737815333239 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:53.241Z [INFO] server-24dd8be8-8623-ee91-0673-2a99a935066a: Installed remote snapshot -2025-01-25T14:28:53.241Z [WARN] server-24dd8be8-8623-ee91-0673-2a99a935066a: skipping application of old log: index=22 -2025-01-25T14:28:53.241Z [INFO] server-3dd21ff8-240f-f532-b9f2-77487d6b4b82: pipelining replication: peer="{Voter server-24dd8be8-8623-ee91-0673-2a99a935066a 24dd8be8-8623-ee91-0673-2a99a935066a}" -2025-01-25T14:28:53.241Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:28:53.243Z [ERROR] server-3dd21ff8-240f-f532-b9f2-77487d6b4b82: failed to get log: index=23 error="log not found" -2025-01-25T14:28:53.244Z [INFO] creating new snapshot: path=/tmp/raft3682433092/snapshots/2-23-1737815333244.tmp -2025-01-25T14:28:53.244Z [INFO] server-908fbfd9-b228-2fec-6428-268069742cda: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:53.245Z [INFO] server-908fbfd9-b228-2fec-6428-268069742cda: copied to local snapshot: bytes=71 -2025-01-25T14:28:53.245Z [INFO] server-908fbfd9-b228-2fec-6428-268069742cda: snapshot restore progress: id=2-23-1737815333244 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:53.245Z [INFO] server-908fbfd9-b228-2fec-6428-268069742cda: Installed remote snapshot -2025-01-25T14:28:53.246Z [INFO] server-3dd21ff8-240f-f532-b9f2-77487d6b4b82: pipelining replication: peer="{Voter server-908fbfd9-b228-2fec-6428-268069742cda 908fbfd9-b228-2fec-6428-268069742cda}" -2025-01-25T14:28:54.247Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:28:53.241958618 +0000 UTC m=+43.096283600 (2 nodes), 1.56µs from start of poll, 1.523286264s from cluster start. Timeout at 2025-01-25 14:28:54.247063974 +0000 UTC m=+44.101388996, 1.005105396s after stability -2025-01-25T14:28:54.263Z [INFO] server-3dd21ff8-240f-f532-b9f2-77487d6b4b82: aborting pipeline replication: peer="{Voter server-24dd8be8-8623-ee91-0673-2a99a935066a 24dd8be8-8623-ee91-0673-2a99a935066a}" -2025-01-25T14:28:54.263Z [INFO] server-3dd21ff8-240f-f532-b9f2-77487d6b4b82: aborting pipeline replication: peer="{Voter server-908fbfd9-b228-2fec-6428-268069742cda 908fbfd9-b228-2fec-6428-268069742cda}" +2025-01-25T14:39:36.139Z [INFO] server-6520a9a6-9764-c47c-31c0-eb4ebdee6b37: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6520a9a6-9764-c47c-31c0-eb4ebdee6b37 Address:6520a9a6-9764-c47c-31c0-eb4ebdee6b37} {Suffrage:Voter ID:server-985c3563-c072-68f9-4d80-1caba51635fa Address:985c3563-c072-68f9-4d80-1caba51635fa} {Suffrage:Voter ID:server-4b18dac6-1a6d-4b73-d89f-7b6048bc5bcd Address:4b18dac6-1a6d-4b73-d89f-7b6048bc5bcd}]" +2025-01-25T14:39:36.139Z [INFO] server-6520a9a6-9764-c47c-31c0-eb4ebdee6b37: entering follower state: follower="Node at 6520a9a6-9764-c47c-31c0-eb4ebdee6b37 [Follower]" leader-address= leader-id= +2025-01-25T14:39:36.140Z [INFO] server-985c3563-c072-68f9-4d80-1caba51635fa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6520a9a6-9764-c47c-31c0-eb4ebdee6b37 Address:6520a9a6-9764-c47c-31c0-eb4ebdee6b37} {Suffrage:Voter ID:server-985c3563-c072-68f9-4d80-1caba51635fa Address:985c3563-c072-68f9-4d80-1caba51635fa} {Suffrage:Voter ID:server-4b18dac6-1a6d-4b73-d89f-7b6048bc5bcd Address:4b18dac6-1a6d-4b73-d89f-7b6048bc5bcd}]" +2025-01-25T14:39:36.140Z [INFO] server-985c3563-c072-68f9-4d80-1caba51635fa: entering follower state: follower="Node at 985c3563-c072-68f9-4d80-1caba51635fa [Follower]" leader-address= leader-id= +2025-01-25T14:39:36.141Z [INFO] server-4b18dac6-1a6d-4b73-d89f-7b6048bc5bcd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6520a9a6-9764-c47c-31c0-eb4ebdee6b37 Address:6520a9a6-9764-c47c-31c0-eb4ebdee6b37} {Suffrage:Voter ID:server-985c3563-c072-68f9-4d80-1caba51635fa Address:985c3563-c072-68f9-4d80-1caba51635fa} {Suffrage:Voter ID:server-4b18dac6-1a6d-4b73-d89f-7b6048bc5bcd Address:4b18dac6-1a6d-4b73-d89f-7b6048bc5bcd}]" +2025-01-25T14:39:36.141Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:36.141Z [INFO] server-4b18dac6-1a6d-4b73-d89f-7b6048bc5bcd: entering follower state: follower="Node at 4b18dac6-1a6d-4b73-d89f-7b6048bc5bcd [Follower]" leader-address= leader-id= +2025-01-25T14:39:36.820Z [WARN] server-6520a9a6-9764-c47c-31c0-eb4ebdee6b37: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:36.820Z [INFO] server-6520a9a6-9764-c47c-31c0-eb4ebdee6b37: entering candidate state: node="Node at 6520a9a6-9764-c47c-31c0-eb4ebdee6b37 [Candidate]" term=2 +2025-01-25T14:39:36.821Z [INFO] server-6520a9a6-9764-c47c-31c0-eb4ebdee6b37: election won: term=2 tally=2 +2025-01-25T14:39:36.821Z [INFO] server-6520a9a6-9764-c47c-31c0-eb4ebdee6b37: entering leader state: leader="Node at 6520a9a6-9764-c47c-31c0-eb4ebdee6b37 [Leader]" +2025-01-25T14:39:36.821Z [INFO] server-6520a9a6-9764-c47c-31c0-eb4ebdee6b37: added peer, starting replication: peer=server-985c3563-c072-68f9-4d80-1caba51635fa +2025-01-25T14:39:36.821Z [INFO] server-6520a9a6-9764-c47c-31c0-eb4ebdee6b37: added peer, starting replication: peer=server-4b18dac6-1a6d-4b73-d89f-7b6048bc5bcd +2025-01-25T14:39:36.822Z [INFO] server-6520a9a6-9764-c47c-31c0-eb4ebdee6b37: pipelining replication: peer="{Voter server-985c3563-c072-68f9-4d80-1caba51635fa 985c3563-c072-68f9-4d80-1caba51635fa}" +2025-01-25T14:39:36.822Z [INFO] server-6520a9a6-9764-c47c-31c0-eb4ebdee6b37: pipelining replication: peer="{Voter server-4b18dac6-1a6d-4b73-d89f-7b6048bc5bcd 4b18dac6-1a6d-4b73-d89f-7b6048bc5bcd}" +2025-01-25T14:39:37.826Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:36.821795685 +0000 UTC m=+45.867635003 (1 nodes), 680.160272ms from start of poll, 682.910113ms from cluster start. Timeout at 2025-01-25 14:39:37.826812019 +0000 UTC m=+46.872651337, 1.005016334s after stability +2025-01-25T14:39:37.830Z [INFO] server-6520a9a6-9764-c47c-31c0-eb4ebdee6b37: starting snapshot up to: index=12 +2025-01-25T14:39:37.830Z [INFO] creating new snapshot: path=/tmp/raft581069264/snapshots/2-12-1737815977830.tmp +2025-01-25T14:39:37.834Z [INFO] server-6520a9a6-9764-c47c-31c0-eb4ebdee6b37: snapshot complete up to: index=12 +2025-01-25T14:39:37.838Z [INFO] creating new snapshot: path=/tmp/raft581069264/snapshots/2-23-1737815977838.tmp +2025-01-25T14:39:37.840Z [INFO] server-6520a9a6-9764-c47c-31c0-eb4ebdee6b37: copied to local snapshot: bytes=71 +2025-01-25T14:39:37.841Z [INFO] server-6520a9a6-9764-c47c-31c0-eb4ebdee6b37: snapshot restore progress: id=2-23-1737815977838 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:37.841Z [INFO] server-6520a9a6-9764-c47c-31c0-eb4ebdee6b37: restored user snapshot: index=23 +2025-01-25T14:39:37.842Z [ERROR] server-6520a9a6-9764-c47c-31c0-eb4ebdee6b37: failed to get log: index=23 error="log not found" +2025-01-25T14:39:37.842Z [INFO] server-6520a9a6-9764-c47c-31c0-eb4ebdee6b37: aborting pipeline replication: peer="{Voter server-4b18dac6-1a6d-4b73-d89f-7b6048bc5bcd 4b18dac6-1a6d-4b73-d89f-7b6048bc5bcd}" +2025-01-25T14:39:37.842Z [ERROR] server-6520a9a6-9764-c47c-31c0-eb4ebdee6b37: failed to get log: index=23 error="log not found" +2025-01-25T14:39:37.842Z [INFO] server-6520a9a6-9764-c47c-31c0-eb4ebdee6b37: aborting pipeline replication: peer="{Voter server-985c3563-c072-68f9-4d80-1caba51635fa 985c3563-c072-68f9-4d80-1caba51635fa}" +2025-01-25T14:39:37.851Z [ERROR] server-6520a9a6-9764-c47c-31c0-eb4ebdee6b37: failed to get log: index=23 error="log not found" +2025-01-25T14:39:37.852Z [ERROR] server-6520a9a6-9764-c47c-31c0-eb4ebdee6b37: failed to get log: index=23 error="log not found" +2025-01-25T14:39:37.853Z [INFO] creating new snapshot: path=/tmp/raft1644726598/snapshots/2-23-1737815977853.tmp +2025-01-25T14:39:37.854Z [INFO] creating new snapshot: path=/tmp/raft3036312517/snapshots/2-23-1737815977854.tmp +2025-01-25T14:39:37.855Z [INFO] server-985c3563-c072-68f9-4d80-1caba51635fa: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:37.858Z [INFO] server-985c3563-c072-68f9-4d80-1caba51635fa: copied to local snapshot: bytes=71 +2025-01-25T14:39:37.862Z [INFO] server-4b18dac6-1a6d-4b73-d89f-7b6048bc5bcd: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:37.866Z [INFO] server-985c3563-c072-68f9-4d80-1caba51635fa: snapshot restore progress: id=2-23-1737815977853 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:37.866Z [INFO] server-985c3563-c072-68f9-4d80-1caba51635fa: Installed remote snapshot +2025-01-25T14:39:37.866Z [INFO] server-4b18dac6-1a6d-4b73-d89f-7b6048bc5bcd: copied to local snapshot: bytes=71 +2025-01-25T14:39:37.866Z [WARN] server-985c3563-c072-68f9-4d80-1caba51635fa: skipping application of old log: index=22 +2025-01-25T14:39:37.867Z [INFO] server-6520a9a6-9764-c47c-31c0-eb4ebdee6b37: pipelining replication: peer="{Voter server-985c3563-c072-68f9-4d80-1caba51635fa 985c3563-c072-68f9-4d80-1caba51635fa}" +2025-01-25T14:39:37.867Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:39:37.867Z [INFO] server-4b18dac6-1a6d-4b73-d89f-7b6048bc5bcd: snapshot restore progress: id=2-23-1737815977854 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:37.867Z [INFO] server-4b18dac6-1a6d-4b73-d89f-7b6048bc5bcd: Installed remote snapshot +2025-01-25T14:39:37.868Z [INFO] server-6520a9a6-9764-c47c-31c0-eb4ebdee6b37: pipelining replication: peer="{Voter server-4b18dac6-1a6d-4b73-d89f-7b6048bc5bcd 4b18dac6-1a6d-4b73-d89f-7b6048bc5bcd}" +2025-01-25T14:39:38.873Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:39:37.867779981 +0000 UTC m=+46.913619298 (2 nodes), 6.125µs from start of poll, 1.728894408s from cluster start. Timeout at 2025-01-25 14:39:38.872916773 +0000 UTC m=+47.918756090, 1.005136792s after stability +2025-01-25T14:39:38.900Z [INFO] server-6520a9a6-9764-c47c-31c0-eb4ebdee6b37: aborting pipeline replication: peer="{Voter server-4b18dac6-1a6d-4b73-d89f-7b6048bc5bcd 4b18dac6-1a6d-4b73-d89f-7b6048bc5bcd}" +2025-01-25T14:39:38.900Z [INFO] server-6520a9a6-9764-c47c-31c0-eb4ebdee6b37: aborting pipeline replication: peer="{Voter server-985c3563-c072-68f9-4d80-1caba51635fa 985c3563-c072-68f9-4d80-1caba51635fa}" === RUN TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_false -2025-01-25T14:28:54.266Z [INFO] server-4d815684-c949-f313-f177-342ae959dca0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4d815684-c949-f313-f177-342ae959dca0 Address:4d815684-c949-f313-f177-342ae959dca0} {Suffrage:Voter ID:server-6b939789-b4ce-8c7b-b999-1b7fdd2c2f4c Address:6b939789-b4ce-8c7b-b999-1b7fdd2c2f4c} {Suffrage:Voter ID:server-4ed9fa5c-8311-d15e-920b-14b47705590d Address:4ed9fa5c-8311-d15e-920b-14b47705590d}]" -2025-01-25T14:28:54.266Z [INFO] server-4d815684-c949-f313-f177-342ae959dca0: entering follower state: follower="Node at 4d815684-c949-f313-f177-342ae959dca0 [Follower]" leader-address= leader-id= -2025-01-25T14:28:54.266Z [INFO] server-6b939789-b4ce-8c7b-b999-1b7fdd2c2f4c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4d815684-c949-f313-f177-342ae959dca0 Address:4d815684-c949-f313-f177-342ae959dca0} {Suffrage:Voter ID:server-6b939789-b4ce-8c7b-b999-1b7fdd2c2f4c Address:6b939789-b4ce-8c7b-b999-1b7fdd2c2f4c} {Suffrage:Voter ID:server-4ed9fa5c-8311-d15e-920b-14b47705590d Address:4ed9fa5c-8311-d15e-920b-14b47705590d}]" -2025-01-25T14:28:54.267Z [INFO] server-6b939789-b4ce-8c7b-b999-1b7fdd2c2f4c: entering follower state: follower="Node at 6b939789-b4ce-8c7b-b999-1b7fdd2c2f4c [Follower]" leader-address= leader-id= -2025-01-25T14:28:54.267Z [INFO] server-4ed9fa5c-8311-d15e-920b-14b47705590d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4d815684-c949-f313-f177-342ae959dca0 Address:4d815684-c949-f313-f177-342ae959dca0} {Suffrage:Voter ID:server-6b939789-b4ce-8c7b-b999-1b7fdd2c2f4c Address:6b939789-b4ce-8c7b-b999-1b7fdd2c2f4c} {Suffrage:Voter ID:server-4ed9fa5c-8311-d15e-920b-14b47705590d Address:4ed9fa5c-8311-d15e-920b-14b47705590d}]" -2025-01-25T14:28:54.267Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:54.267Z [INFO] server-4ed9fa5c-8311-d15e-920b-14b47705590d: entering follower state: follower="Node at 4ed9fa5c-8311-d15e-920b-14b47705590d [Follower]" leader-address= leader-id= -2025-01-25T14:28:54.835Z [WARN] server-4d815684-c949-f313-f177-342ae959dca0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:54.835Z [INFO] server-4d815684-c949-f313-f177-342ae959dca0: entering candidate state: node="Node at 4d815684-c949-f313-f177-342ae959dca0 [Candidate]" term=2 -2025-01-25T14:28:54.836Z [INFO] server-4d815684-c949-f313-f177-342ae959dca0: election won: term=2 tally=2 -2025-01-25T14:28:54.836Z [INFO] server-4d815684-c949-f313-f177-342ae959dca0: entering leader state: leader="Node at 4d815684-c949-f313-f177-342ae959dca0 [Leader]" -2025-01-25T14:28:54.836Z [INFO] server-4d815684-c949-f313-f177-342ae959dca0: added peer, starting replication: peer=server-6b939789-b4ce-8c7b-b999-1b7fdd2c2f4c -2025-01-25T14:28:54.836Z [INFO] server-4d815684-c949-f313-f177-342ae959dca0: added peer, starting replication: peer=server-4ed9fa5c-8311-d15e-920b-14b47705590d -2025-01-25T14:28:54.836Z [INFO] server-4d815684-c949-f313-f177-342ae959dca0: pipelining replication: peer="{Voter server-4ed9fa5c-8311-d15e-920b-14b47705590d 4ed9fa5c-8311-d15e-920b-14b47705590d}" -2025-01-25T14:28:54.836Z [INFO] server-4d815684-c949-f313-f177-342ae959dca0: pipelining replication: peer="{Voter server-6b939789-b4ce-8c7b-b999-1b7fdd2c2f4c 6b939789-b4ce-8c7b-b999-1b7fdd2c2f4c}" -2025-01-25T14:28:55.842Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:54.836387162 +0000 UTC m=+44.690712144 (1 nodes), 569.175198ms from start of poll, 569.904372ms from cluster start. Timeout at 2025-01-25 14:28:55.841481039 +0000 UTC m=+45.695806061, 1.005093917s after stability -2025-01-25T14:28:55.843Z [INFO] server-4d815684-c949-f313-f177-342ae959dca0: starting snapshot up to: index=12 -2025-01-25T14:28:55.843Z [INFO] creating new snapshot: path=/tmp/raft1812817363/snapshots/2-12-1737815335843.tmp -2025-01-25T14:28:55.844Z [INFO] server-4d815684-c949-f313-f177-342ae959dca0: snapshot complete up to: index=12 -2025-01-25T14:28:55.846Z [INFO] creating new snapshot: path=/tmp/raft1812817363/snapshots/2-23-1737815335846.tmp -2025-01-25T14:28:55.847Z [INFO] server-4d815684-c949-f313-f177-342ae959dca0: copied to local snapshot: bytes=71 -2025-01-25T14:28:55.847Z [INFO] server-4d815684-c949-f313-f177-342ae959dca0: snapshot restore progress: id=2-23-1737815335846 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:55.848Z [INFO] server-4d815684-c949-f313-f177-342ae959dca0: removing all old logs from log store -2025-01-25T14:28:55.848Z [INFO] server-4d815684-c949-f313-f177-342ae959dca0: compacting logs: from=1 to=22 -2025-01-25T14:28:55.848Z [INFO] server-4d815684-c949-f313-f177-342ae959dca0: restored user snapshot: index=23 -2025-01-25T14:28:55.848Z [ERROR] server-4d815684-c949-f313-f177-342ae959dca0: failed to get log: index=22 error="log not found" -2025-01-25T14:28:55.848Z [INFO] server-4d815684-c949-f313-f177-342ae959dca0: aborting pipeline replication: peer="{Voter server-4ed9fa5c-8311-d15e-920b-14b47705590d 4ed9fa5c-8311-d15e-920b-14b47705590d}" -2025-01-25T14:28:55.848Z [ERROR] server-4d815684-c949-f313-f177-342ae959dca0: failed to get log: index=22 error="log not found" -2025-01-25T14:28:55.848Z [INFO] server-4d815684-c949-f313-f177-342ae959dca0: aborting pipeline replication: peer="{Voter server-6b939789-b4ce-8c7b-b999-1b7fdd2c2f4c 6b939789-b4ce-8c7b-b999-1b7fdd2c2f4c}" -2025-01-25T14:28:55.855Z [ERROR] server-4d815684-c949-f313-f177-342ae959dca0: failed to get log: index=22 error="log not found" -2025-01-25T14:28:55.856Z [INFO] creating new snapshot: path=/tmp/raft2739535423/snapshots/2-23-1737815335856.tmp -2025-01-25T14:28:55.856Z [INFO] server-6b939789-b4ce-8c7b-b999-1b7fdd2c2f4c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:55.857Z [INFO] server-6b939789-b4ce-8c7b-b999-1b7fdd2c2f4c: copied to local snapshot: bytes=71 -2025-01-25T14:28:55.858Z [INFO] server-6b939789-b4ce-8c7b-b999-1b7fdd2c2f4c: snapshot restore progress: id=2-23-1737815335856 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:55.858Z [INFO] server-6b939789-b4ce-8c7b-b999-1b7fdd2c2f4c: removing all old logs from log store -2025-01-25T14:28:55.858Z [INFO] server-6b939789-b4ce-8c7b-b999-1b7fdd2c2f4c: compacting logs: from=1 to=22 -2025-01-25T14:28:55.858Z [INFO] server-6b939789-b4ce-8c7b-b999-1b7fdd2c2f4c: Installed remote snapshot -2025-01-25T14:28:55.858Z [ERROR] server-4d815684-c949-f313-f177-342ae959dca0: failed to get log: index=22 error="log not found" -2025-01-25T14:28:55.858Z [WARN] server-6b939789-b4ce-8c7b-b999-1b7fdd2c2f4c: skipping application of old log: index=22 -2025-01-25T14:28:55.858Z [INFO] server-4d815684-c949-f313-f177-342ae959dca0: pipelining replication: peer="{Voter server-6b939789-b4ce-8c7b-b999-1b7fdd2c2f4c 6b939789-b4ce-8c7b-b999-1b7fdd2c2f4c}" -2025-01-25T14:28:55.858Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:28:55.859Z [INFO] creating new snapshot: path=/tmp/raft2573522237/snapshots/2-23-1737815335859.tmp -2025-01-25T14:28:55.859Z [INFO] server-4ed9fa5c-8311-d15e-920b-14b47705590d: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:55.860Z [INFO] server-4ed9fa5c-8311-d15e-920b-14b47705590d: copied to local snapshot: bytes=71 -2025-01-25T14:28:55.861Z [INFO] server-4ed9fa5c-8311-d15e-920b-14b47705590d: snapshot restore progress: id=2-23-1737815335859 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:28:55.861Z [INFO] server-4ed9fa5c-8311-d15e-920b-14b47705590d: removing all old logs from log store -2025-01-25T14:28:55.861Z [INFO] server-4ed9fa5c-8311-d15e-920b-14b47705590d: compacting logs: from=1 to=22 -2025-01-25T14:28:55.861Z [INFO] server-4ed9fa5c-8311-d15e-920b-14b47705590d: Installed remote snapshot -2025-01-25T14:28:55.861Z [INFO] server-4d815684-c949-f313-f177-342ae959dca0: pipelining replication: peer="{Voter server-4ed9fa5c-8311-d15e-920b-14b47705590d 4ed9fa5c-8311-d15e-920b-14b47705590d}" -2025-01-25T14:28:56.864Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:28:55.858537259 +0000 UTC m=+45.712862241 (2 nodes), 2.92µs from start of poll, 1.592054469s from cluster start. Timeout at 2025-01-25 14:28:56.863646256 +0000 UTC m=+46.717971198, 1.005108957s after stability -2025-01-25T14:28:56.877Z [INFO] server-4d815684-c949-f313-f177-342ae959dca0: aborting pipeline replication: peer="{Voter server-4ed9fa5c-8311-d15e-920b-14b47705590d 4ed9fa5c-8311-d15e-920b-14b47705590d}" -2025-01-25T14:28:56.877Z [INFO] server-4d815684-c949-f313-f177-342ae959dca0: aborting pipeline replication: peer="{Voter server-6b939789-b4ce-8c7b-b999-1b7fdd2c2f4c 6b939789-b4ce-8c7b-b999-1b7fdd2c2f4c}" +2025-01-25T14:39:38.911Z [INFO] server-100fd7d8-5984-6051-58ec-464cd54922f3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-100fd7d8-5984-6051-58ec-464cd54922f3 Address:100fd7d8-5984-6051-58ec-464cd54922f3} {Suffrage:Voter ID:server-b964111d-cc53-dda4-a231-0bbe8d794d92 Address:b964111d-cc53-dda4-a231-0bbe8d794d92} {Suffrage:Voter ID:server-8df90267-666b-4e6b-0577-50c8d356c9da Address:8df90267-666b-4e6b-0577-50c8d356c9da}]" +2025-01-25T14:39:38.912Z [INFO] server-100fd7d8-5984-6051-58ec-464cd54922f3: entering follower state: follower="Node at 100fd7d8-5984-6051-58ec-464cd54922f3 [Follower]" leader-address= leader-id= +2025-01-25T14:39:38.912Z [INFO] server-b964111d-cc53-dda4-a231-0bbe8d794d92: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-100fd7d8-5984-6051-58ec-464cd54922f3 Address:100fd7d8-5984-6051-58ec-464cd54922f3} {Suffrage:Voter ID:server-b964111d-cc53-dda4-a231-0bbe8d794d92 Address:b964111d-cc53-dda4-a231-0bbe8d794d92} {Suffrage:Voter ID:server-8df90267-666b-4e6b-0577-50c8d356c9da Address:8df90267-666b-4e6b-0577-50c8d356c9da}]" +2025-01-25T14:39:38.913Z [INFO] server-b964111d-cc53-dda4-a231-0bbe8d794d92: entering follower state: follower="Node at b964111d-cc53-dda4-a231-0bbe8d794d92 [Follower]" leader-address= leader-id= +2025-01-25T14:39:38.913Z [INFO] server-8df90267-666b-4e6b-0577-50c8d356c9da: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-100fd7d8-5984-6051-58ec-464cd54922f3 Address:100fd7d8-5984-6051-58ec-464cd54922f3} {Suffrage:Voter ID:server-b964111d-cc53-dda4-a231-0bbe8d794d92 Address:b964111d-cc53-dda4-a231-0bbe8d794d92} {Suffrage:Voter ID:server-8df90267-666b-4e6b-0577-50c8d356c9da Address:8df90267-666b-4e6b-0577-50c8d356c9da}]" +2025-01-25T14:39:38.913Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:38.913Z [INFO] server-8df90267-666b-4e6b-0577-50c8d356c9da: entering follower state: follower="Node at 8df90267-666b-4e6b-0577-50c8d356c9da [Follower]" leader-address= leader-id= +2025-01-25T14:39:39.753Z [WARN] server-8df90267-666b-4e6b-0577-50c8d356c9da: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:39.753Z [INFO] server-8df90267-666b-4e6b-0577-50c8d356c9da: entering candidate state: node="Node at 8df90267-666b-4e6b-0577-50c8d356c9da [Candidate]" term=2 +2025-01-25T14:39:39.753Z [INFO] server-8df90267-666b-4e6b-0577-50c8d356c9da: election won: term=2 tally=2 +2025-01-25T14:39:39.753Z [INFO] server-8df90267-666b-4e6b-0577-50c8d356c9da: entering leader state: leader="Node at 8df90267-666b-4e6b-0577-50c8d356c9da [Leader]" +2025-01-25T14:39:39.753Z [INFO] server-8df90267-666b-4e6b-0577-50c8d356c9da: added peer, starting replication: peer=server-100fd7d8-5984-6051-58ec-464cd54922f3 +2025-01-25T14:39:39.753Z [INFO] server-8df90267-666b-4e6b-0577-50c8d356c9da: added peer, starting replication: peer=server-b964111d-cc53-dda4-a231-0bbe8d794d92 +2025-01-25T14:39:39.754Z [INFO] server-8df90267-666b-4e6b-0577-50c8d356c9da: pipelining replication: peer="{Voter server-b964111d-cc53-dda4-a231-0bbe8d794d92 b964111d-cc53-dda4-a231-0bbe8d794d92}" +2025-01-25T14:39:39.754Z [INFO] server-8df90267-666b-4e6b-0577-50c8d356c9da: pipelining replication: peer="{Voter server-100fd7d8-5984-6051-58ec-464cd54922f3 100fd7d8-5984-6051-58ec-464cd54922f3}" +2025-01-25T14:39:40.759Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:39.753889264 +0000 UTC m=+48.799728581 (1 nodes), 839.883195ms from start of poll, 842.568286ms from cluster start. Timeout at 2025-01-25 14:39:40.758921347 +0000 UTC m=+49.804760664, 1.005032083s after stability +2025-01-25T14:39:40.762Z [INFO] server-8df90267-666b-4e6b-0577-50c8d356c9da: starting snapshot up to: index=12 +2025-01-25T14:39:40.762Z [INFO] creating new snapshot: path=/tmp/raft86236423/snapshots/2-12-1737815980762.tmp +2025-01-25T14:39:40.766Z [INFO] server-8df90267-666b-4e6b-0577-50c8d356c9da: snapshot complete up to: index=12 +2025-01-25T14:39:40.773Z [INFO] creating new snapshot: path=/tmp/raft86236423/snapshots/2-23-1737815980773.tmp +2025-01-25T14:39:40.778Z [INFO] server-8df90267-666b-4e6b-0577-50c8d356c9da: copied to local snapshot: bytes=71 +2025-01-25T14:39:40.779Z [INFO] server-8df90267-666b-4e6b-0577-50c8d356c9da: snapshot restore progress: id=2-23-1737815980773 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:40.780Z [INFO] server-8df90267-666b-4e6b-0577-50c8d356c9da: removing all old logs from log store +2025-01-25T14:39:40.780Z [INFO] server-8df90267-666b-4e6b-0577-50c8d356c9da: compacting logs: from=1 to=22 +2025-01-25T14:39:40.780Z [INFO] server-8df90267-666b-4e6b-0577-50c8d356c9da: restored user snapshot: index=23 +2025-01-25T14:39:40.780Z [ERROR] server-8df90267-666b-4e6b-0577-50c8d356c9da: failed to get log: index=22 error="log not found" +2025-01-25T14:39:40.780Z [INFO] server-8df90267-666b-4e6b-0577-50c8d356c9da: aborting pipeline replication: peer="{Voter server-100fd7d8-5984-6051-58ec-464cd54922f3 100fd7d8-5984-6051-58ec-464cd54922f3}" +2025-01-25T14:39:40.780Z [ERROR] server-8df90267-666b-4e6b-0577-50c8d356c9da: failed to get log: index=22 error="log not found" +2025-01-25T14:39:40.780Z [INFO] server-8df90267-666b-4e6b-0577-50c8d356c9da: aborting pipeline replication: peer="{Voter server-b964111d-cc53-dda4-a231-0bbe8d794d92 b964111d-cc53-dda4-a231-0bbe8d794d92}" +2025-01-25T14:39:40.780Z [ERROR] server-8df90267-666b-4e6b-0577-50c8d356c9da: failed to get log: index=22 error="log not found" +2025-01-25T14:39:40.783Z [INFO] creating new snapshot: path=/tmp/raft3450168551/snapshots/2-23-1737815980783.tmp +2025-01-25T14:39:40.785Z [INFO] server-100fd7d8-5984-6051-58ec-464cd54922f3: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:40.787Z [INFO] server-100fd7d8-5984-6051-58ec-464cd54922f3: copied to local snapshot: bytes=71 +2025-01-25T14:39:40.787Z [ERROR] server-8df90267-666b-4e6b-0577-50c8d356c9da: failed to get log: index=22 error="log not found" +2025-01-25T14:39:40.789Z [INFO] server-100fd7d8-5984-6051-58ec-464cd54922f3: snapshot restore progress: id=2-23-1737815980783 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:40.789Z [INFO] server-100fd7d8-5984-6051-58ec-464cd54922f3: removing all old logs from log store +2025-01-25T14:39:40.789Z [INFO] server-100fd7d8-5984-6051-58ec-464cd54922f3: compacting logs: from=1 to=22 +2025-01-25T14:39:40.789Z [INFO] server-100fd7d8-5984-6051-58ec-464cd54922f3: Installed remote snapshot +2025-01-25T14:39:40.790Z [WARN] server-100fd7d8-5984-6051-58ec-464cd54922f3: skipping application of old log: index=22 +2025-01-25T14:39:40.791Z [INFO] creating new snapshot: path=/tmp/raft1107860118/snapshots/2-23-1737815980790.tmp +2025-01-25T14:39:40.791Z [INFO] server-8df90267-666b-4e6b-0577-50c8d356c9da: pipelining replication: peer="{Voter server-100fd7d8-5984-6051-58ec-464cd54922f3 100fd7d8-5984-6051-58ec-464cd54922f3}" +2025-01-25T14:39:40.791Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:39:40.794Z [INFO] server-b964111d-cc53-dda4-a231-0bbe8d794d92: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:40.802Z [INFO] server-b964111d-cc53-dda4-a231-0bbe8d794d92: copied to local snapshot: bytes=71 +2025-01-25T14:39:40.807Z [INFO] server-b964111d-cc53-dda4-a231-0bbe8d794d92: snapshot restore progress: id=2-23-1737815980790 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:40.808Z [INFO] server-b964111d-cc53-dda4-a231-0bbe8d794d92: removing all old logs from log store +2025-01-25T14:39:40.808Z [INFO] server-b964111d-cc53-dda4-a231-0bbe8d794d92: compacting logs: from=1 to=22 +2025-01-25T14:39:40.808Z [INFO] server-b964111d-cc53-dda4-a231-0bbe8d794d92: Installed remote snapshot +2025-01-25T14:39:40.809Z [INFO] server-8df90267-666b-4e6b-0577-50c8d356c9da: pipelining replication: peer="{Voter server-b964111d-cc53-dda4-a231-0bbe8d794d92 b964111d-cc53-dda4-a231-0bbe8d794d92}" +2025-01-25T14:39:41.797Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:39:40.791601743 +0000 UTC m=+49.837441061 (2 nodes), 6.709µs from start of poll, 1.880280766s from cluster start. Timeout at 2025-01-25 14:39:41.796746118 +0000 UTC m=+50.842585436, 1.005144375s after stability +2025-01-25T14:39:41.811Z [INFO] server-8df90267-666b-4e6b-0577-50c8d356c9da: aborting pipeline replication: peer="{Voter server-100fd7d8-5984-6051-58ec-464cd54922f3 100fd7d8-5984-6051-58ec-464cd54922f3}" +2025-01-25T14:39:41.811Z [INFO] server-8df90267-666b-4e6b-0577-50c8d356c9da: aborting pipeline replication: peer="{Voter server-b964111d-cc53-dda4-a231-0bbe8d794d92 b964111d-cc53-dda4-a231-0bbe8d794d92}" === RUN TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_true -2025-01-25T14:28:56.880Z [INFO] server-c9862a21-3a64-5896-7082-4cab656e5f00: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c9862a21-3a64-5896-7082-4cab656e5f00 Address:c9862a21-3a64-5896-7082-4cab656e5f00} {Suffrage:Voter ID:server-3adee50b-dc2a-6a94-7e1d-fc1222d447ed Address:3adee50b-dc2a-6a94-7e1d-fc1222d447ed} {Suffrage:Voter ID:server-b576e696-78c3-27da-8bda-ff1db8feca07 Address:b576e696-78c3-27da-8bda-ff1db8feca07}]" -2025-01-25T14:28:56.880Z [INFO] server-c9862a21-3a64-5896-7082-4cab656e5f00: entering follower state: follower="Node at c9862a21-3a64-5896-7082-4cab656e5f00 [Follower]" leader-address= leader-id= -2025-01-25T14:28:56.880Z [INFO] server-3adee50b-dc2a-6a94-7e1d-fc1222d447ed: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c9862a21-3a64-5896-7082-4cab656e5f00 Address:c9862a21-3a64-5896-7082-4cab656e5f00} {Suffrage:Voter ID:server-3adee50b-dc2a-6a94-7e1d-fc1222d447ed Address:3adee50b-dc2a-6a94-7e1d-fc1222d447ed} {Suffrage:Voter ID:server-b576e696-78c3-27da-8bda-ff1db8feca07 Address:b576e696-78c3-27da-8bda-ff1db8feca07}]" -2025-01-25T14:28:56.880Z [INFO] server-3adee50b-dc2a-6a94-7e1d-fc1222d447ed: entering follower state: follower="Node at 3adee50b-dc2a-6a94-7e1d-fc1222d447ed [Follower]" leader-address= leader-id= -2025-01-25T14:28:56.880Z [INFO] server-b576e696-78c3-27da-8bda-ff1db8feca07: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c9862a21-3a64-5896-7082-4cab656e5f00 Address:c9862a21-3a64-5896-7082-4cab656e5f00} {Suffrage:Voter ID:server-3adee50b-dc2a-6a94-7e1d-fc1222d447ed Address:3adee50b-dc2a-6a94-7e1d-fc1222d447ed} {Suffrage:Voter ID:server-b576e696-78c3-27da-8bda-ff1db8feca07 Address:b576e696-78c3-27da-8bda-ff1db8feca07}]" -2025-01-25T14:28:56.880Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:56.880Z [INFO] server-b576e696-78c3-27da-8bda-ff1db8feca07: entering follower state: follower="Node at b576e696-78c3-27da-8bda-ff1db8feca07 [Follower]" leader-address= leader-id= -2025-01-25T14:28:57.454Z [WARN] server-b576e696-78c3-27da-8bda-ff1db8feca07: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:57.454Z [INFO] server-b576e696-78c3-27da-8bda-ff1db8feca07: entering candidate state: node="Node at b576e696-78c3-27da-8bda-ff1db8feca07 [Candidate]" term=2 -2025-01-25T14:28:57.454Z [INFO] server-b576e696-78c3-27da-8bda-ff1db8feca07: election won: term=2 tally=2 -2025-01-25T14:28:57.454Z [INFO] server-b576e696-78c3-27da-8bda-ff1db8feca07: entering leader state: leader="Node at b576e696-78c3-27da-8bda-ff1db8feca07 [Leader]" -2025-01-25T14:28:57.454Z [INFO] server-b576e696-78c3-27da-8bda-ff1db8feca07: added peer, starting replication: peer=server-c9862a21-3a64-5896-7082-4cab656e5f00 -2025-01-25T14:28:57.454Z [INFO] server-b576e696-78c3-27da-8bda-ff1db8feca07: added peer, starting replication: peer=server-3adee50b-dc2a-6a94-7e1d-fc1222d447ed -2025-01-25T14:28:57.454Z [INFO] server-b576e696-78c3-27da-8bda-ff1db8feca07: pipelining replication: peer="{Voter server-3adee50b-dc2a-6a94-7e1d-fc1222d447ed 3adee50b-dc2a-6a94-7e1d-fc1222d447ed}" -2025-01-25T14:28:57.455Z [INFO] server-b576e696-78c3-27da-8bda-ff1db8feca07: pipelining replication: peer="{Voter server-c9862a21-3a64-5896-7082-4cab656e5f00 c9862a21-3a64-5896-7082-4cab656e5f00}" -2025-01-25T14:28:58.459Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:57.454889974 +0000 UTC m=+47.309214956 (1 nodes), 574.158219ms from start of poll, 574.807196ms from cluster start. Timeout at 2025-01-25 14:28:58.459894294 +0000 UTC m=+48.314219236, 1.00500428s after stability -2025-01-25T14:28:58.461Z [INFO] server-b576e696-78c3-27da-8bda-ff1db8feca07: starting snapshot up to: index=12 -2025-01-25T14:28:58.461Z [INFO] creating new snapshot: path=/tmp/raft1181848739/snapshots/2-12-1737815338461.tmp -2025-01-25T14:28:58.462Z [INFO] server-b576e696-78c3-27da-8bda-ff1db8feca07: snapshot complete up to: index=12 -2025-01-25T14:28:58.464Z [INFO] server-eb5e61bd-f0ca-a289-e33d-226a0a786170: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eb5e61bd-f0ca-a289-e33d-226a0a786170 Address:eb5e61bd-f0ca-a289-e33d-226a0a786170} {Suffrage:Voter ID:server-e0a8ce1e-d964-e763-1c82-d2ddc66b713b Address:e0a8ce1e-d964-e763-1c82-d2ddc66b713b} {Suffrage:Voter ID:server-9b42780b-af98-44d1-e0a3-220357490865 Address:9b42780b-af98-44d1-e0a3-220357490865}]" -2025-01-25T14:28:58.464Z [INFO] server-e0a8ce1e-d964-e763-1c82-d2ddc66b713b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eb5e61bd-f0ca-a289-e33d-226a0a786170 Address:eb5e61bd-f0ca-a289-e33d-226a0a786170} {Suffrage:Voter ID:server-e0a8ce1e-d964-e763-1c82-d2ddc66b713b Address:e0a8ce1e-d964-e763-1c82-d2ddc66b713b} {Suffrage:Voter ID:server-9b42780b-af98-44d1-e0a3-220357490865 Address:9b42780b-af98-44d1-e0a3-220357490865}]" -2025-01-25T14:28:58.464Z [INFO] server-eb5e61bd-f0ca-a289-e33d-226a0a786170: entering follower state: follower="Node at eb5e61bd-f0ca-a289-e33d-226a0a786170 [Follower]" leader-address= leader-id= -2025-01-25T14:28:58.464Z [INFO] server-e0a8ce1e-d964-e763-1c82-d2ddc66b713b: entering follower state: follower="Node at e0a8ce1e-d964-e763-1c82-d2ddc66b713b [Follower]" leader-address= leader-id= -2025-01-25T14:28:58.464Z [INFO] server-9b42780b-af98-44d1-e0a3-220357490865: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eb5e61bd-f0ca-a289-e33d-226a0a786170 Address:eb5e61bd-f0ca-a289-e33d-226a0a786170} {Suffrage:Voter ID:server-e0a8ce1e-d964-e763-1c82-d2ddc66b713b Address:e0a8ce1e-d964-e763-1c82-d2ddc66b713b} {Suffrage:Voter ID:server-9b42780b-af98-44d1-e0a3-220357490865 Address:9b42780b-af98-44d1-e0a3-220357490865}]" -2025-01-25T14:28:58.465Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:28:58.465Z [INFO] server-9b42780b-af98-44d1-e0a3-220357490865: entering follower state: follower="Node at 9b42780b-af98-44d1-e0a3-220357490865 [Follower]" leader-address= leader-id= -2025-01-25T14:28:59.045Z [WARN] server-eb5e61bd-f0ca-a289-e33d-226a0a786170: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:28:59.045Z [INFO] server-eb5e61bd-f0ca-a289-e33d-226a0a786170: entering candidate state: node="Node at eb5e61bd-f0ca-a289-e33d-226a0a786170 [Candidate]" term=2 -2025-01-25T14:28:59.045Z [INFO] server-eb5e61bd-f0ca-a289-e33d-226a0a786170: election won: term=2 tally=2 -2025-01-25T14:28:59.045Z [INFO] server-eb5e61bd-f0ca-a289-e33d-226a0a786170: entering leader state: leader="Node at eb5e61bd-f0ca-a289-e33d-226a0a786170 [Leader]" -2025-01-25T14:28:59.045Z [INFO] server-eb5e61bd-f0ca-a289-e33d-226a0a786170: added peer, starting replication: peer=server-e0a8ce1e-d964-e763-1c82-d2ddc66b713b -2025-01-25T14:28:59.045Z [INFO] server-eb5e61bd-f0ca-a289-e33d-226a0a786170: added peer, starting replication: peer=server-9b42780b-af98-44d1-e0a3-220357490865 -2025-01-25T14:28:59.045Z [INFO] server-eb5e61bd-f0ca-a289-e33d-226a0a786170: pipelining replication: peer="{Voter server-e0a8ce1e-d964-e763-1c82-d2ddc66b713b e0a8ce1e-d964-e763-1c82-d2ddc66b713b}" -2025-01-25T14:28:59.045Z [INFO] server-eb5e61bd-f0ca-a289-e33d-226a0a786170: pipelining replication: peer="{Voter server-9b42780b-af98-44d1-e0a3-220357490865 9b42780b-af98-44d1-e0a3-220357490865}" -2025-01-25T14:29:00.050Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:28:59.045437097 +0000 UTC m=+48.899762079 (1 nodes), 580.395594ms from start of poll, 581.134768ms from cluster start. Timeout at 2025-01-25 14:29:00.050442457 +0000 UTC m=+49.904767439, 1.00500536s after stability -2025-01-25T14:29:00.051Z [INFO] creating new snapshot: path=/tmp/raft345270667/snapshots/2-15-1737815340051.tmp -2025-01-25T14:29:00.052Z [INFO] server-eb5e61bd-f0ca-a289-e33d-226a0a786170: copied to local snapshot: bytes=71 -2025-01-25T14:29:00.053Z [INFO] server-eb5e61bd-f0ca-a289-e33d-226a0a786170: snapshot restore progress: id=2-15-1737815340051 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:00.053Z [INFO] server-eb5e61bd-f0ca-a289-e33d-226a0a786170: restored user snapshot: index=15 -2025-01-25T14:29:00.053Z [ERROR] server-eb5e61bd-f0ca-a289-e33d-226a0a786170: failed to get log: index=3 error="log not found" -2025-01-25T14:29:00.053Z [INFO] server-eb5e61bd-f0ca-a289-e33d-226a0a786170: aborting pipeline replication: peer="{Voter server-9b42780b-af98-44d1-e0a3-220357490865 9b42780b-af98-44d1-e0a3-220357490865}" -2025-01-25T14:29:00.053Z [ERROR] server-eb5e61bd-f0ca-a289-e33d-226a0a786170: failed to get log: index=3 error="log not found" -2025-01-25T14:29:00.053Z [INFO] server-eb5e61bd-f0ca-a289-e33d-226a0a786170: aborting pipeline replication: peer="{Voter server-e0a8ce1e-d964-e763-1c82-d2ddc66b713b e0a8ce1e-d964-e763-1c82-d2ddc66b713b}" -2025-01-25T14:29:00.062Z [ERROR] server-eb5e61bd-f0ca-a289-e33d-226a0a786170: failed to get log: index=3 error="log not found" -2025-01-25T14:29:00.063Z [ERROR] server-eb5e61bd-f0ca-a289-e33d-226a0a786170: failed to get log: index=3 error="log not found" -2025-01-25T14:29:00.063Z [INFO] creating new snapshot: path=/tmp/raft2194126534/snapshots/2-15-1737815340063.tmp -2025-01-25T14:29:00.064Z [INFO] creating new snapshot: path=/tmp/raft3376604060/snapshots/2-15-1737815340064.tmp -2025-01-25T14:29:00.064Z [INFO] server-9b42780b-af98-44d1-e0a3-220357490865: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:00.064Z [INFO] server-e0a8ce1e-d964-e763-1c82-d2ddc66b713b: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:00.066Z [INFO] server-9b42780b-af98-44d1-e0a3-220357490865: copied to local snapshot: bytes=71 -2025-01-25T14:29:00.068Z [INFO] server-9b42780b-af98-44d1-e0a3-220357490865: snapshot restore progress: id=2-15-1737815340063 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:00.068Z [INFO] server-9b42780b-af98-44d1-e0a3-220357490865: Installed remote snapshot -2025-01-25T14:29:00.068Z [INFO] server-eb5e61bd-f0ca-a289-e33d-226a0a786170: pipelining replication: peer="{Voter server-9b42780b-af98-44d1-e0a3-220357490865 9b42780b-af98-44d1-e0a3-220357490865}" -2025-01-25T14:29:00.068Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:29:00.069Z [INFO] server-e0a8ce1e-d964-e763-1c82-d2ddc66b713b: copied to local snapshot: bytes=71 -2025-01-25T14:29:00.071Z [INFO] server-e0a8ce1e-d964-e763-1c82-d2ddc66b713b: snapshot restore progress: id=2-15-1737815340064 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:00.071Z [INFO] server-e0a8ce1e-d964-e763-1c82-d2ddc66b713b: Installed remote snapshot -2025-01-25T14:29:00.071Z [INFO] server-eb5e61bd-f0ca-a289-e33d-226a0a786170: pipelining replication: peer="{Voter server-e0a8ce1e-d964-e763-1c82-d2ddc66b713b e0a8ce1e-d964-e763-1c82-d2ddc66b713b}" -2025-01-25T14:29:01.074Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:29:00.069029502 +0000 UTC m=+49.923354484 (2 nodes), 2.28µs from start of poll, 1.604727173s from cluster start. Timeout at 2025-01-25 14:29:01.074120779 +0000 UTC m=+50.928445761, 1.005091277s after stability -2025-01-25T14:29:01.086Z [INFO] server-b576e696-78c3-27da-8bda-ff1db8feca07: aborting pipeline replication: peer="{Voter server-3adee50b-dc2a-6a94-7e1d-fc1222d447ed 3adee50b-dc2a-6a94-7e1d-fc1222d447ed}" -2025-01-25T14:29:01.086Z [INFO] server-b576e696-78c3-27da-8bda-ff1db8feca07: aborting pipeline replication: peer="{Voter server-c9862a21-3a64-5896-7082-4cab656e5f00 c9862a21-3a64-5896-7082-4cab656e5f00}" +2025-01-25T14:39:41.820Z [INFO] server-24a9fc4a-4d6e-156d-fb35-4d15cd908868: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-24a9fc4a-4d6e-156d-fb35-4d15cd908868 Address:24a9fc4a-4d6e-156d-fb35-4d15cd908868} {Suffrage:Voter ID:server-d47727c1-6ec3-5bd2-59b2-0624504f41ff Address:d47727c1-6ec3-5bd2-59b2-0624504f41ff} {Suffrage:Voter ID:server-73149e1d-7272-ac66-19ad-0af72269f652 Address:73149e1d-7272-ac66-19ad-0af72269f652}]" +2025-01-25T14:39:41.821Z [INFO] server-d47727c1-6ec3-5bd2-59b2-0624504f41ff: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-24a9fc4a-4d6e-156d-fb35-4d15cd908868 Address:24a9fc4a-4d6e-156d-fb35-4d15cd908868} {Suffrage:Voter ID:server-d47727c1-6ec3-5bd2-59b2-0624504f41ff Address:d47727c1-6ec3-5bd2-59b2-0624504f41ff} {Suffrage:Voter ID:server-73149e1d-7272-ac66-19ad-0af72269f652 Address:73149e1d-7272-ac66-19ad-0af72269f652}]" +2025-01-25T14:39:41.820Z [INFO] server-24a9fc4a-4d6e-156d-fb35-4d15cd908868: entering follower state: follower="Node at 24a9fc4a-4d6e-156d-fb35-4d15cd908868 [Follower]" leader-address= leader-id= +2025-01-25T14:39:41.821Z [INFO] server-d47727c1-6ec3-5bd2-59b2-0624504f41ff: entering follower state: follower="Node at d47727c1-6ec3-5bd2-59b2-0624504f41ff [Follower]" leader-address= leader-id= +2025-01-25T14:39:41.821Z [INFO] server-73149e1d-7272-ac66-19ad-0af72269f652: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-24a9fc4a-4d6e-156d-fb35-4d15cd908868 Address:24a9fc4a-4d6e-156d-fb35-4d15cd908868} {Suffrage:Voter ID:server-d47727c1-6ec3-5bd2-59b2-0624504f41ff Address:d47727c1-6ec3-5bd2-59b2-0624504f41ff} {Suffrage:Voter ID:server-73149e1d-7272-ac66-19ad-0af72269f652 Address:73149e1d-7272-ac66-19ad-0af72269f652}]" +2025-01-25T14:39:41.821Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:41.822Z [INFO] server-73149e1d-7272-ac66-19ad-0af72269f652: entering follower state: follower="Node at 73149e1d-7272-ac66-19ad-0af72269f652 [Follower]" leader-address= leader-id= +2025-01-25T14:39:42.588Z [WARN] server-73149e1d-7272-ac66-19ad-0af72269f652: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:42.589Z [INFO] server-73149e1d-7272-ac66-19ad-0af72269f652: entering candidate state: node="Node at 73149e1d-7272-ac66-19ad-0af72269f652 [Candidate]" term=2 +2025-01-25T14:39:42.589Z [INFO] server-73149e1d-7272-ac66-19ad-0af72269f652: election won: term=2 tally=2 +2025-01-25T14:39:42.589Z [INFO] server-73149e1d-7272-ac66-19ad-0af72269f652: entering leader state: leader="Node at 73149e1d-7272-ac66-19ad-0af72269f652 [Leader]" +2025-01-25T14:39:42.589Z [INFO] server-73149e1d-7272-ac66-19ad-0af72269f652: added peer, starting replication: peer=server-24a9fc4a-4d6e-156d-fb35-4d15cd908868 +2025-01-25T14:39:42.590Z [INFO] server-73149e1d-7272-ac66-19ad-0af72269f652: added peer, starting replication: peer=server-d47727c1-6ec3-5bd2-59b2-0624504f41ff +2025-01-25T14:39:42.590Z [INFO] server-73149e1d-7272-ac66-19ad-0af72269f652: pipelining replication: peer="{Voter server-24a9fc4a-4d6e-156d-fb35-4d15cd908868 24a9fc4a-4d6e-156d-fb35-4d15cd908868}" +2025-01-25T14:39:42.590Z [INFO] server-73149e1d-7272-ac66-19ad-0af72269f652: pipelining replication: peer="{Voter server-d47727c1-6ec3-5bd2-59b2-0624504f41ff d47727c1-6ec3-5bd2-59b2-0624504f41ff}" +2025-01-25T14:39:43.595Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:42.590008812 +0000 UTC m=+51.635848130 (1 nodes), 768.018736ms from start of poll, 770.144992ms from cluster start. Timeout at 2025-01-25 14:39:43.595033604 +0000 UTC m=+52.640872922, 1.005024792s after stability +2025-01-25T14:39:43.599Z [INFO] server-73149e1d-7272-ac66-19ad-0af72269f652: starting snapshot up to: index=12 +2025-01-25T14:39:43.599Z [INFO] creating new snapshot: path=/tmp/raft3828748839/snapshots/2-12-1737815983599.tmp +2025-01-25T14:39:43.602Z [INFO] server-73149e1d-7272-ac66-19ad-0af72269f652: snapshot complete up to: index=12 +2025-01-25T14:39:43.610Z [INFO] server-89c48af7-ee4c-ef4e-4787-c3675f3260e9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-89c48af7-ee4c-ef4e-4787-c3675f3260e9 Address:89c48af7-ee4c-ef4e-4787-c3675f3260e9} {Suffrage:Voter ID:server-5ade61e2-caf7-8fa9-884b-578e15d7a0f3 Address:5ade61e2-caf7-8fa9-884b-578e15d7a0f3} {Suffrage:Voter ID:server-ff4e7f20-1639-d0f9-8076-c928907da7c0 Address:ff4e7f20-1639-d0f9-8076-c928907da7c0}]" +2025-01-25T14:39:43.610Z [INFO] server-89c48af7-ee4c-ef4e-4787-c3675f3260e9: entering follower state: follower="Node at 89c48af7-ee4c-ef4e-4787-c3675f3260e9 [Follower]" leader-address= leader-id= +2025-01-25T14:39:43.610Z [INFO] server-5ade61e2-caf7-8fa9-884b-578e15d7a0f3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-89c48af7-ee4c-ef4e-4787-c3675f3260e9 Address:89c48af7-ee4c-ef4e-4787-c3675f3260e9} {Suffrage:Voter ID:server-5ade61e2-caf7-8fa9-884b-578e15d7a0f3 Address:5ade61e2-caf7-8fa9-884b-578e15d7a0f3} {Suffrage:Voter ID:server-ff4e7f20-1639-d0f9-8076-c928907da7c0 Address:ff4e7f20-1639-d0f9-8076-c928907da7c0}]" +2025-01-25T14:39:43.611Z [INFO] server-5ade61e2-caf7-8fa9-884b-578e15d7a0f3: entering follower state: follower="Node at 5ade61e2-caf7-8fa9-884b-578e15d7a0f3 [Follower]" leader-address= leader-id= +2025-01-25T14:39:43.611Z [INFO] server-ff4e7f20-1639-d0f9-8076-c928907da7c0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-89c48af7-ee4c-ef4e-4787-c3675f3260e9 Address:89c48af7-ee4c-ef4e-4787-c3675f3260e9} {Suffrage:Voter ID:server-5ade61e2-caf7-8fa9-884b-578e15d7a0f3 Address:5ade61e2-caf7-8fa9-884b-578e15d7a0f3} {Suffrage:Voter ID:server-ff4e7f20-1639-d0f9-8076-c928907da7c0 Address:ff4e7f20-1639-d0f9-8076-c928907da7c0}]" +2025-01-25T14:39:43.611Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:43.611Z [INFO] server-ff4e7f20-1639-d0f9-8076-c928907da7c0: entering follower state: follower="Node at ff4e7f20-1639-d0f9-8076-c928907da7c0 [Follower]" leader-address= leader-id= +2025-01-25T14:39:44.130Z [WARN] server-5ade61e2-caf7-8fa9-884b-578e15d7a0f3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:44.130Z [INFO] server-5ade61e2-caf7-8fa9-884b-578e15d7a0f3: entering candidate state: node="Node at 5ade61e2-caf7-8fa9-884b-578e15d7a0f3 [Candidate]" term=2 +2025-01-25T14:39:44.131Z [INFO] server-5ade61e2-caf7-8fa9-884b-578e15d7a0f3: election won: term=2 tally=2 +2025-01-25T14:39:44.131Z [INFO] server-5ade61e2-caf7-8fa9-884b-578e15d7a0f3: entering leader state: leader="Node at 5ade61e2-caf7-8fa9-884b-578e15d7a0f3 [Leader]" +2025-01-25T14:39:44.131Z [INFO] server-5ade61e2-caf7-8fa9-884b-578e15d7a0f3: added peer, starting replication: peer=server-89c48af7-ee4c-ef4e-4787-c3675f3260e9 +2025-01-25T14:39:44.131Z [INFO] server-5ade61e2-caf7-8fa9-884b-578e15d7a0f3: added peer, starting replication: peer=server-ff4e7f20-1639-d0f9-8076-c928907da7c0 +2025-01-25T14:39:44.132Z [INFO] server-5ade61e2-caf7-8fa9-884b-578e15d7a0f3: pipelining replication: peer="{Voter server-ff4e7f20-1639-d0f9-8076-c928907da7c0 ff4e7f20-1639-d0f9-8076-c928907da7c0}" +2025-01-25T14:39:44.132Z [INFO] server-5ade61e2-caf7-8fa9-884b-578e15d7a0f3: pipelining replication: peer="{Voter server-89c48af7-ee4c-ef4e-4787-c3675f3260e9 89c48af7-ee4c-ef4e-4787-c3675f3260e9}" +2025-01-25T14:39:45.137Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:44.131753884 +0000 UTC m=+53.177593202 (1 nodes), 519.788347ms from start of poll, 522.328479ms from cluster start. Timeout at 2025-01-25 14:39:45.136761467 +0000 UTC m=+54.182600785, 1.005007583s after stability +2025-01-25T14:39:45.138Z [INFO] creating new snapshot: path=/tmp/raft1990409646/snapshots/2-15-1737815985138.tmp +2025-01-25T14:39:45.140Z [INFO] server-5ade61e2-caf7-8fa9-884b-578e15d7a0f3: copied to local snapshot: bytes=71 +2025-01-25T14:39:45.141Z [INFO] server-5ade61e2-caf7-8fa9-884b-578e15d7a0f3: snapshot restore progress: id=2-15-1737815985138 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:45.142Z [INFO] server-5ade61e2-caf7-8fa9-884b-578e15d7a0f3: restored user snapshot: index=15 +2025-01-25T14:39:45.142Z [ERROR] server-5ade61e2-caf7-8fa9-884b-578e15d7a0f3: failed to get log: index=3 error="log not found" +2025-01-25T14:39:45.142Z [ERROR] server-5ade61e2-caf7-8fa9-884b-578e15d7a0f3: failed to get log: index=3 error="log not found" +2025-01-25T14:39:45.142Z [INFO] server-5ade61e2-caf7-8fa9-884b-578e15d7a0f3: aborting pipeline replication: peer="{Voter server-89c48af7-ee4c-ef4e-4787-c3675f3260e9 89c48af7-ee4c-ef4e-4787-c3675f3260e9}" +2025-01-25T14:39:45.142Z [INFO] server-5ade61e2-caf7-8fa9-884b-578e15d7a0f3: aborting pipeline replication: peer="{Voter server-ff4e7f20-1639-d0f9-8076-c928907da7c0 ff4e7f20-1639-d0f9-8076-c928907da7c0}" +2025-01-25T14:39:45.150Z [ERROR] server-5ade61e2-caf7-8fa9-884b-578e15d7a0f3: failed to get log: index=3 error="log not found" +2025-01-25T14:39:45.151Z [ERROR] server-5ade61e2-caf7-8fa9-884b-578e15d7a0f3: failed to get log: index=3 error="log not found" +2025-01-25T14:39:45.152Z [INFO] creating new snapshot: path=/tmp/raft2411860784/snapshots/2-15-1737815985151.tmp +2025-01-25T14:39:45.153Z [INFO] server-ff4e7f20-1639-d0f9-8076-c928907da7c0: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:45.154Z [INFO] creating new snapshot: path=/tmp/raft1232502272/snapshots/2-15-1737815985154.tmp +2025-01-25T14:39:45.155Z [INFO] server-ff4e7f20-1639-d0f9-8076-c928907da7c0: copied to local snapshot: bytes=71 +2025-01-25T14:39:45.155Z [INFO] server-89c48af7-ee4c-ef4e-4787-c3675f3260e9: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:45.157Z [INFO] server-ff4e7f20-1639-d0f9-8076-c928907da7c0: snapshot restore progress: id=2-15-1737815985151 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:45.157Z [INFO] server-ff4e7f20-1639-d0f9-8076-c928907da7c0: Installed remote snapshot +2025-01-25T14:39:45.158Z [INFO] server-5ade61e2-caf7-8fa9-884b-578e15d7a0f3: pipelining replication: peer="{Voter server-ff4e7f20-1639-d0f9-8076-c928907da7c0 ff4e7f20-1639-d0f9-8076-c928907da7c0}" +2025-01-25T14:39:45.158Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:39:45.161Z [INFO] server-89c48af7-ee4c-ef4e-4787-c3675f3260e9: copied to local snapshot: bytes=71 +2025-01-25T14:39:45.169Z [INFO] server-89c48af7-ee4c-ef4e-4787-c3675f3260e9: snapshot restore progress: id=2-15-1737815985154 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:45.170Z [INFO] server-89c48af7-ee4c-ef4e-4787-c3675f3260e9: Installed remote snapshot +2025-01-25T14:39:45.170Z [INFO] server-5ade61e2-caf7-8fa9-884b-578e15d7a0f3: pipelining replication: peer="{Voter server-89c48af7-ee4c-ef4e-4787-c3675f3260e9 89c48af7-ee4c-ef4e-4787-c3675f3260e9}" +2025-01-25T14:39:46.164Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:39:45.158693666 +0000 UTC m=+54.204532984 (2 nodes), 6.125µs from start of poll, 1.549268261s from cluster start. Timeout at 2025-01-25 14:39:46.163784084 +0000 UTC m=+55.209623401, 1.005090417s after stability +2025-01-25T14:39:46.178Z [INFO] server-73149e1d-7272-ac66-19ad-0af72269f652: aborting pipeline replication: peer="{Voter server-d47727c1-6ec3-5bd2-59b2-0624504f41ff d47727c1-6ec3-5bd2-59b2-0624504f41ff}" +2025-01-25T14:39:46.178Z [INFO] server-73149e1d-7272-ac66-19ad-0af72269f652: aborting pipeline replication: peer="{Voter server-24a9fc4a-4d6e-156d-fb35-4d15cd908868 24a9fc4a-4d6e-156d-fb35-4d15cd908868}" === RUN TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_true -2025-01-25T14:29:01.090Z [INFO] server-81a5ded8-c6bc-e8f7-620f-68829f653f2b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-81a5ded8-c6bc-e8f7-620f-68829f653f2b Address:81a5ded8-c6bc-e8f7-620f-68829f653f2b} {Suffrage:Voter ID:server-6b72411d-f485-bd29-ad17-20e98c399204 Address:6b72411d-f485-bd29-ad17-20e98c399204} {Suffrage:Voter ID:server-7f3bea51-66dc-f7d3-4fa6-21da2d1d0c81 Address:7f3bea51-66dc-f7d3-4fa6-21da2d1d0c81}]" -2025-01-25T14:29:01.090Z [INFO] server-6b72411d-f485-bd29-ad17-20e98c399204: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-81a5ded8-c6bc-e8f7-620f-68829f653f2b Address:81a5ded8-c6bc-e8f7-620f-68829f653f2b} {Suffrage:Voter ID:server-6b72411d-f485-bd29-ad17-20e98c399204 Address:6b72411d-f485-bd29-ad17-20e98c399204} {Suffrage:Voter ID:server-7f3bea51-66dc-f7d3-4fa6-21da2d1d0c81 Address:7f3bea51-66dc-f7d3-4fa6-21da2d1d0c81}]" -2025-01-25T14:29:01.090Z [INFO] server-7f3bea51-66dc-f7d3-4fa6-21da2d1d0c81: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-81a5ded8-c6bc-e8f7-620f-68829f653f2b Address:81a5ded8-c6bc-e8f7-620f-68829f653f2b} {Suffrage:Voter ID:server-6b72411d-f485-bd29-ad17-20e98c399204 Address:6b72411d-f485-bd29-ad17-20e98c399204} {Suffrage:Voter ID:server-7f3bea51-66dc-f7d3-4fa6-21da2d1d0c81 Address:7f3bea51-66dc-f7d3-4fa6-21da2d1d0c81}]" -2025-01-25T14:29:01.090Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:01.095Z [INFO] server-81a5ded8-c6bc-e8f7-620f-68829f653f2b: entering follower state: follower="Node at 81a5ded8-c6bc-e8f7-620f-68829f653f2b [Follower]" leader-address= leader-id= -2025-01-25T14:29:01.095Z [INFO] server-6b72411d-f485-bd29-ad17-20e98c399204: entering follower state: follower="Node at 6b72411d-f485-bd29-ad17-20e98c399204 [Follower]" leader-address= leader-id= -2025-01-25T14:29:01.095Z [INFO] server-7f3bea51-66dc-f7d3-4fa6-21da2d1d0c81: entering follower state: follower="Node at 7f3bea51-66dc-f7d3-4fa6-21da2d1d0c81 [Follower]" leader-address= leader-id= -2025-01-25T14:29:01.695Z [WARN] server-7f3bea51-66dc-f7d3-4fa6-21da2d1d0c81: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:01.695Z [INFO] server-7f3bea51-66dc-f7d3-4fa6-21da2d1d0c81: entering candidate state: node="Node at 7f3bea51-66dc-f7d3-4fa6-21da2d1d0c81 [Candidate]" term=2 -2025-01-25T14:29:01.695Z [INFO] server-7f3bea51-66dc-f7d3-4fa6-21da2d1d0c81: election won: term=2 tally=2 -2025-01-25T14:29:01.695Z [INFO] server-7f3bea51-66dc-f7d3-4fa6-21da2d1d0c81: entering leader state: leader="Node at 7f3bea51-66dc-f7d3-4fa6-21da2d1d0c81 [Leader]" -2025-01-25T14:29:01.695Z [INFO] server-7f3bea51-66dc-f7d3-4fa6-21da2d1d0c81: added peer, starting replication: peer=server-81a5ded8-c6bc-e8f7-620f-68829f653f2b -2025-01-25T14:29:01.695Z [INFO] server-7f3bea51-66dc-f7d3-4fa6-21da2d1d0c81: added peer, starting replication: peer=server-6b72411d-f485-bd29-ad17-20e98c399204 -2025-01-25T14:29:01.695Z [INFO] server-7f3bea51-66dc-f7d3-4fa6-21da2d1d0c81: pipelining replication: peer="{Voter server-6b72411d-f485-bd29-ad17-20e98c399204 6b72411d-f485-bd29-ad17-20e98c399204}" -2025-01-25T14:29:01.695Z [INFO] server-7f3bea51-66dc-f7d3-4fa6-21da2d1d0c81: pipelining replication: peer="{Voter server-81a5ded8-c6bc-e8f7-620f-68829f653f2b 81a5ded8-c6bc-e8f7-620f-68829f653f2b}" -2025-01-25T14:29:02.700Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:01.695723765 +0000 UTC m=+51.550048747 (1 nodes), 604.95899ms from start of poll, 606.36286ms from cluster start. Timeout at 2025-01-25 14:29:02.700805282 +0000 UTC m=+52.555130384, 1.005081637s after stability -2025-01-25T14:29:02.702Z [INFO] server-7f3bea51-66dc-f7d3-4fa6-21da2d1d0c81: starting snapshot up to: index=12 -2025-01-25T14:29:02.702Z [INFO] creating new snapshot: path=/tmp/raft359287863/snapshots/2-12-1737815342702.tmp -2025-01-25T14:29:02.704Z [INFO] server-7f3bea51-66dc-f7d3-4fa6-21da2d1d0c81: snapshot complete up to: index=12 -2025-01-25T14:29:02.707Z [INFO] server-9ea6adc0-08bd-23b6-ada9-e03113001cd6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9ea6adc0-08bd-23b6-ada9-e03113001cd6 Address:9ea6adc0-08bd-23b6-ada9-e03113001cd6} {Suffrage:Voter ID:server-478ea1b6-6ebc-7753-81e5-cd82f2dbbbdb Address:478ea1b6-6ebc-7753-81e5-cd82f2dbbbdb} {Suffrage:Voter ID:server-33ba07e9-155d-372e-58cb-89ce36575d4d Address:33ba07e9-155d-372e-58cb-89ce36575d4d}]" -2025-01-25T14:29:02.708Z [INFO] server-478ea1b6-6ebc-7753-81e5-cd82f2dbbbdb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9ea6adc0-08bd-23b6-ada9-e03113001cd6 Address:9ea6adc0-08bd-23b6-ada9-e03113001cd6} {Suffrage:Voter ID:server-478ea1b6-6ebc-7753-81e5-cd82f2dbbbdb Address:478ea1b6-6ebc-7753-81e5-cd82f2dbbbdb} {Suffrage:Voter ID:server-33ba07e9-155d-372e-58cb-89ce36575d4d Address:33ba07e9-155d-372e-58cb-89ce36575d4d}]" -2025-01-25T14:29:02.708Z [INFO] server-33ba07e9-155d-372e-58cb-89ce36575d4d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9ea6adc0-08bd-23b6-ada9-e03113001cd6 Address:9ea6adc0-08bd-23b6-ada9-e03113001cd6} {Suffrage:Voter ID:server-478ea1b6-6ebc-7753-81e5-cd82f2dbbbdb Address:478ea1b6-6ebc-7753-81e5-cd82f2dbbbdb} {Suffrage:Voter ID:server-33ba07e9-155d-372e-58cb-89ce36575d4d Address:33ba07e9-155d-372e-58cb-89ce36575d4d}]" -2025-01-25T14:29:02.708Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:02.709Z [INFO] server-9ea6adc0-08bd-23b6-ada9-e03113001cd6: entering follower state: follower="Node at 9ea6adc0-08bd-23b6-ada9-e03113001cd6 [Follower]" leader-address= leader-id= -2025-01-25T14:29:02.709Z [INFO] server-478ea1b6-6ebc-7753-81e5-cd82f2dbbbdb: entering follower state: follower="Node at 478ea1b6-6ebc-7753-81e5-cd82f2dbbbdb [Follower]" leader-address= leader-id= -2025-01-25T14:29:02.710Z [INFO] server-33ba07e9-155d-372e-58cb-89ce36575d4d: entering follower state: follower="Node at 33ba07e9-155d-372e-58cb-89ce36575d4d [Follower]" leader-address= leader-id= -2025-01-25T14:29:03.264Z [WARN] server-9ea6adc0-08bd-23b6-ada9-e03113001cd6: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:03.264Z [INFO] server-9ea6adc0-08bd-23b6-ada9-e03113001cd6: entering candidate state: node="Node at 9ea6adc0-08bd-23b6-ada9-e03113001cd6 [Candidate]" term=2 -2025-01-25T14:29:03.264Z [INFO] server-9ea6adc0-08bd-23b6-ada9-e03113001cd6: election won: term=2 tally=2 -2025-01-25T14:29:03.264Z [INFO] server-9ea6adc0-08bd-23b6-ada9-e03113001cd6: entering leader state: leader="Node at 9ea6adc0-08bd-23b6-ada9-e03113001cd6 [Leader]" -2025-01-25T14:29:03.264Z [INFO] server-9ea6adc0-08bd-23b6-ada9-e03113001cd6: added peer, starting replication: peer=server-478ea1b6-6ebc-7753-81e5-cd82f2dbbbdb -2025-01-25T14:29:03.264Z [INFO] server-9ea6adc0-08bd-23b6-ada9-e03113001cd6: added peer, starting replication: peer=server-33ba07e9-155d-372e-58cb-89ce36575d4d -2025-01-25T14:29:03.264Z [INFO] server-9ea6adc0-08bd-23b6-ada9-e03113001cd6: pipelining replication: peer="{Voter server-33ba07e9-155d-372e-58cb-89ce36575d4d 33ba07e9-155d-372e-58cb-89ce36575d4d}" -2025-01-25T14:29:03.265Z [INFO] server-9ea6adc0-08bd-23b6-ada9-e03113001cd6: pipelining replication: peer="{Voter server-478ea1b6-6ebc-7753-81e5-cd82f2dbbbdb 478ea1b6-6ebc-7753-81e5-cd82f2dbbbdb}" -2025-01-25T14:29:04.271Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:03.265123529 +0000 UTC m=+53.119448511 (1 nodes), 556.103348ms from start of poll, 557.651773ms from cluster start. Timeout at 2025-01-25 14:29:04.270145888 +0000 UTC m=+54.124470870, 1.005022359s after stability -2025-01-25T14:29:04.272Z [INFO] creating new snapshot: path=/tmp/raft3325980079/snapshots/2-15-1737815344272.tmp -2025-01-25T14:29:04.273Z [INFO] server-9ea6adc0-08bd-23b6-ada9-e03113001cd6: copied to local snapshot: bytes=71 -2025-01-25T14:29:04.273Z [INFO] server-9ea6adc0-08bd-23b6-ada9-e03113001cd6: snapshot restore progress: id=2-15-1737815344272 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:04.273Z [INFO] server-9ea6adc0-08bd-23b6-ada9-e03113001cd6: removing all old logs from log store -2025-01-25T14:29:04.273Z [INFO] server-9ea6adc0-08bd-23b6-ada9-e03113001cd6: compacting logs: from=1 to=2 -2025-01-25T14:29:04.273Z [INFO] server-9ea6adc0-08bd-23b6-ada9-e03113001cd6: restored user snapshot: index=15 -2025-01-25T14:29:04.273Z [ERROR] server-9ea6adc0-08bd-23b6-ada9-e03113001cd6: failed to get log: index=2 error="log not found" -2025-01-25T14:29:04.273Z [INFO] server-9ea6adc0-08bd-23b6-ada9-e03113001cd6: aborting pipeline replication: peer="{Voter server-33ba07e9-155d-372e-58cb-89ce36575d4d 33ba07e9-155d-372e-58cb-89ce36575d4d}" -2025-01-25T14:29:04.273Z [ERROR] server-9ea6adc0-08bd-23b6-ada9-e03113001cd6: failed to get log: index=2 error="log not found" -2025-01-25T14:29:04.274Z [INFO] server-9ea6adc0-08bd-23b6-ada9-e03113001cd6: aborting pipeline replication: peer="{Voter server-478ea1b6-6ebc-7753-81e5-cd82f2dbbbdb 478ea1b6-6ebc-7753-81e5-cd82f2dbbbdb}" -2025-01-25T14:29:04.281Z [ERROR] server-9ea6adc0-08bd-23b6-ada9-e03113001cd6: failed to get log: index=2 error="log not found" -2025-01-25T14:29:04.282Z [INFO] creating new snapshot: path=/tmp/raft2141987800/snapshots/2-15-1737815344282.tmp -2025-01-25T14:29:04.282Z [INFO] server-33ba07e9-155d-372e-58cb-89ce36575d4d: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:04.283Z [ERROR] server-9ea6adc0-08bd-23b6-ada9-e03113001cd6: failed to get log: index=2 error="log not found" -2025-01-25T14:29:04.283Z [INFO] server-33ba07e9-155d-372e-58cb-89ce36575d4d: copied to local snapshot: bytes=71 -2025-01-25T14:29:04.283Z [INFO] creating new snapshot: path=/tmp/raft3942468882/snapshots/2-15-1737815344283.tmp -2025-01-25T14:29:04.284Z [INFO] server-33ba07e9-155d-372e-58cb-89ce36575d4d: snapshot restore progress: id=2-15-1737815344282 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:04.284Z [INFO] server-33ba07e9-155d-372e-58cb-89ce36575d4d: removing all old logs from log store -2025-01-25T14:29:04.284Z [INFO] server-33ba07e9-155d-372e-58cb-89ce36575d4d: compacting logs: from=1 to=2 -2025-01-25T14:29:04.284Z [INFO] server-33ba07e9-155d-372e-58cb-89ce36575d4d: Installed remote snapshot -2025-01-25T14:29:04.284Z [INFO] server-9ea6adc0-08bd-23b6-ada9-e03113001cd6: pipelining replication: peer="{Voter server-33ba07e9-155d-372e-58cb-89ce36575d4d 33ba07e9-155d-372e-58cb-89ce36575d4d}" -2025-01-25T14:29:04.284Z [INFO] server-478ea1b6-6ebc-7753-81e5-cd82f2dbbbdb: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:04.284Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:29:04.284Z [INFO] server-478ea1b6-6ebc-7753-81e5-cd82f2dbbbdb: copied to local snapshot: bytes=71 -2025-01-25T14:29:04.285Z [INFO] server-478ea1b6-6ebc-7753-81e5-cd82f2dbbbdb: snapshot restore progress: id=2-15-1737815344283 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:04.285Z [INFO] server-478ea1b6-6ebc-7753-81e5-cd82f2dbbbdb: removing all old logs from log store -2025-01-25T14:29:04.285Z [INFO] server-478ea1b6-6ebc-7753-81e5-cd82f2dbbbdb: compacting logs: from=1 to=2 -2025-01-25T14:29:04.285Z [INFO] server-478ea1b6-6ebc-7753-81e5-cd82f2dbbbdb: Installed remote snapshot -2025-01-25T14:29:04.285Z [INFO] server-9ea6adc0-08bd-23b6-ada9-e03113001cd6: pipelining replication: peer="{Voter server-478ea1b6-6ebc-7753-81e5-cd82f2dbbbdb 478ea1b6-6ebc-7753-81e5-cd82f2dbbbdb}" -2025-01-25T14:29:05.290Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:29:04.284499286 +0000 UTC m=+54.138824268 (2 nodes), 1.6µs from start of poll, 1.57702753s from cluster start. Timeout at 2025-01-25 14:29:05.289619722 +0000 UTC m=+55.143944664, 1.005120396s after stability -2025-01-25T14:29:05.302Z [INFO] server-7f3bea51-66dc-f7d3-4fa6-21da2d1d0c81: aborting pipeline replication: peer="{Voter server-6b72411d-f485-bd29-ad17-20e98c399204 6b72411d-f485-bd29-ad17-20e98c399204}" -2025-01-25T14:29:05.303Z [INFO] server-7f3bea51-66dc-f7d3-4fa6-21da2d1d0c81: aborting pipeline replication: peer="{Voter server-81a5ded8-c6bc-e8f7-620f-68829f653f2b 81a5ded8-c6bc-e8f7-620f-68829f653f2b}" +2025-01-25T14:39:46.187Z [INFO] server-c91f2fbc-0a2e-6899-9a36-28cf7f91cd9a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c91f2fbc-0a2e-6899-9a36-28cf7f91cd9a Address:c91f2fbc-0a2e-6899-9a36-28cf7f91cd9a} {Suffrage:Voter ID:server-e4b86bdd-c1ed-f7a0-aa03-b8450862347d Address:e4b86bdd-c1ed-f7a0-aa03-b8450862347d} {Suffrage:Voter ID:server-2d2cce3d-ed68-627e-cc69-afcaa7c4e895 Address:2d2cce3d-ed68-627e-cc69-afcaa7c4e895}]" +2025-01-25T14:39:46.188Z [INFO] server-c91f2fbc-0a2e-6899-9a36-28cf7f91cd9a: entering follower state: follower="Node at c91f2fbc-0a2e-6899-9a36-28cf7f91cd9a [Follower]" leader-address= leader-id= +2025-01-25T14:39:46.188Z [INFO] server-e4b86bdd-c1ed-f7a0-aa03-b8450862347d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c91f2fbc-0a2e-6899-9a36-28cf7f91cd9a Address:c91f2fbc-0a2e-6899-9a36-28cf7f91cd9a} {Suffrage:Voter ID:server-e4b86bdd-c1ed-f7a0-aa03-b8450862347d Address:e4b86bdd-c1ed-f7a0-aa03-b8450862347d} {Suffrage:Voter ID:server-2d2cce3d-ed68-627e-cc69-afcaa7c4e895 Address:2d2cce3d-ed68-627e-cc69-afcaa7c4e895}]" +2025-01-25T14:39:46.189Z [INFO] server-e4b86bdd-c1ed-f7a0-aa03-b8450862347d: entering follower state: follower="Node at e4b86bdd-c1ed-f7a0-aa03-b8450862347d [Follower]" leader-address= leader-id= +2025-01-25T14:39:46.189Z [INFO] server-2d2cce3d-ed68-627e-cc69-afcaa7c4e895: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c91f2fbc-0a2e-6899-9a36-28cf7f91cd9a Address:c91f2fbc-0a2e-6899-9a36-28cf7f91cd9a} {Suffrage:Voter ID:server-e4b86bdd-c1ed-f7a0-aa03-b8450862347d Address:e4b86bdd-c1ed-f7a0-aa03-b8450862347d} {Suffrage:Voter ID:server-2d2cce3d-ed68-627e-cc69-afcaa7c4e895 Address:2d2cce3d-ed68-627e-cc69-afcaa7c4e895}]" +2025-01-25T14:39:46.190Z [INFO] server-2d2cce3d-ed68-627e-cc69-afcaa7c4e895: entering follower state: follower="Node at 2d2cce3d-ed68-627e-cc69-afcaa7c4e895 [Follower]" leader-address= leader-id= +2025-01-25T14:39:46.190Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:46.789Z [WARN] server-e4b86bdd-c1ed-f7a0-aa03-b8450862347d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:46.789Z [INFO] server-e4b86bdd-c1ed-f7a0-aa03-b8450862347d: entering candidate state: node="Node at e4b86bdd-c1ed-f7a0-aa03-b8450862347d [Candidate]" term=2 +2025-01-25T14:39:46.789Z [INFO] server-e4b86bdd-c1ed-f7a0-aa03-b8450862347d: election won: term=2 tally=2 +2025-01-25T14:39:46.789Z [INFO] server-e4b86bdd-c1ed-f7a0-aa03-b8450862347d: entering leader state: leader="Node at e4b86bdd-c1ed-f7a0-aa03-b8450862347d [Leader]" +2025-01-25T14:39:46.789Z [INFO] server-e4b86bdd-c1ed-f7a0-aa03-b8450862347d: added peer, starting replication: peer=server-c91f2fbc-0a2e-6899-9a36-28cf7f91cd9a +2025-01-25T14:39:46.790Z [INFO] server-e4b86bdd-c1ed-f7a0-aa03-b8450862347d: added peer, starting replication: peer=server-2d2cce3d-ed68-627e-cc69-afcaa7c4e895 +2025-01-25T14:39:46.792Z [INFO] server-e4b86bdd-c1ed-f7a0-aa03-b8450862347d: pipelining replication: peer="{Voter server-c91f2fbc-0a2e-6899-9a36-28cf7f91cd9a c91f2fbc-0a2e-6899-9a36-28cf7f91cd9a}" +2025-01-25T14:39:46.797Z [INFO] server-e4b86bdd-c1ed-f7a0-aa03-b8450862347d: pipelining replication: peer="{Voter server-2d2cce3d-ed68-627e-cc69-afcaa7c4e895 2d2cce3d-ed68-627e-cc69-afcaa7c4e895}" +2025-01-25T14:39:47.795Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:46.789983374 +0000 UTC m=+55.835822691 (1 nodes), 599.454412ms from start of poll, 602.812088ms from cluster start. Timeout at 2025-01-25 14:39:47.795000582 +0000 UTC m=+56.840839899, 1.005017208s after stability +2025-01-25T14:39:47.799Z [INFO] server-e4b86bdd-c1ed-f7a0-aa03-b8450862347d: starting snapshot up to: index=12 +2025-01-25T14:39:47.799Z [INFO] creating new snapshot: path=/tmp/raft2266289093/snapshots/2-12-1737815987799.tmp +2025-01-25T14:39:47.801Z [INFO] server-e4b86bdd-c1ed-f7a0-aa03-b8450862347d: snapshot complete up to: index=12 +2025-01-25T14:39:47.807Z [INFO] server-47d5cd55-3616-dcd4-a34f-7222b80160a3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-47d5cd55-3616-dcd4-a34f-7222b80160a3 Address:47d5cd55-3616-dcd4-a34f-7222b80160a3} {Suffrage:Voter ID:server-abbb3c10-2846-f953-0a69-e233412e2b88 Address:abbb3c10-2846-f953-0a69-e233412e2b88} {Suffrage:Voter ID:server-fa94a9a4-9f2d-2311-d303-3694ed07d55f Address:fa94a9a4-9f2d-2311-d303-3694ed07d55f}]" +2025-01-25T14:39:47.807Z [INFO] server-47d5cd55-3616-dcd4-a34f-7222b80160a3: entering follower state: follower="Node at 47d5cd55-3616-dcd4-a34f-7222b80160a3 [Follower]" leader-address= leader-id= +2025-01-25T14:39:47.808Z [INFO] server-abbb3c10-2846-f953-0a69-e233412e2b88: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-47d5cd55-3616-dcd4-a34f-7222b80160a3 Address:47d5cd55-3616-dcd4-a34f-7222b80160a3} {Suffrage:Voter ID:server-abbb3c10-2846-f953-0a69-e233412e2b88 Address:abbb3c10-2846-f953-0a69-e233412e2b88} {Suffrage:Voter ID:server-fa94a9a4-9f2d-2311-d303-3694ed07d55f Address:fa94a9a4-9f2d-2311-d303-3694ed07d55f}]" +2025-01-25T14:39:47.808Z [INFO] server-abbb3c10-2846-f953-0a69-e233412e2b88: entering follower state: follower="Node at abbb3c10-2846-f953-0a69-e233412e2b88 [Follower]" leader-address= leader-id= +2025-01-25T14:39:47.808Z [INFO] server-fa94a9a4-9f2d-2311-d303-3694ed07d55f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-47d5cd55-3616-dcd4-a34f-7222b80160a3 Address:47d5cd55-3616-dcd4-a34f-7222b80160a3} {Suffrage:Voter ID:server-abbb3c10-2846-f953-0a69-e233412e2b88 Address:abbb3c10-2846-f953-0a69-e233412e2b88} {Suffrage:Voter ID:server-fa94a9a4-9f2d-2311-d303-3694ed07d55f Address:fa94a9a4-9f2d-2311-d303-3694ed07d55f}]" +2025-01-25T14:39:47.809Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:47.809Z [INFO] server-fa94a9a4-9f2d-2311-d303-3694ed07d55f: entering follower state: follower="Node at fa94a9a4-9f2d-2311-d303-3694ed07d55f [Follower]" leader-address= leader-id= +2025-01-25T14:39:48.311Z [WARN] server-abbb3c10-2846-f953-0a69-e233412e2b88: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:48.311Z [INFO] server-abbb3c10-2846-f953-0a69-e233412e2b88: entering candidate state: node="Node at abbb3c10-2846-f953-0a69-e233412e2b88 [Candidate]" term=2 +2025-01-25T14:39:48.312Z [INFO] server-abbb3c10-2846-f953-0a69-e233412e2b88: election won: term=2 tally=2 +2025-01-25T14:39:48.312Z [INFO] server-abbb3c10-2846-f953-0a69-e233412e2b88: entering leader state: leader="Node at abbb3c10-2846-f953-0a69-e233412e2b88 [Leader]" +2025-01-25T14:39:48.312Z [INFO] server-abbb3c10-2846-f953-0a69-e233412e2b88: added peer, starting replication: peer=server-47d5cd55-3616-dcd4-a34f-7222b80160a3 +2025-01-25T14:39:48.312Z [INFO] server-abbb3c10-2846-f953-0a69-e233412e2b88: added peer, starting replication: peer=server-fa94a9a4-9f2d-2311-d303-3694ed07d55f +2025-01-25T14:39:48.313Z [INFO] server-abbb3c10-2846-f953-0a69-e233412e2b88: pipelining replication: peer="{Voter server-47d5cd55-3616-dcd4-a34f-7222b80160a3 47d5cd55-3616-dcd4-a34f-7222b80160a3}" +2025-01-25T14:39:48.313Z [INFO] server-abbb3c10-2846-f953-0a69-e233412e2b88: pipelining replication: peer="{Voter server-fa94a9a4-9f2d-2311-d303-3694ed07d55f fa94a9a4-9f2d-2311-d303-3694ed07d55f}" +2025-01-25T14:39:49.317Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:48.31251989 +0000 UTC m=+57.358359207 (1 nodes), 503.370965ms from start of poll, 505.17172ms from cluster start. Timeout at 2025-01-25 14:39:49.31756539 +0000 UTC m=+58.363404708, 1.005045501s after stability +2025-01-25T14:39:49.319Z [INFO] creating new snapshot: path=/tmp/raft376900651/snapshots/2-15-1737815989319.tmp +2025-01-25T14:39:49.323Z [INFO] server-abbb3c10-2846-f953-0a69-e233412e2b88: copied to local snapshot: bytes=71 +2025-01-25T14:39:49.324Z [INFO] server-abbb3c10-2846-f953-0a69-e233412e2b88: snapshot restore progress: id=2-15-1737815989319 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:49.324Z [INFO] server-abbb3c10-2846-f953-0a69-e233412e2b88: removing all old logs from log store +2025-01-25T14:39:49.324Z [INFO] server-abbb3c10-2846-f953-0a69-e233412e2b88: compacting logs: from=1 to=2 +2025-01-25T14:39:49.325Z [INFO] server-abbb3c10-2846-f953-0a69-e233412e2b88: restored user snapshot: index=15 +2025-01-25T14:39:49.325Z [ERROR] server-abbb3c10-2846-f953-0a69-e233412e2b88: failed to get log: index=2 error="log not found" +2025-01-25T14:39:49.325Z [ERROR] server-abbb3c10-2846-f953-0a69-e233412e2b88: failed to get log: index=2 error="log not found" +2025-01-25T14:39:49.325Z [INFO] server-abbb3c10-2846-f953-0a69-e233412e2b88: aborting pipeline replication: peer="{Voter server-47d5cd55-3616-dcd4-a34f-7222b80160a3 47d5cd55-3616-dcd4-a34f-7222b80160a3}" +2025-01-25T14:39:49.325Z [INFO] server-abbb3c10-2846-f953-0a69-e233412e2b88: aborting pipeline replication: peer="{Voter server-fa94a9a4-9f2d-2311-d303-3694ed07d55f fa94a9a4-9f2d-2311-d303-3694ed07d55f}" +2025-01-25T14:39:49.325Z [ERROR] server-abbb3c10-2846-f953-0a69-e233412e2b88: failed to get log: index=2 error="log not found" +2025-01-25T14:39:49.327Z [INFO] creating new snapshot: path=/tmp/raft3439771974/snapshots/2-15-1737815989327.tmp +2025-01-25T14:39:49.330Z [INFO] server-47d5cd55-3616-dcd4-a34f-7222b80160a3: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:49.331Z [ERROR] server-abbb3c10-2846-f953-0a69-e233412e2b88: failed to get log: index=2 error="log not found" +2025-01-25T14:39:49.332Z [INFO] server-47d5cd55-3616-dcd4-a34f-7222b80160a3: copied to local snapshot: bytes=71 +2025-01-25T14:39:49.333Z [INFO] creating new snapshot: path=/tmp/raft1647242486/snapshots/2-15-1737815989333.tmp +2025-01-25T14:39:49.334Z [INFO] server-fa94a9a4-9f2d-2311-d303-3694ed07d55f: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:49.334Z [INFO] server-47d5cd55-3616-dcd4-a34f-7222b80160a3: snapshot restore progress: id=2-15-1737815989327 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:49.335Z [INFO] server-47d5cd55-3616-dcd4-a34f-7222b80160a3: removing all old logs from log store +2025-01-25T14:39:49.335Z [INFO] server-47d5cd55-3616-dcd4-a34f-7222b80160a3: compacting logs: from=1 to=2 +2025-01-25T14:39:49.335Z [INFO] server-47d5cd55-3616-dcd4-a34f-7222b80160a3: Installed remote snapshot +2025-01-25T14:39:49.335Z [INFO] server-abbb3c10-2846-f953-0a69-e233412e2b88: pipelining replication: peer="{Voter server-47d5cd55-3616-dcd4-a34f-7222b80160a3 47d5cd55-3616-dcd4-a34f-7222b80160a3}" +2025-01-25T14:39:49.336Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:39:49.337Z [INFO] server-fa94a9a4-9f2d-2311-d303-3694ed07d55f: copied to local snapshot: bytes=71 +2025-01-25T14:39:49.338Z [INFO] server-fa94a9a4-9f2d-2311-d303-3694ed07d55f: snapshot restore progress: id=2-15-1737815989333 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:49.338Z [INFO] server-fa94a9a4-9f2d-2311-d303-3694ed07d55f: removing all old logs from log store +2025-01-25T14:39:49.338Z [INFO] server-fa94a9a4-9f2d-2311-d303-3694ed07d55f: compacting logs: from=1 to=2 +2025-01-25T14:39:49.338Z [INFO] server-fa94a9a4-9f2d-2311-d303-3694ed07d55f: Installed remote snapshot +2025-01-25T14:39:49.339Z [INFO] server-abbb3c10-2846-f953-0a69-e233412e2b88: pipelining replication: peer="{Voter server-fa94a9a4-9f2d-2311-d303-3694ed07d55f fa94a9a4-9f2d-2311-d303-3694ed07d55f}" +2025-01-25T14:39:50.341Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:39:49.336238203 +0000 UTC m=+58.382077521 (2 nodes), 6.708µs from start of poll, 1.528890034s from cluster start. Timeout at 2025-01-25 14:39:50.341334162 +0000 UTC m=+59.387173772, 1.005096251s after stability +2025-01-25T14:39:50.358Z [INFO] server-e4b86bdd-c1ed-f7a0-aa03-b8450862347d: aborting pipeline replication: peer="{Voter server-c91f2fbc-0a2e-6899-9a36-28cf7f91cd9a c91f2fbc-0a2e-6899-9a36-28cf7f91cd9a}" +2025-01-25T14:39:50.358Z [INFO] server-e4b86bdd-c1ed-f7a0-aa03-b8450862347d: aborting pipeline replication: peer="{Voter server-2d2cce3d-ed68-627e-cc69-afcaa7c4e895 2d2cce3d-ed68-627e-cc69-afcaa7c4e895}" === RUN TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_false -2025-01-25T14:29:05.305Z [INFO] server-60a0aeb6-2bbb-e914-e26b-58c2a774307f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-60a0aeb6-2bbb-e914-e26b-58c2a774307f Address:60a0aeb6-2bbb-e914-e26b-58c2a774307f} {Suffrage:Voter ID:server-8f192e0d-4f48-540f-b398-63441088583c Address:8f192e0d-4f48-540f-b398-63441088583c} {Suffrage:Voter ID:server-0fdc1750-f512-61e0-7d96-61c5d384ae0a Address:0fdc1750-f512-61e0-7d96-61c5d384ae0a}]" -2025-01-25T14:29:05.305Z [INFO] server-8f192e0d-4f48-540f-b398-63441088583c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-60a0aeb6-2bbb-e914-e26b-58c2a774307f Address:60a0aeb6-2bbb-e914-e26b-58c2a774307f} {Suffrage:Voter ID:server-8f192e0d-4f48-540f-b398-63441088583c Address:8f192e0d-4f48-540f-b398-63441088583c} {Suffrage:Voter ID:server-0fdc1750-f512-61e0-7d96-61c5d384ae0a Address:0fdc1750-f512-61e0-7d96-61c5d384ae0a}]" -2025-01-25T14:29:05.306Z [INFO] server-0fdc1750-f512-61e0-7d96-61c5d384ae0a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-60a0aeb6-2bbb-e914-e26b-58c2a774307f Address:60a0aeb6-2bbb-e914-e26b-58c2a774307f} {Suffrage:Voter ID:server-8f192e0d-4f48-540f-b398-63441088583c Address:8f192e0d-4f48-540f-b398-63441088583c} {Suffrage:Voter ID:server-0fdc1750-f512-61e0-7d96-61c5d384ae0a Address:0fdc1750-f512-61e0-7d96-61c5d384ae0a}]" -2025-01-25T14:29:05.306Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:05.306Z [INFO] server-60a0aeb6-2bbb-e914-e26b-58c2a774307f: entering follower state: follower="Node at 60a0aeb6-2bbb-e914-e26b-58c2a774307f [Follower]" leader-address= leader-id= -2025-01-25T14:29:05.306Z [INFO] server-0fdc1750-f512-61e0-7d96-61c5d384ae0a: entering follower state: follower="Node at 0fdc1750-f512-61e0-7d96-61c5d384ae0a [Follower]" leader-address= leader-id= -2025-01-25T14:29:05.306Z [INFO] server-8f192e0d-4f48-540f-b398-63441088583c: entering follower state: follower="Node at 8f192e0d-4f48-540f-b398-63441088583c [Follower]" leader-address= leader-id= -2025-01-25T14:29:05.872Z [WARN] server-60a0aeb6-2bbb-e914-e26b-58c2a774307f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:05.872Z [INFO] server-60a0aeb6-2bbb-e914-e26b-58c2a774307f: entering candidate state: node="Node at 60a0aeb6-2bbb-e914-e26b-58c2a774307f [Candidate]" term=2 -2025-01-25T14:29:05.872Z [INFO] server-60a0aeb6-2bbb-e914-e26b-58c2a774307f: election won: term=2 tally=2 -2025-01-25T14:29:05.872Z [INFO] server-60a0aeb6-2bbb-e914-e26b-58c2a774307f: entering leader state: leader="Node at 60a0aeb6-2bbb-e914-e26b-58c2a774307f [Leader]" -2025-01-25T14:29:05.872Z [INFO] server-60a0aeb6-2bbb-e914-e26b-58c2a774307f: added peer, starting replication: peer=server-8f192e0d-4f48-540f-b398-63441088583c -2025-01-25T14:29:05.872Z [INFO] server-60a0aeb6-2bbb-e914-e26b-58c2a774307f: added peer, starting replication: peer=server-0fdc1750-f512-61e0-7d96-61c5d384ae0a -2025-01-25T14:29:05.872Z [INFO] server-60a0aeb6-2bbb-e914-e26b-58c2a774307f: pipelining replication: peer="{Voter server-8f192e0d-4f48-540f-b398-63441088583c 8f192e0d-4f48-540f-b398-63441088583c}" -2025-01-25T14:29:05.872Z [INFO] server-60a0aeb6-2bbb-e914-e26b-58c2a774307f: pipelining replication: peer="{Voter server-0fdc1750-f512-61e0-7d96-61c5d384ae0a 0fdc1750-f512-61e0-7d96-61c5d384ae0a}" -2025-01-25T14:29:06.878Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:05.872391984 +0000 UTC m=+55.726716966 (1 nodes), 566.185825ms from start of poll, 566.88652ms from cluster start. Timeout at 2025-01-25 14:29:06.877396344 +0000 UTC m=+56.731721326, 1.00500436s after stability -2025-01-25T14:29:06.879Z [INFO] server-60a0aeb6-2bbb-e914-e26b-58c2a774307f: starting snapshot up to: index=12 -2025-01-25T14:29:06.879Z [INFO] creating new snapshot: path=/tmp/raft1578067068/snapshots/2-12-1737815346879.tmp -2025-01-25T14:29:06.883Z [INFO] server-60a0aeb6-2bbb-e914-e26b-58c2a774307f: snapshot complete up to: index=12 -2025-01-25T14:29:06.885Z [INFO] creating new snapshot: path=/tmp/raft1578067068/snapshots/2-113-1737815346885.tmp -2025-01-25T14:29:06.886Z [INFO] server-60a0aeb6-2bbb-e914-e26b-58c2a774307f: copied to local snapshot: bytes=71 -2025-01-25T14:29:06.886Z [INFO] server-60a0aeb6-2bbb-e914-e26b-58c2a774307f: snapshot restore progress: id=2-113-1737815346885 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:06.886Z [INFO] server-60a0aeb6-2bbb-e914-e26b-58c2a774307f: restored user snapshot: index=113 -2025-01-25T14:29:06.886Z [ERROR] server-60a0aeb6-2bbb-e914-e26b-58c2a774307f: failed to get log: index=23 error="log not found" -2025-01-25T14:29:06.886Z [INFO] server-60a0aeb6-2bbb-e914-e26b-58c2a774307f: aborting pipeline replication: peer="{Voter server-0fdc1750-f512-61e0-7d96-61c5d384ae0a 0fdc1750-f512-61e0-7d96-61c5d384ae0a}" -2025-01-25T14:29:06.886Z [ERROR] server-60a0aeb6-2bbb-e914-e26b-58c2a774307f: failed to get log: index=23 error="log not found" -2025-01-25T14:29:06.887Z [INFO] server-60a0aeb6-2bbb-e914-e26b-58c2a774307f: aborting pipeline replication: peer="{Voter server-8f192e0d-4f48-540f-b398-63441088583c 8f192e0d-4f48-540f-b398-63441088583c}" -2025-01-25T14:29:06.893Z [ERROR] server-60a0aeb6-2bbb-e914-e26b-58c2a774307f: failed to get log: index=23 error="log not found" -2025-01-25T14:29:06.894Z [INFO] creating new snapshot: path=/tmp/raft1507187908/snapshots/2-113-1737815346894.tmp -2025-01-25T14:29:06.894Z [INFO] server-8f192e0d-4f48-540f-b398-63441088583c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:06.895Z [INFO] server-8f192e0d-4f48-540f-b398-63441088583c: copied to local snapshot: bytes=71 -2025-01-25T14:29:06.895Z [ERROR] server-60a0aeb6-2bbb-e914-e26b-58c2a774307f: failed to get log: index=23 error="log not found" -2025-01-25T14:29:06.895Z [INFO] server-8f192e0d-4f48-540f-b398-63441088583c: snapshot restore progress: id=2-113-1737815346894 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:06.896Z [INFO] server-8f192e0d-4f48-540f-b398-63441088583c: Installed remote snapshot -2025-01-25T14:29:06.896Z [WARN] server-8f192e0d-4f48-540f-b398-63441088583c: skipping application of old log: index=22 -2025-01-25T14:29:06.896Z [INFO] server-60a0aeb6-2bbb-e914-e26b-58c2a774307f: pipelining replication: peer="{Voter server-8f192e0d-4f48-540f-b398-63441088583c 8f192e0d-4f48-540f-b398-63441088583c}" -2025-01-25T14:29:06.896Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:29:06.896Z [INFO] creating new snapshot: path=/tmp/raft1148884927/snapshots/2-113-1737815346896.tmp -2025-01-25T14:29:06.897Z [INFO] server-0fdc1750-f512-61e0-7d96-61c5d384ae0a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:06.897Z [INFO] server-0fdc1750-f512-61e0-7d96-61c5d384ae0a: copied to local snapshot: bytes=71 -2025-01-25T14:29:06.898Z [INFO] server-0fdc1750-f512-61e0-7d96-61c5d384ae0a: snapshot restore progress: id=2-113-1737815346896 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:06.898Z [INFO] server-0fdc1750-f512-61e0-7d96-61c5d384ae0a: Installed remote snapshot -2025-01-25T14:29:06.899Z [INFO] server-60a0aeb6-2bbb-e914-e26b-58c2a774307f: pipelining replication: peer="{Voter server-0fdc1750-f512-61e0-7d96-61c5d384ae0a 0fdc1750-f512-61e0-7d96-61c5d384ae0a}" -2025-01-25T14:29:07.901Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:29:06.896420374 +0000 UTC m=+56.750745356 (2 nodes), 1.92µs from start of poll, 1.59091491s from cluster start. Timeout at 2025-01-25 14:29:07.90153801 +0000 UTC m=+57.755862992, 1.005117636s after stability -2025-01-25T14:29:07.913Z [INFO] server-60a0aeb6-2bbb-e914-e26b-58c2a774307f: aborting pipeline replication: peer="{Voter server-0fdc1750-f512-61e0-7d96-61c5d384ae0a 0fdc1750-f512-61e0-7d96-61c5d384ae0a}" -2025-01-25T14:29:07.913Z [INFO] server-60a0aeb6-2bbb-e914-e26b-58c2a774307f: aborting pipeline replication: peer="{Voter server-8f192e0d-4f48-540f-b398-63441088583c 8f192e0d-4f48-540f-b398-63441088583c}" +2025-01-25T14:39:50.380Z [INFO] server-eb84812c-55a3-719a-9f06-b8f72a15555c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eb84812c-55a3-719a-9f06-b8f72a15555c Address:eb84812c-55a3-719a-9f06-b8f72a15555c} {Suffrage:Voter ID:server-7fd4b08d-76f7-1dc3-c566-196de3c90fe5 Address:7fd4b08d-76f7-1dc3-c566-196de3c90fe5} {Suffrage:Voter ID:server-8187e575-a308-4f2a-1ccc-ca501e6657d9 Address:8187e575-a308-4f2a-1ccc-ca501e6657d9}]" +2025-01-25T14:39:50.385Z [INFO] server-eb84812c-55a3-719a-9f06-b8f72a15555c: entering follower state: follower="Node at eb84812c-55a3-719a-9f06-b8f72a15555c [Follower]" leader-address= leader-id= +2025-01-25T14:39:50.391Z [INFO] server-7fd4b08d-76f7-1dc3-c566-196de3c90fe5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eb84812c-55a3-719a-9f06-b8f72a15555c Address:eb84812c-55a3-719a-9f06-b8f72a15555c} {Suffrage:Voter ID:server-7fd4b08d-76f7-1dc3-c566-196de3c90fe5 Address:7fd4b08d-76f7-1dc3-c566-196de3c90fe5} {Suffrage:Voter ID:server-8187e575-a308-4f2a-1ccc-ca501e6657d9 Address:8187e575-a308-4f2a-1ccc-ca501e6657d9}]" +2025-01-25T14:39:50.391Z [INFO] server-7fd4b08d-76f7-1dc3-c566-196de3c90fe5: entering follower state: follower="Node at 7fd4b08d-76f7-1dc3-c566-196de3c90fe5 [Follower]" leader-address= leader-id= +2025-01-25T14:39:50.392Z [INFO] server-8187e575-a308-4f2a-1ccc-ca501e6657d9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eb84812c-55a3-719a-9f06-b8f72a15555c Address:eb84812c-55a3-719a-9f06-b8f72a15555c} {Suffrage:Voter ID:server-7fd4b08d-76f7-1dc3-c566-196de3c90fe5 Address:7fd4b08d-76f7-1dc3-c566-196de3c90fe5} {Suffrage:Voter ID:server-8187e575-a308-4f2a-1ccc-ca501e6657d9 Address:8187e575-a308-4f2a-1ccc-ca501e6657d9}]" +2025-01-25T14:39:50.392Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:50.392Z [INFO] server-8187e575-a308-4f2a-1ccc-ca501e6657d9: entering follower state: follower="Node at 8187e575-a308-4f2a-1ccc-ca501e6657d9 [Follower]" leader-address= leader-id= +2025-01-25T14:39:51.009Z [WARN] server-eb84812c-55a3-719a-9f06-b8f72a15555c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:51.009Z [INFO] server-eb84812c-55a3-719a-9f06-b8f72a15555c: entering candidate state: node="Node at eb84812c-55a3-719a-9f06-b8f72a15555c [Candidate]" term=2 +2025-01-25T14:39:51.016Z [INFO] server-eb84812c-55a3-719a-9f06-b8f72a15555c: election won: term=2 tally=2 +2025-01-25T14:39:51.016Z [INFO] server-eb84812c-55a3-719a-9f06-b8f72a15555c: entering leader state: leader="Node at eb84812c-55a3-719a-9f06-b8f72a15555c [Leader]" +2025-01-25T14:39:51.017Z [INFO] server-eb84812c-55a3-719a-9f06-b8f72a15555c: added peer, starting replication: peer=server-7fd4b08d-76f7-1dc3-c566-196de3c90fe5 +2025-01-25T14:39:51.017Z [INFO] server-eb84812c-55a3-719a-9f06-b8f72a15555c: added peer, starting replication: peer=server-8187e575-a308-4f2a-1ccc-ca501e6657d9 +2025-01-25T14:39:51.022Z [INFO] server-eb84812c-55a3-719a-9f06-b8f72a15555c: pipelining replication: peer="{Voter server-7fd4b08d-76f7-1dc3-c566-196de3c90fe5 7fd4b08d-76f7-1dc3-c566-196de3c90fe5}" +2025-01-25T14:39:51.022Z [INFO] server-eb84812c-55a3-719a-9f06-b8f72a15555c: pipelining replication: peer="{Voter server-8187e575-a308-4f2a-1ccc-ca501e6657d9 8187e575-a308-4f2a-1ccc-ca501e6657d9}" +2025-01-25T14:39:52.028Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:51.022043653 +0000 UTC m=+60.067882971 (1 nodes), 629.609833ms from start of poll, 654.420239ms from cluster start. Timeout at 2025-01-25 14:39:52.027193862 +0000 UTC m=+61.073033471, 1.0051505s after stability +2025-01-25T14:39:52.031Z [INFO] server-eb84812c-55a3-719a-9f06-b8f72a15555c: starting snapshot up to: index=12 +2025-01-25T14:39:52.031Z [INFO] creating new snapshot: path=/tmp/raft29061264/snapshots/2-12-1737815992031.tmp +2025-01-25T14:39:52.035Z [INFO] server-eb84812c-55a3-719a-9f06-b8f72a15555c: snapshot complete up to: index=12 +2025-01-25T14:39:52.039Z [INFO] creating new snapshot: path=/tmp/raft29061264/snapshots/2-113-1737815992039.tmp +2025-01-25T14:39:52.042Z [INFO] server-eb84812c-55a3-719a-9f06-b8f72a15555c: copied to local snapshot: bytes=71 +2025-01-25T14:39:52.043Z [INFO] server-eb84812c-55a3-719a-9f06-b8f72a15555c: snapshot restore progress: id=2-113-1737815992039 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:52.043Z [INFO] server-eb84812c-55a3-719a-9f06-b8f72a15555c: restored user snapshot: index=113 +2025-01-25T14:39:52.044Z [ERROR] server-eb84812c-55a3-719a-9f06-b8f72a15555c: failed to get log: index=23 error="log not found" +2025-01-25T14:39:52.044Z [ERROR] server-eb84812c-55a3-719a-9f06-b8f72a15555c: failed to get log: index=23 error="log not found" +2025-01-25T14:39:52.044Z [INFO] server-eb84812c-55a3-719a-9f06-b8f72a15555c: aborting pipeline replication: peer="{Voter server-8187e575-a308-4f2a-1ccc-ca501e6657d9 8187e575-a308-4f2a-1ccc-ca501e6657d9}" +2025-01-25T14:39:52.044Z [INFO] server-eb84812c-55a3-719a-9f06-b8f72a15555c: aborting pipeline replication: peer="{Voter server-7fd4b08d-76f7-1dc3-c566-196de3c90fe5 7fd4b08d-76f7-1dc3-c566-196de3c90fe5}" +2025-01-25T14:39:52.052Z [ERROR] server-eb84812c-55a3-719a-9f06-b8f72a15555c: failed to get log: index=23 error="log not found" +2025-01-25T14:39:52.056Z [ERROR] server-eb84812c-55a3-719a-9f06-b8f72a15555c: failed to get log: index=23 error="log not found" +2025-01-25T14:39:52.065Z [INFO] creating new snapshot: path=/tmp/raft416693552/snapshots/2-113-1737815992065.tmp +2025-01-25T14:39:52.065Z [INFO] creating new snapshot: path=/tmp/raft1559983849/snapshots/2-113-1737815992065.tmp +2025-01-25T14:39:52.067Z [INFO] server-7fd4b08d-76f7-1dc3-c566-196de3c90fe5: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:52.067Z [INFO] server-8187e575-a308-4f2a-1ccc-ca501e6657d9: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:52.069Z [INFO] server-7fd4b08d-76f7-1dc3-c566-196de3c90fe5: copied to local snapshot: bytes=71 +2025-01-25T14:39:52.070Z [INFO] server-8187e575-a308-4f2a-1ccc-ca501e6657d9: copied to local snapshot: bytes=71 +2025-01-25T14:39:52.070Z [INFO] server-7fd4b08d-76f7-1dc3-c566-196de3c90fe5: snapshot restore progress: id=2-113-1737815992065 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:52.070Z [INFO] server-7fd4b08d-76f7-1dc3-c566-196de3c90fe5: Installed remote snapshot +2025-01-25T14:39:52.070Z [WARN] server-7fd4b08d-76f7-1dc3-c566-196de3c90fe5: skipping application of old log: index=22 +2025-01-25T14:39:52.070Z [INFO] server-eb84812c-55a3-719a-9f06-b8f72a15555c: pipelining replication: peer="{Voter server-7fd4b08d-76f7-1dc3-c566-196de3c90fe5 7fd4b08d-76f7-1dc3-c566-196de3c90fe5}" +2025-01-25T14:39:52.071Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:39:52.071Z [INFO] server-8187e575-a308-4f2a-1ccc-ca501e6657d9: snapshot restore progress: id=2-113-1737815992065 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:52.071Z [INFO] server-8187e575-a308-4f2a-1ccc-ca501e6657d9: Installed remote snapshot +2025-01-25T14:39:52.072Z [INFO] server-eb84812c-55a3-719a-9f06-b8f72a15555c: pipelining replication: peer="{Voter server-8187e575-a308-4f2a-1ccc-ca501e6657d9 8187e575-a308-4f2a-1ccc-ca501e6657d9}" +2025-01-25T14:39:53.076Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:39:52.071260208 +0000 UTC m=+61.117099525 (2 nodes), 6.708µs from start of poll, 1.703636793s from cluster start. Timeout at 2025-01-25 14:39:53.076631792 +0000 UTC m=+62.122471401, 1.005371876s after stability +2025-01-25T14:39:53.091Z [INFO] server-eb84812c-55a3-719a-9f06-b8f72a15555c: aborting pipeline replication: peer="{Voter server-8187e575-a308-4f2a-1ccc-ca501e6657d9 8187e575-a308-4f2a-1ccc-ca501e6657d9}" +2025-01-25T14:39:53.091Z [INFO] server-eb84812c-55a3-719a-9f06-b8f72a15555c: aborting pipeline replication: peer="{Voter server-7fd4b08d-76f7-1dc3-c566-196de3c90fe5 7fd4b08d-76f7-1dc3-c566-196de3c90fe5}" === RUN TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_false -2025-01-25T14:29:07.916Z [INFO] server-47e6297d-dcde-53b0-1987-3d1bf19aee22: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-47e6297d-dcde-53b0-1987-3d1bf19aee22 Address:47e6297d-dcde-53b0-1987-3d1bf19aee22} {Suffrage:Voter ID:server-87895c1b-cd49-9b7e-8c44-87a3a6fdbf1d Address:87895c1b-cd49-9b7e-8c44-87a3a6fdbf1d} {Suffrage:Voter ID:server-709cdcd2-8b8f-49ad-a0ec-e70d24c41240 Address:709cdcd2-8b8f-49ad-a0ec-e70d24c41240}]" -2025-01-25T14:29:07.916Z [INFO] server-87895c1b-cd49-9b7e-8c44-87a3a6fdbf1d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-47e6297d-dcde-53b0-1987-3d1bf19aee22 Address:47e6297d-dcde-53b0-1987-3d1bf19aee22} {Suffrage:Voter ID:server-87895c1b-cd49-9b7e-8c44-87a3a6fdbf1d Address:87895c1b-cd49-9b7e-8c44-87a3a6fdbf1d} {Suffrage:Voter ID:server-709cdcd2-8b8f-49ad-a0ec-e70d24c41240 Address:709cdcd2-8b8f-49ad-a0ec-e70d24c41240}]" -2025-01-25T14:29:07.916Z [INFO] server-47e6297d-dcde-53b0-1987-3d1bf19aee22: entering follower state: follower="Node at 47e6297d-dcde-53b0-1987-3d1bf19aee22 [Follower]" leader-address= leader-id= -2025-01-25T14:29:07.916Z [INFO] server-87895c1b-cd49-9b7e-8c44-87a3a6fdbf1d: entering follower state: follower="Node at 87895c1b-cd49-9b7e-8c44-87a3a6fdbf1d [Follower]" leader-address= leader-id= -2025-01-25T14:29:07.916Z [INFO] server-709cdcd2-8b8f-49ad-a0ec-e70d24c41240: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-47e6297d-dcde-53b0-1987-3d1bf19aee22 Address:47e6297d-dcde-53b0-1987-3d1bf19aee22} {Suffrage:Voter ID:server-87895c1b-cd49-9b7e-8c44-87a3a6fdbf1d Address:87895c1b-cd49-9b7e-8c44-87a3a6fdbf1d} {Suffrage:Voter ID:server-709cdcd2-8b8f-49ad-a0ec-e70d24c41240 Address:709cdcd2-8b8f-49ad-a0ec-e70d24c41240}]" -2025-01-25T14:29:07.917Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:07.917Z [INFO] server-709cdcd2-8b8f-49ad-a0ec-e70d24c41240: entering follower state: follower="Node at 709cdcd2-8b8f-49ad-a0ec-e70d24c41240 [Follower]" leader-address= leader-id= -2025-01-25T14:29:08.648Z [WARN] server-709cdcd2-8b8f-49ad-a0ec-e70d24c41240: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:08.648Z [INFO] server-709cdcd2-8b8f-49ad-a0ec-e70d24c41240: entering candidate state: node="Node at 709cdcd2-8b8f-49ad-a0ec-e70d24c41240 [Candidate]" term=2 -2025-01-25T14:29:08.648Z [INFO] server-709cdcd2-8b8f-49ad-a0ec-e70d24c41240: election won: term=2 tally=2 -2025-01-25T14:29:08.648Z [INFO] server-709cdcd2-8b8f-49ad-a0ec-e70d24c41240: entering leader state: leader="Node at 709cdcd2-8b8f-49ad-a0ec-e70d24c41240 [Leader]" -2025-01-25T14:29:08.648Z [INFO] server-709cdcd2-8b8f-49ad-a0ec-e70d24c41240: added peer, starting replication: peer=server-47e6297d-dcde-53b0-1987-3d1bf19aee22 -2025-01-25T14:29:08.648Z [INFO] server-709cdcd2-8b8f-49ad-a0ec-e70d24c41240: added peer, starting replication: peer=server-87895c1b-cd49-9b7e-8c44-87a3a6fdbf1d -2025-01-25T14:29:08.648Z [INFO] server-709cdcd2-8b8f-49ad-a0ec-e70d24c41240: pipelining replication: peer="{Voter server-47e6297d-dcde-53b0-1987-3d1bf19aee22 47e6297d-dcde-53b0-1987-3d1bf19aee22}" -2025-01-25T14:29:08.648Z [INFO] server-709cdcd2-8b8f-49ad-a0ec-e70d24c41240: pipelining replication: peer="{Voter server-87895c1b-cd49-9b7e-8c44-87a3a6fdbf1d 87895c1b-cd49-9b7e-8c44-87a3a6fdbf1d}" -2025-01-25T14:29:09.654Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:08.648464325 +0000 UTC m=+58.502789307 (1 nodes), 731.416879ms from start of poll, 732.175772ms from cluster start. Timeout at 2025-01-25 14:29:09.653467525 +0000 UTC m=+59.507792467, 1.00500316s after stability -2025-01-25T14:29:09.655Z [INFO] server-709cdcd2-8b8f-49ad-a0ec-e70d24c41240: starting snapshot up to: index=12 -2025-01-25T14:29:09.655Z [INFO] creating new snapshot: path=/tmp/raft3744132961/snapshots/2-12-1737815349655.tmp -2025-01-25T14:29:09.657Z [INFO] server-709cdcd2-8b8f-49ad-a0ec-e70d24c41240: snapshot complete up to: index=12 -2025-01-25T14:29:09.658Z [INFO] creating new snapshot: path=/tmp/raft3744132961/snapshots/2-113-1737815349658.tmp -2025-01-25T14:29:09.659Z [INFO] server-709cdcd2-8b8f-49ad-a0ec-e70d24c41240: copied to local snapshot: bytes=71 -2025-01-25T14:29:09.659Z [INFO] server-709cdcd2-8b8f-49ad-a0ec-e70d24c41240: snapshot restore progress: id=2-113-1737815349658 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:09.660Z [INFO] server-709cdcd2-8b8f-49ad-a0ec-e70d24c41240: removing all old logs from log store -2025-01-25T14:29:09.660Z [INFO] server-709cdcd2-8b8f-49ad-a0ec-e70d24c41240: compacting logs: from=1 to=22 -2025-01-25T14:29:09.660Z [INFO] server-709cdcd2-8b8f-49ad-a0ec-e70d24c41240: restored user snapshot: index=113 -2025-01-25T14:29:09.660Z [ERROR] server-709cdcd2-8b8f-49ad-a0ec-e70d24c41240: failed to get log: index=22 error="log not found" -2025-01-25T14:29:09.660Z [INFO] server-709cdcd2-8b8f-49ad-a0ec-e70d24c41240: aborting pipeline replication: peer="{Voter server-87895c1b-cd49-9b7e-8c44-87a3a6fdbf1d 87895c1b-cd49-9b7e-8c44-87a3a6fdbf1d}" -2025-01-25T14:29:09.660Z [ERROR] server-709cdcd2-8b8f-49ad-a0ec-e70d24c41240: failed to get log: index=22 error="log not found" -2025-01-25T14:29:09.660Z [INFO] server-709cdcd2-8b8f-49ad-a0ec-e70d24c41240: aborting pipeline replication: peer="{Voter server-47e6297d-dcde-53b0-1987-3d1bf19aee22 47e6297d-dcde-53b0-1987-3d1bf19aee22}" -2025-01-25T14:29:09.668Z [ERROR] server-709cdcd2-8b8f-49ad-a0ec-e70d24c41240: failed to get log: index=22 error="log not found" -2025-01-25T14:29:09.670Z [ERROR] server-709cdcd2-8b8f-49ad-a0ec-e70d24c41240: failed to get log: index=22 error="log not found" -2025-01-25T14:29:09.673Z [INFO] creating new snapshot: path=/tmp/raft3954771927/snapshots/2-113-1737815349673.tmp -2025-01-25T14:29:09.674Z [INFO] creating new snapshot: path=/tmp/raft3553680851/snapshots/2-113-1737815349674.tmp -2025-01-25T14:29:09.674Z [INFO] server-87895c1b-cd49-9b7e-8c44-87a3a6fdbf1d: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:09.674Z [INFO] server-47e6297d-dcde-53b0-1987-3d1bf19aee22: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:09.674Z [INFO] server-87895c1b-cd49-9b7e-8c44-87a3a6fdbf1d: copied to local snapshot: bytes=71 -2025-01-25T14:29:09.675Z [INFO] server-87895c1b-cd49-9b7e-8c44-87a3a6fdbf1d: snapshot restore progress: id=2-113-1737815349673 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:09.675Z [INFO] server-87895c1b-cd49-9b7e-8c44-87a3a6fdbf1d: removing all old logs from log store -2025-01-25T14:29:09.675Z [INFO] server-87895c1b-cd49-9b7e-8c44-87a3a6fdbf1d: compacting logs: from=1 to=22 -2025-01-25T14:29:09.675Z [INFO] server-87895c1b-cd49-9b7e-8c44-87a3a6fdbf1d: Installed remote snapshot -2025-01-25T14:29:09.675Z [INFO] server-47e6297d-dcde-53b0-1987-3d1bf19aee22: copied to local snapshot: bytes=71 -2025-01-25T14:29:09.675Z [WARN] server-87895c1b-cd49-9b7e-8c44-87a3a6fdbf1d: skipping application of old log: index=22 -2025-01-25T14:29:09.675Z [INFO] server-709cdcd2-8b8f-49ad-a0ec-e70d24c41240: pipelining replication: peer="{Voter server-87895c1b-cd49-9b7e-8c44-87a3a6fdbf1d 87895c1b-cd49-9b7e-8c44-87a3a6fdbf1d}" -2025-01-25T14:29:09.675Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:29:09.676Z [INFO] server-47e6297d-dcde-53b0-1987-3d1bf19aee22: snapshot restore progress: id=2-113-1737815349674 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:09.676Z [INFO] server-47e6297d-dcde-53b0-1987-3d1bf19aee22: removing all old logs from log store -2025-01-25T14:29:09.676Z [INFO] server-47e6297d-dcde-53b0-1987-3d1bf19aee22: compacting logs: from=1 to=22 -2025-01-25T14:29:09.676Z [INFO] server-47e6297d-dcde-53b0-1987-3d1bf19aee22: Installed remote snapshot -2025-01-25T14:29:09.676Z [INFO] server-709cdcd2-8b8f-49ad-a0ec-e70d24c41240: pipelining replication: peer="{Voter server-47e6297d-dcde-53b0-1987-3d1bf19aee22 47e6297d-dcde-53b0-1987-3d1bf19aee22}" -2025-01-25T14:29:10.682Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:29:09.675995109 +0000 UTC m=+59.530320091 (2 nodes), 1.8µs from start of poll, 1.759706556s from cluster start. Timeout at 2025-01-25 14:29:10.681126305 +0000 UTC m=+60.535451287, 1.005131196s after stability -2025-01-25T14:29:10.696Z [INFO] server-709cdcd2-8b8f-49ad-a0ec-e70d24c41240: aborting pipeline replication: peer="{Voter server-87895c1b-cd49-9b7e-8c44-87a3a6fdbf1d 87895c1b-cd49-9b7e-8c44-87a3a6fdbf1d}" -2025-01-25T14:29:10.696Z [INFO] server-709cdcd2-8b8f-49ad-a0ec-e70d24c41240: aborting pipeline replication: peer="{Voter server-47e6297d-dcde-53b0-1987-3d1bf19aee22 47e6297d-dcde-53b0-1987-3d1bf19aee22}" +2025-01-25T14:39:53.099Z [INFO] server-4428c97f-8ff3-22ff-0afe-1407b2977e00: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4428c97f-8ff3-22ff-0afe-1407b2977e00 Address:4428c97f-8ff3-22ff-0afe-1407b2977e00} {Suffrage:Voter ID:server-44dbc8f4-ea21-8fee-781f-d9e3ad6f898e Address:44dbc8f4-ea21-8fee-781f-d9e3ad6f898e} {Suffrage:Voter ID:server-58cfe16d-c36a-8ad6-4716-bb69f56daab2 Address:58cfe16d-c36a-8ad6-4716-bb69f56daab2}]" +2025-01-25T14:39:53.099Z [INFO] server-44dbc8f4-ea21-8fee-781f-d9e3ad6f898e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4428c97f-8ff3-22ff-0afe-1407b2977e00 Address:4428c97f-8ff3-22ff-0afe-1407b2977e00} {Suffrage:Voter ID:server-44dbc8f4-ea21-8fee-781f-d9e3ad6f898e Address:44dbc8f4-ea21-8fee-781f-d9e3ad6f898e} {Suffrage:Voter ID:server-58cfe16d-c36a-8ad6-4716-bb69f56daab2 Address:58cfe16d-c36a-8ad6-4716-bb69f56daab2}]" +2025-01-25T14:39:53.099Z [INFO] server-4428c97f-8ff3-22ff-0afe-1407b2977e00: entering follower state: follower="Node at 4428c97f-8ff3-22ff-0afe-1407b2977e00 [Follower]" leader-address= leader-id= +2025-01-25T14:39:53.100Z [INFO] server-44dbc8f4-ea21-8fee-781f-d9e3ad6f898e: entering follower state: follower="Node at 44dbc8f4-ea21-8fee-781f-d9e3ad6f898e [Follower]" leader-address= leader-id= +2025-01-25T14:39:53.100Z [INFO] server-58cfe16d-c36a-8ad6-4716-bb69f56daab2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4428c97f-8ff3-22ff-0afe-1407b2977e00 Address:4428c97f-8ff3-22ff-0afe-1407b2977e00} {Suffrage:Voter ID:server-44dbc8f4-ea21-8fee-781f-d9e3ad6f898e Address:44dbc8f4-ea21-8fee-781f-d9e3ad6f898e} {Suffrage:Voter ID:server-58cfe16d-c36a-8ad6-4716-bb69f56daab2 Address:58cfe16d-c36a-8ad6-4716-bb69f56daab2}]" +2025-01-25T14:39:53.100Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:53.100Z [INFO] server-58cfe16d-c36a-8ad6-4716-bb69f56daab2: entering follower state: follower="Node at 58cfe16d-c36a-8ad6-4716-bb69f56daab2 [Follower]" leader-address= leader-id= +2025-01-25T14:39:53.627Z [WARN] server-58cfe16d-c36a-8ad6-4716-bb69f56daab2: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:53.627Z [INFO] server-58cfe16d-c36a-8ad6-4716-bb69f56daab2: entering candidate state: node="Node at 58cfe16d-c36a-8ad6-4716-bb69f56daab2 [Candidate]" term=2 +2025-01-25T14:39:53.628Z [INFO] server-58cfe16d-c36a-8ad6-4716-bb69f56daab2: election won: term=2 tally=2 +2025-01-25T14:39:53.628Z [INFO] server-58cfe16d-c36a-8ad6-4716-bb69f56daab2: entering leader state: leader="Node at 58cfe16d-c36a-8ad6-4716-bb69f56daab2 [Leader]" +2025-01-25T14:39:53.629Z [INFO] server-58cfe16d-c36a-8ad6-4716-bb69f56daab2: added peer, starting replication: peer=server-4428c97f-8ff3-22ff-0afe-1407b2977e00 +2025-01-25T14:39:53.629Z [INFO] server-58cfe16d-c36a-8ad6-4716-bb69f56daab2: added peer, starting replication: peer=server-44dbc8f4-ea21-8fee-781f-d9e3ad6f898e +2025-01-25T14:39:53.629Z [INFO] server-58cfe16d-c36a-8ad6-4716-bb69f56daab2: pipelining replication: peer="{Voter server-4428c97f-8ff3-22ff-0afe-1407b2977e00 4428c97f-8ff3-22ff-0afe-1407b2977e00}" +2025-01-25T14:39:53.630Z [INFO] server-58cfe16d-c36a-8ad6-4716-bb69f56daab2: pipelining replication: peer="{Voter server-44dbc8f4-ea21-8fee-781f-d9e3ad6f898e 44dbc8f4-ea21-8fee-781f-d9e3ad6f898e}" +2025-01-25T14:39:54.634Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:53.62923166 +0000 UTC m=+62.675070978 (1 nodes), 528.484164ms from start of poll, 530.491128ms from cluster start. Timeout at 2025-01-25 14:39:54.634344536 +0000 UTC m=+63.680183854, 1.005112876s after stability +2025-01-25T14:39:54.637Z [INFO] server-58cfe16d-c36a-8ad6-4716-bb69f56daab2: starting snapshot up to: index=12 +2025-01-25T14:39:54.637Z [INFO] creating new snapshot: path=/tmp/raft486870383/snapshots/2-12-1737815994637.tmp +2025-01-25T14:39:54.640Z [INFO] server-58cfe16d-c36a-8ad6-4716-bb69f56daab2: snapshot complete up to: index=12 +2025-01-25T14:39:54.644Z [INFO] creating new snapshot: path=/tmp/raft486870383/snapshots/2-113-1737815994644.tmp +2025-01-25T14:39:54.648Z [INFO] server-58cfe16d-c36a-8ad6-4716-bb69f56daab2: copied to local snapshot: bytes=71 +2025-01-25T14:39:54.649Z [INFO] server-58cfe16d-c36a-8ad6-4716-bb69f56daab2: snapshot restore progress: id=2-113-1737815994644 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:54.650Z [INFO] server-58cfe16d-c36a-8ad6-4716-bb69f56daab2: removing all old logs from log store +2025-01-25T14:39:54.650Z [INFO] server-58cfe16d-c36a-8ad6-4716-bb69f56daab2: compacting logs: from=1 to=22 +2025-01-25T14:39:54.650Z [INFO] server-58cfe16d-c36a-8ad6-4716-bb69f56daab2: restored user snapshot: index=113 +2025-01-25T14:39:54.650Z [ERROR] server-58cfe16d-c36a-8ad6-4716-bb69f56daab2: failed to get log: index=22 error="log not found" +2025-01-25T14:39:54.650Z [ERROR] server-58cfe16d-c36a-8ad6-4716-bb69f56daab2: failed to get log: index=22 error="log not found" +2025-01-25T14:39:54.650Z [INFO] server-58cfe16d-c36a-8ad6-4716-bb69f56daab2: aborting pipeline replication: peer="{Voter server-44dbc8f4-ea21-8fee-781f-d9e3ad6f898e 44dbc8f4-ea21-8fee-781f-d9e3ad6f898e}" +2025-01-25T14:39:54.650Z [INFO] server-58cfe16d-c36a-8ad6-4716-bb69f56daab2: aborting pipeline replication: peer="{Voter server-4428c97f-8ff3-22ff-0afe-1407b2977e00 4428c97f-8ff3-22ff-0afe-1407b2977e00}" +2025-01-25T14:39:54.657Z [ERROR] server-58cfe16d-c36a-8ad6-4716-bb69f56daab2: failed to get log: index=22 error="log not found" +2025-01-25T14:39:54.658Z [ERROR] server-58cfe16d-c36a-8ad6-4716-bb69f56daab2: failed to get log: index=22 error="log not found" +2025-01-25T14:39:54.659Z [INFO] creating new snapshot: path=/tmp/raft3065517522/snapshots/2-113-1737815994659.tmp +2025-01-25T14:39:54.660Z [INFO] server-44dbc8f4-ea21-8fee-781f-d9e3ad6f898e: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:54.661Z [INFO] server-44dbc8f4-ea21-8fee-781f-d9e3ad6f898e: copied to local snapshot: bytes=71 +2025-01-25T14:39:54.661Z [INFO] creating new snapshot: path=/tmp/raft2523015633/snapshots/2-113-1737815994661.tmp +2025-01-25T14:39:54.662Z [INFO] server-44dbc8f4-ea21-8fee-781f-d9e3ad6f898e: snapshot restore progress: id=2-113-1737815994659 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:54.663Z [INFO] server-44dbc8f4-ea21-8fee-781f-d9e3ad6f898e: removing all old logs from log store +2025-01-25T14:39:54.663Z [INFO] server-44dbc8f4-ea21-8fee-781f-d9e3ad6f898e: compacting logs: from=1 to=22 +2025-01-25T14:39:54.663Z [INFO] server-44dbc8f4-ea21-8fee-781f-d9e3ad6f898e: Installed remote snapshot +2025-01-25T14:39:54.667Z [INFO] server-4428c97f-8ff3-22ff-0afe-1407b2977e00: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:54.669Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:39:54.665Z [INFO] server-58cfe16d-c36a-8ad6-4716-bb69f56daab2: pipelining replication: peer="{Voter server-44dbc8f4-ea21-8fee-781f-d9e3ad6f898e 44dbc8f4-ea21-8fee-781f-d9e3ad6f898e}" +2025-01-25T14:39:54.674Z [INFO] server-4428c97f-8ff3-22ff-0afe-1407b2977e00: copied to local snapshot: bytes=71 +2025-01-25T14:39:54.676Z [INFO] server-4428c97f-8ff3-22ff-0afe-1407b2977e00: snapshot restore progress: id=2-113-1737815994661 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:54.676Z [INFO] server-4428c97f-8ff3-22ff-0afe-1407b2977e00: removing all old logs from log store +2025-01-25T14:39:54.676Z [INFO] server-4428c97f-8ff3-22ff-0afe-1407b2977e00: compacting logs: from=1 to=22 +2025-01-25T14:39:54.676Z [INFO] server-4428c97f-8ff3-22ff-0afe-1407b2977e00: Installed remote snapshot +2025-01-25T14:39:54.677Z [INFO] server-58cfe16d-c36a-8ad6-4716-bb69f56daab2: pipelining replication: peer="{Voter server-4428c97f-8ff3-22ff-0afe-1407b2977e00 4428c97f-8ff3-22ff-0afe-1407b2977e00}" +2025-01-25T14:39:55.675Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:39:54.670025732 +0000 UTC m=+63.715865050 (2 nodes), 9.625µs from start of poll, 1.5712852s from cluster start. Timeout at 2025-01-25 14:39:55.675141816 +0000 UTC m=+64.720981425, 1.005116375s after stability +2025-01-25T14:39:55.690Z [INFO] server-58cfe16d-c36a-8ad6-4716-bb69f56daab2: aborting pipeline replication: peer="{Voter server-44dbc8f4-ea21-8fee-781f-d9e3ad6f898e 44dbc8f4-ea21-8fee-781f-d9e3ad6f898e}" +2025-01-25T14:39:55.690Z [INFO] server-58cfe16d-c36a-8ad6-4716-bb69f56daab2: aborting pipeline replication: peer="{Voter server-4428c97f-8ff3-22ff-0afe-1407b2977e00 4428c97f-8ff3-22ff-0afe-1407b2977e00}" === RUN TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_true -2025-01-25T14:29:10.700Z [INFO] server-2e270c31-33c1-8d85-3ca6-32058f0cfb45: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2e270c31-33c1-8d85-3ca6-32058f0cfb45 Address:2e270c31-33c1-8d85-3ca6-32058f0cfb45} {Suffrage:Voter ID:server-d13575af-1ae0-6f27-be3b-ce0f235f7a45 Address:d13575af-1ae0-6f27-be3b-ce0f235f7a45} {Suffrage:Voter ID:server-0d12f722-391f-8b5c-3382-260ef915d052 Address:0d12f722-391f-8b5c-3382-260ef915d052}]" -2025-01-25T14:29:10.700Z [INFO] server-d13575af-1ae0-6f27-be3b-ce0f235f7a45: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2e270c31-33c1-8d85-3ca6-32058f0cfb45 Address:2e270c31-33c1-8d85-3ca6-32058f0cfb45} {Suffrage:Voter ID:server-d13575af-1ae0-6f27-be3b-ce0f235f7a45 Address:d13575af-1ae0-6f27-be3b-ce0f235f7a45} {Suffrage:Voter ID:server-0d12f722-391f-8b5c-3382-260ef915d052 Address:0d12f722-391f-8b5c-3382-260ef915d052}]" -2025-01-25T14:29:10.700Z [INFO] server-0d12f722-391f-8b5c-3382-260ef915d052: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2e270c31-33c1-8d85-3ca6-32058f0cfb45 Address:2e270c31-33c1-8d85-3ca6-32058f0cfb45} {Suffrage:Voter ID:server-d13575af-1ae0-6f27-be3b-ce0f235f7a45 Address:d13575af-1ae0-6f27-be3b-ce0f235f7a45} {Suffrage:Voter ID:server-0d12f722-391f-8b5c-3382-260ef915d052 Address:0d12f722-391f-8b5c-3382-260ef915d052}]" -2025-01-25T14:29:10.700Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:10.700Z [INFO] server-2e270c31-33c1-8d85-3ca6-32058f0cfb45: entering follower state: follower="Node at 2e270c31-33c1-8d85-3ca6-32058f0cfb45 [Follower]" leader-address= leader-id= -2025-01-25T14:29:10.701Z [INFO] server-d13575af-1ae0-6f27-be3b-ce0f235f7a45: entering follower state: follower="Node at d13575af-1ae0-6f27-be3b-ce0f235f7a45 [Follower]" leader-address= leader-id= -2025-01-25T14:29:10.701Z [INFO] server-0d12f722-391f-8b5c-3382-260ef915d052: entering follower state: follower="Node at 0d12f722-391f-8b5c-3382-260ef915d052 [Follower]" leader-address= leader-id= -2025-01-25T14:29:11.261Z [WARN] server-d13575af-1ae0-6f27-be3b-ce0f235f7a45: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:11.261Z [INFO] server-d13575af-1ae0-6f27-be3b-ce0f235f7a45: entering candidate state: node="Node at d13575af-1ae0-6f27-be3b-ce0f235f7a45 [Candidate]" term=2 -2025-01-25T14:29:11.261Z [INFO] server-d13575af-1ae0-6f27-be3b-ce0f235f7a45: election won: term=2 tally=2 -2025-01-25T14:29:11.262Z [INFO] server-d13575af-1ae0-6f27-be3b-ce0f235f7a45: entering leader state: leader="Node at d13575af-1ae0-6f27-be3b-ce0f235f7a45 [Leader]" -2025-01-25T14:29:11.262Z [INFO] server-d13575af-1ae0-6f27-be3b-ce0f235f7a45: added peer, starting replication: peer=server-2e270c31-33c1-8d85-3ca6-32058f0cfb45 -2025-01-25T14:29:11.262Z [INFO] server-d13575af-1ae0-6f27-be3b-ce0f235f7a45: added peer, starting replication: peer=server-0d12f722-391f-8b5c-3382-260ef915d052 -2025-01-25T14:29:11.262Z [INFO] server-d13575af-1ae0-6f27-be3b-ce0f235f7a45: pipelining replication: peer="{Voter server-2e270c31-33c1-8d85-3ca6-32058f0cfb45 2e270c31-33c1-8d85-3ca6-32058f0cfb45}" -2025-01-25T14:29:11.262Z [INFO] server-d13575af-1ae0-6f27-be3b-ce0f235f7a45: pipelining replication: peer="{Voter server-0d12f722-391f-8b5c-3382-260ef915d052 0d12f722-391f-8b5c-3382-260ef915d052}" -2025-01-25T14:29:12.268Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:11.262031714 +0000 UTC m=+61.116356696 (1 nodes), 561.115888ms from start of poll, 561.790104ms from cluster start. Timeout at 2025-01-25 14:29:12.267046953 +0000 UTC m=+62.121371935, 1.005015239s after stability -2025-01-25T14:29:12.269Z [INFO] server-d13575af-1ae0-6f27-be3b-ce0f235f7a45: starting snapshot up to: index=12 -2025-01-25T14:29:12.269Z [INFO] creating new snapshot: path=/tmp/raft3277779116/snapshots/2-12-1737815352269.tmp -2025-01-25T14:29:12.270Z [INFO] server-d13575af-1ae0-6f27-be3b-ce0f235f7a45: snapshot complete up to: index=12 -2025-01-25T14:29:12.273Z [INFO] server-4c93850d-69b2-b675-eec5-d238b7c865c6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4c93850d-69b2-b675-eec5-d238b7c865c6 Address:4c93850d-69b2-b675-eec5-d238b7c865c6} {Suffrage:Voter ID:server-b7356e8b-ceb7-e553-a82d-832d4d96dcb8 Address:b7356e8b-ceb7-e553-a82d-832d4d96dcb8} {Suffrage:Voter ID:server-5c30a755-0453-9d1e-e6c1-9e7b81e9fbdd Address:5c30a755-0453-9d1e-e6c1-9e7b81e9fbdd}]" -2025-01-25T14:29:12.273Z [INFO] server-4c93850d-69b2-b675-eec5-d238b7c865c6: entering follower state: follower="Node at 4c93850d-69b2-b675-eec5-d238b7c865c6 [Follower]" leader-address= leader-id= -2025-01-25T14:29:12.273Z [INFO] server-b7356e8b-ceb7-e553-a82d-832d4d96dcb8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4c93850d-69b2-b675-eec5-d238b7c865c6 Address:4c93850d-69b2-b675-eec5-d238b7c865c6} {Suffrage:Voter ID:server-b7356e8b-ceb7-e553-a82d-832d4d96dcb8 Address:b7356e8b-ceb7-e553-a82d-832d4d96dcb8} {Suffrage:Voter ID:server-5c30a755-0453-9d1e-e6c1-9e7b81e9fbdd Address:5c30a755-0453-9d1e-e6c1-9e7b81e9fbdd}]" -2025-01-25T14:29:12.273Z [INFO] server-5c30a755-0453-9d1e-e6c1-9e7b81e9fbdd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4c93850d-69b2-b675-eec5-d238b7c865c6 Address:4c93850d-69b2-b675-eec5-d238b7c865c6} {Suffrage:Voter ID:server-b7356e8b-ceb7-e553-a82d-832d4d96dcb8 Address:b7356e8b-ceb7-e553-a82d-832d4d96dcb8} {Suffrage:Voter ID:server-5c30a755-0453-9d1e-e6c1-9e7b81e9fbdd Address:5c30a755-0453-9d1e-e6c1-9e7b81e9fbdd}]" -2025-01-25T14:29:12.273Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:12.273Z [INFO] server-b7356e8b-ceb7-e553-a82d-832d4d96dcb8: entering follower state: follower="Node at b7356e8b-ceb7-e553-a82d-832d4d96dcb8 [Follower]" leader-address= leader-id= -2025-01-25T14:29:12.273Z [INFO] server-5c30a755-0453-9d1e-e6c1-9e7b81e9fbdd: entering follower state: follower="Node at 5c30a755-0453-9d1e-e6c1-9e7b81e9fbdd [Follower]" leader-address= leader-id= -2025-01-25T14:29:12.788Z [WARN] server-5c30a755-0453-9d1e-e6c1-9e7b81e9fbdd: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:12.788Z [INFO] server-5c30a755-0453-9d1e-e6c1-9e7b81e9fbdd: entering candidate state: node="Node at 5c30a755-0453-9d1e-e6c1-9e7b81e9fbdd [Candidate]" term=2 -2025-01-25T14:29:12.789Z [INFO] server-5c30a755-0453-9d1e-e6c1-9e7b81e9fbdd: election won: term=2 tally=2 -2025-01-25T14:29:12.789Z [INFO] server-5c30a755-0453-9d1e-e6c1-9e7b81e9fbdd: entering leader state: leader="Node at 5c30a755-0453-9d1e-e6c1-9e7b81e9fbdd [Leader]" -2025-01-25T14:29:12.789Z [INFO] server-5c30a755-0453-9d1e-e6c1-9e7b81e9fbdd: added peer, starting replication: peer=server-4c93850d-69b2-b675-eec5-d238b7c865c6 -2025-01-25T14:29:12.789Z [INFO] server-5c30a755-0453-9d1e-e6c1-9e7b81e9fbdd: added peer, starting replication: peer=server-b7356e8b-ceb7-e553-a82d-832d4d96dcb8 -2025-01-25T14:29:12.789Z [INFO] server-5c30a755-0453-9d1e-e6c1-9e7b81e9fbdd: pipelining replication: peer="{Voter server-b7356e8b-ceb7-e553-a82d-832d4d96dcb8 b7356e8b-ceb7-e553-a82d-832d4d96dcb8}" -2025-01-25T14:29:12.789Z [INFO] server-5c30a755-0453-9d1e-e6c1-9e7b81e9fbdd: pipelining replication: peer="{Voter server-4c93850d-69b2-b675-eec5-d238b7c865c6 4c93850d-69b2-b675-eec5-d238b7c865c6}" -2025-01-25T14:29:13.794Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:12.789430749 +0000 UTC m=+62.643755691 (1 nodes), 515.863397ms from start of poll, 516.556292ms from cluster start. Timeout at 2025-01-25 14:29:13.794432909 +0000 UTC m=+63.648757891, 1.0050022s after stability -2025-01-25T14:29:13.795Z [INFO] creating new snapshot: path=/tmp/raft3385829941/snapshots/2-113-1737815353795.tmp -2025-01-25T14:29:13.796Z [INFO] server-5c30a755-0453-9d1e-e6c1-9e7b81e9fbdd: copied to local snapshot: bytes=71 -2025-01-25T14:29:13.797Z [INFO] server-5c30a755-0453-9d1e-e6c1-9e7b81e9fbdd: snapshot restore progress: id=2-113-1737815353795 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:13.797Z [INFO] server-5c30a755-0453-9d1e-e6c1-9e7b81e9fbdd: restored user snapshot: index=113 -2025-01-25T14:29:13.797Z [ERROR] server-5c30a755-0453-9d1e-e6c1-9e7b81e9fbdd: failed to get log: index=3 error="log not found" -2025-01-25T14:29:13.797Z [INFO] server-5c30a755-0453-9d1e-e6c1-9e7b81e9fbdd: aborting pipeline replication: peer="{Voter server-b7356e8b-ceb7-e553-a82d-832d4d96dcb8 b7356e8b-ceb7-e553-a82d-832d4d96dcb8}" -2025-01-25T14:29:13.797Z [ERROR] server-5c30a755-0453-9d1e-e6c1-9e7b81e9fbdd: failed to get log: index=3 error="log not found" -2025-01-25T14:29:13.797Z [INFO] server-5c30a755-0453-9d1e-e6c1-9e7b81e9fbdd: aborting pipeline replication: peer="{Voter server-4c93850d-69b2-b675-eec5-d238b7c865c6 4c93850d-69b2-b675-eec5-d238b7c865c6}" -2025-01-25T14:29:13.807Z [ERROR] server-5c30a755-0453-9d1e-e6c1-9e7b81e9fbdd: failed to get log: index=3 error="log not found" -2025-01-25T14:29:13.807Z [ERROR] server-5c30a755-0453-9d1e-e6c1-9e7b81e9fbdd: failed to get log: index=3 error="log not found" -2025-01-25T14:29:13.808Z [INFO] creating new snapshot: path=/tmp/raft1627652999/snapshots/2-113-1737815353807.tmp -2025-01-25T14:29:13.808Z [INFO] creating new snapshot: path=/tmp/raft1111282116/snapshots/2-113-1737815353808.tmp -2025-01-25T14:29:13.808Z [INFO] server-b7356e8b-ceb7-e553-a82d-832d4d96dcb8: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:13.812Z [INFO] server-b7356e8b-ceb7-e553-a82d-832d4d96dcb8: copied to local snapshot: bytes=71 -2025-01-25T14:29:13.815Z [INFO] server-4c93850d-69b2-b675-eec5-d238b7c865c6: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:13.816Z [INFO] server-b7356e8b-ceb7-e553-a82d-832d4d96dcb8: snapshot restore progress: id=2-113-1737815353807 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:13.816Z [INFO] server-b7356e8b-ceb7-e553-a82d-832d4d96dcb8: Installed remote snapshot -2025-01-25T14:29:13.816Z [INFO] server-4c93850d-69b2-b675-eec5-d238b7c865c6: copied to local snapshot: bytes=71 -2025-01-25T14:29:13.816Z [INFO] server-5c30a755-0453-9d1e-e6c1-9e7b81e9fbdd: pipelining replication: peer="{Voter server-b7356e8b-ceb7-e553-a82d-832d4d96dcb8 b7356e8b-ceb7-e553-a82d-832d4d96dcb8}" -2025-01-25T14:29:13.817Z [INFO] server-4c93850d-69b2-b675-eec5-d238b7c865c6: snapshot restore progress: id=2-113-1737815353808 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:13.817Z [INFO] server-4c93850d-69b2-b675-eec5-d238b7c865c6: Installed remote snapshot -2025-01-25T14:29:13.817Z [INFO] server-5c30a755-0453-9d1e-e6c1-9e7b81e9fbdd: pipelining replication: peer="{Voter server-4c93850d-69b2-b675-eec5-d238b7c865c6 4c93850d-69b2-b675-eec5-d238b7c865c6}" -2025-01-25T14:29:13.817Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:29:14.823Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:29:13.817759343 +0000 UTC m=+63.672084325 (2 nodes), 1.56µs from start of poll, 1.544884926s from cluster start. Timeout at 2025-01-25 14:29:14.822879139 +0000 UTC m=+64.677204081, 1.005119756s after stability -2025-01-25T14:29:14.835Z [INFO] server-d13575af-1ae0-6f27-be3b-ce0f235f7a45: aborting pipeline replication: peer="{Voter server-0d12f722-391f-8b5c-3382-260ef915d052 0d12f722-391f-8b5c-3382-260ef915d052}" -2025-01-25T14:29:14.835Z [INFO] server-d13575af-1ae0-6f27-be3b-ce0f235f7a45: aborting pipeline replication: peer="{Voter server-2e270c31-33c1-8d85-3ca6-32058f0cfb45 2e270c31-33c1-8d85-3ca6-32058f0cfb45}" +2025-01-25T14:39:55.710Z [INFO] server-e8548364-630a-d182-f28a-ed43cc9bd156: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e8548364-630a-d182-f28a-ed43cc9bd156 Address:e8548364-630a-d182-f28a-ed43cc9bd156} {Suffrage:Voter ID:server-09e3c037-4ff2-3108-c046-cecd838d9f06 Address:09e3c037-4ff2-3108-c046-cecd838d9f06} {Suffrage:Voter ID:server-0a038091-710b-b8a7-cea5-fd957f1212db Address:0a038091-710b-b8a7-cea5-fd957f1212db}]" +2025-01-25T14:39:55.711Z [INFO] server-e8548364-630a-d182-f28a-ed43cc9bd156: entering follower state: follower="Node at e8548364-630a-d182-f28a-ed43cc9bd156 [Follower]" leader-address= leader-id= +2025-01-25T14:39:55.711Z [INFO] server-09e3c037-4ff2-3108-c046-cecd838d9f06: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e8548364-630a-d182-f28a-ed43cc9bd156 Address:e8548364-630a-d182-f28a-ed43cc9bd156} {Suffrage:Voter ID:server-09e3c037-4ff2-3108-c046-cecd838d9f06 Address:09e3c037-4ff2-3108-c046-cecd838d9f06} {Suffrage:Voter ID:server-0a038091-710b-b8a7-cea5-fd957f1212db Address:0a038091-710b-b8a7-cea5-fd957f1212db}]" +2025-01-25T14:39:55.711Z [INFO] server-09e3c037-4ff2-3108-c046-cecd838d9f06: entering follower state: follower="Node at 09e3c037-4ff2-3108-c046-cecd838d9f06 [Follower]" leader-address= leader-id= +2025-01-25T14:39:55.712Z [INFO] server-0a038091-710b-b8a7-cea5-fd957f1212db: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e8548364-630a-d182-f28a-ed43cc9bd156 Address:e8548364-630a-d182-f28a-ed43cc9bd156} {Suffrage:Voter ID:server-09e3c037-4ff2-3108-c046-cecd838d9f06 Address:09e3c037-4ff2-3108-c046-cecd838d9f06} {Suffrage:Voter ID:server-0a038091-710b-b8a7-cea5-fd957f1212db Address:0a038091-710b-b8a7-cea5-fd957f1212db}]" +2025-01-25T14:39:55.712Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:55.712Z [INFO] server-0a038091-710b-b8a7-cea5-fd957f1212db: entering follower state: follower="Node at 0a038091-710b-b8a7-cea5-fd957f1212db [Follower]" leader-address= leader-id= +2025-01-25T14:39:56.301Z [WARN] server-09e3c037-4ff2-3108-c046-cecd838d9f06: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:56.302Z [INFO] server-09e3c037-4ff2-3108-c046-cecd838d9f06: entering candidate state: node="Node at 09e3c037-4ff2-3108-c046-cecd838d9f06 [Candidate]" term=2 +2025-01-25T14:39:56.302Z [INFO] server-09e3c037-4ff2-3108-c046-cecd838d9f06: election won: term=2 tally=2 +2025-01-25T14:39:56.302Z [INFO] server-09e3c037-4ff2-3108-c046-cecd838d9f06: entering leader state: leader="Node at 09e3c037-4ff2-3108-c046-cecd838d9f06 [Leader]" +2025-01-25T14:39:56.302Z [INFO] server-09e3c037-4ff2-3108-c046-cecd838d9f06: added peer, starting replication: peer=server-e8548364-630a-d182-f28a-ed43cc9bd156 +2025-01-25T14:39:56.302Z [INFO] server-09e3c037-4ff2-3108-c046-cecd838d9f06: added peer, starting replication: peer=server-0a038091-710b-b8a7-cea5-fd957f1212db +2025-01-25T14:39:56.303Z [INFO] server-09e3c037-4ff2-3108-c046-cecd838d9f06: pipelining replication: peer="{Voter server-e8548364-630a-d182-f28a-ed43cc9bd156 e8548364-630a-d182-f28a-ed43cc9bd156}" +2025-01-25T14:39:56.304Z [INFO] server-09e3c037-4ff2-3108-c046-cecd838d9f06: pipelining replication: peer="{Voter server-0a038091-710b-b8a7-cea5-fd957f1212db 0a038091-710b-b8a7-cea5-fd957f1212db}" +2025-01-25T14:39:57.308Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:56.302646319 +0000 UTC m=+65.348485636 (1 nodes), 589.86955ms from start of poll, 592.457808ms from cluster start. Timeout at 2025-01-25 14:39:57.307824236 +0000 UTC m=+66.353663554, 1.005177918s after stability +2025-01-25T14:39:57.312Z [INFO] server-09e3c037-4ff2-3108-c046-cecd838d9f06: starting snapshot up to: index=12 +2025-01-25T14:39:57.312Z [INFO] creating new snapshot: path=/tmp/raft1204418711/snapshots/2-12-1737815997312.tmp +2025-01-25T14:39:57.316Z [INFO] server-09e3c037-4ff2-3108-c046-cecd838d9f06: snapshot complete up to: index=12 +2025-01-25T14:39:57.326Z [INFO] server-c4099e29-19da-5d54-9590-d60d2f6a80de: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c4099e29-19da-5d54-9590-d60d2f6a80de Address:c4099e29-19da-5d54-9590-d60d2f6a80de} {Suffrage:Voter ID:server-e3a70510-475f-345c-61be-d887bc0a0849 Address:e3a70510-475f-345c-61be-d887bc0a0849} {Suffrage:Voter ID:server-edb08f0d-5c87-abd2-becd-19ce987db660 Address:edb08f0d-5c87-abd2-becd-19ce987db660}]" +2025-01-25T14:39:57.329Z [INFO] server-c4099e29-19da-5d54-9590-d60d2f6a80de: entering follower state: follower="Node at c4099e29-19da-5d54-9590-d60d2f6a80de [Follower]" leader-address= leader-id= +2025-01-25T14:39:57.332Z [INFO] server-e3a70510-475f-345c-61be-d887bc0a0849: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c4099e29-19da-5d54-9590-d60d2f6a80de Address:c4099e29-19da-5d54-9590-d60d2f6a80de} {Suffrage:Voter ID:server-e3a70510-475f-345c-61be-d887bc0a0849 Address:e3a70510-475f-345c-61be-d887bc0a0849} {Suffrage:Voter ID:server-edb08f0d-5c87-abd2-becd-19ce987db660 Address:edb08f0d-5c87-abd2-becd-19ce987db660}]" +2025-01-25T14:39:57.333Z [INFO] server-e3a70510-475f-345c-61be-d887bc0a0849: entering follower state: follower="Node at e3a70510-475f-345c-61be-d887bc0a0849 [Follower]" leader-address= leader-id= +2025-01-25T14:39:57.334Z [INFO] server-edb08f0d-5c87-abd2-becd-19ce987db660: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c4099e29-19da-5d54-9590-d60d2f6a80de Address:c4099e29-19da-5d54-9590-d60d2f6a80de} {Suffrage:Voter ID:server-e3a70510-475f-345c-61be-d887bc0a0849 Address:e3a70510-475f-345c-61be-d887bc0a0849} {Suffrage:Voter ID:server-edb08f0d-5c87-abd2-becd-19ce987db660 Address:edb08f0d-5c87-abd2-becd-19ce987db660}]" +2025-01-25T14:39:57.334Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:39:57.334Z [INFO] server-edb08f0d-5c87-abd2-becd-19ce987db660: entering follower state: follower="Node at edb08f0d-5c87-abd2-becd-19ce987db660 [Follower]" leader-address= leader-id= +2025-01-25T14:39:58.136Z [WARN] server-c4099e29-19da-5d54-9590-d60d2f6a80de: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:39:58.136Z [INFO] server-c4099e29-19da-5d54-9590-d60d2f6a80de: entering candidate state: node="Node at c4099e29-19da-5d54-9590-d60d2f6a80de [Candidate]" term=2 +2025-01-25T14:39:58.137Z [INFO] server-c4099e29-19da-5d54-9590-d60d2f6a80de: election won: term=2 tally=2 +2025-01-25T14:39:58.137Z [INFO] server-c4099e29-19da-5d54-9590-d60d2f6a80de: entering leader state: leader="Node at c4099e29-19da-5d54-9590-d60d2f6a80de [Leader]" +2025-01-25T14:39:58.137Z [INFO] server-c4099e29-19da-5d54-9590-d60d2f6a80de: added peer, starting replication: peer=server-e3a70510-475f-345c-61be-d887bc0a0849 +2025-01-25T14:39:58.137Z [INFO] server-c4099e29-19da-5d54-9590-d60d2f6a80de: added peer, starting replication: peer=server-edb08f0d-5c87-abd2-becd-19ce987db660 +2025-01-25T14:39:58.138Z [INFO] server-c4099e29-19da-5d54-9590-d60d2f6a80de: pipelining replication: peer="{Voter server-e3a70510-475f-345c-61be-d887bc0a0849 e3a70510-475f-345c-61be-d887bc0a0849}" +2025-01-25T14:39:58.138Z [INFO] server-c4099e29-19da-5d54-9590-d60d2f6a80de: pipelining replication: peer="{Voter server-edb08f0d-5c87-abd2-becd-19ce987db660 edb08f0d-5c87-abd2-becd-19ce987db660}" +2025-01-25T14:39:59.143Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:39:58.137509369 +0000 UTC m=+67.183348686 (1 nodes), 802.630878ms from start of poll, 813.553243ms from cluster start. Timeout at 2025-01-25 14:39:59.142525994 +0000 UTC m=+68.188365311, 1.005016625s after stability +2025-01-25T14:39:59.144Z [INFO] creating new snapshot: path=/tmp/raft3847685016/snapshots/2-113-1737815999144.tmp +2025-01-25T14:39:59.148Z [INFO] server-c4099e29-19da-5d54-9590-d60d2f6a80de: copied to local snapshot: bytes=71 +2025-01-25T14:39:59.150Z [INFO] server-c4099e29-19da-5d54-9590-d60d2f6a80de: snapshot restore progress: id=2-113-1737815999144 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:59.150Z [INFO] server-c4099e29-19da-5d54-9590-d60d2f6a80de: restored user snapshot: index=113 +2025-01-25T14:39:59.150Z [ERROR] server-c4099e29-19da-5d54-9590-d60d2f6a80de: failed to get log: index=3 error="log not found" +2025-01-25T14:39:59.150Z [INFO] server-c4099e29-19da-5d54-9590-d60d2f6a80de: aborting pipeline replication: peer="{Voter server-e3a70510-475f-345c-61be-d887bc0a0849 e3a70510-475f-345c-61be-d887bc0a0849}" +2025-01-25T14:39:59.150Z [ERROR] server-c4099e29-19da-5d54-9590-d60d2f6a80de: failed to get log: index=3 error="log not found" +2025-01-25T14:39:59.151Z [INFO] server-c4099e29-19da-5d54-9590-d60d2f6a80de: aborting pipeline replication: peer="{Voter server-edb08f0d-5c87-abd2-becd-19ce987db660 edb08f0d-5c87-abd2-becd-19ce987db660}" +2025-01-25T14:39:59.158Z [ERROR] server-c4099e29-19da-5d54-9590-d60d2f6a80de: failed to get log: index=3 error="log not found" +2025-01-25T14:39:59.158Z [ERROR] server-c4099e29-19da-5d54-9590-d60d2f6a80de: failed to get log: index=3 error="log not found" +2025-01-25T14:39:59.160Z [INFO] creating new snapshot: path=/tmp/raft1753885065/snapshots/2-113-1737815999160.tmp +2025-01-25T14:39:59.161Z [INFO] creating new snapshot: path=/tmp/raft293925439/snapshots/2-113-1737815999161.tmp +2025-01-25T14:39:59.161Z [INFO] server-edb08f0d-5c87-abd2-becd-19ce987db660: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:59.162Z [INFO] server-e3a70510-475f-345c-61be-d887bc0a0849: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:59.162Z [INFO] server-edb08f0d-5c87-abd2-becd-19ce987db660: copied to local snapshot: bytes=71 +2025-01-25T14:39:59.164Z [INFO] server-edb08f0d-5c87-abd2-becd-19ce987db660: snapshot restore progress: id=2-113-1737815999160 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:59.164Z [INFO] server-edb08f0d-5c87-abd2-becd-19ce987db660: Installed remote snapshot +2025-01-25T14:39:59.164Z [WARN] server-edb08f0d-5c87-abd2-becd-19ce987db660: skipping application of old log: index=113 +2025-01-25T14:39:59.165Z [INFO] server-c4099e29-19da-5d54-9590-d60d2f6a80de: pipelining replication: peer="{Voter server-edb08f0d-5c87-abd2-becd-19ce987db660 edb08f0d-5c87-abd2-becd-19ce987db660}" +2025-01-25T14:39:59.165Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:39:59.165Z [INFO] server-e3a70510-475f-345c-61be-d887bc0a0849: copied to local snapshot: bytes=71 +2025-01-25T14:39:59.167Z [INFO] server-e3a70510-475f-345c-61be-d887bc0a0849: snapshot restore progress: id=2-113-1737815999161 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:39:59.167Z [INFO] server-e3a70510-475f-345c-61be-d887bc0a0849: Installed remote snapshot +2025-01-25T14:39:59.168Z [INFO] server-c4099e29-19da-5d54-9590-d60d2f6a80de: pipelining replication: peer="{Voter server-e3a70510-475f-345c-61be-d887bc0a0849 e3a70510-475f-345c-61be-d887bc0a0849}" +2025-01-25T14:40:00.170Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:39:59.165215527 +0000 UTC m=+68.211054553 (2 nodes), 4.375µs from start of poll, 1.84125911s from cluster start. Timeout at 2025-01-25 14:40:00.170345611 +0000 UTC m=+69.216184929, 1.005130376s after stability +2025-01-25T14:40:00.184Z [INFO] server-09e3c037-4ff2-3108-c046-cecd838d9f06: aborting pipeline replication: peer="{Voter server-0a038091-710b-b8a7-cea5-fd957f1212db 0a038091-710b-b8a7-cea5-fd957f1212db}" +2025-01-25T14:40:00.184Z [INFO] server-09e3c037-4ff2-3108-c046-cecd838d9f06: aborting pipeline replication: peer="{Voter server-e8548364-630a-d182-f28a-ed43cc9bd156 e8548364-630a-d182-f28a-ed43cc9bd156}" === RUN TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_true -2025-01-25T14:29:14.838Z [INFO] server-15c5dab3-22d4-23b7-e103-bf5a63a3ccd1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-15c5dab3-22d4-23b7-e103-bf5a63a3ccd1 Address:15c5dab3-22d4-23b7-e103-bf5a63a3ccd1} {Suffrage:Voter ID:server-977f50ab-3940-9f74-c6ca-9e4cffe8927a Address:977f50ab-3940-9f74-c6ca-9e4cffe8927a} {Suffrage:Voter ID:server-65501e75-bc18-c9d2-9e54-6a5136e6decd Address:65501e75-bc18-c9d2-9e54-6a5136e6decd}]" -2025-01-25T14:29:14.838Z [INFO] server-977f50ab-3940-9f74-c6ca-9e4cffe8927a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-15c5dab3-22d4-23b7-e103-bf5a63a3ccd1 Address:15c5dab3-22d4-23b7-e103-bf5a63a3ccd1} {Suffrage:Voter ID:server-977f50ab-3940-9f74-c6ca-9e4cffe8927a Address:977f50ab-3940-9f74-c6ca-9e4cffe8927a} {Suffrage:Voter ID:server-65501e75-bc18-c9d2-9e54-6a5136e6decd Address:65501e75-bc18-c9d2-9e54-6a5136e6decd}]" -2025-01-25T14:29:14.838Z [INFO] server-15c5dab3-22d4-23b7-e103-bf5a63a3ccd1: entering follower state: follower="Node at 15c5dab3-22d4-23b7-e103-bf5a63a3ccd1 [Follower]" leader-address= leader-id= -2025-01-25T14:29:14.838Z [INFO] server-977f50ab-3940-9f74-c6ca-9e4cffe8927a: entering follower state: follower="Node at 977f50ab-3940-9f74-c6ca-9e4cffe8927a [Follower]" leader-address= leader-id= -2025-01-25T14:29:14.838Z [INFO] server-65501e75-bc18-c9d2-9e54-6a5136e6decd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-15c5dab3-22d4-23b7-e103-bf5a63a3ccd1 Address:15c5dab3-22d4-23b7-e103-bf5a63a3ccd1} {Suffrage:Voter ID:server-977f50ab-3940-9f74-c6ca-9e4cffe8927a Address:977f50ab-3940-9f74-c6ca-9e4cffe8927a} {Suffrage:Voter ID:server-65501e75-bc18-c9d2-9e54-6a5136e6decd Address:65501e75-bc18-c9d2-9e54-6a5136e6decd}]" -2025-01-25T14:29:14.838Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:14.838Z [INFO] server-65501e75-bc18-c9d2-9e54-6a5136e6decd: entering follower state: follower="Node at 65501e75-bc18-c9d2-9e54-6a5136e6decd [Follower]" leader-address= leader-id= -2025-01-25T14:29:15.341Z [WARN] server-65501e75-bc18-c9d2-9e54-6a5136e6decd: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:15.341Z [INFO] server-65501e75-bc18-c9d2-9e54-6a5136e6decd: entering candidate state: node="Node at 65501e75-bc18-c9d2-9e54-6a5136e6decd [Candidate]" term=2 -2025-01-25T14:29:15.341Z [INFO] server-65501e75-bc18-c9d2-9e54-6a5136e6decd: election won: term=2 tally=2 -2025-01-25T14:29:15.341Z [INFO] server-65501e75-bc18-c9d2-9e54-6a5136e6decd: entering leader state: leader="Node at 65501e75-bc18-c9d2-9e54-6a5136e6decd [Leader]" -2025-01-25T14:29:15.341Z [INFO] server-65501e75-bc18-c9d2-9e54-6a5136e6decd: added peer, starting replication: peer=server-15c5dab3-22d4-23b7-e103-bf5a63a3ccd1 -2025-01-25T14:29:15.341Z [INFO] server-65501e75-bc18-c9d2-9e54-6a5136e6decd: added peer, starting replication: peer=server-977f50ab-3940-9f74-c6ca-9e4cffe8927a -2025-01-25T14:29:15.342Z [INFO] server-65501e75-bc18-c9d2-9e54-6a5136e6decd: pipelining replication: peer="{Voter server-15c5dab3-22d4-23b7-e103-bf5a63a3ccd1 15c5dab3-22d4-23b7-e103-bf5a63a3ccd1}" -2025-01-25T14:29:15.342Z [INFO] server-65501e75-bc18-c9d2-9e54-6a5136e6decd: pipelining replication: peer="{Voter server-977f50ab-3940-9f74-c6ca-9e4cffe8927a 977f50ab-3940-9f74-c6ca-9e4cffe8927a}" -2025-01-25T14:29:16.347Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:15.341867136 +0000 UTC m=+65.196192078 (1 nodes), 503.021378ms from start of poll, 503.717313ms from cluster start. Timeout at 2025-01-25 14:29:16.346876376 +0000 UTC m=+66.201201318, 1.00500924s after stability -2025-01-25T14:29:16.348Z [INFO] server-65501e75-bc18-c9d2-9e54-6a5136e6decd: starting snapshot up to: index=12 -2025-01-25T14:29:16.348Z [INFO] creating new snapshot: path=/tmp/raft2135019359/snapshots/2-12-1737815356348.tmp -2025-01-25T14:29:16.349Z [INFO] server-65501e75-bc18-c9d2-9e54-6a5136e6decd: snapshot complete up to: index=12 -2025-01-25T14:29:16.352Z [INFO] server-8b4378bc-30e0-cfe0-f4a3-123ea14f8cf3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8b4378bc-30e0-cfe0-f4a3-123ea14f8cf3 Address:8b4378bc-30e0-cfe0-f4a3-123ea14f8cf3} {Suffrage:Voter ID:server-f956397f-36b4-3bf2-464a-844f7e6dbe06 Address:f956397f-36b4-3bf2-464a-844f7e6dbe06} {Suffrage:Voter ID:server-b78d0a09-bc3b-fa5d-bf7c-e150112bf577 Address:b78d0a09-bc3b-fa5d-bf7c-e150112bf577}]" -2025-01-25T14:29:16.352Z [INFO] server-8b4378bc-30e0-cfe0-f4a3-123ea14f8cf3: entering follower state: follower="Node at 8b4378bc-30e0-cfe0-f4a3-123ea14f8cf3 [Follower]" leader-address= leader-id= -2025-01-25T14:29:16.352Z [INFO] server-f956397f-36b4-3bf2-464a-844f7e6dbe06: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8b4378bc-30e0-cfe0-f4a3-123ea14f8cf3 Address:8b4378bc-30e0-cfe0-f4a3-123ea14f8cf3} {Suffrage:Voter ID:server-f956397f-36b4-3bf2-464a-844f7e6dbe06 Address:f956397f-36b4-3bf2-464a-844f7e6dbe06} {Suffrage:Voter ID:server-b78d0a09-bc3b-fa5d-bf7c-e150112bf577 Address:b78d0a09-bc3b-fa5d-bf7c-e150112bf577}]" -2025-01-25T14:29:16.352Z [INFO] server-f956397f-36b4-3bf2-464a-844f7e6dbe06: entering follower state: follower="Node at f956397f-36b4-3bf2-464a-844f7e6dbe06 [Follower]" leader-address= leader-id= -2025-01-25T14:29:16.352Z [INFO] server-b78d0a09-bc3b-fa5d-bf7c-e150112bf577: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8b4378bc-30e0-cfe0-f4a3-123ea14f8cf3 Address:8b4378bc-30e0-cfe0-f4a3-123ea14f8cf3} {Suffrage:Voter ID:server-f956397f-36b4-3bf2-464a-844f7e6dbe06 Address:f956397f-36b4-3bf2-464a-844f7e6dbe06} {Suffrage:Voter ID:server-b78d0a09-bc3b-fa5d-bf7c-e150112bf577 Address:b78d0a09-bc3b-fa5d-bf7c-e150112bf577}]" -2025-01-25T14:29:16.352Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:16.352Z [INFO] server-b78d0a09-bc3b-fa5d-bf7c-e150112bf577: entering follower state: follower="Node at b78d0a09-bc3b-fa5d-bf7c-e150112bf577 [Follower]" leader-address= leader-id= -2025-01-25T14:29:16.995Z [WARN] server-f956397f-36b4-3bf2-464a-844f7e6dbe06: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:16.995Z [INFO] server-f956397f-36b4-3bf2-464a-844f7e6dbe06: entering candidate state: node="Node at f956397f-36b4-3bf2-464a-844f7e6dbe06 [Candidate]" term=2 -2025-01-25T14:29:16.995Z [INFO] server-f956397f-36b4-3bf2-464a-844f7e6dbe06: election won: term=2 tally=2 -2025-01-25T14:29:16.995Z [INFO] server-f956397f-36b4-3bf2-464a-844f7e6dbe06: entering leader state: leader="Node at f956397f-36b4-3bf2-464a-844f7e6dbe06 [Leader]" -2025-01-25T14:29:16.995Z [INFO] server-f956397f-36b4-3bf2-464a-844f7e6dbe06: added peer, starting replication: peer=server-8b4378bc-30e0-cfe0-f4a3-123ea14f8cf3 -2025-01-25T14:29:16.995Z [INFO] server-f956397f-36b4-3bf2-464a-844f7e6dbe06: added peer, starting replication: peer=server-b78d0a09-bc3b-fa5d-bf7c-e150112bf577 -2025-01-25T14:29:16.995Z [INFO] server-f956397f-36b4-3bf2-464a-844f7e6dbe06: pipelining replication: peer="{Voter server-b78d0a09-bc3b-fa5d-bf7c-e150112bf577 b78d0a09-bc3b-fa5d-bf7c-e150112bf577}" -2025-01-25T14:29:16.995Z [INFO] server-f956397f-36b4-3bf2-464a-844f7e6dbe06: pipelining replication: peer="{Voter server-8b4378bc-30e0-cfe0-f4a3-123ea14f8cf3 8b4378bc-30e0-cfe0-f4a3-123ea14f8cf3}" -2025-01-25T14:29:18.000Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:16.995606785 +0000 UTC m=+66.849931767 (1 nodes), 643.095538ms from start of poll, 643.748314ms from cluster start. Timeout at 2025-01-25 14:29:18.000615224 +0000 UTC m=+67.854940246, 1.005008479s after stability -2025-01-25T14:29:18.001Z [INFO] creating new snapshot: path=/tmp/raft719616705/snapshots/2-113-1737815358001.tmp -2025-01-25T14:29:18.002Z [INFO] server-f956397f-36b4-3bf2-464a-844f7e6dbe06: copied to local snapshot: bytes=71 -2025-01-25T14:29:18.002Z [INFO] server-f956397f-36b4-3bf2-464a-844f7e6dbe06: snapshot restore progress: id=2-113-1737815358001 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:18.003Z [INFO] server-f956397f-36b4-3bf2-464a-844f7e6dbe06: removing all old logs from log store -2025-01-25T14:29:18.003Z [INFO] server-f956397f-36b4-3bf2-464a-844f7e6dbe06: compacting logs: from=1 to=2 -2025-01-25T14:29:18.003Z [INFO] server-f956397f-36b4-3bf2-464a-844f7e6dbe06: restored user snapshot: index=113 -2025-01-25T14:29:18.003Z [ERROR] server-f956397f-36b4-3bf2-464a-844f7e6dbe06: failed to get log: index=2 error="log not found" -2025-01-25T14:29:18.003Z [INFO] server-f956397f-36b4-3bf2-464a-844f7e6dbe06: aborting pipeline replication: peer="{Voter server-8b4378bc-30e0-cfe0-f4a3-123ea14f8cf3 8b4378bc-30e0-cfe0-f4a3-123ea14f8cf3}" -2025-01-25T14:29:18.003Z [ERROR] server-f956397f-36b4-3bf2-464a-844f7e6dbe06: failed to get log: index=2 error="log not found" -2025-01-25T14:29:18.003Z [ERROR] server-f956397f-36b4-3bf2-464a-844f7e6dbe06: failed to get log: index=2 error="log not found" -2025-01-25T14:29:18.003Z [INFO] server-f956397f-36b4-3bf2-464a-844f7e6dbe06: aborting pipeline replication: peer="{Voter server-b78d0a09-bc3b-fa5d-bf7c-e150112bf577 b78d0a09-bc3b-fa5d-bf7c-e150112bf577}" -2025-01-25T14:29:18.004Z [INFO] creating new snapshot: path=/tmp/raft4026948096/snapshots/2-113-1737815358004.tmp -2025-01-25T14:29:18.004Z [INFO] server-8b4378bc-30e0-cfe0-f4a3-123ea14f8cf3: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:18.005Z [INFO] server-8b4378bc-30e0-cfe0-f4a3-123ea14f8cf3: copied to local snapshot: bytes=71 -2025-01-25T14:29:18.005Z [INFO] server-8b4378bc-30e0-cfe0-f4a3-123ea14f8cf3: snapshot restore progress: id=2-113-1737815358004 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:18.005Z [INFO] server-8b4378bc-30e0-cfe0-f4a3-123ea14f8cf3: removing all old logs from log store -2025-01-25T14:29:18.005Z [INFO] server-8b4378bc-30e0-cfe0-f4a3-123ea14f8cf3: compacting logs: from=1 to=2 -2025-01-25T14:29:18.005Z [INFO] server-8b4378bc-30e0-cfe0-f4a3-123ea14f8cf3: Installed remote snapshot -2025-01-25T14:29:18.006Z [INFO] server-f956397f-36b4-3bf2-464a-844f7e6dbe06: pipelining replication: peer="{Voter server-8b4378bc-30e0-cfe0-f4a3-123ea14f8cf3 8b4378bc-30e0-cfe0-f4a3-123ea14f8cf3}" -2025-01-25T14:29:18.006Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:29:18.011Z [ERROR] server-f956397f-36b4-3bf2-464a-844f7e6dbe06: failed to get log: index=2 error="log not found" -2025-01-25T14:29:18.012Z [INFO] creating new snapshot: path=/tmp/raft2371475202/snapshots/2-113-1737815358012.tmp -2025-01-25T14:29:18.012Z [INFO] server-b78d0a09-bc3b-fa5d-bf7c-e150112bf577: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:18.013Z [INFO] server-b78d0a09-bc3b-fa5d-bf7c-e150112bf577: copied to local snapshot: bytes=71 -2025-01-25T14:29:18.014Z [INFO] server-b78d0a09-bc3b-fa5d-bf7c-e150112bf577: snapshot restore progress: id=2-113-1737815358012 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:18.014Z [INFO] server-b78d0a09-bc3b-fa5d-bf7c-e150112bf577: removing all old logs from log store -2025-01-25T14:29:18.014Z [INFO] server-b78d0a09-bc3b-fa5d-bf7c-e150112bf577: compacting logs: from=1 to=2 -2025-01-25T14:29:18.014Z [INFO] server-b78d0a09-bc3b-fa5d-bf7c-e150112bf577: Installed remote snapshot -2025-01-25T14:29:18.014Z [INFO] server-f956397f-36b4-3bf2-464a-844f7e6dbe06: pipelining replication: peer="{Voter server-b78d0a09-bc3b-fa5d-bf7c-e150112bf577 b78d0a09-bc3b-fa5d-bf7c-e150112bf577}" -2025-01-25T14:29:19.011Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:29:18.006275775 +0000 UTC m=+67.860600757 (2 nodes), 1.64µs from start of poll, 1.654417304s from cluster start. Timeout at 2025-01-25 14:29:19.01142569 +0000 UTC m=+68.865750672, 1.005149915s after stability -2025-01-25T14:29:19.023Z [INFO] server-65501e75-bc18-c9d2-9e54-6a5136e6decd: aborting pipeline replication: peer="{Voter server-977f50ab-3940-9f74-c6ca-9e4cffe8927a 977f50ab-3940-9f74-c6ca-9e4cffe8927a}" -2025-01-25T14:29:19.023Z [INFO] server-65501e75-bc18-c9d2-9e54-6a5136e6decd: aborting pipeline replication: peer="{Voter server-15c5dab3-22d4-23b7-e103-bf5a63a3ccd1 15c5dab3-22d4-23b7-e103-bf5a63a3ccd1}" +2025-01-25T14:40:00.193Z [INFO] server-e63e906c-43ce-e1bf-f059-234e7b10fea5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e63e906c-43ce-e1bf-f059-234e7b10fea5 Address:e63e906c-43ce-e1bf-f059-234e7b10fea5} {Suffrage:Voter ID:server-548025c0-e9c0-71ce-c2a8-21080b3e815d Address:548025c0-e9c0-71ce-c2a8-21080b3e815d} {Suffrage:Voter ID:server-c82a7e7e-acf0-71c6-a8bc-fdfbf6fbbbb7 Address:c82a7e7e-acf0-71c6-a8bc-fdfbf6fbbbb7}]" +2025-01-25T14:40:00.193Z [INFO] server-e63e906c-43ce-e1bf-f059-234e7b10fea5: entering follower state: follower="Node at e63e906c-43ce-e1bf-f059-234e7b10fea5 [Follower]" leader-address= leader-id= +2025-01-25T14:40:00.194Z [INFO] server-548025c0-e9c0-71ce-c2a8-21080b3e815d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e63e906c-43ce-e1bf-f059-234e7b10fea5 Address:e63e906c-43ce-e1bf-f059-234e7b10fea5} {Suffrage:Voter ID:server-548025c0-e9c0-71ce-c2a8-21080b3e815d Address:548025c0-e9c0-71ce-c2a8-21080b3e815d} {Suffrage:Voter ID:server-c82a7e7e-acf0-71c6-a8bc-fdfbf6fbbbb7 Address:c82a7e7e-acf0-71c6-a8bc-fdfbf6fbbbb7}]" +2025-01-25T14:40:00.194Z [INFO] server-548025c0-e9c0-71ce-c2a8-21080b3e815d: entering follower state: follower="Node at 548025c0-e9c0-71ce-c2a8-21080b3e815d [Follower]" leader-address= leader-id= +2025-01-25T14:40:00.195Z [INFO] server-c82a7e7e-acf0-71c6-a8bc-fdfbf6fbbbb7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e63e906c-43ce-e1bf-f059-234e7b10fea5 Address:e63e906c-43ce-e1bf-f059-234e7b10fea5} {Suffrage:Voter ID:server-548025c0-e9c0-71ce-c2a8-21080b3e815d Address:548025c0-e9c0-71ce-c2a8-21080b3e815d} {Suffrage:Voter ID:server-c82a7e7e-acf0-71c6-a8bc-fdfbf6fbbbb7 Address:c82a7e7e-acf0-71c6-a8bc-fdfbf6fbbbb7}]" +2025-01-25T14:40:00.195Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:00.195Z [INFO] server-c82a7e7e-acf0-71c6-a8bc-fdfbf6fbbbb7: entering follower state: follower="Node at c82a7e7e-acf0-71c6-a8bc-fdfbf6fbbbb7 [Follower]" leader-address= leader-id= +2025-01-25T14:40:00.765Z [WARN] server-c82a7e7e-acf0-71c6-a8bc-fdfbf6fbbbb7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:00.765Z [INFO] server-c82a7e7e-acf0-71c6-a8bc-fdfbf6fbbbb7: entering candidate state: node="Node at c82a7e7e-acf0-71c6-a8bc-fdfbf6fbbbb7 [Candidate]" term=2 +2025-01-25T14:40:00.766Z [INFO] server-c82a7e7e-acf0-71c6-a8bc-fdfbf6fbbbb7: election won: term=2 tally=2 +2025-01-25T14:40:00.766Z [INFO] server-c82a7e7e-acf0-71c6-a8bc-fdfbf6fbbbb7: entering leader state: leader="Node at c82a7e7e-acf0-71c6-a8bc-fdfbf6fbbbb7 [Leader]" +2025-01-25T14:40:00.766Z [INFO] server-c82a7e7e-acf0-71c6-a8bc-fdfbf6fbbbb7: added peer, starting replication: peer=server-e63e906c-43ce-e1bf-f059-234e7b10fea5 +2025-01-25T14:40:00.766Z [INFO] server-c82a7e7e-acf0-71c6-a8bc-fdfbf6fbbbb7: added peer, starting replication: peer=server-548025c0-e9c0-71ce-c2a8-21080b3e815d +2025-01-25T14:40:00.767Z [INFO] server-c82a7e7e-acf0-71c6-a8bc-fdfbf6fbbbb7: pipelining replication: peer="{Voter server-e63e906c-43ce-e1bf-f059-234e7b10fea5 e63e906c-43ce-e1bf-f059-234e7b10fea5}" +2025-01-25T14:40:00.767Z [INFO] server-c82a7e7e-acf0-71c6-a8bc-fdfbf6fbbbb7: pipelining replication: peer="{Voter server-548025c0-e9c0-71ce-c2a8-21080b3e815d 548025c0-e9c0-71ce-c2a8-21080b3e815d}" +2025-01-25T14:40:01.773Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:00.766409647 +0000 UTC m=+69.812248965 (1 nodes), 570.826453ms from start of poll, 573.484711ms from cluster start. Timeout at 2025-01-25 14:40:01.771425105 +0000 UTC m=+70.817264423, 1.005015458s after stability +2025-01-25T14:40:01.785Z [INFO] server-c82a7e7e-acf0-71c6-a8bc-fdfbf6fbbbb7: starting snapshot up to: index=12 +2025-01-25T14:40:01.785Z [INFO] creating new snapshot: path=/tmp/raft400316007/snapshots/2-12-1737816001785.tmp +2025-01-25T14:40:01.789Z [INFO] server-c82a7e7e-acf0-71c6-a8bc-fdfbf6fbbbb7: snapshot complete up to: index=12 +2025-01-25T14:40:01.796Z [INFO] server-dd6eb6f6-cc2b-30e4-031b-554446c52d61: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dd6eb6f6-cc2b-30e4-031b-554446c52d61 Address:dd6eb6f6-cc2b-30e4-031b-554446c52d61} {Suffrage:Voter ID:server-36537e12-7b06-1676-1250-fa2857e1ea00 Address:36537e12-7b06-1676-1250-fa2857e1ea00} {Suffrage:Voter ID:server-a67c46a0-b27b-e6ce-9d1b-0fd649cc6e15 Address:a67c46a0-b27b-e6ce-9d1b-0fd649cc6e15}]" +2025-01-25T14:40:01.797Z [INFO] server-dd6eb6f6-cc2b-30e4-031b-554446c52d61: entering follower state: follower="Node at dd6eb6f6-cc2b-30e4-031b-554446c52d61 [Follower]" leader-address= leader-id= +2025-01-25T14:40:01.797Z [INFO] server-36537e12-7b06-1676-1250-fa2857e1ea00: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dd6eb6f6-cc2b-30e4-031b-554446c52d61 Address:dd6eb6f6-cc2b-30e4-031b-554446c52d61} {Suffrage:Voter ID:server-36537e12-7b06-1676-1250-fa2857e1ea00 Address:36537e12-7b06-1676-1250-fa2857e1ea00} {Suffrage:Voter ID:server-a67c46a0-b27b-e6ce-9d1b-0fd649cc6e15 Address:a67c46a0-b27b-e6ce-9d1b-0fd649cc6e15}]" +2025-01-25T14:40:01.797Z [INFO] server-36537e12-7b06-1676-1250-fa2857e1ea00: entering follower state: follower="Node at 36537e12-7b06-1676-1250-fa2857e1ea00 [Follower]" leader-address= leader-id= +2025-01-25T14:40:01.797Z [INFO] server-a67c46a0-b27b-e6ce-9d1b-0fd649cc6e15: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dd6eb6f6-cc2b-30e4-031b-554446c52d61 Address:dd6eb6f6-cc2b-30e4-031b-554446c52d61} {Suffrage:Voter ID:server-36537e12-7b06-1676-1250-fa2857e1ea00 Address:36537e12-7b06-1676-1250-fa2857e1ea00} {Suffrage:Voter ID:server-a67c46a0-b27b-e6ce-9d1b-0fd649cc6e15 Address:a67c46a0-b27b-e6ce-9d1b-0fd649cc6e15}]" +2025-01-25T14:40:01.798Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:01.798Z [INFO] server-a67c46a0-b27b-e6ce-9d1b-0fd649cc6e15: entering follower state: follower="Node at a67c46a0-b27b-e6ce-9d1b-0fd649cc6e15 [Follower]" leader-address= leader-id= +2025-01-25T14:40:02.324Z [WARN] server-a67c46a0-b27b-e6ce-9d1b-0fd649cc6e15: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:02.324Z [INFO] server-a67c46a0-b27b-e6ce-9d1b-0fd649cc6e15: entering candidate state: node="Node at a67c46a0-b27b-e6ce-9d1b-0fd649cc6e15 [Candidate]" term=2 +2025-01-25T14:40:02.324Z [INFO] server-a67c46a0-b27b-e6ce-9d1b-0fd649cc6e15: election won: term=2 tally=2 +2025-01-25T14:40:02.324Z [INFO] server-a67c46a0-b27b-e6ce-9d1b-0fd649cc6e15: entering leader state: leader="Node at a67c46a0-b27b-e6ce-9d1b-0fd649cc6e15 [Leader]" +2025-01-25T14:40:02.324Z [INFO] server-a67c46a0-b27b-e6ce-9d1b-0fd649cc6e15: added peer, starting replication: peer=server-dd6eb6f6-cc2b-30e4-031b-554446c52d61 +2025-01-25T14:40:02.325Z [INFO] server-a67c46a0-b27b-e6ce-9d1b-0fd649cc6e15: added peer, starting replication: peer=server-36537e12-7b06-1676-1250-fa2857e1ea00 +2025-01-25T14:40:02.325Z [INFO] server-a67c46a0-b27b-e6ce-9d1b-0fd649cc6e15: pipelining replication: peer="{Voter server-dd6eb6f6-cc2b-30e4-031b-554446c52d61 dd6eb6f6-cc2b-30e4-031b-554446c52d61}" +2025-01-25T14:40:02.325Z [INFO] server-a67c46a0-b27b-e6ce-9d1b-0fd649cc6e15: pipelining replication: peer="{Voter server-36537e12-7b06-1676-1250-fa2857e1ea00 36537e12-7b06-1676-1250-fa2857e1ea00}" +2025-01-25T14:40:03.330Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:02.324907601 +0000 UTC m=+71.370746919 (1 nodes), 526.772534ms from start of poll, 528.52283ms from cluster start. Timeout at 2025-01-25 14:40:03.329919268 +0000 UTC m=+72.375758586, 1.005011667s after stability +2025-01-25T14:40:03.331Z [INFO] creating new snapshot: path=/tmp/raft738740473/snapshots/2-113-1737816003331.tmp +2025-01-25T14:40:03.333Z [INFO] server-a67c46a0-b27b-e6ce-9d1b-0fd649cc6e15: copied to local snapshot: bytes=71 +2025-01-25T14:40:03.334Z [INFO] server-a67c46a0-b27b-e6ce-9d1b-0fd649cc6e15: snapshot restore progress: id=2-113-1737816003331 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:03.334Z [INFO] server-a67c46a0-b27b-e6ce-9d1b-0fd649cc6e15: removing all old logs from log store +2025-01-25T14:40:03.334Z [INFO] server-a67c46a0-b27b-e6ce-9d1b-0fd649cc6e15: compacting logs: from=1 to=2 +2025-01-25T14:40:03.335Z [INFO] server-a67c46a0-b27b-e6ce-9d1b-0fd649cc6e15: restored user snapshot: index=113 +2025-01-25T14:40:03.335Z [ERROR] server-a67c46a0-b27b-e6ce-9d1b-0fd649cc6e15: failed to get log: index=2 error="log not found" +2025-01-25T14:40:03.335Z [ERROR] server-a67c46a0-b27b-e6ce-9d1b-0fd649cc6e15: failed to get log: index=2 error="log not found" +2025-01-25T14:40:03.335Z [INFO] server-a67c46a0-b27b-e6ce-9d1b-0fd649cc6e15: aborting pipeline replication: peer="{Voter server-36537e12-7b06-1676-1250-fa2857e1ea00 36537e12-7b06-1676-1250-fa2857e1ea00}" +2025-01-25T14:40:03.335Z [INFO] server-a67c46a0-b27b-e6ce-9d1b-0fd649cc6e15: aborting pipeline replication: peer="{Voter server-dd6eb6f6-cc2b-30e4-031b-554446c52d61 dd6eb6f6-cc2b-30e4-031b-554446c52d61}" +2025-01-25T14:40:03.345Z [ERROR] server-a67c46a0-b27b-e6ce-9d1b-0fd649cc6e15: failed to get log: index=2 error="log not found" +2025-01-25T14:40:03.346Z [ERROR] server-a67c46a0-b27b-e6ce-9d1b-0fd649cc6e15: failed to get log: index=2 error="log not found" +2025-01-25T14:40:03.347Z [INFO] creating new snapshot: path=/tmp/raft187743725/snapshots/2-113-1737816003347.tmp +2025-01-25T14:40:03.349Z [INFO] creating new snapshot: path=/tmp/raft20720472/snapshots/2-113-1737816003349.tmp +2025-01-25T14:40:03.349Z [INFO] server-36537e12-7b06-1676-1250-fa2857e1ea00: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:03.353Z [INFO] server-36537e12-7b06-1676-1250-fa2857e1ea00: copied to local snapshot: bytes=71 +2025-01-25T14:40:03.359Z [INFO] server-dd6eb6f6-cc2b-30e4-031b-554446c52d61: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:03.364Z [INFO] server-dd6eb6f6-cc2b-30e4-031b-554446c52d61: copied to local snapshot: bytes=71 +2025-01-25T14:40:03.365Z [INFO] server-dd6eb6f6-cc2b-30e4-031b-554446c52d61: snapshot restore progress: id=2-113-1737816003349 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:03.365Z [INFO] server-36537e12-7b06-1676-1250-fa2857e1ea00: snapshot restore progress: id=2-113-1737816003347 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:03.365Z [INFO] server-dd6eb6f6-cc2b-30e4-031b-554446c52d61: removing all old logs from log store +2025-01-25T14:40:03.365Z [INFO] server-dd6eb6f6-cc2b-30e4-031b-554446c52d61: compacting logs: from=1 to=2 +2025-01-25T14:40:03.365Z [INFO] server-36537e12-7b06-1676-1250-fa2857e1ea00: removing all old logs from log store +2025-01-25T14:40:03.365Z [INFO] server-dd6eb6f6-cc2b-30e4-031b-554446c52d61: Installed remote snapshot +2025-01-25T14:40:03.365Z [INFO] server-36537e12-7b06-1676-1250-fa2857e1ea00: compacting logs: from=1 to=2 +2025-01-25T14:40:03.365Z [INFO] server-36537e12-7b06-1676-1250-fa2857e1ea00: Installed remote snapshot +2025-01-25T14:40:03.366Z [WARN] server-36537e12-7b06-1676-1250-fa2857e1ea00: skipping application of old log: index=113 +2025-01-25T14:40:03.366Z [INFO] server-a67c46a0-b27b-e6ce-9d1b-0fd649cc6e15: pipelining replication: peer="{Voter server-dd6eb6f6-cc2b-30e4-031b-554446c52d61 dd6eb6f6-cc2b-30e4-031b-554446c52d61}" +2025-01-25T14:40:03.366Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:40:03.366Z [INFO] server-a67c46a0-b27b-e6ce-9d1b-0fd649cc6e15: pipelining replication: peer="{Voter server-36537e12-7b06-1676-1250-fa2857e1ea00 36537e12-7b06-1676-1250-fa2857e1ea00}" +2025-01-25T14:40:04.372Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:40:03.366936427 +0000 UTC m=+72.412775744 (2 nodes), 4.666µs from start of poll, 1.570551655s from cluster start. Timeout at 2025-01-25 14:40:04.372066803 +0000 UTC m=+73.417906412, 1.005130668s after stability +2025-01-25T14:40:04.387Z [INFO] server-c82a7e7e-acf0-71c6-a8bc-fdfbf6fbbbb7: aborting pipeline replication: peer="{Voter server-548025c0-e9c0-71ce-c2a8-21080b3e815d 548025c0-e9c0-71ce-c2a8-21080b3e815d}" +2025-01-25T14:40:04.387Z [INFO] server-c82a7e7e-acf0-71c6-a8bc-fdfbf6fbbbb7: aborting pipeline replication: peer="{Voter server-e63e906c-43ce-e1bf-f059-234e7b10fea5 e63e906c-43ce-e1bf-f059-234e7b10fea5}" === RUN TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_false -2025-01-25T14:29:19.026Z [INFO] server-a4fc1263-f4dd-5b81-3e5e-143819cefc59: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a4fc1263-f4dd-5b81-3e5e-143819cefc59 Address:a4fc1263-f4dd-5b81-3e5e-143819cefc59} {Suffrage:Voter ID:server-ef0b42df-2ed6-b5ce-48ea-d4ec9a38b275 Address:ef0b42df-2ed6-b5ce-48ea-d4ec9a38b275} {Suffrage:Voter ID:server-3f2a5ce6-8faf-d30a-b490-3e69de51833c Address:3f2a5ce6-8faf-d30a-b490-3e69de51833c}]" -2025-01-25T14:29:19.026Z [INFO] server-ef0b42df-2ed6-b5ce-48ea-d4ec9a38b275: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a4fc1263-f4dd-5b81-3e5e-143819cefc59 Address:a4fc1263-f4dd-5b81-3e5e-143819cefc59} {Suffrage:Voter ID:server-ef0b42df-2ed6-b5ce-48ea-d4ec9a38b275 Address:ef0b42df-2ed6-b5ce-48ea-d4ec9a38b275} {Suffrage:Voter ID:server-3f2a5ce6-8faf-d30a-b490-3e69de51833c Address:3f2a5ce6-8faf-d30a-b490-3e69de51833c}]" -2025-01-25T14:29:19.026Z [INFO] server-3f2a5ce6-8faf-d30a-b490-3e69de51833c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a4fc1263-f4dd-5b81-3e5e-143819cefc59 Address:a4fc1263-f4dd-5b81-3e5e-143819cefc59} {Suffrage:Voter ID:server-ef0b42df-2ed6-b5ce-48ea-d4ec9a38b275 Address:ef0b42df-2ed6-b5ce-48ea-d4ec9a38b275} {Suffrage:Voter ID:server-3f2a5ce6-8faf-d30a-b490-3e69de51833c Address:3f2a5ce6-8faf-d30a-b490-3e69de51833c}]" -2025-01-25T14:29:19.026Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:19.026Z [INFO] server-a4fc1263-f4dd-5b81-3e5e-143819cefc59: entering follower state: follower="Node at a4fc1263-f4dd-5b81-3e5e-143819cefc59 [Follower]" leader-address= leader-id= -2025-01-25T14:29:19.027Z [INFO] server-ef0b42df-2ed6-b5ce-48ea-d4ec9a38b275: entering follower state: follower="Node at ef0b42df-2ed6-b5ce-48ea-d4ec9a38b275 [Follower]" leader-address= leader-id= -2025-01-25T14:29:19.027Z [INFO] server-3f2a5ce6-8faf-d30a-b490-3e69de51833c: entering follower state: follower="Node at 3f2a5ce6-8faf-d30a-b490-3e69de51833c [Follower]" leader-address= leader-id= -2025-01-25T14:29:19.531Z [WARN] server-3f2a5ce6-8faf-d30a-b490-3e69de51833c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:19.531Z [INFO] server-3f2a5ce6-8faf-d30a-b490-3e69de51833c: entering candidate state: node="Node at 3f2a5ce6-8faf-d30a-b490-3e69de51833c [Candidate]" term=2 -2025-01-25T14:29:19.531Z [INFO] server-3f2a5ce6-8faf-d30a-b490-3e69de51833c: election won: term=2 tally=2 -2025-01-25T14:29:19.531Z [INFO] server-3f2a5ce6-8faf-d30a-b490-3e69de51833c: entering leader state: leader="Node at 3f2a5ce6-8faf-d30a-b490-3e69de51833c [Leader]" -2025-01-25T14:29:19.531Z [INFO] server-3f2a5ce6-8faf-d30a-b490-3e69de51833c: added peer, starting replication: peer=server-a4fc1263-f4dd-5b81-3e5e-143819cefc59 -2025-01-25T14:29:19.531Z [INFO] server-3f2a5ce6-8faf-d30a-b490-3e69de51833c: added peer, starting replication: peer=server-ef0b42df-2ed6-b5ce-48ea-d4ec9a38b275 -2025-01-25T14:29:19.531Z [INFO] server-3f2a5ce6-8faf-d30a-b490-3e69de51833c: pipelining replication: peer="{Voter server-a4fc1263-f4dd-5b81-3e5e-143819cefc59 a4fc1263-f4dd-5b81-3e5e-143819cefc59}" -2025-01-25T14:29:19.531Z [INFO] server-3f2a5ce6-8faf-d30a-b490-3e69de51833c: pipelining replication: peer="{Voter server-ef0b42df-2ed6-b5ce-48ea-d4ec9a38b275 ef0b42df-2ed6-b5ce-48ea-d4ec9a38b275}" -2025-01-25T14:29:20.537Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:19.531558205 +0000 UTC m=+69.385883187 (1 nodes), 504.801514ms from start of poll, 505.393892ms from cluster start. Timeout at 2025-01-25 14:29:20.536563365 +0000 UTC m=+70.390888347, 1.00500516s after stability -2025-01-25T14:29:20.539Z [INFO] server-3f2a5ce6-8faf-d30a-b490-3e69de51833c: starting snapshot up to: index=12 -2025-01-25T14:29:20.539Z [INFO] creating new snapshot: path=/tmp/raft4170409730/snapshots/2-12-1737815360539.tmp -2025-01-25T14:29:20.540Z [INFO] server-3f2a5ce6-8faf-d30a-b490-3e69de51833c: snapshot complete up to: index=12 -2025-01-25T14:29:20.541Z [INFO] creating new snapshot: path=/tmp/raft4170409730/snapshots/2-1013-1737815360541.tmp -2025-01-25T14:29:20.543Z [INFO] server-3f2a5ce6-8faf-d30a-b490-3e69de51833c: copied to local snapshot: bytes=71 -2025-01-25T14:29:20.543Z [INFO] server-3f2a5ce6-8faf-d30a-b490-3e69de51833c: snapshot restore progress: id=2-1013-1737815360541 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:20.543Z [INFO] server-3f2a5ce6-8faf-d30a-b490-3e69de51833c: restored user snapshot: index=1013 -2025-01-25T14:29:20.543Z [ERROR] server-3f2a5ce6-8faf-d30a-b490-3e69de51833c: failed to get log: index=23 error="log not found" -2025-01-25T14:29:20.543Z [INFO] server-3f2a5ce6-8faf-d30a-b490-3e69de51833c: aborting pipeline replication: peer="{Voter server-ef0b42df-2ed6-b5ce-48ea-d4ec9a38b275 ef0b42df-2ed6-b5ce-48ea-d4ec9a38b275}" -2025-01-25T14:29:20.543Z [ERROR] server-3f2a5ce6-8faf-d30a-b490-3e69de51833c: failed to get log: index=23 error="log not found" -2025-01-25T14:29:20.543Z [INFO] server-3f2a5ce6-8faf-d30a-b490-3e69de51833c: aborting pipeline replication: peer="{Voter server-a4fc1263-f4dd-5b81-3e5e-143819cefc59 a4fc1263-f4dd-5b81-3e5e-143819cefc59}" -2025-01-25T14:29:20.550Z [ERROR] server-3f2a5ce6-8faf-d30a-b490-3e69de51833c: failed to get log: index=23 error="log not found" -2025-01-25T14:29:20.551Z [INFO] creating new snapshot: path=/tmp/raft2983815109/snapshots/2-1013-1737815360551.tmp -2025-01-25T14:29:20.552Z [INFO] server-ef0b42df-2ed6-b5ce-48ea-d4ec9a38b275: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:20.553Z [ERROR] server-3f2a5ce6-8faf-d30a-b490-3e69de51833c: failed to get log: index=23 error="log not found" -2025-01-25T14:29:20.553Z [INFO] server-ef0b42df-2ed6-b5ce-48ea-d4ec9a38b275: copied to local snapshot: bytes=71 -2025-01-25T14:29:20.555Z [INFO] creating new snapshot: path=/tmp/raft2505429516/snapshots/2-1013-1737815360555.tmp -2025-01-25T14:29:20.556Z [INFO] server-ef0b42df-2ed6-b5ce-48ea-d4ec9a38b275: snapshot restore progress: id=2-1013-1737815360551 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:20.556Z [INFO] server-ef0b42df-2ed6-b5ce-48ea-d4ec9a38b275: Installed remote snapshot -2025-01-25T14:29:20.557Z [WARN] server-ef0b42df-2ed6-b5ce-48ea-d4ec9a38b275: skipping application of old log: index=22 -2025-01-25T14:29:20.557Z [INFO] server-3f2a5ce6-8faf-d30a-b490-3e69de51833c: pipelining replication: peer="{Voter server-ef0b42df-2ed6-b5ce-48ea-d4ec9a38b275 ef0b42df-2ed6-b5ce-48ea-d4ec9a38b275}" -2025-01-25T14:29:20.557Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:29:20.557Z [INFO] server-a4fc1263-f4dd-5b81-3e5e-143819cefc59: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:20.558Z [INFO] server-a4fc1263-f4dd-5b81-3e5e-143819cefc59: copied to local snapshot: bytes=71 -2025-01-25T14:29:20.559Z [INFO] server-a4fc1263-f4dd-5b81-3e5e-143819cefc59: snapshot restore progress: id=2-1013-1737815360555 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:20.559Z [INFO] server-a4fc1263-f4dd-5b81-3e5e-143819cefc59: Installed remote snapshot -2025-01-25T14:29:20.559Z [INFO] server-3f2a5ce6-8faf-d30a-b490-3e69de51833c: pipelining replication: peer="{Voter server-a4fc1263-f4dd-5b81-3e5e-143819cefc59 a4fc1263-f4dd-5b81-3e5e-143819cefc59}" -2025-01-25T14:29:21.563Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:29:20.557842233 +0000 UTC m=+70.412167215 (2 nodes), 1.44µs from start of poll, 1.53167792s from cluster start. Timeout at 2025-01-25 14:29:21.562989267 +0000 UTC m=+71.417314249, 1.005147034s after stability -2025-01-25T14:29:21.581Z [INFO] server-3f2a5ce6-8faf-d30a-b490-3e69de51833c: aborting pipeline replication: peer="{Voter server-ef0b42df-2ed6-b5ce-48ea-d4ec9a38b275 ef0b42df-2ed6-b5ce-48ea-d4ec9a38b275}" -2025-01-25T14:29:21.581Z [INFO] server-3f2a5ce6-8faf-d30a-b490-3e69de51833c: aborting pipeline replication: peer="{Voter server-a4fc1263-f4dd-5b81-3e5e-143819cefc59 a4fc1263-f4dd-5b81-3e5e-143819cefc59}" +2025-01-25T14:40:04.395Z [INFO] server-06c46638-5c15-5e42-4d61-fdab63747d2e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-06c46638-5c15-5e42-4d61-fdab63747d2e Address:06c46638-5c15-5e42-4d61-fdab63747d2e} {Suffrage:Voter ID:server-06358017-1616-d6fb-c12a-c2a2b7fb749b Address:06358017-1616-d6fb-c12a-c2a2b7fb749b} {Suffrage:Voter ID:server-48c93e89-3d04-ccba-70ad-66396594897e Address:48c93e89-3d04-ccba-70ad-66396594897e}]" +2025-01-25T14:40:04.395Z [INFO] server-06c46638-5c15-5e42-4d61-fdab63747d2e: entering follower state: follower="Node at 06c46638-5c15-5e42-4d61-fdab63747d2e [Follower]" leader-address= leader-id= +2025-01-25T14:40:04.395Z [INFO] server-06358017-1616-d6fb-c12a-c2a2b7fb749b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-06c46638-5c15-5e42-4d61-fdab63747d2e Address:06c46638-5c15-5e42-4d61-fdab63747d2e} {Suffrage:Voter ID:server-06358017-1616-d6fb-c12a-c2a2b7fb749b Address:06358017-1616-d6fb-c12a-c2a2b7fb749b} {Suffrage:Voter ID:server-48c93e89-3d04-ccba-70ad-66396594897e Address:48c93e89-3d04-ccba-70ad-66396594897e}]" +2025-01-25T14:40:04.396Z [INFO] server-06358017-1616-d6fb-c12a-c2a2b7fb749b: entering follower state: follower="Node at 06358017-1616-d6fb-c12a-c2a2b7fb749b [Follower]" leader-address= leader-id= +2025-01-25T14:40:04.396Z [INFO] server-48c93e89-3d04-ccba-70ad-66396594897e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-06c46638-5c15-5e42-4d61-fdab63747d2e Address:06c46638-5c15-5e42-4d61-fdab63747d2e} {Suffrage:Voter ID:server-06358017-1616-d6fb-c12a-c2a2b7fb749b Address:06358017-1616-d6fb-c12a-c2a2b7fb749b} {Suffrage:Voter ID:server-48c93e89-3d04-ccba-70ad-66396594897e Address:48c93e89-3d04-ccba-70ad-66396594897e}]" +2025-01-25T14:40:04.397Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:04.397Z [INFO] server-48c93e89-3d04-ccba-70ad-66396594897e: entering follower state: follower="Node at 48c93e89-3d04-ccba-70ad-66396594897e [Follower]" leader-address= leader-id= +2025-01-25T14:40:04.947Z [WARN] server-06c46638-5c15-5e42-4d61-fdab63747d2e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:04.947Z [INFO] server-06c46638-5c15-5e42-4d61-fdab63747d2e: entering candidate state: node="Node at 06c46638-5c15-5e42-4d61-fdab63747d2e [Candidate]" term=2 +2025-01-25T14:40:04.947Z [INFO] server-06c46638-5c15-5e42-4d61-fdab63747d2e: election won: term=2 tally=2 +2025-01-25T14:40:04.948Z [INFO] server-06c46638-5c15-5e42-4d61-fdab63747d2e: entering leader state: leader="Node at 06c46638-5c15-5e42-4d61-fdab63747d2e [Leader]" +2025-01-25T14:40:04.948Z [INFO] server-06c46638-5c15-5e42-4d61-fdab63747d2e: added peer, starting replication: peer=server-06358017-1616-d6fb-c12a-c2a2b7fb749b +2025-01-25T14:40:04.948Z [INFO] server-06c46638-5c15-5e42-4d61-fdab63747d2e: added peer, starting replication: peer=server-48c93e89-3d04-ccba-70ad-66396594897e +2025-01-25T14:40:04.948Z [INFO] server-06c46638-5c15-5e42-4d61-fdab63747d2e: pipelining replication: peer="{Voter server-06358017-1616-d6fb-c12a-c2a2b7fb749b 06358017-1616-d6fb-c12a-c2a2b7fb749b}" +2025-01-25T14:40:04.949Z [INFO] server-06c46638-5c15-5e42-4d61-fdab63747d2e: pipelining replication: peer="{Voter server-48c93e89-3d04-ccba-70ad-66396594897e 48c93e89-3d04-ccba-70ad-66396594897e}" +2025-01-25T14:40:05.953Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:04.948278656 +0000 UTC m=+73.994117973 (1 nodes), 550.94652ms from start of poll, 553.886237ms from cluster start. Timeout at 2025-01-25 14:40:05.953326489 +0000 UTC m=+74.999165806, 1.005047833s after stability +2025-01-25T14:40:05.957Z [INFO] server-06c46638-5c15-5e42-4d61-fdab63747d2e: starting snapshot up to: index=12 +2025-01-25T14:40:05.957Z [INFO] creating new snapshot: path=/tmp/raft571891524/snapshots/2-12-1737816005957.tmp +2025-01-25T14:40:05.962Z [INFO] server-06c46638-5c15-5e42-4d61-fdab63747d2e: snapshot complete up to: index=12 +2025-01-25T14:40:05.967Z [INFO] creating new snapshot: path=/tmp/raft571891524/snapshots/2-1013-1737816005967.tmp +2025-01-25T14:40:05.972Z [INFO] server-06c46638-5c15-5e42-4d61-fdab63747d2e: copied to local snapshot: bytes=71 +2025-01-25T14:40:05.973Z [INFO] server-06c46638-5c15-5e42-4d61-fdab63747d2e: snapshot restore progress: id=2-1013-1737816005967 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:05.973Z [INFO] server-06c46638-5c15-5e42-4d61-fdab63747d2e: restored user snapshot: index=1013 +2025-01-25T14:40:05.974Z [ERROR] server-06c46638-5c15-5e42-4d61-fdab63747d2e: failed to get log: index=23 error="log not found" +2025-01-25T14:40:05.974Z [ERROR] server-06c46638-5c15-5e42-4d61-fdab63747d2e: failed to get log: index=23 error="log not found" +2025-01-25T14:40:05.974Z [INFO] server-06c46638-5c15-5e42-4d61-fdab63747d2e: aborting pipeline replication: peer="{Voter server-48c93e89-3d04-ccba-70ad-66396594897e 48c93e89-3d04-ccba-70ad-66396594897e}" +2025-01-25T14:40:05.974Z [INFO] server-06c46638-5c15-5e42-4d61-fdab63747d2e: aborting pipeline replication: peer="{Voter server-06358017-1616-d6fb-c12a-c2a2b7fb749b 06358017-1616-d6fb-c12a-c2a2b7fb749b}" +2025-01-25T14:40:05.980Z [ERROR] server-06c46638-5c15-5e42-4d61-fdab63747d2e: failed to get log: index=23 error="log not found" +2025-01-25T14:40:05.983Z [INFO] creating new snapshot: path=/tmp/raft1020148577/snapshots/2-1013-1737816005983.tmp +2025-01-25T14:40:05.984Z [ERROR] server-06c46638-5c15-5e42-4d61-fdab63747d2e: failed to get log: index=23 error="log not found" +2025-01-25T14:40:05.985Z [INFO] server-06358017-1616-d6fb-c12a-c2a2b7fb749b: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:05.986Z [INFO] creating new snapshot: path=/tmp/raft2756079466/snapshots/2-1013-1737816005986.tmp +2025-01-25T14:40:05.987Z [INFO] server-06358017-1616-d6fb-c12a-c2a2b7fb749b: copied to local snapshot: bytes=71 +2025-01-25T14:40:05.988Z [INFO] server-06358017-1616-d6fb-c12a-c2a2b7fb749b: snapshot restore progress: id=2-1013-1737816005983 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:05.988Z [INFO] server-48c93e89-3d04-ccba-70ad-66396594897e: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:05.988Z [INFO] server-06358017-1616-d6fb-c12a-c2a2b7fb749b: Installed remote snapshot +2025-01-25T14:40:05.989Z [INFO] server-06c46638-5c15-5e42-4d61-fdab63747d2e: pipelining replication: peer="{Voter server-06358017-1616-d6fb-c12a-c2a2b7fb749b 06358017-1616-d6fb-c12a-c2a2b7fb749b}" +2025-01-25T14:40:05.989Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:40:05.990Z [INFO] server-48c93e89-3d04-ccba-70ad-66396594897e: copied to local snapshot: bytes=71 +2025-01-25T14:40:05.997Z [INFO] server-48c93e89-3d04-ccba-70ad-66396594897e: snapshot restore progress: id=2-1013-1737816005986 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:06.000Z [INFO] server-48c93e89-3d04-ccba-70ad-66396594897e: Installed remote snapshot +2025-01-25T14:40:06.005Z [INFO] server-06c46638-5c15-5e42-4d61-fdab63747d2e: pipelining replication: peer="{Voter server-48c93e89-3d04-ccba-70ad-66396594897e 48c93e89-3d04-ccba-70ad-66396594897e}" +2025-01-25T14:40:06.995Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:40:05.989602937 +0000 UTC m=+75.035442255 (2 nodes), 6.125µs from start of poll, 1.595210519s from cluster start. Timeout at 2025-01-25 14:40:06.994763937 +0000 UTC m=+76.040603255, 1.005161s after stability +2025-01-25T14:40:07.008Z [INFO] server-06c46638-5c15-5e42-4d61-fdab63747d2e: aborting pipeline replication: peer="{Voter server-48c93e89-3d04-ccba-70ad-66396594897e 48c93e89-3d04-ccba-70ad-66396594897e}" +2025-01-25T14:40:07.008Z [INFO] server-06c46638-5c15-5e42-4d61-fdab63747d2e: aborting pipeline replication: peer="{Voter server-06358017-1616-d6fb-c12a-c2a2b7fb749b 06358017-1616-d6fb-c12a-c2a2b7fb749b}" === RUN TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_false -2025-01-25T14:29:21.584Z [INFO] server-87f24fd1-dcea-5868-dbe7-1c7d10a8f524: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-87f24fd1-dcea-5868-dbe7-1c7d10a8f524 Address:87f24fd1-dcea-5868-dbe7-1c7d10a8f524} {Suffrage:Voter ID:server-d6ffaeb2-f826-a3d6-e63e-e5b8b7427335 Address:d6ffaeb2-f826-a3d6-e63e-e5b8b7427335} {Suffrage:Voter ID:server-1c34bbb4-c6eb-17ae-08c8-3329b23ec47a Address:1c34bbb4-c6eb-17ae-08c8-3329b23ec47a}]" -2025-01-25T14:29:21.584Z [INFO] server-d6ffaeb2-f826-a3d6-e63e-e5b8b7427335: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-87f24fd1-dcea-5868-dbe7-1c7d10a8f524 Address:87f24fd1-dcea-5868-dbe7-1c7d10a8f524} {Suffrage:Voter ID:server-d6ffaeb2-f826-a3d6-e63e-e5b8b7427335 Address:d6ffaeb2-f826-a3d6-e63e-e5b8b7427335} {Suffrage:Voter ID:server-1c34bbb4-c6eb-17ae-08c8-3329b23ec47a Address:1c34bbb4-c6eb-17ae-08c8-3329b23ec47a}]" -2025-01-25T14:29:21.585Z [INFO] server-1c34bbb4-c6eb-17ae-08c8-3329b23ec47a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-87f24fd1-dcea-5868-dbe7-1c7d10a8f524 Address:87f24fd1-dcea-5868-dbe7-1c7d10a8f524} {Suffrage:Voter ID:server-d6ffaeb2-f826-a3d6-e63e-e5b8b7427335 Address:d6ffaeb2-f826-a3d6-e63e-e5b8b7427335} {Suffrage:Voter ID:server-1c34bbb4-c6eb-17ae-08c8-3329b23ec47a Address:1c34bbb4-c6eb-17ae-08c8-3329b23ec47a}]" -2025-01-25T14:29:21.585Z [INFO] server-d6ffaeb2-f826-a3d6-e63e-e5b8b7427335: entering follower state: follower="Node at d6ffaeb2-f826-a3d6-e63e-e5b8b7427335 [Follower]" leader-address= leader-id= -2025-01-25T14:29:21.585Z [INFO] server-1c34bbb4-c6eb-17ae-08c8-3329b23ec47a: entering follower state: follower="Node at 1c34bbb4-c6eb-17ae-08c8-3329b23ec47a [Follower]" leader-address= leader-id= -2025-01-25T14:29:21.584Z [INFO] server-87f24fd1-dcea-5868-dbe7-1c7d10a8f524: entering follower state: follower="Node at 87f24fd1-dcea-5868-dbe7-1c7d10a8f524 [Follower]" leader-address= leader-id= -2025-01-25T14:29:21.585Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:22.163Z [WARN] server-87f24fd1-dcea-5868-dbe7-1c7d10a8f524: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:22.163Z [INFO] server-87f24fd1-dcea-5868-dbe7-1c7d10a8f524: entering candidate state: node="Node at 87f24fd1-dcea-5868-dbe7-1c7d10a8f524 [Candidate]" term=2 -2025-01-25T14:29:22.163Z [INFO] server-87f24fd1-dcea-5868-dbe7-1c7d10a8f524: election won: term=2 tally=2 -2025-01-25T14:29:22.163Z [INFO] server-87f24fd1-dcea-5868-dbe7-1c7d10a8f524: entering leader state: leader="Node at 87f24fd1-dcea-5868-dbe7-1c7d10a8f524 [Leader]" -2025-01-25T14:29:22.163Z [INFO] server-87f24fd1-dcea-5868-dbe7-1c7d10a8f524: added peer, starting replication: peer=server-d6ffaeb2-f826-a3d6-e63e-e5b8b7427335 -2025-01-25T14:29:22.163Z [INFO] server-87f24fd1-dcea-5868-dbe7-1c7d10a8f524: added peer, starting replication: peer=server-1c34bbb4-c6eb-17ae-08c8-3329b23ec47a -2025-01-25T14:29:22.163Z [INFO] server-87f24fd1-dcea-5868-dbe7-1c7d10a8f524: pipelining replication: peer="{Voter server-1c34bbb4-c6eb-17ae-08c8-3329b23ec47a 1c34bbb4-c6eb-17ae-08c8-3329b23ec47a}" -2025-01-25T14:29:22.163Z [INFO] server-87f24fd1-dcea-5868-dbe7-1c7d10a8f524: pipelining replication: peer="{Voter server-d6ffaeb2-f826-a3d6-e63e-e5b8b7427335 d6ffaeb2-f826-a3d6-e63e-e5b8b7427335}" -2025-01-25T14:29:23.169Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:22.163517051 +0000 UTC m=+72.017842033 (1 nodes), 578.334748ms from start of poll, 579.053802ms from cluster start. Timeout at 2025-01-25 14:29:23.168524131 +0000 UTC m=+73.022849113, 1.00500708s after stability -2025-01-25T14:29:23.175Z [INFO] server-87f24fd1-dcea-5868-dbe7-1c7d10a8f524: starting snapshot up to: index=12 -2025-01-25T14:29:23.175Z [INFO] creating new snapshot: path=/tmp/raft3204552013/snapshots/2-12-1737815363175.tmp -2025-01-25T14:29:23.176Z [INFO] server-87f24fd1-dcea-5868-dbe7-1c7d10a8f524: snapshot complete up to: index=12 -2025-01-25T14:29:23.178Z [INFO] creating new snapshot: path=/tmp/raft3204552013/snapshots/2-1013-1737815363178.tmp -2025-01-25T14:29:23.180Z [INFO] server-87f24fd1-dcea-5868-dbe7-1c7d10a8f524: copied to local snapshot: bytes=71 -2025-01-25T14:29:23.180Z [INFO] server-87f24fd1-dcea-5868-dbe7-1c7d10a8f524: snapshot restore progress: id=2-1013-1737815363178 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:23.180Z [INFO] server-87f24fd1-dcea-5868-dbe7-1c7d10a8f524: removing all old logs from log store -2025-01-25T14:29:23.180Z [INFO] server-87f24fd1-dcea-5868-dbe7-1c7d10a8f524: compacting logs: from=1 to=22 -2025-01-25T14:29:23.180Z [INFO] server-87f24fd1-dcea-5868-dbe7-1c7d10a8f524: restored user snapshot: index=1013 -2025-01-25T14:29:23.180Z [ERROR] server-87f24fd1-dcea-5868-dbe7-1c7d10a8f524: failed to get log: index=22 error="log not found" -2025-01-25T14:29:23.180Z [INFO] server-87f24fd1-dcea-5868-dbe7-1c7d10a8f524: aborting pipeline replication: peer="{Voter server-1c34bbb4-c6eb-17ae-08c8-3329b23ec47a 1c34bbb4-c6eb-17ae-08c8-3329b23ec47a}" -2025-01-25T14:29:23.180Z [ERROR] server-87f24fd1-dcea-5868-dbe7-1c7d10a8f524: failed to get log: index=22 error="log not found" -2025-01-25T14:29:23.180Z [INFO] server-87f24fd1-dcea-5868-dbe7-1c7d10a8f524: aborting pipeline replication: peer="{Voter server-d6ffaeb2-f826-a3d6-e63e-e5b8b7427335 d6ffaeb2-f826-a3d6-e63e-e5b8b7427335}" -2025-01-25T14:29:23.186Z [ERROR] server-87f24fd1-dcea-5868-dbe7-1c7d10a8f524: failed to get log: index=22 error="log not found" -2025-01-25T14:29:23.186Z [ERROR] server-87f24fd1-dcea-5868-dbe7-1c7d10a8f524: failed to get log: index=22 error="log not found" -2025-01-25T14:29:23.186Z [INFO] creating new snapshot: path=/tmp/raft1608611309/snapshots/2-1013-1737815363186.tmp -2025-01-25T14:29:23.187Z [INFO] creating new snapshot: path=/tmp/raft3898775036/snapshots/2-1013-1737815363187.tmp -2025-01-25T14:29:23.187Z [INFO] server-1c34bbb4-c6eb-17ae-08c8-3329b23ec47a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:23.188Z [INFO] server-d6ffaeb2-f826-a3d6-e63e-e5b8b7427335: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:23.188Z [INFO] server-1c34bbb4-c6eb-17ae-08c8-3329b23ec47a: copied to local snapshot: bytes=71 -2025-01-25T14:29:23.188Z [INFO] server-d6ffaeb2-f826-a3d6-e63e-e5b8b7427335: copied to local snapshot: bytes=71 -2025-01-25T14:29:23.189Z [INFO] server-1c34bbb4-c6eb-17ae-08c8-3329b23ec47a: snapshot restore progress: id=2-1013-1737815363186 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:23.189Z [INFO] server-1c34bbb4-c6eb-17ae-08c8-3329b23ec47a: removing all old logs from log store -2025-01-25T14:29:23.189Z [INFO] server-d6ffaeb2-f826-a3d6-e63e-e5b8b7427335: snapshot restore progress: id=2-1013-1737815363187 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:23.189Z [INFO] server-1c34bbb4-c6eb-17ae-08c8-3329b23ec47a: compacting logs: from=1 to=22 -2025-01-25T14:29:23.189Z [INFO] server-d6ffaeb2-f826-a3d6-e63e-e5b8b7427335: removing all old logs from log store -2025-01-25T14:29:23.189Z [INFO] server-d6ffaeb2-f826-a3d6-e63e-e5b8b7427335: compacting logs: from=1 to=22 -2025-01-25T14:29:23.189Z [INFO] server-1c34bbb4-c6eb-17ae-08c8-3329b23ec47a: Installed remote snapshot -2025-01-25T14:29:23.189Z [INFO] server-d6ffaeb2-f826-a3d6-e63e-e5b8b7427335: Installed remote snapshot -2025-01-25T14:29:23.189Z [WARN] server-d6ffaeb2-f826-a3d6-e63e-e5b8b7427335: skipping application of old log: index=1013 -2025-01-25T14:29:23.189Z [INFO] server-87f24fd1-dcea-5868-dbe7-1c7d10a8f524: pipelining replication: peer="{Voter server-d6ffaeb2-f826-a3d6-e63e-e5b8b7427335 d6ffaeb2-f826-a3d6-e63e-e5b8b7427335}" -2025-01-25T14:29:23.189Z [WARN] server-1c34bbb4-c6eb-17ae-08c8-3329b23ec47a: skipping application of old log: index=22 -2025-01-25T14:29:23.189Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:29:23.189Z [INFO] server-87f24fd1-dcea-5868-dbe7-1c7d10a8f524: pipelining replication: peer="{Voter server-1c34bbb4-c6eb-17ae-08c8-3329b23ec47a 1c34bbb4-c6eb-17ae-08c8-3329b23ec47a}" -2025-01-25T14:29:24.194Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:29:23.189732721 +0000 UTC m=+73.044057703 (2 nodes), 1.48µs from start of poll, 1.605269472s from cluster start. Timeout at 2025-01-25 14:29:24.194866956 +0000 UTC m=+74.049191938, 1.005134235s after stability -2025-01-25T14:29:24.208Z [INFO] server-87f24fd1-dcea-5868-dbe7-1c7d10a8f524: aborting pipeline replication: peer="{Voter server-1c34bbb4-c6eb-17ae-08c8-3329b23ec47a 1c34bbb4-c6eb-17ae-08c8-3329b23ec47a}" -2025-01-25T14:29:24.208Z [INFO] server-87f24fd1-dcea-5868-dbe7-1c7d10a8f524: aborting pipeline replication: peer="{Voter server-d6ffaeb2-f826-a3d6-e63e-e5b8b7427335 d6ffaeb2-f826-a3d6-e63e-e5b8b7427335}" +2025-01-25T14:40:07.019Z [INFO] server-cb3222e7-78d0-1f9b-1602-f18961051f6a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cb3222e7-78d0-1f9b-1602-f18961051f6a Address:cb3222e7-78d0-1f9b-1602-f18961051f6a} {Suffrage:Voter ID:server-72c73d59-9e9d-d35c-14cd-942137f5fb80 Address:72c73d59-9e9d-d35c-14cd-942137f5fb80} {Suffrage:Voter ID:server-832b4947-8b21-744a-ef0b-c58a1fd264d9 Address:832b4947-8b21-744a-ef0b-c58a1fd264d9}]" +2025-01-25T14:40:07.020Z [INFO] server-cb3222e7-78d0-1f9b-1602-f18961051f6a: entering follower state: follower="Node at cb3222e7-78d0-1f9b-1602-f18961051f6a [Follower]" leader-address= leader-id= +2025-01-25T14:40:07.021Z [INFO] server-72c73d59-9e9d-d35c-14cd-942137f5fb80: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cb3222e7-78d0-1f9b-1602-f18961051f6a Address:cb3222e7-78d0-1f9b-1602-f18961051f6a} {Suffrage:Voter ID:server-72c73d59-9e9d-d35c-14cd-942137f5fb80 Address:72c73d59-9e9d-d35c-14cd-942137f5fb80} {Suffrage:Voter ID:server-832b4947-8b21-744a-ef0b-c58a1fd264d9 Address:832b4947-8b21-744a-ef0b-c58a1fd264d9}]" +2025-01-25T14:40:07.021Z [INFO] server-72c73d59-9e9d-d35c-14cd-942137f5fb80: entering follower state: follower="Node at 72c73d59-9e9d-d35c-14cd-942137f5fb80 [Follower]" leader-address= leader-id= +2025-01-25T14:40:07.021Z [INFO] server-832b4947-8b21-744a-ef0b-c58a1fd264d9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cb3222e7-78d0-1f9b-1602-f18961051f6a Address:cb3222e7-78d0-1f9b-1602-f18961051f6a} {Suffrage:Voter ID:server-72c73d59-9e9d-d35c-14cd-942137f5fb80 Address:72c73d59-9e9d-d35c-14cd-942137f5fb80} {Suffrage:Voter ID:server-832b4947-8b21-744a-ef0b-c58a1fd264d9 Address:832b4947-8b21-744a-ef0b-c58a1fd264d9}]" +2025-01-25T14:40:07.022Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:07.022Z [INFO] server-832b4947-8b21-744a-ef0b-c58a1fd264d9: entering follower state: follower="Node at 832b4947-8b21-744a-ef0b-c58a1fd264d9 [Follower]" leader-address= leader-id= +2025-01-25T14:40:07.571Z [WARN] server-72c73d59-9e9d-d35c-14cd-942137f5fb80: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:07.572Z [INFO] server-72c73d59-9e9d-d35c-14cd-942137f5fb80: entering candidate state: node="Node at 72c73d59-9e9d-d35c-14cd-942137f5fb80 [Candidate]" term=2 +2025-01-25T14:40:07.572Z [INFO] server-72c73d59-9e9d-d35c-14cd-942137f5fb80: election won: term=2 tally=2 +2025-01-25T14:40:07.572Z [INFO] server-72c73d59-9e9d-d35c-14cd-942137f5fb80: entering leader state: leader="Node at 72c73d59-9e9d-d35c-14cd-942137f5fb80 [Leader]" +2025-01-25T14:40:07.572Z [INFO] server-72c73d59-9e9d-d35c-14cd-942137f5fb80: added peer, starting replication: peer=server-cb3222e7-78d0-1f9b-1602-f18961051f6a +2025-01-25T14:40:07.573Z [INFO] server-72c73d59-9e9d-d35c-14cd-942137f5fb80: added peer, starting replication: peer=server-832b4947-8b21-744a-ef0b-c58a1fd264d9 +2025-01-25T14:40:07.573Z [INFO] server-72c73d59-9e9d-d35c-14cd-942137f5fb80: pipelining replication: peer="{Voter server-cb3222e7-78d0-1f9b-1602-f18961051f6a cb3222e7-78d0-1f9b-1602-f18961051f6a}" +2025-01-25T14:40:07.576Z [INFO] server-72c73d59-9e9d-d35c-14cd-942137f5fb80: pipelining replication: peer="{Voter server-832b4947-8b21-744a-ef0b-c58a1fd264d9 832b4947-8b21-744a-ef0b-c58a1fd264d9}" +2025-01-25T14:40:08.578Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:07.573097881 +0000 UTC m=+76.618937198 (1 nodes), 550.888437ms from start of poll, 553.890862ms from cluster start. Timeout at 2025-01-25 14:40:08.578115672 +0000 UTC m=+77.623954990, 1.005017792s after stability +2025-01-25T14:40:08.581Z [INFO] server-72c73d59-9e9d-d35c-14cd-942137f5fb80: starting snapshot up to: index=12 +2025-01-25T14:40:08.581Z [INFO] creating new snapshot: path=/tmp/raft1714959880/snapshots/2-12-1737816008581.tmp +2025-01-25T14:40:08.584Z [INFO] server-72c73d59-9e9d-d35c-14cd-942137f5fb80: snapshot complete up to: index=12 +2025-01-25T14:40:08.589Z [INFO] creating new snapshot: path=/tmp/raft1714959880/snapshots/2-1013-1737816008589.tmp +2025-01-25T14:40:08.593Z [INFO] server-72c73d59-9e9d-d35c-14cd-942137f5fb80: copied to local snapshot: bytes=71 +2025-01-25T14:40:08.594Z [INFO] server-72c73d59-9e9d-d35c-14cd-942137f5fb80: snapshot restore progress: id=2-1013-1737816008589 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:08.595Z [INFO] server-72c73d59-9e9d-d35c-14cd-942137f5fb80: removing all old logs from log store +2025-01-25T14:40:08.595Z [INFO] server-72c73d59-9e9d-d35c-14cd-942137f5fb80: compacting logs: from=1 to=22 +2025-01-25T14:40:08.595Z [INFO] server-72c73d59-9e9d-d35c-14cd-942137f5fb80: restored user snapshot: index=1013 +2025-01-25T14:40:08.595Z [ERROR] server-72c73d59-9e9d-d35c-14cd-942137f5fb80: failed to get log: index=22 error="log not found" +2025-01-25T14:40:08.595Z [INFO] server-72c73d59-9e9d-d35c-14cd-942137f5fb80: aborting pipeline replication: peer="{Voter server-cb3222e7-78d0-1f9b-1602-f18961051f6a cb3222e7-78d0-1f9b-1602-f18961051f6a}" +2025-01-25T14:40:08.595Z [ERROR] server-72c73d59-9e9d-d35c-14cd-942137f5fb80: failed to get log: index=22 error="log not found" +2025-01-25T14:40:08.596Z [INFO] server-72c73d59-9e9d-d35c-14cd-942137f5fb80: aborting pipeline replication: peer="{Voter server-832b4947-8b21-744a-ef0b-c58a1fd264d9 832b4947-8b21-744a-ef0b-c58a1fd264d9}" +2025-01-25T14:40:08.603Z [ERROR] server-72c73d59-9e9d-d35c-14cd-942137f5fb80: failed to get log: index=22 error="log not found" +2025-01-25T14:40:08.603Z [ERROR] server-72c73d59-9e9d-d35c-14cd-942137f5fb80: failed to get log: index=22 error="log not found" +2025-01-25T14:40:08.605Z [INFO] creating new snapshot: path=/tmp/raft1684161558/snapshots/2-1013-1737816008605.tmp +2025-01-25T14:40:08.606Z [INFO] creating new snapshot: path=/tmp/raft955767798/snapshots/2-1013-1737816008606.tmp +2025-01-25T14:40:08.606Z [INFO] server-cb3222e7-78d0-1f9b-1602-f18961051f6a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:08.616Z [INFO] server-832b4947-8b21-744a-ef0b-c58a1fd264d9: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:08.618Z [INFO] server-cb3222e7-78d0-1f9b-1602-f18961051f6a: copied to local snapshot: bytes=71 +2025-01-25T14:40:08.621Z [INFO] server-832b4947-8b21-744a-ef0b-c58a1fd264d9: copied to local snapshot: bytes=71 +2025-01-25T14:40:08.622Z [INFO] server-cb3222e7-78d0-1f9b-1602-f18961051f6a: snapshot restore progress: id=2-1013-1737816008605 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:08.622Z [INFO] server-cb3222e7-78d0-1f9b-1602-f18961051f6a: removing all old logs from log store +2025-01-25T14:40:08.622Z [INFO] server-cb3222e7-78d0-1f9b-1602-f18961051f6a: compacting logs: from=1 to=22 +2025-01-25T14:40:08.622Z [INFO] server-cb3222e7-78d0-1f9b-1602-f18961051f6a: Installed remote snapshot +2025-01-25T14:40:08.622Z [INFO] server-832b4947-8b21-744a-ef0b-c58a1fd264d9: snapshot restore progress: id=2-1013-1737816008606 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:08.623Z [INFO] server-832b4947-8b21-744a-ef0b-c58a1fd264d9: removing all old logs from log store +2025-01-25T14:40:08.623Z [INFO] server-832b4947-8b21-744a-ef0b-c58a1fd264d9: compacting logs: from=1 to=22 +2025-01-25T14:40:08.623Z [INFO] server-832b4947-8b21-744a-ef0b-c58a1fd264d9: Installed remote snapshot +2025-01-25T14:40:08.623Z [WARN] server-cb3222e7-78d0-1f9b-1602-f18961051f6a: skipping application of old log: index=22 +2025-01-25T14:40:08.623Z [INFO] server-72c73d59-9e9d-d35c-14cd-942137f5fb80: pipelining replication: peer="{Voter server-cb3222e7-78d0-1f9b-1602-f18961051f6a cb3222e7-78d0-1f9b-1602-f18961051f6a}" +2025-01-25T14:40:08.623Z [WARN] server-832b4947-8b21-744a-ef0b-c58a1fd264d9: skipping application of old log: index=22 +2025-01-25T14:40:08.623Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:40:08.624Z [INFO] server-72c73d59-9e9d-d35c-14cd-942137f5fb80: pipelining replication: peer="{Voter server-832b4947-8b21-744a-ef0b-c58a1fd264d9 832b4947-8b21-744a-ef0b-c58a1fd264d9}" +2025-01-25T14:40:09.629Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:40:08.623879898 +0000 UTC m=+77.669719216 (2 nodes), 6.709µs from start of poll, 1.60467288s from cluster start. Timeout at 2025-01-25 14:40:09.629099524 +0000 UTC m=+78.674938841, 1.005219625s after stability +2025-01-25T14:40:09.644Z [INFO] server-72c73d59-9e9d-d35c-14cd-942137f5fb80: aborting pipeline replication: peer="{Voter server-832b4947-8b21-744a-ef0b-c58a1fd264d9 832b4947-8b21-744a-ef0b-c58a1fd264d9}" +2025-01-25T14:40:09.645Z [INFO] server-72c73d59-9e9d-d35c-14cd-942137f5fb80: aborting pipeline replication: peer="{Voter server-cb3222e7-78d0-1f9b-1602-f18961051f6a cb3222e7-78d0-1f9b-1602-f18961051f6a}" +2025-01-25T14:40:10.146Z [ERROR] server-72c73d59-9e9d-d35c-14cd-942137f5fb80: failed to heartbeat to: peer=832b4947-8b21-744a-ef0b-c58a1fd264d9 backoff time=10ms error="command timed out" === RUN TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_true -2025-01-25T14:29:24.212Z [INFO] server-c6b73303-998f-ce4b-f5dd-bffc533ff251: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c6b73303-998f-ce4b-f5dd-bffc533ff251 Address:c6b73303-998f-ce4b-f5dd-bffc533ff251} {Suffrage:Voter ID:server-d2cba660-3a81-b8dc-0633-a287f4965a9f Address:d2cba660-3a81-b8dc-0633-a287f4965a9f} {Suffrage:Voter ID:server-1ac1e9af-e75c-22d1-90a3-59a684cfc086 Address:1ac1e9af-e75c-22d1-90a3-59a684cfc086}]" -2025-01-25T14:29:24.212Z [INFO] server-c6b73303-998f-ce4b-f5dd-bffc533ff251: entering follower state: follower="Node at c6b73303-998f-ce4b-f5dd-bffc533ff251 [Follower]" leader-address= leader-id= -2025-01-25T14:29:24.212Z [INFO] server-d2cba660-3a81-b8dc-0633-a287f4965a9f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c6b73303-998f-ce4b-f5dd-bffc533ff251 Address:c6b73303-998f-ce4b-f5dd-bffc533ff251} {Suffrage:Voter ID:server-d2cba660-3a81-b8dc-0633-a287f4965a9f Address:d2cba660-3a81-b8dc-0633-a287f4965a9f} {Suffrage:Voter ID:server-1ac1e9af-e75c-22d1-90a3-59a684cfc086 Address:1ac1e9af-e75c-22d1-90a3-59a684cfc086}]" -2025-01-25T14:29:24.212Z [INFO] server-d2cba660-3a81-b8dc-0633-a287f4965a9f: entering follower state: follower="Node at d2cba660-3a81-b8dc-0633-a287f4965a9f [Follower]" leader-address= leader-id= -2025-01-25T14:29:24.212Z [INFO] server-1ac1e9af-e75c-22d1-90a3-59a684cfc086: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c6b73303-998f-ce4b-f5dd-bffc533ff251 Address:c6b73303-998f-ce4b-f5dd-bffc533ff251} {Suffrage:Voter ID:server-d2cba660-3a81-b8dc-0633-a287f4965a9f Address:d2cba660-3a81-b8dc-0633-a287f4965a9f} {Suffrage:Voter ID:server-1ac1e9af-e75c-22d1-90a3-59a684cfc086 Address:1ac1e9af-e75c-22d1-90a3-59a684cfc086}]" -2025-01-25T14:29:24.212Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:24.212Z [INFO] server-1ac1e9af-e75c-22d1-90a3-59a684cfc086: entering follower state: follower="Node at 1ac1e9af-e75c-22d1-90a3-59a684cfc086 [Follower]" leader-address= leader-id= -2025-01-25T14:29:24.780Z [WARN] server-c6b73303-998f-ce4b-f5dd-bffc533ff251: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:24.780Z [INFO] server-c6b73303-998f-ce4b-f5dd-bffc533ff251: entering candidate state: node="Node at c6b73303-998f-ce4b-f5dd-bffc533ff251 [Candidate]" term=2 -2025-01-25T14:29:24.780Z [INFO] server-c6b73303-998f-ce4b-f5dd-bffc533ff251: election won: term=2 tally=2 -2025-01-25T14:29:24.780Z [INFO] server-c6b73303-998f-ce4b-f5dd-bffc533ff251: entering leader state: leader="Node at c6b73303-998f-ce4b-f5dd-bffc533ff251 [Leader]" -2025-01-25T14:29:24.780Z [INFO] server-c6b73303-998f-ce4b-f5dd-bffc533ff251: added peer, starting replication: peer=server-d2cba660-3a81-b8dc-0633-a287f4965a9f -2025-01-25T14:29:24.780Z [INFO] server-c6b73303-998f-ce4b-f5dd-bffc533ff251: added peer, starting replication: peer=server-1ac1e9af-e75c-22d1-90a3-59a684cfc086 -2025-01-25T14:29:24.781Z [INFO] server-c6b73303-998f-ce4b-f5dd-bffc533ff251: pipelining replication: peer="{Voter server-d2cba660-3a81-b8dc-0633-a287f4965a9f d2cba660-3a81-b8dc-0633-a287f4965a9f}" -2025-01-25T14:29:24.781Z [INFO] server-c6b73303-998f-ce4b-f5dd-bffc533ff251: pipelining replication: peer="{Voter server-1ac1e9af-e75c-22d1-90a3-59a684cfc086 1ac1e9af-e75c-22d1-90a3-59a684cfc086}" -2025-01-25T14:29:25.785Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:24.780739347 +0000 UTC m=+74.635064329 (1 nodes), 567.756209ms from start of poll, 568.606418ms from cluster start. Timeout at 2025-01-25 14:29:25.785741507 +0000 UTC m=+75.640066449, 1.00500212s after stability -2025-01-25T14:29:25.786Z [INFO] server-c6b73303-998f-ce4b-f5dd-bffc533ff251: starting snapshot up to: index=12 -2025-01-25T14:29:25.786Z [INFO] creating new snapshot: path=/tmp/raft1575114220/snapshots/2-12-1737815365786.tmp -2025-01-25T14:29:25.788Z [INFO] server-c6b73303-998f-ce4b-f5dd-bffc533ff251: snapshot complete up to: index=12 -2025-01-25T14:29:25.790Z [INFO] server-ec2ddbed-adf2-a8fc-f94b-a00c4ae311f7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ec2ddbed-adf2-a8fc-f94b-a00c4ae311f7 Address:ec2ddbed-adf2-a8fc-f94b-a00c4ae311f7} {Suffrage:Voter ID:server-df5fff9a-770c-262a-937e-8fe7e82babf9 Address:df5fff9a-770c-262a-937e-8fe7e82babf9} {Suffrage:Voter ID:server-d7eceecb-fa8d-b19f-218a-ebb7a4dc4e9f Address:d7eceecb-fa8d-b19f-218a-ebb7a4dc4e9f}]" -2025-01-25T14:29:25.790Z [INFO] server-ec2ddbed-adf2-a8fc-f94b-a00c4ae311f7: entering follower state: follower="Node at ec2ddbed-adf2-a8fc-f94b-a00c4ae311f7 [Follower]" leader-address= leader-id= -2025-01-25T14:29:25.791Z [INFO] server-df5fff9a-770c-262a-937e-8fe7e82babf9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ec2ddbed-adf2-a8fc-f94b-a00c4ae311f7 Address:ec2ddbed-adf2-a8fc-f94b-a00c4ae311f7} {Suffrage:Voter ID:server-df5fff9a-770c-262a-937e-8fe7e82babf9 Address:df5fff9a-770c-262a-937e-8fe7e82babf9} {Suffrage:Voter ID:server-d7eceecb-fa8d-b19f-218a-ebb7a4dc4e9f Address:d7eceecb-fa8d-b19f-218a-ebb7a4dc4e9f}]" -2025-01-25T14:29:25.791Z [INFO] server-df5fff9a-770c-262a-937e-8fe7e82babf9: entering follower state: follower="Node at df5fff9a-770c-262a-937e-8fe7e82babf9 [Follower]" leader-address= leader-id= -2025-01-25T14:29:25.791Z [INFO] server-d7eceecb-fa8d-b19f-218a-ebb7a4dc4e9f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ec2ddbed-adf2-a8fc-f94b-a00c4ae311f7 Address:ec2ddbed-adf2-a8fc-f94b-a00c4ae311f7} {Suffrage:Voter ID:server-df5fff9a-770c-262a-937e-8fe7e82babf9 Address:df5fff9a-770c-262a-937e-8fe7e82babf9} {Suffrage:Voter ID:server-d7eceecb-fa8d-b19f-218a-ebb7a4dc4e9f Address:d7eceecb-fa8d-b19f-218a-ebb7a4dc4e9f}]" -2025-01-25T14:29:25.791Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:25.791Z [INFO] server-d7eceecb-fa8d-b19f-218a-ebb7a4dc4e9f: entering follower state: follower="Node at d7eceecb-fa8d-b19f-218a-ebb7a4dc4e9f [Follower]" leader-address= leader-id= -2025-01-25T14:29:26.301Z [WARN] server-df5fff9a-770c-262a-937e-8fe7e82babf9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:26.301Z [INFO] server-df5fff9a-770c-262a-937e-8fe7e82babf9: entering candidate state: node="Node at df5fff9a-770c-262a-937e-8fe7e82babf9 [Candidate]" term=2 -2025-01-25T14:29:26.307Z [INFO] server-df5fff9a-770c-262a-937e-8fe7e82babf9: election won: term=2 tally=2 -2025-01-25T14:29:26.308Z [INFO] server-df5fff9a-770c-262a-937e-8fe7e82babf9: entering leader state: leader="Node at df5fff9a-770c-262a-937e-8fe7e82babf9 [Leader]" -2025-01-25T14:29:26.308Z [INFO] server-df5fff9a-770c-262a-937e-8fe7e82babf9: added peer, starting replication: peer=server-ec2ddbed-adf2-a8fc-f94b-a00c4ae311f7 -2025-01-25T14:29:26.308Z [INFO] server-df5fff9a-770c-262a-937e-8fe7e82babf9: added peer, starting replication: peer=server-d7eceecb-fa8d-b19f-218a-ebb7a4dc4e9f -2025-01-25T14:29:26.308Z [INFO] server-df5fff9a-770c-262a-937e-8fe7e82babf9: pipelining replication: peer="{Voter server-d7eceecb-fa8d-b19f-218a-ebb7a4dc4e9f d7eceecb-fa8d-b19f-218a-ebb7a4dc4e9f}" -2025-01-25T14:29:26.308Z [WARN] server-ec2ddbed-adf2-a8fc-f94b-a00c4ae311f7: rejecting vote request since our last term is greater: candidate=df5fff9a-770c-262a-937e-8fe7e82babf9 last-term=2 last-candidate-term=1 -2025-01-25T14:29:26.309Z [INFO] server-df5fff9a-770c-262a-937e-8fe7e82babf9: pipelining replication: peer="{Voter server-ec2ddbed-adf2-a8fc-f94b-a00c4ae311f7 ec2ddbed-adf2-a8fc-f94b-a00c4ae311f7}" -2025-01-25T14:29:27.314Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:26.308962951 +0000 UTC m=+76.163287933 (1 nodes), 517.497697ms from start of poll, 518.427383ms from cluster start. Timeout at 2025-01-25 14:29:27.313964871 +0000 UTC m=+77.168289853, 1.00500192s after stability -2025-01-25T14:29:27.316Z [INFO] creating new snapshot: path=/tmp/raft3102055552/snapshots/2-1013-1737815367316.tmp -2025-01-25T14:29:27.317Z [INFO] server-df5fff9a-770c-262a-937e-8fe7e82babf9: copied to local snapshot: bytes=71 -2025-01-25T14:29:27.317Z [INFO] server-df5fff9a-770c-262a-937e-8fe7e82babf9: snapshot restore progress: id=2-1013-1737815367316 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:27.318Z [INFO] server-df5fff9a-770c-262a-937e-8fe7e82babf9: restored user snapshot: index=1013 -2025-01-25T14:29:27.318Z [ERROR] server-df5fff9a-770c-262a-937e-8fe7e82babf9: failed to get log: index=3 error="log not found" -2025-01-25T14:29:27.318Z [INFO] server-df5fff9a-770c-262a-937e-8fe7e82babf9: aborting pipeline replication: peer="{Voter server-d7eceecb-fa8d-b19f-218a-ebb7a4dc4e9f d7eceecb-fa8d-b19f-218a-ebb7a4dc4e9f}" -2025-01-25T14:29:27.318Z [ERROR] server-df5fff9a-770c-262a-937e-8fe7e82babf9: failed to get log: index=3 error="log not found" -2025-01-25T14:29:27.318Z [INFO] server-df5fff9a-770c-262a-937e-8fe7e82babf9: aborting pipeline replication: peer="{Voter server-ec2ddbed-adf2-a8fc-f94b-a00c4ae311f7 ec2ddbed-adf2-a8fc-f94b-a00c4ae311f7}" -2025-01-25T14:29:27.325Z [ERROR] server-df5fff9a-770c-262a-937e-8fe7e82babf9: failed to get log: index=3 error="log not found" -2025-01-25T14:29:27.326Z [INFO] creating new snapshot: path=/tmp/raft563240831/snapshots/2-1013-1737815367326.tmp -2025-01-25T14:29:27.326Z [INFO] server-d7eceecb-fa8d-b19f-218a-ebb7a4dc4e9f: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:27.326Z [ERROR] server-df5fff9a-770c-262a-937e-8fe7e82babf9: failed to get log: index=3 error="log not found" -2025-01-25T14:29:27.327Z [INFO] server-d7eceecb-fa8d-b19f-218a-ebb7a4dc4e9f: copied to local snapshot: bytes=71 -2025-01-25T14:29:27.327Z [INFO] creating new snapshot: path=/tmp/raft819682955/snapshots/2-1013-1737815367327.tmp -2025-01-25T14:29:27.327Z [INFO] server-d7eceecb-fa8d-b19f-218a-ebb7a4dc4e9f: snapshot restore progress: id=2-1013-1737815367326 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:27.328Z [INFO] server-d7eceecb-fa8d-b19f-218a-ebb7a4dc4e9f: Installed remote snapshot -2025-01-25T14:29:27.328Z [INFO] server-df5fff9a-770c-262a-937e-8fe7e82babf9: pipelining replication: peer="{Voter server-d7eceecb-fa8d-b19f-218a-ebb7a4dc4e9f d7eceecb-fa8d-b19f-218a-ebb7a4dc4e9f}" -2025-01-25T14:29:27.328Z [INFO] server-ec2ddbed-adf2-a8fc-f94b-a00c4ae311f7: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:27.328Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:29:27.328Z [INFO] server-ec2ddbed-adf2-a8fc-f94b-a00c4ae311f7: copied to local snapshot: bytes=71 -2025-01-25T14:29:27.329Z [INFO] server-ec2ddbed-adf2-a8fc-f94b-a00c4ae311f7: snapshot restore progress: id=2-1013-1737815367327 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:27.329Z [INFO] server-ec2ddbed-adf2-a8fc-f94b-a00c4ae311f7: Installed remote snapshot -2025-01-25T14:29:27.329Z [INFO] server-df5fff9a-770c-262a-937e-8fe7e82babf9: pipelining replication: peer="{Voter server-ec2ddbed-adf2-a8fc-f94b-a00c4ae311f7 ec2ddbed-adf2-a8fc-f94b-a00c4ae311f7}" -2025-01-25T14:29:28.333Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:29:27.328450463 +0000 UTC m=+77.182775445 (2 nodes), 1.72µs from start of poll, 1.537914895s from cluster start. Timeout at 2025-01-25 14:29:28.333590658 +0000 UTC m=+78.187915680, 1.005140235s after stability -2025-01-25T14:29:28.346Z [INFO] server-c6b73303-998f-ce4b-f5dd-bffc533ff251: aborting pipeline replication: peer="{Voter server-d2cba660-3a81-b8dc-0633-a287f4965a9f d2cba660-3a81-b8dc-0633-a287f4965a9f}" -2025-01-25T14:29:28.346Z [INFO] server-c6b73303-998f-ce4b-f5dd-bffc533ff251: aborting pipeline replication: peer="{Voter server-1ac1e9af-e75c-22d1-90a3-59a684cfc086 1ac1e9af-e75c-22d1-90a3-59a684cfc086}" +2025-01-25T14:40:10.153Z [INFO] server-a2d31918-567d-87a0-4260-efc4c5820704: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a2d31918-567d-87a0-4260-efc4c5820704 Address:a2d31918-567d-87a0-4260-efc4c5820704} {Suffrage:Voter ID:server-54472967-957d-1e57-ee00-da1226b33e62 Address:54472967-957d-1e57-ee00-da1226b33e62} {Suffrage:Voter ID:server-552d6c9c-beef-3df3-5527-d8a44a0bbf8d Address:552d6c9c-beef-3df3-5527-d8a44a0bbf8d}]" +2025-01-25T14:40:10.154Z [INFO] server-a2d31918-567d-87a0-4260-efc4c5820704: entering follower state: follower="Node at a2d31918-567d-87a0-4260-efc4c5820704 [Follower]" leader-address= leader-id= +2025-01-25T14:40:10.154Z [INFO] server-54472967-957d-1e57-ee00-da1226b33e62: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a2d31918-567d-87a0-4260-efc4c5820704 Address:a2d31918-567d-87a0-4260-efc4c5820704} {Suffrage:Voter ID:server-54472967-957d-1e57-ee00-da1226b33e62 Address:54472967-957d-1e57-ee00-da1226b33e62} {Suffrage:Voter ID:server-552d6c9c-beef-3df3-5527-d8a44a0bbf8d Address:552d6c9c-beef-3df3-5527-d8a44a0bbf8d}]" +2025-01-25T14:40:10.154Z [INFO] server-54472967-957d-1e57-ee00-da1226b33e62: entering follower state: follower="Node at 54472967-957d-1e57-ee00-da1226b33e62 [Follower]" leader-address= leader-id= +2025-01-25T14:40:10.154Z [INFO] server-552d6c9c-beef-3df3-5527-d8a44a0bbf8d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a2d31918-567d-87a0-4260-efc4c5820704 Address:a2d31918-567d-87a0-4260-efc4c5820704} {Suffrage:Voter ID:server-54472967-957d-1e57-ee00-da1226b33e62 Address:54472967-957d-1e57-ee00-da1226b33e62} {Suffrage:Voter ID:server-552d6c9c-beef-3df3-5527-d8a44a0bbf8d Address:552d6c9c-beef-3df3-5527-d8a44a0bbf8d}]" +2025-01-25T14:40:10.155Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:10.155Z [INFO] server-552d6c9c-beef-3df3-5527-d8a44a0bbf8d: entering follower state: follower="Node at 552d6c9c-beef-3df3-5527-d8a44a0bbf8d [Follower]" leader-address= leader-id= +2025-01-25T14:40:10.674Z [WARN] server-a2d31918-567d-87a0-4260-efc4c5820704: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:10.674Z [INFO] server-a2d31918-567d-87a0-4260-efc4c5820704: entering candidate state: node="Node at a2d31918-567d-87a0-4260-efc4c5820704 [Candidate]" term=2 +2025-01-25T14:40:10.675Z [INFO] server-a2d31918-567d-87a0-4260-efc4c5820704: election won: term=2 tally=2 +2025-01-25T14:40:10.675Z [INFO] server-a2d31918-567d-87a0-4260-efc4c5820704: entering leader state: leader="Node at a2d31918-567d-87a0-4260-efc4c5820704 [Leader]" +2025-01-25T14:40:10.675Z [INFO] server-a2d31918-567d-87a0-4260-efc4c5820704: added peer, starting replication: peer=server-54472967-957d-1e57-ee00-da1226b33e62 +2025-01-25T14:40:10.675Z [INFO] server-a2d31918-567d-87a0-4260-efc4c5820704: added peer, starting replication: peer=server-552d6c9c-beef-3df3-5527-d8a44a0bbf8d +2025-01-25T14:40:10.676Z [INFO] server-a2d31918-567d-87a0-4260-efc4c5820704: pipelining replication: peer="{Voter server-54472967-957d-1e57-ee00-da1226b33e62 54472967-957d-1e57-ee00-da1226b33e62}" +2025-01-25T14:40:10.676Z [INFO] server-a2d31918-567d-87a0-4260-efc4c5820704: pipelining replication: peer="{Voter server-552d6c9c-beef-3df3-5527-d8a44a0bbf8d 552d6c9c-beef-3df3-5527-d8a44a0bbf8d}" +2025-01-25T14:40:11.681Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:10.675653538 +0000 UTC m=+79.721492856 (1 nodes), 520.562474ms from start of poll, 522.320937ms from cluster start. Timeout at 2025-01-25 14:40:11.680699913 +0000 UTC m=+80.726539231, 1.005046375s after stability +2025-01-25T14:40:11.684Z [INFO] server-a2d31918-567d-87a0-4260-efc4c5820704: starting snapshot up to: index=12 +2025-01-25T14:40:11.684Z [INFO] creating new snapshot: path=/tmp/raft2746938458/snapshots/2-12-1737816011684.tmp +2025-01-25T14:40:11.686Z [INFO] server-a2d31918-567d-87a0-4260-efc4c5820704: snapshot complete up to: index=12 +2025-01-25T14:40:11.694Z [INFO] server-392f6568-deb4-5e14-75d7-d58f7eea3f73: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-392f6568-deb4-5e14-75d7-d58f7eea3f73 Address:392f6568-deb4-5e14-75d7-d58f7eea3f73} {Suffrage:Voter ID:server-f0ad42c8-ada3-a5a8-6b77-425324987bfb Address:f0ad42c8-ada3-a5a8-6b77-425324987bfb} {Suffrage:Voter ID:server-37eb2160-cbaf-bdf5-a182-4da4930bce9e Address:37eb2160-cbaf-bdf5-a182-4da4930bce9e}]" +2025-01-25T14:40:11.695Z [INFO] server-392f6568-deb4-5e14-75d7-d58f7eea3f73: entering follower state: follower="Node at 392f6568-deb4-5e14-75d7-d58f7eea3f73 [Follower]" leader-address= leader-id= +2025-01-25T14:40:11.695Z [INFO] server-f0ad42c8-ada3-a5a8-6b77-425324987bfb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-392f6568-deb4-5e14-75d7-d58f7eea3f73 Address:392f6568-deb4-5e14-75d7-d58f7eea3f73} {Suffrage:Voter ID:server-f0ad42c8-ada3-a5a8-6b77-425324987bfb Address:f0ad42c8-ada3-a5a8-6b77-425324987bfb} {Suffrage:Voter ID:server-37eb2160-cbaf-bdf5-a182-4da4930bce9e Address:37eb2160-cbaf-bdf5-a182-4da4930bce9e}]" +2025-01-25T14:40:11.696Z [INFO] server-f0ad42c8-ada3-a5a8-6b77-425324987bfb: entering follower state: follower="Node at f0ad42c8-ada3-a5a8-6b77-425324987bfb [Follower]" leader-address= leader-id= +2025-01-25T14:40:11.696Z [INFO] server-37eb2160-cbaf-bdf5-a182-4da4930bce9e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-392f6568-deb4-5e14-75d7-d58f7eea3f73 Address:392f6568-deb4-5e14-75d7-d58f7eea3f73} {Suffrage:Voter ID:server-f0ad42c8-ada3-a5a8-6b77-425324987bfb Address:f0ad42c8-ada3-a5a8-6b77-425324987bfb} {Suffrage:Voter ID:server-37eb2160-cbaf-bdf5-a182-4da4930bce9e Address:37eb2160-cbaf-bdf5-a182-4da4930bce9e}]" +2025-01-25T14:40:11.697Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:11.697Z [INFO] server-37eb2160-cbaf-bdf5-a182-4da4930bce9e: entering follower state: follower="Node at 37eb2160-cbaf-bdf5-a182-4da4930bce9e [Follower]" leader-address= leader-id= +2025-01-25T14:40:12.354Z [WARN] server-392f6568-deb4-5e14-75d7-d58f7eea3f73: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:12.354Z [INFO] server-392f6568-deb4-5e14-75d7-d58f7eea3f73: entering candidate state: node="Node at 392f6568-deb4-5e14-75d7-d58f7eea3f73 [Candidate]" term=2 +2025-01-25T14:40:12.355Z [INFO] server-392f6568-deb4-5e14-75d7-d58f7eea3f73: election won: term=2 tally=2 +2025-01-25T14:40:12.355Z [INFO] server-392f6568-deb4-5e14-75d7-d58f7eea3f73: entering leader state: leader="Node at 392f6568-deb4-5e14-75d7-d58f7eea3f73 [Leader]" +2025-01-25T14:40:12.355Z [INFO] server-392f6568-deb4-5e14-75d7-d58f7eea3f73: added peer, starting replication: peer=server-f0ad42c8-ada3-a5a8-6b77-425324987bfb +2025-01-25T14:40:12.355Z [INFO] server-392f6568-deb4-5e14-75d7-d58f7eea3f73: added peer, starting replication: peer=server-37eb2160-cbaf-bdf5-a182-4da4930bce9e +2025-01-25T14:40:12.359Z [INFO] server-392f6568-deb4-5e14-75d7-d58f7eea3f73: pipelining replication: peer="{Voter server-f0ad42c8-ada3-a5a8-6b77-425324987bfb f0ad42c8-ada3-a5a8-6b77-425324987bfb}" +2025-01-25T14:40:12.371Z [INFO] server-392f6568-deb4-5e14-75d7-d58f7eea3f73: pipelining replication: peer="{Voter server-37eb2160-cbaf-bdf5-a182-4da4930bce9e 37eb2160-cbaf-bdf5-a182-4da4930bce9e}" +2025-01-25T14:40:13.360Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:12.355456595 +0000 UTC m=+81.401295913 (1 nodes), 658.241375ms from start of poll, 661.303592ms from cluster start. Timeout at 2025-01-25 14:40:13.360461262 +0000 UTC m=+82.406300580, 1.005004667s after stability +2025-01-25T14:40:13.361Z [INFO] creating new snapshot: path=/tmp/raft1745915871/snapshots/2-1013-1737816013361.tmp +2025-01-25T14:40:13.363Z [INFO] server-392f6568-deb4-5e14-75d7-d58f7eea3f73: copied to local snapshot: bytes=71 +2025-01-25T14:40:13.365Z [INFO] server-392f6568-deb4-5e14-75d7-d58f7eea3f73: snapshot restore progress: id=2-1013-1737816013361 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:13.365Z [INFO] server-392f6568-deb4-5e14-75d7-d58f7eea3f73: restored user snapshot: index=1013 +2025-01-25T14:40:13.365Z [ERROR] server-392f6568-deb4-5e14-75d7-d58f7eea3f73: failed to get log: index=3 error="log not found" +2025-01-25T14:40:13.365Z [INFO] server-392f6568-deb4-5e14-75d7-d58f7eea3f73: aborting pipeline replication: peer="{Voter server-37eb2160-cbaf-bdf5-a182-4da4930bce9e 37eb2160-cbaf-bdf5-a182-4da4930bce9e}" +2025-01-25T14:40:13.365Z [ERROR] server-392f6568-deb4-5e14-75d7-d58f7eea3f73: failed to get log: index=3 error="log not found" +2025-01-25T14:40:13.365Z [INFO] server-392f6568-deb4-5e14-75d7-d58f7eea3f73: aborting pipeline replication: peer="{Voter server-f0ad42c8-ada3-a5a8-6b77-425324987bfb f0ad42c8-ada3-a5a8-6b77-425324987bfb}" +2025-01-25T14:40:13.371Z [ERROR] server-392f6568-deb4-5e14-75d7-d58f7eea3f73: failed to get log: index=3 error="log not found" +2025-01-25T14:40:13.372Z [ERROR] server-392f6568-deb4-5e14-75d7-d58f7eea3f73: failed to get log: index=3 error="log not found" +2025-01-25T14:40:13.374Z [INFO] creating new snapshot: path=/tmp/raft407616547/snapshots/2-1013-1737816013374.tmp +2025-01-25T14:40:13.374Z [INFO] creating new snapshot: path=/tmp/raft3704104438/snapshots/2-1013-1737816013374.tmp +2025-01-25T14:40:13.375Z [INFO] server-f0ad42c8-ada3-a5a8-6b77-425324987bfb: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:13.375Z [INFO] server-37eb2160-cbaf-bdf5-a182-4da4930bce9e: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:13.378Z [INFO] server-37eb2160-cbaf-bdf5-a182-4da4930bce9e: copied to local snapshot: bytes=71 +2025-01-25T14:40:13.378Z [INFO] server-f0ad42c8-ada3-a5a8-6b77-425324987bfb: copied to local snapshot: bytes=71 +2025-01-25T14:40:13.379Z [INFO] server-37eb2160-cbaf-bdf5-a182-4da4930bce9e: snapshot restore progress: id=2-1013-1737816013374 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:13.379Z [INFO] server-37eb2160-cbaf-bdf5-a182-4da4930bce9e: Installed remote snapshot +2025-01-25T14:40:13.380Z [INFO] server-392f6568-deb4-5e14-75d7-d58f7eea3f73: pipelining replication: peer="{Voter server-37eb2160-cbaf-bdf5-a182-4da4930bce9e 37eb2160-cbaf-bdf5-a182-4da4930bce9e}" +2025-01-25T14:40:13.380Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:40:13.380Z [INFO] server-f0ad42c8-ada3-a5a8-6b77-425324987bfb: snapshot restore progress: id=2-1013-1737816013374 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:13.381Z [INFO] server-f0ad42c8-ada3-a5a8-6b77-425324987bfb: Installed remote snapshot +2025-01-25T14:40:13.382Z [INFO] server-392f6568-deb4-5e14-75d7-d58f7eea3f73: pipelining replication: peer="{Voter server-f0ad42c8-ada3-a5a8-6b77-425324987bfb f0ad42c8-ada3-a5a8-6b77-425324987bfb}" +2025-01-25T14:40:14.386Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:40:13.380853247 +0000 UTC m=+82.426692565 (2 nodes), 4.667µs from start of poll, 1.686700244s from cluster start. Timeout at 2025-01-25 14:40:14.386019789 +0000 UTC m=+83.431859107, 1.005166542s after stability +2025-01-25T14:40:14.411Z [INFO] server-a2d31918-567d-87a0-4260-efc4c5820704: aborting pipeline replication: peer="{Voter server-552d6c9c-beef-3df3-5527-d8a44a0bbf8d 552d6c9c-beef-3df3-5527-d8a44a0bbf8d}" +2025-01-25T14:40:14.411Z [INFO] server-a2d31918-567d-87a0-4260-efc4c5820704: aborting pipeline replication: peer="{Voter server-54472967-957d-1e57-ee00-da1226b33e62 54472967-957d-1e57-ee00-da1226b33e62}" === RUN TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_true -2025-01-25T14:29:28.348Z [INFO] server-deac9130-839c-f54e-185d-246a460b2f82: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-deac9130-839c-f54e-185d-246a460b2f82 Address:deac9130-839c-f54e-185d-246a460b2f82} {Suffrage:Voter ID:server-edd04b33-d20b-5083-c904-d62c59699a66 Address:edd04b33-d20b-5083-c904-d62c59699a66} {Suffrage:Voter ID:server-27d2c5f2-30d3-851d-cb8e-504c5b5f4446 Address:27d2c5f2-30d3-851d-cb8e-504c5b5f4446}]" -2025-01-25T14:29:28.349Z [INFO] server-edd04b33-d20b-5083-c904-d62c59699a66: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-deac9130-839c-f54e-185d-246a460b2f82 Address:deac9130-839c-f54e-185d-246a460b2f82} {Suffrage:Voter ID:server-edd04b33-d20b-5083-c904-d62c59699a66 Address:edd04b33-d20b-5083-c904-d62c59699a66} {Suffrage:Voter ID:server-27d2c5f2-30d3-851d-cb8e-504c5b5f4446 Address:27d2c5f2-30d3-851d-cb8e-504c5b5f4446}]" -2025-01-25T14:29:28.349Z [INFO] server-deac9130-839c-f54e-185d-246a460b2f82: entering follower state: follower="Node at deac9130-839c-f54e-185d-246a460b2f82 [Follower]" leader-address= leader-id= -2025-01-25T14:29:28.349Z [INFO] server-edd04b33-d20b-5083-c904-d62c59699a66: entering follower state: follower="Node at edd04b33-d20b-5083-c904-d62c59699a66 [Follower]" leader-address= leader-id= -2025-01-25T14:29:28.349Z [INFO] server-27d2c5f2-30d3-851d-cb8e-504c5b5f4446: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-deac9130-839c-f54e-185d-246a460b2f82 Address:deac9130-839c-f54e-185d-246a460b2f82} {Suffrage:Voter ID:server-edd04b33-d20b-5083-c904-d62c59699a66 Address:edd04b33-d20b-5083-c904-d62c59699a66} {Suffrage:Voter ID:server-27d2c5f2-30d3-851d-cb8e-504c5b5f4446 Address:27d2c5f2-30d3-851d-cb8e-504c5b5f4446}]" -2025-01-25T14:29:28.349Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:28.349Z [INFO] server-27d2c5f2-30d3-851d-cb8e-504c5b5f4446: entering follower state: follower="Node at 27d2c5f2-30d3-851d-cb8e-504c5b5f4446 [Follower]" leader-address= leader-id= -2025-01-25T14:29:28.860Z [WARN] server-27d2c5f2-30d3-851d-cb8e-504c5b5f4446: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:28.860Z [INFO] server-27d2c5f2-30d3-851d-cb8e-504c5b5f4446: entering candidate state: node="Node at 27d2c5f2-30d3-851d-cb8e-504c5b5f4446 [Candidate]" term=2 -2025-01-25T14:29:28.860Z [INFO] server-27d2c5f2-30d3-851d-cb8e-504c5b5f4446: election won: term=2 tally=2 -2025-01-25T14:29:28.860Z [INFO] server-27d2c5f2-30d3-851d-cb8e-504c5b5f4446: entering leader state: leader="Node at 27d2c5f2-30d3-851d-cb8e-504c5b5f4446 [Leader]" -2025-01-25T14:29:28.860Z [INFO] server-27d2c5f2-30d3-851d-cb8e-504c5b5f4446: added peer, starting replication: peer=server-deac9130-839c-f54e-185d-246a460b2f82 -2025-01-25T14:29:28.861Z [INFO] server-27d2c5f2-30d3-851d-cb8e-504c5b5f4446: added peer, starting replication: peer=server-edd04b33-d20b-5083-c904-d62c59699a66 -2025-01-25T14:29:28.861Z [INFO] server-27d2c5f2-30d3-851d-cb8e-504c5b5f4446: pipelining replication: peer="{Voter server-deac9130-839c-f54e-185d-246a460b2f82 deac9130-839c-f54e-185d-246a460b2f82}" -2025-01-25T14:29:28.861Z [INFO] server-27d2c5f2-30d3-851d-cb8e-504c5b5f4446: pipelining replication: peer="{Voter server-edd04b33-d20b-5083-c904-d62c59699a66 edd04b33-d20b-5083-c904-d62c59699a66}" -2025-01-25T14:29:29.866Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:28.861012632 +0000 UTC m=+78.715337614 (1 nodes), 511.563271ms from start of poll, 512.236327ms from cluster start. Timeout at 2025-01-25 14:29:29.866015672 +0000 UTC m=+79.720340614, 1.005003s after stability -2025-01-25T14:29:29.872Z [INFO] server-27d2c5f2-30d3-851d-cb8e-504c5b5f4446: starting snapshot up to: index=12 -2025-01-25T14:29:29.872Z [INFO] creating new snapshot: path=/tmp/raft2053591696/snapshots/2-12-1737815369872.tmp -2025-01-25T14:29:29.874Z [INFO] server-27d2c5f2-30d3-851d-cb8e-504c5b5f4446: snapshot complete up to: index=12 -2025-01-25T14:29:29.876Z [INFO] server-75afaf31-1898-20fb-9d23-a94b8b344e23: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-75afaf31-1898-20fb-9d23-a94b8b344e23 Address:75afaf31-1898-20fb-9d23-a94b8b344e23} {Suffrage:Voter ID:server-a627ff05-dcb8-1937-5bd4-fdc8d3c04893 Address:a627ff05-dcb8-1937-5bd4-fdc8d3c04893} {Suffrage:Voter ID:server-16902e3d-3e90-0290-977b-4c48d469fc00 Address:16902e3d-3e90-0290-977b-4c48d469fc00}]" -2025-01-25T14:29:29.877Z [INFO] server-75afaf31-1898-20fb-9d23-a94b8b344e23: entering follower state: follower="Node at 75afaf31-1898-20fb-9d23-a94b8b344e23 [Follower]" leader-address= leader-id= -2025-01-25T14:29:29.877Z [INFO] server-a627ff05-dcb8-1937-5bd4-fdc8d3c04893: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-75afaf31-1898-20fb-9d23-a94b8b344e23 Address:75afaf31-1898-20fb-9d23-a94b8b344e23} {Suffrage:Voter ID:server-a627ff05-dcb8-1937-5bd4-fdc8d3c04893 Address:a627ff05-dcb8-1937-5bd4-fdc8d3c04893} {Suffrage:Voter ID:server-16902e3d-3e90-0290-977b-4c48d469fc00 Address:16902e3d-3e90-0290-977b-4c48d469fc00}]" -2025-01-25T14:29:29.877Z [INFO] server-16902e3d-3e90-0290-977b-4c48d469fc00: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-75afaf31-1898-20fb-9d23-a94b8b344e23 Address:75afaf31-1898-20fb-9d23-a94b8b344e23} {Suffrage:Voter ID:server-a627ff05-dcb8-1937-5bd4-fdc8d3c04893 Address:a627ff05-dcb8-1937-5bd4-fdc8d3c04893} {Suffrage:Voter ID:server-16902e3d-3e90-0290-977b-4c48d469fc00 Address:16902e3d-3e90-0290-977b-4c48d469fc00}]" -2025-01-25T14:29:29.877Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:29.877Z [INFO] server-a627ff05-dcb8-1937-5bd4-fdc8d3c04893: entering follower state: follower="Node at a627ff05-dcb8-1937-5bd4-fdc8d3c04893 [Follower]" leader-address= leader-id= -2025-01-25T14:29:29.877Z [INFO] server-16902e3d-3e90-0290-977b-4c48d469fc00: entering follower state: follower="Node at 16902e3d-3e90-0290-977b-4c48d469fc00 [Follower]" leader-address= leader-id= -2025-01-25T14:29:30.433Z [WARN] server-16902e3d-3e90-0290-977b-4c48d469fc00: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:30.433Z [INFO] server-16902e3d-3e90-0290-977b-4c48d469fc00: entering candidate state: node="Node at 16902e3d-3e90-0290-977b-4c48d469fc00 [Candidate]" term=2 -2025-01-25T14:29:30.434Z [INFO] server-16902e3d-3e90-0290-977b-4c48d469fc00: election won: term=2 tally=2 -2025-01-25T14:29:30.434Z [INFO] server-16902e3d-3e90-0290-977b-4c48d469fc00: entering leader state: leader="Node at 16902e3d-3e90-0290-977b-4c48d469fc00 [Leader]" -2025-01-25T14:29:30.434Z [INFO] server-16902e3d-3e90-0290-977b-4c48d469fc00: added peer, starting replication: peer=server-75afaf31-1898-20fb-9d23-a94b8b344e23 -2025-01-25T14:29:30.434Z [INFO] server-16902e3d-3e90-0290-977b-4c48d469fc00: added peer, starting replication: peer=server-a627ff05-dcb8-1937-5bd4-fdc8d3c04893 -2025-01-25T14:29:30.434Z [INFO] server-16902e3d-3e90-0290-977b-4c48d469fc00: pipelining replication: peer="{Voter server-a627ff05-dcb8-1937-5bd4-fdc8d3c04893 a627ff05-dcb8-1937-5bd4-fdc8d3c04893}" -2025-01-25T14:29:30.434Z [INFO] server-16902e3d-3e90-0290-977b-4c48d469fc00: pipelining replication: peer="{Voter server-75afaf31-1898-20fb-9d23-a94b8b344e23 75afaf31-1898-20fb-9d23-a94b8b344e23}" -2025-01-25T14:29:31.439Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:30.434184579 +0000 UTC m=+80.288509561 (1 nodes), 556.775244ms from start of poll, 557.458579ms from cluster start. Timeout at 2025-01-25 14:29:31.439187539 +0000 UTC m=+81.293512521, 1.00500296s after stability -2025-01-25T14:29:31.440Z [INFO] creating new snapshot: path=/tmp/raft3524174746/snapshots/2-1013-1737815371440.tmp -2025-01-25T14:29:31.441Z [INFO] server-16902e3d-3e90-0290-977b-4c48d469fc00: copied to local snapshot: bytes=71 -2025-01-25T14:29:31.441Z [INFO] server-16902e3d-3e90-0290-977b-4c48d469fc00: snapshot restore progress: id=2-1013-1737815371440 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:31.441Z [INFO] server-16902e3d-3e90-0290-977b-4c48d469fc00: removing all old logs from log store -2025-01-25T14:29:31.441Z [INFO] server-16902e3d-3e90-0290-977b-4c48d469fc00: compacting logs: from=1 to=2 -2025-01-25T14:29:31.441Z [INFO] server-16902e3d-3e90-0290-977b-4c48d469fc00: restored user snapshot: index=1013 -2025-01-25T14:29:31.441Z [ERROR] server-16902e3d-3e90-0290-977b-4c48d469fc00: failed to get log: index=2 error="log not found" -2025-01-25T14:29:31.441Z [INFO] server-16902e3d-3e90-0290-977b-4c48d469fc00: aborting pipeline replication: peer="{Voter server-a627ff05-dcb8-1937-5bd4-fdc8d3c04893 a627ff05-dcb8-1937-5bd4-fdc8d3c04893}" -2025-01-25T14:29:31.441Z [ERROR] server-16902e3d-3e90-0290-977b-4c48d469fc00: failed to get log: index=2 error="log not found" -2025-01-25T14:29:31.441Z [ERROR] server-16902e3d-3e90-0290-977b-4c48d469fc00: failed to get log: index=2 error="log not found" -2025-01-25T14:29:31.441Z [INFO] server-16902e3d-3e90-0290-977b-4c48d469fc00: aborting pipeline replication: peer="{Voter server-75afaf31-1898-20fb-9d23-a94b8b344e23 75afaf31-1898-20fb-9d23-a94b8b344e23}" -2025-01-25T14:29:31.442Z [INFO] creating new snapshot: path=/tmp/raft1353825785/snapshots/2-1013-1737815371442.tmp -2025-01-25T14:29:31.443Z [INFO] server-a627ff05-dcb8-1937-5bd4-fdc8d3c04893: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:31.443Z [INFO] server-a627ff05-dcb8-1937-5bd4-fdc8d3c04893: copied to local snapshot: bytes=71 -2025-01-25T14:29:31.444Z [INFO] server-a627ff05-dcb8-1937-5bd4-fdc8d3c04893: snapshot restore progress: id=2-1013-1737815371442 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:31.444Z [INFO] server-a627ff05-dcb8-1937-5bd4-fdc8d3c04893: removing all old logs from log store -2025-01-25T14:29:31.444Z [INFO] server-a627ff05-dcb8-1937-5bd4-fdc8d3c04893: compacting logs: from=1 to=2 -2025-01-25T14:29:31.444Z [INFO] server-a627ff05-dcb8-1937-5bd4-fdc8d3c04893: Installed remote snapshot -2025-01-25T14:29:31.444Z [INFO] server-16902e3d-3e90-0290-977b-4c48d469fc00: pipelining replication: peer="{Voter server-a627ff05-dcb8-1937-5bd4-fdc8d3c04893 a627ff05-dcb8-1937-5bd4-fdc8d3c04893}" -2025-01-25T14:29:31.444Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:29:31.452Z [ERROR] server-16902e3d-3e90-0290-977b-4c48d469fc00: failed to get log: index=2 error="log not found" -2025-01-25T14:29:31.455Z [INFO] creating new snapshot: path=/tmp/raft3794580954/snapshots/2-1013-1737815371455.tmp -2025-01-25T14:29:31.456Z [INFO] server-75afaf31-1898-20fb-9d23-a94b8b344e23: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:31.456Z [INFO] server-75afaf31-1898-20fb-9d23-a94b8b344e23: copied to local snapshot: bytes=71 -2025-01-25T14:29:31.457Z [INFO] server-75afaf31-1898-20fb-9d23-a94b8b344e23: snapshot restore progress: id=2-1013-1737815371455 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:31.457Z [INFO] server-75afaf31-1898-20fb-9d23-a94b8b344e23: removing all old logs from log store -2025-01-25T14:29:31.457Z [INFO] server-75afaf31-1898-20fb-9d23-a94b8b344e23: compacting logs: from=1 to=2 -2025-01-25T14:29:31.457Z [INFO] server-75afaf31-1898-20fb-9d23-a94b8b344e23: Installed remote snapshot -2025-01-25T14:29:31.457Z [INFO] server-16902e3d-3e90-0290-977b-4c48d469fc00: pipelining replication: peer="{Voter server-75afaf31-1898-20fb-9d23-a94b8b344e23 75afaf31-1898-20fb-9d23-a94b8b344e23}" -2025-01-25T14:29:32.451Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:29:31.444969204 +0000 UTC m=+81.299294186 (2 nodes), 1.48µs from start of poll, 1.568243204s from cluster start. Timeout at 2025-01-25 14:29:32.450147718 +0000 UTC m=+82.304472660, 1.005178474s after stability -2025-01-25T14:29:32.466Z [INFO] server-27d2c5f2-30d3-851d-cb8e-504c5b5f4446: aborting pipeline replication: peer="{Voter server-edd04b33-d20b-5083-c904-d62c59699a66 edd04b33-d20b-5083-c904-d62c59699a66}" -2025-01-25T14:29:32.466Z [INFO] server-27d2c5f2-30d3-851d-cb8e-504c5b5f4446: aborting pipeline replication: peer="{Voter server-deac9130-839c-f54e-185d-246a460b2f82 deac9130-839c-f54e-185d-246a460b2f82}" +2025-01-25T14:40:14.421Z [INFO] server-7e338f17-662e-9db1-4270-b1a2d9785db0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7e338f17-662e-9db1-4270-b1a2d9785db0 Address:7e338f17-662e-9db1-4270-b1a2d9785db0} {Suffrage:Voter ID:server-ae9ac15f-52e9-7be4-1770-b7443ee67c3b Address:ae9ac15f-52e9-7be4-1770-b7443ee67c3b} {Suffrage:Voter ID:server-9bb82b36-08ef-5ca8-7bd6-74dc6c6d8f25 Address:9bb82b36-08ef-5ca8-7bd6-74dc6c6d8f25}]" +2025-01-25T14:40:14.421Z [INFO] server-7e338f17-662e-9db1-4270-b1a2d9785db0: entering follower state: follower="Node at 7e338f17-662e-9db1-4270-b1a2d9785db0 [Follower]" leader-address= leader-id= +2025-01-25T14:40:14.422Z [INFO] server-ae9ac15f-52e9-7be4-1770-b7443ee67c3b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7e338f17-662e-9db1-4270-b1a2d9785db0 Address:7e338f17-662e-9db1-4270-b1a2d9785db0} {Suffrage:Voter ID:server-ae9ac15f-52e9-7be4-1770-b7443ee67c3b Address:ae9ac15f-52e9-7be4-1770-b7443ee67c3b} {Suffrage:Voter ID:server-9bb82b36-08ef-5ca8-7bd6-74dc6c6d8f25 Address:9bb82b36-08ef-5ca8-7bd6-74dc6c6d8f25}]" +2025-01-25T14:40:14.422Z [INFO] server-ae9ac15f-52e9-7be4-1770-b7443ee67c3b: entering follower state: follower="Node at ae9ac15f-52e9-7be4-1770-b7443ee67c3b [Follower]" leader-address= leader-id= +2025-01-25T14:40:14.423Z [INFO] server-9bb82b36-08ef-5ca8-7bd6-74dc6c6d8f25: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7e338f17-662e-9db1-4270-b1a2d9785db0 Address:7e338f17-662e-9db1-4270-b1a2d9785db0} {Suffrage:Voter ID:server-ae9ac15f-52e9-7be4-1770-b7443ee67c3b Address:ae9ac15f-52e9-7be4-1770-b7443ee67c3b} {Suffrage:Voter ID:server-9bb82b36-08ef-5ca8-7bd6-74dc6c6d8f25 Address:9bb82b36-08ef-5ca8-7bd6-74dc6c6d8f25}]" +2025-01-25T14:40:14.423Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:14.423Z [INFO] server-9bb82b36-08ef-5ca8-7bd6-74dc6c6d8f25: entering follower state: follower="Node at 9bb82b36-08ef-5ca8-7bd6-74dc6c6d8f25 [Follower]" leader-address= leader-id= +2025-01-25T14:40:15.018Z [WARN] server-9bb82b36-08ef-5ca8-7bd6-74dc6c6d8f25: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:15.018Z [INFO] server-9bb82b36-08ef-5ca8-7bd6-74dc6c6d8f25: entering candidate state: node="Node at 9bb82b36-08ef-5ca8-7bd6-74dc6c6d8f25 [Candidate]" term=2 +2025-01-25T14:40:15.019Z [INFO] server-9bb82b36-08ef-5ca8-7bd6-74dc6c6d8f25: election won: term=2 tally=2 +2025-01-25T14:40:15.019Z [INFO] server-9bb82b36-08ef-5ca8-7bd6-74dc6c6d8f25: entering leader state: leader="Node at 9bb82b36-08ef-5ca8-7bd6-74dc6c6d8f25 [Leader]" +2025-01-25T14:40:15.019Z [INFO] server-9bb82b36-08ef-5ca8-7bd6-74dc6c6d8f25: added peer, starting replication: peer=server-7e338f17-662e-9db1-4270-b1a2d9785db0 +2025-01-25T14:40:15.019Z [INFO] server-9bb82b36-08ef-5ca8-7bd6-74dc6c6d8f25: added peer, starting replication: peer=server-ae9ac15f-52e9-7be4-1770-b7443ee67c3b +2025-01-25T14:40:15.020Z [INFO] server-9bb82b36-08ef-5ca8-7bd6-74dc6c6d8f25: pipelining replication: peer="{Voter server-7e338f17-662e-9db1-4270-b1a2d9785db0 7e338f17-662e-9db1-4270-b1a2d9785db0}" +2025-01-25T14:40:15.020Z [INFO] server-9bb82b36-08ef-5ca8-7bd6-74dc6c6d8f25: pipelining replication: peer="{Voter server-ae9ac15f-52e9-7be4-1770-b7443ee67c3b ae9ac15f-52e9-7be4-1770-b7443ee67c3b}" +2025-01-25T14:40:16.026Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:15.019469518 +0000 UTC m=+84.065308835 (1 nodes), 595.840859ms from start of poll, 598.687534ms from cluster start. Timeout at 2025-01-25 14:40:16.024491684 +0000 UTC m=+85.070331002, 1.005022167s after stability +2025-01-25T14:40:16.029Z [INFO] server-9bb82b36-08ef-5ca8-7bd6-74dc6c6d8f25: starting snapshot up to: index=12 +2025-01-25T14:40:16.029Z [INFO] creating new snapshot: path=/tmp/raft1424512567/snapshots/2-12-1737816016029.tmp +2025-01-25T14:40:16.033Z [INFO] server-9bb82b36-08ef-5ca8-7bd6-74dc6c6d8f25: snapshot complete up to: index=12 +2025-01-25T14:40:16.040Z [INFO] server-13bee13e-95ca-40da-11f2-0e2fed67d712: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-13bee13e-95ca-40da-11f2-0e2fed67d712 Address:13bee13e-95ca-40da-11f2-0e2fed67d712} {Suffrage:Voter ID:server-5c0b15f5-5e2e-fe94-edf6-ab2432635ca1 Address:5c0b15f5-5e2e-fe94-edf6-ab2432635ca1} {Suffrage:Voter ID:server-52feb266-b2cc-d60e-a3e4-71e037df82e1 Address:52feb266-b2cc-d60e-a3e4-71e037df82e1}]" +2025-01-25T14:40:16.041Z [INFO] server-13bee13e-95ca-40da-11f2-0e2fed67d712: entering follower state: follower="Node at 13bee13e-95ca-40da-11f2-0e2fed67d712 [Follower]" leader-address= leader-id= +2025-01-25T14:40:16.041Z [INFO] server-5c0b15f5-5e2e-fe94-edf6-ab2432635ca1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-13bee13e-95ca-40da-11f2-0e2fed67d712 Address:13bee13e-95ca-40da-11f2-0e2fed67d712} {Suffrage:Voter ID:server-5c0b15f5-5e2e-fe94-edf6-ab2432635ca1 Address:5c0b15f5-5e2e-fe94-edf6-ab2432635ca1} {Suffrage:Voter ID:server-52feb266-b2cc-d60e-a3e4-71e037df82e1 Address:52feb266-b2cc-d60e-a3e4-71e037df82e1}]" +2025-01-25T14:40:16.042Z [INFO] server-52feb266-b2cc-d60e-a3e4-71e037df82e1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-13bee13e-95ca-40da-11f2-0e2fed67d712 Address:13bee13e-95ca-40da-11f2-0e2fed67d712} {Suffrage:Voter ID:server-5c0b15f5-5e2e-fe94-edf6-ab2432635ca1 Address:5c0b15f5-5e2e-fe94-edf6-ab2432635ca1} {Suffrage:Voter ID:server-52feb266-b2cc-d60e-a3e4-71e037df82e1 Address:52feb266-b2cc-d60e-a3e4-71e037df82e1}]" +2025-01-25T14:40:16.042Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:16.042Z [INFO] server-5c0b15f5-5e2e-fe94-edf6-ab2432635ca1: entering follower state: follower="Node at 5c0b15f5-5e2e-fe94-edf6-ab2432635ca1 [Follower]" leader-address= leader-id= +2025-01-25T14:40:16.042Z [INFO] server-52feb266-b2cc-d60e-a3e4-71e037df82e1: entering follower state: follower="Node at 52feb266-b2cc-d60e-a3e4-71e037df82e1 [Follower]" leader-address= leader-id= +2025-01-25T14:40:16.543Z [WARN] server-13bee13e-95ca-40da-11f2-0e2fed67d712: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:16.543Z [INFO] server-13bee13e-95ca-40da-11f2-0e2fed67d712: entering candidate state: node="Node at 13bee13e-95ca-40da-11f2-0e2fed67d712 [Candidate]" term=2 +2025-01-25T14:40:16.544Z [INFO] server-13bee13e-95ca-40da-11f2-0e2fed67d712: election won: term=2 tally=2 +2025-01-25T14:40:16.544Z [INFO] server-13bee13e-95ca-40da-11f2-0e2fed67d712: entering leader state: leader="Node at 13bee13e-95ca-40da-11f2-0e2fed67d712 [Leader]" +2025-01-25T14:40:16.544Z [INFO] server-13bee13e-95ca-40da-11f2-0e2fed67d712: added peer, starting replication: peer=server-5c0b15f5-5e2e-fe94-edf6-ab2432635ca1 +2025-01-25T14:40:16.545Z [INFO] server-13bee13e-95ca-40da-11f2-0e2fed67d712: added peer, starting replication: peer=server-52feb266-b2cc-d60e-a3e4-71e037df82e1 +2025-01-25T14:40:16.545Z [INFO] server-13bee13e-95ca-40da-11f2-0e2fed67d712: pipelining replication: peer="{Voter server-52feb266-b2cc-d60e-a3e4-71e037df82e1 52feb266-b2cc-d60e-a3e4-71e037df82e1}" +2025-01-25T14:40:16.546Z [INFO] server-13bee13e-95ca-40da-11f2-0e2fed67d712: pipelining replication: peer="{Voter server-5c0b15f5-5e2e-fe94-edf6-ab2432635ca1 5c0b15f5-5e2e-fe94-edf6-ab2432635ca1}" +2025-01-25T14:40:17.550Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:16.545052876 +0000 UTC m=+85.590892485 (1 nodes), 502.253004ms from start of poll, 504.954428ms from cluster start. Timeout at 2025-01-25 14:40:17.550065709 +0000 UTC m=+86.595904735, 1.00501225s after stability +2025-01-25T14:40:17.551Z [INFO] creating new snapshot: path=/tmp/raft3919648531/snapshots/2-1013-1737816017551.tmp +2025-01-25T14:40:17.554Z [INFO] server-13bee13e-95ca-40da-11f2-0e2fed67d712: copied to local snapshot: bytes=71 +2025-01-25T14:40:17.555Z [INFO] server-13bee13e-95ca-40da-11f2-0e2fed67d712: snapshot restore progress: id=2-1013-1737816017551 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:17.555Z [INFO] server-13bee13e-95ca-40da-11f2-0e2fed67d712: removing all old logs from log store +2025-01-25T14:40:17.555Z [INFO] server-13bee13e-95ca-40da-11f2-0e2fed67d712: compacting logs: from=1 to=2 +2025-01-25T14:40:17.555Z [INFO] server-13bee13e-95ca-40da-11f2-0e2fed67d712: restored user snapshot: index=1013 +2025-01-25T14:40:17.555Z [ERROR] server-13bee13e-95ca-40da-11f2-0e2fed67d712: failed to get log: index=2 error="log not found" +2025-01-25T14:40:17.555Z [INFO] server-13bee13e-95ca-40da-11f2-0e2fed67d712: aborting pipeline replication: peer="{Voter server-5c0b15f5-5e2e-fe94-edf6-ab2432635ca1 5c0b15f5-5e2e-fe94-edf6-ab2432635ca1}" +2025-01-25T14:40:17.555Z [ERROR] server-13bee13e-95ca-40da-11f2-0e2fed67d712: failed to get log: index=2 error="log not found" +2025-01-25T14:40:17.555Z [INFO] server-13bee13e-95ca-40da-11f2-0e2fed67d712: aborting pipeline replication: peer="{Voter server-52feb266-b2cc-d60e-a3e4-71e037df82e1 52feb266-b2cc-d60e-a3e4-71e037df82e1}" +2025-01-25T14:40:17.561Z [ERROR] server-13bee13e-95ca-40da-11f2-0e2fed67d712: failed to get log: index=2 error="log not found" +2025-01-25T14:40:17.562Z [ERROR] server-13bee13e-95ca-40da-11f2-0e2fed67d712: failed to get log: index=2 error="log not found" +2025-01-25T14:40:17.564Z [INFO] creating new snapshot: path=/tmp/raft110672445/snapshots/2-1013-1737816017564.tmp +2025-01-25T14:40:17.565Z [INFO] creating new snapshot: path=/tmp/raft1682964489/snapshots/2-1013-1737816017565.tmp +2025-01-25T14:40:17.566Z [INFO] server-52feb266-b2cc-d60e-a3e4-71e037df82e1: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:17.566Z [INFO] server-5c0b15f5-5e2e-fe94-edf6-ab2432635ca1: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:17.568Z [INFO] server-5c0b15f5-5e2e-fe94-edf6-ab2432635ca1: copied to local snapshot: bytes=71 +2025-01-25T14:40:17.569Z [INFO] server-52feb266-b2cc-d60e-a3e4-71e037df82e1: copied to local snapshot: bytes=71 +2025-01-25T14:40:17.570Z [INFO] server-5c0b15f5-5e2e-fe94-edf6-ab2432635ca1: snapshot restore progress: id=2-1013-1737816017564 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:17.570Z [INFO] server-52feb266-b2cc-d60e-a3e4-71e037df82e1: snapshot restore progress: id=2-1013-1737816017565 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:17.570Z [INFO] server-5c0b15f5-5e2e-fe94-edf6-ab2432635ca1: removing all old logs from log store +2025-01-25T14:40:17.570Z [INFO] server-5c0b15f5-5e2e-fe94-edf6-ab2432635ca1: compacting logs: from=1 to=2 +2025-01-25T14:40:17.570Z [INFO] server-5c0b15f5-5e2e-fe94-edf6-ab2432635ca1: Installed remote snapshot +2025-01-25T14:40:17.570Z [INFO] server-52feb266-b2cc-d60e-a3e4-71e037df82e1: removing all old logs from log store +2025-01-25T14:40:17.570Z [INFO] server-52feb266-b2cc-d60e-a3e4-71e037df82e1: compacting logs: from=1 to=2 +2025-01-25T14:40:17.570Z [INFO] server-52feb266-b2cc-d60e-a3e4-71e037df82e1: Installed remote snapshot +2025-01-25T14:40:17.571Z [INFO] server-13bee13e-95ca-40da-11f2-0e2fed67d712: pipelining replication: peer="{Voter server-52feb266-b2cc-d60e-a3e4-71e037df82e1 52feb266-b2cc-d60e-a3e4-71e037df82e1}" +2025-01-25T14:40:17.571Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:40:17.571Z [INFO] server-13bee13e-95ca-40da-11f2-0e2fed67d712: pipelining replication: peer="{Voter server-5c0b15f5-5e2e-fe94-edf6-ab2432635ca1 5c0b15f5-5e2e-fe94-edf6-ab2432635ca1}" +2025-01-25T14:40:18.577Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:40:17.571941407 +0000 UTC m=+86.617780725 (2 nodes), 6.125µs from start of poll, 1.531842668s from cluster start. Timeout at 2025-01-25 14:40:18.577141491 +0000 UTC m=+87.622980809, 1.005200084s after stability +2025-01-25T14:40:18.592Z [INFO] server-9bb82b36-08ef-5ca8-7bd6-74dc6c6d8f25: aborting pipeline replication: peer="{Voter server-ae9ac15f-52e9-7be4-1770-b7443ee67c3b ae9ac15f-52e9-7be4-1770-b7443ee67c3b}" +2025-01-25T14:40:18.592Z [INFO] server-9bb82b36-08ef-5ca8-7bd6-74dc6c6d8f25: aborting pipeline replication: peer="{Voter server-7e338f17-662e-9db1-4270-b1a2d9785db0 7e338f17-662e-9db1-4270-b1a2d9785db0}" === RUN TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_false -2025-01-25T14:29:32.469Z [INFO] server-23e3abb6-f312-acc9-15cd-330a48fc9f67: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-23e3abb6-f312-acc9-15cd-330a48fc9f67 Address:23e3abb6-f312-acc9-15cd-330a48fc9f67} {Suffrage:Voter ID:server-95f24c46-890b-9e09-638e-f956eeb2397b Address:95f24c46-890b-9e09-638e-f956eeb2397b} {Suffrage:Voter ID:server-c4659b27-d1b5-5fde-eee3-1131794b82d3 Address:c4659b27-d1b5-5fde-eee3-1131794b82d3}]" -2025-01-25T14:29:32.469Z [INFO] server-95f24c46-890b-9e09-638e-f956eeb2397b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-23e3abb6-f312-acc9-15cd-330a48fc9f67 Address:23e3abb6-f312-acc9-15cd-330a48fc9f67} {Suffrage:Voter ID:server-95f24c46-890b-9e09-638e-f956eeb2397b Address:95f24c46-890b-9e09-638e-f956eeb2397b} {Suffrage:Voter ID:server-c4659b27-d1b5-5fde-eee3-1131794b82d3 Address:c4659b27-d1b5-5fde-eee3-1131794b82d3}]" -2025-01-25T14:29:32.470Z [INFO] server-c4659b27-d1b5-5fde-eee3-1131794b82d3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-23e3abb6-f312-acc9-15cd-330a48fc9f67 Address:23e3abb6-f312-acc9-15cd-330a48fc9f67} {Suffrage:Voter ID:server-95f24c46-890b-9e09-638e-f956eeb2397b Address:95f24c46-890b-9e09-638e-f956eeb2397b} {Suffrage:Voter ID:server-c4659b27-d1b5-5fde-eee3-1131794b82d3 Address:c4659b27-d1b5-5fde-eee3-1131794b82d3}]" -2025-01-25T14:29:32.470Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:32.470Z [INFO] server-95f24c46-890b-9e09-638e-f956eeb2397b: entering follower state: follower="Node at 95f24c46-890b-9e09-638e-f956eeb2397b [Follower]" leader-address= leader-id= -2025-01-25T14:29:32.470Z [INFO] server-23e3abb6-f312-acc9-15cd-330a48fc9f67: entering follower state: follower="Node at 23e3abb6-f312-acc9-15cd-330a48fc9f67 [Follower]" leader-address= leader-id= -2025-01-25T14:29:32.470Z [INFO] server-c4659b27-d1b5-5fde-eee3-1131794b82d3: entering follower state: follower="Node at c4659b27-d1b5-5fde-eee3-1131794b82d3 [Follower]" leader-address= leader-id= -2025-01-25T14:29:33.201Z [WARN] server-23e3abb6-f312-acc9-15cd-330a48fc9f67: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:33.201Z [INFO] server-23e3abb6-f312-acc9-15cd-330a48fc9f67: entering candidate state: node="Node at 23e3abb6-f312-acc9-15cd-330a48fc9f67 [Candidate]" term=2 -2025-01-25T14:29:33.201Z [INFO] server-23e3abb6-f312-acc9-15cd-330a48fc9f67: election won: term=2 tally=2 -2025-01-25T14:29:33.201Z [INFO] server-23e3abb6-f312-acc9-15cd-330a48fc9f67: entering leader state: leader="Node at 23e3abb6-f312-acc9-15cd-330a48fc9f67 [Leader]" -2025-01-25T14:29:33.201Z [INFO] server-23e3abb6-f312-acc9-15cd-330a48fc9f67: added peer, starting replication: peer=server-95f24c46-890b-9e09-638e-f956eeb2397b -2025-01-25T14:29:33.201Z [INFO] server-23e3abb6-f312-acc9-15cd-330a48fc9f67: added peer, starting replication: peer=server-c4659b27-d1b5-5fde-eee3-1131794b82d3 -2025-01-25T14:29:33.201Z [INFO] server-23e3abb6-f312-acc9-15cd-330a48fc9f67: pipelining replication: peer="{Voter server-c4659b27-d1b5-5fde-eee3-1131794b82d3 c4659b27-d1b5-5fde-eee3-1131794b82d3}" -2025-01-25T14:29:33.201Z [INFO] server-23e3abb6-f312-acc9-15cd-330a48fc9f67: pipelining replication: peer="{Voter server-95f24c46-890b-9e09-638e-f956eeb2397b 95f24c46-890b-9e09-638e-f956eeb2397b}" -2025-01-25T14:29:34.208Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:33.201780462 +0000 UTC m=+83.056105444 (1 nodes), 731.64711ms from start of poll, 732.361605ms from cluster start. Timeout at 2025-01-25 14:29:34.206785422 +0000 UTC m=+84.061110444, 1.005005s after stability -2025-01-25T14:29:34.208Z [INFO] server-23e3abb6-f312-acc9-15cd-330a48fc9f67: starting snapshot up to: index=12 -2025-01-25T14:29:34.209Z [INFO] creating new snapshot: path=/tmp/raft322268888/snapshots/2-12-1737815374209.tmp -2025-01-25T14:29:34.210Z [INFO] server-23e3abb6-f312-acc9-15cd-330a48fc9f67: snapshot complete up to: index=12 -2025-01-25T14:29:34.211Z [INFO] creating new snapshot: path=/tmp/raft322268888/snapshots/2-10013-1737815374211.tmp -2025-01-25T14:29:34.212Z [INFO] server-23e3abb6-f312-acc9-15cd-330a48fc9f67: copied to local snapshot: bytes=71 -2025-01-25T14:29:34.213Z [INFO] server-23e3abb6-f312-acc9-15cd-330a48fc9f67: snapshot restore progress: id=2-10013-1737815374211 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:34.213Z [INFO] server-23e3abb6-f312-acc9-15cd-330a48fc9f67: restored user snapshot: index=10013 -2025-01-25T14:29:34.213Z [ERROR] server-23e3abb6-f312-acc9-15cd-330a48fc9f67: failed to get log: index=23 error="log not found" -2025-01-25T14:29:34.213Z [ERROR] server-23e3abb6-f312-acc9-15cd-330a48fc9f67: failed to get log: index=23 error="log not found" -2025-01-25T14:29:34.213Z [INFO] server-23e3abb6-f312-acc9-15cd-330a48fc9f67: aborting pipeline replication: peer="{Voter server-c4659b27-d1b5-5fde-eee3-1131794b82d3 c4659b27-d1b5-5fde-eee3-1131794b82d3}" -2025-01-25T14:29:34.213Z [INFO] server-23e3abb6-f312-acc9-15cd-330a48fc9f67: aborting pipeline replication: peer="{Voter server-95f24c46-890b-9e09-638e-f956eeb2397b 95f24c46-890b-9e09-638e-f956eeb2397b}" -2025-01-25T14:29:34.223Z [ERROR] server-23e3abb6-f312-acc9-15cd-330a48fc9f67: failed to get log: index=23 error="log not found" -2025-01-25T14:29:34.223Z [ERROR] server-23e3abb6-f312-acc9-15cd-330a48fc9f67: failed to get log: index=23 error="log not found" -2025-01-25T14:29:34.224Z [INFO] creating new snapshot: path=/tmp/raft1531839359/snapshots/2-10013-1737815374224.tmp -2025-01-25T14:29:34.224Z [INFO] creating new snapshot: path=/tmp/raft1184432685/snapshots/2-10013-1737815374224.tmp -2025-01-25T14:29:34.224Z [INFO] server-95f24c46-890b-9e09-638e-f956eeb2397b: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:34.225Z [INFO] server-c4659b27-d1b5-5fde-eee3-1131794b82d3: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:34.225Z [INFO] server-95f24c46-890b-9e09-638e-f956eeb2397b: copied to local snapshot: bytes=71 -2025-01-25T14:29:34.225Z [INFO] server-c4659b27-d1b5-5fde-eee3-1131794b82d3: copied to local snapshot: bytes=71 -2025-01-25T14:29:34.226Z [INFO] server-95f24c46-890b-9e09-638e-f956eeb2397b: snapshot restore progress: id=2-10013-1737815374224 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:34.226Z [INFO] server-95f24c46-890b-9e09-638e-f956eeb2397b: Installed remote snapshot -2025-01-25T14:29:34.226Z [WARN] server-95f24c46-890b-9e09-638e-f956eeb2397b: skipping application of old log: index=22 -2025-01-25T14:29:34.226Z [INFO] server-23e3abb6-f312-acc9-15cd-330a48fc9f67: pipelining replication: peer="{Voter server-95f24c46-890b-9e09-638e-f956eeb2397b 95f24c46-890b-9e09-638e-f956eeb2397b}" -2025-01-25T14:29:34.226Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:29:34.227Z [INFO] server-c4659b27-d1b5-5fde-eee3-1131794b82d3: snapshot restore progress: id=2-10013-1737815374224 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:34.227Z [INFO] server-c4659b27-d1b5-5fde-eee3-1131794b82d3: Installed remote snapshot -2025-01-25T14:29:34.227Z [INFO] server-23e3abb6-f312-acc9-15cd-330a48fc9f67: pipelining replication: peer="{Voter server-c4659b27-d1b5-5fde-eee3-1131794b82d3 c4659b27-d1b5-5fde-eee3-1131794b82d3}" -2025-01-25T14:29:35.232Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:29:34.226898851 +0000 UTC m=+84.081223833 (2 nodes), 1.68µs from start of poll, 1.757479994s from cluster start. Timeout at 2025-01-25 14:29:35.232051526 +0000 UTC m=+85.086376468, 1.005152635s after stability -2025-01-25T14:29:35.245Z [INFO] server-23e3abb6-f312-acc9-15cd-330a48fc9f67: aborting pipeline replication: peer="{Voter server-c4659b27-d1b5-5fde-eee3-1131794b82d3 c4659b27-d1b5-5fde-eee3-1131794b82d3}" -2025-01-25T14:29:35.245Z [INFO] server-23e3abb6-f312-acc9-15cd-330a48fc9f67: aborting pipeline replication: peer="{Voter server-95f24c46-890b-9e09-638e-f956eeb2397b 95f24c46-890b-9e09-638e-f956eeb2397b}" +2025-01-25T14:40:18.602Z [INFO] server-3f7b8e48-d493-a316-dcc5-57aad63939f3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3f7b8e48-d493-a316-dcc5-57aad63939f3 Address:3f7b8e48-d493-a316-dcc5-57aad63939f3} {Suffrage:Voter ID:server-3a435977-db2d-a940-6f2e-5f8ca2f55b5a Address:3a435977-db2d-a940-6f2e-5f8ca2f55b5a} {Suffrage:Voter ID:server-348b27c7-9fe1-3b70-ee8e-aff02a349fb9 Address:348b27c7-9fe1-3b70-ee8e-aff02a349fb9}]" +2025-01-25T14:40:18.602Z [INFO] server-3f7b8e48-d493-a316-dcc5-57aad63939f3: entering follower state: follower="Node at 3f7b8e48-d493-a316-dcc5-57aad63939f3 [Follower]" leader-address= leader-id= +2025-01-25T14:40:18.602Z [INFO] server-3a435977-db2d-a940-6f2e-5f8ca2f55b5a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3f7b8e48-d493-a316-dcc5-57aad63939f3 Address:3f7b8e48-d493-a316-dcc5-57aad63939f3} {Suffrage:Voter ID:server-3a435977-db2d-a940-6f2e-5f8ca2f55b5a Address:3a435977-db2d-a940-6f2e-5f8ca2f55b5a} {Suffrage:Voter ID:server-348b27c7-9fe1-3b70-ee8e-aff02a349fb9 Address:348b27c7-9fe1-3b70-ee8e-aff02a349fb9}]" +2025-01-25T14:40:18.603Z [INFO] server-3a435977-db2d-a940-6f2e-5f8ca2f55b5a: entering follower state: follower="Node at 3a435977-db2d-a940-6f2e-5f8ca2f55b5a [Follower]" leader-address= leader-id= +2025-01-25T14:40:18.603Z [INFO] server-348b27c7-9fe1-3b70-ee8e-aff02a349fb9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3f7b8e48-d493-a316-dcc5-57aad63939f3 Address:3f7b8e48-d493-a316-dcc5-57aad63939f3} {Suffrage:Voter ID:server-3a435977-db2d-a940-6f2e-5f8ca2f55b5a Address:3a435977-db2d-a940-6f2e-5f8ca2f55b5a} {Suffrage:Voter ID:server-348b27c7-9fe1-3b70-ee8e-aff02a349fb9 Address:348b27c7-9fe1-3b70-ee8e-aff02a349fb9}]" +2025-01-25T14:40:18.604Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:18.604Z [INFO] server-348b27c7-9fe1-3b70-ee8e-aff02a349fb9: entering follower state: follower="Node at 348b27c7-9fe1-3b70-ee8e-aff02a349fb9 [Follower]" leader-address= leader-id= +2025-01-25T14:40:19.468Z [WARN] server-348b27c7-9fe1-3b70-ee8e-aff02a349fb9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:19.468Z [INFO] server-348b27c7-9fe1-3b70-ee8e-aff02a349fb9: entering candidate state: node="Node at 348b27c7-9fe1-3b70-ee8e-aff02a349fb9 [Candidate]" term=2 +2025-01-25T14:40:19.468Z [INFO] server-348b27c7-9fe1-3b70-ee8e-aff02a349fb9: election won: term=2 tally=2 +2025-01-25T14:40:19.468Z [INFO] server-348b27c7-9fe1-3b70-ee8e-aff02a349fb9: entering leader state: leader="Node at 348b27c7-9fe1-3b70-ee8e-aff02a349fb9 [Leader]" +2025-01-25T14:40:19.469Z [INFO] server-348b27c7-9fe1-3b70-ee8e-aff02a349fb9: added peer, starting replication: peer=server-3f7b8e48-d493-a316-dcc5-57aad63939f3 +2025-01-25T14:40:19.469Z [INFO] server-348b27c7-9fe1-3b70-ee8e-aff02a349fb9: added peer, starting replication: peer=server-3a435977-db2d-a940-6f2e-5f8ca2f55b5a +2025-01-25T14:40:19.469Z [INFO] server-348b27c7-9fe1-3b70-ee8e-aff02a349fb9: pipelining replication: peer="{Voter server-3f7b8e48-d493-a316-dcc5-57aad63939f3 3f7b8e48-d493-a316-dcc5-57aad63939f3}" +2025-01-25T14:40:19.470Z [INFO] server-348b27c7-9fe1-3b70-ee8e-aff02a349fb9: pipelining replication: peer="{Voter server-3a435977-db2d-a940-6f2e-5f8ca2f55b5a 3a435977-db2d-a940-6f2e-5f8ca2f55b5a}" +2025-01-25T14:40:20.475Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:19.469258389 +0000 UTC m=+88.515097998 (1 nodes), 865.121435ms from start of poll, 867.842693ms from cluster start. Timeout at 2025-01-25 14:40:20.474269472 +0000 UTC m=+89.520108790, 1.005010792s after stability +2025-01-25T14:40:20.479Z [INFO] server-348b27c7-9fe1-3b70-ee8e-aff02a349fb9: starting snapshot up to: index=12 +2025-01-25T14:40:20.479Z [INFO] creating new snapshot: path=/tmp/raft4169431295/snapshots/2-12-1737816020479.tmp +2025-01-25T14:40:20.484Z [INFO] server-348b27c7-9fe1-3b70-ee8e-aff02a349fb9: snapshot complete up to: index=12 +2025-01-25T14:40:20.488Z [INFO] creating new snapshot: path=/tmp/raft4169431295/snapshots/2-10013-1737816020488.tmp +2025-01-25T14:40:20.492Z [INFO] server-348b27c7-9fe1-3b70-ee8e-aff02a349fb9: copied to local snapshot: bytes=71 +2025-01-25T14:40:20.494Z [INFO] server-348b27c7-9fe1-3b70-ee8e-aff02a349fb9: snapshot restore progress: id=2-10013-1737816020488 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:20.494Z [INFO] server-348b27c7-9fe1-3b70-ee8e-aff02a349fb9: restored user snapshot: index=10013 +2025-01-25T14:40:20.495Z [ERROR] server-348b27c7-9fe1-3b70-ee8e-aff02a349fb9: failed to get log: index=23 error="log not found" +2025-01-25T14:40:20.495Z [ERROR] server-348b27c7-9fe1-3b70-ee8e-aff02a349fb9: failed to get log: index=23 error="log not found" +2025-01-25T14:40:20.495Z [INFO] server-348b27c7-9fe1-3b70-ee8e-aff02a349fb9: aborting pipeline replication: peer="{Voter server-3f7b8e48-d493-a316-dcc5-57aad63939f3 3f7b8e48-d493-a316-dcc5-57aad63939f3}" +2025-01-25T14:40:20.495Z [INFO] server-348b27c7-9fe1-3b70-ee8e-aff02a349fb9: aborting pipeline replication: peer="{Voter server-3a435977-db2d-a940-6f2e-5f8ca2f55b5a 3a435977-db2d-a940-6f2e-5f8ca2f55b5a}" +2025-01-25T14:40:20.501Z [ERROR] server-348b27c7-9fe1-3b70-ee8e-aff02a349fb9: failed to get log: index=23 error="log not found" +2025-01-25T14:40:20.504Z [INFO] creating new snapshot: path=/tmp/raft1460476360/snapshots/2-10013-1737816020504.tmp +2025-01-25T14:40:20.504Z [ERROR] server-348b27c7-9fe1-3b70-ee8e-aff02a349fb9: failed to get log: index=23 error="log not found" +2025-01-25T14:40:20.506Z [INFO] server-3f7b8e48-d493-a316-dcc5-57aad63939f3: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:20.506Z [INFO] creating new snapshot: path=/tmp/raft1705459359/snapshots/2-10013-1737816020506.tmp +2025-01-25T14:40:20.507Z [INFO] server-3a435977-db2d-a940-6f2e-5f8ca2f55b5a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:20.515Z [INFO] server-3a435977-db2d-a940-6f2e-5f8ca2f55b5a: copied to local snapshot: bytes=71 +2025-01-25T14:40:20.523Z [INFO] server-3f7b8e48-d493-a316-dcc5-57aad63939f3: copied to local snapshot: bytes=71 +2025-01-25T14:40:20.525Z [INFO] server-3a435977-db2d-a940-6f2e-5f8ca2f55b5a: snapshot restore progress: id=2-10013-1737816020506 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:20.525Z [INFO] server-3f7b8e48-d493-a316-dcc5-57aad63939f3: snapshot restore progress: id=2-10013-1737816020504 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:20.525Z [INFO] server-3a435977-db2d-a940-6f2e-5f8ca2f55b5a: Installed remote snapshot +2025-01-25T14:40:20.525Z [INFO] server-3f7b8e48-d493-a316-dcc5-57aad63939f3: Installed remote snapshot +2025-01-25T14:40:20.525Z [WARN] server-3a435977-db2d-a940-6f2e-5f8ca2f55b5a: skipping application of old log: index=10013 +2025-01-25T14:40:20.526Z [INFO] server-348b27c7-9fe1-3b70-ee8e-aff02a349fb9: pipelining replication: peer="{Voter server-3f7b8e48-d493-a316-dcc5-57aad63939f3 3f7b8e48-d493-a316-dcc5-57aad63939f3}" +2025-01-25T14:40:20.526Z [INFO] server-348b27c7-9fe1-3b70-ee8e-aff02a349fb9: pipelining replication: peer="{Voter server-3a435977-db2d-a940-6f2e-5f8ca2f55b5a 3a435977-db2d-a940-6f2e-5f8ca2f55b5a}" +2025-01-25T14:40:20.526Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:40:21.531Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:40:20.526360717 +0000 UTC m=+89.572200034 (2 nodes), 7µs from start of poll, 1.924944729s from cluster start. Timeout at 2025-01-25 14:40:21.5315678 +0000 UTC m=+90.577407118, 1.005207084s after stability +2025-01-25T14:40:21.546Z [INFO] server-348b27c7-9fe1-3b70-ee8e-aff02a349fb9: aborting pipeline replication: peer="{Voter server-3f7b8e48-d493-a316-dcc5-57aad63939f3 3f7b8e48-d493-a316-dcc5-57aad63939f3}" +2025-01-25T14:40:21.546Z [INFO] server-348b27c7-9fe1-3b70-ee8e-aff02a349fb9: aborting pipeline replication: peer="{Voter server-3a435977-db2d-a940-6f2e-5f8ca2f55b5a 3a435977-db2d-a940-6f2e-5f8ca2f55b5a}" === RUN TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_false -2025-01-25T14:29:35.248Z [INFO] server-4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b Address:4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b} {Suffrage:Voter ID:server-3a7a655e-2090-5c6c-0f06-ecd30723cde6 Address:3a7a655e-2090-5c6c-0f06-ecd30723cde6} {Suffrage:Voter ID:server-b4acdb5e-008f-1684-0d80-4b1b1a2cd6ca Address:b4acdb5e-008f-1684-0d80-4b1b1a2cd6ca}]" -2025-01-25T14:29:35.248Z [INFO] server-4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b: entering follower state: follower="Node at 4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b [Follower]" leader-address= leader-id= -2025-01-25T14:29:35.248Z [INFO] server-3a7a655e-2090-5c6c-0f06-ecd30723cde6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b Address:4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b} {Suffrage:Voter ID:server-3a7a655e-2090-5c6c-0f06-ecd30723cde6 Address:3a7a655e-2090-5c6c-0f06-ecd30723cde6} {Suffrage:Voter ID:server-b4acdb5e-008f-1684-0d80-4b1b1a2cd6ca Address:b4acdb5e-008f-1684-0d80-4b1b1a2cd6ca}]" -2025-01-25T14:29:35.248Z [INFO] server-3a7a655e-2090-5c6c-0f06-ecd30723cde6: entering follower state: follower="Node at 3a7a655e-2090-5c6c-0f06-ecd30723cde6 [Follower]" leader-address= leader-id= -2025-01-25T14:29:35.248Z [INFO] server-b4acdb5e-008f-1684-0d80-4b1b1a2cd6ca: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b Address:4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b} {Suffrage:Voter ID:server-3a7a655e-2090-5c6c-0f06-ecd30723cde6 Address:3a7a655e-2090-5c6c-0f06-ecd30723cde6} {Suffrage:Voter ID:server-b4acdb5e-008f-1684-0d80-4b1b1a2cd6ca Address:b4acdb5e-008f-1684-0d80-4b1b1a2cd6ca}]" -2025-01-25T14:29:35.248Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:35.248Z [INFO] server-b4acdb5e-008f-1684-0d80-4b1b1a2cd6ca: entering follower state: follower="Node at b4acdb5e-008f-1684-0d80-4b1b1a2cd6ca [Follower]" leader-address= leader-id= -2025-01-25T14:29:35.834Z [WARN] server-4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:35.834Z [INFO] server-4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b: entering candidate state: node="Node at 4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b [Candidate]" term=2 -2025-01-25T14:29:35.834Z [INFO] server-4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b: election won: term=2 tally=2 -2025-01-25T14:29:35.834Z [INFO] server-4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b: entering leader state: leader="Node at 4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b [Leader]" -2025-01-25T14:29:35.834Z [INFO] server-4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b: added peer, starting replication: peer=server-3a7a655e-2090-5c6c-0f06-ecd30723cde6 -2025-01-25T14:29:35.834Z [INFO] server-4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b: added peer, starting replication: peer=server-b4acdb5e-008f-1684-0d80-4b1b1a2cd6ca -2025-01-25T14:29:35.835Z [INFO] server-4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b: pipelining replication: peer="{Voter server-b4acdb5e-008f-1684-0d80-4b1b1a2cd6ca b4acdb5e-008f-1684-0d80-4b1b1a2cd6ca}" -2025-01-25T14:29:35.835Z [INFO] server-4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b: pipelining replication: peer="{Voter server-3a7a655e-2090-5c6c-0f06-ecd30723cde6 3a7a655e-2090-5c6c-0f06-ecd30723cde6}" -2025-01-25T14:29:36.840Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:35.834949263 +0000 UTC m=+85.689274205 (1 nodes), 586.06011ms from start of poll, 586.778604ms from cluster start. Timeout at 2025-01-25 14:29:36.839952103 +0000 UTC m=+86.694277765, 1.00500356s after stability -2025-01-25T14:29:36.841Z [INFO] server-4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b: starting snapshot up to: index=12 -2025-01-25T14:29:36.841Z [INFO] creating new snapshot: path=/tmp/raft824119493/snapshots/2-12-1737815376841.tmp -2025-01-25T14:29:36.842Z [INFO] server-4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b: snapshot complete up to: index=12 -2025-01-25T14:29:36.844Z [INFO] creating new snapshot: path=/tmp/raft824119493/snapshots/2-10013-1737815376844.tmp -2025-01-25T14:29:36.845Z [INFO] server-4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b: copied to local snapshot: bytes=71 -2025-01-25T14:29:36.845Z [INFO] server-4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b: snapshot restore progress: id=2-10013-1737815376844 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:36.845Z [INFO] server-4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b: removing all old logs from log store -2025-01-25T14:29:36.845Z [INFO] server-4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b: compacting logs: from=1 to=22 -2025-01-25T14:29:36.845Z [INFO] server-4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b: restored user snapshot: index=10013 -2025-01-25T14:29:36.845Z [ERROR] server-4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b: failed to get log: index=22 error="log not found" -2025-01-25T14:29:36.845Z [ERROR] server-4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b: failed to get log: index=22 error="log not found" -2025-01-25T14:29:36.845Z [INFO] server-4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b: aborting pipeline replication: peer="{Voter server-3a7a655e-2090-5c6c-0f06-ecd30723cde6 3a7a655e-2090-5c6c-0f06-ecd30723cde6}" -2025-01-25T14:29:36.845Z [INFO] server-4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b: aborting pipeline replication: peer="{Voter server-b4acdb5e-008f-1684-0d80-4b1b1a2cd6ca b4acdb5e-008f-1684-0d80-4b1b1a2cd6ca}" -2025-01-25T14:29:36.851Z [ERROR] server-4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b: failed to get log: index=22 error="log not found" -2025-01-25T14:29:36.851Z [INFO] creating new snapshot: path=/tmp/raft1759923620/snapshots/2-10013-1737815376851.tmp -2025-01-25T14:29:36.852Z [INFO] server-b4acdb5e-008f-1684-0d80-4b1b1a2cd6ca: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:36.853Z [INFO] server-b4acdb5e-008f-1684-0d80-4b1b1a2cd6ca: copied to local snapshot: bytes=71 -2025-01-25T14:29:36.853Z [INFO] server-b4acdb5e-008f-1684-0d80-4b1b1a2cd6ca: snapshot restore progress: id=2-10013-1737815376851 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:36.853Z [INFO] server-b4acdb5e-008f-1684-0d80-4b1b1a2cd6ca: removing all old logs from log store -2025-01-25T14:29:36.853Z [INFO] server-b4acdb5e-008f-1684-0d80-4b1b1a2cd6ca: compacting logs: from=1 to=22 -2025-01-25T14:29:36.853Z [INFO] server-b4acdb5e-008f-1684-0d80-4b1b1a2cd6ca: Installed remote snapshot -2025-01-25T14:29:36.853Z [WARN] server-b4acdb5e-008f-1684-0d80-4b1b1a2cd6ca: skipping application of old log: index=22 -2025-01-25T14:29:36.853Z [INFO] server-4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b: pipelining replication: peer="{Voter server-b4acdb5e-008f-1684-0d80-4b1b1a2cd6ca b4acdb5e-008f-1684-0d80-4b1b1a2cd6ca}" -2025-01-25T14:29:36.854Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:29:36.855Z [ERROR] server-4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b: failed to get log: index=22 error="log not found" -2025-01-25T14:29:36.856Z [INFO] creating new snapshot: path=/tmp/raft1530357578/snapshots/2-10013-1737815376856.tmp -2025-01-25T14:29:36.856Z [INFO] server-3a7a655e-2090-5c6c-0f06-ecd30723cde6: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:36.857Z [INFO] server-3a7a655e-2090-5c6c-0f06-ecd30723cde6: copied to local snapshot: bytes=71 -2025-01-25T14:29:36.857Z [INFO] server-3a7a655e-2090-5c6c-0f06-ecd30723cde6: snapshot restore progress: id=2-10013-1737815376856 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:36.857Z [INFO] server-3a7a655e-2090-5c6c-0f06-ecd30723cde6: removing all old logs from log store -2025-01-25T14:29:36.857Z [INFO] server-3a7a655e-2090-5c6c-0f06-ecd30723cde6: compacting logs: from=1 to=22 -2025-01-25T14:29:36.857Z [INFO] server-3a7a655e-2090-5c6c-0f06-ecd30723cde6: Installed remote snapshot -2025-01-25T14:29:36.858Z [INFO] server-4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b: pipelining replication: peer="{Voter server-3a7a655e-2090-5c6c-0f06-ecd30723cde6 3a7a655e-2090-5c6c-0f06-ecd30723cde6}" -2025-01-25T14:29:37.859Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:29:36.854139825 +0000 UTC m=+86.708464807 (2 nodes), 1.56µs from start of poll, 1.605969206s from cluster start. Timeout at 2025-01-25 14:29:37.85930522 +0000 UTC m=+87.713630202, 1.005165395s after stability -2025-01-25T14:29:37.872Z [INFO] server-4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b: aborting pipeline replication: peer="{Voter server-b4acdb5e-008f-1684-0d80-4b1b1a2cd6ca b4acdb5e-008f-1684-0d80-4b1b1a2cd6ca}" -2025-01-25T14:29:37.872Z [INFO] server-4d9d5fe9-fb2a-23bd-c840-122e74f9cb9b: aborting pipeline replication: peer="{Voter server-3a7a655e-2090-5c6c-0f06-ecd30723cde6 3a7a655e-2090-5c6c-0f06-ecd30723cde6}" +2025-01-25T14:40:21.559Z [INFO] server-9495fc5c-374c-b4b3-85c5-434c5eb407c3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9495fc5c-374c-b4b3-85c5-434c5eb407c3 Address:9495fc5c-374c-b4b3-85c5-434c5eb407c3} {Suffrage:Voter ID:server-23bce12e-447f-e766-6947-a2721aba9179 Address:23bce12e-447f-e766-6947-a2721aba9179} {Suffrage:Voter ID:server-e8b1e53a-a91d-72f4-599e-68490d0e9421 Address:e8b1e53a-a91d-72f4-599e-68490d0e9421}]" +2025-01-25T14:40:21.560Z [INFO] server-9495fc5c-374c-b4b3-85c5-434c5eb407c3: entering follower state: follower="Node at 9495fc5c-374c-b4b3-85c5-434c5eb407c3 [Follower]" leader-address= leader-id= +2025-01-25T14:40:21.560Z [INFO] server-23bce12e-447f-e766-6947-a2721aba9179: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9495fc5c-374c-b4b3-85c5-434c5eb407c3 Address:9495fc5c-374c-b4b3-85c5-434c5eb407c3} {Suffrage:Voter ID:server-23bce12e-447f-e766-6947-a2721aba9179 Address:23bce12e-447f-e766-6947-a2721aba9179} {Suffrage:Voter ID:server-e8b1e53a-a91d-72f4-599e-68490d0e9421 Address:e8b1e53a-a91d-72f4-599e-68490d0e9421}]" +2025-01-25T14:40:21.560Z [INFO] server-23bce12e-447f-e766-6947-a2721aba9179: entering follower state: follower="Node at 23bce12e-447f-e766-6947-a2721aba9179 [Follower]" leader-address= leader-id= +2025-01-25T14:40:21.560Z [INFO] server-e8b1e53a-a91d-72f4-599e-68490d0e9421: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9495fc5c-374c-b4b3-85c5-434c5eb407c3 Address:9495fc5c-374c-b4b3-85c5-434c5eb407c3} {Suffrage:Voter ID:server-23bce12e-447f-e766-6947-a2721aba9179 Address:23bce12e-447f-e766-6947-a2721aba9179} {Suffrage:Voter ID:server-e8b1e53a-a91d-72f4-599e-68490d0e9421 Address:e8b1e53a-a91d-72f4-599e-68490d0e9421}]" +2025-01-25T14:40:21.561Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:21.561Z [INFO] server-e8b1e53a-a91d-72f4-599e-68490d0e9421: entering follower state: follower="Node at e8b1e53a-a91d-72f4-599e-68490d0e9421 [Follower]" leader-address= leader-id= +2025-01-25T14:40:22.393Z [WARN] server-e8b1e53a-a91d-72f4-599e-68490d0e9421: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:22.394Z [INFO] server-e8b1e53a-a91d-72f4-599e-68490d0e9421: entering candidate state: node="Node at e8b1e53a-a91d-72f4-599e-68490d0e9421 [Candidate]" term=2 +2025-01-25T14:40:22.394Z [INFO] server-e8b1e53a-a91d-72f4-599e-68490d0e9421: election won: term=2 tally=2 +2025-01-25T14:40:22.394Z [INFO] server-e8b1e53a-a91d-72f4-599e-68490d0e9421: entering leader state: leader="Node at e8b1e53a-a91d-72f4-599e-68490d0e9421 [Leader]" +2025-01-25T14:40:22.394Z [INFO] server-e8b1e53a-a91d-72f4-599e-68490d0e9421: added peer, starting replication: peer=server-9495fc5c-374c-b4b3-85c5-434c5eb407c3 +2025-01-25T14:40:22.394Z [INFO] server-e8b1e53a-a91d-72f4-599e-68490d0e9421: added peer, starting replication: peer=server-23bce12e-447f-e766-6947-a2721aba9179 +2025-01-25T14:40:22.395Z [INFO] server-e8b1e53a-a91d-72f4-599e-68490d0e9421: pipelining replication: peer="{Voter server-9495fc5c-374c-b4b3-85c5-434c5eb407c3 9495fc5c-374c-b4b3-85c5-434c5eb407c3}" +2025-01-25T14:40:22.395Z [INFO] server-e8b1e53a-a91d-72f4-599e-68490d0e9421: pipelining replication: peer="{Voter server-23bce12e-447f-e766-6947-a2721aba9179 23bce12e-447f-e766-6947-a2721aba9179}" +2025-01-25T14:40:23.399Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:22.394741989 +0000 UTC m=+91.440581307 (1 nodes), 833.603302ms from start of poll, 835.551058ms from cluster start. Timeout at 2025-01-25 14:40:23.399751614 +0000 UTC m=+92.445590932, 1.005009625s after stability +2025-01-25T14:40:23.412Z [INFO] server-e8b1e53a-a91d-72f4-599e-68490d0e9421: starting snapshot up to: index=12 +2025-01-25T14:40:23.413Z [INFO] creating new snapshot: path=/tmp/raft332669086/snapshots/2-12-1737816023413.tmp +2025-01-25T14:40:23.418Z [INFO] server-e8b1e53a-a91d-72f4-599e-68490d0e9421: snapshot complete up to: index=12 +2025-01-25T14:40:23.423Z [INFO] creating new snapshot: path=/tmp/raft332669086/snapshots/2-10013-1737816023423.tmp +2025-01-25T14:40:23.426Z [INFO] server-e8b1e53a-a91d-72f4-599e-68490d0e9421: copied to local snapshot: bytes=71 +2025-01-25T14:40:23.428Z [INFO] server-e8b1e53a-a91d-72f4-599e-68490d0e9421: snapshot restore progress: id=2-10013-1737816023423 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:23.428Z [INFO] server-e8b1e53a-a91d-72f4-599e-68490d0e9421: removing all old logs from log store +2025-01-25T14:40:23.428Z [INFO] server-e8b1e53a-a91d-72f4-599e-68490d0e9421: compacting logs: from=1 to=22 +2025-01-25T14:40:23.428Z [INFO] server-e8b1e53a-a91d-72f4-599e-68490d0e9421: restored user snapshot: index=10013 +2025-01-25T14:40:23.429Z [ERROR] server-e8b1e53a-a91d-72f4-599e-68490d0e9421: failed to get log: index=22 error="log not found" +2025-01-25T14:40:23.429Z [ERROR] server-e8b1e53a-a91d-72f4-599e-68490d0e9421: failed to get log: index=22 error="log not found" +2025-01-25T14:40:23.429Z [INFO] server-e8b1e53a-a91d-72f4-599e-68490d0e9421: aborting pipeline replication: peer="{Voter server-9495fc5c-374c-b4b3-85c5-434c5eb407c3 9495fc5c-374c-b4b3-85c5-434c5eb407c3}" +2025-01-25T14:40:23.429Z [INFO] server-e8b1e53a-a91d-72f4-599e-68490d0e9421: aborting pipeline replication: peer="{Voter server-23bce12e-447f-e766-6947-a2721aba9179 23bce12e-447f-e766-6947-a2721aba9179}" +2025-01-25T14:40:23.435Z [ERROR] server-e8b1e53a-a91d-72f4-599e-68490d0e9421: failed to get log: index=22 error="log not found" +2025-01-25T14:40:23.437Z [ERROR] server-e8b1e53a-a91d-72f4-599e-68490d0e9421: failed to get log: index=22 error="log not found" +2025-01-25T14:40:23.438Z [INFO] creating new snapshot: path=/tmp/raft1181006379/snapshots/2-10013-1737816023438.tmp +2025-01-25T14:40:23.439Z [INFO] creating new snapshot: path=/tmp/raft784497317/snapshots/2-10013-1737816023439.tmp +2025-01-25T14:40:23.440Z [INFO] server-23bce12e-447f-e766-6947-a2721aba9179: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:23.442Z [INFO] server-9495fc5c-374c-b4b3-85c5-434c5eb407c3: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:23.442Z [INFO] server-23bce12e-447f-e766-6947-a2721aba9179: copied to local snapshot: bytes=71 +2025-01-25T14:40:23.444Z [INFO] server-23bce12e-447f-e766-6947-a2721aba9179: snapshot restore progress: id=2-10013-1737816023438 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:23.444Z [INFO] server-23bce12e-447f-e766-6947-a2721aba9179: removing all old logs from log store +2025-01-25T14:40:23.444Z [INFO] server-23bce12e-447f-e766-6947-a2721aba9179: compacting logs: from=1 to=22 +2025-01-25T14:40:23.444Z [INFO] server-23bce12e-447f-e766-6947-a2721aba9179: Installed remote snapshot +2025-01-25T14:40:23.445Z [WARN] server-23bce12e-447f-e766-6947-a2721aba9179: skipping application of old log: index=10013 +2025-01-25T14:40:23.445Z [INFO] server-e8b1e53a-a91d-72f4-599e-68490d0e9421: pipelining replication: peer="{Voter server-23bce12e-447f-e766-6947-a2721aba9179 23bce12e-447f-e766-6947-a2721aba9179}" +2025-01-25T14:40:23.445Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:40:23.446Z [INFO] server-9495fc5c-374c-b4b3-85c5-434c5eb407c3: copied to local snapshot: bytes=71 +2025-01-25T14:40:23.447Z [INFO] server-9495fc5c-374c-b4b3-85c5-434c5eb407c3: snapshot restore progress: id=2-10013-1737816023439 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:23.447Z [INFO] server-9495fc5c-374c-b4b3-85c5-434c5eb407c3: removing all old logs from log store +2025-01-25T14:40:23.447Z [INFO] server-9495fc5c-374c-b4b3-85c5-434c5eb407c3: compacting logs: from=1 to=22 +2025-01-25T14:40:23.448Z [INFO] server-9495fc5c-374c-b4b3-85c5-434c5eb407c3: Installed remote snapshot +2025-01-25T14:40:23.448Z [INFO] server-e8b1e53a-a91d-72f4-599e-68490d0e9421: pipelining replication: peer="{Voter server-9495fc5c-374c-b4b3-85c5-434c5eb407c3 9495fc5c-374c-b4b3-85c5-434c5eb407c3}" +2025-01-25T14:40:24.451Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:40:23.445930383 +0000 UTC m=+92.491769701 (2 nodes), 6.126µs from start of poll, 1.886739452s from cluster start. Timeout at 2025-01-25 14:40:24.451114717 +0000 UTC m=+93.496954035, 1.005184334s after stability +2025-01-25T14:40:24.465Z [INFO] server-e8b1e53a-a91d-72f4-599e-68490d0e9421: aborting pipeline replication: peer="{Voter server-23bce12e-447f-e766-6947-a2721aba9179 23bce12e-447f-e766-6947-a2721aba9179}" +2025-01-25T14:40:24.465Z [INFO] server-e8b1e53a-a91d-72f4-599e-68490d0e9421: aborting pipeline replication: peer="{Voter server-9495fc5c-374c-b4b3-85c5-434c5eb407c3 9495fc5c-374c-b4b3-85c5-434c5eb407c3}" === RUN TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_true -2025-01-25T14:29:37.876Z [INFO] server-1ec4db90-9040-e72f-d2cd-01b640a11559: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1ec4db90-9040-e72f-d2cd-01b640a11559 Address:1ec4db90-9040-e72f-d2cd-01b640a11559} {Suffrage:Voter ID:server-5544a573-752b-b9a4-f76d-b65b3d6d03b7 Address:5544a573-752b-b9a4-f76d-b65b3d6d03b7} {Suffrage:Voter ID:server-cb831d4f-3295-6d3c-d0dd-bbcc930dae56 Address:cb831d4f-3295-6d3c-d0dd-bbcc930dae56}]" -2025-01-25T14:29:37.876Z [INFO] server-1ec4db90-9040-e72f-d2cd-01b640a11559: entering follower state: follower="Node at 1ec4db90-9040-e72f-d2cd-01b640a11559 [Follower]" leader-address= leader-id= -2025-01-25T14:29:37.876Z [INFO] server-5544a573-752b-b9a4-f76d-b65b3d6d03b7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1ec4db90-9040-e72f-d2cd-01b640a11559 Address:1ec4db90-9040-e72f-d2cd-01b640a11559} {Suffrage:Voter ID:server-5544a573-752b-b9a4-f76d-b65b3d6d03b7 Address:5544a573-752b-b9a4-f76d-b65b3d6d03b7} {Suffrage:Voter ID:server-cb831d4f-3295-6d3c-d0dd-bbcc930dae56 Address:cb831d4f-3295-6d3c-d0dd-bbcc930dae56}]" -2025-01-25T14:29:37.876Z [INFO] server-5544a573-752b-b9a4-f76d-b65b3d6d03b7: entering follower state: follower="Node at 5544a573-752b-b9a4-f76d-b65b3d6d03b7 [Follower]" leader-address= leader-id= -2025-01-25T14:29:37.876Z [INFO] server-cb831d4f-3295-6d3c-d0dd-bbcc930dae56: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1ec4db90-9040-e72f-d2cd-01b640a11559 Address:1ec4db90-9040-e72f-d2cd-01b640a11559} {Suffrage:Voter ID:server-5544a573-752b-b9a4-f76d-b65b3d6d03b7 Address:5544a573-752b-b9a4-f76d-b65b3d6d03b7} {Suffrage:Voter ID:server-cb831d4f-3295-6d3c-d0dd-bbcc930dae56 Address:cb831d4f-3295-6d3c-d0dd-bbcc930dae56}]" -2025-01-25T14:29:37.876Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:37.876Z [INFO] server-cb831d4f-3295-6d3c-d0dd-bbcc930dae56: entering follower state: follower="Node at cb831d4f-3295-6d3c-d0dd-bbcc930dae56 [Follower]" leader-address= leader-id= -2025-01-25T14:29:38.426Z [WARN] server-cb831d4f-3295-6d3c-d0dd-bbcc930dae56: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:38.426Z [INFO] server-cb831d4f-3295-6d3c-d0dd-bbcc930dae56: entering candidate state: node="Node at cb831d4f-3295-6d3c-d0dd-bbcc930dae56 [Candidate]" term=2 -2025-01-25T14:29:38.426Z [INFO] server-cb831d4f-3295-6d3c-d0dd-bbcc930dae56: election won: term=2 tally=2 -2025-01-25T14:29:38.426Z [INFO] server-cb831d4f-3295-6d3c-d0dd-bbcc930dae56: entering leader state: leader="Node at cb831d4f-3295-6d3c-d0dd-bbcc930dae56 [Leader]" -2025-01-25T14:29:38.426Z [INFO] server-cb831d4f-3295-6d3c-d0dd-bbcc930dae56: added peer, starting replication: peer=server-1ec4db90-9040-e72f-d2cd-01b640a11559 -2025-01-25T14:29:38.426Z [INFO] server-cb831d4f-3295-6d3c-d0dd-bbcc930dae56: added peer, starting replication: peer=server-5544a573-752b-b9a4-f76d-b65b3d6d03b7 -2025-01-25T14:29:38.426Z [INFO] server-cb831d4f-3295-6d3c-d0dd-bbcc930dae56: pipelining replication: peer="{Voter server-5544a573-752b-b9a4-f76d-b65b3d6d03b7 5544a573-752b-b9a4-f76d-b65b3d6d03b7}" -2025-01-25T14:29:38.427Z [INFO] server-cb831d4f-3295-6d3c-d0dd-bbcc930dae56: pipelining replication: peer="{Voter server-1ec4db90-9040-e72f-d2cd-01b640a11559 1ec4db90-9040-e72f-d2cd-01b640a11559}" -2025-01-25T14:29:39.432Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:38.426879507 +0000 UTC m=+88.281204489 (1 nodes), 550.399033ms from start of poll, 551.098327ms from cluster start. Timeout at 2025-01-25 14:29:39.431885187 +0000 UTC m=+89.286210129, 1.00500564s after stability -2025-01-25T14:29:39.433Z [INFO] server-cb831d4f-3295-6d3c-d0dd-bbcc930dae56: starting snapshot up to: index=12 -2025-01-25T14:29:39.433Z [INFO] creating new snapshot: path=/tmp/raft3090776554/snapshots/2-12-1737815379433.tmp -2025-01-25T14:29:39.434Z [INFO] server-cb831d4f-3295-6d3c-d0dd-bbcc930dae56: snapshot complete up to: index=12 -2025-01-25T14:29:39.437Z [INFO] server-66b15622-b3e2-29c3-9e5e-3ad3d8464cdc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-66b15622-b3e2-29c3-9e5e-3ad3d8464cdc Address:66b15622-b3e2-29c3-9e5e-3ad3d8464cdc} {Suffrage:Voter ID:server-0e9b5732-88e1-d4bd-f2e7-e25aa13be55f Address:0e9b5732-88e1-d4bd-f2e7-e25aa13be55f} {Suffrage:Voter ID:server-9288e411-bf17-9328-4b25-c3fa1906c367 Address:9288e411-bf17-9328-4b25-c3fa1906c367}]" -2025-01-25T14:29:39.437Z [INFO] server-0e9b5732-88e1-d4bd-f2e7-e25aa13be55f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-66b15622-b3e2-29c3-9e5e-3ad3d8464cdc Address:66b15622-b3e2-29c3-9e5e-3ad3d8464cdc} {Suffrage:Voter ID:server-0e9b5732-88e1-d4bd-f2e7-e25aa13be55f Address:0e9b5732-88e1-d4bd-f2e7-e25aa13be55f} {Suffrage:Voter ID:server-9288e411-bf17-9328-4b25-c3fa1906c367 Address:9288e411-bf17-9328-4b25-c3fa1906c367}]" -2025-01-25T14:29:39.437Z [INFO] server-66b15622-b3e2-29c3-9e5e-3ad3d8464cdc: entering follower state: follower="Node at 66b15622-b3e2-29c3-9e5e-3ad3d8464cdc [Follower]" leader-address= leader-id= -2025-01-25T14:29:39.437Z [INFO] server-0e9b5732-88e1-d4bd-f2e7-e25aa13be55f: entering follower state: follower="Node at 0e9b5732-88e1-d4bd-f2e7-e25aa13be55f [Follower]" leader-address= leader-id= -2025-01-25T14:29:39.437Z [INFO] server-9288e411-bf17-9328-4b25-c3fa1906c367: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-66b15622-b3e2-29c3-9e5e-3ad3d8464cdc Address:66b15622-b3e2-29c3-9e5e-3ad3d8464cdc} {Suffrage:Voter ID:server-0e9b5732-88e1-d4bd-f2e7-e25aa13be55f Address:0e9b5732-88e1-d4bd-f2e7-e25aa13be55f} {Suffrage:Voter ID:server-9288e411-bf17-9328-4b25-c3fa1906c367 Address:9288e411-bf17-9328-4b25-c3fa1906c367}]" -2025-01-25T14:29:39.437Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:39.437Z [INFO] server-9288e411-bf17-9328-4b25-c3fa1906c367: entering follower state: follower="Node at 9288e411-bf17-9328-4b25-c3fa1906c367 [Follower]" leader-address= leader-id= -2025-01-25T14:29:40.177Z [WARN] server-0e9b5732-88e1-d4bd-f2e7-e25aa13be55f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:40.177Z [INFO] server-0e9b5732-88e1-d4bd-f2e7-e25aa13be55f: entering candidate state: node="Node at 0e9b5732-88e1-d4bd-f2e7-e25aa13be55f [Candidate]" term=2 -2025-01-25T14:29:40.177Z [INFO] server-0e9b5732-88e1-d4bd-f2e7-e25aa13be55f: election won: term=2 tally=2 -2025-01-25T14:29:40.177Z [INFO] server-0e9b5732-88e1-d4bd-f2e7-e25aa13be55f: entering leader state: leader="Node at 0e9b5732-88e1-d4bd-f2e7-e25aa13be55f [Leader]" -2025-01-25T14:29:40.177Z [INFO] server-0e9b5732-88e1-d4bd-f2e7-e25aa13be55f: added peer, starting replication: peer=server-66b15622-b3e2-29c3-9e5e-3ad3d8464cdc -2025-01-25T14:29:40.177Z [INFO] server-0e9b5732-88e1-d4bd-f2e7-e25aa13be55f: added peer, starting replication: peer=server-9288e411-bf17-9328-4b25-c3fa1906c367 -2025-01-25T14:29:40.177Z [INFO] server-0e9b5732-88e1-d4bd-f2e7-e25aa13be55f: pipelining replication: peer="{Voter server-66b15622-b3e2-29c3-9e5e-3ad3d8464cdc 66b15622-b3e2-29c3-9e5e-3ad3d8464cdc}" -2025-01-25T14:29:40.177Z [INFO] server-0e9b5732-88e1-d4bd-f2e7-e25aa13be55f: pipelining replication: peer="{Voter server-9288e411-bf17-9328-4b25-c3fa1906c367 9288e411-bf17-9328-4b25-c3fa1906c367}" -2025-01-25T14:29:41.182Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:40.177590345 +0000 UTC m=+90.031915287 (1 nodes), 739.775058ms from start of poll, 740.487553ms from cluster start. Timeout at 2025-01-25 14:29:41.182601745 +0000 UTC m=+91.036926687, 1.0050114s after stability -2025-01-25T14:29:41.183Z [INFO] creating new snapshot: path=/tmp/raft1459660841/snapshots/2-10013-1737815381183.tmp -2025-01-25T14:29:41.184Z [INFO] server-0e9b5732-88e1-d4bd-f2e7-e25aa13be55f: copied to local snapshot: bytes=71 -2025-01-25T14:29:41.184Z [INFO] server-0e9b5732-88e1-d4bd-f2e7-e25aa13be55f: snapshot restore progress: id=2-10013-1737815381183 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:41.184Z [INFO] server-0e9b5732-88e1-d4bd-f2e7-e25aa13be55f: restored user snapshot: index=10013 -2025-01-25T14:29:41.185Z [ERROR] server-0e9b5732-88e1-d4bd-f2e7-e25aa13be55f: failed to get log: index=3 error="log not found" -2025-01-25T14:29:41.185Z [INFO] server-0e9b5732-88e1-d4bd-f2e7-e25aa13be55f: aborting pipeline replication: peer="{Voter server-9288e411-bf17-9328-4b25-c3fa1906c367 9288e411-bf17-9328-4b25-c3fa1906c367}" -2025-01-25T14:29:41.185Z [ERROR] server-0e9b5732-88e1-d4bd-f2e7-e25aa13be55f: failed to get log: index=3 error="log not found" -2025-01-25T14:29:41.185Z [INFO] server-0e9b5732-88e1-d4bd-f2e7-e25aa13be55f: aborting pipeline replication: peer="{Voter server-66b15622-b3e2-29c3-9e5e-3ad3d8464cdc 66b15622-b3e2-29c3-9e5e-3ad3d8464cdc}" -2025-01-25T14:29:41.190Z [ERROR] server-0e9b5732-88e1-d4bd-f2e7-e25aa13be55f: failed to get log: index=3 error="log not found" -2025-01-25T14:29:41.191Z [INFO] creating new snapshot: path=/tmp/raft1722530050/snapshots/2-10013-1737815381191.tmp -2025-01-25T14:29:41.191Z [ERROR] server-0e9b5732-88e1-d4bd-f2e7-e25aa13be55f: failed to get log: index=3 error="log not found" -2025-01-25T14:29:41.191Z [INFO] server-9288e411-bf17-9328-4b25-c3fa1906c367: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:41.192Z [INFO] server-9288e411-bf17-9328-4b25-c3fa1906c367: copied to local snapshot: bytes=71 -2025-01-25T14:29:41.192Z [INFO] creating new snapshot: path=/tmp/raft14806582/snapshots/2-10013-1737815381192.tmp -2025-01-25T14:29:41.192Z [INFO] server-9288e411-bf17-9328-4b25-c3fa1906c367: snapshot restore progress: id=2-10013-1737815381191 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:41.192Z [INFO] server-9288e411-bf17-9328-4b25-c3fa1906c367: Installed remote snapshot -2025-01-25T14:29:41.192Z [INFO] server-66b15622-b3e2-29c3-9e5e-3ad3d8464cdc: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:41.193Z [INFO] server-0e9b5732-88e1-d4bd-f2e7-e25aa13be55f: pipelining replication: peer="{Voter server-9288e411-bf17-9328-4b25-c3fa1906c367 9288e411-bf17-9328-4b25-c3fa1906c367}" -2025-01-25T14:29:41.193Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:29:41.193Z [INFO] server-66b15622-b3e2-29c3-9e5e-3ad3d8464cdc: copied to local snapshot: bytes=71 -2025-01-25T14:29:41.194Z [INFO] server-66b15622-b3e2-29c3-9e5e-3ad3d8464cdc: snapshot restore progress: id=2-10013-1737815381192 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:41.194Z [INFO] server-66b15622-b3e2-29c3-9e5e-3ad3d8464cdc: Installed remote snapshot -2025-01-25T14:29:41.194Z [INFO] server-0e9b5732-88e1-d4bd-f2e7-e25aa13be55f: pipelining replication: peer="{Voter server-66b15622-b3e2-29c3-9e5e-3ad3d8464cdc 66b15622-b3e2-29c3-9e5e-3ad3d8464cdc}" -2025-01-25T14:29:42.199Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:29:41.193207756 +0000 UTC m=+91.047532738 (2 nodes), 1.56µs from start of poll, 1.756105004s from cluster start. Timeout at 2025-01-25 14:29:42.198430988 +0000 UTC m=+92.052755970, 1.005223232s after stability -2025-01-25T14:29:42.211Z [INFO] server-cb831d4f-3295-6d3c-d0dd-bbcc930dae56: aborting pipeline replication: peer="{Voter server-1ec4db90-9040-e72f-d2cd-01b640a11559 1ec4db90-9040-e72f-d2cd-01b640a11559}" -2025-01-25T14:29:42.211Z [INFO] server-cb831d4f-3295-6d3c-d0dd-bbcc930dae56: aborting pipeline replication: peer="{Voter server-5544a573-752b-b9a4-f76d-b65b3d6d03b7 5544a573-752b-b9a4-f76d-b65b3d6d03b7}" +2025-01-25T14:40:24.477Z [INFO] server-9fe82a04-3e06-3a85-c835-999c789789a6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9fe82a04-3e06-3a85-c835-999c789789a6 Address:9fe82a04-3e06-3a85-c835-999c789789a6} {Suffrage:Voter ID:server-f95a7e81-b38b-c5b7-69c2-246d072827c5 Address:f95a7e81-b38b-c5b7-69c2-246d072827c5} {Suffrage:Voter ID:server-4abd66d9-e960-5896-a29e-4de47be8ce1a Address:4abd66d9-e960-5896-a29e-4de47be8ce1a}]" +2025-01-25T14:40:24.478Z [INFO] server-f95a7e81-b38b-c5b7-69c2-246d072827c5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9fe82a04-3e06-3a85-c835-999c789789a6 Address:9fe82a04-3e06-3a85-c835-999c789789a6} {Suffrage:Voter ID:server-f95a7e81-b38b-c5b7-69c2-246d072827c5 Address:f95a7e81-b38b-c5b7-69c2-246d072827c5} {Suffrage:Voter ID:server-4abd66d9-e960-5896-a29e-4de47be8ce1a Address:4abd66d9-e960-5896-a29e-4de47be8ce1a}]" +2025-01-25T14:40:24.477Z [INFO] server-9fe82a04-3e06-3a85-c835-999c789789a6: entering follower state: follower="Node at 9fe82a04-3e06-3a85-c835-999c789789a6 [Follower]" leader-address= leader-id= +2025-01-25T14:40:24.478Z [INFO] server-f95a7e81-b38b-c5b7-69c2-246d072827c5: entering follower state: follower="Node at f95a7e81-b38b-c5b7-69c2-246d072827c5 [Follower]" leader-address= leader-id= +2025-01-25T14:40:24.478Z [INFO] server-4abd66d9-e960-5896-a29e-4de47be8ce1a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9fe82a04-3e06-3a85-c835-999c789789a6 Address:9fe82a04-3e06-3a85-c835-999c789789a6} {Suffrage:Voter ID:server-f95a7e81-b38b-c5b7-69c2-246d072827c5 Address:f95a7e81-b38b-c5b7-69c2-246d072827c5} {Suffrage:Voter ID:server-4abd66d9-e960-5896-a29e-4de47be8ce1a Address:4abd66d9-e960-5896-a29e-4de47be8ce1a}]" +2025-01-25T14:40:24.478Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:24.478Z [INFO] server-4abd66d9-e960-5896-a29e-4de47be8ce1a: entering follower state: follower="Node at 4abd66d9-e960-5896-a29e-4de47be8ce1a [Follower]" leader-address= leader-id= +2025-01-25T14:40:25.029Z [WARN] server-9fe82a04-3e06-3a85-c835-999c789789a6: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:25.029Z [INFO] server-9fe82a04-3e06-3a85-c835-999c789789a6: entering candidate state: node="Node at 9fe82a04-3e06-3a85-c835-999c789789a6 [Candidate]" term=2 +2025-01-25T14:40:25.029Z [INFO] server-9fe82a04-3e06-3a85-c835-999c789789a6: election won: term=2 tally=2 +2025-01-25T14:40:25.030Z [INFO] server-9fe82a04-3e06-3a85-c835-999c789789a6: entering leader state: leader="Node at 9fe82a04-3e06-3a85-c835-999c789789a6 [Leader]" +2025-01-25T14:40:25.030Z [INFO] server-9fe82a04-3e06-3a85-c835-999c789789a6: added peer, starting replication: peer=server-f95a7e81-b38b-c5b7-69c2-246d072827c5 +2025-01-25T14:40:25.030Z [INFO] server-9fe82a04-3e06-3a85-c835-999c789789a6: added peer, starting replication: peer=server-4abd66d9-e960-5896-a29e-4de47be8ce1a +2025-01-25T14:40:25.030Z [INFO] server-9fe82a04-3e06-3a85-c835-999c789789a6: pipelining replication: peer="{Voter server-f95a7e81-b38b-c5b7-69c2-246d072827c5 f95a7e81-b38b-c5b7-69c2-246d072827c5}" +2025-01-25T14:40:25.031Z [INFO] server-9fe82a04-3e06-3a85-c835-999c789789a6: pipelining replication: peer="{Voter server-4abd66d9-e960-5896-a29e-4de47be8ce1a 4abd66d9-e960-5896-a29e-4de47be8ce1a}" +2025-01-25T14:40:26.035Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:25.030147328 +0000 UTC m=+94.075986646 (1 nodes), 551.344772ms from start of poll, 553.693945ms from cluster start. Timeout at 2025-01-25 14:40:26.035269829 +0000 UTC m=+95.081109146, 1.0051225s after stability +2025-01-25T14:40:26.039Z [INFO] server-9fe82a04-3e06-3a85-c835-999c789789a6: starting snapshot up to: index=12 +2025-01-25T14:40:26.039Z [INFO] creating new snapshot: path=/tmp/raft3156912799/snapshots/2-12-1737816026039.tmp +2025-01-25T14:40:26.042Z [INFO] server-9fe82a04-3e06-3a85-c835-999c789789a6: snapshot complete up to: index=12 +2025-01-25T14:40:26.049Z [INFO] server-799772df-b3e1-e03e-058e-50f0f39c0ac1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-799772df-b3e1-e03e-058e-50f0f39c0ac1 Address:799772df-b3e1-e03e-058e-50f0f39c0ac1} {Suffrage:Voter ID:server-f8fd2b7d-3a6b-cc52-976a-eb4a774bc3af Address:f8fd2b7d-3a6b-cc52-976a-eb4a774bc3af} {Suffrage:Voter ID:server-9cc5d13f-8165-bab4-8ac0-c9f24706a5c3 Address:9cc5d13f-8165-bab4-8ac0-c9f24706a5c3}]" +2025-01-25T14:40:26.050Z [INFO] server-799772df-b3e1-e03e-058e-50f0f39c0ac1: entering follower state: follower="Node at 799772df-b3e1-e03e-058e-50f0f39c0ac1 [Follower]" leader-address= leader-id= +2025-01-25T14:40:26.050Z [INFO] server-f8fd2b7d-3a6b-cc52-976a-eb4a774bc3af: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-799772df-b3e1-e03e-058e-50f0f39c0ac1 Address:799772df-b3e1-e03e-058e-50f0f39c0ac1} {Suffrage:Voter ID:server-f8fd2b7d-3a6b-cc52-976a-eb4a774bc3af Address:f8fd2b7d-3a6b-cc52-976a-eb4a774bc3af} {Suffrage:Voter ID:server-9cc5d13f-8165-bab4-8ac0-c9f24706a5c3 Address:9cc5d13f-8165-bab4-8ac0-c9f24706a5c3}]" +2025-01-25T14:40:26.050Z [INFO] server-f8fd2b7d-3a6b-cc52-976a-eb4a774bc3af: entering follower state: follower="Node at f8fd2b7d-3a6b-cc52-976a-eb4a774bc3af [Follower]" leader-address= leader-id= +2025-01-25T14:40:26.051Z [INFO] server-9cc5d13f-8165-bab4-8ac0-c9f24706a5c3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-799772df-b3e1-e03e-058e-50f0f39c0ac1 Address:799772df-b3e1-e03e-058e-50f0f39c0ac1} {Suffrage:Voter ID:server-f8fd2b7d-3a6b-cc52-976a-eb4a774bc3af Address:f8fd2b7d-3a6b-cc52-976a-eb4a774bc3af} {Suffrage:Voter ID:server-9cc5d13f-8165-bab4-8ac0-c9f24706a5c3 Address:9cc5d13f-8165-bab4-8ac0-c9f24706a5c3}]" +2025-01-25T14:40:26.051Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:26.051Z [INFO] server-9cc5d13f-8165-bab4-8ac0-c9f24706a5c3: entering follower state: follower="Node at 9cc5d13f-8165-bab4-8ac0-c9f24706a5c3 [Follower]" leader-address= leader-id= +2025-01-25T14:40:26.677Z [WARN] server-9cc5d13f-8165-bab4-8ac0-c9f24706a5c3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:26.677Z [INFO] server-9cc5d13f-8165-bab4-8ac0-c9f24706a5c3: entering candidate state: node="Node at 9cc5d13f-8165-bab4-8ac0-c9f24706a5c3 [Candidate]" term=2 +2025-01-25T14:40:26.677Z [INFO] server-9cc5d13f-8165-bab4-8ac0-c9f24706a5c3: election won: term=2 tally=2 +2025-01-25T14:40:26.678Z [INFO] server-9cc5d13f-8165-bab4-8ac0-c9f24706a5c3: entering leader state: leader="Node at 9cc5d13f-8165-bab4-8ac0-c9f24706a5c3 [Leader]" +2025-01-25T14:40:26.678Z [INFO] server-9cc5d13f-8165-bab4-8ac0-c9f24706a5c3: added peer, starting replication: peer=server-799772df-b3e1-e03e-058e-50f0f39c0ac1 +2025-01-25T14:40:26.678Z [INFO] server-9cc5d13f-8165-bab4-8ac0-c9f24706a5c3: added peer, starting replication: peer=server-f8fd2b7d-3a6b-cc52-976a-eb4a774bc3af +2025-01-25T14:40:26.678Z [INFO] server-9cc5d13f-8165-bab4-8ac0-c9f24706a5c3: pipelining replication: peer="{Voter server-799772df-b3e1-e03e-058e-50f0f39c0ac1 799772df-b3e1-e03e-058e-50f0f39c0ac1}" +2025-01-25T14:40:26.681Z [INFO] server-9cc5d13f-8165-bab4-8ac0-c9f24706a5c3: pipelining replication: peer="{Voter server-f8fd2b7d-3a6b-cc52-976a-eb4a774bc3af f8fd2b7d-3a6b-cc52-976a-eb4a774bc3af}" +2025-01-25T14:40:27.683Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:26.678312001 +0000 UTC m=+95.724151319 (1 nodes), 626.678949ms from start of poll, 629.267498ms from cluster start. Timeout at 2025-01-25 14:40:27.683330669 +0000 UTC m=+96.729169986, 1.005018667s after stability +2025-01-25T14:40:27.685Z [INFO] creating new snapshot: path=/tmp/raft3100313404/snapshots/2-10013-1737816027685.tmp +2025-01-25T14:40:27.688Z [INFO] server-9cc5d13f-8165-bab4-8ac0-c9f24706a5c3: copied to local snapshot: bytes=71 +2025-01-25T14:40:27.699Z [INFO] server-9cc5d13f-8165-bab4-8ac0-c9f24706a5c3: snapshot restore progress: id=2-10013-1737816027685 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:27.702Z [INFO] server-9cc5d13f-8165-bab4-8ac0-c9f24706a5c3: restored user snapshot: index=10013 +2025-01-25T14:40:27.702Z [ERROR] server-9cc5d13f-8165-bab4-8ac0-c9f24706a5c3: failed to get log: index=3 error="log not found" +2025-01-25T14:40:27.702Z [INFO] server-9cc5d13f-8165-bab4-8ac0-c9f24706a5c3: aborting pipeline replication: peer="{Voter server-f8fd2b7d-3a6b-cc52-976a-eb4a774bc3af f8fd2b7d-3a6b-cc52-976a-eb4a774bc3af}" +2025-01-25T14:40:27.702Z [ERROR] server-9cc5d13f-8165-bab4-8ac0-c9f24706a5c3: failed to get log: index=3 error="log not found" +2025-01-25T14:40:27.703Z [INFO] server-9cc5d13f-8165-bab4-8ac0-c9f24706a5c3: aborting pipeline replication: peer="{Voter server-799772df-b3e1-e03e-058e-50f0f39c0ac1 799772df-b3e1-e03e-058e-50f0f39c0ac1}" +2025-01-25T14:40:27.709Z [ERROR] server-9cc5d13f-8165-bab4-8ac0-c9f24706a5c3: failed to get log: index=3 error="log not found" +2025-01-25T14:40:27.710Z [ERROR] server-9cc5d13f-8165-bab4-8ac0-c9f24706a5c3: failed to get log: index=3 error="log not found" +2025-01-25T14:40:27.711Z [INFO] creating new snapshot: path=/tmp/raft1619378591/snapshots/2-10013-1737816027711.tmp +2025-01-25T14:40:27.711Z [INFO] creating new snapshot: path=/tmp/raft3940019591/snapshots/2-10013-1737816027711.tmp +2025-01-25T14:40:27.712Z [INFO] server-799772df-b3e1-e03e-058e-50f0f39c0ac1: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:27.713Z [INFO] server-f8fd2b7d-3a6b-cc52-976a-eb4a774bc3af: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:27.714Z [INFO] server-799772df-b3e1-e03e-058e-50f0f39c0ac1: copied to local snapshot: bytes=71 +2025-01-25T14:40:27.715Z [INFO] server-799772df-b3e1-e03e-058e-50f0f39c0ac1: snapshot restore progress: id=2-10013-1737816027711 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:27.715Z [INFO] server-799772df-b3e1-e03e-058e-50f0f39c0ac1: Installed remote snapshot +2025-01-25T14:40:27.715Z [INFO] server-f8fd2b7d-3a6b-cc52-976a-eb4a774bc3af: copied to local snapshot: bytes=71 +2025-01-25T14:40:27.715Z [INFO] server-9cc5d13f-8165-bab4-8ac0-c9f24706a5c3: pipelining replication: peer="{Voter server-799772df-b3e1-e03e-058e-50f0f39c0ac1 799772df-b3e1-e03e-058e-50f0f39c0ac1}" +2025-01-25T14:40:27.716Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:40:27.717Z [INFO] server-f8fd2b7d-3a6b-cc52-976a-eb4a774bc3af: snapshot restore progress: id=2-10013-1737816027711 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:27.717Z [INFO] server-f8fd2b7d-3a6b-cc52-976a-eb4a774bc3af: Installed remote snapshot +2025-01-25T14:40:27.718Z [INFO] server-9cc5d13f-8165-bab4-8ac0-c9f24706a5c3: pipelining replication: peer="{Voter server-f8fd2b7d-3a6b-cc52-976a-eb4a774bc3af f8fd2b7d-3a6b-cc52-976a-eb4a774bc3af}" +2025-01-25T14:40:28.721Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:40:27.716078898 +0000 UTC m=+96.761918215 (2 nodes), 4.666µs from start of poll, 1.667034394s from cluster start. Timeout at 2025-01-25 14:40:28.721293565 +0000 UTC m=+97.767132882, 1.005214667s after stability +2025-01-25T14:40:28.749Z [INFO] server-9fe82a04-3e06-3a85-c835-999c789789a6: aborting pipeline replication: peer="{Voter server-4abd66d9-e960-5896-a29e-4de47be8ce1a 4abd66d9-e960-5896-a29e-4de47be8ce1a}" +2025-01-25T14:40:28.749Z [INFO] server-9fe82a04-3e06-3a85-c835-999c789789a6: aborting pipeline replication: peer="{Voter server-f95a7e81-b38b-c5b7-69c2-246d072827c5 f95a7e81-b38b-c5b7-69c2-246d072827c5}" === RUN TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_true -2025-01-25T14:29:42.214Z [INFO] server-2d4bbf4a-b755-51fe-7de1-eb34e5a1c47e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2d4bbf4a-b755-51fe-7de1-eb34e5a1c47e Address:2d4bbf4a-b755-51fe-7de1-eb34e5a1c47e} {Suffrage:Voter ID:server-04ef2b72-148b-89d7-9562-cc9d60eba792 Address:04ef2b72-148b-89d7-9562-cc9d60eba792} {Suffrage:Voter ID:server-777f47a4-665c-dae8-ed72-6302f2e20c9c Address:777f47a4-665c-dae8-ed72-6302f2e20c9c}]" -2025-01-25T14:29:42.214Z [INFO] server-04ef2b72-148b-89d7-9562-cc9d60eba792: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2d4bbf4a-b755-51fe-7de1-eb34e5a1c47e Address:2d4bbf4a-b755-51fe-7de1-eb34e5a1c47e} {Suffrage:Voter ID:server-04ef2b72-148b-89d7-9562-cc9d60eba792 Address:04ef2b72-148b-89d7-9562-cc9d60eba792} {Suffrage:Voter ID:server-777f47a4-665c-dae8-ed72-6302f2e20c9c Address:777f47a4-665c-dae8-ed72-6302f2e20c9c}]" -2025-01-25T14:29:42.214Z [INFO] server-2d4bbf4a-b755-51fe-7de1-eb34e5a1c47e: entering follower state: follower="Node at 2d4bbf4a-b755-51fe-7de1-eb34e5a1c47e [Follower]" leader-address= leader-id= -2025-01-25T14:29:42.214Z [INFO] server-777f47a4-665c-dae8-ed72-6302f2e20c9c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2d4bbf4a-b755-51fe-7de1-eb34e5a1c47e Address:2d4bbf4a-b755-51fe-7de1-eb34e5a1c47e} {Suffrage:Voter ID:server-04ef2b72-148b-89d7-9562-cc9d60eba792 Address:04ef2b72-148b-89d7-9562-cc9d60eba792} {Suffrage:Voter ID:server-777f47a4-665c-dae8-ed72-6302f2e20c9c Address:777f47a4-665c-dae8-ed72-6302f2e20c9c}]" -2025-01-25T14:29:42.214Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:42.214Z [INFO] server-777f47a4-665c-dae8-ed72-6302f2e20c9c: entering follower state: follower="Node at 777f47a4-665c-dae8-ed72-6302f2e20c9c [Follower]" leader-address= leader-id= -2025-01-25T14:29:42.214Z [INFO] server-04ef2b72-148b-89d7-9562-cc9d60eba792: entering follower state: follower="Node at 04ef2b72-148b-89d7-9562-cc9d60eba792 [Follower]" leader-address= leader-id= -2025-01-25T14:29:42.792Z [WARN] server-04ef2b72-148b-89d7-9562-cc9d60eba792: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:42.792Z [INFO] server-04ef2b72-148b-89d7-9562-cc9d60eba792: entering candidate state: node="Node at 04ef2b72-148b-89d7-9562-cc9d60eba792 [Candidate]" term=2 -2025-01-25T14:29:42.792Z [INFO] server-04ef2b72-148b-89d7-9562-cc9d60eba792: election won: term=2 tally=2 -2025-01-25T14:29:42.792Z [INFO] server-04ef2b72-148b-89d7-9562-cc9d60eba792: entering leader state: leader="Node at 04ef2b72-148b-89d7-9562-cc9d60eba792 [Leader]" -2025-01-25T14:29:42.792Z [INFO] server-04ef2b72-148b-89d7-9562-cc9d60eba792: added peer, starting replication: peer=server-2d4bbf4a-b755-51fe-7de1-eb34e5a1c47e -2025-01-25T14:29:42.792Z [INFO] server-04ef2b72-148b-89d7-9562-cc9d60eba792: added peer, starting replication: peer=server-777f47a4-665c-dae8-ed72-6302f2e20c9c -2025-01-25T14:29:42.793Z [INFO] server-04ef2b72-148b-89d7-9562-cc9d60eba792: pipelining replication: peer="{Voter server-777f47a4-665c-dae8-ed72-6302f2e20c9c 777f47a4-665c-dae8-ed72-6302f2e20c9c}" -2025-01-25T14:29:42.793Z [INFO] server-04ef2b72-148b-89d7-9562-cc9d60eba792: pipelining replication: peer="{Voter server-2d4bbf4a-b755-51fe-7de1-eb34e5a1c47e 2d4bbf4a-b755-51fe-7de1-eb34e5a1c47e}" -2025-01-25T14:29:43.798Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:42.793489846 +0000 UTC m=+92.647814828 (1 nodes), 578.80433ms from start of poll, 579.508425ms from cluster start. Timeout at 2025-01-25 14:29:43.798502566 +0000 UTC m=+93.652827548, 1.00501272s after stability -2025-01-25T14:29:43.799Z [INFO] server-04ef2b72-148b-89d7-9562-cc9d60eba792: starting snapshot up to: index=12 -2025-01-25T14:29:43.799Z [INFO] creating new snapshot: path=/tmp/raft2785895767/snapshots/2-12-1737815383799.tmp -2025-01-25T14:29:43.800Z [INFO] server-04ef2b72-148b-89d7-9562-cc9d60eba792: snapshot complete up to: index=12 -2025-01-25T14:29:43.803Z [INFO] server-ab73106a-aebb-13bf-c751-17b2f518886c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ab73106a-aebb-13bf-c751-17b2f518886c Address:ab73106a-aebb-13bf-c751-17b2f518886c} {Suffrage:Voter ID:server-ddd30d47-14a1-11c1-b864-16ac8f6862b7 Address:ddd30d47-14a1-11c1-b864-16ac8f6862b7} {Suffrage:Voter ID:server-678fc9c7-1974-ff73-c888-d3fb679aa89c Address:678fc9c7-1974-ff73-c888-d3fb679aa89c}]" -2025-01-25T14:29:43.803Z [INFO] server-ab73106a-aebb-13bf-c751-17b2f518886c: entering follower state: follower="Node at ab73106a-aebb-13bf-c751-17b2f518886c [Follower]" leader-address= leader-id= -2025-01-25T14:29:43.803Z [INFO] server-ddd30d47-14a1-11c1-b864-16ac8f6862b7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ab73106a-aebb-13bf-c751-17b2f518886c Address:ab73106a-aebb-13bf-c751-17b2f518886c} {Suffrage:Voter ID:server-ddd30d47-14a1-11c1-b864-16ac8f6862b7 Address:ddd30d47-14a1-11c1-b864-16ac8f6862b7} {Suffrage:Voter ID:server-678fc9c7-1974-ff73-c888-d3fb679aa89c Address:678fc9c7-1974-ff73-c888-d3fb679aa89c}]" -2025-01-25T14:29:43.803Z [INFO] server-678fc9c7-1974-ff73-c888-d3fb679aa89c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ab73106a-aebb-13bf-c751-17b2f518886c Address:ab73106a-aebb-13bf-c751-17b2f518886c} {Suffrage:Voter ID:server-ddd30d47-14a1-11c1-b864-16ac8f6862b7 Address:ddd30d47-14a1-11c1-b864-16ac8f6862b7} {Suffrage:Voter ID:server-678fc9c7-1974-ff73-c888-d3fb679aa89c Address:678fc9c7-1974-ff73-c888-d3fb679aa89c}]" -2025-01-25T14:29:43.803Z [INFO] server-ddd30d47-14a1-11c1-b864-16ac8f6862b7: entering follower state: follower="Node at ddd30d47-14a1-11c1-b864-16ac8f6862b7 [Follower]" leader-address= leader-id= -2025-01-25T14:29:43.803Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:43.803Z [INFO] server-678fc9c7-1974-ff73-c888-d3fb679aa89c: entering follower state: follower="Node at 678fc9c7-1974-ff73-c888-d3fb679aa89c [Follower]" leader-address= leader-id= -2025-01-25T14:29:44.352Z [WARN] server-ddd30d47-14a1-11c1-b864-16ac8f6862b7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:44.352Z [INFO] server-ddd30d47-14a1-11c1-b864-16ac8f6862b7: entering candidate state: node="Node at ddd30d47-14a1-11c1-b864-16ac8f6862b7 [Candidate]" term=2 -2025-01-25T14:29:44.353Z [INFO] server-ddd30d47-14a1-11c1-b864-16ac8f6862b7: election won: term=2 tally=2 -2025-01-25T14:29:44.353Z [INFO] server-ddd30d47-14a1-11c1-b864-16ac8f6862b7: entering leader state: leader="Node at ddd30d47-14a1-11c1-b864-16ac8f6862b7 [Leader]" -2025-01-25T14:29:44.353Z [INFO] server-ddd30d47-14a1-11c1-b864-16ac8f6862b7: added peer, starting replication: peer=server-ab73106a-aebb-13bf-c751-17b2f518886c -2025-01-25T14:29:44.353Z [INFO] server-ddd30d47-14a1-11c1-b864-16ac8f6862b7: added peer, starting replication: peer=server-678fc9c7-1974-ff73-c888-d3fb679aa89c -2025-01-25T14:29:44.353Z [INFO] server-ddd30d47-14a1-11c1-b864-16ac8f6862b7: pipelining replication: peer="{Voter server-678fc9c7-1974-ff73-c888-d3fb679aa89c 678fc9c7-1974-ff73-c888-d3fb679aa89c}" -2025-01-25T14:29:44.353Z [INFO] server-ddd30d47-14a1-11c1-b864-16ac8f6862b7: pipelining replication: peer="{Voter server-ab73106a-aebb-13bf-c751-17b2f518886c ab73106a-aebb-13bf-c751-17b2f518886c}" -2025-01-25T14:29:45.359Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:44.353157318 +0000 UTC m=+94.207482300 (1 nodes), 549.339951ms from start of poll, 550.18792ms from cluster start. Timeout at 2025-01-25 14:29:45.358160238 +0000 UTC m=+95.212485220, 1.00500292s after stability -2025-01-25T14:29:45.360Z [INFO] creating new snapshot: path=/tmp/raft3009688639/snapshots/2-10013-1737815385360.tmp -2025-01-25T14:29:45.361Z [INFO] server-ddd30d47-14a1-11c1-b864-16ac8f6862b7: copied to local snapshot: bytes=71 -2025-01-25T14:29:45.361Z [INFO] server-ddd30d47-14a1-11c1-b864-16ac8f6862b7: snapshot restore progress: id=2-10013-1737815385360 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:45.362Z [INFO] server-ddd30d47-14a1-11c1-b864-16ac8f6862b7: removing all old logs from log store -2025-01-25T14:29:45.362Z [INFO] server-ddd30d47-14a1-11c1-b864-16ac8f6862b7: compacting logs: from=1 to=2 -2025-01-25T14:29:45.362Z [INFO] server-ddd30d47-14a1-11c1-b864-16ac8f6862b7: restored user snapshot: index=10013 -2025-01-25T14:29:45.362Z [ERROR] server-ddd30d47-14a1-11c1-b864-16ac8f6862b7: failed to get log: index=2 error="log not found" -2025-01-25T14:29:45.362Z [INFO] server-ddd30d47-14a1-11c1-b864-16ac8f6862b7: aborting pipeline replication: peer="{Voter server-678fc9c7-1974-ff73-c888-d3fb679aa89c 678fc9c7-1974-ff73-c888-d3fb679aa89c}" -2025-01-25T14:29:45.362Z [ERROR] server-ddd30d47-14a1-11c1-b864-16ac8f6862b7: failed to get log: index=2 error="log not found" -2025-01-25T14:29:45.362Z [INFO] server-ddd30d47-14a1-11c1-b864-16ac8f6862b7: aborting pipeline replication: peer="{Voter server-ab73106a-aebb-13bf-c751-17b2f518886c ab73106a-aebb-13bf-c751-17b2f518886c}" -2025-01-25T14:29:45.368Z [ERROR] server-ddd30d47-14a1-11c1-b864-16ac8f6862b7: failed to get log: index=2 error="log not found" -2025-01-25T14:29:45.368Z [INFO] creating new snapshot: path=/tmp/raft2273850496/snapshots/2-10013-1737815385368.tmp -2025-01-25T14:29:45.369Z [INFO] server-ab73106a-aebb-13bf-c751-17b2f518886c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:45.370Z [INFO] server-ab73106a-aebb-13bf-c751-17b2f518886c: copied to local snapshot: bytes=71 -2025-01-25T14:29:45.370Z [INFO] server-ab73106a-aebb-13bf-c751-17b2f518886c: snapshot restore progress: id=2-10013-1737815385368 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:45.370Z [INFO] server-ab73106a-aebb-13bf-c751-17b2f518886c: removing all old logs from log store -2025-01-25T14:29:45.370Z [INFO] server-ab73106a-aebb-13bf-c751-17b2f518886c: compacting logs: from=1 to=2 -2025-01-25T14:29:45.370Z [INFO] server-ab73106a-aebb-13bf-c751-17b2f518886c: Installed remote snapshot -2025-01-25T14:29:45.371Z [INFO] server-ddd30d47-14a1-11c1-b864-16ac8f6862b7: pipelining replication: peer="{Voter server-ab73106a-aebb-13bf-c751-17b2f518886c ab73106a-aebb-13bf-c751-17b2f518886c}" -2025-01-25T14:29:45.371Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:29:45.371Z [ERROR] server-ddd30d47-14a1-11c1-b864-16ac8f6862b7: failed to get log: index=2 error="log not found" -2025-01-25T14:29:45.372Z [INFO] creating new snapshot: path=/tmp/raft1251686499/snapshots/2-10013-1737815385372.tmp -2025-01-25T14:29:45.372Z [INFO] server-678fc9c7-1974-ff73-c888-d3fb679aa89c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:45.373Z [INFO] server-678fc9c7-1974-ff73-c888-d3fb679aa89c: copied to local snapshot: bytes=71 -2025-01-25T14:29:45.373Z [INFO] server-678fc9c7-1974-ff73-c888-d3fb679aa89c: snapshot restore progress: id=2-10013-1737815385372 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-25T14:29:45.374Z [INFO] server-678fc9c7-1974-ff73-c888-d3fb679aa89c: removing all old logs from log store -2025-01-25T14:29:45.374Z [INFO] server-678fc9c7-1974-ff73-c888-d3fb679aa89c: compacting logs: from=1 to=2 -2025-01-25T14:29:45.374Z [INFO] server-678fc9c7-1974-ff73-c888-d3fb679aa89c: Installed remote snapshot -2025-01-25T14:29:45.374Z [INFO] server-ddd30d47-14a1-11c1-b864-16ac8f6862b7: pipelining replication: peer="{Voter server-678fc9c7-1974-ff73-c888-d3fb679aa89c 678fc9c7-1974-ff73-c888-d3fb679aa89c}" -2025-01-25T14:29:46.376Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:29:45.371324557 +0000 UTC m=+95.225649499 (2 nodes), 1.96µs from start of poll, 1.568355119s from cluster start. Timeout at 2025-01-25 14:29:46.376580507 +0000 UTC m=+96.230905489, 1.00525599s after stability -2025-01-25T14:29:46.388Z [INFO] server-04ef2b72-148b-89d7-9562-cc9d60eba792: aborting pipeline replication: peer="{Voter server-777f47a4-665c-dae8-ed72-6302f2e20c9c 777f47a4-665c-dae8-ed72-6302f2e20c9c}" -2025-01-25T14:29:46.388Z [INFO] server-04ef2b72-148b-89d7-9562-cc9d60eba792: aborting pipeline replication: peer="{Voter server-2d4bbf4a-b755-51fe-7de1-eb34e5a1c47e 2d4bbf4a-b755-51fe-7de1-eb34e5a1c47e}" ---- PASS: TestRaft_UserRestore (82.81s) - --- PASS: TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_false (2.71s) +2025-01-25T14:40:28.758Z [INFO] server-e8d3b8d7-7c51-8ecd-852b-e3afaa2d89f2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e8d3b8d7-7c51-8ecd-852b-e3afaa2d89f2 Address:e8d3b8d7-7c51-8ecd-852b-e3afaa2d89f2} {Suffrage:Voter ID:server-eef70d67-69e3-20a4-7616-6fd79f261684 Address:eef70d67-69e3-20a4-7616-6fd79f261684} {Suffrage:Voter ID:server-45711fed-3580-6374-e489-1abafac07794 Address:45711fed-3580-6374-e489-1abafac07794}]" +2025-01-25T14:40:28.758Z [INFO] server-e8d3b8d7-7c51-8ecd-852b-e3afaa2d89f2: entering follower state: follower="Node at e8d3b8d7-7c51-8ecd-852b-e3afaa2d89f2 [Follower]" leader-address= leader-id= +2025-01-25T14:40:28.759Z [INFO] server-eef70d67-69e3-20a4-7616-6fd79f261684: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e8d3b8d7-7c51-8ecd-852b-e3afaa2d89f2 Address:e8d3b8d7-7c51-8ecd-852b-e3afaa2d89f2} {Suffrage:Voter ID:server-eef70d67-69e3-20a4-7616-6fd79f261684 Address:eef70d67-69e3-20a4-7616-6fd79f261684} {Suffrage:Voter ID:server-45711fed-3580-6374-e489-1abafac07794 Address:45711fed-3580-6374-e489-1abafac07794}]" +2025-01-25T14:40:28.759Z [INFO] server-eef70d67-69e3-20a4-7616-6fd79f261684: entering follower state: follower="Node at eef70d67-69e3-20a4-7616-6fd79f261684 [Follower]" leader-address= leader-id= +2025-01-25T14:40:28.760Z [INFO] server-45711fed-3580-6374-e489-1abafac07794: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e8d3b8d7-7c51-8ecd-852b-e3afaa2d89f2 Address:e8d3b8d7-7c51-8ecd-852b-e3afaa2d89f2} {Suffrage:Voter ID:server-eef70d67-69e3-20a4-7616-6fd79f261684 Address:eef70d67-69e3-20a4-7616-6fd79f261684} {Suffrage:Voter ID:server-45711fed-3580-6374-e489-1abafac07794 Address:45711fed-3580-6374-e489-1abafac07794}]" +2025-01-25T14:40:28.760Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:28.760Z [INFO] server-45711fed-3580-6374-e489-1abafac07794: entering follower state: follower="Node at 45711fed-3580-6374-e489-1abafac07794 [Follower]" leader-address= leader-id= +2025-01-25T14:40:29.381Z [WARN] server-e8d3b8d7-7c51-8ecd-852b-e3afaa2d89f2: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:29.381Z [INFO] server-e8d3b8d7-7c51-8ecd-852b-e3afaa2d89f2: entering candidate state: node="Node at e8d3b8d7-7c51-8ecd-852b-e3afaa2d89f2 [Candidate]" term=2 +2025-01-25T14:40:29.382Z [INFO] server-e8d3b8d7-7c51-8ecd-852b-e3afaa2d89f2: election won: term=2 tally=2 +2025-01-25T14:40:29.382Z [INFO] server-e8d3b8d7-7c51-8ecd-852b-e3afaa2d89f2: entering leader state: leader="Node at e8d3b8d7-7c51-8ecd-852b-e3afaa2d89f2 [Leader]" +2025-01-25T14:40:29.382Z [INFO] server-e8d3b8d7-7c51-8ecd-852b-e3afaa2d89f2: added peer, starting replication: peer=server-eef70d67-69e3-20a4-7616-6fd79f261684 +2025-01-25T14:40:29.382Z [INFO] server-e8d3b8d7-7c51-8ecd-852b-e3afaa2d89f2: added peer, starting replication: peer=server-45711fed-3580-6374-e489-1abafac07794 +2025-01-25T14:40:29.383Z [INFO] server-e8d3b8d7-7c51-8ecd-852b-e3afaa2d89f2: pipelining replication: peer="{Voter server-eef70d67-69e3-20a4-7616-6fd79f261684 eef70d67-69e3-20a4-7616-6fd79f261684}" +2025-01-25T14:40:29.383Z [INFO] server-e8d3b8d7-7c51-8ecd-852b-e3afaa2d89f2: pipelining replication: peer="{Voter server-45711fed-3580-6374-e489-1abafac07794 45711fed-3580-6374-e489-1abafac07794}" +2025-01-25T14:40:30.387Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:29.382522541 +0000 UTC m=+98.428361858 (1 nodes), 621.918435ms from start of poll, 624.582818ms from cluster start. Timeout at 2025-01-25 14:40:30.387543832 +0000 UTC m=+99.433383150, 1.005021292s after stability +2025-01-25T14:40:30.391Z [INFO] server-e8d3b8d7-7c51-8ecd-852b-e3afaa2d89f2: starting snapshot up to: index=12 +2025-01-25T14:40:30.392Z [INFO] creating new snapshot: path=/tmp/raft1569310466/snapshots/2-12-1737816030392.tmp +2025-01-25T14:40:30.395Z [INFO] server-e8d3b8d7-7c51-8ecd-852b-e3afaa2d89f2: snapshot complete up to: index=12 +2025-01-25T14:40:30.405Z [INFO] server-36ce9746-b007-ae40-39c9-19752406d5c7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-36ce9746-b007-ae40-39c9-19752406d5c7 Address:36ce9746-b007-ae40-39c9-19752406d5c7} {Suffrage:Voter ID:server-673fe147-0250-5d84-c892-01032787a846 Address:673fe147-0250-5d84-c892-01032787a846} {Suffrage:Voter ID:server-7c07c237-a25f-f4ce-ca7d-8168ee14a7ee Address:7c07c237-a25f-f4ce-ca7d-8168ee14a7ee}]" +2025-01-25T14:40:30.406Z [INFO] server-36ce9746-b007-ae40-39c9-19752406d5c7: entering follower state: follower="Node at 36ce9746-b007-ae40-39c9-19752406d5c7 [Follower]" leader-address= leader-id= +2025-01-25T14:40:30.406Z [INFO] server-673fe147-0250-5d84-c892-01032787a846: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-36ce9746-b007-ae40-39c9-19752406d5c7 Address:36ce9746-b007-ae40-39c9-19752406d5c7} {Suffrage:Voter ID:server-673fe147-0250-5d84-c892-01032787a846 Address:673fe147-0250-5d84-c892-01032787a846} {Suffrage:Voter ID:server-7c07c237-a25f-f4ce-ca7d-8168ee14a7ee Address:7c07c237-a25f-f4ce-ca7d-8168ee14a7ee}]" +2025-01-25T14:40:30.407Z [INFO] server-673fe147-0250-5d84-c892-01032787a846: entering follower state: follower="Node at 673fe147-0250-5d84-c892-01032787a846 [Follower]" leader-address= leader-id= +2025-01-25T14:40:30.407Z [INFO] server-7c07c237-a25f-f4ce-ca7d-8168ee14a7ee: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-36ce9746-b007-ae40-39c9-19752406d5c7 Address:36ce9746-b007-ae40-39c9-19752406d5c7} {Suffrage:Voter ID:server-673fe147-0250-5d84-c892-01032787a846 Address:673fe147-0250-5d84-c892-01032787a846} {Suffrage:Voter ID:server-7c07c237-a25f-f4ce-ca7d-8168ee14a7ee Address:7c07c237-a25f-f4ce-ca7d-8168ee14a7ee}]" +2025-01-25T14:40:30.408Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:30.407Z [INFO] server-7c07c237-a25f-f4ce-ca7d-8168ee14a7ee: entering follower state: follower="Node at 7c07c237-a25f-f4ce-ca7d-8168ee14a7ee [Follower]" leader-address= leader-id= +2025-01-25T14:40:31.181Z [WARN] server-36ce9746-b007-ae40-39c9-19752406d5c7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:31.181Z [INFO] server-36ce9746-b007-ae40-39c9-19752406d5c7: entering candidate state: node="Node at 36ce9746-b007-ae40-39c9-19752406d5c7 [Candidate]" term=2 +2025-01-25T14:40:31.182Z [INFO] server-36ce9746-b007-ae40-39c9-19752406d5c7: election won: term=2 tally=2 +2025-01-25T14:40:31.182Z [INFO] server-36ce9746-b007-ae40-39c9-19752406d5c7: entering leader state: leader="Node at 36ce9746-b007-ae40-39c9-19752406d5c7 [Leader]" +2025-01-25T14:40:31.182Z [INFO] server-36ce9746-b007-ae40-39c9-19752406d5c7: added peer, starting replication: peer=server-673fe147-0250-5d84-c892-01032787a846 +2025-01-25T14:40:31.182Z [INFO] server-36ce9746-b007-ae40-39c9-19752406d5c7: added peer, starting replication: peer=server-7c07c237-a25f-f4ce-ca7d-8168ee14a7ee +2025-01-25T14:40:31.183Z [INFO] server-36ce9746-b007-ae40-39c9-19752406d5c7: pipelining replication: peer="{Voter server-673fe147-0250-5d84-c892-01032787a846 673fe147-0250-5d84-c892-01032787a846}" +2025-01-25T14:40:31.183Z [INFO] server-36ce9746-b007-ae40-39c9-19752406d5c7: pipelining replication: peer="{Voter server-7c07c237-a25f-f4ce-ca7d-8168ee14a7ee 7c07c237-a25f-f4ce-ca7d-8168ee14a7ee}" +2025-01-25T14:40:32.187Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:31.182308655 +0000 UTC m=+100.228147681 (1 nodes), 774.223129ms from start of poll, 777.026345ms from cluster start. Timeout at 2025-01-25 14:40:32.187331405 +0000 UTC m=+101.233170723, 1.005023042s after stability +2025-01-25T14:40:32.189Z [INFO] creating new snapshot: path=/tmp/raft1889553920/snapshots/2-10013-1737816032189.tmp +2025-01-25T14:40:32.193Z [INFO] server-36ce9746-b007-ae40-39c9-19752406d5c7: copied to local snapshot: bytes=71 +2025-01-25T14:40:32.194Z [INFO] server-36ce9746-b007-ae40-39c9-19752406d5c7: snapshot restore progress: id=2-10013-1737816032189 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:32.195Z [INFO] server-36ce9746-b007-ae40-39c9-19752406d5c7: removing all old logs from log store +2025-01-25T14:40:32.195Z [INFO] server-36ce9746-b007-ae40-39c9-19752406d5c7: compacting logs: from=1 to=2 +2025-01-25T14:40:32.195Z [INFO] server-36ce9746-b007-ae40-39c9-19752406d5c7: restored user snapshot: index=10013 +2025-01-25T14:40:32.195Z [ERROR] server-36ce9746-b007-ae40-39c9-19752406d5c7: failed to get log: index=2 error="log not found" +2025-01-25T14:40:32.195Z [ERROR] server-36ce9746-b007-ae40-39c9-19752406d5c7: failed to get log: index=2 error="log not found" +2025-01-25T14:40:32.195Z [INFO] server-36ce9746-b007-ae40-39c9-19752406d5c7: aborting pipeline replication: peer="{Voter server-7c07c237-a25f-f4ce-ca7d-8168ee14a7ee 7c07c237-a25f-f4ce-ca7d-8168ee14a7ee}" +2025-01-25T14:40:32.195Z [INFO] server-36ce9746-b007-ae40-39c9-19752406d5c7: aborting pipeline replication: peer="{Voter server-673fe147-0250-5d84-c892-01032787a846 673fe147-0250-5d84-c892-01032787a846}" +2025-01-25T14:40:32.203Z [ERROR] server-36ce9746-b007-ae40-39c9-19752406d5c7: failed to get log: index=2 error="log not found" +2025-01-25T14:40:32.204Z [ERROR] server-36ce9746-b007-ae40-39c9-19752406d5c7: failed to get log: index=2 error="log not found" +2025-01-25T14:40:32.206Z [INFO] creating new snapshot: path=/tmp/raft3571742711/snapshots/2-10013-1737816032206.tmp +2025-01-25T14:40:32.206Z [INFO] creating new snapshot: path=/tmp/raft2911222235/snapshots/2-10013-1737816032206.tmp +2025-01-25T14:40:32.207Z [INFO] server-7c07c237-a25f-f4ce-ca7d-8168ee14a7ee: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:32.208Z [INFO] server-673fe147-0250-5d84-c892-01032787a846: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:32.209Z [INFO] server-7c07c237-a25f-f4ce-ca7d-8168ee14a7ee: copied to local snapshot: bytes=71 +2025-01-25T14:40:32.210Z [INFO] server-7c07c237-a25f-f4ce-ca7d-8168ee14a7ee: snapshot restore progress: id=2-10013-1737816032206 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:32.210Z [INFO] server-7c07c237-a25f-f4ce-ca7d-8168ee14a7ee: removing all old logs from log store +2025-01-25T14:40:32.210Z [INFO] server-7c07c237-a25f-f4ce-ca7d-8168ee14a7ee: compacting logs: from=1 to=2 +2025-01-25T14:40:32.210Z [INFO] server-7c07c237-a25f-f4ce-ca7d-8168ee14a7ee: Installed remote snapshot +2025-01-25T14:40:32.211Z [INFO] server-36ce9746-b007-ae40-39c9-19752406d5c7: pipelining replication: peer="{Voter server-7c07c237-a25f-f4ce-ca7d-8168ee14a7ee 7c07c237-a25f-f4ce-ca7d-8168ee14a7ee}" +2025-01-25T14:40:32.211Z [INFO] server-673fe147-0250-5d84-c892-01032787a846: copied to local snapshot: bytes=71 +2025-01-25T14:40:32.211Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:40:32.213Z [INFO] server-673fe147-0250-5d84-c892-01032787a846: snapshot restore progress: id=2-10013-1737816032206 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-25T14:40:32.213Z [INFO] server-673fe147-0250-5d84-c892-01032787a846: removing all old logs from log store +2025-01-25T14:40:32.213Z [INFO] server-673fe147-0250-5d84-c892-01032787a846: compacting logs: from=1 to=2 +2025-01-25T14:40:32.213Z [INFO] server-673fe147-0250-5d84-c892-01032787a846: Installed remote snapshot +2025-01-25T14:40:32.214Z [INFO] server-36ce9746-b007-ae40-39c9-19752406d5c7: pipelining replication: peer="{Voter server-673fe147-0250-5d84-c892-01032787a846 673fe147-0250-5d84-c892-01032787a846}" +2025-01-25T14:40:33.226Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:40:32.211898694 +0000 UTC m=+101.257737720 (2 nodes), 6.416µs from start of poll, 1.806616384s from cluster start. Timeout at 2025-01-25 14:40:33.217130278 +0000 UTC m=+102.262969596, 1.005231876s after stability +2025-01-25T14:40:33.240Z [INFO] server-e8d3b8d7-7c51-8ecd-852b-e3afaa2d89f2: aborting pipeline replication: peer="{Voter server-45711fed-3580-6374-e489-1abafac07794 45711fed-3580-6374-e489-1abafac07794}" +2025-01-25T14:40:33.240Z [INFO] server-e8d3b8d7-7c51-8ecd-852b-e3afaa2d89f2: aborting pipeline replication: peer="{Voter server-eef70d67-69e3-20a4-7616-6fd79f261684 eef70d67-69e3-20a4-7616-6fd79f261684}" +--- PASS: TestRaft_UserRestore (85.30s) + --- PASS: TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_false (2.86s) --- PASS: TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_false (2.58s) - --- PASS: TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_true (4.08s) + --- PASS: TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_true (4.25s) --- PASS: TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_true (4.26s) - --- PASS: TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_false (2.90s) - --- PASS: TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_false (2.61s) + --- PASS: TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_false (2.79s) + --- PASS: TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_false (2.82s) --- PASS: TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_true (4.43s) - --- PASS: TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_true (4.56s) - --- PASS: TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_false (2.55s) - --- PASS: TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_false (2.61s) - --- PASS: TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_true (4.21s) - --- PASS: TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_true (4.22s) - --- PASS: TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_false (2.61s) - --- PASS: TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_false (2.78s) - --- PASS: TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_true (4.14s) - --- PASS: TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_true (4.19s) - --- PASS: TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_false (2.56s) - --- PASS: TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_false (2.63s) - --- PASS: TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_true (4.14s) - --- PASS: TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_true (4.12s) - --- PASS: TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_false (2.78s) - --- PASS: TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_false (2.63s) - --- PASS: TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_true (4.34s) - --- PASS: TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_true (4.18s) + --- PASS: TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_true (4.20s) + --- PASS: TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_false (2.77s) + --- PASS: TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_false (2.91s) + --- PASS: TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_true (4.37s) + --- PASS: TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_true (4.18s) + --- PASS: TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_false (2.73s) + --- PASS: TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_false (2.61s) + --- PASS: TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_true (4.48s) + --- PASS: TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_true (4.20s) + --- PASS: TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_false (2.62s) + --- PASS: TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_false (3.14s) + --- PASS: TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_true (4.27s) + --- PASS: TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_true (4.18s) + --- PASS: TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_false (2.96s) + --- PASS: TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_false (2.92s) + --- PASS: TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_true (4.28s) + --- PASS: TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_true (4.49s) === RUN TestRaft_SendSnapshotFollower -2025-01-25T14:29:46.398Z [INFO] server-df7d3529-fcf7-edee-33ba-4dc1c04ad324: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-df7d3529-fcf7-edee-33ba-4dc1c04ad324 Address:df7d3529-fcf7-edee-33ba-4dc1c04ad324} {Suffrage:Voter ID:server-558d93a0-fba7-31db-1efc-5f3d1d067252 Address:558d93a0-fba7-31db-1efc-5f3d1d067252} {Suffrage:Voter ID:server-966e830d-4b6e-96dd-70de-88075b220f08 Address:966e830d-4b6e-96dd-70de-88075b220f08}]" -2025-01-25T14:29:46.401Z [INFO] server-df7d3529-fcf7-edee-33ba-4dc1c04ad324: entering follower state: follower="Node at df7d3529-fcf7-edee-33ba-4dc1c04ad324 [Follower]" leader-address= leader-id= -2025-01-25T14:29:46.401Z [INFO] server-558d93a0-fba7-31db-1efc-5f3d1d067252: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-df7d3529-fcf7-edee-33ba-4dc1c04ad324 Address:df7d3529-fcf7-edee-33ba-4dc1c04ad324} {Suffrage:Voter ID:server-558d93a0-fba7-31db-1efc-5f3d1d067252 Address:558d93a0-fba7-31db-1efc-5f3d1d067252} {Suffrage:Voter ID:server-966e830d-4b6e-96dd-70de-88075b220f08 Address:966e830d-4b6e-96dd-70de-88075b220f08}]" -2025-01-25T14:29:46.401Z [INFO] server-966e830d-4b6e-96dd-70de-88075b220f08: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-df7d3529-fcf7-edee-33ba-4dc1c04ad324 Address:df7d3529-fcf7-edee-33ba-4dc1c04ad324} {Suffrage:Voter ID:server-558d93a0-fba7-31db-1efc-5f3d1d067252 Address:558d93a0-fba7-31db-1efc-5f3d1d067252} {Suffrage:Voter ID:server-966e830d-4b6e-96dd-70de-88075b220f08 Address:966e830d-4b6e-96dd-70de-88075b220f08}]" -2025-01-25T14:29:46.401Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:29:46.401Z [INFO] server-558d93a0-fba7-31db-1efc-5f3d1d067252: entering follower state: follower="Node at 558d93a0-fba7-31db-1efc-5f3d1d067252 [Follower]" leader-address= leader-id= -2025-01-25T14:29:46.401Z [INFO] server-966e830d-4b6e-96dd-70de-88075b220f08: entering follower state: follower="Node at 966e830d-4b6e-96dd-70de-88075b220f08 [Follower]" leader-address= leader-id= -2025-01-25T14:29:46.460Z [WARN] server-966e830d-4b6e-96dd-70de-88075b220f08: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:46.460Z [INFO] server-966e830d-4b6e-96dd-70de-88075b220f08: entering candidate state: node="Node at 966e830d-4b6e-96dd-70de-88075b220f08 [Candidate]" term=2 -2025-01-25T14:29:46.460Z [INFO] server-966e830d-4b6e-96dd-70de-88075b220f08: election won: term=2 tally=2 -2025-01-25T14:29:46.460Z [INFO] server-966e830d-4b6e-96dd-70de-88075b220f08: entering leader state: leader="Node at 966e830d-4b6e-96dd-70de-88075b220f08 [Leader]" -2025-01-25T14:29:46.460Z [INFO] server-966e830d-4b6e-96dd-70de-88075b220f08: added peer, starting replication: peer=server-df7d3529-fcf7-edee-33ba-4dc1c04ad324 -2025-01-25T14:29:46.460Z [INFO] server-966e830d-4b6e-96dd-70de-88075b220f08: added peer, starting replication: peer=server-558d93a0-fba7-31db-1efc-5f3d1d067252 -2025-01-25T14:29:46.460Z [INFO] server-966e830d-4b6e-96dd-70de-88075b220f08: pipelining replication: peer="{Voter server-558d93a0-fba7-31db-1efc-5f3d1d067252 558d93a0-fba7-31db-1efc-5f3d1d067252}" -2025-01-25T14:29:46.460Z [INFO] server-966e830d-4b6e-96dd-70de-88075b220f08: pipelining replication: peer="{Voter server-df7d3529-fcf7-edee-33ba-4dc1c04ad324 df7d3529-fcf7-edee-33ba-4dc1c04ad324}" -2025-01-25T14:29:46.566Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:29:46.460646636 +0000 UTC m=+96.314971618 (2 nodes), 58.799244ms from start of poll, 68.37718ms from cluster start. Timeout at 2025-01-25 14:29:46.565650316 +0000 UTC m=+96.419975298, 105.00368ms after stability -2025-01-25T14:29:46.566Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:46.672Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:46.566366431 +0000 UTC m=+96.420691413 (1 nodes), 1.68µs from start of poll, 174.096975ms from cluster start. Timeout at 2025-01-25 14:29:46.671521466 +0000 UTC m=+96.525846488, 105.155075ms after stability -2025-01-25T14:29:46.674Z [ERROR] server-966e830d-4b6e-96dd-70de-88075b220f08: failed to pipeline appendEntries: peer="{Voter server-df7d3529-fcf7-edee-33ba-4dc1c04ad324 df7d3529-fcf7-edee-33ba-4dc1c04ad324}" error="append pipeline closed" -2025-01-25T14:29:46.674Z [INFO] server-966e830d-4b6e-96dd-70de-88075b220f08: aborting pipeline replication: peer="{Voter server-df7d3529-fcf7-edee-33ba-4dc1c04ad324 df7d3529-fcf7-edee-33ba-4dc1c04ad324}" -2025-01-25T14:29:46.674Z [ERROR] server-966e830d-4b6e-96dd-70de-88075b220f08: failed to appendEntries to: peer="{Voter server-df7d3529-fcf7-edee-33ba-4dc1c04ad324 df7d3529-fcf7-edee-33ba-4dc1c04ad324}" error="failed to connect to peer: df7d3529-fcf7-edee-33ba-4dc1c04ad324" -2025-01-25T14:29:46.676Z [ERROR] server-966e830d-4b6e-96dd-70de-88075b220f08: failed to heartbeat to: peer=df7d3529-fcf7-edee-33ba-4dc1c04ad324 backoff time=10ms error="failed to connect to peer: df7d3529-fcf7-edee-33ba-4dc1c04ad324" +2025-01-25T14:40:33.250Z [INFO] server-b27defbd-0ce9-4a65-5da9-07f2e13933ea: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b27defbd-0ce9-4a65-5da9-07f2e13933ea Address:b27defbd-0ce9-4a65-5da9-07f2e13933ea} {Suffrage:Voter ID:server-ebdcbc21-64ce-19f6-c15b-68f8dd07cc41 Address:ebdcbc21-64ce-19f6-c15b-68f8dd07cc41} {Suffrage:Voter ID:server-5f86487b-bc80-abb5-2fbd-0dc5ffdb0634 Address:5f86487b-bc80-abb5-2fbd-0dc5ffdb0634}]" +2025-01-25T14:40:33.250Z [INFO] server-b27defbd-0ce9-4a65-5da9-07f2e13933ea: entering follower state: follower="Node at b27defbd-0ce9-4a65-5da9-07f2e13933ea [Follower]" leader-address= leader-id= +2025-01-25T14:40:33.251Z [INFO] server-ebdcbc21-64ce-19f6-c15b-68f8dd07cc41: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b27defbd-0ce9-4a65-5da9-07f2e13933ea Address:b27defbd-0ce9-4a65-5da9-07f2e13933ea} {Suffrage:Voter ID:server-ebdcbc21-64ce-19f6-c15b-68f8dd07cc41 Address:ebdcbc21-64ce-19f6-c15b-68f8dd07cc41} {Suffrage:Voter ID:server-5f86487b-bc80-abb5-2fbd-0dc5ffdb0634 Address:5f86487b-bc80-abb5-2fbd-0dc5ffdb0634}]" +2025-01-25T14:40:33.251Z [INFO] server-ebdcbc21-64ce-19f6-c15b-68f8dd07cc41: entering follower state: follower="Node at ebdcbc21-64ce-19f6-c15b-68f8dd07cc41 [Follower]" leader-address= leader-id= +2025-01-25T14:40:33.251Z [INFO] server-5f86487b-bc80-abb5-2fbd-0dc5ffdb0634: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b27defbd-0ce9-4a65-5da9-07f2e13933ea Address:b27defbd-0ce9-4a65-5da9-07f2e13933ea} {Suffrage:Voter ID:server-ebdcbc21-64ce-19f6-c15b-68f8dd07cc41 Address:ebdcbc21-64ce-19f6-c15b-68f8dd07cc41} {Suffrage:Voter ID:server-5f86487b-bc80-abb5-2fbd-0dc5ffdb0634 Address:5f86487b-bc80-abb5-2fbd-0dc5ffdb0634}]" +2025-01-25T14:40:33.252Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:40:33.252Z [INFO] server-5f86487b-bc80-abb5-2fbd-0dc5ffdb0634: entering follower state: follower="Node at 5f86487b-bc80-abb5-2fbd-0dc5ffdb0634 [Follower]" leader-address= leader-id= +2025-01-25T14:40:33.307Z [WARN] server-5f86487b-bc80-abb5-2fbd-0dc5ffdb0634: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:33.307Z [INFO] server-5f86487b-bc80-abb5-2fbd-0dc5ffdb0634: entering candidate state: node="Node at 5f86487b-bc80-abb5-2fbd-0dc5ffdb0634 [Candidate]" term=2 +2025-01-25T14:40:33.307Z [INFO] server-5f86487b-bc80-abb5-2fbd-0dc5ffdb0634: election won: term=2 tally=2 +2025-01-25T14:40:33.307Z [INFO] server-5f86487b-bc80-abb5-2fbd-0dc5ffdb0634: entering leader state: leader="Node at 5f86487b-bc80-abb5-2fbd-0dc5ffdb0634 [Leader]" +2025-01-25T14:40:33.308Z [INFO] server-5f86487b-bc80-abb5-2fbd-0dc5ffdb0634: added peer, starting replication: peer=server-b27defbd-0ce9-4a65-5da9-07f2e13933ea +2025-01-25T14:40:33.308Z [INFO] server-5f86487b-bc80-abb5-2fbd-0dc5ffdb0634: added peer, starting replication: peer=server-ebdcbc21-64ce-19f6-c15b-68f8dd07cc41 +2025-01-25T14:40:33.308Z [INFO] server-5f86487b-bc80-abb5-2fbd-0dc5ffdb0634: pipelining replication: peer="{Voter server-ebdcbc21-64ce-19f6-c15b-68f8dd07cc41 ebdcbc21-64ce-19f6-c15b-68f8dd07cc41}" +2025-01-25T14:40:33.309Z [INFO] server-5f86487b-bc80-abb5-2fbd-0dc5ffdb0634: pipelining replication: peer="{Voter server-b27defbd-0ce9-4a65-5da9-07f2e13933ea b27defbd-0ce9-4a65-5da9-07f2e13933ea}" +2025-01-25T14:40:33.413Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:40:33.308031886 +0000 UTC m=+102.353871203 (2 nodes), 55.806787ms from start of poll, 58.460962ms from cluster start. Timeout at 2025-01-25 14:40:33.413039177 +0000 UTC m=+102.458878495, 105.007292ms after stability +2025-01-25T14:40:33.413Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:33.519Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:33.413539234 +0000 UTC m=+102.459378552 (1 nodes), 6.125µs from start of poll, 163.968311ms from cluster start. Timeout at 2025-01-25 14:40:33.518728818 +0000 UTC m=+102.564568136, 105.189584ms after stability +2025-01-25T14:40:33.520Z [ERROR] server-5f86487b-bc80-abb5-2fbd-0dc5ffdb0634: failed to pipeline appendEntries: peer="{Voter server-b27defbd-0ce9-4a65-5da9-07f2e13933ea b27defbd-0ce9-4a65-5da9-07f2e13933ea}" error="append pipeline closed" +2025-01-25T14:40:33.520Z [INFO] server-5f86487b-bc80-abb5-2fbd-0dc5ffdb0634: aborting pipeline replication: peer="{Voter server-b27defbd-0ce9-4a65-5da9-07f2e13933ea b27defbd-0ce9-4a65-5da9-07f2e13933ea}" +2025-01-25T14:40:33.520Z [ERROR] server-5f86487b-bc80-abb5-2fbd-0dc5ffdb0634: failed to appendEntries to: peer="{Voter server-b27defbd-0ce9-4a65-5da9-07f2e13933ea b27defbd-0ce9-4a65-5da9-07f2e13933ea}" error="failed to connect to peer: b27defbd-0ce9-4a65-5da9-07f2e13933ea" +2025-01-25T14:40:33.523Z [ERROR] server-5f86487b-bc80-abb5-2fbd-0dc5ffdb0634: failed to heartbeat to: peer=b27defbd-0ce9-4a65-5da9-07f2e13933ea backoff time=10ms error="failed to connect to peer: b27defbd-0ce9-4a65-5da9-07f2e13933ea" +2025-01-25T14:40:33.531Z [ERROR] server-5f86487b-bc80-abb5-2fbd-0dc5ffdb0634: failed to appendEntries to: peer="{Voter server-b27defbd-0ce9-4a65-5da9-07f2e13933ea b27defbd-0ce9-4a65-5da9-07f2e13933ea}" error="failed to connect to peer: b27defbd-0ce9-4a65-5da9-07f2e13933ea" +2025-01-25T14:40:33.538Z [ERROR] server-5f86487b-bc80-abb5-2fbd-0dc5ffdb0634: failed to heartbeat to: peer=b27defbd-0ce9-4a65-5da9-07f2e13933ea backoff time=10ms error="failed to connect to peer: b27defbd-0ce9-4a65-5da9-07f2e13933ea" +2025-01-25T14:40:33.541Z [ERROR] server-5f86487b-bc80-abb5-2fbd-0dc5ffdb0634: failed to appendEntries to: peer="{Voter server-b27defbd-0ce9-4a65-5da9-07f2e13933ea b27defbd-0ce9-4a65-5da9-07f2e13933ea}" error="failed to connect to peer: b27defbd-0ce9-4a65-5da9-07f2e13933ea" raft_test.go:1627: [INFO] Finished apply without behind follower -2025-01-25T14:29:46.683Z [ERROR] server-df7d3529-fcf7-edee-33ba-4dc1c04ad324: failed to take snapshot: error="nothing new to snapshot" -2025-01-25T14:29:46.683Z [INFO] server-558d93a0-fba7-31db-1efc-5f3d1d067252: starting snapshot up to: index=80 -2025-01-25T14:29:46.683Z [INFO] creating new snapshot: path=/tmp/raft213672628/snapshots/2-80-1737815386683.tmp -2025-01-25T14:29:46.685Z [ERROR] server-966e830d-4b6e-96dd-70de-88075b220f08: failed to appendEntries to: peer="{Voter server-df7d3529-fcf7-edee-33ba-4dc1c04ad324 df7d3529-fcf7-edee-33ba-4dc1c04ad324}" error="failed to connect to peer: df7d3529-fcf7-edee-33ba-4dc1c04ad324" -2025-01-25T14:29:46.694Z [INFO] server-558d93a0-fba7-31db-1efc-5f3d1d067252: compacting logs: from=1 to=80 -2025-01-25T14:29:46.694Z [INFO] server-558d93a0-fba7-31db-1efc-5f3d1d067252: snapshot complete up to: index=80 -2025-01-25T14:29:46.695Z [ERROR] server-966e830d-4b6e-96dd-70de-88075b220f08: failed to appendEntries to: peer="{Voter server-df7d3529-fcf7-edee-33ba-4dc1c04ad324 df7d3529-fcf7-edee-33ba-4dc1c04ad324}" error="failed to connect to peer: df7d3529-fcf7-edee-33ba-4dc1c04ad324" -2025-01-25T14:29:46.696Z [INFO] server-966e830d-4b6e-96dd-70de-88075b220f08: starting snapshot up to: index=102 -2025-01-25T14:29:46.696Z [INFO] creating new snapshot: path=/tmp/raft408683412/snapshots/2-102-1737815386696.tmp -2025-01-25T14:29:46.699Z [INFO] server-966e830d-4b6e-96dd-70de-88075b220f08: compacting logs: from=1 to=92 -2025-01-25T14:29:46.699Z [INFO] server-966e830d-4b6e-96dd-70de-88075b220f08: snapshot complete up to: index=102 -2025-01-25T14:29:46.725Z [ERROR] server-966e830d-4b6e-96dd-70de-88075b220f08: failed to get log: index=2 error="log not found" -2025-01-25T14:29:46.726Z [INFO] creating new snapshot: path=/tmp/raft2294705873/snapshots/2-102-1737815386726.tmp -2025-01-25T14:29:46.726Z [INFO] server-df7d3529-fcf7-edee-33ba-4dc1c04ad324: snapshot network transfer progress: read-bytes=693 percent-complete=100.00% -2025-01-25T14:29:46.727Z [INFO] server-df7d3529-fcf7-edee-33ba-4dc1c04ad324: copied to local snapshot: bytes=693 -2025-01-25T14:29:46.727Z [INFO] server-df7d3529-fcf7-edee-33ba-4dc1c04ad324: snapshot restore progress: id=2-102-1737815386726 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% -2025-01-25T14:29:46.728Z [INFO] server-df7d3529-fcf7-edee-33ba-4dc1c04ad324: Installed remote snapshot -2025-01-25T14:29:46.728Z [INFO] server-966e830d-4b6e-96dd-70de-88075b220f08: aborting pipeline replication: peer="{Voter server-558d93a0-fba7-31db-1efc-5f3d1d067252 558d93a0-fba7-31db-1efc-5f3d1d067252}" ---- PASS: TestRaft_SendSnapshotFollower (0.34s) +2025-01-25T14:40:33.545Z [ERROR] server-b27defbd-0ce9-4a65-5da9-07f2e13933ea: failed to take snapshot: error="nothing new to snapshot" +2025-01-25T14:40:33.546Z [INFO] server-ebdcbc21-64ce-19f6-c15b-68f8dd07cc41: starting snapshot up to: index=101 +2025-01-25T14:40:33.546Z [INFO] creating new snapshot: path=/tmp/raft1776403662/snapshots/2-101-1737816033546.tmp +2025-01-25T14:40:33.550Z [INFO] server-ebdcbc21-64ce-19f6-c15b-68f8dd07cc41: compacting logs: from=1 to=92 +2025-01-25T14:40:33.550Z [INFO] server-ebdcbc21-64ce-19f6-c15b-68f8dd07cc41: snapshot complete up to: index=101 +2025-01-25T14:40:33.550Z [INFO] server-5f86487b-bc80-abb5-2fbd-0dc5ffdb0634: starting snapshot up to: index=102 +2025-01-25T14:40:33.550Z [INFO] creating new snapshot: path=/tmp/raft162098002/snapshots/2-102-1737816033550.tmp +2025-01-25T14:40:33.553Z [INFO] server-5f86487b-bc80-abb5-2fbd-0dc5ffdb0634: compacting logs: from=1 to=92 +2025-01-25T14:40:33.553Z [INFO] server-5f86487b-bc80-abb5-2fbd-0dc5ffdb0634: snapshot complete up to: index=102 +2025-01-25T14:40:33.562Z [ERROR] server-5f86487b-bc80-abb5-2fbd-0dc5ffdb0634: failed to get log: index=2 error="log not found" +2025-01-25T14:40:33.564Z [INFO] creating new snapshot: path=/tmp/raft3077626271/snapshots/2-102-1737816033564.tmp +2025-01-25T14:40:33.566Z [INFO] server-b27defbd-0ce9-4a65-5da9-07f2e13933ea: snapshot network transfer progress: read-bytes=693 percent-complete=100.00% +2025-01-25T14:40:33.568Z [INFO] server-b27defbd-0ce9-4a65-5da9-07f2e13933ea: copied to local snapshot: bytes=693 +2025-01-25T14:40:33.570Z [INFO] server-b27defbd-0ce9-4a65-5da9-07f2e13933ea: snapshot restore progress: id=2-102-1737816033564 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% +2025-01-25T14:40:33.570Z [ERROR] server-b27defbd-0ce9-4a65-5da9-07f2e13933ea: failed to restore snapshot: error="raft is already shutdown" +2025-01-25T14:40:33.571Z [INFO] server-5f86487b-bc80-abb5-2fbd-0dc5ffdb0634: aborting pipeline replication: peer="{Voter server-ebdcbc21-64ce-19f6-c15b-68f8dd07cc41 ebdcbc21-64ce-19f6-c15b-68f8dd07cc41}" +2025-01-25T14:40:33.571Z [ERROR] server-5f86487b-bc80-abb5-2fbd-0dc5ffdb0634: failed to install snapshot: id=2-102-1737816033550 error="raft is already shutdown" +2025-01-25T14:40:33.571Z [ERROR] server-5f86487b-bc80-abb5-2fbd-0dc5ffdb0634: failed to send snapshot to: peer="{Voter server-b27defbd-0ce9-4a65-5da9-07f2e13933ea b27defbd-0ce9-4a65-5da9-07f2e13933ea}" error="raft is already shutdown" +2025-01-25T14:40:34.068Z [ERROR] server-5f86487b-bc80-abb5-2fbd-0dc5ffdb0634: failed to heartbeat to: peer=b27defbd-0ce9-4a65-5da9-07f2e13933ea backoff time=10ms error="command timed out" +--- PASS: TestRaft_SendSnapshotFollower (0.83s) === RUN TestRaft_SendSnapshotAndLogsFollower -2025-01-25T14:29:46.732Z [INFO] server-3ac21012-71e3-68a6-a999-f85fbd9242ad: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3ac21012-71e3-68a6-a999-f85fbd9242ad Address:3ac21012-71e3-68a6-a999-f85fbd9242ad} {Suffrage:Voter ID:server-c62c5dcf-3dac-4404-82e6-619ff7dba9f8 Address:c62c5dcf-3dac-4404-82e6-619ff7dba9f8} {Suffrage:Voter ID:server-16af2563-94ff-cd32-6c94-112692f1aa3d Address:16af2563-94ff-cd32-6c94-112692f1aa3d}]" -2025-01-25T14:29:46.732Z [INFO] server-3ac21012-71e3-68a6-a999-f85fbd9242ad: entering follower state: follower="Node at 3ac21012-71e3-68a6-a999-f85fbd9242ad [Follower]" leader-address= leader-id= -2025-01-25T14:29:46.732Z [INFO] server-c62c5dcf-3dac-4404-82e6-619ff7dba9f8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3ac21012-71e3-68a6-a999-f85fbd9242ad Address:3ac21012-71e3-68a6-a999-f85fbd9242ad} {Suffrage:Voter ID:server-c62c5dcf-3dac-4404-82e6-619ff7dba9f8 Address:c62c5dcf-3dac-4404-82e6-619ff7dba9f8} {Suffrage:Voter ID:server-16af2563-94ff-cd32-6c94-112692f1aa3d Address:16af2563-94ff-cd32-6c94-112692f1aa3d}]" -2025-01-25T14:29:46.732Z [INFO] server-c62c5dcf-3dac-4404-82e6-619ff7dba9f8: entering follower state: follower="Node at c62c5dcf-3dac-4404-82e6-619ff7dba9f8 [Follower]" leader-address= leader-id= -2025-01-25T14:29:46.732Z [INFO] server-16af2563-94ff-cd32-6c94-112692f1aa3d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3ac21012-71e3-68a6-a999-f85fbd9242ad Address:3ac21012-71e3-68a6-a999-f85fbd9242ad} {Suffrage:Voter ID:server-c62c5dcf-3dac-4404-82e6-619ff7dba9f8 Address:c62c5dcf-3dac-4404-82e6-619ff7dba9f8} {Suffrage:Voter ID:server-16af2563-94ff-cd32-6c94-112692f1aa3d Address:16af2563-94ff-cd32-6c94-112692f1aa3d}]" -2025-01-25T14:29:46.732Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:29:46.732Z [INFO] server-16af2563-94ff-cd32-6c94-112692f1aa3d: entering follower state: follower="Node at 16af2563-94ff-cd32-6c94-112692f1aa3d [Follower]" leader-address= leader-id= -2025-01-25T14:29:46.795Z [WARN] server-16af2563-94ff-cd32-6c94-112692f1aa3d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:46.795Z [INFO] server-16af2563-94ff-cd32-6c94-112692f1aa3d: entering candidate state: node="Node at 16af2563-94ff-cd32-6c94-112692f1aa3d [Candidate]" term=2 -2025-01-25T14:29:46.796Z [INFO] server-16af2563-94ff-cd32-6c94-112692f1aa3d: election won: term=2 tally=2 -2025-01-25T14:29:46.796Z [INFO] server-16af2563-94ff-cd32-6c94-112692f1aa3d: entering leader state: leader="Node at 16af2563-94ff-cd32-6c94-112692f1aa3d [Leader]" -2025-01-25T14:29:46.796Z [INFO] server-16af2563-94ff-cd32-6c94-112692f1aa3d: added peer, starting replication: peer=server-3ac21012-71e3-68a6-a999-f85fbd9242ad -2025-01-25T14:29:46.796Z [INFO] server-16af2563-94ff-cd32-6c94-112692f1aa3d: added peer, starting replication: peer=server-c62c5dcf-3dac-4404-82e6-619ff7dba9f8 -2025-01-25T14:29:46.796Z [INFO] server-16af2563-94ff-cd32-6c94-112692f1aa3d: pipelining replication: peer="{Voter server-3ac21012-71e3-68a6-a999-f85fbd9242ad 3ac21012-71e3-68a6-a999-f85fbd9242ad}" -2025-01-25T14:29:46.796Z [INFO] server-16af2563-94ff-cd32-6c94-112692f1aa3d: pipelining replication: peer="{Voter server-c62c5dcf-3dac-4404-82e6-619ff7dba9f8 c62c5dcf-3dac-4404-82e6-619ff7dba9f8}" -2025-01-25T14:29:46.901Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:29:46.796246038 +0000 UTC m=+96.650571020 (2 nodes), 63.650909ms from start of poll, 64.317965ms from cluster start. Timeout at 2025-01-25 14:29:46.901251478 +0000 UTC m=+96.755576460, 105.00544ms after stability -2025-01-25T14:29:46.901Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:47.007Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:46.901430973 +0000 UTC m=+96.755755915 (1 nodes), 2.12µs from start of poll, 169.50286ms from cluster start. Timeout at 2025-01-25 14:29:47.006659445 +0000 UTC m=+96.860984387, 105.228472ms after stability -2025-01-25T14:29:47.007Z [ERROR] server-16af2563-94ff-cd32-6c94-112692f1aa3d: failed to pipeline appendEntries: peer="{Voter server-3ac21012-71e3-68a6-a999-f85fbd9242ad 3ac21012-71e3-68a6-a999-f85fbd9242ad}" error="append pipeline closed" -2025-01-25T14:29:47.007Z [INFO] server-16af2563-94ff-cd32-6c94-112692f1aa3d: aborting pipeline replication: peer="{Voter server-3ac21012-71e3-68a6-a999-f85fbd9242ad 3ac21012-71e3-68a6-a999-f85fbd9242ad}" -2025-01-25T14:29:47.007Z [ERROR] server-16af2563-94ff-cd32-6c94-112692f1aa3d: failed to appendEntries to: peer="{Voter server-3ac21012-71e3-68a6-a999-f85fbd9242ad 3ac21012-71e3-68a6-a999-f85fbd9242ad}" error="failed to connect to peer: 3ac21012-71e3-68a6-a999-f85fbd9242ad" -2025-01-25T14:29:47.009Z [ERROR] server-16af2563-94ff-cd32-6c94-112692f1aa3d: failed to heartbeat to: peer=3ac21012-71e3-68a6-a999-f85fbd9242ad backoff time=10ms error="failed to connect to peer: 3ac21012-71e3-68a6-a999-f85fbd9242ad" +2025-01-25T14:40:34.078Z [INFO] server-d3ddde7c-8a7e-7411-71bc-d92aa5a663c1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d3ddde7c-8a7e-7411-71bc-d92aa5a663c1 Address:d3ddde7c-8a7e-7411-71bc-d92aa5a663c1} {Suffrage:Voter ID:server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1 Address:ea5e3072-f5f5-198d-4167-29ecc4eb22d1} {Suffrage:Voter ID:server-c1a91cd1-dd7b-3eec-9995-19216a926acd Address:c1a91cd1-dd7b-3eec-9995-19216a926acd}]" +2025-01-25T14:40:34.079Z [INFO] server-d3ddde7c-8a7e-7411-71bc-d92aa5a663c1: entering follower state: follower="Node at d3ddde7c-8a7e-7411-71bc-d92aa5a663c1 [Follower]" leader-address= leader-id= +2025-01-25T14:40:34.079Z [INFO] server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d3ddde7c-8a7e-7411-71bc-d92aa5a663c1 Address:d3ddde7c-8a7e-7411-71bc-d92aa5a663c1} {Suffrage:Voter ID:server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1 Address:ea5e3072-f5f5-198d-4167-29ecc4eb22d1} {Suffrage:Voter ID:server-c1a91cd1-dd7b-3eec-9995-19216a926acd Address:c1a91cd1-dd7b-3eec-9995-19216a926acd}]" +2025-01-25T14:40:34.079Z [INFO] server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1: entering follower state: follower="Node at ea5e3072-f5f5-198d-4167-29ecc4eb22d1 [Follower]" leader-address= leader-id= +2025-01-25T14:40:34.080Z [INFO] server-c1a91cd1-dd7b-3eec-9995-19216a926acd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d3ddde7c-8a7e-7411-71bc-d92aa5a663c1 Address:d3ddde7c-8a7e-7411-71bc-d92aa5a663c1} {Suffrage:Voter ID:server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1 Address:ea5e3072-f5f5-198d-4167-29ecc4eb22d1} {Suffrage:Voter ID:server-c1a91cd1-dd7b-3eec-9995-19216a926acd Address:c1a91cd1-dd7b-3eec-9995-19216a926acd}]" +2025-01-25T14:40:34.080Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:40:34.081Z [INFO] server-c1a91cd1-dd7b-3eec-9995-19216a926acd: entering follower state: follower="Node at c1a91cd1-dd7b-3eec-9995-19216a926acd [Follower]" leader-address= leader-id= +2025-01-25T14:40:34.135Z [WARN] server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:34.135Z [INFO] server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1: entering candidate state: node="Node at ea5e3072-f5f5-198d-4167-29ecc4eb22d1 [Candidate]" term=2 +2025-01-25T14:40:34.147Z [INFO] server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1: election won: term=2 tally=2 +2025-01-25T14:40:34.147Z [INFO] server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1: entering leader state: leader="Node at ea5e3072-f5f5-198d-4167-29ecc4eb22d1 [Leader]" +2025-01-25T14:40:34.147Z [INFO] server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1: added peer, starting replication: peer=server-d3ddde7c-8a7e-7411-71bc-d92aa5a663c1 +2025-01-25T14:40:34.147Z [INFO] server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1: added peer, starting replication: peer=server-c1a91cd1-dd7b-3eec-9995-19216a926acd +2025-01-25T14:40:34.148Z [INFO] server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1: pipelining replication: peer="{Voter server-d3ddde7c-8a7e-7411-71bc-d92aa5a663c1 d3ddde7c-8a7e-7411-71bc-d92aa5a663c1}" +2025-01-25T14:40:34.149Z [WARN] server-c1a91cd1-dd7b-3eec-9995-19216a926acd: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:34.149Z [INFO] server-c1a91cd1-dd7b-3eec-9995-19216a926acd: entering candidate state: node="Node at c1a91cd1-dd7b-3eec-9995-19216a926acd [Candidate]" term=2 +2025-01-25T14:40:34.149Z [INFO] server-c1a91cd1-dd7b-3eec-9995-19216a926acd: duplicate requestVote for same term: term=2 +2025-01-25T14:40:34.149Z [INFO] server-c1a91cd1-dd7b-3eec-9995-19216a926acd: entering follower state: follower="Node at c1a91cd1-dd7b-3eec-9995-19216a926acd [Follower]" leader-address=ea5e3072-f5f5-198d-4167-29ecc4eb22d1 leader-id=server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1 +2025-01-25T14:40:34.150Z [WARN] server-d3ddde7c-8a7e-7411-71bc-d92aa5a663c1: rejecting vote request since we have a leader: from=c1a91cd1-dd7b-3eec-9995-19216a926acd leader=ea5e3072-f5f5-198d-4167-29ecc4eb22d1 leader-id=server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1 +2025-01-25T14:40:34.150Z [WARN] server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1: rejecting vote request since we have a leader: from=c1a91cd1-dd7b-3eec-9995-19216a926acd leader=ea5e3072-f5f5-198d-4167-29ecc4eb22d1 leader-id=server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1 +2025-01-25T14:40:34.150Z [INFO] server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1: pipelining replication: peer="{Voter server-c1a91cd1-dd7b-3eec-9995-19216a926acd c1a91cd1-dd7b-3eec-9995-19216a926acd}" +2025-01-25T14:40:34.255Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:40:34.150589838 +0000 UTC m=+103.196429156 (2 nodes), 69.505286ms from start of poll, 72.342044ms from cluster start. Timeout at 2025-01-25 14:40:34.25560938 +0000 UTC m=+103.301448697, 105.019541ms after stability +2025-01-25T14:40:34.256Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:34.362Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:34.256124187 +0000 UTC m=+103.301963505 (1 nodes), 4.667µs from start of poll, 177.876393ms from cluster start. Timeout at 2025-01-25 14:40:34.361279938 +0000 UTC m=+103.407119255, 105.15575ms after stability +2025-01-25T14:40:34.363Z [ERROR] server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1: failed to pipeline appendEntries: peer="{Voter server-d3ddde7c-8a7e-7411-71bc-d92aa5a663c1 d3ddde7c-8a7e-7411-71bc-d92aa5a663c1}" error="append pipeline closed" +2025-01-25T14:40:34.363Z [INFO] server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1: aborting pipeline replication: peer="{Voter server-d3ddde7c-8a7e-7411-71bc-d92aa5a663c1 d3ddde7c-8a7e-7411-71bc-d92aa5a663c1}" +2025-01-25T14:40:34.363Z [ERROR] server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1: failed to appendEntries to: peer="{Voter server-d3ddde7c-8a7e-7411-71bc-d92aa5a663c1 d3ddde7c-8a7e-7411-71bc-d92aa5a663c1}" error="failed to connect to peer: d3ddde7c-8a7e-7411-71bc-d92aa5a663c1" +2025-01-25T14:40:34.363Z [ERROR] server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1: failed to heartbeat to: peer=d3ddde7c-8a7e-7411-71bc-d92aa5a663c1 backoff time=10ms error="failed to connect to peer: d3ddde7c-8a7e-7411-71bc-d92aa5a663c1" +2025-01-25T14:40:34.374Z [ERROR] server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1: failed to appendEntries to: peer="{Voter server-d3ddde7c-8a7e-7411-71bc-d92aa5a663c1 d3ddde7c-8a7e-7411-71bc-d92aa5a663c1}" error="failed to connect to peer: d3ddde7c-8a7e-7411-71bc-d92aa5a663c1" +2025-01-25T14:40:34.383Z [ERROR] server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1: failed to heartbeat to: peer=d3ddde7c-8a7e-7411-71bc-d92aa5a663c1 backoff time=10ms error="failed to connect to peer: d3ddde7c-8a7e-7411-71bc-d92aa5a663c1" +2025-01-25T14:40:34.384Z [ERROR] server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1: failed to appendEntries to: peer="{Voter server-d3ddde7c-8a7e-7411-71bc-d92aa5a663c1 d3ddde7c-8a7e-7411-71bc-d92aa5a663c1}" error="failed to connect to peer: d3ddde7c-8a7e-7411-71bc-d92aa5a663c1" raft_test.go:1669: [INFO] Finished apply without behind follower -2025-01-25T14:29:47.018Z [ERROR] server-3ac21012-71e3-68a6-a999-f85fbd9242ad: failed to take snapshot: error="nothing new to snapshot" -2025-01-25T14:29:47.018Z [INFO] server-c62c5dcf-3dac-4404-82e6-619ff7dba9f8: starting snapshot up to: index=97 -2025-01-25T14:29:47.018Z [INFO] creating new snapshot: path=/tmp/raft2276402575/snapshots/2-97-1737815387018.tmp -2025-01-25T14:29:47.018Z [ERROR] server-16af2563-94ff-cd32-6c94-112692f1aa3d: failed to appendEntries to: peer="{Voter server-3ac21012-71e3-68a6-a999-f85fbd9242ad 3ac21012-71e3-68a6-a999-f85fbd9242ad}" error="failed to connect to peer: 3ac21012-71e3-68a6-a999-f85fbd9242ad" -2025-01-25T14:29:47.019Z [INFO] server-c62c5dcf-3dac-4404-82e6-619ff7dba9f8: compacting logs: from=1 to=92 -2025-01-25T14:29:47.019Z [INFO] server-c62c5dcf-3dac-4404-82e6-619ff7dba9f8: snapshot complete up to: index=97 -2025-01-25T14:29:47.019Z [INFO] server-16af2563-94ff-cd32-6c94-112692f1aa3d: starting snapshot up to: index=102 -2025-01-25T14:29:47.020Z [INFO] creating new snapshot: path=/tmp/raft2369143322/snapshots/2-102-1737815387019.tmp -2025-01-25T14:29:47.021Z [INFO] server-16af2563-94ff-cd32-6c94-112692f1aa3d: compacting logs: from=1 to=92 -2025-01-25T14:29:47.021Z [INFO] server-16af2563-94ff-cd32-6c94-112692f1aa3d: snapshot complete up to: index=102 -2025-01-25T14:29:47.035Z [ERROR] server-16af2563-94ff-cd32-6c94-112692f1aa3d: failed to heartbeat to: peer=3ac21012-71e3-68a6-a999-f85fbd9242ad backoff time=10ms error="failed to connect to peer: 3ac21012-71e3-68a6-a999-f85fbd9242ad" -2025-01-25T14:29:47.038Z [ERROR] server-16af2563-94ff-cd32-6c94-112692f1aa3d: failed to get log: index=2 error="log not found" +2025-01-25T14:40:34.387Z [ERROR] server-d3ddde7c-8a7e-7411-71bc-d92aa5a663c1: failed to take snapshot: error="nothing new to snapshot" +2025-01-25T14:40:34.388Z [INFO] server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1: starting snapshot up to: index=102 +2025-01-25T14:40:34.388Z [INFO] creating new snapshot: path=/tmp/raft1153411593/snapshots/2-102-1737816034388.tmp +2025-01-25T14:40:34.393Z [INFO] server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1: compacting logs: from=1 to=92 +2025-01-25T14:40:34.393Z [INFO] server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1: snapshot complete up to: index=102 +2025-01-25T14:40:34.393Z [INFO] server-c1a91cd1-dd7b-3eec-9995-19216a926acd: starting snapshot up to: index=95 +2025-01-25T14:40:34.393Z [INFO] creating new snapshot: path=/tmp/raft3021658089/snapshots/2-95-1737816034393.tmp +2025-01-25T14:40:34.397Z [INFO] server-c1a91cd1-dd7b-3eec-9995-19216a926acd: compacting logs: from=1 to=92 +2025-01-25T14:40:34.397Z [INFO] server-c1a91cd1-dd7b-3eec-9995-19216a926acd: snapshot complete up to: index=95 +2025-01-25T14:40:34.402Z [ERROR] server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1: failed to heartbeat to: peer=d3ddde7c-8a7e-7411-71bc-d92aa5a663c1 backoff time=10ms error="failed to connect to peer: d3ddde7c-8a7e-7411-71bc-d92aa5a663c1" +2025-01-25T14:40:34.404Z [ERROR] server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1: failed to get log: index=2 error="log not found" +2025-01-25T14:40:34.417Z [WARN] server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1: failed to contact: server-id=server-d3ddde7c-8a7e-7411-71bc-d92aa5a663c1 time=57.713168ms +2025-01-25T14:40:34.427Z [ERROR] server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1: failed to install snapshot: id=2-102-1737816034388 error="failed to connect to peer: d3ddde7c-8a7e-7411-71bc-d92aa5a663c1" +2025-01-25T14:40:34.427Z [ERROR] server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1: failed to send snapshot to: peer="{Voter server-d3ddde7c-8a7e-7411-71bc-d92aa5a663c1 d3ddde7c-8a7e-7411-71bc-d92aa5a663c1}" error="failed to connect to peer: d3ddde7c-8a7e-7411-71bc-d92aa5a663c1" +2025-01-25T14:40:34.435Z [ERROR] server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1: failed to heartbeat to: peer=d3ddde7c-8a7e-7411-71bc-d92aa5a663c1 backoff time=20ms error="failed to connect to peer: d3ddde7c-8a7e-7411-71bc-d92aa5a663c1" raft_test.go:1690: [INFO] Finished apply without behind follower -2025-01-25T14:29:47.041Z [INFO] creating new snapshot: path=/tmp/raft2386578095/snapshots/2-102-1737815387041.tmp -2025-01-25T14:29:47.041Z [INFO] server-3ac21012-71e3-68a6-a999-f85fbd9242ad: snapshot network transfer progress: read-bytes=693 percent-complete=100.00% -2025-01-25T14:29:47.043Z [INFO] server-3ac21012-71e3-68a6-a999-f85fbd9242ad: copied to local snapshot: bytes=693 -2025-01-25T14:29:47.043Z [INFO] server-3ac21012-71e3-68a6-a999-f85fbd9242ad: snapshot restore progress: id=2-102-1737815387041 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% -2025-01-25T14:29:47.044Z [INFO] server-3ac21012-71e3-68a6-a999-f85fbd9242ad: Installed remote snapshot -2025-01-25T14:29:47.044Z [INFO] server-16af2563-94ff-cd32-6c94-112692f1aa3d: pipelining replication: peer="{Voter server-3ac21012-71e3-68a6-a999-f85fbd9242ad 3ac21012-71e3-68a6-a999-f85fbd9242ad}" -2025-01-25T14:29:47.052Z [INFO] server-16af2563-94ff-cd32-6c94-112692f1aa3d: aborting pipeline replication: peer="{Voter server-c62c5dcf-3dac-4404-82e6-619ff7dba9f8 c62c5dcf-3dac-4404-82e6-619ff7dba9f8}" -2025-01-25T14:29:47.052Z [INFO] server-16af2563-94ff-cd32-6c94-112692f1aa3d: aborting pipeline replication: peer="{Voter server-3ac21012-71e3-68a6-a999-f85fbd9242ad 3ac21012-71e3-68a6-a999-f85fbd9242ad}" ---- PASS: TestRaft_SendSnapshotAndLogsFollower (0.32s) +2025-01-25T14:40:34.459Z [WARN] server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1: failed to contact: server-id=server-d3ddde7c-8a7e-7411-71bc-d92aa5a663c1 time=99.478498ms +2025-01-25T14:40:34.469Z [ERROR] server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1: failed to get log: index=2 error="log not found" +2025-01-25T14:40:34.471Z [INFO] creating new snapshot: path=/tmp/raft1638327693/snapshots/2-102-1737816034471.tmp +2025-01-25T14:40:34.472Z [INFO] server-d3ddde7c-8a7e-7411-71bc-d92aa5a663c1: snapshot network transfer progress: read-bytes=693 percent-complete=100.00% +2025-01-25T14:40:34.474Z [INFO] server-d3ddde7c-8a7e-7411-71bc-d92aa5a663c1: copied to local snapshot: bytes=693 +2025-01-25T14:40:34.475Z [INFO] server-d3ddde7c-8a7e-7411-71bc-d92aa5a663c1: snapshot restore progress: id=2-102-1737816034471 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% +2025-01-25T14:40:34.476Z [INFO] server-d3ddde7c-8a7e-7411-71bc-d92aa5a663c1: Installed remote snapshot +2025-01-25T14:40:34.477Z [INFO] server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1: pipelining replication: peer="{Voter server-d3ddde7c-8a7e-7411-71bc-d92aa5a663c1 d3ddde7c-8a7e-7411-71bc-d92aa5a663c1}" +2025-01-25T14:40:34.480Z [INFO] server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1: aborting pipeline replication: peer="{Voter server-d3ddde7c-8a7e-7411-71bc-d92aa5a663c1 d3ddde7c-8a7e-7411-71bc-d92aa5a663c1}" +2025-01-25T14:40:34.481Z [INFO] server-ea5e3072-f5f5-198d-4167-29ecc4eb22d1: aborting pipeline replication: peer="{Voter server-c1a91cd1-dd7b-3eec-9995-19216a926acd c1a91cd1-dd7b-3eec-9995-19216a926acd}" +--- PASS: TestRaft_SendSnapshotAndLogsFollower (0.41s) === RUN TestRaft_ReJoinFollower -2025-01-25T14:29:47.054Z [INFO] server-a8d59acf-d76b-52ee-086a-cdd42ce1445f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a8d59acf-d76b-52ee-086a-cdd42ce1445f Address:a8d59acf-d76b-52ee-086a-cdd42ce1445f} {Suffrage:Voter ID:server-e2404cea-3363-8a08-25b1-5ab57c105b86 Address:e2404cea-3363-8a08-25b1-5ab57c105b86} {Suffrage:Voter ID:server-4c6a32da-d560-903c-980f-ddd2a15bfa6d Address:4c6a32da-d560-903c-980f-ddd2a15bfa6d}]" -2025-01-25T14:29:47.055Z [INFO] server-a8d59acf-d76b-52ee-086a-cdd42ce1445f: entering follower state: follower="Node at a8d59acf-d76b-52ee-086a-cdd42ce1445f [Follower]" leader-address= leader-id= -2025-01-25T14:29:47.055Z [INFO] server-e2404cea-3363-8a08-25b1-5ab57c105b86: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a8d59acf-d76b-52ee-086a-cdd42ce1445f Address:a8d59acf-d76b-52ee-086a-cdd42ce1445f} {Suffrage:Voter ID:server-e2404cea-3363-8a08-25b1-5ab57c105b86 Address:e2404cea-3363-8a08-25b1-5ab57c105b86} {Suffrage:Voter ID:server-4c6a32da-d560-903c-980f-ddd2a15bfa6d Address:4c6a32da-d560-903c-980f-ddd2a15bfa6d}]" -2025-01-25T14:29:47.055Z [INFO] server-e2404cea-3363-8a08-25b1-5ab57c105b86: entering follower state: follower="Node at e2404cea-3363-8a08-25b1-5ab57c105b86 [Follower]" leader-address= leader-id= -2025-01-25T14:29:47.055Z [INFO] server-4c6a32da-d560-903c-980f-ddd2a15bfa6d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a8d59acf-d76b-52ee-086a-cdd42ce1445f Address:a8d59acf-d76b-52ee-086a-cdd42ce1445f} {Suffrage:Voter ID:server-e2404cea-3363-8a08-25b1-5ab57c105b86 Address:e2404cea-3363-8a08-25b1-5ab57c105b86} {Suffrage:Voter ID:server-4c6a32da-d560-903c-980f-ddd2a15bfa6d Address:4c6a32da-d560-903c-980f-ddd2a15bfa6d}]" -2025-01-25T14:29:47.055Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:47.055Z [INFO] server-4c6a32da-d560-903c-980f-ddd2a15bfa6d: entering follower state: follower="Node at 4c6a32da-d560-903c-980f-ddd2a15bfa6d [Follower]" leader-address= leader-id= -2025-01-25T14:29:47.108Z [WARN] server-a8d59acf-d76b-52ee-086a-cdd42ce1445f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:47.108Z [INFO] server-a8d59acf-d76b-52ee-086a-cdd42ce1445f: entering candidate state: node="Node at a8d59acf-d76b-52ee-086a-cdd42ce1445f [Candidate]" term=2 -2025-01-25T14:29:47.108Z [INFO] server-a8d59acf-d76b-52ee-086a-cdd42ce1445f: election won: term=2 tally=2 -2025-01-25T14:29:47.108Z [INFO] server-a8d59acf-d76b-52ee-086a-cdd42ce1445f: entering leader state: leader="Node at a8d59acf-d76b-52ee-086a-cdd42ce1445f [Leader]" -2025-01-25T14:29:47.108Z [INFO] server-a8d59acf-d76b-52ee-086a-cdd42ce1445f: added peer, starting replication: peer=server-e2404cea-3363-8a08-25b1-5ab57c105b86 -2025-01-25T14:29:47.108Z [INFO] server-a8d59acf-d76b-52ee-086a-cdd42ce1445f: added peer, starting replication: peer=server-4c6a32da-d560-903c-980f-ddd2a15bfa6d -2025-01-25T14:29:47.109Z [INFO] server-a8d59acf-d76b-52ee-086a-cdd42ce1445f: pipelining replication: peer="{Voter server-4c6a32da-d560-903c-980f-ddd2a15bfa6d 4c6a32da-d560-903c-980f-ddd2a15bfa6d}" -2025-01-25T14:29:47.109Z [INFO] server-a8d59acf-d76b-52ee-086a-cdd42ce1445f: pipelining replication: peer="{Voter server-e2404cea-3363-8a08-25b1-5ab57c105b86 e2404cea-3363-8a08-25b1-5ab57c105b86}" -2025-01-25T14:29:47.215Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:47.108916186 +0000 UTC m=+96.963241168 (1 nodes), 53.022492ms from start of poll, 54.104773ms from cluster start. Timeout at 2025-01-25 14:29:47.213931666 +0000 UTC m=+97.068256608, 105.01544ms after stability -2025-01-25T14:29:47.220Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:29:47.327Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:29:47.220946715 +0000 UTC m=+97.075271697 (2 nodes), 2.24µs from start of poll, 166.135302ms from cluster start. Timeout at 2025-01-25 14:29:47.326166147 +0000 UTC m=+97.180491129, 105.219432ms after stability -2025-01-25T14:29:47.327Z [INFO] server-a8d59acf-d76b-52ee-086a-cdd42ce1445f: updating configuration: command=RemoveServer server-id=server-e2404cea-3363-8a08-25b1-5ab57c105b86 server-addr= servers="[{Suffrage:Voter ID:server-a8d59acf-d76b-52ee-086a-cdd42ce1445f Address:a8d59acf-d76b-52ee-086a-cdd42ce1445f} {Suffrage:Voter ID:server-4c6a32da-d560-903c-980f-ddd2a15bfa6d Address:4c6a32da-d560-903c-980f-ddd2a15bfa6d}]" -2025-01-25T14:29:47.327Z [INFO] server-a8d59acf-d76b-52ee-086a-cdd42ce1445f: removed peer, stopping replication: peer=server-e2404cea-3363-8a08-25b1-5ab57c105b86 last-index=3 -2025-01-25T14:29:47.327Z [INFO] server-a8d59acf-d76b-52ee-086a-cdd42ce1445f: aborting pipeline replication: peer="{Voter server-e2404cea-3363-8a08-25b1-5ab57c105b86 e2404cea-3363-8a08-25b1-5ab57c105b86}" -2025-01-25T14:29:47.433Z [INFO] server-a8d59acf-d76b-52ee-086a-cdd42ce1445f: updating configuration: command=AddVoter server-id=server-e2404cea-3363-8a08-25b1-5ab57c105b86 server-addr=e2404cea-3363-8a08-25b1-5ab57c105b86 servers="[{Suffrage:Voter ID:server-a8d59acf-d76b-52ee-086a-cdd42ce1445f Address:a8d59acf-d76b-52ee-086a-cdd42ce1445f} {Suffrage:Voter ID:server-4c6a32da-d560-903c-980f-ddd2a15bfa6d Address:4c6a32da-d560-903c-980f-ddd2a15bfa6d} {Suffrage:Voter ID:server-e2404cea-3363-8a08-25b1-5ab57c105b86 Address:e2404cea-3363-8a08-25b1-5ab57c105b86}]" -2025-01-25T14:29:47.433Z [INFO] server-a8d59acf-d76b-52ee-086a-cdd42ce1445f: added peer, starting replication: peer=server-e2404cea-3363-8a08-25b1-5ab57c105b86 -2025-01-25T14:29:47.433Z [WARN] server-e2404cea-3363-8a08-25b1-5ab57c105b86: failed to get previous log: previous-index=4 last-index=3 error="log not found" -2025-01-25T14:29:47.434Z [WARN] server-a8d59acf-d76b-52ee-086a-cdd42ce1445f: appendEntries rejected, sending older logs: peer="{Voter server-e2404cea-3363-8a08-25b1-5ab57c105b86 e2404cea-3363-8a08-25b1-5ab57c105b86}" next=4 -2025-01-25T14:29:47.434Z [INFO] server-a8d59acf-d76b-52ee-086a-cdd42ce1445f: pipelining replication: peer="{Voter server-e2404cea-3363-8a08-25b1-5ab57c105b86 e2404cea-3363-8a08-25b1-5ab57c105b86}" -2025-01-25T14:29:47.434Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:47.540Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:47.434352795 +0000 UTC m=+97.288677777 (1 nodes), 1.12µs from start of poll, 379.541382ms from cluster start. Timeout at 2025-01-25 14:29:47.539541948 +0000 UTC m=+97.393866930, 105.189153ms after stability -2025-01-25T14:29:47.540Z [INFO] server-a8d59acf-d76b-52ee-086a-cdd42ce1445f: aborting pipeline replication: peer="{Voter server-4c6a32da-d560-903c-980f-ddd2a15bfa6d 4c6a32da-d560-903c-980f-ddd2a15bfa6d}" -2025-01-25T14:29:47.540Z [INFO] server-a8d59acf-d76b-52ee-086a-cdd42ce1445f: aborting pipeline replication: peer="{Voter server-e2404cea-3363-8a08-25b1-5ab57c105b86 e2404cea-3363-8a08-25b1-5ab57c105b86}" ---- PASS: TestRaft_ReJoinFollower (0.49s) +2025-01-25T14:40:34.491Z [INFO] server-3e093765-f61e-f217-2b2d-73f323862093: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3e093765-f61e-f217-2b2d-73f323862093 Address:3e093765-f61e-f217-2b2d-73f323862093} {Suffrage:Voter ID:server-00e9bf5e-5fcc-3a37-fdac-b7c0d67b1f8a Address:00e9bf5e-5fcc-3a37-fdac-b7c0d67b1f8a} {Suffrage:Voter ID:server-2ea6ef68-09f4-506e-a435-e4233efdaa01 Address:2ea6ef68-09f4-506e-a435-e4233efdaa01}]" +2025-01-25T14:40:34.491Z [INFO] server-3e093765-f61e-f217-2b2d-73f323862093: entering follower state: follower="Node at 3e093765-f61e-f217-2b2d-73f323862093 [Follower]" leader-address= leader-id= +2025-01-25T14:40:34.491Z [INFO] server-00e9bf5e-5fcc-3a37-fdac-b7c0d67b1f8a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3e093765-f61e-f217-2b2d-73f323862093 Address:3e093765-f61e-f217-2b2d-73f323862093} {Suffrage:Voter ID:server-00e9bf5e-5fcc-3a37-fdac-b7c0d67b1f8a Address:00e9bf5e-5fcc-3a37-fdac-b7c0d67b1f8a} {Suffrage:Voter ID:server-2ea6ef68-09f4-506e-a435-e4233efdaa01 Address:2ea6ef68-09f4-506e-a435-e4233efdaa01}]" +2025-01-25T14:40:34.492Z [INFO] server-00e9bf5e-5fcc-3a37-fdac-b7c0d67b1f8a: entering follower state: follower="Node at 00e9bf5e-5fcc-3a37-fdac-b7c0d67b1f8a [Follower]" leader-address= leader-id= +2025-01-25T14:40:34.492Z [INFO] server-2ea6ef68-09f4-506e-a435-e4233efdaa01: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3e093765-f61e-f217-2b2d-73f323862093 Address:3e093765-f61e-f217-2b2d-73f323862093} {Suffrage:Voter ID:server-00e9bf5e-5fcc-3a37-fdac-b7c0d67b1f8a Address:00e9bf5e-5fcc-3a37-fdac-b7c0d67b1f8a} {Suffrage:Voter ID:server-2ea6ef68-09f4-506e-a435-e4233efdaa01 Address:2ea6ef68-09f4-506e-a435-e4233efdaa01}]" +2025-01-25T14:40:34.492Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:34.492Z [INFO] server-2ea6ef68-09f4-506e-a435-e4233efdaa01: entering follower state: follower="Node at 2ea6ef68-09f4-506e-a435-e4233efdaa01 [Follower]" leader-address= leader-id= +2025-01-25T14:40:34.584Z [WARN] server-2ea6ef68-09f4-506e-a435-e4233efdaa01: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:34.584Z [INFO] server-2ea6ef68-09f4-506e-a435-e4233efdaa01: entering candidate state: node="Node at 2ea6ef68-09f4-506e-a435-e4233efdaa01 [Candidate]" term=2 +2025-01-25T14:40:34.584Z [INFO] server-2ea6ef68-09f4-506e-a435-e4233efdaa01: election won: term=2 tally=2 +2025-01-25T14:40:34.584Z [INFO] server-2ea6ef68-09f4-506e-a435-e4233efdaa01: entering leader state: leader="Node at 2ea6ef68-09f4-506e-a435-e4233efdaa01 [Leader]" +2025-01-25T14:40:34.584Z [INFO] server-2ea6ef68-09f4-506e-a435-e4233efdaa01: added peer, starting replication: peer=server-3e093765-f61e-f217-2b2d-73f323862093 +2025-01-25T14:40:34.584Z [INFO] server-2ea6ef68-09f4-506e-a435-e4233efdaa01: added peer, starting replication: peer=server-00e9bf5e-5fcc-3a37-fdac-b7c0d67b1f8a +2025-01-25T14:40:34.585Z [INFO] server-2ea6ef68-09f4-506e-a435-e4233efdaa01: pipelining replication: peer="{Voter server-00e9bf5e-5fcc-3a37-fdac-b7c0d67b1f8a 00e9bf5e-5fcc-3a37-fdac-b7c0d67b1f8a}" +2025-01-25T14:40:34.585Z [INFO] server-2ea6ef68-09f4-506e-a435-e4233efdaa01: pipelining replication: peer="{Voter server-3e093765-f61e-f217-2b2d-73f323862093 3e093765-f61e-f217-2b2d-73f323862093}" +2025-01-25T14:40:34.690Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:34.584809103 +0000 UTC m=+103.630648420 (1 nodes), 91.746434ms from start of poll, 94.42715ms from cluster start. Timeout at 2025-01-25 14:40:34.689853145 +0000 UTC m=+103.735692463, 105.044043ms after stability +2025-01-25T14:40:34.695Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:40:34.800Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:40:34.695438258 +0000 UTC m=+103.741277576 (2 nodes), 7.292µs from start of poll, 205.056306ms from cluster start. Timeout at 2025-01-25 14:40:34.800640092 +0000 UTC m=+103.846479409, 105.201833ms after stability +2025-01-25T14:40:34.801Z [INFO] server-2ea6ef68-09f4-506e-a435-e4233efdaa01: updating configuration: command=RemoveServer server-id=server-3e093765-f61e-f217-2b2d-73f323862093 server-addr= servers="[{Suffrage:Voter ID:server-00e9bf5e-5fcc-3a37-fdac-b7c0d67b1f8a Address:00e9bf5e-5fcc-3a37-fdac-b7c0d67b1f8a} {Suffrage:Voter ID:server-2ea6ef68-09f4-506e-a435-e4233efdaa01 Address:2ea6ef68-09f4-506e-a435-e4233efdaa01}]" +2025-01-25T14:40:34.801Z [INFO] server-2ea6ef68-09f4-506e-a435-e4233efdaa01: removed peer, stopping replication: peer=server-3e093765-f61e-f217-2b2d-73f323862093 last-index=3 +2025-01-25T14:40:34.801Z [INFO] server-2ea6ef68-09f4-506e-a435-e4233efdaa01: aborting pipeline replication: peer="{Voter server-3e093765-f61e-f217-2b2d-73f323862093 3e093765-f61e-f217-2b2d-73f323862093}" +2025-01-25T14:40:34.858Z [WARN] server-3e093765-f61e-f217-2b2d-73f323862093: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +2025-01-25T14:40:34.907Z [INFO] server-2ea6ef68-09f4-506e-a435-e4233efdaa01: updating configuration: command=AddVoter server-id=server-3e093765-f61e-f217-2b2d-73f323862093 server-addr=3e093765-f61e-f217-2b2d-73f323862093 servers="[{Suffrage:Voter ID:server-00e9bf5e-5fcc-3a37-fdac-b7c0d67b1f8a Address:00e9bf5e-5fcc-3a37-fdac-b7c0d67b1f8a} {Suffrage:Voter ID:server-2ea6ef68-09f4-506e-a435-e4233efdaa01 Address:2ea6ef68-09f4-506e-a435-e4233efdaa01} {Suffrage:Voter ID:server-3e093765-f61e-f217-2b2d-73f323862093 Address:3e093765-f61e-f217-2b2d-73f323862093}]" +2025-01-25T14:40:34.908Z [INFO] server-2ea6ef68-09f4-506e-a435-e4233efdaa01: added peer, starting replication: peer=server-3e093765-f61e-f217-2b2d-73f323862093 +2025-01-25T14:40:34.908Z [WARN] server-3e093765-f61e-f217-2b2d-73f323862093: failed to get previous log: previous-index=4 last-index=3 error="log not found" +2025-01-25T14:40:34.908Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:34.909Z [WARN] server-2ea6ef68-09f4-506e-a435-e4233efdaa01: appendEntries rejected, sending older logs: peer="{Voter server-3e093765-f61e-f217-2b2d-73f323862093 3e093765-f61e-f217-2b2d-73f323862093}" next=4 +2025-01-25T14:40:34.909Z [INFO] server-2ea6ef68-09f4-506e-a435-e4233efdaa01: pipelining replication: peer="{Voter server-3e093765-f61e-f217-2b2d-73f323862093 3e093765-f61e-f217-2b2d-73f323862093}" +2025-01-25T14:40:35.014Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:34.90907588 +0000 UTC m=+103.954915198 (1 nodes), 3.5µs from start of poll, 418.693928ms from cluster start. Timeout at 2025-01-25 14:40:35.014326131 +0000 UTC m=+104.060165449, 105.250251ms after stability +2025-01-25T14:40:35.015Z [INFO] server-2ea6ef68-09f4-506e-a435-e4233efdaa01: aborting pipeline replication: peer="{Voter server-00e9bf5e-5fcc-3a37-fdac-b7c0d67b1f8a 00e9bf5e-5fcc-3a37-fdac-b7c0d67b1f8a}" +2025-01-25T14:40:35.015Z [INFO] server-2ea6ef68-09f4-506e-a435-e4233efdaa01: aborting pipeline replication: peer="{Voter server-3e093765-f61e-f217-2b2d-73f323862093 3e093765-f61e-f217-2b2d-73f323862093}" +--- PASS: TestRaft_ReJoinFollower (0.53s) === RUN TestRaft_LeaderLeaseExpire -2025-01-25T14:29:47.542Z [INFO] server-25ccac42-f31d-35a1-5318-6bc169312cec: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-25ccac42-f31d-35a1-5318-6bc169312cec Address:25ccac42-f31d-35a1-5318-6bc169312cec} {Suffrage:Voter ID:server-68c8e1c7-8678-32c2-504d-98f1683e16c6 Address:68c8e1c7-8678-32c2-504d-98f1683e16c6}]" -2025-01-25T14:29:47.542Z [INFO] server-68c8e1c7-8678-32c2-504d-98f1683e16c6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-25ccac42-f31d-35a1-5318-6bc169312cec Address:25ccac42-f31d-35a1-5318-6bc169312cec} {Suffrage:Voter ID:server-68c8e1c7-8678-32c2-504d-98f1683e16c6 Address:68c8e1c7-8678-32c2-504d-98f1683e16c6}]" -2025-01-25T14:29:47.542Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:47.542Z [INFO] server-25ccac42-f31d-35a1-5318-6bc169312cec: entering follower state: follower="Node at 25ccac42-f31d-35a1-5318-6bc169312cec [Follower]" leader-address= leader-id= -2025-01-25T14:29:47.542Z [INFO] server-68c8e1c7-8678-32c2-504d-98f1683e16c6: entering follower state: follower="Node at 68c8e1c7-8678-32c2-504d-98f1683e16c6 [Follower]" leader-address= leader-id= -2025-01-25T14:29:47.606Z [WARN] server-68c8e1c7-8678-32c2-504d-98f1683e16c6: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:47.606Z [INFO] server-68c8e1c7-8678-32c2-504d-98f1683e16c6: entering candidate state: node="Node at 68c8e1c7-8678-32c2-504d-98f1683e16c6 [Candidate]" term=2 -2025-01-25T14:29:47.606Z [INFO] server-68c8e1c7-8678-32c2-504d-98f1683e16c6: election won: term=2 tally=2 -2025-01-25T14:29:47.606Z [INFO] server-68c8e1c7-8678-32c2-504d-98f1683e16c6: entering leader state: leader="Node at 68c8e1c7-8678-32c2-504d-98f1683e16c6 [Leader]" -2025-01-25T14:29:47.606Z [INFO] server-68c8e1c7-8678-32c2-504d-98f1683e16c6: added peer, starting replication: peer=server-25ccac42-f31d-35a1-5318-6bc169312cec -2025-01-25T14:29:47.606Z [INFO] server-68c8e1c7-8678-32c2-504d-98f1683e16c6: pipelining replication: peer="{Voter server-25ccac42-f31d-35a1-5318-6bc169312cec 25ccac42-f31d-35a1-5318-6bc169312cec}" -2025-01-25T14:29:47.711Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:47.606626155 +0000 UTC m=+97.460951137 (1 nodes), 63.870781ms from start of poll, 64.359084ms from cluster start. Timeout at 2025-01-25 14:29:47.711628714 +0000 UTC m=+97.565953696, 105.002559ms after stability -2025-01-25T14:29:47.717Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:29:47.822Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:29:47.717510205 +0000 UTC m=+97.571835187 (1 nodes), 1.76µs from start of poll, 175.243134ms from cluster start. Timeout at 2025-01-25 14:29:47.822705478 +0000 UTC m=+97.677030500, 105.195313ms after stability - raft_test.go:1799: [INFO] Disconnecting Node at 25ccac42-f31d-35a1-5318-6bc169312cec [Follower] -2025-01-25T14:29:47.826Z [ERROR] server-68c8e1c7-8678-32c2-504d-98f1683e16c6: failed to pipeline appendEntries: peer="{Voter server-25ccac42-f31d-35a1-5318-6bc169312cec 25ccac42-f31d-35a1-5318-6bc169312cec}" error="append pipeline closed" -2025-01-25T14:29:47.826Z [INFO] server-68c8e1c7-8678-32c2-504d-98f1683e16c6: aborting pipeline replication: peer="{Voter server-25ccac42-f31d-35a1-5318-6bc169312cec 25ccac42-f31d-35a1-5318-6bc169312cec}" -2025-01-25T14:29:47.829Z [ERROR] server-68c8e1c7-8678-32c2-504d-98f1683e16c6: failed to heartbeat to: peer=25ccac42-f31d-35a1-5318-6bc169312cec backoff time=10ms error="failed to connect to peer: 25ccac42-f31d-35a1-5318-6bc169312cec" -2025-01-25T14:29:47.832Z [ERROR] server-68c8e1c7-8678-32c2-504d-98f1683e16c6: failed to appendEntries to: peer="{Voter server-25ccac42-f31d-35a1-5318-6bc169312cec 25ccac42-f31d-35a1-5318-6bc169312cec}" error="failed to connect to peer: 25ccac42-f31d-35a1-5318-6bc169312cec" -2025-01-25T14:29:47.848Z [ERROR] server-68c8e1c7-8678-32c2-504d-98f1683e16c6: failed to heartbeat to: peer=25ccac42-f31d-35a1-5318-6bc169312cec backoff time=10ms error="failed to connect to peer: 25ccac42-f31d-35a1-5318-6bc169312cec" -2025-01-25T14:29:47.851Z [ERROR] server-68c8e1c7-8678-32c2-504d-98f1683e16c6: failed to appendEntries to: peer="{Voter server-25ccac42-f31d-35a1-5318-6bc169312cec 25ccac42-f31d-35a1-5318-6bc169312cec}" error="failed to connect to peer: 25ccac42-f31d-35a1-5318-6bc169312cec" -2025-01-25T14:29:47.865Z [ERROR] server-68c8e1c7-8678-32c2-504d-98f1683e16c6: failed to heartbeat to: peer=25ccac42-f31d-35a1-5318-6bc169312cec backoff time=10ms error="failed to connect to peer: 25ccac42-f31d-35a1-5318-6bc169312cec" -2025-01-25T14:29:47.867Z [ERROR] server-68c8e1c7-8678-32c2-504d-98f1683e16c6: failed to appendEntries to: peer="{Voter server-25ccac42-f31d-35a1-5318-6bc169312cec 25ccac42-f31d-35a1-5318-6bc169312cec}" error="failed to connect to peer: 25ccac42-f31d-35a1-5318-6bc169312cec" -2025-01-25T14:29:47.872Z [WARN] server-68c8e1c7-8678-32c2-504d-98f1683e16c6: failed to contact: server-id=server-25ccac42-f31d-35a1-5318-6bc169312cec time=50.499982ms -2025-01-25T14:29:47.872Z [WARN] server-68c8e1c7-8678-32c2-504d-98f1683e16c6: failed to contact quorum of nodes, stepping down -2025-01-25T14:29:47.872Z [INFO] server-68c8e1c7-8678-32c2-504d-98f1683e16c6: entering follower state: follower="Node at 68c8e1c7-8678-32c2-504d-98f1683e16c6 [Follower]" leader-address= leader-id= -2025-01-25T14:29:47.872Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:47.876Z [WARN] server-25ccac42-f31d-35a1-5318-6bc169312cec: heartbeat timeout reached, starting election: last-leader-addr=68c8e1c7-8678-32c2-504d-98f1683e16c6 last-leader-id=server-68c8e1c7-8678-32c2-504d-98f1683e16c6 -2025-01-25T14:29:47.876Z [INFO] server-25ccac42-f31d-35a1-5318-6bc169312cec: entering candidate state: node="Node at 25ccac42-f31d-35a1-5318-6bc169312cec [Candidate]" term=3 -2025-01-25T14:29:47.876Z [ERROR] server-25ccac42-f31d-35a1-5318-6bc169312cec: failed to make requestVote RPC: target="{Voter server-68c8e1c7-8678-32c2-504d-98f1683e16c6 68c8e1c7-8678-32c2-504d-98f1683e16c6}" error="failed to connect to peer: 68c8e1c7-8678-32c2-504d-98f1683e16c6" term=3 -2025-01-25T14:29:47.939Z [WARN] server-25ccac42-f31d-35a1-5318-6bc169312cec: Election timeout reached, restarting election -2025-01-25T14:29:47.939Z [INFO] server-25ccac42-f31d-35a1-5318-6bc169312cec: entering candidate state: node="Node at 25ccac42-f31d-35a1-5318-6bc169312cec [Candidate]" term=4 -2025-01-25T14:29:47.939Z [ERROR] server-25ccac42-f31d-35a1-5318-6bc169312cec: failed to make requestVote RPC: target="{Voter server-68c8e1c7-8678-32c2-504d-98f1683e16c6 68c8e1c7-8678-32c2-504d-98f1683e16c6}" error="failed to connect to peer: 68c8e1c7-8678-32c2-504d-98f1683e16c6" term=4 -2025-01-25T14:29:47.942Z [WARN] server-68c8e1c7-8678-32c2-504d-98f1683e16c6: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:47.942Z [INFO] server-68c8e1c7-8678-32c2-504d-98f1683e16c6: entering candidate state: node="Node at 68c8e1c7-8678-32c2-504d-98f1683e16c6 [Candidate]" term=3 -2025-01-25T14:29:47.942Z [ERROR] server-68c8e1c7-8678-32c2-504d-98f1683e16c6: failed to make requestVote RPC: target="{Voter server-25ccac42-f31d-35a1-5318-6bc169312cec 25ccac42-f31d-35a1-5318-6bc169312cec}" error="failed to connect to peer: 25ccac42-f31d-35a1-5318-6bc169312cec" term=3 -2025-01-25T14:29:47.993Z [WARN] server-25ccac42-f31d-35a1-5318-6bc169312cec: Election timeout reached, restarting election -2025-01-25T14:29:47.993Z [INFO] server-25ccac42-f31d-35a1-5318-6bc169312cec: entering candidate state: node="Node at 25ccac42-f31d-35a1-5318-6bc169312cec [Candidate]" term=5 -2025-01-25T14:29:47.993Z [ERROR] server-25ccac42-f31d-35a1-5318-6bc169312cec: failed to make requestVote RPC: target="{Voter server-68c8e1c7-8678-32c2-504d-98f1683e16c6 68c8e1c7-8678-32c2-504d-98f1683e16c6}" error="failed to connect to peer: 68c8e1c7-8678-32c2-504d-98f1683e16c6" term=5 -2025-01-25T14:29:48.031Z [WARN] server-68c8e1c7-8678-32c2-504d-98f1683e16c6: Election timeout reached, restarting election -2025-01-25T14:29:48.031Z [INFO] server-68c8e1c7-8678-32c2-504d-98f1683e16c6: entering candidate state: node="Node at 68c8e1c7-8678-32c2-504d-98f1683e16c6 [Candidate]" term=4 -2025-01-25T14:29:48.031Z [ERROR] server-68c8e1c7-8678-32c2-504d-98f1683e16c6: failed to make requestVote RPC: target="{Voter server-25ccac42-f31d-35a1-5318-6bc169312cec 25ccac42-f31d-35a1-5318-6bc169312cec}" error="failed to connect to peer: 25ccac42-f31d-35a1-5318-6bc169312cec" term=4 -2025-01-25T14:29:48.048Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:47.942665622 +0000 UTC m=+97.796990604 (0 nodes), 69.835007ms from start of poll, 400.398551ms from cluster start. Timeout at 2025-01-25 14:29:48.047668902 +0000 UTC m=+97.901993884, 105.00328ms after stability -2025-01-25T14:29:48.077Z [WARN] server-25ccac42-f31d-35a1-5318-6bc169312cec: Election timeout reached, restarting election -2025-01-25T14:29:48.077Z [INFO] server-25ccac42-f31d-35a1-5318-6bc169312cec: entering candidate state: node="Node at 25ccac42-f31d-35a1-5318-6bc169312cec [Candidate]" term=6 -2025-01-25T14:29:48.077Z [ERROR] server-25ccac42-f31d-35a1-5318-6bc169312cec: failed to make requestVote RPC: target="{Voter server-68c8e1c7-8678-32c2-504d-98f1683e16c6 68c8e1c7-8678-32c2-504d-98f1683e16c6}" error="failed to connect to peer: 68c8e1c7-8678-32c2-504d-98f1683e16c6" term=6 -2025-01-25T14:29:48.092Z [WARN] server-68c8e1c7-8678-32c2-504d-98f1683e16c6: Election timeout reached, restarting election -2025-01-25T14:29:48.092Z [INFO] server-68c8e1c7-8678-32c2-504d-98f1683e16c6: entering candidate state: node="Node at 68c8e1c7-8678-32c2-504d-98f1683e16c6 [Candidate]" term=5 -2025-01-25T14:29:48.092Z [ERROR] server-68c8e1c7-8678-32c2-504d-98f1683e16c6: failed to make requestVote RPC: target="{Voter server-25ccac42-f31d-35a1-5318-6bc169312cec 25ccac42-f31d-35a1-5318-6bc169312cec}" error="failed to connect to peer: 25ccac42-f31d-35a1-5318-6bc169312cec" term=5 -2025-01-25T14:29:48.149Z [WARN] server-68c8e1c7-8678-32c2-504d-98f1683e16c6: Election timeout reached, restarting election -2025-01-25T14:29:48.149Z [INFO] server-68c8e1c7-8678-32c2-504d-98f1683e16c6: entering candidate state: node="Node at 68c8e1c7-8678-32c2-504d-98f1683e16c6 [Candidate]" term=6 -2025-01-25T14:29:48.149Z [ERROR] server-68c8e1c7-8678-32c2-504d-98f1683e16c6: failed to make requestVote RPC: target="{Voter server-25ccac42-f31d-35a1-5318-6bc169312cec 25ccac42-f31d-35a1-5318-6bc169312cec}" error="failed to connect to peer: 25ccac42-f31d-35a1-5318-6bc169312cec" term=6 ---- PASS: TestRaft_LeaderLeaseExpire (0.61s) +2025-01-25T14:40:35.022Z [INFO] server-bf09adf7-a12e-f669-d3e8-1208118cf4d8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bf09adf7-a12e-f669-d3e8-1208118cf4d8 Address:bf09adf7-a12e-f669-d3e8-1208118cf4d8} {Suffrage:Voter ID:server-71010ac1-f2ea-6373-e72f-69e6285e4846 Address:71010ac1-f2ea-6373-e72f-69e6285e4846}]" +2025-01-25T14:40:35.023Z [INFO] server-bf09adf7-a12e-f669-d3e8-1208118cf4d8: entering follower state: follower="Node at bf09adf7-a12e-f669-d3e8-1208118cf4d8 [Follower]" leader-address= leader-id= +2025-01-25T14:40:35.023Z [INFO] server-71010ac1-f2ea-6373-e72f-69e6285e4846: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bf09adf7-a12e-f669-d3e8-1208118cf4d8 Address:bf09adf7-a12e-f669-d3e8-1208118cf4d8} {Suffrage:Voter ID:server-71010ac1-f2ea-6373-e72f-69e6285e4846 Address:71010ac1-f2ea-6373-e72f-69e6285e4846}]" +2025-01-25T14:40:35.023Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:35.023Z [INFO] server-71010ac1-f2ea-6373-e72f-69e6285e4846: entering follower state: follower="Node at 71010ac1-f2ea-6373-e72f-69e6285e4846 [Follower]" leader-address= leader-id= +2025-01-25T14:40:35.099Z [WARN] server-bf09adf7-a12e-f669-d3e8-1208118cf4d8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:35.100Z [INFO] server-bf09adf7-a12e-f669-d3e8-1208118cf4d8: entering candidate state: node="Node at bf09adf7-a12e-f669-d3e8-1208118cf4d8 [Candidate]" term=2 +2025-01-25T14:40:35.100Z [INFO] server-bf09adf7-a12e-f669-d3e8-1208118cf4d8: election won: term=2 tally=2 +2025-01-25T14:40:35.100Z [INFO] server-bf09adf7-a12e-f669-d3e8-1208118cf4d8: entering leader state: leader="Node at bf09adf7-a12e-f669-d3e8-1208118cf4d8 [Leader]" +2025-01-25T14:40:35.100Z [INFO] server-bf09adf7-a12e-f669-d3e8-1208118cf4d8: added peer, starting replication: peer=server-71010ac1-f2ea-6373-e72f-69e6285e4846 +2025-01-25T14:40:35.101Z [INFO] server-bf09adf7-a12e-f669-d3e8-1208118cf4d8: pipelining replication: peer="{Voter server-71010ac1-f2ea-6373-e72f-69e6285e4846 71010ac1-f2ea-6373-e72f-69e6285e4846}" +2025-01-25T14:40:35.206Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:35.100724688 +0000 UTC m=+104.146587047 (1 nodes), 77.040974ms from start of poll, 78.712229ms from cluster start. Timeout at 2025-01-25 14:40:35.205767855 +0000 UTC m=+104.251607173, 105.020126ms after stability +2025-01-25T14:40:35.211Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:40:35.317Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:40:35.211403385 +0000 UTC m=+104.257242411 (1 nodes), 4.667µs from start of poll, 189.367593ms from cluster start. Timeout at 2025-01-25 14:40:35.316638761 +0000 UTC m=+104.362478078, 105.235667ms after stability + raft_test.go:1799: [INFO] Disconnecting Node at 71010ac1-f2ea-6373-e72f-69e6285e4846 [Follower] +2025-01-25T14:40:35.319Z [ERROR] server-bf09adf7-a12e-f669-d3e8-1208118cf4d8: failed to pipeline appendEntries: peer="{Voter server-71010ac1-f2ea-6373-e72f-69e6285e4846 71010ac1-f2ea-6373-e72f-69e6285e4846}" error="append pipeline closed" +2025-01-25T14:40:35.319Z [INFO] server-bf09adf7-a12e-f669-d3e8-1208118cf4d8: aborting pipeline replication: peer="{Voter server-71010ac1-f2ea-6373-e72f-69e6285e4846 71010ac1-f2ea-6373-e72f-69e6285e4846}" +2025-01-25T14:40:35.326Z [ERROR] server-bf09adf7-a12e-f669-d3e8-1208118cf4d8: failed to heartbeat to: peer=71010ac1-f2ea-6373-e72f-69e6285e4846 backoff time=10ms error="failed to connect to peer: 71010ac1-f2ea-6373-e72f-69e6285e4846" +2025-01-25T14:40:35.329Z [ERROR] server-bf09adf7-a12e-f669-d3e8-1208118cf4d8: failed to appendEntries to: peer="{Voter server-71010ac1-f2ea-6373-e72f-69e6285e4846 71010ac1-f2ea-6373-e72f-69e6285e4846}" error="failed to connect to peer: 71010ac1-f2ea-6373-e72f-69e6285e4846" +2025-01-25T14:40:35.346Z [ERROR] server-bf09adf7-a12e-f669-d3e8-1208118cf4d8: failed to heartbeat to: peer=71010ac1-f2ea-6373-e72f-69e6285e4846 backoff time=10ms error="failed to connect to peer: 71010ac1-f2ea-6373-e72f-69e6285e4846" +2025-01-25T14:40:35.348Z [ERROR] server-bf09adf7-a12e-f669-d3e8-1208118cf4d8: failed to appendEntries to: peer="{Voter server-71010ac1-f2ea-6373-e72f-69e6285e4846 71010ac1-f2ea-6373-e72f-69e6285e4846}" error="failed to connect to peer: 71010ac1-f2ea-6373-e72f-69e6285e4846" +2025-01-25T14:40:35.366Z [WARN] server-bf09adf7-a12e-f669-d3e8-1208118cf4d8: failed to contact: server-id=server-71010ac1-f2ea-6373-e72f-69e6285e4846 time=51.112107ms +2025-01-25T14:40:35.367Z [WARN] server-bf09adf7-a12e-f669-d3e8-1208118cf4d8: failed to contact quorum of nodes, stepping down +2025-01-25T14:40:35.367Z [INFO] server-bf09adf7-a12e-f669-d3e8-1208118cf4d8: entering follower state: follower="Node at bf09adf7-a12e-f669-d3e8-1208118cf4d8 [Follower]" leader-address= leader-id= +2025-01-25T14:40:35.367Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:35.367Z [ERROR] server-bf09adf7-a12e-f669-d3e8-1208118cf4d8: failed to appendEntries to: peer="{Voter server-71010ac1-f2ea-6373-e72f-69e6285e4846 71010ac1-f2ea-6373-e72f-69e6285e4846}" error="failed to connect to peer: 71010ac1-f2ea-6373-e72f-69e6285e4846" +2025-01-25T14:40:35.390Z [WARN] server-71010ac1-f2ea-6373-e72f-69e6285e4846: heartbeat timeout reached, starting election: last-leader-addr=bf09adf7-a12e-f669-d3e8-1208118cf4d8 last-leader-id=server-bf09adf7-a12e-f669-d3e8-1208118cf4d8 +2025-01-25T14:40:35.390Z [INFO] server-71010ac1-f2ea-6373-e72f-69e6285e4846: entering candidate state: node="Node at 71010ac1-f2ea-6373-e72f-69e6285e4846 [Candidate]" term=3 +2025-01-25T14:40:35.391Z [ERROR] server-71010ac1-f2ea-6373-e72f-69e6285e4846: failed to make requestVote RPC: target="{Voter server-bf09adf7-a12e-f669-d3e8-1208118cf4d8 bf09adf7-a12e-f669-d3e8-1208118cf4d8}" error="failed to connect to peer: bf09adf7-a12e-f669-d3e8-1208118cf4d8" term=3 +2025-01-25T14:40:35.444Z [WARN] server-bf09adf7-a12e-f669-d3e8-1208118cf4d8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:35.445Z [INFO] server-bf09adf7-a12e-f669-d3e8-1208118cf4d8: entering candidate state: node="Node at bf09adf7-a12e-f669-d3e8-1208118cf4d8 [Candidate]" term=3 +2025-01-25T14:40:35.445Z [ERROR] server-bf09adf7-a12e-f669-d3e8-1208118cf4d8: failed to make requestVote RPC: target="{Voter server-71010ac1-f2ea-6373-e72f-69e6285e4846 71010ac1-f2ea-6373-e72f-69e6285e4846}" error="failed to connect to peer: 71010ac1-f2ea-6373-e72f-69e6285e4846" term=3 +2025-01-25T14:40:35.483Z [WARN] server-71010ac1-f2ea-6373-e72f-69e6285e4846: Election timeout reached, restarting election +2025-01-25T14:40:35.483Z [INFO] server-71010ac1-f2ea-6373-e72f-69e6285e4846: entering candidate state: node="Node at 71010ac1-f2ea-6373-e72f-69e6285e4846 [Candidate]" term=4 +2025-01-25T14:40:35.483Z [ERROR] server-71010ac1-f2ea-6373-e72f-69e6285e4846: failed to make requestVote RPC: target="{Voter server-bf09adf7-a12e-f669-d3e8-1208118cf4d8 bf09adf7-a12e-f669-d3e8-1208118cf4d8}" error="failed to connect to peer: bf09adf7-a12e-f669-d3e8-1208118cf4d8" term=4 +2025-01-25T14:40:35.537Z [WARN] server-bf09adf7-a12e-f669-d3e8-1208118cf4d8: Election timeout reached, restarting election +2025-01-25T14:40:35.537Z [INFO] server-bf09adf7-a12e-f669-d3e8-1208118cf4d8: entering candidate state: node="Node at bf09adf7-a12e-f669-d3e8-1208118cf4d8 [Candidate]" term=4 +2025-01-25T14:40:35.537Z [ERROR] server-bf09adf7-a12e-f669-d3e8-1208118cf4d8: failed to make requestVote RPC: target="{Voter server-71010ac1-f2ea-6373-e72f-69e6285e4846 71010ac1-f2ea-6373-e72f-69e6285e4846}" error="failed to connect to peer: 71010ac1-f2ea-6373-e72f-69e6285e4846" term=4 +2025-01-25T14:40:35.550Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:35.445301649 +0000 UTC m=+104.491140967 (0 nodes), 77.914769ms from start of poll, 423.266149ms from cluster start. Timeout at 2025-01-25 14:40:35.550309524 +0000 UTC m=+104.596148842, 105.007875ms after stability +2025-01-25T14:40:35.591Z [WARN] server-71010ac1-f2ea-6373-e72f-69e6285e4846: Election timeout reached, restarting election +2025-01-25T14:40:35.595Z [INFO] server-71010ac1-f2ea-6373-e72f-69e6285e4846: entering candidate state: node="Node at 71010ac1-f2ea-6373-e72f-69e6285e4846 [Candidate]" term=5 +2025-01-25T14:40:35.596Z [ERROR] server-71010ac1-f2ea-6373-e72f-69e6285e4846: failed to make requestVote RPC: target="{Voter server-bf09adf7-a12e-f669-d3e8-1208118cf4d8 bf09adf7-a12e-f669-d3e8-1208118cf4d8}" error="failed to connect to peer: bf09adf7-a12e-f669-d3e8-1208118cf4d8" term=5 +2025-01-25T14:40:35.634Z [WARN] server-bf09adf7-a12e-f669-d3e8-1208118cf4d8: Election timeout reached, restarting election +2025-01-25T14:40:35.634Z [INFO] server-bf09adf7-a12e-f669-d3e8-1208118cf4d8: entering candidate state: node="Node at bf09adf7-a12e-f669-d3e8-1208118cf4d8 [Candidate]" term=5 +2025-01-25T14:40:35.635Z [ERROR] server-bf09adf7-a12e-f669-d3e8-1208118cf4d8: failed to make requestVote RPC: target="{Voter server-71010ac1-f2ea-6373-e72f-69e6285e4846 71010ac1-f2ea-6373-e72f-69e6285e4846}" error="failed to connect to peer: 71010ac1-f2ea-6373-e72f-69e6285e4846" term=5 +2025-01-25T14:40:35.651Z [WARN] server-71010ac1-f2ea-6373-e72f-69e6285e4846: Election timeout reached, restarting election +2025-01-25T14:40:35.651Z [INFO] server-71010ac1-f2ea-6373-e72f-69e6285e4846: entering candidate state: node="Node at 71010ac1-f2ea-6373-e72f-69e6285e4846 [Candidate]" term=6 +2025-01-25T14:40:35.652Z [ERROR] server-71010ac1-f2ea-6373-e72f-69e6285e4846: failed to make requestVote RPC: target="{Voter server-bf09adf7-a12e-f669-d3e8-1208118cf4d8 bf09adf7-a12e-f669-d3e8-1208118cf4d8}" error="failed to connect to peer: bf09adf7-a12e-f669-d3e8-1208118cf4d8" term=6 +--- PASS: TestRaft_LeaderLeaseExpire (0.64s) === RUN TestRaft_Barrier -2025-01-25T14:29:48.156Z [INFO] server-141d2975-5452-45bc-1596-0b8528b97cf2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-141d2975-5452-45bc-1596-0b8528b97cf2 Address:141d2975-5452-45bc-1596-0b8528b97cf2} {Suffrage:Voter ID:server-a992ec2a-25c9-0cec-a38f-b3ac52a83f0d Address:a992ec2a-25c9-0cec-a38f-b3ac52a83f0d} {Suffrage:Voter ID:server-4562e26a-b643-1f1f-c30c-50cb7a04bae2 Address:4562e26a-b643-1f1f-c30c-50cb7a04bae2}]" -2025-01-25T14:29:48.156Z [INFO] server-a992ec2a-25c9-0cec-a38f-b3ac52a83f0d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-141d2975-5452-45bc-1596-0b8528b97cf2 Address:141d2975-5452-45bc-1596-0b8528b97cf2} {Suffrage:Voter ID:server-a992ec2a-25c9-0cec-a38f-b3ac52a83f0d Address:a992ec2a-25c9-0cec-a38f-b3ac52a83f0d} {Suffrage:Voter ID:server-4562e26a-b643-1f1f-c30c-50cb7a04bae2 Address:4562e26a-b643-1f1f-c30c-50cb7a04bae2}]" -2025-01-25T14:29:48.157Z [INFO] server-4562e26a-b643-1f1f-c30c-50cb7a04bae2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-141d2975-5452-45bc-1596-0b8528b97cf2 Address:141d2975-5452-45bc-1596-0b8528b97cf2} {Suffrage:Voter ID:server-a992ec2a-25c9-0cec-a38f-b3ac52a83f0d Address:a992ec2a-25c9-0cec-a38f-b3ac52a83f0d} {Suffrage:Voter ID:server-4562e26a-b643-1f1f-c30c-50cb7a04bae2 Address:4562e26a-b643-1f1f-c30c-50cb7a04bae2}]" -2025-01-25T14:29:48.157Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:48.157Z [INFO] server-a992ec2a-25c9-0cec-a38f-b3ac52a83f0d: entering follower state: follower="Node at a992ec2a-25c9-0cec-a38f-b3ac52a83f0d [Follower]" leader-address= leader-id= -2025-01-25T14:29:48.157Z [INFO] server-141d2975-5452-45bc-1596-0b8528b97cf2: entering follower state: follower="Node at 141d2975-5452-45bc-1596-0b8528b97cf2 [Follower]" leader-address= leader-id= -2025-01-25T14:29:48.157Z [INFO] server-4562e26a-b643-1f1f-c30c-50cb7a04bae2: entering follower state: follower="Node at 4562e26a-b643-1f1f-c30c-50cb7a04bae2 [Follower]" leader-address= leader-id= -2025-01-25T14:29:48.221Z [WARN] server-4562e26a-b643-1f1f-c30c-50cb7a04bae2: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:48.221Z [INFO] server-4562e26a-b643-1f1f-c30c-50cb7a04bae2: entering candidate state: node="Node at 4562e26a-b643-1f1f-c30c-50cb7a04bae2 [Candidate]" term=2 -2025-01-25T14:29:48.221Z [INFO] server-4562e26a-b643-1f1f-c30c-50cb7a04bae2: election won: term=2 tally=2 -2025-01-25T14:29:48.222Z [INFO] server-4562e26a-b643-1f1f-c30c-50cb7a04bae2: entering leader state: leader="Node at 4562e26a-b643-1f1f-c30c-50cb7a04bae2 [Leader]" -2025-01-25T14:29:48.222Z [INFO] server-4562e26a-b643-1f1f-c30c-50cb7a04bae2: added peer, starting replication: peer=server-141d2975-5452-45bc-1596-0b8528b97cf2 -2025-01-25T14:29:48.222Z [INFO] server-4562e26a-b643-1f1f-c30c-50cb7a04bae2: added peer, starting replication: peer=server-a992ec2a-25c9-0cec-a38f-b3ac52a83f0d -2025-01-25T14:29:48.222Z [INFO] server-4562e26a-b643-1f1f-c30c-50cb7a04bae2: pipelining replication: peer="{Voter server-a992ec2a-25c9-0cec-a38f-b3ac52a83f0d a992ec2a-25c9-0cec-a38f-b3ac52a83f0d}" -2025-01-25T14:29:48.222Z [INFO] server-4562e26a-b643-1f1f-c30c-50cb7a04bae2: pipelining replication: peer="{Voter server-141d2975-5452-45bc-1596-0b8528b97cf2 141d2975-5452-45bc-1596-0b8528b97cf2}" -2025-01-25T14:29:48.327Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:48.222125405 +0000 UTC m=+98.076450387 (1 nodes), 64.920424ms from start of poll, 65.563841ms from cluster start. Timeout at 2025-01-25 14:29:48.327130124 +0000 UTC m=+98.181455106, 105.004719ms after stability -2025-01-25T14:29:48.352Z [INFO] server-4562e26a-b643-1f1f-c30c-50cb7a04bae2: aborting pipeline replication: peer="{Voter server-a992ec2a-25c9-0cec-a38f-b3ac52a83f0d a992ec2a-25c9-0cec-a38f-b3ac52a83f0d}" -2025-01-25T14:29:48.352Z [INFO] server-4562e26a-b643-1f1f-c30c-50cb7a04bae2: aborting pipeline replication: peer="{Voter server-141d2975-5452-45bc-1596-0b8528b97cf2 141d2975-5452-45bc-1596-0b8528b97cf2}" ---- PASS: TestRaft_Barrier (0.20s) +2025-01-25T14:40:35.660Z [INFO] server-ca471045-4fd0-be3b-5f0b-fe4056330202: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ca471045-4fd0-be3b-5f0b-fe4056330202 Address:ca471045-4fd0-be3b-5f0b-fe4056330202} {Suffrage:Voter ID:server-6e4630e7-2cc1-ad4c-2abb-6136c30ec55d Address:6e4630e7-2cc1-ad4c-2abb-6136c30ec55d} {Suffrage:Voter ID:server-a584e3b7-2b45-8ed0-2f5d-f55d3a13df41 Address:a584e3b7-2b45-8ed0-2f5d-f55d3a13df41}]" +2025-01-25T14:40:35.660Z [INFO] server-ca471045-4fd0-be3b-5f0b-fe4056330202: entering follower state: follower="Node at ca471045-4fd0-be3b-5f0b-fe4056330202 [Follower]" leader-address= leader-id= +2025-01-25T14:40:35.661Z [INFO] server-6e4630e7-2cc1-ad4c-2abb-6136c30ec55d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ca471045-4fd0-be3b-5f0b-fe4056330202 Address:ca471045-4fd0-be3b-5f0b-fe4056330202} {Suffrage:Voter ID:server-6e4630e7-2cc1-ad4c-2abb-6136c30ec55d Address:6e4630e7-2cc1-ad4c-2abb-6136c30ec55d} {Suffrage:Voter ID:server-a584e3b7-2b45-8ed0-2f5d-f55d3a13df41 Address:a584e3b7-2b45-8ed0-2f5d-f55d3a13df41}]" +2025-01-25T14:40:35.661Z [INFO] server-6e4630e7-2cc1-ad4c-2abb-6136c30ec55d: entering follower state: follower="Node at 6e4630e7-2cc1-ad4c-2abb-6136c30ec55d [Follower]" leader-address= leader-id= +2025-01-25T14:40:35.661Z [INFO] server-a584e3b7-2b45-8ed0-2f5d-f55d3a13df41: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ca471045-4fd0-be3b-5f0b-fe4056330202 Address:ca471045-4fd0-be3b-5f0b-fe4056330202} {Suffrage:Voter ID:server-6e4630e7-2cc1-ad4c-2abb-6136c30ec55d Address:6e4630e7-2cc1-ad4c-2abb-6136c30ec55d} {Suffrage:Voter ID:server-a584e3b7-2b45-8ed0-2f5d-f55d3a13df41 Address:a584e3b7-2b45-8ed0-2f5d-f55d3a13df41}]" +2025-01-25T14:40:35.661Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:35.661Z [INFO] server-a584e3b7-2b45-8ed0-2f5d-f55d3a13df41: entering follower state: follower="Node at a584e3b7-2b45-8ed0-2f5d-f55d3a13df41 [Follower]" leader-address= leader-id= +2025-01-25T14:40:35.729Z [WARN] server-ca471045-4fd0-be3b-5f0b-fe4056330202: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:35.729Z [INFO] server-ca471045-4fd0-be3b-5f0b-fe4056330202: entering candidate state: node="Node at ca471045-4fd0-be3b-5f0b-fe4056330202 [Candidate]" term=2 +2025-01-25T14:40:35.729Z [INFO] server-ca471045-4fd0-be3b-5f0b-fe4056330202: election won: term=2 tally=2 +2025-01-25T14:40:35.729Z [INFO] server-ca471045-4fd0-be3b-5f0b-fe4056330202: entering leader state: leader="Node at ca471045-4fd0-be3b-5f0b-fe4056330202 [Leader]" +2025-01-25T14:40:35.729Z [INFO] server-ca471045-4fd0-be3b-5f0b-fe4056330202: added peer, starting replication: peer=server-6e4630e7-2cc1-ad4c-2abb-6136c30ec55d +2025-01-25T14:40:35.730Z [INFO] server-ca471045-4fd0-be3b-5f0b-fe4056330202: added peer, starting replication: peer=server-a584e3b7-2b45-8ed0-2f5d-f55d3a13df41 +2025-01-25T14:40:35.730Z [INFO] server-ca471045-4fd0-be3b-5f0b-fe4056330202: pipelining replication: peer="{Voter server-6e4630e7-2cc1-ad4c-2abb-6136c30ec55d 6e4630e7-2cc1-ad4c-2abb-6136c30ec55d}" +2025-01-25T14:40:35.730Z [INFO] server-ca471045-4fd0-be3b-5f0b-fe4056330202: pipelining replication: peer="{Voter server-a584e3b7-2b45-8ed0-2f5d-f55d3a13df41 a584e3b7-2b45-8ed0-2f5d-f55d3a13df41}" +2025-01-25T14:40:35.835Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:35.729808478 +0000 UTC m=+104.775647795 (1 nodes), 67.932614ms from start of poll, 69.701869ms from cluster start. Timeout at 2025-01-25 14:40:35.83481606 +0000 UTC m=+104.880655378, 105.007583ms after stability +2025-01-25T14:40:35.893Z [INFO] server-ca471045-4fd0-be3b-5f0b-fe4056330202: aborting pipeline replication: peer="{Voter server-6e4630e7-2cc1-ad4c-2abb-6136c30ec55d 6e4630e7-2cc1-ad4c-2abb-6136c30ec55d}" +2025-01-25T14:40:35.893Z [INFO] server-ca471045-4fd0-be3b-5f0b-fe4056330202: aborting pipeline replication: peer="{Voter server-a584e3b7-2b45-8ed0-2f5d-f55d3a13df41 a584e3b7-2b45-8ed0-2f5d-f55d3a13df41}" +--- PASS: TestRaft_Barrier (0.24s) === RUN TestRaft_VerifyLeader -2025-01-25T14:29:48.355Z [INFO] server-b40c6640-37f9-2e08-55ba-4e899981f78e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b40c6640-37f9-2e08-55ba-4e899981f78e Address:b40c6640-37f9-2e08-55ba-4e899981f78e} {Suffrage:Voter ID:server-cf4885b8-2887-6f72-3406-23e40561e3aa Address:cf4885b8-2887-6f72-3406-23e40561e3aa} {Suffrage:Voter ID:server-8de764b5-dd07-04dc-34c0-cdf4de098bf0 Address:8de764b5-dd07-04dc-34c0-cdf4de098bf0}]" -2025-01-25T14:29:48.355Z [INFO] server-b40c6640-37f9-2e08-55ba-4e899981f78e: entering follower state: follower="Node at b40c6640-37f9-2e08-55ba-4e899981f78e [Follower]" leader-address= leader-id= -2025-01-25T14:29:48.355Z [INFO] server-cf4885b8-2887-6f72-3406-23e40561e3aa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b40c6640-37f9-2e08-55ba-4e899981f78e Address:b40c6640-37f9-2e08-55ba-4e899981f78e} {Suffrage:Voter ID:server-cf4885b8-2887-6f72-3406-23e40561e3aa Address:cf4885b8-2887-6f72-3406-23e40561e3aa} {Suffrage:Voter ID:server-8de764b5-dd07-04dc-34c0-cdf4de098bf0 Address:8de764b5-dd07-04dc-34c0-cdf4de098bf0}]" -2025-01-25T14:29:48.355Z [INFO] server-cf4885b8-2887-6f72-3406-23e40561e3aa: entering follower state: follower="Node at cf4885b8-2887-6f72-3406-23e40561e3aa [Follower]" leader-address= leader-id= -2025-01-25T14:29:48.356Z [INFO] server-8de764b5-dd07-04dc-34c0-cdf4de098bf0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b40c6640-37f9-2e08-55ba-4e899981f78e Address:b40c6640-37f9-2e08-55ba-4e899981f78e} {Suffrage:Voter ID:server-cf4885b8-2887-6f72-3406-23e40561e3aa Address:cf4885b8-2887-6f72-3406-23e40561e3aa} {Suffrage:Voter ID:server-8de764b5-dd07-04dc-34c0-cdf4de098bf0 Address:8de764b5-dd07-04dc-34c0-cdf4de098bf0}]" -2025-01-25T14:29:48.356Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:48.356Z [INFO] server-8de764b5-dd07-04dc-34c0-cdf4de098bf0: entering follower state: follower="Node at 8de764b5-dd07-04dc-34c0-cdf4de098bf0 [Follower]" leader-address= leader-id= -2025-01-25T14:29:48.430Z [WARN] server-b40c6640-37f9-2e08-55ba-4e899981f78e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:48.430Z [INFO] server-b40c6640-37f9-2e08-55ba-4e899981f78e: entering candidate state: node="Node at b40c6640-37f9-2e08-55ba-4e899981f78e [Candidate]" term=2 -2025-01-25T14:29:48.431Z [INFO] server-b40c6640-37f9-2e08-55ba-4e899981f78e: election won: term=2 tally=2 -2025-01-25T14:29:48.431Z [INFO] server-b40c6640-37f9-2e08-55ba-4e899981f78e: entering leader state: leader="Node at b40c6640-37f9-2e08-55ba-4e899981f78e [Leader]" -2025-01-25T14:29:48.431Z [INFO] server-b40c6640-37f9-2e08-55ba-4e899981f78e: added peer, starting replication: peer=server-cf4885b8-2887-6f72-3406-23e40561e3aa -2025-01-25T14:29:48.431Z [INFO] server-b40c6640-37f9-2e08-55ba-4e899981f78e: added peer, starting replication: peer=server-8de764b5-dd07-04dc-34c0-cdf4de098bf0 -2025-01-25T14:29:48.431Z [WARN] server-8de764b5-dd07-04dc-34c0-cdf4de098bf0: rejecting vote request since our last term is greater: candidate=b40c6640-37f9-2e08-55ba-4e899981f78e last-term=2 last-candidate-term=1 -2025-01-25T14:29:48.431Z [INFO] server-b40c6640-37f9-2e08-55ba-4e899981f78e: pipelining replication: peer="{Voter server-cf4885b8-2887-6f72-3406-23e40561e3aa cf4885b8-2887-6f72-3406-23e40561e3aa}" -2025-01-25T14:29:48.431Z [INFO] server-b40c6640-37f9-2e08-55ba-4e899981f78e: pipelining replication: peer="{Voter server-8de764b5-dd07-04dc-34c0-cdf4de098bf0 8de764b5-dd07-04dc-34c0-cdf4de098bf0}" -2025-01-25T14:29:48.537Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:48.431617746 +0000 UTC m=+98.285942688 (1 nodes), 75.504203ms from start of poll, 76.262656ms from cluster start. Timeout at 2025-01-25 14:29:48.536630746 +0000 UTC m=+98.390955688, 105.013ms after stability -2025-01-25T14:29:48.537Z [INFO] server-b40c6640-37f9-2e08-55ba-4e899981f78e: aborting pipeline replication: peer="{Voter server-8de764b5-dd07-04dc-34c0-cdf4de098bf0 8de764b5-dd07-04dc-34c0-cdf4de098bf0}" -2025-01-25T14:29:48.537Z [INFO] server-b40c6640-37f9-2e08-55ba-4e899981f78e: aborting pipeline replication: peer="{Voter server-cf4885b8-2887-6f72-3406-23e40561e3aa cf4885b8-2887-6f72-3406-23e40561e3aa}" ---- PASS: TestRaft_VerifyLeader (0.18s) +2025-01-25T14:40:35.902Z [INFO] server-78ce5cce-278b-0052-d650-159378c782c7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-78ce5cce-278b-0052-d650-159378c782c7 Address:78ce5cce-278b-0052-d650-159378c782c7} {Suffrage:Voter ID:server-5249fa5d-b4f1-207e-1c8f-abb8dc7020e1 Address:5249fa5d-b4f1-207e-1c8f-abb8dc7020e1} {Suffrage:Voter ID:server-f707de82-b311-2542-9909-809d67323a2c Address:f707de82-b311-2542-9909-809d67323a2c}]" +2025-01-25T14:40:35.902Z [INFO] server-78ce5cce-278b-0052-d650-159378c782c7: entering follower state: follower="Node at 78ce5cce-278b-0052-d650-159378c782c7 [Follower]" leader-address= leader-id= +2025-01-25T14:40:35.903Z [INFO] server-5249fa5d-b4f1-207e-1c8f-abb8dc7020e1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-78ce5cce-278b-0052-d650-159378c782c7 Address:78ce5cce-278b-0052-d650-159378c782c7} {Suffrage:Voter ID:server-5249fa5d-b4f1-207e-1c8f-abb8dc7020e1 Address:5249fa5d-b4f1-207e-1c8f-abb8dc7020e1} {Suffrage:Voter ID:server-f707de82-b311-2542-9909-809d67323a2c Address:f707de82-b311-2542-9909-809d67323a2c}]" +2025-01-25T14:40:35.903Z [INFO] server-5249fa5d-b4f1-207e-1c8f-abb8dc7020e1: entering follower state: follower="Node at 5249fa5d-b4f1-207e-1c8f-abb8dc7020e1 [Follower]" leader-address= leader-id= +2025-01-25T14:40:35.904Z [INFO] server-f707de82-b311-2542-9909-809d67323a2c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-78ce5cce-278b-0052-d650-159378c782c7 Address:78ce5cce-278b-0052-d650-159378c782c7} {Suffrage:Voter ID:server-5249fa5d-b4f1-207e-1c8f-abb8dc7020e1 Address:5249fa5d-b4f1-207e-1c8f-abb8dc7020e1} {Suffrage:Voter ID:server-f707de82-b311-2542-9909-809d67323a2c Address:f707de82-b311-2542-9909-809d67323a2c}]" +2025-01-25T14:40:35.904Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:35.904Z [INFO] server-f707de82-b311-2542-9909-809d67323a2c: entering follower state: follower="Node at f707de82-b311-2542-9909-809d67323a2c [Follower]" leader-address= leader-id= +2025-01-25T14:40:35.973Z [WARN] server-78ce5cce-278b-0052-d650-159378c782c7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:35.973Z [INFO] server-78ce5cce-278b-0052-d650-159378c782c7: entering candidate state: node="Node at 78ce5cce-278b-0052-d650-159378c782c7 [Candidate]" term=2 +2025-01-25T14:40:35.973Z [WARN] server-f707de82-b311-2542-9909-809d67323a2c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:35.973Z [INFO] server-f707de82-b311-2542-9909-809d67323a2c: entering candidate state: node="Node at f707de82-b311-2542-9909-809d67323a2c [Candidate]" term=2 +2025-01-25T14:40:35.974Z [INFO] server-f707de82-b311-2542-9909-809d67323a2c: duplicate requestVote for same term: term=2 +2025-01-25T14:40:35.974Z [INFO] server-5249fa5d-b4f1-207e-1c8f-abb8dc7020e1: duplicate requestVote for same term: term=2 +2025-01-25T14:40:35.974Z [INFO] server-78ce5cce-278b-0052-d650-159378c782c7: duplicate requestVote for same term: term=2 +2025-01-25T14:40:35.974Z [INFO] server-78ce5cce-278b-0052-d650-159378c782c7: election won: term=2 tally=2 +2025-01-25T14:40:35.974Z [INFO] server-78ce5cce-278b-0052-d650-159378c782c7: entering leader state: leader="Node at 78ce5cce-278b-0052-d650-159378c782c7 [Leader]" +2025-01-25T14:40:35.974Z [INFO] server-78ce5cce-278b-0052-d650-159378c782c7: added peer, starting replication: peer=server-5249fa5d-b4f1-207e-1c8f-abb8dc7020e1 +2025-01-25T14:40:35.974Z [INFO] server-78ce5cce-278b-0052-d650-159378c782c7: added peer, starting replication: peer=server-f707de82-b311-2542-9909-809d67323a2c +2025-01-25T14:40:35.975Z [INFO] server-78ce5cce-278b-0052-d650-159378c782c7: pipelining replication: peer="{Voter server-5249fa5d-b4f1-207e-1c8f-abb8dc7020e1 5249fa5d-b4f1-207e-1c8f-abb8dc7020e1}" +2025-01-25T14:40:35.975Z [INFO] server-f707de82-b311-2542-9909-809d67323a2c: entering follower state: follower="Node at f707de82-b311-2542-9909-809d67323a2c [Follower]" leader-address=78ce5cce-278b-0052-d650-159378c782c7 leader-id=server-78ce5cce-278b-0052-d650-159378c782c7 +2025-01-25T14:40:35.975Z [INFO] server-78ce5cce-278b-0052-d650-159378c782c7: pipelining replication: peer="{Voter server-f707de82-b311-2542-9909-809d67323a2c f707de82-b311-2542-9909-809d67323a2c}" +2025-01-25T14:40:36.080Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:35.975489776 +0000 UTC m=+105.021329094 (1 nodes), 71.074457ms from start of poll, 74.108091ms from cluster start. Timeout at 2025-01-25 14:40:36.080507859 +0000 UTC m=+105.126347177, 105.018083ms after stability +2025-01-25T14:40:36.081Z [INFO] server-78ce5cce-278b-0052-d650-159378c782c7: aborting pipeline replication: peer="{Voter server-f707de82-b311-2542-9909-809d67323a2c f707de82-b311-2542-9909-809d67323a2c}" +2025-01-25T14:40:36.081Z [INFO] server-78ce5cce-278b-0052-d650-159378c782c7: aborting pipeline replication: peer="{Voter server-5249fa5d-b4f1-207e-1c8f-abb8dc7020e1 5249fa5d-b4f1-207e-1c8f-abb8dc7020e1}" +--- PASS: TestRaft_VerifyLeader (0.19s) === RUN TestRaft_VerifyLeader_Single -2025-01-25T14:29:48.539Z [INFO] server-9d692c05-8179-c61c-5f9b-441a1663aa5b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9d692c05-8179-c61c-5f9b-441a1663aa5b Address:9d692c05-8179-c61c-5f9b-441a1663aa5b}]" -2025-01-25T14:29:48.539Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:48.539Z [INFO] server-9d692c05-8179-c61c-5f9b-441a1663aa5b: entering follower state: follower="Node at 9d692c05-8179-c61c-5f9b-441a1663aa5b [Follower]" leader-address= leader-id= -2025-01-25T14:29:48.602Z [WARN] server-9d692c05-8179-c61c-5f9b-441a1663aa5b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:48.602Z [INFO] server-9d692c05-8179-c61c-5f9b-441a1663aa5b: entering candidate state: node="Node at 9d692c05-8179-c61c-5f9b-441a1663aa5b [Candidate]" term=2 -2025-01-25T14:29:48.602Z [INFO] server-9d692c05-8179-c61c-5f9b-441a1663aa5b: election won: term=2 tally=1 -2025-01-25T14:29:48.602Z [INFO] server-9d692c05-8179-c61c-5f9b-441a1663aa5b: entering leader state: leader="Node at 9d692c05-8179-c61c-5f9b-441a1663aa5b [Leader]" -2025-01-25T14:29:48.707Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:48.602275284 +0000 UTC m=+98.456600266 (1 nodes), 62.654505ms from start of poll, 63.34016ms from cluster start. Timeout at 2025-01-25 14:29:48.707282804 +0000 UTC m=+98.561607786, 105.00752ms after stability ---- PASS: TestRaft_VerifyLeader_Single (0.17s) +2025-01-25T14:40:36.087Z [INFO] server-864ea926-bfca-703a-e65a-b3795c01c638: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-864ea926-bfca-703a-e65a-b3795c01c638 Address:864ea926-bfca-703a-e65a-b3795c01c638}]" +2025-01-25T14:40:36.087Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:36.087Z [INFO] server-864ea926-bfca-703a-e65a-b3795c01c638: entering follower state: follower="Node at 864ea926-bfca-703a-e65a-b3795c01c638 [Follower]" leader-address= leader-id= +2025-01-25T14:40:36.156Z [WARN] server-864ea926-bfca-703a-e65a-b3795c01c638: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:36.156Z [INFO] server-864ea926-bfca-703a-e65a-b3795c01c638: entering candidate state: node="Node at 864ea926-bfca-703a-e65a-b3795c01c638 [Candidate]" term=2 +2025-01-25T14:40:36.156Z [INFO] server-864ea926-bfca-703a-e65a-b3795c01c638: election won: term=2 tally=1 +2025-01-25T14:40:36.156Z [INFO] server-864ea926-bfca-703a-e65a-b3795c01c638: entering leader state: leader="Node at 864ea926-bfca-703a-e65a-b3795c01c638 [Leader]" +2025-01-25T14:40:36.262Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:36.157151222 +0000 UTC m=+105.202990539 (1 nodes), 69.273785ms from start of poll, 70.458538ms from cluster start. Timeout at 2025-01-25 14:40:36.26216493 +0000 UTC m=+105.308004248, 105.013709ms after stability +--- PASS: TestRaft_VerifyLeader_Single (0.18s) === RUN TestRaft_VerifyLeader_Fail -2025-01-25T14:29:48.710Z [INFO] server-a3c10448-efa2-4c57-99b1-abb6032f5eb4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a3c10448-efa2-4c57-99b1-abb6032f5eb4 Address:a3c10448-efa2-4c57-99b1-abb6032f5eb4} {Suffrage:Voter ID:server-f1ee4f06-f9bc-d6e0-713a-7423eb2722d0 Address:f1ee4f06-f9bc-d6e0-713a-7423eb2722d0}]" -2025-01-25T14:29:48.710Z [INFO] server-f1ee4f06-f9bc-d6e0-713a-7423eb2722d0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a3c10448-efa2-4c57-99b1-abb6032f5eb4 Address:a3c10448-efa2-4c57-99b1-abb6032f5eb4} {Suffrage:Voter ID:server-f1ee4f06-f9bc-d6e0-713a-7423eb2722d0 Address:f1ee4f06-f9bc-d6e0-713a-7423eb2722d0}]" -2025-01-25T14:29:48.710Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:48.710Z [INFO] server-a3c10448-efa2-4c57-99b1-abb6032f5eb4: entering follower state: follower="Node at a3c10448-efa2-4c57-99b1-abb6032f5eb4 [Follower]" leader-address= leader-id= -2025-01-25T14:29:48.710Z [INFO] server-f1ee4f06-f9bc-d6e0-713a-7423eb2722d0: entering follower state: follower="Node at f1ee4f06-f9bc-d6e0-713a-7423eb2722d0 [Follower]" leader-address= leader-id= -2025-01-25T14:29:48.764Z [WARN] server-f1ee4f06-f9bc-d6e0-713a-7423eb2722d0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:48.764Z [INFO] server-f1ee4f06-f9bc-d6e0-713a-7423eb2722d0: entering candidate state: node="Node at f1ee4f06-f9bc-d6e0-713a-7423eb2722d0 [Candidate]" term=2 -2025-01-25T14:29:48.764Z [INFO] server-f1ee4f06-f9bc-d6e0-713a-7423eb2722d0: election won: term=2 tally=2 -2025-01-25T14:29:48.764Z [INFO] server-f1ee4f06-f9bc-d6e0-713a-7423eb2722d0: entering leader state: leader="Node at f1ee4f06-f9bc-d6e0-713a-7423eb2722d0 [Leader]" -2025-01-25T14:29:48.764Z [INFO] server-f1ee4f06-f9bc-d6e0-713a-7423eb2722d0: added peer, starting replication: peer=server-a3c10448-efa2-4c57-99b1-abb6032f5eb4 -2025-01-25T14:29:48.765Z [INFO] server-f1ee4f06-f9bc-d6e0-713a-7423eb2722d0: pipelining replication: peer="{Voter server-a3c10448-efa2-4c57-99b1-abb6032f5eb4 a3c10448-efa2-4c57-99b1-abb6032f5eb4}" -2025-01-25T14:29:48.870Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:48.765272698 +0000 UTC m=+98.619597680 (1 nodes), 54.868225ms from start of poll, 55.367727ms from cluster start. Timeout at 2025-01-25 14:29:48.870276578 +0000 UTC m=+98.724601560, 105.00388ms after stability -2025-01-25T14:29:48.870Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:29:48.976Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:29:48.87082918 +0000 UTC m=+98.725154162 (1 nodes), 1.32µs from start of poll, 160.924209ms from cluster start. Timeout at 2025-01-25 14:29:48.976021853 +0000 UTC m=+98.830346835, 105.192673ms after stability -2025-01-25T14:29:48.978Z [ERROR] server-f1ee4f06-f9bc-d6e0-713a-7423eb2722d0: peer has newer term, stopping replication: peer="{Voter server-a3c10448-efa2-4c57-99b1-abb6032f5eb4 a3c10448-efa2-4c57-99b1-abb6032f5eb4}" -2025-01-25T14:29:48.978Z [INFO] server-f1ee4f06-f9bc-d6e0-713a-7423eb2722d0: aborting pipeline replication: peer="{Voter server-a3c10448-efa2-4c57-99b1-abb6032f5eb4 a3c10448-efa2-4c57-99b1-abb6032f5eb4}" -2025-01-25T14:29:48.978Z [INFO] server-f1ee4f06-f9bc-d6e0-713a-7423eb2722d0: entering follower state: follower="Node at f1ee4f06-f9bc-d6e0-713a-7423eb2722d0 [Follower]" leader-address= leader-id= ---- PASS: TestRaft_VerifyLeader_Fail (0.27s) +2025-01-25T14:40:36.266Z [INFO] server-5e0955e7-f2bb-86a5-f7b5-cfd626899091: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5e0955e7-f2bb-86a5-f7b5-cfd626899091 Address:5e0955e7-f2bb-86a5-f7b5-cfd626899091} {Suffrage:Voter ID:server-8381cfd7-e4c7-4256-ed7c-fa7402d73612 Address:8381cfd7-e4c7-4256-ed7c-fa7402d73612}]" +2025-01-25T14:40:36.266Z [INFO] server-5e0955e7-f2bb-86a5-f7b5-cfd626899091: entering follower state: follower="Node at 5e0955e7-f2bb-86a5-f7b5-cfd626899091 [Follower]" leader-address= leader-id= +2025-01-25T14:40:36.266Z [INFO] server-8381cfd7-e4c7-4256-ed7c-fa7402d73612: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5e0955e7-f2bb-86a5-f7b5-cfd626899091 Address:5e0955e7-f2bb-86a5-f7b5-cfd626899091} {Suffrage:Voter ID:server-8381cfd7-e4c7-4256-ed7c-fa7402d73612 Address:8381cfd7-e4c7-4256-ed7c-fa7402d73612}]" +2025-01-25T14:40:36.266Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:36.266Z [INFO] server-8381cfd7-e4c7-4256-ed7c-fa7402d73612: entering follower state: follower="Node at 8381cfd7-e4c7-4256-ed7c-fa7402d73612 [Follower]" leader-address= leader-id= +2025-01-25T14:40:36.346Z [WARN] server-8381cfd7-e4c7-4256-ed7c-fa7402d73612: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:36.346Z [INFO] server-8381cfd7-e4c7-4256-ed7c-fa7402d73612: entering candidate state: node="Node at 8381cfd7-e4c7-4256-ed7c-fa7402d73612 [Candidate]" term=2 +2025-01-25T14:40:36.347Z [INFO] server-8381cfd7-e4c7-4256-ed7c-fa7402d73612: election won: term=2 tally=2 +2025-01-25T14:40:36.347Z [INFO] server-8381cfd7-e4c7-4256-ed7c-fa7402d73612: entering leader state: leader="Node at 8381cfd7-e4c7-4256-ed7c-fa7402d73612 [Leader]" +2025-01-25T14:40:36.347Z [INFO] server-8381cfd7-e4c7-4256-ed7c-fa7402d73612: added peer, starting replication: peer=server-5e0955e7-f2bb-86a5-f7b5-cfd626899091 +2025-01-25T14:40:36.348Z [INFO] server-8381cfd7-e4c7-4256-ed7c-fa7402d73612: pipelining replication: peer="{Voter server-5e0955e7-f2bb-86a5-f7b5-cfd626899091 5e0955e7-f2bb-86a5-f7b5-cfd626899091}" +2025-01-25T14:40:36.452Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:36.347553151 +0000 UTC m=+105.393392469 (1 nodes), 80.695569ms from start of poll, 81.909197ms from cluster start. Timeout at 2025-01-25 14:40:36.452560734 +0000 UTC m=+105.498400052, 105.007583ms after stability +2025-01-25T14:40:36.452Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:40:36.558Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:40:36.453005958 +0000 UTC m=+105.498845276 (1 nodes), 4.667µs from start of poll, 187.362004ms from cluster start. Timeout at 2025-01-25 14:40:36.558263792 +0000 UTC m=+105.604103110, 105.257834ms after stability +2025-01-25T14:40:36.563Z [ERROR] server-8381cfd7-e4c7-4256-ed7c-fa7402d73612: peer has newer term, stopping replication: peer="{Voter server-5e0955e7-f2bb-86a5-f7b5-cfd626899091 5e0955e7-f2bb-86a5-f7b5-cfd626899091}" +2025-01-25T14:40:36.563Z [INFO] server-8381cfd7-e4c7-4256-ed7c-fa7402d73612: aborting pipeline replication: peer="{Voter server-5e0955e7-f2bb-86a5-f7b5-cfd626899091 5e0955e7-f2bb-86a5-f7b5-cfd626899091}" +2025-01-25T14:40:36.563Z [INFO] server-8381cfd7-e4c7-4256-ed7c-fa7402d73612: entering follower state: follower="Node at 8381cfd7-e4c7-4256-ed7c-fa7402d73612 [Follower]" leader-address= leader-id= +--- PASS: TestRaft_VerifyLeader_Fail (0.30s) === RUN TestRaft_VerifyLeader_PartialConnect -2025-01-25T14:29:48.981Z [INFO] server-51eeb3cc-b3e1-5b59-cc9f-3886c06f4123: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-51eeb3cc-b3e1-5b59-cc9f-3886c06f4123 Address:51eeb3cc-b3e1-5b59-cc9f-3886c06f4123} {Suffrage:Voter ID:server-a78708f3-8e3e-b88e-73c1-51063f4f36cb Address:a78708f3-8e3e-b88e-73c1-51063f4f36cb} {Suffrage:Voter ID:server-7d8797c8-f722-9ede-6de0-0b4d6c7b1041 Address:7d8797c8-f722-9ede-6de0-0b4d6c7b1041}]" -2025-01-25T14:29:48.981Z [INFO] server-a78708f3-8e3e-b88e-73c1-51063f4f36cb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-51eeb3cc-b3e1-5b59-cc9f-3886c06f4123 Address:51eeb3cc-b3e1-5b59-cc9f-3886c06f4123} {Suffrage:Voter ID:server-a78708f3-8e3e-b88e-73c1-51063f4f36cb Address:a78708f3-8e3e-b88e-73c1-51063f4f36cb} {Suffrage:Voter ID:server-7d8797c8-f722-9ede-6de0-0b4d6c7b1041 Address:7d8797c8-f722-9ede-6de0-0b4d6c7b1041}]" -2025-01-25T14:29:48.981Z [INFO] server-51eeb3cc-b3e1-5b59-cc9f-3886c06f4123: entering follower state: follower="Node at 51eeb3cc-b3e1-5b59-cc9f-3886c06f4123 [Follower]" leader-address= leader-id= -2025-01-25T14:29:48.981Z [INFO] server-a78708f3-8e3e-b88e-73c1-51063f4f36cb: entering follower state: follower="Node at a78708f3-8e3e-b88e-73c1-51063f4f36cb [Follower]" leader-address= leader-id= -2025-01-25T14:29:48.981Z [INFO] server-7d8797c8-f722-9ede-6de0-0b4d6c7b1041: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-51eeb3cc-b3e1-5b59-cc9f-3886c06f4123 Address:51eeb3cc-b3e1-5b59-cc9f-3886c06f4123} {Suffrage:Voter ID:server-a78708f3-8e3e-b88e-73c1-51063f4f36cb Address:a78708f3-8e3e-b88e-73c1-51063f4f36cb} {Suffrage:Voter ID:server-7d8797c8-f722-9ede-6de0-0b4d6c7b1041 Address:7d8797c8-f722-9ede-6de0-0b4d6c7b1041}]" -2025-01-25T14:29:48.981Z [INFO] server-7d8797c8-f722-9ede-6de0-0b4d6c7b1041: entering follower state: follower="Node at 7d8797c8-f722-9ede-6de0-0b4d6c7b1041 [Follower]" leader-address= leader-id= -2025-01-25T14:29:48.981Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:49.051Z [WARN] server-7d8797c8-f722-9ede-6de0-0b4d6c7b1041: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:49.052Z [INFO] server-7d8797c8-f722-9ede-6de0-0b4d6c7b1041: entering candidate state: node="Node at 7d8797c8-f722-9ede-6de0-0b4d6c7b1041 [Candidate]" term=2 -2025-01-25T14:29:49.052Z [INFO] server-7d8797c8-f722-9ede-6de0-0b4d6c7b1041: election won: term=2 tally=2 -2025-01-25T14:29:49.052Z [INFO] server-7d8797c8-f722-9ede-6de0-0b4d6c7b1041: entering leader state: leader="Node at 7d8797c8-f722-9ede-6de0-0b4d6c7b1041 [Leader]" -2025-01-25T14:29:49.053Z [INFO] server-7d8797c8-f722-9ede-6de0-0b4d6c7b1041: added peer, starting replication: peer=server-51eeb3cc-b3e1-5b59-cc9f-3886c06f4123 -2025-01-25T14:29:49.053Z [INFO] server-7d8797c8-f722-9ede-6de0-0b4d6c7b1041: added peer, starting replication: peer=server-a78708f3-8e3e-b88e-73c1-51063f4f36cb -2025-01-25T14:29:49.053Z [INFO] server-7d8797c8-f722-9ede-6de0-0b4d6c7b1041: pipelining replication: peer="{Voter server-a78708f3-8e3e-b88e-73c1-51063f4f36cb a78708f3-8e3e-b88e-73c1-51063f4f36cb}" -2025-01-25T14:29:49.056Z [INFO] server-7d8797c8-f722-9ede-6de0-0b4d6c7b1041: pipelining replication: peer="{Voter server-51eeb3cc-b3e1-5b59-cc9f-3886c06f4123 51eeb3cc-b3e1-5b59-cc9f-3886c06f4123}" -2025-01-25T14:29:49.160Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:49.055654528 +0000 UTC m=+98.909979510 (1 nodes), 73.707947ms from start of poll, 74.405282ms from cluster start. Timeout at 2025-01-25 14:29:49.160658688 +0000 UTC m=+99.014983670, 105.00416ms after stability -2025-01-25T14:29:49.166Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:29:49.272Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:29:49.16676305 +0000 UTC m=+99.021088032 (2 nodes), 1.96µs from start of poll, 185.513804ms from cluster start. Timeout at 2025-01-25 14:29:49.271973082 +0000 UTC m=+99.126298104, 105.210072ms after stability - raft_test.go:1968: [INFO] Disconnecting Node at 51eeb3cc-b3e1-5b59-cc9f-3886c06f4123 [Follower] -2025-01-25T14:29:49.272Z [ERROR] server-7d8797c8-f722-9ede-6de0-0b4d6c7b1041: failed to heartbeat to: peer=51eeb3cc-b3e1-5b59-cc9f-3886c06f4123 backoff time=10ms error="failed to connect to peer: 51eeb3cc-b3e1-5b59-cc9f-3886c06f4123" -2025-01-25T14:29:49.273Z [INFO] server-7d8797c8-f722-9ede-6de0-0b4d6c7b1041: aborting pipeline replication: peer="{Voter server-a78708f3-8e3e-b88e-73c1-51063f4f36cb a78708f3-8e3e-b88e-73c1-51063f4f36cb}" -2025-01-25T14:29:49.273Z [INFO] server-7d8797c8-f722-9ede-6de0-0b4d6c7b1041: aborting pipeline replication: peer="{Voter server-51eeb3cc-b3e1-5b59-cc9f-3886c06f4123 51eeb3cc-b3e1-5b59-cc9f-3886c06f4123}" ---- PASS: TestRaft_VerifyLeader_PartialConnect (0.29s) +2025-01-25T14:40:36.571Z [INFO] server-70992817-07be-2980-c952-6cb88c2a9505: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-70992817-07be-2980-c952-6cb88c2a9505 Address:70992817-07be-2980-c952-6cb88c2a9505} {Suffrage:Voter ID:server-d471963d-12b8-50f1-fa3e-53ecacb601de Address:d471963d-12b8-50f1-fa3e-53ecacb601de} {Suffrage:Voter ID:server-b9927782-4c08-bd6c-a369-af3e37508772 Address:b9927782-4c08-bd6c-a369-af3e37508772}]" +2025-01-25T14:40:36.571Z [INFO] server-70992817-07be-2980-c952-6cb88c2a9505: entering follower state: follower="Node at 70992817-07be-2980-c952-6cb88c2a9505 [Follower]" leader-address= leader-id= +2025-01-25T14:40:36.572Z [INFO] server-d471963d-12b8-50f1-fa3e-53ecacb601de: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-70992817-07be-2980-c952-6cb88c2a9505 Address:70992817-07be-2980-c952-6cb88c2a9505} {Suffrage:Voter ID:server-d471963d-12b8-50f1-fa3e-53ecacb601de Address:d471963d-12b8-50f1-fa3e-53ecacb601de} {Suffrage:Voter ID:server-b9927782-4c08-bd6c-a369-af3e37508772 Address:b9927782-4c08-bd6c-a369-af3e37508772}]" +2025-01-25T14:40:36.573Z [INFO] server-d471963d-12b8-50f1-fa3e-53ecacb601de: entering follower state: follower="Node at d471963d-12b8-50f1-fa3e-53ecacb601de [Follower]" leader-address= leader-id= +2025-01-25T14:40:36.573Z [INFO] server-b9927782-4c08-bd6c-a369-af3e37508772: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-70992817-07be-2980-c952-6cb88c2a9505 Address:70992817-07be-2980-c952-6cb88c2a9505} {Suffrage:Voter ID:server-d471963d-12b8-50f1-fa3e-53ecacb601de Address:d471963d-12b8-50f1-fa3e-53ecacb601de} {Suffrage:Voter ID:server-b9927782-4c08-bd6c-a369-af3e37508772 Address:b9927782-4c08-bd6c-a369-af3e37508772}]" +2025-01-25T14:40:36.573Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:36.573Z [INFO] server-b9927782-4c08-bd6c-a369-af3e37508772: entering follower state: follower="Node at b9927782-4c08-bd6c-a369-af3e37508772 [Follower]" leader-address= leader-id= +2025-01-25T14:40:36.659Z [WARN] server-b9927782-4c08-bd6c-a369-af3e37508772: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:36.659Z [INFO] server-b9927782-4c08-bd6c-a369-af3e37508772: entering candidate state: node="Node at b9927782-4c08-bd6c-a369-af3e37508772 [Candidate]" term=2 +2025-01-25T14:40:36.660Z [INFO] server-b9927782-4c08-bd6c-a369-af3e37508772: election won: term=2 tally=2 +2025-01-25T14:40:36.660Z [INFO] server-b9927782-4c08-bd6c-a369-af3e37508772: entering leader state: leader="Node at b9927782-4c08-bd6c-a369-af3e37508772 [Leader]" +2025-01-25T14:40:36.660Z [INFO] server-b9927782-4c08-bd6c-a369-af3e37508772: added peer, starting replication: peer=server-70992817-07be-2980-c952-6cb88c2a9505 +2025-01-25T14:40:36.660Z [INFO] server-b9927782-4c08-bd6c-a369-af3e37508772: added peer, starting replication: peer=server-d471963d-12b8-50f1-fa3e-53ecacb601de +2025-01-25T14:40:36.660Z [INFO] server-b9927782-4c08-bd6c-a369-af3e37508772: pipelining replication: peer="{Voter server-70992817-07be-2980-c952-6cb88c2a9505 70992817-07be-2980-c952-6cb88c2a9505}" +2025-01-25T14:40:36.661Z [INFO] server-b9927782-4c08-bd6c-a369-af3e37508772: pipelining replication: peer="{Voter server-d471963d-12b8-50f1-fa3e-53ecacb601de d471963d-12b8-50f1-fa3e-53ecacb601de}" +2025-01-25T14:40:36.765Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:36.660206436 +0000 UTC m=+105.706045754 (1 nodes), 86.442085ms from start of poll, 89.251427ms from cluster start. Timeout at 2025-01-25 14:40:36.765229187 +0000 UTC m=+105.811068504, 105.02275ms after stability +2025-01-25T14:40:36.771Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:40:36.876Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:40:36.77135451 +0000 UTC m=+105.817193827 (2 nodes), 7µs from start of poll, 200.3995ms from cluster start. Timeout at 2025-01-25 14:40:36.876599219 +0000 UTC m=+105.922438245, 105.244418ms after stability + raft_test.go:1968: [INFO] Disconnecting Node at 70992817-07be-2980-c952-6cb88c2a9505 [Follower] +2025-01-25T14:40:36.877Z [ERROR] server-b9927782-4c08-bd6c-a369-af3e37508772: failed to heartbeat to: peer=70992817-07be-2980-c952-6cb88c2a9505 backoff time=10ms error="failed to connect to peer: 70992817-07be-2980-c952-6cb88c2a9505" +2025-01-25T14:40:36.877Z [INFO] server-b9927782-4c08-bd6c-a369-af3e37508772: aborting pipeline replication: peer="{Voter server-70992817-07be-2980-c952-6cb88c2a9505 70992817-07be-2980-c952-6cb88c2a9505}" +2025-01-25T14:40:36.877Z [INFO] server-b9927782-4c08-bd6c-a369-af3e37508772: aborting pipeline replication: peer="{Voter server-d471963d-12b8-50f1-fa3e-53ecacb601de d471963d-12b8-50f1-fa3e-53ecacb601de}" +--- PASS: TestRaft_VerifyLeader_PartialConnect (0.31s) === RUN TestRaft_NotifyCh -2025-01-25T14:29:49.274Z [INFO] server-d3105446-26dd-4d5c-a665-d43a0fb786a2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d3105446-26dd-4d5c-a665-d43a0fb786a2 Address:d3105446-26dd-4d5c-a665-d43a0fb786a2}]" -2025-01-25T14:29:49.274Z [INFO] server-d3105446-26dd-4d5c-a665-d43a0fb786a2: entering follower state: follower="Node at d3105446-26dd-4d5c-a665-d43a0fb786a2 [Follower]" leader-address= leader-id= -2025-01-25T14:29:49.337Z [WARN] server-d3105446-26dd-4d5c-a665-d43a0fb786a2: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:49.337Z [INFO] server-d3105446-26dd-4d5c-a665-d43a0fb786a2: entering candidate state: node="Node at d3105446-26dd-4d5c-a665-d43a0fb786a2 [Candidate]" term=2 -2025-01-25T14:29:49.338Z [INFO] server-d3105446-26dd-4d5c-a665-d43a0fb786a2: election won: term=2 tally=1 -2025-01-25T14:29:49.338Z [INFO] server-d3105446-26dd-4d5c-a665-d43a0fb786a2: entering leader state: leader="Node at d3105446-26dd-4d5c-a665-d43a0fb786a2 [Leader]" ---- PASS: TestRaft_NotifyCh (0.06s) +2025-01-25T14:40:36.884Z [INFO] server-e9539374-1b2e-c1fa-8e51-b338fca16da1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e9539374-1b2e-c1fa-8e51-b338fca16da1 Address:e9539374-1b2e-c1fa-8e51-b338fca16da1}]" +2025-01-25T14:40:36.884Z [INFO] server-e9539374-1b2e-c1fa-8e51-b338fca16da1: entering follower state: follower="Node at e9539374-1b2e-c1fa-8e51-b338fca16da1 [Follower]" leader-address= leader-id= +2025-01-25T14:40:36.975Z [WARN] server-e9539374-1b2e-c1fa-8e51-b338fca16da1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:36.975Z [INFO] server-e9539374-1b2e-c1fa-8e51-b338fca16da1: entering candidate state: node="Node at e9539374-1b2e-c1fa-8e51-b338fca16da1 [Candidate]" term=2 +2025-01-25T14:40:36.975Z [INFO] server-e9539374-1b2e-c1fa-8e51-b338fca16da1: election won: term=2 tally=1 +2025-01-25T14:40:36.975Z [INFO] server-e9539374-1b2e-c1fa-8e51-b338fca16da1: entering leader state: leader="Node at e9539374-1b2e-c1fa-8e51-b338fca16da1 [Leader]" +--- PASS: TestRaft_NotifyCh (0.10s) === RUN TestRaft_AppendEntry -2025-01-25T14:29:49.340Z [INFO] server-94acb135-878e-eb19-b4ab-718c571a6bdb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-94acb135-878e-eb19-b4ab-718c571a6bdb Address:94acb135-878e-eb19-b4ab-718c571a6bdb} {Suffrage:Voter ID:server-242e5f70-7bd4-d6d3-4f9e-98828be08e1a Address:242e5f70-7bd4-d6d3-4f9e-98828be08e1a} {Suffrage:Voter ID:server-1e3277e5-461d-8989-fa76-45160a8b152e Address:1e3277e5-461d-8989-fa76-45160a8b152e}]" -2025-01-25T14:29:49.340Z [INFO] server-242e5f70-7bd4-d6d3-4f9e-98828be08e1a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-94acb135-878e-eb19-b4ab-718c571a6bdb Address:94acb135-878e-eb19-b4ab-718c571a6bdb} {Suffrage:Voter ID:server-242e5f70-7bd4-d6d3-4f9e-98828be08e1a Address:242e5f70-7bd4-d6d3-4f9e-98828be08e1a} {Suffrage:Voter ID:server-1e3277e5-461d-8989-fa76-45160a8b152e Address:1e3277e5-461d-8989-fa76-45160a8b152e}]" -2025-01-25T14:29:49.340Z [INFO] server-94acb135-878e-eb19-b4ab-718c571a6bdb: entering follower state: follower="Node at 94acb135-878e-eb19-b4ab-718c571a6bdb [Follower]" leader-address= leader-id= -2025-01-25T14:29:49.340Z [INFO] server-242e5f70-7bd4-d6d3-4f9e-98828be08e1a: entering follower state: follower="Node at 242e5f70-7bd4-d6d3-4f9e-98828be08e1a [Follower]" leader-address= leader-id= -2025-01-25T14:29:49.340Z [INFO] server-1e3277e5-461d-8989-fa76-45160a8b152e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-94acb135-878e-eb19-b4ab-718c571a6bdb Address:94acb135-878e-eb19-b4ab-718c571a6bdb} {Suffrage:Voter ID:server-242e5f70-7bd4-d6d3-4f9e-98828be08e1a Address:242e5f70-7bd4-d6d3-4f9e-98828be08e1a} {Suffrage:Voter ID:server-1e3277e5-461d-8989-fa76-45160a8b152e Address:1e3277e5-461d-8989-fa76-45160a8b152e}]" -2025-01-25T14:29:49.340Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:29:49.341Z [INFO] server-1e3277e5-461d-8989-fa76-45160a8b152e: entering follower state: follower="Node at 1e3277e5-461d-8989-fa76-45160a8b152e [Follower]" leader-address= leader-id= -2025-01-25T14:29:49.391Z [WARN] server-1e3277e5-461d-8989-fa76-45160a8b152e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:49.391Z [INFO] server-1e3277e5-461d-8989-fa76-45160a8b152e: entering candidate state: node="Node at 1e3277e5-461d-8989-fa76-45160a8b152e [Candidate]" term=2 -2025-01-25T14:29:49.391Z [INFO] server-1e3277e5-461d-8989-fa76-45160a8b152e: election won: term=2 tally=2 -2025-01-25T14:29:49.391Z [INFO] server-1e3277e5-461d-8989-fa76-45160a8b152e: entering leader state: leader="Node at 1e3277e5-461d-8989-fa76-45160a8b152e [Leader]" -2025-01-25T14:29:49.391Z [INFO] server-1e3277e5-461d-8989-fa76-45160a8b152e: added peer, starting replication: peer=server-94acb135-878e-eb19-b4ab-718c571a6bdb -2025-01-25T14:29:49.391Z [INFO] server-1e3277e5-461d-8989-fa76-45160a8b152e: added peer, starting replication: peer=server-242e5f70-7bd4-d6d3-4f9e-98828be08e1a -2025-01-25T14:29:49.392Z [INFO] server-1e3277e5-461d-8989-fa76-45160a8b152e: pipelining replication: peer="{Voter server-242e5f70-7bd4-d6d3-4f9e-98828be08e1a 242e5f70-7bd4-d6d3-4f9e-98828be08e1a}" -2025-01-25T14:29:49.392Z [INFO] server-1e3277e5-461d-8989-fa76-45160a8b152e: pipelining replication: peer="{Voter server-94acb135-878e-eb19-b4ab-718c571a6bdb 94acb135-878e-eb19-b4ab-718c571a6bdb}" -2025-01-25T14:29:49.497Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:29:49.392011144 +0000 UTC m=+99.246336126 (2 nodes), 50.976926ms from start of poll, 51.7025ms from cluster start. Timeout at 2025-01-25 14:29:49.497015183 +0000 UTC m=+99.351340165, 105.004039ms after stability -2025-01-25T14:29:49.497Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:49.603Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:49.497622503 +0000 UTC m=+99.351947485 (1 nodes), 2.48µs from start of poll, 157.313859ms from cluster start. Timeout at 2025-01-25 14:29:49.602885333 +0000 UTC m=+99.457210355, 105.26287ms after stability -2025-01-25T14:29:49.603Z [WARN] server-94acb135-878e-eb19-b4ab-718c571a6bdb: clearing log suffix: from=1 to=2 -2025-01-25T14:29:49.603Z [WARN] server-94acb135-878e-eb19-b4ab-718c571a6bdb: skipping application of old log: index=1 -2025-01-25T14:29:49.603Z [WARN] server-94acb135-878e-eb19-b4ab-718c571a6bdb: skipping application of old log: index=1 -2025-01-25T14:29:49.603Z [INFO] server-1e3277e5-461d-8989-fa76-45160a8b152e: aborting pipeline replication: peer="{Voter server-242e5f70-7bd4-d6d3-4f9e-98828be08e1a 242e5f70-7bd4-d6d3-4f9e-98828be08e1a}" -2025-01-25T14:29:49.603Z [INFO] server-1e3277e5-461d-8989-fa76-45160a8b152e: aborting pipeline replication: peer="{Voter server-94acb135-878e-eb19-b4ab-718c571a6bdb 94acb135-878e-eb19-b4ab-718c571a6bdb}" ---- PASS: TestRaft_AppendEntry (0.27s) +2025-01-25T14:40:36.982Z [INFO] server-4542b37b-bd49-28eb-6edc-b11e8c28810c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4542b37b-bd49-28eb-6edc-b11e8c28810c Address:4542b37b-bd49-28eb-6edc-b11e8c28810c} {Suffrage:Voter ID:server-04973041-1a3c-d24c-63ab-269748397d46 Address:04973041-1a3c-d24c-63ab-269748397d46} {Suffrage:Voter ID:server-9a130ef6-1b21-e361-5205-edeefa1b18ff Address:9a130ef6-1b21-e361-5205-edeefa1b18ff}]" +2025-01-25T14:40:36.982Z [INFO] server-4542b37b-bd49-28eb-6edc-b11e8c28810c: entering follower state: follower="Node at 4542b37b-bd49-28eb-6edc-b11e8c28810c [Follower]" leader-address= leader-id= +2025-01-25T14:40:36.983Z [INFO] server-04973041-1a3c-d24c-63ab-269748397d46: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4542b37b-bd49-28eb-6edc-b11e8c28810c Address:4542b37b-bd49-28eb-6edc-b11e8c28810c} {Suffrage:Voter ID:server-04973041-1a3c-d24c-63ab-269748397d46 Address:04973041-1a3c-d24c-63ab-269748397d46} {Suffrage:Voter ID:server-9a130ef6-1b21-e361-5205-edeefa1b18ff Address:9a130ef6-1b21-e361-5205-edeefa1b18ff}]" +2025-01-25T14:40:36.983Z [INFO] server-04973041-1a3c-d24c-63ab-269748397d46: entering follower state: follower="Node at 04973041-1a3c-d24c-63ab-269748397d46 [Follower]" leader-address= leader-id= +2025-01-25T14:40:36.983Z [INFO] server-9a130ef6-1b21-e361-5205-edeefa1b18ff: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4542b37b-bd49-28eb-6edc-b11e8c28810c Address:4542b37b-bd49-28eb-6edc-b11e8c28810c} {Suffrage:Voter ID:server-04973041-1a3c-d24c-63ab-269748397d46 Address:04973041-1a3c-d24c-63ab-269748397d46} {Suffrage:Voter ID:server-9a130ef6-1b21-e361-5205-edeefa1b18ff Address:9a130ef6-1b21-e361-5205-edeefa1b18ff}]" +2025-01-25T14:40:36.984Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:40:36.984Z [INFO] server-9a130ef6-1b21-e361-5205-edeefa1b18ff: entering follower state: follower="Node at 9a130ef6-1b21-e361-5205-edeefa1b18ff [Follower]" leader-address= leader-id= +2025-01-25T14:40:37.058Z [WARN] server-04973041-1a3c-d24c-63ab-269748397d46: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:37.058Z [INFO] server-04973041-1a3c-d24c-63ab-269748397d46: entering candidate state: node="Node at 04973041-1a3c-d24c-63ab-269748397d46 [Candidate]" term=2 +2025-01-25T14:40:37.059Z [INFO] server-04973041-1a3c-d24c-63ab-269748397d46: election won: term=2 tally=2 +2025-01-25T14:40:37.059Z [INFO] server-04973041-1a3c-d24c-63ab-269748397d46: entering leader state: leader="Node at 04973041-1a3c-d24c-63ab-269748397d46 [Leader]" +2025-01-25T14:40:37.059Z [INFO] server-04973041-1a3c-d24c-63ab-269748397d46: added peer, starting replication: peer=server-4542b37b-bd49-28eb-6edc-b11e8c28810c +2025-01-25T14:40:37.059Z [INFO] server-04973041-1a3c-d24c-63ab-269748397d46: added peer, starting replication: peer=server-9a130ef6-1b21-e361-5205-edeefa1b18ff +2025-01-25T14:40:37.060Z [INFO] server-04973041-1a3c-d24c-63ab-269748397d46: pipelining replication: peer="{Voter server-9a130ef6-1b21-e361-5205-edeefa1b18ff 9a130ef6-1b21-e361-5205-edeefa1b18ff}" +2025-01-25T14:40:37.060Z [INFO] server-04973041-1a3c-d24c-63ab-269748397d46: pipelining replication: peer="{Voter server-4542b37b-bd49-28eb-6edc-b11e8c28810c 4542b37b-bd49-28eb-6edc-b11e8c28810c}" +2025-01-25T14:40:37.166Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:40:37.059699724 +0000 UTC m=+106.105539042 (2 nodes), 75.551303ms from start of poll, 78.152978ms from cluster start. Timeout at 2025-01-25 14:40:37.164721307 +0000 UTC m=+106.210560625, 105.021583ms after stability +2025-01-25T14:40:37.166Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:37.273Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:37.16695112 +0000 UTC m=+106.212790437 (1 nodes), 6.125µs from start of poll, 185.404373ms from cluster start. Timeout at 2025-01-25 14:40:37.27218912 +0000 UTC m=+106.318028438, 105.238001ms after stability +2025-01-25T14:40:37.273Z [WARN] server-4542b37b-bd49-28eb-6edc-b11e8c28810c: clearing log suffix: from=1 to=2 +2025-01-25T14:40:37.273Z [WARN] server-4542b37b-bd49-28eb-6edc-b11e8c28810c: skipping application of old log: index=1 +2025-01-25T14:40:37.273Z [WARN] server-4542b37b-bd49-28eb-6edc-b11e8c28810c: skipping application of old log: index=1 +2025-01-25T14:40:37.274Z [INFO] server-04973041-1a3c-d24c-63ab-269748397d46: aborting pipeline replication: peer="{Voter server-9a130ef6-1b21-e361-5205-edeefa1b18ff 9a130ef6-1b21-e361-5205-edeefa1b18ff}" +2025-01-25T14:40:37.274Z [INFO] server-04973041-1a3c-d24c-63ab-269748397d46: aborting pipeline replication: peer="{Voter server-4542b37b-bd49-28eb-6edc-b11e8c28810c 4542b37b-bd49-28eb-6edc-b11e8c28810c}" +--- PASS: TestRaft_AppendEntry (0.30s) === RUN TestRaft_VotingGrant_WhenLeaderAvailable -2025-01-25T14:29:49.605Z [INFO] server-e35948a9-9d50-b227-eb75-61073348b35e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e35948a9-9d50-b227-eb75-61073348b35e Address:e35948a9-9d50-b227-eb75-61073348b35e} {Suffrage:Voter ID:server-07fa1e12-081c-37c5-dbd9-0194a7bf379a Address:07fa1e12-081c-37c5-dbd9-0194a7bf379a} {Suffrage:Voter ID:server-57aeb195-dcc1-871a-fba0-c2771a5fef27 Address:57aeb195-dcc1-871a-fba0-c2771a5fef27}]" -2025-01-25T14:29:49.605Z [INFO] server-e35948a9-9d50-b227-eb75-61073348b35e: entering follower state: follower="Node at e35948a9-9d50-b227-eb75-61073348b35e [Follower]" leader-address= leader-id= -2025-01-25T14:29:49.606Z [INFO] server-07fa1e12-081c-37c5-dbd9-0194a7bf379a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e35948a9-9d50-b227-eb75-61073348b35e Address:e35948a9-9d50-b227-eb75-61073348b35e} {Suffrage:Voter ID:server-07fa1e12-081c-37c5-dbd9-0194a7bf379a Address:07fa1e12-081c-37c5-dbd9-0194a7bf379a} {Suffrage:Voter ID:server-57aeb195-dcc1-871a-fba0-c2771a5fef27 Address:57aeb195-dcc1-871a-fba0-c2771a5fef27}]" -2025-01-25T14:29:49.606Z [INFO] server-07fa1e12-081c-37c5-dbd9-0194a7bf379a: entering follower state: follower="Node at 07fa1e12-081c-37c5-dbd9-0194a7bf379a [Follower]" leader-address= leader-id= -2025-01-25T14:29:49.606Z [INFO] server-57aeb195-dcc1-871a-fba0-c2771a5fef27: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e35948a9-9d50-b227-eb75-61073348b35e Address:e35948a9-9d50-b227-eb75-61073348b35e} {Suffrage:Voter ID:server-07fa1e12-081c-37c5-dbd9-0194a7bf379a Address:07fa1e12-081c-37c5-dbd9-0194a7bf379a} {Suffrage:Voter ID:server-57aeb195-dcc1-871a-fba0-c2771a5fef27 Address:57aeb195-dcc1-871a-fba0-c2771a5fef27}]" -2025-01-25T14:29:49.606Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:29:49.606Z [INFO] server-57aeb195-dcc1-871a-fba0-c2771a5fef27: entering follower state: follower="Node at 57aeb195-dcc1-871a-fba0-c2771a5fef27 [Follower]" leader-address= leader-id= -2025-01-25T14:29:49.663Z [WARN] server-57aeb195-dcc1-871a-fba0-c2771a5fef27: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:49.663Z [INFO] server-57aeb195-dcc1-871a-fba0-c2771a5fef27: entering candidate state: node="Node at 57aeb195-dcc1-871a-fba0-c2771a5fef27 [Candidate]" term=2 -2025-01-25T14:29:49.663Z [INFO] server-57aeb195-dcc1-871a-fba0-c2771a5fef27: election won: term=2 tally=2 -2025-01-25T14:29:49.663Z [INFO] server-57aeb195-dcc1-871a-fba0-c2771a5fef27: entering leader state: leader="Node at 57aeb195-dcc1-871a-fba0-c2771a5fef27 [Leader]" -2025-01-25T14:29:49.663Z [INFO] server-57aeb195-dcc1-871a-fba0-c2771a5fef27: added peer, starting replication: peer=server-e35948a9-9d50-b227-eb75-61073348b35e -2025-01-25T14:29:49.663Z [INFO] server-57aeb195-dcc1-871a-fba0-c2771a5fef27: added peer, starting replication: peer=server-07fa1e12-081c-37c5-dbd9-0194a7bf379a -2025-01-25T14:29:49.664Z [INFO] server-57aeb195-dcc1-871a-fba0-c2771a5fef27: pipelining replication: peer="{Voter server-e35948a9-9d50-b227-eb75-61073348b35e e35948a9-9d50-b227-eb75-61073348b35e}" -2025-01-25T14:29:49.664Z [INFO] server-57aeb195-dcc1-871a-fba0-c2771a5fef27: pipelining replication: peer="{Voter server-07fa1e12-081c-37c5-dbd9-0194a7bf379a 07fa1e12-081c-37c5-dbd9-0194a7bf379a}" -2025-01-25T14:29:49.769Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:29:49.663853081 +0000 UTC m=+99.518178023 (2 nodes), 57.535769ms from start of poll, 58.194186ms from cluster start. Timeout at 2025-01-25 14:29:49.768858081 +0000 UTC m=+99.623183063, 105.00504ms after stability -2025-01-25T14:29:49.769Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:49.874Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:49.769557477 +0000 UTC m=+99.623882459 (1 nodes), 1.92µs from start of poll, 163.898622ms from cluster start. Timeout at 2025-01-25 14:29:49.874764429 +0000 UTC m=+99.729089571, 105.207112ms after stability -2025-01-25T14:29:49.875Z [WARN] server-07fa1e12-081c-37c5-dbd9-0194a7bf379a: rejecting vote request since we have a leader: from=e35948a9-9d50-b227-eb75-61073348b35e leader=57aeb195-dcc1-871a-fba0-c2771a5fef27 leader-id=server-57aeb195-dcc1-871a-fba0-c2771a5fef27 -2025-01-25T14:29:49.875Z [ERROR] server-57aeb195-dcc1-871a-fba0-c2771a5fef27: peer has newer term, stopping replication: peer="{Voter server-e35948a9-9d50-b227-eb75-61073348b35e e35948a9-9d50-b227-eb75-61073348b35e}" -2025-01-25T14:29:49.875Z [INFO] server-57aeb195-dcc1-871a-fba0-c2771a5fef27: aborting pipeline replication: peer="{Voter server-e35948a9-9d50-b227-eb75-61073348b35e e35948a9-9d50-b227-eb75-61073348b35e}" -2025-01-25T14:29:49.875Z [INFO] server-57aeb195-dcc1-871a-fba0-c2771a5fef27: aborting pipeline replication: peer="{Voter server-07fa1e12-081c-37c5-dbd9-0194a7bf379a 07fa1e12-081c-37c5-dbd9-0194a7bf379a}" ---- PASS: TestRaft_VotingGrant_WhenLeaderAvailable (0.27s) +2025-01-25T14:40:37.282Z [INFO] server-60e1154b-c6fd-99ca-b5c4-554abef400d9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-60e1154b-c6fd-99ca-b5c4-554abef400d9 Address:60e1154b-c6fd-99ca-b5c4-554abef400d9} {Suffrage:Voter ID:server-00cb6dea-19ab-a4c3-00c4-27c5482baa29 Address:00cb6dea-19ab-a4c3-00c4-27c5482baa29} {Suffrage:Voter ID:server-f10036f2-2bba-03cd-422c-65c8907981f8 Address:f10036f2-2bba-03cd-422c-65c8907981f8}]" +2025-01-25T14:40:37.282Z [INFO] server-60e1154b-c6fd-99ca-b5c4-554abef400d9: entering follower state: follower="Node at 60e1154b-c6fd-99ca-b5c4-554abef400d9 [Follower]" leader-address= leader-id= +2025-01-25T14:40:37.283Z [INFO] server-00cb6dea-19ab-a4c3-00c4-27c5482baa29: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-60e1154b-c6fd-99ca-b5c4-554abef400d9 Address:60e1154b-c6fd-99ca-b5c4-554abef400d9} {Suffrage:Voter ID:server-00cb6dea-19ab-a4c3-00c4-27c5482baa29 Address:00cb6dea-19ab-a4c3-00c4-27c5482baa29} {Suffrage:Voter ID:server-f10036f2-2bba-03cd-422c-65c8907981f8 Address:f10036f2-2bba-03cd-422c-65c8907981f8}]" +2025-01-25T14:40:37.283Z [INFO] server-00cb6dea-19ab-a4c3-00c4-27c5482baa29: entering follower state: follower="Node at 00cb6dea-19ab-a4c3-00c4-27c5482baa29 [Follower]" leader-address= leader-id= +2025-01-25T14:40:37.284Z [INFO] server-f10036f2-2bba-03cd-422c-65c8907981f8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-60e1154b-c6fd-99ca-b5c4-554abef400d9 Address:60e1154b-c6fd-99ca-b5c4-554abef400d9} {Suffrage:Voter ID:server-00cb6dea-19ab-a4c3-00c4-27c5482baa29 Address:00cb6dea-19ab-a4c3-00c4-27c5482baa29} {Suffrage:Voter ID:server-f10036f2-2bba-03cd-422c-65c8907981f8 Address:f10036f2-2bba-03cd-422c-65c8907981f8}]" +2025-01-25T14:40:37.284Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:40:37.284Z [INFO] server-f10036f2-2bba-03cd-422c-65c8907981f8: entering follower state: follower="Node at f10036f2-2bba-03cd-422c-65c8907981f8 [Follower]" leader-address= leader-id= +2025-01-25T14:40:37.352Z [WARN] server-00cb6dea-19ab-a4c3-00c4-27c5482baa29: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:37.352Z [INFO] server-00cb6dea-19ab-a4c3-00c4-27c5482baa29: entering candidate state: node="Node at 00cb6dea-19ab-a4c3-00c4-27c5482baa29 [Candidate]" term=2 +2025-01-25T14:40:37.352Z [WARN] server-f10036f2-2bba-03cd-422c-65c8907981f8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:37.353Z [INFO] server-f10036f2-2bba-03cd-422c-65c8907981f8: entering candidate state: node="Node at f10036f2-2bba-03cd-422c-65c8907981f8 [Candidate]" term=2 +2025-01-25T14:40:37.353Z [INFO] server-f10036f2-2bba-03cd-422c-65c8907981f8: duplicate requestVote for same term: term=2 +2025-01-25T14:40:37.353Z [INFO] server-00cb6dea-19ab-a4c3-00c4-27c5482baa29: election won: term=2 tally=2 +2025-01-25T14:40:37.353Z [INFO] server-00cb6dea-19ab-a4c3-00c4-27c5482baa29: entering leader state: leader="Node at 00cb6dea-19ab-a4c3-00c4-27c5482baa29 [Leader]" +2025-01-25T14:40:37.353Z [INFO] server-60e1154b-c6fd-99ca-b5c4-554abef400d9: duplicate requestVote for same term: term=2 +2025-01-25T14:40:37.353Z [INFO] server-00cb6dea-19ab-a4c3-00c4-27c5482baa29: added peer, starting replication: peer=server-60e1154b-c6fd-99ca-b5c4-554abef400d9 +2025-01-25T14:40:37.353Z [INFO] server-00cb6dea-19ab-a4c3-00c4-27c5482baa29: added peer, starting replication: peer=server-f10036f2-2bba-03cd-422c-65c8907981f8 +2025-01-25T14:40:37.353Z [WARN] server-00cb6dea-19ab-a4c3-00c4-27c5482baa29: rejecting vote request since we have a leader: from=f10036f2-2bba-03cd-422c-65c8907981f8 leader=00cb6dea-19ab-a4c3-00c4-27c5482baa29 leader-id=server-00cb6dea-19ab-a4c3-00c4-27c5482baa29 +2025-01-25T14:40:37.354Z [INFO] server-f10036f2-2bba-03cd-422c-65c8907981f8: entering follower state: follower="Node at f10036f2-2bba-03cd-422c-65c8907981f8 [Follower]" leader-address=00cb6dea-19ab-a4c3-00c4-27c5482baa29 leader-id=server-00cb6dea-19ab-a4c3-00c4-27c5482baa29 +2025-01-25T14:40:37.354Z [INFO] server-00cb6dea-19ab-a4c3-00c4-27c5482baa29: pipelining replication: peer="{Voter server-60e1154b-c6fd-99ca-b5c4-554abef400d9 60e1154b-c6fd-99ca-b5c4-554abef400d9}" +2025-01-25T14:40:37.354Z [INFO] server-00cb6dea-19ab-a4c3-00c4-27c5482baa29: pipelining replication: peer="{Voter server-f10036f2-2bba-03cd-422c-65c8907981f8 f10036f2-2bba-03cd-422c-65c8907981f8}" +2025-01-25T14:40:37.460Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:40:37.35424329 +0000 UTC m=+106.400082608 (2 nodes), 69.70612ms from start of poll, 72.288544ms from cluster start. Timeout at 2025-01-25 14:40:37.459252623 +0000 UTC m=+106.505091940, 105.009332ms after stability +2025-01-25T14:40:37.460Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:37.566Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:37.460604308 +0000 UTC m=+106.506443334 (1 nodes), 4.958µs from start of poll, 178.64927ms from cluster start. Timeout at 2025-01-25 14:40:37.565799434 +0000 UTC m=+106.611638751, 105.195417ms after stability +2025-01-25T14:40:37.566Z [WARN] server-f10036f2-2bba-03cd-422c-65c8907981f8: rejecting vote request since we have a leader: from=60e1154b-c6fd-99ca-b5c4-554abef400d9 leader=00cb6dea-19ab-a4c3-00c4-27c5482baa29 leader-id=server-00cb6dea-19ab-a4c3-00c4-27c5482baa29 +2025-01-25T14:40:37.567Z [INFO] server-00cb6dea-19ab-a4c3-00c4-27c5482baa29: aborting pipeline replication: peer="{Voter server-f10036f2-2bba-03cd-422c-65c8907981f8 f10036f2-2bba-03cd-422c-65c8907981f8}" +2025-01-25T14:40:37.567Z [INFO] server-00cb6dea-19ab-a4c3-00c4-27c5482baa29: aborting pipeline replication: peer="{Voter server-60e1154b-c6fd-99ca-b5c4-554abef400d9 60e1154b-c6fd-99ca-b5c4-554abef400d9}" +--- PASS: TestRaft_VotingGrant_WhenLeaderAvailable (0.29s) === RUN TestRaft_ProtocolVersion_RejectRPC -2025-01-25T14:29:49.877Z [INFO] server-869b7e59-f8f1-fce0-5626-fd59907b12b1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-869b7e59-f8f1-fce0-5626-fd59907b12b1 Address:869b7e59-f8f1-fce0-5626-fd59907b12b1} {Suffrage:Voter ID:server-bbb55135-deaa-e666-8235-97f50969b083 Address:bbb55135-deaa-e666-8235-97f50969b083} {Suffrage:Voter ID:server-9c8d64aa-c31b-ae1b-00b5-3da07b4cd2ce Address:9c8d64aa-c31b-ae1b-00b5-3da07b4cd2ce}]" -2025-01-25T14:29:49.877Z [INFO] server-869b7e59-f8f1-fce0-5626-fd59907b12b1: entering follower state: follower="Node at 869b7e59-f8f1-fce0-5626-fd59907b12b1 [Follower]" leader-address= leader-id= -2025-01-25T14:29:49.878Z [INFO] server-bbb55135-deaa-e666-8235-97f50969b083: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-869b7e59-f8f1-fce0-5626-fd59907b12b1 Address:869b7e59-f8f1-fce0-5626-fd59907b12b1} {Suffrage:Voter ID:server-bbb55135-deaa-e666-8235-97f50969b083 Address:bbb55135-deaa-e666-8235-97f50969b083} {Suffrage:Voter ID:server-9c8d64aa-c31b-ae1b-00b5-3da07b4cd2ce Address:9c8d64aa-c31b-ae1b-00b5-3da07b4cd2ce}]" -2025-01-25T14:29:49.878Z [INFO] server-bbb55135-deaa-e666-8235-97f50969b083: entering follower state: follower="Node at bbb55135-deaa-e666-8235-97f50969b083 [Follower]" leader-address= leader-id= -2025-01-25T14:29:49.878Z [INFO] server-9c8d64aa-c31b-ae1b-00b5-3da07b4cd2ce: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-869b7e59-f8f1-fce0-5626-fd59907b12b1 Address:869b7e59-f8f1-fce0-5626-fd59907b12b1} {Suffrage:Voter ID:server-bbb55135-deaa-e666-8235-97f50969b083 Address:bbb55135-deaa-e666-8235-97f50969b083} {Suffrage:Voter ID:server-9c8d64aa-c31b-ae1b-00b5-3da07b4cd2ce Address:9c8d64aa-c31b-ae1b-00b5-3da07b4cd2ce}]" -2025-01-25T14:29:49.878Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:29:49.878Z [INFO] server-9c8d64aa-c31b-ae1b-00b5-3da07b4cd2ce: entering follower state: follower="Node at 9c8d64aa-c31b-ae1b-00b5-3da07b4cd2ce [Follower]" leader-address= leader-id= -2025-01-25T14:29:49.937Z [WARN] server-9c8d64aa-c31b-ae1b-00b5-3da07b4cd2ce: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:49.937Z [INFO] server-9c8d64aa-c31b-ae1b-00b5-3da07b4cd2ce: entering candidate state: node="Node at 9c8d64aa-c31b-ae1b-00b5-3da07b4cd2ce [Candidate]" term=2 -2025-01-25T14:29:49.937Z [INFO] server-9c8d64aa-c31b-ae1b-00b5-3da07b4cd2ce: election won: term=2 tally=2 -2025-01-25T14:29:49.937Z [INFO] server-9c8d64aa-c31b-ae1b-00b5-3da07b4cd2ce: entering leader state: leader="Node at 9c8d64aa-c31b-ae1b-00b5-3da07b4cd2ce [Leader]" -2025-01-25T14:29:49.937Z [INFO] server-9c8d64aa-c31b-ae1b-00b5-3da07b4cd2ce: added peer, starting replication: peer=server-869b7e59-f8f1-fce0-5626-fd59907b12b1 -2025-01-25T14:29:49.937Z [INFO] server-9c8d64aa-c31b-ae1b-00b5-3da07b4cd2ce: added peer, starting replication: peer=server-bbb55135-deaa-e666-8235-97f50969b083 -2025-01-25T14:29:49.937Z [INFO] server-9c8d64aa-c31b-ae1b-00b5-3da07b4cd2ce: pipelining replication: peer="{Voter server-bbb55135-deaa-e666-8235-97f50969b083 bbb55135-deaa-e666-8235-97f50969b083}" -2025-01-25T14:29:49.937Z [INFO] server-9c8d64aa-c31b-ae1b-00b5-3da07b4cd2ce: pipelining replication: peer="{Voter server-869b7e59-f8f1-fce0-5626-fd59907b12b1 869b7e59-f8f1-fce0-5626-fd59907b12b1}" -2025-01-25T14:29:50.043Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:29:49.93757635 +0000 UTC m=+99.791901332 (2 nodes), 59.293626ms from start of poll, 59.968081ms from cluster start. Timeout at 2025-01-25 14:29:50.042615509 +0000 UTC m=+99.896940491, 105.039159ms after stability -2025-01-25T14:29:50.044Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:50.150Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:50.044039399 +0000 UTC m=+99.898364381 (1 nodes), 2.52µs from start of poll, 166.43113ms from cluster start. Timeout at 2025-01-25 14:29:50.149198193 +0000 UTC m=+100.003523175, 105.158794ms after stability -2025-01-25T14:29:50.150Z [INFO] server-9c8d64aa-c31b-ae1b-00b5-3da07b4cd2ce: aborting pipeline replication: peer="{Voter server-bbb55135-deaa-e666-8235-97f50969b083 bbb55135-deaa-e666-8235-97f50969b083}" -2025-01-25T14:29:50.150Z [INFO] server-9c8d64aa-c31b-ae1b-00b5-3da07b4cd2ce: aborting pipeline replication: peer="{Voter server-869b7e59-f8f1-fce0-5626-fd59907b12b1 869b7e59-f8f1-fce0-5626-fd59907b12b1}" ---- PASS: TestRaft_ProtocolVersion_RejectRPC (0.27s) +2025-01-25T14:40:37.575Z [INFO] server-0899f58f-8038-2cea-17fb-04e6c3cd4bd6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0899f58f-8038-2cea-17fb-04e6c3cd4bd6 Address:0899f58f-8038-2cea-17fb-04e6c3cd4bd6} {Suffrage:Voter ID:server-1bcac4d0-c0e8-3d66-4248-61a8e6720085 Address:1bcac4d0-c0e8-3d66-4248-61a8e6720085} {Suffrage:Voter ID:server-bf4ebf56-52b8-507d-c619-b95885c00ecc Address:bf4ebf56-52b8-507d-c619-b95885c00ecc}]" +2025-01-25T14:40:37.576Z [INFO] server-0899f58f-8038-2cea-17fb-04e6c3cd4bd6: entering follower state: follower="Node at 0899f58f-8038-2cea-17fb-04e6c3cd4bd6 [Follower]" leader-address= leader-id= +2025-01-25T14:40:37.576Z [INFO] server-1bcac4d0-c0e8-3d66-4248-61a8e6720085: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0899f58f-8038-2cea-17fb-04e6c3cd4bd6 Address:0899f58f-8038-2cea-17fb-04e6c3cd4bd6} {Suffrage:Voter ID:server-1bcac4d0-c0e8-3d66-4248-61a8e6720085 Address:1bcac4d0-c0e8-3d66-4248-61a8e6720085} {Suffrage:Voter ID:server-bf4ebf56-52b8-507d-c619-b95885c00ecc Address:bf4ebf56-52b8-507d-c619-b95885c00ecc}]" +2025-01-25T14:40:37.577Z [INFO] server-1bcac4d0-c0e8-3d66-4248-61a8e6720085: entering follower state: follower="Node at 1bcac4d0-c0e8-3d66-4248-61a8e6720085 [Follower]" leader-address= leader-id= +2025-01-25T14:40:37.577Z [INFO] server-bf4ebf56-52b8-507d-c619-b95885c00ecc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0899f58f-8038-2cea-17fb-04e6c3cd4bd6 Address:0899f58f-8038-2cea-17fb-04e6c3cd4bd6} {Suffrage:Voter ID:server-1bcac4d0-c0e8-3d66-4248-61a8e6720085 Address:1bcac4d0-c0e8-3d66-4248-61a8e6720085} {Suffrage:Voter ID:server-bf4ebf56-52b8-507d-c619-b95885c00ecc Address:bf4ebf56-52b8-507d-c619-b95885c00ecc}]" +2025-01-25T14:40:37.577Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:40:37.577Z [INFO] server-bf4ebf56-52b8-507d-c619-b95885c00ecc: entering follower state: follower="Node at bf4ebf56-52b8-507d-c619-b95885c00ecc [Follower]" leader-address= leader-id= +2025-01-25T14:40:37.642Z [WARN] server-1bcac4d0-c0e8-3d66-4248-61a8e6720085: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:37.642Z [INFO] server-1bcac4d0-c0e8-3d66-4248-61a8e6720085: entering candidate state: node="Node at 1bcac4d0-c0e8-3d66-4248-61a8e6720085 [Candidate]" term=2 +2025-01-25T14:40:37.643Z [INFO] server-1bcac4d0-c0e8-3d66-4248-61a8e6720085: election won: term=2 tally=2 +2025-01-25T14:40:37.643Z [INFO] server-1bcac4d0-c0e8-3d66-4248-61a8e6720085: entering leader state: leader="Node at 1bcac4d0-c0e8-3d66-4248-61a8e6720085 [Leader]" +2025-01-25T14:40:37.643Z [INFO] server-1bcac4d0-c0e8-3d66-4248-61a8e6720085: added peer, starting replication: peer=server-0899f58f-8038-2cea-17fb-04e6c3cd4bd6 +2025-01-25T14:40:37.643Z [INFO] server-1bcac4d0-c0e8-3d66-4248-61a8e6720085: added peer, starting replication: peer=server-bf4ebf56-52b8-507d-c619-b95885c00ecc +2025-01-25T14:40:37.644Z [INFO] server-1bcac4d0-c0e8-3d66-4248-61a8e6720085: pipelining replication: peer="{Voter server-0899f58f-8038-2cea-17fb-04e6c3cd4bd6 0899f58f-8038-2cea-17fb-04e6c3cd4bd6}" +2025-01-25T14:40:37.644Z [INFO] server-1bcac4d0-c0e8-3d66-4248-61a8e6720085: pipelining replication: peer="{Voter server-bf4ebf56-52b8-507d-c619-b95885c00ecc bf4ebf56-52b8-507d-c619-b95885c00ecc}" +2025-01-25T14:40:37.750Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:40:37.643385757 +0000 UTC m=+106.689225074 (2 nodes), 65.523524ms from start of poll, 68.169823ms from cluster start. Timeout at 2025-01-25 14:40:37.748396256 +0000 UTC m=+106.794235574, 105.0105ms after stability +2025-01-25T14:40:37.750Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:37.855Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:37.750230443 +0000 UTC m=+106.796069760 (1 nodes), 4.083µs from start of poll, 175.014509ms from cluster start. Timeout at 2025-01-25 14:40:37.855464068 +0000 UTC m=+106.901303386, 105.233626ms after stability +2025-01-25T14:40:37.856Z [INFO] server-1bcac4d0-c0e8-3d66-4248-61a8e6720085: aborting pipeline replication: peer="{Voter server-bf4ebf56-52b8-507d-c619-b95885c00ecc bf4ebf56-52b8-507d-c619-b95885c00ecc}" +2025-01-25T14:40:37.856Z [INFO] server-1bcac4d0-c0e8-3d66-4248-61a8e6720085: aborting pipeline replication: peer="{Voter server-0899f58f-8038-2cea-17fb-04e6c3cd4bd6 0899f58f-8038-2cea-17fb-04e6c3cd4bd6}" +--- PASS: TestRaft_ProtocolVersion_RejectRPC (0.29s) === RUN TestRaft_ProtocolVersion_Upgrade_1_2 raft_test.go:2151: Skipping unreliable test --- SKIP: TestRaft_ProtocolVersion_Upgrade_1_2 (0.00s) @@ -3573,284 +3627,284 @@ raft_test.go:2196: Skipping unreliable test --- SKIP: TestRaft_ProtocolVersion_Upgrade_2_3 (0.00s) === RUN TestRaft_LeaderID_Propagated -2025-01-25T14:29:50.153Z [INFO] server-65b71926-a98d-c4c2-708d-4c8ee5270da2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-65b71926-a98d-c4c2-708d-4c8ee5270da2 Address:65b71926-a98d-c4c2-708d-4c8ee5270da2} {Suffrage:Voter ID:server-b24b02b8-59db-e4a4-e245-819a9768d206 Address:b24b02b8-59db-e4a4-e245-819a9768d206} {Suffrage:Voter ID:server-7e4b5d00-4cac-567f-df73-374479edd8e7 Address:7e4b5d00-4cac-567f-df73-374479edd8e7}]" -2025-01-25T14:29:50.153Z [INFO] server-65b71926-a98d-c4c2-708d-4c8ee5270da2: entering follower state: follower="Node at 65b71926-a98d-c4c2-708d-4c8ee5270da2 [Follower]" leader-address= leader-id= -2025-01-25T14:29:50.153Z [INFO] server-b24b02b8-59db-e4a4-e245-819a9768d206: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-65b71926-a98d-c4c2-708d-4c8ee5270da2 Address:65b71926-a98d-c4c2-708d-4c8ee5270da2} {Suffrage:Voter ID:server-b24b02b8-59db-e4a4-e245-819a9768d206 Address:b24b02b8-59db-e4a4-e245-819a9768d206} {Suffrage:Voter ID:server-7e4b5d00-4cac-567f-df73-374479edd8e7 Address:7e4b5d00-4cac-567f-df73-374479edd8e7}]" -2025-01-25T14:29:50.153Z [INFO] server-b24b02b8-59db-e4a4-e245-819a9768d206: entering follower state: follower="Node at b24b02b8-59db-e4a4-e245-819a9768d206 [Follower]" leader-address= leader-id= -2025-01-25T14:29:50.153Z [INFO] server-7e4b5d00-4cac-567f-df73-374479edd8e7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-65b71926-a98d-c4c2-708d-4c8ee5270da2 Address:65b71926-a98d-c4c2-708d-4c8ee5270da2} {Suffrage:Voter ID:server-b24b02b8-59db-e4a4-e245-819a9768d206 Address:b24b02b8-59db-e4a4-e245-819a9768d206} {Suffrage:Voter ID:server-7e4b5d00-4cac-567f-df73-374479edd8e7 Address:7e4b5d00-4cac-567f-df73-374479edd8e7}]" -2025-01-25T14:29:50.153Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:50.153Z [INFO] server-7e4b5d00-4cac-567f-df73-374479edd8e7: entering follower state: follower="Node at 7e4b5d00-4cac-567f-df73-374479edd8e7 [Follower]" leader-address= leader-id= -2025-01-25T14:29:50.215Z [WARN] server-65b71926-a98d-c4c2-708d-4c8ee5270da2: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:50.216Z [INFO] server-65b71926-a98d-c4c2-708d-4c8ee5270da2: entering candidate state: node="Node at 65b71926-a98d-c4c2-708d-4c8ee5270da2 [Candidate]" term=2 -2025-01-25T14:29:50.216Z [INFO] server-65b71926-a98d-c4c2-708d-4c8ee5270da2: election won: term=2 tally=2 -2025-01-25T14:29:50.216Z [INFO] server-65b71926-a98d-c4c2-708d-4c8ee5270da2: entering leader state: leader="Node at 65b71926-a98d-c4c2-708d-4c8ee5270da2 [Leader]" -2025-01-25T14:29:50.216Z [INFO] server-65b71926-a98d-c4c2-708d-4c8ee5270da2: added peer, starting replication: peer=server-b24b02b8-59db-e4a4-e245-819a9768d206 -2025-01-25T14:29:50.216Z [INFO] server-65b71926-a98d-c4c2-708d-4c8ee5270da2: added peer, starting replication: peer=server-7e4b5d00-4cac-567f-df73-374479edd8e7 -2025-01-25T14:29:50.216Z [INFO] server-65b71926-a98d-c4c2-708d-4c8ee5270da2: pipelining replication: peer="{Voter server-7e4b5d00-4cac-567f-df73-374479edd8e7 7e4b5d00-4cac-567f-df73-374479edd8e7}" -2025-01-25T14:29:50.216Z [INFO] server-65b71926-a98d-c4c2-708d-4c8ee5270da2: pipelining replication: peer="{Voter server-b24b02b8-59db-e4a4-e245-819a9768d206 b24b02b8-59db-e4a4-e245-819a9768d206}" -2025-01-25T14:29:50.322Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:50.216462554 +0000 UTC m=+100.070787536 (1 nodes), 62.473672ms from start of poll, 63.189846ms from cluster start. Timeout at 2025-01-25 14:29:50.321466513 +0000 UTC m=+100.175791535, 105.003999ms after stability -2025-01-25T14:29:50.322Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:50.429Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:50.32271633 +0000 UTC m=+100.177041272 (1 nodes), 1.24µs from start of poll, 169.443582ms from cluster start. Timeout at 2025-01-25 14:29:50.42798972 +0000 UTC m=+100.282314662, 105.27339ms after stability -2025-01-25T14:29:50.429Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:50.536Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:50.429667521 +0000 UTC m=+100.283992463 (1 nodes), 1.2µs from start of poll, 276.394773ms from cluster start. Timeout at 2025-01-25 14:29:50.534836075 +0000 UTC m=+100.389161017, 105.168554ms after stability -2025-01-25T14:29:50.536Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:50.642Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:50.536649871 +0000 UTC m=+100.390974853 (1 nodes), 1.88µs from start of poll, 383.377163ms from cluster start. Timeout at 2025-01-25 14:29:50.641891062 +0000 UTC m=+100.496216084, 105.241231ms after stability -2025-01-25T14:29:50.642Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:50.748Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:50.642428985 +0000 UTC m=+100.496753967 (1 nodes), 1.36µs from start of poll, 489.156277ms from cluster start. Timeout at 2025-01-25 14:29:50.747654297 +0000 UTC m=+100.601979279, 105.225312ms after stability -2025-01-25T14:29:50.748Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:50.854Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:50.748400291 +0000 UTC m=+100.602725273 (1 nodes), 1.12µs from start of poll, 595.127583ms from cluster start. Timeout at 2025-01-25 14:29:50.853571604 +0000 UTC m=+100.707896586, 105.171313ms after stability -2025-01-25T14:29:50.960Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:51.066Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:50.960212948 +0000 UTC m=+100.814537930 (1 nodes), 1.159µs from start of poll, 806.94024ms from cluster start. Timeout at 2025-01-25 14:29:51.065431581 +0000 UTC m=+100.919756563, 105.218633ms after stability -2025-01-25T14:29:51.066Z [INFO] server-65b71926-a98d-c4c2-708d-4c8ee5270da2: aborting pipeline replication: peer="{Voter server-b24b02b8-59db-e4a4-e245-819a9768d206 b24b02b8-59db-e4a4-e245-819a9768d206}" -2025-01-25T14:29:51.066Z [INFO] server-65b71926-a98d-c4c2-708d-4c8ee5270da2: aborting pipeline replication: peer="{Voter server-7e4b5d00-4cac-567f-df73-374479edd8e7 7e4b5d00-4cac-567f-df73-374479edd8e7}" ---- PASS: TestRaft_LeaderID_Propagated (0.92s) +2025-01-25T14:40:37.866Z [INFO] server-8e5db2d2-3963-48ba-6bdc-bf49f15992be: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8e5db2d2-3963-48ba-6bdc-bf49f15992be Address:8e5db2d2-3963-48ba-6bdc-bf49f15992be} {Suffrage:Voter ID:server-31836d4e-4042-fdbf-e073-6bd7f935b14c Address:31836d4e-4042-fdbf-e073-6bd7f935b14c} {Suffrage:Voter ID:server-b660cdd2-4d6f-c0e0-be08-b5ebc5a72b94 Address:b660cdd2-4d6f-c0e0-be08-b5ebc5a72b94}]" +2025-01-25T14:40:37.866Z [INFO] server-8e5db2d2-3963-48ba-6bdc-bf49f15992be: entering follower state: follower="Node at 8e5db2d2-3963-48ba-6bdc-bf49f15992be [Follower]" leader-address= leader-id= +2025-01-25T14:40:37.867Z [INFO] server-31836d4e-4042-fdbf-e073-6bd7f935b14c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8e5db2d2-3963-48ba-6bdc-bf49f15992be Address:8e5db2d2-3963-48ba-6bdc-bf49f15992be} {Suffrage:Voter ID:server-31836d4e-4042-fdbf-e073-6bd7f935b14c Address:31836d4e-4042-fdbf-e073-6bd7f935b14c} {Suffrage:Voter ID:server-b660cdd2-4d6f-c0e0-be08-b5ebc5a72b94 Address:b660cdd2-4d6f-c0e0-be08-b5ebc5a72b94}]" +2025-01-25T14:40:37.867Z [INFO] server-31836d4e-4042-fdbf-e073-6bd7f935b14c: entering follower state: follower="Node at 31836d4e-4042-fdbf-e073-6bd7f935b14c [Follower]" leader-address= leader-id= +2025-01-25T14:40:37.868Z [INFO] server-b660cdd2-4d6f-c0e0-be08-b5ebc5a72b94: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8e5db2d2-3963-48ba-6bdc-bf49f15992be Address:8e5db2d2-3963-48ba-6bdc-bf49f15992be} {Suffrage:Voter ID:server-31836d4e-4042-fdbf-e073-6bd7f935b14c Address:31836d4e-4042-fdbf-e073-6bd7f935b14c} {Suffrage:Voter ID:server-b660cdd2-4d6f-c0e0-be08-b5ebc5a72b94 Address:b660cdd2-4d6f-c0e0-be08-b5ebc5a72b94}]" +2025-01-25T14:40:37.868Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:37.868Z [INFO] server-b660cdd2-4d6f-c0e0-be08-b5ebc5a72b94: entering follower state: follower="Node at b660cdd2-4d6f-c0e0-be08-b5ebc5a72b94 [Follower]" leader-address= leader-id= +2025-01-25T14:40:37.936Z [WARN] server-8e5db2d2-3963-48ba-6bdc-bf49f15992be: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:37.936Z [INFO] server-8e5db2d2-3963-48ba-6bdc-bf49f15992be: entering candidate state: node="Node at 8e5db2d2-3963-48ba-6bdc-bf49f15992be [Candidate]" term=2 +2025-01-25T14:40:37.936Z [INFO] server-8e5db2d2-3963-48ba-6bdc-bf49f15992be: election won: term=2 tally=2 +2025-01-25T14:40:37.936Z [INFO] server-8e5db2d2-3963-48ba-6bdc-bf49f15992be: entering leader state: leader="Node at 8e5db2d2-3963-48ba-6bdc-bf49f15992be [Leader]" +2025-01-25T14:40:37.937Z [INFO] server-8e5db2d2-3963-48ba-6bdc-bf49f15992be: added peer, starting replication: peer=server-31836d4e-4042-fdbf-e073-6bd7f935b14c +2025-01-25T14:40:37.937Z [INFO] server-8e5db2d2-3963-48ba-6bdc-bf49f15992be: added peer, starting replication: peer=server-b660cdd2-4d6f-c0e0-be08-b5ebc5a72b94 +2025-01-25T14:40:37.937Z [INFO] server-8e5db2d2-3963-48ba-6bdc-bf49f15992be: pipelining replication: peer="{Voter server-31836d4e-4042-fdbf-e073-6bd7f935b14c 31836d4e-4042-fdbf-e073-6bd7f935b14c}" +2025-01-25T14:40:37.938Z [INFO] server-8e5db2d2-3963-48ba-6bdc-bf49f15992be: pipelining replication: peer="{Voter server-b660cdd2-4d6f-c0e0-be08-b5ebc5a72b94 b660cdd2-4d6f-c0e0-be08-b5ebc5a72b94}" +2025-01-25T14:40:38.042Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:37.937018778 +0000 UTC m=+106.982858096 (1 nodes), 68.484699ms from start of poll, 71.209166ms from cluster start. Timeout at 2025-01-25 14:40:38.04204882 +0000 UTC m=+107.087888138, 105.030042ms after stability +2025-01-25T14:40:38.042Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:38.148Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:38.043026254 +0000 UTC m=+107.088865571 (1 nodes), 4.666µs from start of poll, 177.216641ms from cluster start. Timeout at 2025-01-25 14:40:38.148296629 +0000 UTC m=+107.194135946, 105.270375ms after stability +2025-01-25T14:40:38.149Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:38.254Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:38.149520647 +0000 UTC m=+107.195359965 (1 nodes), 3.5µs from start of poll, 283.711035ms from cluster start. Timeout at 2025-01-25 14:40:38.254746397 +0000 UTC m=+107.300585715, 105.22575ms after stability +2025-01-25T14:40:38.255Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:38.361Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:38.255953623 +0000 UTC m=+107.301792941 (1 nodes), 3.5µs from start of poll, 390.144011ms from cluster start. Timeout at 2025-01-25 14:40:38.361158666 +0000 UTC m=+107.406997983, 105.205042ms after stability +2025-01-25T14:40:38.362Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:38.468Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:38.36308281 +0000 UTC m=+107.408922128 (1 nodes), 5.25µs from start of poll, 497.273198ms from cluster start. Timeout at 2025-01-25 14:40:38.468280853 +0000 UTC m=+107.514120462, 105.198334ms after stability +2025-01-25T14:40:38.469Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:38.575Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:38.469822246 +0000 UTC m=+107.515661564 (1 nodes), 5.542µs from start of poll, 604.012634ms from cluster start. Timeout at 2025-01-25 14:40:38.575077164 +0000 UTC m=+107.620916481, 105.254917ms after stability +2025-01-25T14:40:38.681Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:38.793Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:38.682113989 +0000 UTC m=+107.727953307 (1 nodes), 5.542µs from start of poll, 816.304377ms from cluster start. Timeout at 2025-01-25 14:40:38.78740099 +0000 UTC m=+107.833240308, 105.287001ms after stability +2025-01-25T14:40:38.793Z [INFO] server-8e5db2d2-3963-48ba-6bdc-bf49f15992be: aborting pipeline replication: peer="{Voter server-b660cdd2-4d6f-c0e0-be08-b5ebc5a72b94 b660cdd2-4d6f-c0e0-be08-b5ebc5a72b94}" +2025-01-25T14:40:38.793Z [INFO] server-8e5db2d2-3963-48ba-6bdc-bf49f15992be: aborting pipeline replication: peer="{Voter server-31836d4e-4042-fdbf-e073-6bd7f935b14c 31836d4e-4042-fdbf-e073-6bd7f935b14c}" +--- PASS: TestRaft_LeaderID_Propagated (0.94s) === RUN TestRaft_LeadershipTransferInProgress --- PASS: TestRaft_LeadershipTransferInProgress (0.00s) === RUN TestRaft_LeadershipTransferPickServer --- PASS: TestRaft_LeadershipTransferPickServer (0.00s) === RUN TestRaft_LeadershipTransfer -2025-01-25T14:29:51.070Z [INFO] server-06c471f4-1096-676e-f79e-cba8e1f25656: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-06c471f4-1096-676e-f79e-cba8e1f25656 Address:06c471f4-1096-676e-f79e-cba8e1f25656} {Suffrage:Voter ID:server-a65baebb-6749-785b-1027-899b4cbba662 Address:a65baebb-6749-785b-1027-899b4cbba662} {Suffrage:Voter ID:server-e1f0691e-0ba0-c022-c1c5-175e73e93f7a Address:e1f0691e-0ba0-c022-c1c5-175e73e93f7a}]" -2025-01-25T14:29:51.070Z [INFO] server-06c471f4-1096-676e-f79e-cba8e1f25656: entering follower state: follower="Node at 06c471f4-1096-676e-f79e-cba8e1f25656 [Follower]" leader-address= leader-id= -2025-01-25T14:29:51.070Z [INFO] server-a65baebb-6749-785b-1027-899b4cbba662: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-06c471f4-1096-676e-f79e-cba8e1f25656 Address:06c471f4-1096-676e-f79e-cba8e1f25656} {Suffrage:Voter ID:server-a65baebb-6749-785b-1027-899b4cbba662 Address:a65baebb-6749-785b-1027-899b4cbba662} {Suffrage:Voter ID:server-e1f0691e-0ba0-c022-c1c5-175e73e93f7a Address:e1f0691e-0ba0-c022-c1c5-175e73e93f7a}]" -2025-01-25T14:29:51.070Z [INFO] server-a65baebb-6749-785b-1027-899b4cbba662: entering follower state: follower="Node at a65baebb-6749-785b-1027-899b4cbba662 [Follower]" leader-address= leader-id= -2025-01-25T14:29:51.070Z [INFO] server-e1f0691e-0ba0-c022-c1c5-175e73e93f7a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-06c471f4-1096-676e-f79e-cba8e1f25656 Address:06c471f4-1096-676e-f79e-cba8e1f25656} {Suffrage:Voter ID:server-a65baebb-6749-785b-1027-899b4cbba662 Address:a65baebb-6749-785b-1027-899b4cbba662} {Suffrage:Voter ID:server-e1f0691e-0ba0-c022-c1c5-175e73e93f7a Address:e1f0691e-0ba0-c022-c1c5-175e73e93f7a}]" -2025-01-25T14:29:51.070Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:51.070Z [INFO] server-e1f0691e-0ba0-c022-c1c5-175e73e93f7a: entering follower state: follower="Node at e1f0691e-0ba0-c022-c1c5-175e73e93f7a [Follower]" leader-address= leader-id= -2025-01-25T14:29:51.131Z [WARN] server-06c471f4-1096-676e-f79e-cba8e1f25656: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:51.131Z [INFO] server-06c471f4-1096-676e-f79e-cba8e1f25656: entering candidate state: node="Node at 06c471f4-1096-676e-f79e-cba8e1f25656 [Candidate]" term=2 -2025-01-25T14:29:51.131Z [INFO] server-06c471f4-1096-676e-f79e-cba8e1f25656: election won: term=2 tally=2 -2025-01-25T14:29:51.131Z [INFO] server-06c471f4-1096-676e-f79e-cba8e1f25656: entering leader state: leader="Node at 06c471f4-1096-676e-f79e-cba8e1f25656 [Leader]" -2025-01-25T14:29:51.131Z [INFO] server-06c471f4-1096-676e-f79e-cba8e1f25656: added peer, starting replication: peer=server-a65baebb-6749-785b-1027-899b4cbba662 -2025-01-25T14:29:51.131Z [INFO] server-06c471f4-1096-676e-f79e-cba8e1f25656: added peer, starting replication: peer=server-e1f0691e-0ba0-c022-c1c5-175e73e93f7a -2025-01-25T14:29:51.131Z [INFO] server-06c471f4-1096-676e-f79e-cba8e1f25656: pipelining replication: peer="{Voter server-e1f0691e-0ba0-c022-c1c5-175e73e93f7a e1f0691e-0ba0-c022-c1c5-175e73e93f7a}" -2025-01-25T14:29:51.131Z [INFO] server-06c471f4-1096-676e-f79e-cba8e1f25656: pipelining replication: peer="{Voter server-a65baebb-6749-785b-1027-899b4cbba662 a65baebb-6749-785b-1027-899b4cbba662}" -2025-01-25T14:29:51.236Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:51.131475065 +0000 UTC m=+100.985800047 (1 nodes), 60.656497ms from start of poll, 61.428709ms from cluster start. Timeout at 2025-01-25 14:29:51.236480425 +0000 UTC m=+101.090805367, 105.00532ms after stability -2025-01-25T14:29:51.236Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:51.343Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:51.236797595 +0000 UTC m=+101.091122577 (1 nodes), 2.88µs from start of poll, 166.751239ms from cluster start. Timeout at 2025-01-25 14:29:51.342000708 +0000 UTC m=+101.196325690, 105.203113ms after stability -2025-01-25T14:29:51.344Z [INFO] server-a65baebb-6749-785b-1027-899b4cbba662: entering candidate state: node="Node at a65baebb-6749-785b-1027-899b4cbba662 [Candidate]" term=3 -2025-01-25T14:29:51.344Z [INFO] server-a65baebb-6749-785b-1027-899b4cbba662: election won: term=3 tally=2 -2025-01-25T14:29:51.344Z [INFO] server-a65baebb-6749-785b-1027-899b4cbba662: entering leader state: leader="Node at a65baebb-6749-785b-1027-899b4cbba662 [Leader]" -2025-01-25T14:29:51.344Z [INFO] server-a65baebb-6749-785b-1027-899b4cbba662: added peer, starting replication: peer=server-06c471f4-1096-676e-f79e-cba8e1f25656 -2025-01-25T14:29:51.344Z [INFO] server-a65baebb-6749-785b-1027-899b4cbba662: added peer, starting replication: peer=server-e1f0691e-0ba0-c022-c1c5-175e73e93f7a -2025-01-25T14:29:51.344Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:51.344Z [INFO] server-a65baebb-6749-785b-1027-899b4cbba662: pipelining replication: peer="{Voter server-e1f0691e-0ba0-c022-c1c5-175e73e93f7a e1f0691e-0ba0-c022-c1c5-175e73e93f7a}" -2025-01-25T14:29:51.344Z [INFO] server-06c471f4-1096-676e-f79e-cba8e1f25656: entering follower state: follower="Node at 06c471f4-1096-676e-f79e-cba8e1f25656 [Follower]" leader-address= leader-id= -2025-01-25T14:29:51.344Z [INFO] server-06c471f4-1096-676e-f79e-cba8e1f25656: aborting pipeline replication: peer="{Voter server-a65baebb-6749-785b-1027-899b4cbba662 a65baebb-6749-785b-1027-899b4cbba662}" -2025-01-25T14:29:51.344Z [INFO] server-06c471f4-1096-676e-f79e-cba8e1f25656: aborting pipeline replication: peer="{Voter server-e1f0691e-0ba0-c022-c1c5-175e73e93f7a e1f0691e-0ba0-c022-c1c5-175e73e93f7a}" -2025-01-25T14:29:51.344Z [INFO] server-a65baebb-6749-785b-1027-899b4cbba662: pipelining replication: peer="{Voter server-06c471f4-1096-676e-f79e-cba8e1f25656 06c471f4-1096-676e-f79e-cba8e1f25656}" -2025-01-25T14:29:51.450Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:51.34474427 +0000 UTC m=+101.199069252 (1 nodes), 276.71µs from start of poll, 274.697914ms from cluster start. Timeout at 2025-01-25 14:29:51.44974591 +0000 UTC m=+101.304070892, 105.00164ms after stability -2025-01-25T14:29:51.450Z [INFO] server-a65baebb-6749-785b-1027-899b4cbba662: aborting pipeline replication: peer="{Voter server-e1f0691e-0ba0-c022-c1c5-175e73e93f7a e1f0691e-0ba0-c022-c1c5-175e73e93f7a}" -2025-01-25T14:29:51.450Z [INFO] server-a65baebb-6749-785b-1027-899b4cbba662: aborting pipeline replication: peer="{Voter server-06c471f4-1096-676e-f79e-cba8e1f25656 06c471f4-1096-676e-f79e-cba8e1f25656}" +2025-01-25T14:40:38.803Z [INFO] server-a96dcf4d-0533-06c4-b893-00e74ca4e9c4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a96dcf4d-0533-06c4-b893-00e74ca4e9c4 Address:a96dcf4d-0533-06c4-b893-00e74ca4e9c4} {Suffrage:Voter ID:server-66f1b245-5693-5894-71d0-3e5f57010dc0 Address:66f1b245-5693-5894-71d0-3e5f57010dc0} {Suffrage:Voter ID:server-f2d3d293-0b24-af71-0c9a-cce491c8df4b Address:f2d3d293-0b24-af71-0c9a-cce491c8df4b}]" +2025-01-25T14:40:38.803Z [INFO] server-a96dcf4d-0533-06c4-b893-00e74ca4e9c4: entering follower state: follower="Node at a96dcf4d-0533-06c4-b893-00e74ca4e9c4 [Follower]" leader-address= leader-id= +2025-01-25T14:40:38.804Z [INFO] server-66f1b245-5693-5894-71d0-3e5f57010dc0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a96dcf4d-0533-06c4-b893-00e74ca4e9c4 Address:a96dcf4d-0533-06c4-b893-00e74ca4e9c4} {Suffrage:Voter ID:server-66f1b245-5693-5894-71d0-3e5f57010dc0 Address:66f1b245-5693-5894-71d0-3e5f57010dc0} {Suffrage:Voter ID:server-f2d3d293-0b24-af71-0c9a-cce491c8df4b Address:f2d3d293-0b24-af71-0c9a-cce491c8df4b}]" +2025-01-25T14:40:38.804Z [INFO] server-66f1b245-5693-5894-71d0-3e5f57010dc0: entering follower state: follower="Node at 66f1b245-5693-5894-71d0-3e5f57010dc0 [Follower]" leader-address= leader-id= +2025-01-25T14:40:38.805Z [INFO] server-f2d3d293-0b24-af71-0c9a-cce491c8df4b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a96dcf4d-0533-06c4-b893-00e74ca4e9c4 Address:a96dcf4d-0533-06c4-b893-00e74ca4e9c4} {Suffrage:Voter ID:server-66f1b245-5693-5894-71d0-3e5f57010dc0 Address:66f1b245-5693-5894-71d0-3e5f57010dc0} {Suffrage:Voter ID:server-f2d3d293-0b24-af71-0c9a-cce491c8df4b Address:f2d3d293-0b24-af71-0c9a-cce491c8df4b}]" +2025-01-25T14:40:38.805Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:38.805Z [INFO] server-f2d3d293-0b24-af71-0c9a-cce491c8df4b: entering follower state: follower="Node at f2d3d293-0b24-af71-0c9a-cce491c8df4b [Follower]" leader-address= leader-id= +2025-01-25T14:40:38.857Z [WARN] server-a96dcf4d-0533-06c4-b893-00e74ca4e9c4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:38.857Z [INFO] server-a96dcf4d-0533-06c4-b893-00e74ca4e9c4: entering candidate state: node="Node at a96dcf4d-0533-06c4-b893-00e74ca4e9c4 [Candidate]" term=2 +2025-01-25T14:40:38.857Z [INFO] server-a96dcf4d-0533-06c4-b893-00e74ca4e9c4: election won: term=2 tally=2 +2025-01-25T14:40:38.857Z [INFO] server-a96dcf4d-0533-06c4-b893-00e74ca4e9c4: entering leader state: leader="Node at a96dcf4d-0533-06c4-b893-00e74ca4e9c4 [Leader]" +2025-01-25T14:40:38.857Z [INFO] server-a96dcf4d-0533-06c4-b893-00e74ca4e9c4: added peer, starting replication: peer=server-66f1b245-5693-5894-71d0-3e5f57010dc0 +2025-01-25T14:40:38.857Z [INFO] server-a96dcf4d-0533-06c4-b893-00e74ca4e9c4: added peer, starting replication: peer=server-f2d3d293-0b24-af71-0c9a-cce491c8df4b +2025-01-25T14:40:38.858Z [INFO] server-a96dcf4d-0533-06c4-b893-00e74ca4e9c4: pipelining replication: peer="{Voter server-f2d3d293-0b24-af71-0c9a-cce491c8df4b f2d3d293-0b24-af71-0c9a-cce491c8df4b}" +2025-01-25T14:40:38.858Z [INFO] server-a96dcf4d-0533-06c4-b893-00e74ca4e9c4: pipelining replication: peer="{Voter server-66f1b245-5693-5894-71d0-3e5f57010dc0 66f1b245-5693-5894-71d0-3e5f57010dc0}" +2025-01-25T14:40:38.967Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:38.857756584 +0000 UTC m=+107.903595901 (1 nodes), 52.251943ms from start of poll, 55.090452ms from cluster start. Timeout at 2025-01-25 14:40:38.962775542 +0000 UTC m=+108.008614860, 105.018959ms after stability +2025-01-25T14:40:38.967Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:39.073Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:38.967813696 +0000 UTC m=+108.013653013 (1 nodes), 3.791µs from start of poll, 165.147564ms from cluster start. Timeout at 2025-01-25 14:40:39.072998321 +0000 UTC m=+108.118837639, 105.184626ms after stability +2025-01-25T14:40:39.073Z [INFO] server-66f1b245-5693-5894-71d0-3e5f57010dc0: entering candidate state: node="Node at 66f1b245-5693-5894-71d0-3e5f57010dc0 [Candidate]" term=3 +2025-01-25T14:40:39.073Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:39.074Z [INFO] server-a96dcf4d-0533-06c4-b893-00e74ca4e9c4: entering follower state: follower="Node at a96dcf4d-0533-06c4-b893-00e74ca4e9c4 [Follower]" leader-address= leader-id= +2025-01-25T14:40:39.074Z [INFO] server-66f1b245-5693-5894-71d0-3e5f57010dc0: election won: term=3 tally=2 +2025-01-25T14:40:39.074Z [INFO] server-a96dcf4d-0533-06c4-b893-00e74ca4e9c4: aborting pipeline replication: peer="{Voter server-66f1b245-5693-5894-71d0-3e5f57010dc0 66f1b245-5693-5894-71d0-3e5f57010dc0}" +2025-01-25T14:40:39.074Z [INFO] server-66f1b245-5693-5894-71d0-3e5f57010dc0: entering leader state: leader="Node at 66f1b245-5693-5894-71d0-3e5f57010dc0 [Leader]" +2025-01-25T14:40:39.074Z [INFO] server-a96dcf4d-0533-06c4-b893-00e74ca4e9c4: aborting pipeline replication: peer="{Voter server-f2d3d293-0b24-af71-0c9a-cce491c8df4b f2d3d293-0b24-af71-0c9a-cce491c8df4b}" +2025-01-25T14:40:39.074Z [INFO] server-66f1b245-5693-5894-71d0-3e5f57010dc0: added peer, starting replication: peer=server-a96dcf4d-0533-06c4-b893-00e74ca4e9c4 +2025-01-25T14:40:39.074Z [INFO] server-66f1b245-5693-5894-71d0-3e5f57010dc0: added peer, starting replication: peer=server-f2d3d293-0b24-af71-0c9a-cce491c8df4b +2025-01-25T14:40:39.075Z [INFO] server-66f1b245-5693-5894-71d0-3e5f57010dc0: pipelining replication: peer="{Voter server-a96dcf4d-0533-06c4-b893-00e74ca4e9c4 a96dcf4d-0533-06c4-b893-00e74ca4e9c4}" +2025-01-25T14:40:39.075Z [INFO] server-66f1b245-5693-5894-71d0-3e5f57010dc0: pipelining replication: peer="{Voter server-f2d3d293-0b24-af71-0c9a-cce491c8df4b f2d3d293-0b24-af71-0c9a-cce491c8df4b}" +2025-01-25T14:40:39.180Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:39.07443084 +0000 UTC m=+108.120270157 (1 nodes), 584.043µs from start of poll, 271.764708ms from cluster start. Timeout at 2025-01-25 14:40:39.17968809 +0000 UTC m=+108.225527407, 105.25725ms after stability +2025-01-25T14:40:39.180Z [INFO] server-66f1b245-5693-5894-71d0-3e5f57010dc0: aborting pipeline replication: peer="{Voter server-f2d3d293-0b24-af71-0c9a-cce491c8df4b f2d3d293-0b24-af71-0c9a-cce491c8df4b}" +2025-01-25T14:40:39.180Z [INFO] server-66f1b245-5693-5894-71d0-3e5f57010dc0: aborting pipeline replication: peer="{Voter server-a96dcf4d-0533-06c4-b893-00e74ca4e9c4 a96dcf4d-0533-06c4-b893-00e74ca4e9c4}" --- PASS: TestRaft_LeadershipTransfer (0.38s) === RUN TestRaft_LeadershipTransferWithOneNode -2025-01-25T14:29:51.452Z [INFO] server-c52787a1-0dbf-1315-a4e7-b6d887cfd5b8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c52787a1-0dbf-1315-a4e7-b6d887cfd5b8 Address:c52787a1-0dbf-1315-a4e7-b6d887cfd5b8}]" -2025-01-25T14:29:51.452Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:51.452Z [INFO] server-c52787a1-0dbf-1315-a4e7-b6d887cfd5b8: entering follower state: follower="Node at c52787a1-0dbf-1315-a4e7-b6d887cfd5b8 [Follower]" leader-address= leader-id= -2025-01-25T14:29:51.534Z [WARN] server-c52787a1-0dbf-1315-a4e7-b6d887cfd5b8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:51.534Z [INFO] server-c52787a1-0dbf-1315-a4e7-b6d887cfd5b8: entering candidate state: node="Node at c52787a1-0dbf-1315-a4e7-b6d887cfd5b8 [Candidate]" term=2 -2025-01-25T14:29:51.534Z [INFO] server-c52787a1-0dbf-1315-a4e7-b6d887cfd5b8: election won: term=2 tally=1 -2025-01-25T14:29:51.534Z [INFO] server-c52787a1-0dbf-1315-a4e7-b6d887cfd5b8: entering leader state: leader="Node at c52787a1-0dbf-1315-a4e7-b6d887cfd5b8 [Leader]" -2025-01-25T14:29:51.641Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:51.535146778 +0000 UTC m=+101.389471720 (1 nodes), 82.649506ms from start of poll, 82.890337ms from cluster start. Timeout at 2025-01-25 14:29:51.640151698 +0000 UTC m=+101.494476640, 105.00492ms after stability ---- PASS: TestRaft_LeadershipTransferWithOneNode (0.19s) +2025-01-25T14:40:39.185Z [INFO] server-bc20dcc6-84da-5893-c5a1-157c02fc75b8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bc20dcc6-84da-5893-c5a1-157c02fc75b8 Address:bc20dcc6-84da-5893-c5a1-157c02fc75b8}]" +2025-01-25T14:40:39.185Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:39.185Z [INFO] server-bc20dcc6-84da-5893-c5a1-157c02fc75b8: entering follower state: follower="Node at bc20dcc6-84da-5893-c5a1-157c02fc75b8 [Follower]" leader-address= leader-id= +2025-01-25T14:40:39.258Z [WARN] server-bc20dcc6-84da-5893-c5a1-157c02fc75b8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:39.258Z [INFO] server-bc20dcc6-84da-5893-c5a1-157c02fc75b8: entering candidate state: node="Node at bc20dcc6-84da-5893-c5a1-157c02fc75b8 [Candidate]" term=2 +2025-01-25T14:40:39.258Z [INFO] server-bc20dcc6-84da-5893-c5a1-157c02fc75b8: election won: term=2 tally=1 +2025-01-25T14:40:39.258Z [INFO] server-bc20dcc6-84da-5893-c5a1-157c02fc75b8: entering leader state: leader="Node at bc20dcc6-84da-5893-c5a1-157c02fc75b8 [Leader]" +2025-01-25T14:40:39.364Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:39.258587834 +0000 UTC m=+108.304427152 (1 nodes), 72.960338ms from start of poll, 73.622131ms from cluster start. Timeout at 2025-01-25 14:40:39.363607667 +0000 UTC m=+108.409446985, 105.019833ms after stability +--- PASS: TestRaft_LeadershipTransferWithOneNode (0.18s) === RUN TestRaft_LeadershipTransferWithSevenNodes -2025-01-25T14:29:51.645Z [INFO] server-b1ef68a1-157e-004f-2992-86f439700735: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b1ef68a1-157e-004f-2992-86f439700735 Address:b1ef68a1-157e-004f-2992-86f439700735} {Suffrage:Voter ID:server-63dfeab7-b1db-8c9d-e8ff-9e5a455187ef Address:63dfeab7-b1db-8c9d-e8ff-9e5a455187ef} {Suffrage:Voter ID:server-8a095541-f276-2986-14c9-f10f3a3f537b Address:8a095541-f276-2986-14c9-f10f3a3f537b} {Suffrage:Voter ID:server-c0207010-c564-e72f-006e-a76fcc4ba9e4 Address:c0207010-c564-e72f-006e-a76fcc4ba9e4} {Suffrage:Voter ID:server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d Address:ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d} {Suffrage:Voter ID:server-7731092c-1bb9-a1a7-f8d8-14c99df84e0e Address:7731092c-1bb9-a1a7-f8d8-14c99df84e0e} {Suffrage:Voter ID:server-8c2b8647-cd78-d3a2-15db-31bb4a415706 Address:8c2b8647-cd78-d3a2-15db-31bb4a415706}]" -2025-01-25T14:29:51.645Z [INFO] server-63dfeab7-b1db-8c9d-e8ff-9e5a455187ef: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b1ef68a1-157e-004f-2992-86f439700735 Address:b1ef68a1-157e-004f-2992-86f439700735} {Suffrage:Voter ID:server-63dfeab7-b1db-8c9d-e8ff-9e5a455187ef Address:63dfeab7-b1db-8c9d-e8ff-9e5a455187ef} {Suffrage:Voter ID:server-8a095541-f276-2986-14c9-f10f3a3f537b Address:8a095541-f276-2986-14c9-f10f3a3f537b} {Suffrage:Voter ID:server-c0207010-c564-e72f-006e-a76fcc4ba9e4 Address:c0207010-c564-e72f-006e-a76fcc4ba9e4} {Suffrage:Voter ID:server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d Address:ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d} {Suffrage:Voter ID:server-7731092c-1bb9-a1a7-f8d8-14c99df84e0e Address:7731092c-1bb9-a1a7-f8d8-14c99df84e0e} {Suffrage:Voter ID:server-8c2b8647-cd78-d3a2-15db-31bb4a415706 Address:8c2b8647-cd78-d3a2-15db-31bb4a415706}]" -2025-01-25T14:29:51.645Z [INFO] server-8a095541-f276-2986-14c9-f10f3a3f537b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b1ef68a1-157e-004f-2992-86f439700735 Address:b1ef68a1-157e-004f-2992-86f439700735} {Suffrage:Voter ID:server-63dfeab7-b1db-8c9d-e8ff-9e5a455187ef Address:63dfeab7-b1db-8c9d-e8ff-9e5a455187ef} {Suffrage:Voter ID:server-8a095541-f276-2986-14c9-f10f3a3f537b Address:8a095541-f276-2986-14c9-f10f3a3f537b} {Suffrage:Voter ID:server-c0207010-c564-e72f-006e-a76fcc4ba9e4 Address:c0207010-c564-e72f-006e-a76fcc4ba9e4} {Suffrage:Voter ID:server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d Address:ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d} {Suffrage:Voter ID:server-7731092c-1bb9-a1a7-f8d8-14c99df84e0e Address:7731092c-1bb9-a1a7-f8d8-14c99df84e0e} {Suffrage:Voter ID:server-8c2b8647-cd78-d3a2-15db-31bb4a415706 Address:8c2b8647-cd78-d3a2-15db-31bb4a415706}]" -2025-01-25T14:29:51.646Z [INFO] server-c0207010-c564-e72f-006e-a76fcc4ba9e4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b1ef68a1-157e-004f-2992-86f439700735 Address:b1ef68a1-157e-004f-2992-86f439700735} {Suffrage:Voter ID:server-63dfeab7-b1db-8c9d-e8ff-9e5a455187ef Address:63dfeab7-b1db-8c9d-e8ff-9e5a455187ef} {Suffrage:Voter ID:server-8a095541-f276-2986-14c9-f10f3a3f537b Address:8a095541-f276-2986-14c9-f10f3a3f537b} {Suffrage:Voter ID:server-c0207010-c564-e72f-006e-a76fcc4ba9e4 Address:c0207010-c564-e72f-006e-a76fcc4ba9e4} {Suffrage:Voter ID:server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d Address:ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d} {Suffrage:Voter ID:server-7731092c-1bb9-a1a7-f8d8-14c99df84e0e Address:7731092c-1bb9-a1a7-f8d8-14c99df84e0e} {Suffrage:Voter ID:server-8c2b8647-cd78-d3a2-15db-31bb4a415706 Address:8c2b8647-cd78-d3a2-15db-31bb4a415706}]" -2025-01-25T14:29:51.646Z [INFO] server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b1ef68a1-157e-004f-2992-86f439700735 Address:b1ef68a1-157e-004f-2992-86f439700735} {Suffrage:Voter ID:server-63dfeab7-b1db-8c9d-e8ff-9e5a455187ef Address:63dfeab7-b1db-8c9d-e8ff-9e5a455187ef} {Suffrage:Voter ID:server-8a095541-f276-2986-14c9-f10f3a3f537b Address:8a095541-f276-2986-14c9-f10f3a3f537b} {Suffrage:Voter ID:server-c0207010-c564-e72f-006e-a76fcc4ba9e4 Address:c0207010-c564-e72f-006e-a76fcc4ba9e4} {Suffrage:Voter ID:server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d Address:ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d} {Suffrage:Voter ID:server-7731092c-1bb9-a1a7-f8d8-14c99df84e0e Address:7731092c-1bb9-a1a7-f8d8-14c99df84e0e} {Suffrage:Voter ID:server-8c2b8647-cd78-d3a2-15db-31bb4a415706 Address:8c2b8647-cd78-d3a2-15db-31bb4a415706}]" -2025-01-25T14:29:51.646Z [INFO] server-7731092c-1bb9-a1a7-f8d8-14c99df84e0e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b1ef68a1-157e-004f-2992-86f439700735 Address:b1ef68a1-157e-004f-2992-86f439700735} {Suffrage:Voter ID:server-63dfeab7-b1db-8c9d-e8ff-9e5a455187ef Address:63dfeab7-b1db-8c9d-e8ff-9e5a455187ef} {Suffrage:Voter ID:server-8a095541-f276-2986-14c9-f10f3a3f537b Address:8a095541-f276-2986-14c9-f10f3a3f537b} {Suffrage:Voter ID:server-c0207010-c564-e72f-006e-a76fcc4ba9e4 Address:c0207010-c564-e72f-006e-a76fcc4ba9e4} {Suffrage:Voter ID:server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d Address:ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d} {Suffrage:Voter ID:server-7731092c-1bb9-a1a7-f8d8-14c99df84e0e Address:7731092c-1bb9-a1a7-f8d8-14c99df84e0e} {Suffrage:Voter ID:server-8c2b8647-cd78-d3a2-15db-31bb4a415706 Address:8c2b8647-cd78-d3a2-15db-31bb4a415706}]" -2025-01-25T14:29:51.647Z [INFO] server-b1ef68a1-157e-004f-2992-86f439700735: entering follower state: follower="Node at b1ef68a1-157e-004f-2992-86f439700735 [Follower]" leader-address= leader-id= -2025-01-25T14:29:51.647Z [INFO] server-63dfeab7-b1db-8c9d-e8ff-9e5a455187ef: entering follower state: follower="Node at 63dfeab7-b1db-8c9d-e8ff-9e5a455187ef [Follower]" leader-address= leader-id= -2025-01-25T14:29:51.647Z [INFO] server-8a095541-f276-2986-14c9-f10f3a3f537b: entering follower state: follower="Node at 8a095541-f276-2986-14c9-f10f3a3f537b [Follower]" leader-address= leader-id= -2025-01-25T14:29:51.647Z [INFO] server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d: entering follower state: follower="Node at ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d [Follower]" leader-address= leader-id= -2025-01-25T14:29:51.647Z [INFO] server-c0207010-c564-e72f-006e-a76fcc4ba9e4: entering follower state: follower="Node at c0207010-c564-e72f-006e-a76fcc4ba9e4 [Follower]" leader-address= leader-id= -2025-01-25T14:29:51.647Z [INFO] server-7731092c-1bb9-a1a7-f8d8-14c99df84e0e: entering follower state: follower="Node at 7731092c-1bb9-a1a7-f8d8-14c99df84e0e [Follower]" leader-address= leader-id= -2025-01-25T14:29:51.648Z [INFO] server-8c2b8647-cd78-d3a2-15db-31bb4a415706: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b1ef68a1-157e-004f-2992-86f439700735 Address:b1ef68a1-157e-004f-2992-86f439700735} {Suffrage:Voter ID:server-63dfeab7-b1db-8c9d-e8ff-9e5a455187ef Address:63dfeab7-b1db-8c9d-e8ff-9e5a455187ef} {Suffrage:Voter ID:server-8a095541-f276-2986-14c9-f10f3a3f537b Address:8a095541-f276-2986-14c9-f10f3a3f537b} {Suffrage:Voter ID:server-c0207010-c564-e72f-006e-a76fcc4ba9e4 Address:c0207010-c564-e72f-006e-a76fcc4ba9e4} {Suffrage:Voter ID:server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d Address:ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d} {Suffrage:Voter ID:server-7731092c-1bb9-a1a7-f8d8-14c99df84e0e Address:7731092c-1bb9-a1a7-f8d8-14c99df84e0e} {Suffrage:Voter ID:server-8c2b8647-cd78-d3a2-15db-31bb4a415706 Address:8c2b8647-cd78-d3a2-15db-31bb4a415706}]" -2025-01-25T14:29:51.648Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:51.648Z [INFO] server-8c2b8647-cd78-d3a2-15db-31bb4a415706: entering follower state: follower="Node at 8c2b8647-cd78-d3a2-15db-31bb4a415706 [Follower]" leader-address= leader-id= -2025-01-25T14:29:51.698Z [WARN] server-8a095541-f276-2986-14c9-f10f3a3f537b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:51.698Z [WARN] server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:51.698Z [INFO] server-8a095541-f276-2986-14c9-f10f3a3f537b: entering candidate state: node="Node at 8a095541-f276-2986-14c9-f10f3a3f537b [Candidate]" term=2 -2025-01-25T14:29:51.698Z [INFO] server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d: entering candidate state: node="Node at ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d [Candidate]" term=2 -2025-01-25T14:29:51.698Z [INFO] server-8c2b8647-cd78-d3a2-15db-31bb4a415706: duplicate requestVote for same term: term=2 -2025-01-25T14:29:51.699Z [INFO] server-b1ef68a1-157e-004f-2992-86f439700735: duplicate requestVote for same term: term=2 -2025-01-25T14:29:51.699Z [INFO] server-8a095541-f276-2986-14c9-f10f3a3f537b: duplicate requestVote for same term: term=2 -2025-01-25T14:29:51.699Z [INFO] server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d: election won: term=2 tally=4 -2025-01-25T14:29:51.699Z [INFO] server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d: entering leader state: leader="Node at ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d [Leader]" -2025-01-25T14:29:51.699Z [INFO] server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d: added peer, starting replication: peer=server-b1ef68a1-157e-004f-2992-86f439700735 -2025-01-25T14:29:51.699Z [INFO] server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d: added peer, starting replication: peer=server-63dfeab7-b1db-8c9d-e8ff-9e5a455187ef -2025-01-25T14:29:51.699Z [INFO] server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d: added peer, starting replication: peer=server-8a095541-f276-2986-14c9-f10f3a3f537b -2025-01-25T14:29:51.699Z [INFO] server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d: added peer, starting replication: peer=server-c0207010-c564-e72f-006e-a76fcc4ba9e4 -2025-01-25T14:29:51.699Z [INFO] server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d: added peer, starting replication: peer=server-7731092c-1bb9-a1a7-f8d8-14c99df84e0e -2025-01-25T14:29:51.699Z [INFO] server-63dfeab7-b1db-8c9d-e8ff-9e5a455187ef: duplicate requestVote for same term: term=2 -2025-01-25T14:29:51.699Z [INFO] server-c0207010-c564-e72f-006e-a76fcc4ba9e4: duplicate requestVote for same term: term=2 -2025-01-25T14:29:51.699Z [INFO] server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d: added peer, starting replication: peer=server-8c2b8647-cd78-d3a2-15db-31bb4a415706 -2025-01-25T14:29:51.699Z [INFO] server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d: pipelining replication: peer="{Voter server-b1ef68a1-157e-004f-2992-86f439700735 b1ef68a1-157e-004f-2992-86f439700735}" -2025-01-25T14:29:51.699Z [INFO] server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d: pipelining replication: peer="{Voter server-8c2b8647-cd78-d3a2-15db-31bb4a415706 8c2b8647-cd78-d3a2-15db-31bb4a415706}" -2025-01-25T14:29:51.699Z [INFO] server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d: pipelining replication: peer="{Voter server-63dfeab7-b1db-8c9d-e8ff-9e5a455187ef 63dfeab7-b1db-8c9d-e8ff-9e5a455187ef}" -2025-01-25T14:29:51.699Z [WARN] server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d: rejecting vote request since we have a leader: from=8a095541-f276-2986-14c9-f10f3a3f537b leader=ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d leader-id=server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d -2025-01-25T14:29:51.699Z [INFO] server-7731092c-1bb9-a1a7-f8d8-14c99df84e0e: duplicate requestVote for same term: term=2 -2025-01-25T14:29:51.700Z [INFO] server-8a095541-f276-2986-14c9-f10f3a3f537b: entering follower state: follower="Node at 8a095541-f276-2986-14c9-f10f3a3f537b [Follower]" leader-address=ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d leader-id=server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d -2025-01-25T14:29:51.700Z [INFO] server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d: pipelining replication: peer="{Voter server-7731092c-1bb9-a1a7-f8d8-14c99df84e0e 7731092c-1bb9-a1a7-f8d8-14c99df84e0e}" -2025-01-25T14:29:51.700Z [INFO] server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d: pipelining replication: peer="{Voter server-c0207010-c564-e72f-006e-a76fcc4ba9e4 c0207010-c564-e72f-006e-a76fcc4ba9e4}" -2025-01-25T14:29:51.700Z [INFO] server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d: pipelining replication: peer="{Voter server-8a095541-f276-2986-14c9-f10f3a3f537b 8a095541-f276-2986-14c9-f10f3a3f537b}" -2025-01-25T14:29:51.805Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:51.700383072 +0000 UTC m=+101.554708054 (1 nodes), 51.805336ms from start of poll, 55.322569ms from cluster start. Timeout at 2025-01-25 14:29:51.805385512 +0000 UTC m=+101.659710494, 105.00244ms after stability -2025-01-25T14:29:51.805Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:29:51.911Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:29:51.805791358 +0000 UTC m=+101.660116340 (6 nodes), 3.84µs from start of poll, 160.730855ms from cluster start. Timeout at 2025-01-25 14:29:51.911079228 +0000 UTC m=+101.765404170, 105.28783ms after stability -2025-01-25T14:29:51.911Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:52.017Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:51.911826622 +0000 UTC m=+101.766151604 (1 nodes), 1.92µs from start of poll, 266.766119ms from cluster start. Timeout at 2025-01-25 14:29:52.017041054 +0000 UTC m=+101.871366036, 105.214432ms after stability -2025-01-25T14:29:52.017Z [INFO] server-b1ef68a1-157e-004f-2992-86f439700735: entering candidate state: node="Node at b1ef68a1-157e-004f-2992-86f439700735 [Candidate]" term=3 -2025-01-25T14:29:52.017Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:52.018Z [INFO] server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d: entering follower state: follower="Node at ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d [Follower]" leader-address= leader-id= -2025-01-25T14:29:52.018Z [INFO] server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d: aborting pipeline replication: peer="{Voter server-b1ef68a1-157e-004f-2992-86f439700735 b1ef68a1-157e-004f-2992-86f439700735}" -2025-01-25T14:29:52.018Z [INFO] server-b1ef68a1-157e-004f-2992-86f439700735: election won: term=3 tally=4 -2025-01-25T14:29:52.018Z [INFO] server-b1ef68a1-157e-004f-2992-86f439700735: entering leader state: leader="Node at b1ef68a1-157e-004f-2992-86f439700735 [Leader]" -2025-01-25T14:29:52.018Z [INFO] server-b1ef68a1-157e-004f-2992-86f439700735: added peer, starting replication: peer=server-63dfeab7-b1db-8c9d-e8ff-9e5a455187ef -2025-01-25T14:29:52.018Z [INFO] server-b1ef68a1-157e-004f-2992-86f439700735: added peer, starting replication: peer=server-8a095541-f276-2986-14c9-f10f3a3f537b -2025-01-25T14:29:52.018Z [INFO] server-b1ef68a1-157e-004f-2992-86f439700735: added peer, starting replication: peer=server-c0207010-c564-e72f-006e-a76fcc4ba9e4 -2025-01-25T14:29:52.018Z [INFO] server-b1ef68a1-157e-004f-2992-86f439700735: added peer, starting replication: peer=server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d -2025-01-25T14:29:52.018Z [INFO] server-b1ef68a1-157e-004f-2992-86f439700735: added peer, starting replication: peer=server-7731092c-1bb9-a1a7-f8d8-14c99df84e0e -2025-01-25T14:29:52.018Z [INFO] server-b1ef68a1-157e-004f-2992-86f439700735: added peer, starting replication: peer=server-8c2b8647-cd78-d3a2-15db-31bb4a415706 -2025-01-25T14:29:52.019Z [INFO] server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d: aborting pipeline replication: peer="{Voter server-63dfeab7-b1db-8c9d-e8ff-9e5a455187ef 63dfeab7-b1db-8c9d-e8ff-9e5a455187ef}" -2025-01-25T14:29:52.019Z [INFO] server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d: aborting pipeline replication: peer="{Voter server-8a095541-f276-2986-14c9-f10f3a3f537b 8a095541-f276-2986-14c9-f10f3a3f537b}" -2025-01-25T14:29:52.019Z [INFO] server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d: aborting pipeline replication: peer="{Voter server-c0207010-c564-e72f-006e-a76fcc4ba9e4 c0207010-c564-e72f-006e-a76fcc4ba9e4}" -2025-01-25T14:29:52.019Z [INFO] server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d: aborting pipeline replication: peer="{Voter server-7731092c-1bb9-a1a7-f8d8-14c99df84e0e 7731092c-1bb9-a1a7-f8d8-14c99df84e0e}" -2025-01-25T14:29:52.019Z [INFO] server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d: aborting pipeline replication: peer="{Voter server-8c2b8647-cd78-d3a2-15db-31bb4a415706 8c2b8647-cd78-d3a2-15db-31bb4a415706}" -2025-01-25T14:29:52.019Z [INFO] server-b1ef68a1-157e-004f-2992-86f439700735: pipelining replication: peer="{Voter server-c0207010-c564-e72f-006e-a76fcc4ba9e4 c0207010-c564-e72f-006e-a76fcc4ba9e4}" -2025-01-25T14:29:52.019Z [INFO] server-b1ef68a1-157e-004f-2992-86f439700735: pipelining replication: peer="{Voter server-7731092c-1bb9-a1a7-f8d8-14c99df84e0e 7731092c-1bb9-a1a7-f8d8-14c99df84e0e}" -2025-01-25T14:29:52.020Z [INFO] server-b1ef68a1-157e-004f-2992-86f439700735: pipelining replication: peer="{Voter server-8c2b8647-cd78-d3a2-15db-31bb4a415706 8c2b8647-cd78-d3a2-15db-31bb4a415706}" -2025-01-25T14:29:52.021Z [INFO] server-b1ef68a1-157e-004f-2992-86f439700735: pipelining replication: peer="{Voter server-63dfeab7-b1db-8c9d-e8ff-9e5a455187ef 63dfeab7-b1db-8c9d-e8ff-9e5a455187ef}" -2025-01-25T14:29:52.021Z [INFO] server-b1ef68a1-157e-004f-2992-86f439700735: pipelining replication: peer="{Voter server-8a095541-f276-2986-14c9-f10f3a3f537b 8a095541-f276-2986-14c9-f10f3a3f537b}" -2025-01-25T14:29:52.021Z [INFO] server-b1ef68a1-157e-004f-2992-86f439700735: pipelining replication: peer="{Voter server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d}" -2025-01-25T14:29:52.124Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:52.018952447 +0000 UTC m=+101.873277389 (1 nodes), 993.204µs from start of poll, 373.891904ms from cluster start. Timeout at 2025-01-25 14:29:52.123985045 +0000 UTC m=+101.978309987, 105.032598ms after stability -2025-01-25T14:29:52.125Z [INFO] server-b1ef68a1-157e-004f-2992-86f439700735: aborting pipeline replication: peer="{Voter server-8a095541-f276-2986-14c9-f10f3a3f537b 8a095541-f276-2986-14c9-f10f3a3f537b}" -2025-01-25T14:29:52.126Z [INFO] server-b1ef68a1-157e-004f-2992-86f439700735: aborting pipeline replication: peer="{Voter server-8c2b8647-cd78-d3a2-15db-31bb4a415706 8c2b8647-cd78-d3a2-15db-31bb4a415706}" -2025-01-25T14:29:52.126Z [INFO] server-b1ef68a1-157e-004f-2992-86f439700735: aborting pipeline replication: peer="{Voter server-ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d ef6fc62f-910c-6fae-4bd8-d03f5ebdc08d}" -2025-01-25T14:29:52.126Z [INFO] server-b1ef68a1-157e-004f-2992-86f439700735: aborting pipeline replication: peer="{Voter server-7731092c-1bb9-a1a7-f8d8-14c99df84e0e 7731092c-1bb9-a1a7-f8d8-14c99df84e0e}" -2025-01-25T14:29:52.126Z [INFO] server-b1ef68a1-157e-004f-2992-86f439700735: aborting pipeline replication: peer="{Voter server-c0207010-c564-e72f-006e-a76fcc4ba9e4 c0207010-c564-e72f-006e-a76fcc4ba9e4}" -2025-01-25T14:29:52.127Z [INFO] server-b1ef68a1-157e-004f-2992-86f439700735: aborting pipeline replication: peer="{Voter server-63dfeab7-b1db-8c9d-e8ff-9e5a455187ef 63dfeab7-b1db-8c9d-e8ff-9e5a455187ef}" -2025-01-25T14:29:52.626Z [ERROR] server-b1ef68a1-157e-004f-2992-86f439700735: failed to heartbeat to: peer=7731092c-1bb9-a1a7-f8d8-14c99df84e0e backoff time=10ms error="command timed out" ---- PASS: TestRaft_LeadershipTransferWithSevenNodes (0.99s) +2025-01-25T14:40:39.375Z [INFO] server-1c339361-b2de-8273-74be-e9a5da7dde1a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1c339361-b2de-8273-74be-e9a5da7dde1a Address:1c339361-b2de-8273-74be-e9a5da7dde1a} {Suffrage:Voter ID:server-f072e323-9daa-cea8-dc98-4134dc03681b Address:f072e323-9daa-cea8-dc98-4134dc03681b} {Suffrage:Voter ID:server-342ed6b5-d87e-9359-76a4-eaaa0a7f5f99 Address:342ed6b5-d87e-9359-76a4-eaaa0a7f5f99} {Suffrage:Voter ID:server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7 Address:19d4612b-3d28-6de1-ba6c-88ed4c01b4a7} {Suffrage:Voter ID:server-7c77cc1e-0413-9a40-4c07-7966cdc46751 Address:7c77cc1e-0413-9a40-4c07-7966cdc46751} {Suffrage:Voter ID:server-ea1dbf4a-c6e8-703f-614f-18b4362c5eb6 Address:ea1dbf4a-c6e8-703f-614f-18b4362c5eb6} {Suffrage:Voter ID:server-748d830a-3f32-d4f1-e8e1-edc1c01e8190 Address:748d830a-3f32-d4f1-e8e1-edc1c01e8190}]" +2025-01-25T14:40:39.376Z [INFO] server-1c339361-b2de-8273-74be-e9a5da7dde1a: entering follower state: follower="Node at 1c339361-b2de-8273-74be-e9a5da7dde1a [Follower]" leader-address= leader-id= +2025-01-25T14:40:39.377Z [INFO] server-f072e323-9daa-cea8-dc98-4134dc03681b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1c339361-b2de-8273-74be-e9a5da7dde1a Address:1c339361-b2de-8273-74be-e9a5da7dde1a} {Suffrage:Voter ID:server-f072e323-9daa-cea8-dc98-4134dc03681b Address:f072e323-9daa-cea8-dc98-4134dc03681b} {Suffrage:Voter ID:server-342ed6b5-d87e-9359-76a4-eaaa0a7f5f99 Address:342ed6b5-d87e-9359-76a4-eaaa0a7f5f99} {Suffrage:Voter ID:server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7 Address:19d4612b-3d28-6de1-ba6c-88ed4c01b4a7} {Suffrage:Voter ID:server-7c77cc1e-0413-9a40-4c07-7966cdc46751 Address:7c77cc1e-0413-9a40-4c07-7966cdc46751} {Suffrage:Voter ID:server-ea1dbf4a-c6e8-703f-614f-18b4362c5eb6 Address:ea1dbf4a-c6e8-703f-614f-18b4362c5eb6} {Suffrage:Voter ID:server-748d830a-3f32-d4f1-e8e1-edc1c01e8190 Address:748d830a-3f32-d4f1-e8e1-edc1c01e8190}]" +2025-01-25T14:40:39.377Z [INFO] server-f072e323-9daa-cea8-dc98-4134dc03681b: entering follower state: follower="Node at f072e323-9daa-cea8-dc98-4134dc03681b [Follower]" leader-address= leader-id= +2025-01-25T14:40:39.378Z [INFO] server-342ed6b5-d87e-9359-76a4-eaaa0a7f5f99: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1c339361-b2de-8273-74be-e9a5da7dde1a Address:1c339361-b2de-8273-74be-e9a5da7dde1a} {Suffrage:Voter ID:server-f072e323-9daa-cea8-dc98-4134dc03681b Address:f072e323-9daa-cea8-dc98-4134dc03681b} {Suffrage:Voter ID:server-342ed6b5-d87e-9359-76a4-eaaa0a7f5f99 Address:342ed6b5-d87e-9359-76a4-eaaa0a7f5f99} {Suffrage:Voter ID:server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7 Address:19d4612b-3d28-6de1-ba6c-88ed4c01b4a7} {Suffrage:Voter ID:server-7c77cc1e-0413-9a40-4c07-7966cdc46751 Address:7c77cc1e-0413-9a40-4c07-7966cdc46751} {Suffrage:Voter ID:server-ea1dbf4a-c6e8-703f-614f-18b4362c5eb6 Address:ea1dbf4a-c6e8-703f-614f-18b4362c5eb6} {Suffrage:Voter ID:server-748d830a-3f32-d4f1-e8e1-edc1c01e8190 Address:748d830a-3f32-d4f1-e8e1-edc1c01e8190}]" +2025-01-25T14:40:39.378Z [INFO] server-342ed6b5-d87e-9359-76a4-eaaa0a7f5f99: entering follower state: follower="Node at 342ed6b5-d87e-9359-76a4-eaaa0a7f5f99 [Follower]" leader-address= leader-id= +2025-01-25T14:40:39.379Z [INFO] server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1c339361-b2de-8273-74be-e9a5da7dde1a Address:1c339361-b2de-8273-74be-e9a5da7dde1a} {Suffrage:Voter ID:server-f072e323-9daa-cea8-dc98-4134dc03681b Address:f072e323-9daa-cea8-dc98-4134dc03681b} {Suffrage:Voter ID:server-342ed6b5-d87e-9359-76a4-eaaa0a7f5f99 Address:342ed6b5-d87e-9359-76a4-eaaa0a7f5f99} {Suffrage:Voter ID:server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7 Address:19d4612b-3d28-6de1-ba6c-88ed4c01b4a7} {Suffrage:Voter ID:server-7c77cc1e-0413-9a40-4c07-7966cdc46751 Address:7c77cc1e-0413-9a40-4c07-7966cdc46751} {Suffrage:Voter ID:server-ea1dbf4a-c6e8-703f-614f-18b4362c5eb6 Address:ea1dbf4a-c6e8-703f-614f-18b4362c5eb6} {Suffrage:Voter ID:server-748d830a-3f32-d4f1-e8e1-edc1c01e8190 Address:748d830a-3f32-d4f1-e8e1-edc1c01e8190}]" +2025-01-25T14:40:39.379Z [INFO] server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7: entering follower state: follower="Node at 19d4612b-3d28-6de1-ba6c-88ed4c01b4a7 [Follower]" leader-address= leader-id= +2025-01-25T14:40:39.380Z [INFO] server-7c77cc1e-0413-9a40-4c07-7966cdc46751: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1c339361-b2de-8273-74be-e9a5da7dde1a Address:1c339361-b2de-8273-74be-e9a5da7dde1a} {Suffrage:Voter ID:server-f072e323-9daa-cea8-dc98-4134dc03681b Address:f072e323-9daa-cea8-dc98-4134dc03681b} {Suffrage:Voter ID:server-342ed6b5-d87e-9359-76a4-eaaa0a7f5f99 Address:342ed6b5-d87e-9359-76a4-eaaa0a7f5f99} {Suffrage:Voter ID:server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7 Address:19d4612b-3d28-6de1-ba6c-88ed4c01b4a7} {Suffrage:Voter ID:server-7c77cc1e-0413-9a40-4c07-7966cdc46751 Address:7c77cc1e-0413-9a40-4c07-7966cdc46751} {Suffrage:Voter ID:server-ea1dbf4a-c6e8-703f-614f-18b4362c5eb6 Address:ea1dbf4a-c6e8-703f-614f-18b4362c5eb6} {Suffrage:Voter ID:server-748d830a-3f32-d4f1-e8e1-edc1c01e8190 Address:748d830a-3f32-d4f1-e8e1-edc1c01e8190}]" +2025-01-25T14:40:39.380Z [INFO] server-7c77cc1e-0413-9a40-4c07-7966cdc46751: entering follower state: follower="Node at 7c77cc1e-0413-9a40-4c07-7966cdc46751 [Follower]" leader-address= leader-id= +2025-01-25T14:40:39.381Z [INFO] server-ea1dbf4a-c6e8-703f-614f-18b4362c5eb6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1c339361-b2de-8273-74be-e9a5da7dde1a Address:1c339361-b2de-8273-74be-e9a5da7dde1a} {Suffrage:Voter ID:server-f072e323-9daa-cea8-dc98-4134dc03681b Address:f072e323-9daa-cea8-dc98-4134dc03681b} {Suffrage:Voter ID:server-342ed6b5-d87e-9359-76a4-eaaa0a7f5f99 Address:342ed6b5-d87e-9359-76a4-eaaa0a7f5f99} {Suffrage:Voter ID:server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7 Address:19d4612b-3d28-6de1-ba6c-88ed4c01b4a7} {Suffrage:Voter ID:server-7c77cc1e-0413-9a40-4c07-7966cdc46751 Address:7c77cc1e-0413-9a40-4c07-7966cdc46751} {Suffrage:Voter ID:server-ea1dbf4a-c6e8-703f-614f-18b4362c5eb6 Address:ea1dbf4a-c6e8-703f-614f-18b4362c5eb6} {Suffrage:Voter ID:server-748d830a-3f32-d4f1-e8e1-edc1c01e8190 Address:748d830a-3f32-d4f1-e8e1-edc1c01e8190}]" +2025-01-25T14:40:39.382Z [INFO] server-ea1dbf4a-c6e8-703f-614f-18b4362c5eb6: entering follower state: follower="Node at ea1dbf4a-c6e8-703f-614f-18b4362c5eb6 [Follower]" leader-address= leader-id= +2025-01-25T14:40:39.382Z [INFO] server-748d830a-3f32-d4f1-e8e1-edc1c01e8190: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1c339361-b2de-8273-74be-e9a5da7dde1a Address:1c339361-b2de-8273-74be-e9a5da7dde1a} {Suffrage:Voter ID:server-f072e323-9daa-cea8-dc98-4134dc03681b Address:f072e323-9daa-cea8-dc98-4134dc03681b} {Suffrage:Voter ID:server-342ed6b5-d87e-9359-76a4-eaaa0a7f5f99 Address:342ed6b5-d87e-9359-76a4-eaaa0a7f5f99} {Suffrage:Voter ID:server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7 Address:19d4612b-3d28-6de1-ba6c-88ed4c01b4a7} {Suffrage:Voter ID:server-7c77cc1e-0413-9a40-4c07-7966cdc46751 Address:7c77cc1e-0413-9a40-4c07-7966cdc46751} {Suffrage:Voter ID:server-ea1dbf4a-c6e8-703f-614f-18b4362c5eb6 Address:ea1dbf4a-c6e8-703f-614f-18b4362c5eb6} {Suffrage:Voter ID:server-748d830a-3f32-d4f1-e8e1-edc1c01e8190 Address:748d830a-3f32-d4f1-e8e1-edc1c01e8190}]" +2025-01-25T14:40:39.382Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:39.382Z [INFO] server-748d830a-3f32-d4f1-e8e1-edc1c01e8190: entering follower state: follower="Node at 748d830a-3f32-d4f1-e8e1-edc1c01e8190 [Follower]" leader-address= leader-id= +2025-01-25T14:40:39.433Z [WARN] server-f072e323-9daa-cea8-dc98-4134dc03681b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:39.433Z [WARN] server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:39.433Z [INFO] server-f072e323-9daa-cea8-dc98-4134dc03681b: entering candidate state: node="Node at f072e323-9daa-cea8-dc98-4134dc03681b [Candidate]" term=2 +2025-01-25T14:40:39.433Z [INFO] server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7: entering candidate state: node="Node at 19d4612b-3d28-6de1-ba6c-88ed4c01b4a7 [Candidate]" term=2 +2025-01-25T14:40:39.433Z [INFO] server-748d830a-3f32-d4f1-e8e1-edc1c01e8190: duplicate requestVote for same term: term=2 +2025-01-25T14:40:39.433Z [INFO] server-f072e323-9daa-cea8-dc98-4134dc03681b: duplicate requestVote for same term: term=2 +2025-01-25T14:40:39.433Z [INFO] server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7: election won: term=2 tally=4 +2025-01-25T14:40:39.433Z [INFO] server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7: entering leader state: leader="Node at 19d4612b-3d28-6de1-ba6c-88ed4c01b4a7 [Leader]" +2025-01-25T14:40:39.434Z [INFO] server-ea1dbf4a-c6e8-703f-614f-18b4362c5eb6: duplicate requestVote for same term: term=2 +2025-01-25T14:40:39.434Z [INFO] server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7: added peer, starting replication: peer=server-1c339361-b2de-8273-74be-e9a5da7dde1a +2025-01-25T14:40:39.434Z [INFO] server-1c339361-b2de-8273-74be-e9a5da7dde1a: duplicate requestVote for same term: term=2 +2025-01-25T14:40:39.434Z [INFO] server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7: added peer, starting replication: peer=server-f072e323-9daa-cea8-dc98-4134dc03681b +2025-01-25T14:40:39.434Z [INFO] server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7: added peer, starting replication: peer=server-342ed6b5-d87e-9359-76a4-eaaa0a7f5f99 +2025-01-25T14:40:39.434Z [INFO] server-342ed6b5-d87e-9359-76a4-eaaa0a7f5f99: duplicate requestVote for same term: term=2 +2025-01-25T14:40:39.434Z [INFO] server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7: added peer, starting replication: peer=server-7c77cc1e-0413-9a40-4c07-7966cdc46751 +2025-01-25T14:40:39.434Z [INFO] server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7: added peer, starting replication: peer=server-ea1dbf4a-c6e8-703f-614f-18b4362c5eb6 +2025-01-25T14:40:39.434Z [INFO] server-7c77cc1e-0413-9a40-4c07-7966cdc46751: duplicate requestVote for same term: term=2 +2025-01-25T14:40:39.434Z [INFO] server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7: added peer, starting replication: peer=server-748d830a-3f32-d4f1-e8e1-edc1c01e8190 +2025-01-25T14:40:39.434Z [WARN] server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7: rejecting vote request since we have a leader: from=f072e323-9daa-cea8-dc98-4134dc03681b leader=19d4612b-3d28-6de1-ba6c-88ed4c01b4a7 leader-id=server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7 +2025-01-25T14:40:39.434Z [INFO] server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7: pipelining replication: peer="{Voter server-1c339361-b2de-8273-74be-e9a5da7dde1a 1c339361-b2de-8273-74be-e9a5da7dde1a}" +2025-01-25T14:40:39.435Z [INFO] server-f072e323-9daa-cea8-dc98-4134dc03681b: entering follower state: follower="Node at f072e323-9daa-cea8-dc98-4134dc03681b [Follower]" leader-address=19d4612b-3d28-6de1-ba6c-88ed4c01b4a7 leader-id=server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7 +2025-01-25T14:40:39.435Z [INFO] server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7: pipelining replication: peer="{Voter server-342ed6b5-d87e-9359-76a4-eaaa0a7f5f99 342ed6b5-d87e-9359-76a4-eaaa0a7f5f99}" +2025-01-25T14:40:39.435Z [INFO] server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7: pipelining replication: peer="{Voter server-f072e323-9daa-cea8-dc98-4134dc03681b f072e323-9daa-cea8-dc98-4134dc03681b}" +2025-01-25T14:40:39.435Z [INFO] server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7: pipelining replication: peer="{Voter server-7c77cc1e-0413-9a40-4c07-7966cdc46751 7c77cc1e-0413-9a40-4c07-7966cdc46751}" +2025-01-25T14:40:39.435Z [INFO] server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7: pipelining replication: peer="{Voter server-748d830a-3f32-d4f1-e8e1-edc1c01e8190 748d830a-3f32-d4f1-e8e1-edc1c01e8190}" +2025-01-25T14:40:39.435Z [INFO] server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7: pipelining replication: peer="{Voter server-ea1dbf4a-c6e8-703f-614f-18b4362c5eb6 ea1dbf4a-c6e8-703f-614f-18b4362c5eb6}" +2025-01-25T14:40:39.540Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:39.435300057 +0000 UTC m=+108.481139374 (1 nodes), 52.842778ms from start of poll, 60.180175ms from cluster start. Timeout at 2025-01-25 14:40:39.540314931 +0000 UTC m=+108.586154249, 105.014875ms after stability +2025-01-25T14:40:39.540Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:40:39.646Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:40:39.540757197 +0000 UTC m=+108.586596515 (6 nodes), 8.75µs from start of poll, 165.637316ms from cluster start. Timeout at 2025-01-25 14:40:39.645940947 +0000 UTC m=+108.691779973, 105.183458ms after stability +2025-01-25T14:40:39.646Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:39.753Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:39.646654547 +0000 UTC m=+108.692493865 (1 nodes), 6.125µs from start of poll, 271.534666ms from cluster start. Timeout at 2025-01-25 14:40:39.751893423 +0000 UTC m=+108.797732741, 105.238876ms after stability +2025-01-25T14:40:39.753Z [INFO] server-1c339361-b2de-8273-74be-e9a5da7dde1a: entering candidate state: node="Node at 1c339361-b2de-8273-74be-e9a5da7dde1a [Candidate]" term=3 +2025-01-25T14:40:39.753Z [INFO] server-1c339361-b2de-8273-74be-e9a5da7dde1a: election won: term=3 tally=4 +2025-01-25T14:40:39.753Z [INFO] server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7: entering follower state: follower="Node at 19d4612b-3d28-6de1-ba6c-88ed4c01b4a7 [Follower]" leader-address= leader-id= +2025-01-25T14:40:39.754Z [INFO] server-1c339361-b2de-8273-74be-e9a5da7dde1a: entering leader state: leader="Node at 1c339361-b2de-8273-74be-e9a5da7dde1a [Leader]" +2025-01-25T14:40:39.754Z [INFO] server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7: aborting pipeline replication: peer="{Voter server-f072e323-9daa-cea8-dc98-4134dc03681b f072e323-9daa-cea8-dc98-4134dc03681b}" +2025-01-25T14:40:39.754Z [INFO] server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7: aborting pipeline replication: peer="{Voter server-7c77cc1e-0413-9a40-4c07-7966cdc46751 7c77cc1e-0413-9a40-4c07-7966cdc46751}" +2025-01-25T14:40:39.754Z [INFO] server-1c339361-b2de-8273-74be-e9a5da7dde1a: added peer, starting replication: peer=server-f072e323-9daa-cea8-dc98-4134dc03681b +2025-01-25T14:40:39.754Z [INFO] server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7: aborting pipeline replication: peer="{Voter server-ea1dbf4a-c6e8-703f-614f-18b4362c5eb6 ea1dbf4a-c6e8-703f-614f-18b4362c5eb6}" +2025-01-25T14:40:39.754Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:39.754Z [INFO] server-1c339361-b2de-8273-74be-e9a5da7dde1a: added peer, starting replication: peer=server-342ed6b5-d87e-9359-76a4-eaaa0a7f5f99 +2025-01-25T14:40:39.754Z [INFO] server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7: aborting pipeline replication: peer="{Voter server-748d830a-3f32-d4f1-e8e1-edc1c01e8190 748d830a-3f32-d4f1-e8e1-edc1c01e8190}" +2025-01-25T14:40:39.754Z [INFO] server-1c339361-b2de-8273-74be-e9a5da7dde1a: added peer, starting replication: peer=server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7 +2025-01-25T14:40:39.754Z [INFO] server-1c339361-b2de-8273-74be-e9a5da7dde1a: added peer, starting replication: peer=server-7c77cc1e-0413-9a40-4c07-7966cdc46751 +2025-01-25T14:40:39.754Z [INFO] server-1c339361-b2de-8273-74be-e9a5da7dde1a: added peer, starting replication: peer=server-ea1dbf4a-c6e8-703f-614f-18b4362c5eb6 +2025-01-25T14:40:39.754Z [INFO] server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7: aborting pipeline replication: peer="{Voter server-1c339361-b2de-8273-74be-e9a5da7dde1a 1c339361-b2de-8273-74be-e9a5da7dde1a}" +2025-01-25T14:40:39.754Z [INFO] server-1c339361-b2de-8273-74be-e9a5da7dde1a: added peer, starting replication: peer=server-748d830a-3f32-d4f1-e8e1-edc1c01e8190 +2025-01-25T14:40:39.754Z [INFO] server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7: aborting pipeline replication: peer="{Voter server-342ed6b5-d87e-9359-76a4-eaaa0a7f5f99 342ed6b5-d87e-9359-76a4-eaaa0a7f5f99}" +2025-01-25T14:40:39.754Z [INFO] server-1c339361-b2de-8273-74be-e9a5da7dde1a: pipelining replication: peer="{Voter server-f072e323-9daa-cea8-dc98-4134dc03681b f072e323-9daa-cea8-dc98-4134dc03681b}" +2025-01-25T14:40:39.755Z [INFO] server-1c339361-b2de-8273-74be-e9a5da7dde1a: pipelining replication: peer="{Voter server-342ed6b5-d87e-9359-76a4-eaaa0a7f5f99 342ed6b5-d87e-9359-76a4-eaaa0a7f5f99}" +2025-01-25T14:40:39.755Z [INFO] server-1c339361-b2de-8273-74be-e9a5da7dde1a: pipelining replication: peer="{Voter server-748d830a-3f32-d4f1-e8e1-edc1c01e8190 748d830a-3f32-d4f1-e8e1-edc1c01e8190}" +2025-01-25T14:40:39.755Z [INFO] server-1c339361-b2de-8273-74be-e9a5da7dde1a: pipelining replication: peer="{Voter server-ea1dbf4a-c6e8-703f-614f-18b4362c5eb6 ea1dbf4a-c6e8-703f-614f-18b4362c5eb6}" +2025-01-25T14:40:39.755Z [INFO] server-1c339361-b2de-8273-74be-e9a5da7dde1a: pipelining replication: peer="{Voter server-7c77cc1e-0413-9a40-4c07-7966cdc46751 7c77cc1e-0413-9a40-4c07-7966cdc46751}" +2025-01-25T14:40:39.756Z [INFO] server-1c339361-b2de-8273-74be-e9a5da7dde1a: pipelining replication: peer="{Voter server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7 19d4612b-3d28-6de1-ba6c-88ed4c01b4a7}" +2025-01-25T14:40:39.861Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:39.754942362 +0000 UTC m=+108.800781680 (1 nodes), 546.293µs from start of poll, 379.822481ms from cluster start. Timeout at 2025-01-25 14:40:39.860302572 +0000 UTC m=+108.906141889, 105.360209ms after stability +2025-01-25T14:40:39.861Z [INFO] server-1c339361-b2de-8273-74be-e9a5da7dde1a: aborting pipeline replication: peer="{Voter server-19d4612b-3d28-6de1-ba6c-88ed4c01b4a7 19d4612b-3d28-6de1-ba6c-88ed4c01b4a7}" +2025-01-25T14:40:39.861Z [INFO] server-1c339361-b2de-8273-74be-e9a5da7dde1a: aborting pipeline replication: peer="{Voter server-748d830a-3f32-d4f1-e8e1-edc1c01e8190 748d830a-3f32-d4f1-e8e1-edc1c01e8190}" +2025-01-25T14:40:39.861Z [INFO] server-1c339361-b2de-8273-74be-e9a5da7dde1a: aborting pipeline replication: peer="{Voter server-ea1dbf4a-c6e8-703f-614f-18b4362c5eb6 ea1dbf4a-c6e8-703f-614f-18b4362c5eb6}" +2025-01-25T14:40:39.861Z [INFO] server-1c339361-b2de-8273-74be-e9a5da7dde1a: aborting pipeline replication: peer="{Voter server-f072e323-9daa-cea8-dc98-4134dc03681b f072e323-9daa-cea8-dc98-4134dc03681b}" +2025-01-25T14:40:39.861Z [INFO] server-1c339361-b2de-8273-74be-e9a5da7dde1a: aborting pipeline replication: peer="{Voter server-7c77cc1e-0413-9a40-4c07-7966cdc46751 7c77cc1e-0413-9a40-4c07-7966cdc46751}" +2025-01-25T14:40:39.861Z [INFO] server-1c339361-b2de-8273-74be-e9a5da7dde1a: aborting pipeline replication: peer="{Voter server-342ed6b5-d87e-9359-76a4-eaaa0a7f5f99 342ed6b5-d87e-9359-76a4-eaaa0a7f5f99}" +2025-01-25T14:40:40.361Z [ERROR] server-1c339361-b2de-8273-74be-e9a5da7dde1a: failed to heartbeat to: peer=f072e323-9daa-cea8-dc98-4134dc03681b backoff time=10ms error="command timed out" +--- PASS: TestRaft_LeadershipTransferWithSevenNodes (1.00s) === RUN TestRaft_LeadershipTransferToInvalidID -2025-01-25T14:29:52.630Z [INFO] server-fa0bc42c-ea77-39ac-5ad5-e2f983e94abd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fa0bc42c-ea77-39ac-5ad5-e2f983e94abd Address:fa0bc42c-ea77-39ac-5ad5-e2f983e94abd} {Suffrage:Voter ID:server-cb47b2e9-0a53-82d1-9818-08887336521b Address:cb47b2e9-0a53-82d1-9818-08887336521b} {Suffrage:Voter ID:server-34148589-7b42-c3d5-7425-c8788b1e81d4 Address:34148589-7b42-c3d5-7425-c8788b1e81d4}]" -2025-01-25T14:29:52.630Z [INFO] server-fa0bc42c-ea77-39ac-5ad5-e2f983e94abd: entering follower state: follower="Node at fa0bc42c-ea77-39ac-5ad5-e2f983e94abd [Follower]" leader-address= leader-id= -2025-01-25T14:29:52.630Z [INFO] server-cb47b2e9-0a53-82d1-9818-08887336521b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fa0bc42c-ea77-39ac-5ad5-e2f983e94abd Address:fa0bc42c-ea77-39ac-5ad5-e2f983e94abd} {Suffrage:Voter ID:server-cb47b2e9-0a53-82d1-9818-08887336521b Address:cb47b2e9-0a53-82d1-9818-08887336521b} {Suffrage:Voter ID:server-34148589-7b42-c3d5-7425-c8788b1e81d4 Address:34148589-7b42-c3d5-7425-c8788b1e81d4}]" -2025-01-25T14:29:52.630Z [INFO] server-cb47b2e9-0a53-82d1-9818-08887336521b: entering follower state: follower="Node at cb47b2e9-0a53-82d1-9818-08887336521b [Follower]" leader-address= leader-id= -2025-01-25T14:29:52.630Z [INFO] server-34148589-7b42-c3d5-7425-c8788b1e81d4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fa0bc42c-ea77-39ac-5ad5-e2f983e94abd Address:fa0bc42c-ea77-39ac-5ad5-e2f983e94abd} {Suffrage:Voter ID:server-cb47b2e9-0a53-82d1-9818-08887336521b Address:cb47b2e9-0a53-82d1-9818-08887336521b} {Suffrage:Voter ID:server-34148589-7b42-c3d5-7425-c8788b1e81d4 Address:34148589-7b42-c3d5-7425-c8788b1e81d4}]" -2025-01-25T14:29:52.630Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:52.630Z [INFO] server-34148589-7b42-c3d5-7425-c8788b1e81d4: entering follower state: follower="Node at 34148589-7b42-c3d5-7425-c8788b1e81d4 [Follower]" leader-address= leader-id= -2025-01-25T14:29:52.695Z [WARN] server-cb47b2e9-0a53-82d1-9818-08887336521b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:52.695Z [INFO] server-cb47b2e9-0a53-82d1-9818-08887336521b: entering candidate state: node="Node at cb47b2e9-0a53-82d1-9818-08887336521b [Candidate]" term=2 -2025-01-25T14:29:52.695Z [INFO] server-cb47b2e9-0a53-82d1-9818-08887336521b: election won: term=2 tally=2 -2025-01-25T14:29:52.695Z [INFO] server-cb47b2e9-0a53-82d1-9818-08887336521b: entering leader state: leader="Node at cb47b2e9-0a53-82d1-9818-08887336521b [Leader]" -2025-01-25T14:29:52.695Z [INFO] server-cb47b2e9-0a53-82d1-9818-08887336521b: added peer, starting replication: peer=server-fa0bc42c-ea77-39ac-5ad5-e2f983e94abd -2025-01-25T14:29:52.695Z [INFO] server-cb47b2e9-0a53-82d1-9818-08887336521b: added peer, starting replication: peer=server-34148589-7b42-c3d5-7425-c8788b1e81d4 -2025-01-25T14:29:52.695Z [INFO] server-cb47b2e9-0a53-82d1-9818-08887336521b: pipelining replication: peer="{Voter server-34148589-7b42-c3d5-7425-c8788b1e81d4 34148589-7b42-c3d5-7425-c8788b1e81d4}" -2025-01-25T14:29:52.696Z [INFO] server-cb47b2e9-0a53-82d1-9818-08887336521b: pipelining replication: peer="{Voter server-fa0bc42c-ea77-39ac-5ad5-e2f983e94abd fa0bc42c-ea77-39ac-5ad5-e2f983e94abd}" -2025-01-25T14:29:52.801Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:52.695890886 +0000 UTC m=+102.550215868 (1 nodes), 65.205373ms from start of poll, 65.886989ms from cluster start. Timeout at 2025-01-25 14:29:52.800894406 +0000 UTC m=+102.655219388, 105.00352ms after stability -2025-01-25T14:29:52.801Z [INFO] server-cb47b2e9-0a53-82d1-9818-08887336521b: aborting pipeline replication: peer="{Voter server-34148589-7b42-c3d5-7425-c8788b1e81d4 34148589-7b42-c3d5-7425-c8788b1e81d4}" -2025-01-25T14:29:52.801Z [INFO] server-cb47b2e9-0a53-82d1-9818-08887336521b: aborting pipeline replication: peer="{Voter server-fa0bc42c-ea77-39ac-5ad5-e2f983e94abd fa0bc42c-ea77-39ac-5ad5-e2f983e94abd}" ---- PASS: TestRaft_LeadershipTransferToInvalidID (0.17s) +2025-01-25T14:40:40.374Z [INFO] server-cd348b3d-74e6-6fe1-a4c4-78f3e151c924: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cd348b3d-74e6-6fe1-a4c4-78f3e151c924 Address:cd348b3d-74e6-6fe1-a4c4-78f3e151c924} {Suffrage:Voter ID:server-334cd199-0fe2-24df-8ec7-53233db29112 Address:334cd199-0fe2-24df-8ec7-53233db29112} {Suffrage:Voter ID:server-355d9234-36b2-4a31-f5ff-1c15141e8222 Address:355d9234-36b2-4a31-f5ff-1c15141e8222}]" +2025-01-25T14:40:40.375Z [INFO] server-cd348b3d-74e6-6fe1-a4c4-78f3e151c924: entering follower state: follower="Node at cd348b3d-74e6-6fe1-a4c4-78f3e151c924 [Follower]" leader-address= leader-id= +2025-01-25T14:40:40.375Z [INFO] server-334cd199-0fe2-24df-8ec7-53233db29112: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cd348b3d-74e6-6fe1-a4c4-78f3e151c924 Address:cd348b3d-74e6-6fe1-a4c4-78f3e151c924} {Suffrage:Voter ID:server-334cd199-0fe2-24df-8ec7-53233db29112 Address:334cd199-0fe2-24df-8ec7-53233db29112} {Suffrage:Voter ID:server-355d9234-36b2-4a31-f5ff-1c15141e8222 Address:355d9234-36b2-4a31-f5ff-1c15141e8222}]" +2025-01-25T14:40:40.375Z [INFO] server-334cd199-0fe2-24df-8ec7-53233db29112: entering follower state: follower="Node at 334cd199-0fe2-24df-8ec7-53233db29112 [Follower]" leader-address= leader-id= +2025-01-25T14:40:40.376Z [INFO] server-355d9234-36b2-4a31-f5ff-1c15141e8222: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cd348b3d-74e6-6fe1-a4c4-78f3e151c924 Address:cd348b3d-74e6-6fe1-a4c4-78f3e151c924} {Suffrage:Voter ID:server-334cd199-0fe2-24df-8ec7-53233db29112 Address:334cd199-0fe2-24df-8ec7-53233db29112} {Suffrage:Voter ID:server-355d9234-36b2-4a31-f5ff-1c15141e8222 Address:355d9234-36b2-4a31-f5ff-1c15141e8222}]" +2025-01-25T14:40:40.376Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:40.377Z [INFO] server-355d9234-36b2-4a31-f5ff-1c15141e8222: entering follower state: follower="Node at 355d9234-36b2-4a31-f5ff-1c15141e8222 [Follower]" leader-address= leader-id= +2025-01-25T14:40:40.445Z [WARN] server-334cd199-0fe2-24df-8ec7-53233db29112: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:40.445Z [INFO] server-334cd199-0fe2-24df-8ec7-53233db29112: entering candidate state: node="Node at 334cd199-0fe2-24df-8ec7-53233db29112 [Candidate]" term=2 +2025-01-25T14:40:40.445Z [WARN] server-355d9234-36b2-4a31-f5ff-1c15141e8222: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:40.445Z [INFO] server-355d9234-36b2-4a31-f5ff-1c15141e8222: entering candidate state: node="Node at 355d9234-36b2-4a31-f5ff-1c15141e8222 [Candidate]" term=2 +2025-01-25T14:40:40.446Z [INFO] server-355d9234-36b2-4a31-f5ff-1c15141e8222: duplicate requestVote for same term: term=2 +2025-01-25T14:40:40.446Z [INFO] server-cd348b3d-74e6-6fe1-a4c4-78f3e151c924: duplicate requestVote for same term: term=2 +2025-01-25T14:40:40.446Z [INFO] server-334cd199-0fe2-24df-8ec7-53233db29112: election won: term=2 tally=2 +2025-01-25T14:40:40.446Z [INFO] server-334cd199-0fe2-24df-8ec7-53233db29112: entering leader state: leader="Node at 334cd199-0fe2-24df-8ec7-53233db29112 [Leader]" +2025-01-25T14:40:40.446Z [INFO] server-334cd199-0fe2-24df-8ec7-53233db29112: added peer, starting replication: peer=server-cd348b3d-74e6-6fe1-a4c4-78f3e151c924 +2025-01-25T14:40:40.446Z [INFO] server-334cd199-0fe2-24df-8ec7-53233db29112: added peer, starting replication: peer=server-355d9234-36b2-4a31-f5ff-1c15141e8222 +2025-01-25T14:40:40.446Z [WARN] server-334cd199-0fe2-24df-8ec7-53233db29112: rejecting vote request since we have a leader: from=355d9234-36b2-4a31-f5ff-1c15141e8222 leader=334cd199-0fe2-24df-8ec7-53233db29112 leader-id=server-334cd199-0fe2-24df-8ec7-53233db29112 +2025-01-25T14:40:40.446Z [INFO] server-355d9234-36b2-4a31-f5ff-1c15141e8222: entering follower state: follower="Node at 355d9234-36b2-4a31-f5ff-1c15141e8222 [Follower]" leader-address=334cd199-0fe2-24df-8ec7-53233db29112 leader-id=server-334cd199-0fe2-24df-8ec7-53233db29112 +2025-01-25T14:40:40.447Z [INFO] server-334cd199-0fe2-24df-8ec7-53233db29112: pipelining replication: peer="{Voter server-cd348b3d-74e6-6fe1-a4c4-78f3e151c924 cd348b3d-74e6-6fe1-a4c4-78f3e151c924}" +2025-01-25T14:40:40.447Z [INFO] server-334cd199-0fe2-24df-8ec7-53233db29112: pipelining replication: peer="{Voter server-355d9234-36b2-4a31-f5ff-1c15141e8222 355d9234-36b2-4a31-f5ff-1c15141e8222}" +2025-01-25T14:40:40.552Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:40.44698321 +0000 UTC m=+109.492822528 (1 nodes), 69.974204ms from start of poll, 72.985671ms from cluster start. Timeout at 2025-01-25 14:40:40.552015586 +0000 UTC m=+109.597854903, 105.032375ms after stability +2025-01-25T14:40:40.553Z [INFO] server-334cd199-0fe2-24df-8ec7-53233db29112: aborting pipeline replication: peer="{Voter server-355d9234-36b2-4a31-f5ff-1c15141e8222 355d9234-36b2-4a31-f5ff-1c15141e8222}" +2025-01-25T14:40:40.553Z [INFO] server-334cd199-0fe2-24df-8ec7-53233db29112: aborting pipeline replication: peer="{Voter server-cd348b3d-74e6-6fe1-a4c4-78f3e151c924 cd348b3d-74e6-6fe1-a4c4-78f3e151c924}" +--- PASS: TestRaft_LeadershipTransferToInvalidID (0.19s) === RUN TestRaft_LeadershipTransferToInvalidAddress -2025-01-25T14:29:52.804Z [INFO] server-43e6afaa-ba92-b4a6-edc1-c76bc05ffb2a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-43e6afaa-ba92-b4a6-edc1-c76bc05ffb2a Address:43e6afaa-ba92-b4a6-edc1-c76bc05ffb2a} {Suffrage:Voter ID:server-275a0f32-4be4-3289-2502-faa6a76e0355 Address:275a0f32-4be4-3289-2502-faa6a76e0355} {Suffrage:Voter ID:server-07525e83-5e99-69ea-b3e6-2bff0c91ebe6 Address:07525e83-5e99-69ea-b3e6-2bff0c91ebe6}]" -2025-01-25T14:29:52.804Z [INFO] server-275a0f32-4be4-3289-2502-faa6a76e0355: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-43e6afaa-ba92-b4a6-edc1-c76bc05ffb2a Address:43e6afaa-ba92-b4a6-edc1-c76bc05ffb2a} {Suffrage:Voter ID:server-275a0f32-4be4-3289-2502-faa6a76e0355 Address:275a0f32-4be4-3289-2502-faa6a76e0355} {Suffrage:Voter ID:server-07525e83-5e99-69ea-b3e6-2bff0c91ebe6 Address:07525e83-5e99-69ea-b3e6-2bff0c91ebe6}]" -2025-01-25T14:29:52.804Z [INFO] server-07525e83-5e99-69ea-b3e6-2bff0c91ebe6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-43e6afaa-ba92-b4a6-edc1-c76bc05ffb2a Address:43e6afaa-ba92-b4a6-edc1-c76bc05ffb2a} {Suffrage:Voter ID:server-275a0f32-4be4-3289-2502-faa6a76e0355 Address:275a0f32-4be4-3289-2502-faa6a76e0355} {Suffrage:Voter ID:server-07525e83-5e99-69ea-b3e6-2bff0c91ebe6 Address:07525e83-5e99-69ea-b3e6-2bff0c91ebe6}]" -2025-01-25T14:29:52.804Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:29:52.804Z [INFO] server-43e6afaa-ba92-b4a6-edc1-c76bc05ffb2a: entering follower state: follower="Node at 43e6afaa-ba92-b4a6-edc1-c76bc05ffb2a [Follower]" leader-address= leader-id= -2025-01-25T14:29:52.805Z [INFO] server-275a0f32-4be4-3289-2502-faa6a76e0355: entering follower state: follower="Node at 275a0f32-4be4-3289-2502-faa6a76e0355 [Follower]" leader-address= leader-id= -2025-01-25T14:29:52.805Z [INFO] server-07525e83-5e99-69ea-b3e6-2bff0c91ebe6: entering follower state: follower="Node at 07525e83-5e99-69ea-b3e6-2bff0c91ebe6 [Follower]" leader-address= leader-id= -2025-01-25T14:29:52.894Z [WARN] server-275a0f32-4be4-3289-2502-faa6a76e0355: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:52.894Z [INFO] server-275a0f32-4be4-3289-2502-faa6a76e0355: entering candidate state: node="Node at 275a0f32-4be4-3289-2502-faa6a76e0355 [Candidate]" term=2 -2025-01-25T14:29:52.894Z [INFO] server-275a0f32-4be4-3289-2502-faa6a76e0355: election won: term=2 tally=2 -2025-01-25T14:29:52.894Z [INFO] server-275a0f32-4be4-3289-2502-faa6a76e0355: entering leader state: leader="Node at 275a0f32-4be4-3289-2502-faa6a76e0355 [Leader]" -2025-01-25T14:29:52.894Z [INFO] server-275a0f32-4be4-3289-2502-faa6a76e0355: added peer, starting replication: peer=server-43e6afaa-ba92-b4a6-edc1-c76bc05ffb2a -2025-01-25T14:29:52.894Z [INFO] server-275a0f32-4be4-3289-2502-faa6a76e0355: added peer, starting replication: peer=server-07525e83-5e99-69ea-b3e6-2bff0c91ebe6 -2025-01-25T14:29:52.894Z [INFO] server-275a0f32-4be4-3289-2502-faa6a76e0355: pipelining replication: peer="{Voter server-07525e83-5e99-69ea-b3e6-2bff0c91ebe6 07525e83-5e99-69ea-b3e6-2bff0c91ebe6}" -2025-01-25T14:29:52.895Z [WARN] server-43e6afaa-ba92-b4a6-edc1-c76bc05ffb2a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:52.895Z [INFO] server-43e6afaa-ba92-b4a6-edc1-c76bc05ffb2a: entering candidate state: node="Node at 43e6afaa-ba92-b4a6-edc1-c76bc05ffb2a [Candidate]" term=2 -2025-01-25T14:29:52.895Z [INFO] server-43e6afaa-ba92-b4a6-edc1-c76bc05ffb2a: duplicate requestVote for same term: term=2 -2025-01-25T14:29:52.895Z [WARN] server-275a0f32-4be4-3289-2502-faa6a76e0355: rejecting vote request since we have a leader: from=43e6afaa-ba92-b4a6-edc1-c76bc05ffb2a leader=275a0f32-4be4-3289-2502-faa6a76e0355 leader-id=server-275a0f32-4be4-3289-2502-faa6a76e0355 -2025-01-25T14:29:52.895Z [INFO] server-43e6afaa-ba92-b4a6-edc1-c76bc05ffb2a: entering follower state: follower="Node at 43e6afaa-ba92-b4a6-edc1-c76bc05ffb2a [Follower]" leader-address=275a0f32-4be4-3289-2502-faa6a76e0355 leader-id=server-275a0f32-4be4-3289-2502-faa6a76e0355 -2025-01-25T14:29:52.895Z [WARN] server-07525e83-5e99-69ea-b3e6-2bff0c91ebe6: rejecting vote request since we have a leader: from=43e6afaa-ba92-b4a6-edc1-c76bc05ffb2a leader=275a0f32-4be4-3289-2502-faa6a76e0355 leader-id=server-275a0f32-4be4-3289-2502-faa6a76e0355 -2025-01-25T14:29:52.895Z [INFO] server-275a0f32-4be4-3289-2502-faa6a76e0355: pipelining replication: peer="{Voter server-43e6afaa-ba92-b4a6-edc1-c76bc05ffb2a 43e6afaa-ba92-b4a6-edc1-c76bc05ffb2a}" -2025-01-25T14:29:53.001Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:29:52.895814091 +0000 UTC m=+102.750139073 (2 nodes), 90.935367ms from start of poll, 91.629462ms from cluster start. Timeout at 2025-01-25 14:29:53.000840891 +0000 UTC m=+102.855165833, 105.02676ms after stability -2025-01-25T14:29:53.001Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:53.109Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:53.001128741 +0000 UTC m=+102.855453723 (1 nodes), 1.76µs from start of poll, 196.944112ms from cluster start. Timeout at 2025-01-25 14:29:53.106382092 +0000 UTC m=+102.960707194, 105.253471ms after stability -2025-01-25T14:29:53.109Z [INFO] server-275a0f32-4be4-3289-2502-faa6a76e0355: aborting pipeline replication: peer="{Voter server-43e6afaa-ba92-b4a6-edc1-c76bc05ffb2a 43e6afaa-ba92-b4a6-edc1-c76bc05ffb2a}" -2025-01-25T14:29:53.109Z [INFO] server-275a0f32-4be4-3289-2502-faa6a76e0355: aborting pipeline replication: peer="{Voter server-07525e83-5e99-69ea-b3e6-2bff0c91ebe6 07525e83-5e99-69ea-b3e6-2bff0c91ebe6}" ---- PASS: TestRaft_LeadershipTransferToInvalidAddress (0.31s) +2025-01-25T14:40:40.564Z [INFO] server-0c3fbbe6-5de3-8159-b381-ec17cd6d6b04: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0c3fbbe6-5de3-8159-b381-ec17cd6d6b04 Address:0c3fbbe6-5de3-8159-b381-ec17cd6d6b04} {Suffrage:Voter ID:server-1f0912e4-b87a-4084-5a82-f3195ca48f1a Address:1f0912e4-b87a-4084-5a82-f3195ca48f1a} {Suffrage:Voter ID:server-8b984e73-7726-55e7-9ce9-628655f32659 Address:8b984e73-7726-55e7-9ce9-628655f32659}]" +2025-01-25T14:40:40.565Z [INFO] server-0c3fbbe6-5de3-8159-b381-ec17cd6d6b04: entering follower state: follower="Node at 0c3fbbe6-5de3-8159-b381-ec17cd6d6b04 [Follower]" leader-address= leader-id= +2025-01-25T14:40:40.565Z [INFO] server-1f0912e4-b87a-4084-5a82-f3195ca48f1a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0c3fbbe6-5de3-8159-b381-ec17cd6d6b04 Address:0c3fbbe6-5de3-8159-b381-ec17cd6d6b04} {Suffrage:Voter ID:server-1f0912e4-b87a-4084-5a82-f3195ca48f1a Address:1f0912e4-b87a-4084-5a82-f3195ca48f1a} {Suffrage:Voter ID:server-8b984e73-7726-55e7-9ce9-628655f32659 Address:8b984e73-7726-55e7-9ce9-628655f32659}]" +2025-01-25T14:40:40.566Z [INFO] server-1f0912e4-b87a-4084-5a82-f3195ca48f1a: entering follower state: follower="Node at 1f0912e4-b87a-4084-5a82-f3195ca48f1a [Follower]" leader-address= leader-id= +2025-01-25T14:40:40.567Z [INFO] server-8b984e73-7726-55e7-9ce9-628655f32659: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0c3fbbe6-5de3-8159-b381-ec17cd6d6b04 Address:0c3fbbe6-5de3-8159-b381-ec17cd6d6b04} {Suffrage:Voter ID:server-1f0912e4-b87a-4084-5a82-f3195ca48f1a Address:1f0912e4-b87a-4084-5a82-f3195ca48f1a} {Suffrage:Voter ID:server-8b984e73-7726-55e7-9ce9-628655f32659 Address:8b984e73-7726-55e7-9ce9-628655f32659}]" +2025-01-25T14:40:40.567Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:40:40.567Z [INFO] server-8b984e73-7726-55e7-9ce9-628655f32659: entering follower state: follower="Node at 8b984e73-7726-55e7-9ce9-628655f32659 [Follower]" leader-address= leader-id= +2025-01-25T14:40:40.634Z [WARN] server-8b984e73-7726-55e7-9ce9-628655f32659: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:40.634Z [INFO] server-8b984e73-7726-55e7-9ce9-628655f32659: entering candidate state: node="Node at 8b984e73-7726-55e7-9ce9-628655f32659 [Candidate]" term=2 +2025-01-25T14:40:40.635Z [INFO] server-8b984e73-7726-55e7-9ce9-628655f32659: election won: term=2 tally=2 +2025-01-25T14:40:40.635Z [INFO] server-8b984e73-7726-55e7-9ce9-628655f32659: entering leader state: leader="Node at 8b984e73-7726-55e7-9ce9-628655f32659 [Leader]" +2025-01-25T14:40:40.635Z [INFO] server-8b984e73-7726-55e7-9ce9-628655f32659: added peer, starting replication: peer=server-0c3fbbe6-5de3-8159-b381-ec17cd6d6b04 +2025-01-25T14:40:40.635Z [INFO] server-8b984e73-7726-55e7-9ce9-628655f32659: added peer, starting replication: peer=server-1f0912e4-b87a-4084-5a82-f3195ca48f1a +2025-01-25T14:40:40.636Z [INFO] server-8b984e73-7726-55e7-9ce9-628655f32659: pipelining replication: peer="{Voter server-0c3fbbe6-5de3-8159-b381-ec17cd6d6b04 0c3fbbe6-5de3-8159-b381-ec17cd6d6b04}" +2025-01-25T14:40:40.636Z [INFO] server-8b984e73-7726-55e7-9ce9-628655f32659: pipelining replication: peer="{Voter server-1f0912e4-b87a-4084-5a82-f3195ca48f1a 1f0912e4-b87a-4084-5a82-f3195ca48f1a}" +2025-01-25T14:40:40.742Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:40:40.635418967 +0000 UTC m=+109.681258285 (2 nodes), 68.11049ms from start of poll, 71.5755ms from cluster start. Timeout at 2025-01-25 14:40:40.740430051 +0000 UTC m=+109.786269369, 105.011084ms after stability +2025-01-25T14:40:40.742Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:40.848Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:40.742513446 +0000 UTC m=+109.788352763 (1 nodes), 5.541µs from start of poll, 178.669978ms from cluster start. Timeout at 2025-01-25 14:40:40.847718196 +0000 UTC m=+109.893557805, 105.205042ms after stability +2025-01-25T14:40:40.848Z [INFO] server-8b984e73-7726-55e7-9ce9-628655f32659: aborting pipeline replication: peer="{Voter server-0c3fbbe6-5de3-8159-b381-ec17cd6d6b04 0c3fbbe6-5de3-8159-b381-ec17cd6d6b04}" +2025-01-25T14:40:40.848Z [INFO] server-8b984e73-7726-55e7-9ce9-628655f32659: aborting pipeline replication: peer="{Voter server-1f0912e4-b87a-4084-5a82-f3195ca48f1a 1f0912e4-b87a-4084-5a82-f3195ca48f1a}" +--- PASS: TestRaft_LeadershipTransferToInvalidAddress (0.29s) === RUN TestRaft_LeadershipTransferToBehindServer -2025-01-25T14:29:53.112Z [INFO] server-3f106e1c-4394-bec0-f499-d1b4ddf4bd1d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3f106e1c-4394-bec0-f499-d1b4ddf4bd1d Address:3f106e1c-4394-bec0-f499-d1b4ddf4bd1d} {Suffrage:Voter ID:server-67e445b5-76a8-7f31-47ae-597854d04e3d Address:67e445b5-76a8-7f31-47ae-597854d04e3d} {Suffrage:Voter ID:server-b0e3192f-d65a-3d81-283a-fef84d9d77b4 Address:b0e3192f-d65a-3d81-283a-fef84d9d77b4}]" -2025-01-25T14:29:53.112Z [INFO] server-67e445b5-76a8-7f31-47ae-597854d04e3d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3f106e1c-4394-bec0-f499-d1b4ddf4bd1d Address:3f106e1c-4394-bec0-f499-d1b4ddf4bd1d} {Suffrage:Voter ID:server-67e445b5-76a8-7f31-47ae-597854d04e3d Address:67e445b5-76a8-7f31-47ae-597854d04e3d} {Suffrage:Voter ID:server-b0e3192f-d65a-3d81-283a-fef84d9d77b4 Address:b0e3192f-d65a-3d81-283a-fef84d9d77b4}]" -2025-01-25T14:29:53.113Z [INFO] server-67e445b5-76a8-7f31-47ae-597854d04e3d: entering follower state: follower="Node at 67e445b5-76a8-7f31-47ae-597854d04e3d [Follower]" leader-address= leader-id= -2025-01-25T14:29:53.112Z [INFO] server-3f106e1c-4394-bec0-f499-d1b4ddf4bd1d: entering follower state: follower="Node at 3f106e1c-4394-bec0-f499-d1b4ddf4bd1d [Follower]" leader-address= leader-id= -2025-01-25T14:29:53.113Z [INFO] server-b0e3192f-d65a-3d81-283a-fef84d9d77b4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3f106e1c-4394-bec0-f499-d1b4ddf4bd1d Address:3f106e1c-4394-bec0-f499-d1b4ddf4bd1d} {Suffrage:Voter ID:server-67e445b5-76a8-7f31-47ae-597854d04e3d Address:67e445b5-76a8-7f31-47ae-597854d04e3d} {Suffrage:Voter ID:server-b0e3192f-d65a-3d81-283a-fef84d9d77b4 Address:b0e3192f-d65a-3d81-283a-fef84d9d77b4}]" -2025-01-25T14:29:53.113Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:53.113Z [INFO] server-b0e3192f-d65a-3d81-283a-fef84d9d77b4: entering follower state: follower="Node at b0e3192f-d65a-3d81-283a-fef84d9d77b4 [Follower]" leader-address= leader-id= -2025-01-25T14:29:53.169Z [WARN] server-b0e3192f-d65a-3d81-283a-fef84d9d77b4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:53.170Z [INFO] server-b0e3192f-d65a-3d81-283a-fef84d9d77b4: entering candidate state: node="Node at b0e3192f-d65a-3d81-283a-fef84d9d77b4 [Candidate]" term=2 -2025-01-25T14:29:53.170Z [INFO] server-b0e3192f-d65a-3d81-283a-fef84d9d77b4: election won: term=2 tally=2 -2025-01-25T14:29:53.170Z [INFO] server-b0e3192f-d65a-3d81-283a-fef84d9d77b4: entering leader state: leader="Node at b0e3192f-d65a-3d81-283a-fef84d9d77b4 [Leader]" -2025-01-25T14:29:53.170Z [INFO] server-b0e3192f-d65a-3d81-283a-fef84d9d77b4: added peer, starting replication: peer=server-3f106e1c-4394-bec0-f499-d1b4ddf4bd1d -2025-01-25T14:29:53.170Z [INFO] server-b0e3192f-d65a-3d81-283a-fef84d9d77b4: added peer, starting replication: peer=server-67e445b5-76a8-7f31-47ae-597854d04e3d -2025-01-25T14:29:53.171Z [INFO] server-b0e3192f-d65a-3d81-283a-fef84d9d77b4: pipelining replication: peer="{Voter server-67e445b5-76a8-7f31-47ae-597854d04e3d 67e445b5-76a8-7f31-47ae-597854d04e3d}" -2025-01-25T14:29:53.171Z [INFO] server-b0e3192f-d65a-3d81-283a-fef84d9d77b4: pipelining replication: peer="{Voter server-3f106e1c-4394-bec0-f499-d1b4ddf4bd1d 3f106e1c-4394-bec0-f499-d1b4ddf4bd1d}" -2025-01-25T14:29:53.276Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:53.171477051 +0000 UTC m=+103.025802033 (1 nodes), 58.192946ms from start of poll, 58.981518ms from cluster start. Timeout at 2025-01-25 14:29:53.276479371 +0000 UTC m=+103.130804353, 105.00232ms after stability -2025-01-25T14:29:53.276Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:29:53.382Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:29:53.276642106 +0000 UTC m=+103.130967088 (2 nodes), 1.68µs from start of poll, 164.146573ms from cluster start. Timeout at 2025-01-25 14:29:53.381924256 +0000 UTC m=+103.236249198, 105.28211ms after stability -2025-01-25T14:29:53.534Z [INFO] server-3f106e1c-4394-bec0-f499-d1b4ddf4bd1d: entering candidate state: node="Node at 3f106e1c-4394-bec0-f499-d1b4ddf4bd1d [Candidate]" term=3 -2025-01-25T14:29:53.534Z [INFO] server-b0e3192f-d65a-3d81-283a-fef84d9d77b4: entering follower state: follower="Node at b0e3192f-d65a-3d81-283a-fef84d9d77b4 [Follower]" leader-address= leader-id= -2025-01-25T14:29:53.534Z [INFO] server-b0e3192f-d65a-3d81-283a-fef84d9d77b4: aborting pipeline replication: peer="{Voter server-67e445b5-76a8-7f31-47ae-597854d04e3d 67e445b5-76a8-7f31-47ae-597854d04e3d}" -2025-01-25T14:29:53.534Z [INFO] server-3f106e1c-4394-bec0-f499-d1b4ddf4bd1d: election won: term=3 tally=2 -2025-01-25T14:29:53.534Z [INFO] server-3f106e1c-4394-bec0-f499-d1b4ddf4bd1d: entering leader state: leader="Node at 3f106e1c-4394-bec0-f499-d1b4ddf4bd1d [Leader]" -2025-01-25T14:29:53.535Z [INFO] server-3f106e1c-4394-bec0-f499-d1b4ddf4bd1d: added peer, starting replication: peer=server-67e445b5-76a8-7f31-47ae-597854d04e3d -2025-01-25T14:29:53.535Z [INFO] server-3f106e1c-4394-bec0-f499-d1b4ddf4bd1d: added peer, starting replication: peer=server-b0e3192f-d65a-3d81-283a-fef84d9d77b4 -2025-01-25T14:29:53.535Z [INFO] server-b0e3192f-d65a-3d81-283a-fef84d9d77b4: aborting pipeline replication: peer="{Voter server-3f106e1c-4394-bec0-f499-d1b4ddf4bd1d 3f106e1c-4394-bec0-f499-d1b4ddf4bd1d}" -2025-01-25T14:29:53.535Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:53.535Z [INFO] server-3f106e1c-4394-bec0-f499-d1b4ddf4bd1d: pipelining replication: peer="{Voter server-67e445b5-76a8-7f31-47ae-597854d04e3d 67e445b5-76a8-7f31-47ae-597854d04e3d}" -2025-01-25T14:29:53.535Z [INFO] server-3f106e1c-4394-bec0-f499-d1b4ddf4bd1d: pipelining replication: peer="{Voter server-b0e3192f-d65a-3d81-283a-fef84d9d77b4 b0e3192f-d65a-3d81-283a-fef84d9d77b4}" -2025-01-25T14:29:53.640Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:53.535462672 +0000 UTC m=+103.389787654 (1 nodes), 54.598µs from start of poll, 422.967139ms from cluster start. Timeout at 2025-01-25 14:29:53.640637626 +0000 UTC m=+103.494962768, 105.175114ms after stability -2025-01-25T14:29:53.640Z [INFO] server-3f106e1c-4394-bec0-f499-d1b4ddf4bd1d: aborting pipeline replication: peer="{Voter server-b0e3192f-d65a-3d81-283a-fef84d9d77b4 b0e3192f-d65a-3d81-283a-fef84d9d77b4}" -2025-01-25T14:29:53.640Z [INFO] server-3f106e1c-4394-bec0-f499-d1b4ddf4bd1d: aborting pipeline replication: peer="{Voter server-67e445b5-76a8-7f31-47ae-597854d04e3d 67e445b5-76a8-7f31-47ae-597854d04e3d}" ---- PASS: TestRaft_LeadershipTransferToBehindServer (0.53s) +2025-01-25T14:40:40.857Z [INFO] server-b85d3c99-94dc-a9b6-21c6-cb42ee7fcf30: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b85d3c99-94dc-a9b6-21c6-cb42ee7fcf30 Address:b85d3c99-94dc-a9b6-21c6-cb42ee7fcf30} {Suffrage:Voter ID:server-f6e48ff7-8015-e566-9807-7788269d15bc Address:f6e48ff7-8015-e566-9807-7788269d15bc} {Suffrage:Voter ID:server-8d7b00f1-1dd3-e41f-cbb9-95b606dce620 Address:8d7b00f1-1dd3-e41f-cbb9-95b606dce620}]" +2025-01-25T14:40:40.857Z [INFO] server-b85d3c99-94dc-a9b6-21c6-cb42ee7fcf30: entering follower state: follower="Node at b85d3c99-94dc-a9b6-21c6-cb42ee7fcf30 [Follower]" leader-address= leader-id= +2025-01-25T14:40:40.858Z [INFO] server-f6e48ff7-8015-e566-9807-7788269d15bc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b85d3c99-94dc-a9b6-21c6-cb42ee7fcf30 Address:b85d3c99-94dc-a9b6-21c6-cb42ee7fcf30} {Suffrage:Voter ID:server-f6e48ff7-8015-e566-9807-7788269d15bc Address:f6e48ff7-8015-e566-9807-7788269d15bc} {Suffrage:Voter ID:server-8d7b00f1-1dd3-e41f-cbb9-95b606dce620 Address:8d7b00f1-1dd3-e41f-cbb9-95b606dce620}]" +2025-01-25T14:40:40.858Z [INFO] server-f6e48ff7-8015-e566-9807-7788269d15bc: entering follower state: follower="Node at f6e48ff7-8015-e566-9807-7788269d15bc [Follower]" leader-address= leader-id= +2025-01-25T14:40:40.858Z [INFO] server-8d7b00f1-1dd3-e41f-cbb9-95b606dce620: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b85d3c99-94dc-a9b6-21c6-cb42ee7fcf30 Address:b85d3c99-94dc-a9b6-21c6-cb42ee7fcf30} {Suffrage:Voter ID:server-f6e48ff7-8015-e566-9807-7788269d15bc Address:f6e48ff7-8015-e566-9807-7788269d15bc} {Suffrage:Voter ID:server-8d7b00f1-1dd3-e41f-cbb9-95b606dce620 Address:8d7b00f1-1dd3-e41f-cbb9-95b606dce620}]" +2025-01-25T14:40:40.859Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:40.859Z [INFO] server-8d7b00f1-1dd3-e41f-cbb9-95b606dce620: entering follower state: follower="Node at 8d7b00f1-1dd3-e41f-cbb9-95b606dce620 [Follower]" leader-address= leader-id= +2025-01-25T14:40:40.914Z [WARN] server-8d7b00f1-1dd3-e41f-cbb9-95b606dce620: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:40.914Z [INFO] server-8d7b00f1-1dd3-e41f-cbb9-95b606dce620: entering candidate state: node="Node at 8d7b00f1-1dd3-e41f-cbb9-95b606dce620 [Candidate]" term=2 +2025-01-25T14:40:40.915Z [INFO] server-8d7b00f1-1dd3-e41f-cbb9-95b606dce620: election won: term=2 tally=2 +2025-01-25T14:40:40.915Z [INFO] server-8d7b00f1-1dd3-e41f-cbb9-95b606dce620: entering leader state: leader="Node at 8d7b00f1-1dd3-e41f-cbb9-95b606dce620 [Leader]" +2025-01-25T14:40:40.915Z [INFO] server-8d7b00f1-1dd3-e41f-cbb9-95b606dce620: added peer, starting replication: peer=server-b85d3c99-94dc-a9b6-21c6-cb42ee7fcf30 +2025-01-25T14:40:40.915Z [INFO] server-8d7b00f1-1dd3-e41f-cbb9-95b606dce620: added peer, starting replication: peer=server-f6e48ff7-8015-e566-9807-7788269d15bc +2025-01-25T14:40:40.916Z [INFO] server-8d7b00f1-1dd3-e41f-cbb9-95b606dce620: pipelining replication: peer="{Voter server-b85d3c99-94dc-a9b6-21c6-cb42ee7fcf30 b85d3c99-94dc-a9b6-21c6-cb42ee7fcf30}" +2025-01-25T14:40:40.916Z [INFO] server-8d7b00f1-1dd3-e41f-cbb9-95b606dce620: pipelining replication: peer="{Voter server-f6e48ff7-8015-e566-9807-7788269d15bc f6e48ff7-8015-e566-9807-7788269d15bc}" +2025-01-25T14:40:41.020Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:40.915677616 +0000 UTC m=+109.961516934 (1 nodes), 56.500664ms from start of poll, 59.005505ms from cluster start. Timeout at 2025-01-25 14:40:41.020687242 +0000 UTC m=+110.066526559, 105.009625ms after stability +2025-01-25T14:40:41.021Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:40:41.126Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:40:41.021163549 +0000 UTC m=+110.067002866 (2 nodes), 7.291µs from start of poll, 164.491437ms from cluster start. Timeout at 2025-01-25 14:40:41.126387257 +0000 UTC m=+110.172226575, 105.223709ms after stability +2025-01-25T14:40:41.516Z [INFO] server-b85d3c99-94dc-a9b6-21c6-cb42ee7fcf30: entering candidate state: node="Node at b85d3c99-94dc-a9b6-21c6-cb42ee7fcf30 [Candidate]" term=3 +2025-01-25T14:40:41.516Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:41.517Z [INFO] server-8d7b00f1-1dd3-e41f-cbb9-95b606dce620: entering follower state: follower="Node at 8d7b00f1-1dd3-e41f-cbb9-95b606dce620 [Follower]" leader-address= leader-id= +2025-01-25T14:40:41.517Z [INFO] server-b85d3c99-94dc-a9b6-21c6-cb42ee7fcf30: election won: term=3 tally=2 +2025-01-25T14:40:41.517Z [INFO] server-8d7b00f1-1dd3-e41f-cbb9-95b606dce620: aborting pipeline replication: peer="{Voter server-f6e48ff7-8015-e566-9807-7788269d15bc f6e48ff7-8015-e566-9807-7788269d15bc}" +2025-01-25T14:40:41.517Z [INFO] server-b85d3c99-94dc-a9b6-21c6-cb42ee7fcf30: entering leader state: leader="Node at b85d3c99-94dc-a9b6-21c6-cb42ee7fcf30 [Leader]" +2025-01-25T14:40:41.517Z [INFO] server-b85d3c99-94dc-a9b6-21c6-cb42ee7fcf30: added peer, starting replication: peer=server-f6e48ff7-8015-e566-9807-7788269d15bc +2025-01-25T14:40:41.517Z [INFO] server-8d7b00f1-1dd3-e41f-cbb9-95b606dce620: aborting pipeline replication: peer="{Voter server-b85d3c99-94dc-a9b6-21c6-cb42ee7fcf30 b85d3c99-94dc-a9b6-21c6-cb42ee7fcf30}" +2025-01-25T14:40:41.517Z [INFO] server-b85d3c99-94dc-a9b6-21c6-cb42ee7fcf30: added peer, starting replication: peer=server-8d7b00f1-1dd3-e41f-cbb9-95b606dce620 +2025-01-25T14:40:41.518Z [INFO] server-b85d3c99-94dc-a9b6-21c6-cb42ee7fcf30: pipelining replication: peer="{Voter server-f6e48ff7-8015-e566-9807-7788269d15bc f6e48ff7-8015-e566-9807-7788269d15bc}" +2025-01-25T14:40:41.518Z [INFO] server-b85d3c99-94dc-a9b6-21c6-cb42ee7fcf30: pipelining replication: peer="{Voter server-8d7b00f1-1dd3-e41f-cbb9-95b606dce620 8d7b00f1-1dd3-e41f-cbb9-95b606dce620}" +2025-01-25T14:40:41.625Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:41.517515452 +0000 UTC m=+110.563354770 (1 nodes), 672.419µs from start of poll, 660.843341ms from cluster start. Timeout at 2025-01-25 14:40:41.622945079 +0000 UTC m=+110.668784397, 105.429627ms after stability +2025-01-25T14:40:41.625Z [INFO] server-b85d3c99-94dc-a9b6-21c6-cb42ee7fcf30: aborting pipeline replication: peer="{Voter server-f6e48ff7-8015-e566-9807-7788269d15bc f6e48ff7-8015-e566-9807-7788269d15bc}" +2025-01-25T14:40:41.625Z [INFO] server-b85d3c99-94dc-a9b6-21c6-cb42ee7fcf30: aborting pipeline replication: peer="{Voter server-8d7b00f1-1dd3-e41f-cbb9-95b606dce620 8d7b00f1-1dd3-e41f-cbb9-95b606dce620}" +--- PASS: TestRaft_LeadershipTransferToBehindServer (0.78s) === RUN TestRaft_LeadershipTransferToItself -2025-01-25T14:29:53.644Z [INFO] server-229aa372-85fe-e732-e6ba-ebb33cc9ed7f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-229aa372-85fe-e732-e6ba-ebb33cc9ed7f Address:229aa372-85fe-e732-e6ba-ebb33cc9ed7f} {Suffrage:Voter ID:server-b07a38de-155a-2936-1c63-8fa8864b9eb5 Address:b07a38de-155a-2936-1c63-8fa8864b9eb5} {Suffrage:Voter ID:server-e225ea80-f715-4bd1-9db0-6ff70c578a05 Address:e225ea80-f715-4bd1-9db0-6ff70c578a05}]" -2025-01-25T14:29:53.645Z [INFO] server-b07a38de-155a-2936-1c63-8fa8864b9eb5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-229aa372-85fe-e732-e6ba-ebb33cc9ed7f Address:229aa372-85fe-e732-e6ba-ebb33cc9ed7f} {Suffrage:Voter ID:server-b07a38de-155a-2936-1c63-8fa8864b9eb5 Address:b07a38de-155a-2936-1c63-8fa8864b9eb5} {Suffrage:Voter ID:server-e225ea80-f715-4bd1-9db0-6ff70c578a05 Address:e225ea80-f715-4bd1-9db0-6ff70c578a05}]" -2025-01-25T14:29:53.645Z [INFO] server-e225ea80-f715-4bd1-9db0-6ff70c578a05: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-229aa372-85fe-e732-e6ba-ebb33cc9ed7f Address:229aa372-85fe-e732-e6ba-ebb33cc9ed7f} {Suffrage:Voter ID:server-b07a38de-155a-2936-1c63-8fa8864b9eb5 Address:b07a38de-155a-2936-1c63-8fa8864b9eb5} {Suffrage:Voter ID:server-e225ea80-f715-4bd1-9db0-6ff70c578a05 Address:e225ea80-f715-4bd1-9db0-6ff70c578a05}]" -2025-01-25T14:29:53.645Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:53.645Z [INFO] server-229aa372-85fe-e732-e6ba-ebb33cc9ed7f: entering follower state: follower="Node at 229aa372-85fe-e732-e6ba-ebb33cc9ed7f [Follower]" leader-address= leader-id= -2025-01-25T14:29:53.645Z [INFO] server-b07a38de-155a-2936-1c63-8fa8864b9eb5: entering follower state: follower="Node at b07a38de-155a-2936-1c63-8fa8864b9eb5 [Follower]" leader-address= leader-id= -2025-01-25T14:29:53.653Z [INFO] server-e225ea80-f715-4bd1-9db0-6ff70c578a05: entering follower state: follower="Node at e225ea80-f715-4bd1-9db0-6ff70c578a05 [Follower]" leader-address= leader-id= -2025-01-25T14:29:53.710Z [WARN] server-229aa372-85fe-e732-e6ba-ebb33cc9ed7f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:53.711Z [INFO] server-229aa372-85fe-e732-e6ba-ebb33cc9ed7f: entering candidate state: node="Node at 229aa372-85fe-e732-e6ba-ebb33cc9ed7f [Candidate]" term=2 -2025-01-25T14:29:53.711Z [INFO] server-229aa372-85fe-e732-e6ba-ebb33cc9ed7f: election won: term=2 tally=2 -2025-01-25T14:29:53.711Z [INFO] server-229aa372-85fe-e732-e6ba-ebb33cc9ed7f: entering leader state: leader="Node at 229aa372-85fe-e732-e6ba-ebb33cc9ed7f [Leader]" -2025-01-25T14:29:53.711Z [INFO] server-229aa372-85fe-e732-e6ba-ebb33cc9ed7f: added peer, starting replication: peer=server-b07a38de-155a-2936-1c63-8fa8864b9eb5 -2025-01-25T14:29:53.711Z [INFO] server-229aa372-85fe-e732-e6ba-ebb33cc9ed7f: added peer, starting replication: peer=server-e225ea80-f715-4bd1-9db0-6ff70c578a05 -2025-01-25T14:29:53.711Z [INFO] server-229aa372-85fe-e732-e6ba-ebb33cc9ed7f: pipelining replication: peer="{Voter server-e225ea80-f715-4bd1-9db0-6ff70c578a05 e225ea80-f715-4bd1-9db0-6ff70c578a05}" -2025-01-25T14:29:53.712Z [INFO] server-229aa372-85fe-e732-e6ba-ebb33cc9ed7f: pipelining replication: peer="{Voter server-b07a38de-155a-2936-1c63-8fa8864b9eb5 b07a38de-155a-2936-1c63-8fa8864b9eb5}" -2025-01-25T14:29:53.817Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:53.712147034 +0000 UTC m=+103.566472015 (1 nodes), 66.626442ms from start of poll, 67.421453ms from cluster start. Timeout at 2025-01-25 14:29:53.817167592 +0000 UTC m=+103.671492574, 105.020559ms after stability -2025-01-25T14:29:53.817Z [INFO] server-229aa372-85fe-e732-e6ba-ebb33cc9ed7f: cannot transfer leadership to itself -2025-01-25T14:29:53.817Z [INFO] server-229aa372-85fe-e732-e6ba-ebb33cc9ed7f: aborting pipeline replication: peer="{Voter server-e225ea80-f715-4bd1-9db0-6ff70c578a05 e225ea80-f715-4bd1-9db0-6ff70c578a05}" -2025-01-25T14:29:53.817Z [INFO] server-229aa372-85fe-e732-e6ba-ebb33cc9ed7f: aborting pipeline replication: peer="{Voter server-b07a38de-155a-2936-1c63-8fa8864b9eb5 b07a38de-155a-2936-1c63-8fa8864b9eb5}" +2025-01-25T14:40:41.634Z [INFO] server-1fc6bf41-6fcd-9443-bf49-5702d0a6313b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1fc6bf41-6fcd-9443-bf49-5702d0a6313b Address:1fc6bf41-6fcd-9443-bf49-5702d0a6313b} {Suffrage:Voter ID:server-d1af5b12-3772-3cd3-db43-78d30f654db4 Address:d1af5b12-3772-3cd3-db43-78d30f654db4} {Suffrage:Voter ID:server-d59315d3-9a36-1c38-61f3-0a213e9e49be Address:d59315d3-9a36-1c38-61f3-0a213e9e49be}]" +2025-01-25T14:40:41.635Z [INFO] server-1fc6bf41-6fcd-9443-bf49-5702d0a6313b: entering follower state: follower="Node at 1fc6bf41-6fcd-9443-bf49-5702d0a6313b [Follower]" leader-address= leader-id= +2025-01-25T14:40:41.635Z [INFO] server-d1af5b12-3772-3cd3-db43-78d30f654db4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1fc6bf41-6fcd-9443-bf49-5702d0a6313b Address:1fc6bf41-6fcd-9443-bf49-5702d0a6313b} {Suffrage:Voter ID:server-d1af5b12-3772-3cd3-db43-78d30f654db4 Address:d1af5b12-3772-3cd3-db43-78d30f654db4} {Suffrage:Voter ID:server-d59315d3-9a36-1c38-61f3-0a213e9e49be Address:d59315d3-9a36-1c38-61f3-0a213e9e49be}]" +2025-01-25T14:40:41.635Z [INFO] server-d1af5b12-3772-3cd3-db43-78d30f654db4: entering follower state: follower="Node at d1af5b12-3772-3cd3-db43-78d30f654db4 [Follower]" leader-address= leader-id= +2025-01-25T14:40:41.636Z [INFO] server-d59315d3-9a36-1c38-61f3-0a213e9e49be: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1fc6bf41-6fcd-9443-bf49-5702d0a6313b Address:1fc6bf41-6fcd-9443-bf49-5702d0a6313b} {Suffrage:Voter ID:server-d1af5b12-3772-3cd3-db43-78d30f654db4 Address:d1af5b12-3772-3cd3-db43-78d30f654db4} {Suffrage:Voter ID:server-d59315d3-9a36-1c38-61f3-0a213e9e49be Address:d59315d3-9a36-1c38-61f3-0a213e9e49be}]" +2025-01-25T14:40:41.636Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:41.636Z [INFO] server-d59315d3-9a36-1c38-61f3-0a213e9e49be: entering follower state: follower="Node at d59315d3-9a36-1c38-61f3-0a213e9e49be [Follower]" leader-address= leader-id= +2025-01-25T14:40:41.695Z [WARN] server-d1af5b12-3772-3cd3-db43-78d30f654db4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:41.695Z [INFO] server-d1af5b12-3772-3cd3-db43-78d30f654db4: entering candidate state: node="Node at d1af5b12-3772-3cd3-db43-78d30f654db4 [Candidate]" term=2 +2025-01-25T14:40:41.696Z [INFO] server-d1af5b12-3772-3cd3-db43-78d30f654db4: election won: term=2 tally=2 +2025-01-25T14:40:41.696Z [INFO] server-d1af5b12-3772-3cd3-db43-78d30f654db4: entering leader state: leader="Node at d1af5b12-3772-3cd3-db43-78d30f654db4 [Leader]" +2025-01-25T14:40:41.696Z [INFO] server-d1af5b12-3772-3cd3-db43-78d30f654db4: added peer, starting replication: peer=server-1fc6bf41-6fcd-9443-bf49-5702d0a6313b +2025-01-25T14:40:41.696Z [INFO] server-d1af5b12-3772-3cd3-db43-78d30f654db4: added peer, starting replication: peer=server-d59315d3-9a36-1c38-61f3-0a213e9e49be +2025-01-25T14:40:41.697Z [INFO] server-d1af5b12-3772-3cd3-db43-78d30f654db4: pipelining replication: peer="{Voter server-1fc6bf41-6fcd-9443-bf49-5702d0a6313b 1fc6bf41-6fcd-9443-bf49-5702d0a6313b}" +2025-01-25T14:40:41.697Z [INFO] server-d1af5b12-3772-3cd3-db43-78d30f654db4: pipelining replication: peer="{Voter server-d59315d3-9a36-1c38-61f3-0a213e9e49be d59315d3-9a36-1c38-61f3-0a213e9e49be}" +2025-01-25T14:40:41.802Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:41.696556598 +0000 UTC m=+110.742395916 (1 nodes), 59.868133ms from start of poll, 62.592016ms from cluster start. Timeout at 2025-01-25 14:40:41.80157789 +0000 UTC m=+110.847417208, 105.021292ms after stability +2025-01-25T14:40:41.802Z [INFO] server-d1af5b12-3772-3cd3-db43-78d30f654db4: cannot transfer leadership to itself +2025-01-25T14:40:41.802Z [INFO] server-d1af5b12-3772-3cd3-db43-78d30f654db4: aborting pipeline replication: peer="{Voter server-d59315d3-9a36-1c38-61f3-0a213e9e49be d59315d3-9a36-1c38-61f3-0a213e9e49be}" +2025-01-25T14:40:41.802Z [INFO] server-d1af5b12-3772-3cd3-db43-78d30f654db4: aborting pipeline replication: peer="{Voter server-1fc6bf41-6fcd-9443-bf49-5702d0a6313b 1fc6bf41-6fcd-9443-bf49-5702d0a6313b}" --- PASS: TestRaft_LeadershipTransferToItself (0.18s) === RUN TestRaft_LeadershipTransferLeaderRejectsClientRequests -2025-01-25T14:29:53.821Z [INFO] server-39a99784-482f-8024-85d3-c62d58ad144c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-39a99784-482f-8024-85d3-c62d58ad144c Address:39a99784-482f-8024-85d3-c62d58ad144c} {Suffrage:Voter ID:server-1cfdfc44-782a-7f48-b01d-242982286025 Address:1cfdfc44-782a-7f48-b01d-242982286025} {Suffrage:Voter ID:server-fbd285d9-b5b5-0f40-0a5a-ec87214cadbb Address:fbd285d9-b5b5-0f40-0a5a-ec87214cadbb}]" -2025-01-25T14:29:53.821Z [INFO] server-1cfdfc44-782a-7f48-b01d-242982286025: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-39a99784-482f-8024-85d3-c62d58ad144c Address:39a99784-482f-8024-85d3-c62d58ad144c} {Suffrage:Voter ID:server-1cfdfc44-782a-7f48-b01d-242982286025 Address:1cfdfc44-782a-7f48-b01d-242982286025} {Suffrage:Voter ID:server-fbd285d9-b5b5-0f40-0a5a-ec87214cadbb Address:fbd285d9-b5b5-0f40-0a5a-ec87214cadbb}]" -2025-01-25T14:29:53.821Z [INFO] server-fbd285d9-b5b5-0f40-0a5a-ec87214cadbb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-39a99784-482f-8024-85d3-c62d58ad144c Address:39a99784-482f-8024-85d3-c62d58ad144c} {Suffrage:Voter ID:server-1cfdfc44-782a-7f48-b01d-242982286025 Address:1cfdfc44-782a-7f48-b01d-242982286025} {Suffrage:Voter ID:server-fbd285d9-b5b5-0f40-0a5a-ec87214cadbb Address:fbd285d9-b5b5-0f40-0a5a-ec87214cadbb}]" -2025-01-25T14:29:53.821Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:53.821Z [INFO] server-39a99784-482f-8024-85d3-c62d58ad144c: entering follower state: follower="Node at 39a99784-482f-8024-85d3-c62d58ad144c [Follower]" leader-address= leader-id= -2025-01-25T14:29:53.821Z [INFO] server-1cfdfc44-782a-7f48-b01d-242982286025: entering follower state: follower="Node at 1cfdfc44-782a-7f48-b01d-242982286025 [Follower]" leader-address= leader-id= -2025-01-25T14:29:53.821Z [INFO] server-fbd285d9-b5b5-0f40-0a5a-ec87214cadbb: entering follower state: follower="Node at fbd285d9-b5b5-0f40-0a5a-ec87214cadbb [Follower]" leader-address= leader-id= -2025-01-25T14:29:53.876Z [WARN] server-fbd285d9-b5b5-0f40-0a5a-ec87214cadbb: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:53.876Z [INFO] server-fbd285d9-b5b5-0f40-0a5a-ec87214cadbb: entering candidate state: node="Node at fbd285d9-b5b5-0f40-0a5a-ec87214cadbb [Candidate]" term=2 -2025-01-25T14:29:53.876Z [INFO] server-fbd285d9-b5b5-0f40-0a5a-ec87214cadbb: election won: term=2 tally=2 -2025-01-25T14:29:53.876Z [INFO] server-fbd285d9-b5b5-0f40-0a5a-ec87214cadbb: entering leader state: leader="Node at fbd285d9-b5b5-0f40-0a5a-ec87214cadbb [Leader]" -2025-01-25T14:29:53.876Z [INFO] server-fbd285d9-b5b5-0f40-0a5a-ec87214cadbb: added peer, starting replication: peer=server-39a99784-482f-8024-85d3-c62d58ad144c -2025-01-25T14:29:53.876Z [INFO] server-fbd285d9-b5b5-0f40-0a5a-ec87214cadbb: added peer, starting replication: peer=server-1cfdfc44-782a-7f48-b01d-242982286025 -2025-01-25T14:29:53.876Z [INFO] server-fbd285d9-b5b5-0f40-0a5a-ec87214cadbb: pipelining replication: peer="{Voter server-39a99784-482f-8024-85d3-c62d58ad144c 39a99784-482f-8024-85d3-c62d58ad144c}" -2025-01-25T14:29:53.876Z [INFO] server-fbd285d9-b5b5-0f40-0a5a-ec87214cadbb: pipelining replication: peer="{Voter server-1cfdfc44-782a-7f48-b01d-242982286025 1cfdfc44-782a-7f48-b01d-242982286025}" -2025-01-25T14:29:53.988Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:53.876713551 +0000 UTC m=+103.731038533 (1 nodes), 55.058018ms from start of poll, 55.797712ms from cluster start. Timeout at 2025-01-25 14:29:53.981721911 +0000 UTC m=+103.836046893, 105.00836ms after stability +2025-01-25T14:40:41.811Z [INFO] server-00280c23-c1a8-8387-f35a-c014bc7b49b3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-00280c23-c1a8-8387-f35a-c014bc7b49b3 Address:00280c23-c1a8-8387-f35a-c014bc7b49b3} {Suffrage:Voter ID:server-7997056d-a997-3376-ad7c-532c5ea0d445 Address:7997056d-a997-3376-ad7c-532c5ea0d445} {Suffrage:Voter ID:server-6934bc5d-eef6-ab14-83e8-cd3db36cce43 Address:6934bc5d-eef6-ab14-83e8-cd3db36cce43}]" +2025-01-25T14:40:41.811Z [INFO] server-00280c23-c1a8-8387-f35a-c014bc7b49b3: entering follower state: follower="Node at 00280c23-c1a8-8387-f35a-c014bc7b49b3 [Follower]" leader-address= leader-id= +2025-01-25T14:40:41.811Z [INFO] server-7997056d-a997-3376-ad7c-532c5ea0d445: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-00280c23-c1a8-8387-f35a-c014bc7b49b3 Address:00280c23-c1a8-8387-f35a-c014bc7b49b3} {Suffrage:Voter ID:server-7997056d-a997-3376-ad7c-532c5ea0d445 Address:7997056d-a997-3376-ad7c-532c5ea0d445} {Suffrage:Voter ID:server-6934bc5d-eef6-ab14-83e8-cd3db36cce43 Address:6934bc5d-eef6-ab14-83e8-cd3db36cce43}]" +2025-01-25T14:40:41.812Z [INFO] server-7997056d-a997-3376-ad7c-532c5ea0d445: entering follower state: follower="Node at 7997056d-a997-3376-ad7c-532c5ea0d445 [Follower]" leader-address= leader-id= +2025-01-25T14:40:41.812Z [INFO] server-6934bc5d-eef6-ab14-83e8-cd3db36cce43: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-00280c23-c1a8-8387-f35a-c014bc7b49b3 Address:00280c23-c1a8-8387-f35a-c014bc7b49b3} {Suffrage:Voter ID:server-7997056d-a997-3376-ad7c-532c5ea0d445 Address:7997056d-a997-3376-ad7c-532c5ea0d445} {Suffrage:Voter ID:server-6934bc5d-eef6-ab14-83e8-cd3db36cce43 Address:6934bc5d-eef6-ab14-83e8-cd3db36cce43}]" +2025-01-25T14:40:41.812Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:41.813Z [INFO] server-6934bc5d-eef6-ab14-83e8-cd3db36cce43: entering follower state: follower="Node at 6934bc5d-eef6-ab14-83e8-cd3db36cce43 [Follower]" leader-address= leader-id= +2025-01-25T14:40:41.867Z [WARN] server-7997056d-a997-3376-ad7c-532c5ea0d445: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:41.867Z [INFO] server-7997056d-a997-3376-ad7c-532c5ea0d445: entering candidate state: node="Node at 7997056d-a997-3376-ad7c-532c5ea0d445 [Candidate]" term=2 +2025-01-25T14:40:41.867Z [INFO] server-7997056d-a997-3376-ad7c-532c5ea0d445: election won: term=2 tally=2 +2025-01-25T14:40:41.867Z [INFO] server-7997056d-a997-3376-ad7c-532c5ea0d445: entering leader state: leader="Node at 7997056d-a997-3376-ad7c-532c5ea0d445 [Leader]" +2025-01-25T14:40:41.867Z [INFO] server-7997056d-a997-3376-ad7c-532c5ea0d445: added peer, starting replication: peer=server-00280c23-c1a8-8387-f35a-c014bc7b49b3 +2025-01-25T14:40:41.867Z [INFO] server-7997056d-a997-3376-ad7c-532c5ea0d445: added peer, starting replication: peer=server-6934bc5d-eef6-ab14-83e8-cd3db36cce43 +2025-01-25T14:40:41.868Z [INFO] server-7997056d-a997-3376-ad7c-532c5ea0d445: pipelining replication: peer="{Voter server-6934bc5d-eef6-ab14-83e8-cd3db36cce43 6934bc5d-eef6-ab14-83e8-cd3db36cce43}" +2025-01-25T14:40:41.868Z [INFO] server-7997056d-a997-3376-ad7c-532c5ea0d445: pipelining replication: peer="{Voter server-00280c23-c1a8-8387-f35a-c014bc7b49b3 00280c23-c1a8-8387-f35a-c014bc7b49b3}" +2025-01-25T14:40:41.973Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:41.868114973 +0000 UTC m=+110.913954290 (1 nodes), 55.033577ms from start of poll, 57.73471ms from cluster start. Timeout at 2025-01-25 14:40:41.973150556 +0000 UTC m=+111.018989874, 105.035584ms after stability raft_test.go:2457: waiting on future 0 raft_test.go:2457: waiting on future 1 raft_test.go:2457: waiting on future 2 @@ -3858,342 +3912,357 @@ raft_test.go:2457: waiting on future 4 raft_test.go:2457: waiting on future 5 raft_test.go:2457: waiting on future 6 -2025-01-25T14:29:53.989Z [INFO] server-fbd285d9-b5b5-0f40-0a5a-ec87214cadbb: aborting pipeline replication: peer="{Voter server-1cfdfc44-782a-7f48-b01d-242982286025 1cfdfc44-782a-7f48-b01d-242982286025}" -2025-01-25T14:29:53.989Z [INFO] server-fbd285d9-b5b5-0f40-0a5a-ec87214cadbb: aborting pipeline replication: peer="{Voter server-39a99784-482f-8024-85d3-c62d58ad144c 39a99784-482f-8024-85d3-c62d58ad144c}" +2025-01-25T14:40:41.975Z [INFO] server-7997056d-a997-3376-ad7c-532c5ea0d445: aborting pipeline replication: peer="{Voter server-6934bc5d-eef6-ab14-83e8-cd3db36cce43 6934bc5d-eef6-ab14-83e8-cd3db36cce43}" +2025-01-25T14:40:41.975Z [INFO] server-7997056d-a997-3376-ad7c-532c5ea0d445: aborting pipeline replication: peer="{Voter server-00280c23-c1a8-8387-f35a-c014bc7b49b3 00280c23-c1a8-8387-f35a-c014bc7b49b3}" --- PASS: TestRaft_LeadershipTransferLeaderRejectsClientRequests (0.17s) === RUN TestRaft_LeadershipTransferLeaderReplicationTimeout -2025-01-25T14:29:53.991Z [INFO] server-b5e68fd4-766a-c95e-e2f7-4c6862d2c343: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b5e68fd4-766a-c95e-e2f7-4c6862d2c343 Address:b5e68fd4-766a-c95e-e2f7-4c6862d2c343} {Suffrage:Voter ID:server-bae24b4a-8101-b75e-b80f-181136becf8b Address:bae24b4a-8101-b75e-b80f-181136becf8b} {Suffrage:Voter ID:server-c919d1a7-c505-c3db-d672-60269f366cf0 Address:c919d1a7-c505-c3db-d672-60269f366cf0}]" -2025-01-25T14:29:53.992Z [INFO] server-bae24b4a-8101-b75e-b80f-181136becf8b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b5e68fd4-766a-c95e-e2f7-4c6862d2c343 Address:b5e68fd4-766a-c95e-e2f7-4c6862d2c343} {Suffrage:Voter ID:server-bae24b4a-8101-b75e-b80f-181136becf8b Address:bae24b4a-8101-b75e-b80f-181136becf8b} {Suffrage:Voter ID:server-c919d1a7-c505-c3db-d672-60269f366cf0 Address:c919d1a7-c505-c3db-d672-60269f366cf0}]" -2025-01-25T14:29:53.992Z [INFO] server-c919d1a7-c505-c3db-d672-60269f366cf0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b5e68fd4-766a-c95e-e2f7-4c6862d2c343 Address:b5e68fd4-766a-c95e-e2f7-4c6862d2c343} {Suffrage:Voter ID:server-bae24b4a-8101-b75e-b80f-181136becf8b Address:bae24b4a-8101-b75e-b80f-181136becf8b} {Suffrage:Voter ID:server-c919d1a7-c505-c3db-d672-60269f366cf0 Address:c919d1a7-c505-c3db-d672-60269f366cf0}]" -2025-01-25T14:29:53.992Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:53.992Z [INFO] server-b5e68fd4-766a-c95e-e2f7-4c6862d2c343: entering follower state: follower="Node at b5e68fd4-766a-c95e-e2f7-4c6862d2c343 [Follower]" leader-address= leader-id= -2025-01-25T14:29:53.992Z [INFO] server-bae24b4a-8101-b75e-b80f-181136becf8b: entering follower state: follower="Node at bae24b4a-8101-b75e-b80f-181136becf8b [Follower]" leader-address= leader-id= -2025-01-25T14:29:53.992Z [INFO] server-c919d1a7-c505-c3db-d672-60269f366cf0: entering follower state: follower="Node at c919d1a7-c505-c3db-d672-60269f366cf0 [Follower]" leader-address= leader-id= -2025-01-25T14:29:54.047Z [WARN] server-bae24b4a-8101-b75e-b80f-181136becf8b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:54.047Z [INFO] server-bae24b4a-8101-b75e-b80f-181136becf8b: entering candidate state: node="Node at bae24b4a-8101-b75e-b80f-181136becf8b [Candidate]" term=2 -2025-01-25T14:29:54.047Z [INFO] server-bae24b4a-8101-b75e-b80f-181136becf8b: election won: term=2 tally=2 -2025-01-25T14:29:54.047Z [INFO] server-bae24b4a-8101-b75e-b80f-181136becf8b: entering leader state: leader="Node at bae24b4a-8101-b75e-b80f-181136becf8b [Leader]" -2025-01-25T14:29:54.047Z [INFO] server-bae24b4a-8101-b75e-b80f-181136becf8b: added peer, starting replication: peer=server-b5e68fd4-766a-c95e-e2f7-4c6862d2c343 -2025-01-25T14:29:54.047Z [INFO] server-bae24b4a-8101-b75e-b80f-181136becf8b: added peer, starting replication: peer=server-c919d1a7-c505-c3db-d672-60269f366cf0 -2025-01-25T14:29:54.048Z [INFO] server-bae24b4a-8101-b75e-b80f-181136becf8b: pipelining replication: peer="{Voter server-b5e68fd4-766a-c95e-e2f7-4c6862d2c343 b5e68fd4-766a-c95e-e2f7-4c6862d2c343}" -2025-01-25T14:29:54.048Z [INFO] server-bae24b4a-8101-b75e-b80f-181136becf8b: pipelining replication: peer="{Voter server-c919d1a7-c505-c3db-d672-60269f366cf0 c919d1a7-c505-c3db-d672-60269f366cf0}" -2025-01-25T14:29:54.152Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:54.047763796 +0000 UTC m=+103.902088778 (1 nodes), 55.349449ms from start of poll, 56.022184ms from cluster start. Timeout at 2025-01-25 14:29:54.152776634 +0000 UTC m=+104.007101616, 105.012838ms after stability -2025-01-25T14:29:54.153Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:29:54.264Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:29:54.153079045 +0000 UTC m=+104.007403987 (2 nodes), 2.119µs from start of poll, 161.337393ms from cluster start. Timeout at 2025-01-25 14:29:54.258348276 +0000 UTC m=+104.112673218, 105.269231ms after stability -2025-01-25T14:29:55.643Z [INFO] server-bae24b4a-8101-b75e-b80f-181136becf8b: aborting pipeline replication: peer="{Voter server-c919d1a7-c505-c3db-d672-60269f366cf0 c919d1a7-c505-c3db-d672-60269f366cf0}" -2025-01-25T14:29:55.643Z [INFO] server-bae24b4a-8101-b75e-b80f-181136becf8b: aborting pipeline replication: peer="{Voter server-b5e68fd4-766a-c95e-e2f7-4c6862d2c343 b5e68fd4-766a-c95e-e2f7-4c6862d2c343}" -2025-01-25T14:29:56.140Z [ERROR] server-bae24b4a-8101-b75e-b80f-181136becf8b: failed to heartbeat to: peer=c919d1a7-c505-c3db-d672-60269f366cf0 backoff time=10ms error="command timed out" -2025-01-25T14:29:56.144Z [ERROR] server-bae24b4a-8101-b75e-b80f-181136becf8b: failed to heartbeat to: peer=b5e68fd4-766a-c95e-e2f7-4c6862d2c343 backoff time=10ms error="command timed out" ---- PASS: TestRaft_LeadershipTransferLeaderReplicationTimeout (2.22s) +2025-01-25T14:40:41.981Z [INFO] server-6b32230b-c1b5-c06b-0863-a3d0aeba301d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6b32230b-c1b5-c06b-0863-a3d0aeba301d Address:6b32230b-c1b5-c06b-0863-a3d0aeba301d} {Suffrage:Voter ID:server-645c0379-3bf7-78cc-e178-92387681180a Address:645c0379-3bf7-78cc-e178-92387681180a} {Suffrage:Voter ID:server-eb1a040a-a7df-c129-e4e8-667ec8e1d3c1 Address:eb1a040a-a7df-c129-e4e8-667ec8e1d3c1}]" +2025-01-25T14:40:41.981Z [INFO] server-6b32230b-c1b5-c06b-0863-a3d0aeba301d: entering follower state: follower="Node at 6b32230b-c1b5-c06b-0863-a3d0aeba301d [Follower]" leader-address= leader-id= +2025-01-25T14:40:41.982Z [INFO] server-645c0379-3bf7-78cc-e178-92387681180a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6b32230b-c1b5-c06b-0863-a3d0aeba301d Address:6b32230b-c1b5-c06b-0863-a3d0aeba301d} {Suffrage:Voter ID:server-645c0379-3bf7-78cc-e178-92387681180a Address:645c0379-3bf7-78cc-e178-92387681180a} {Suffrage:Voter ID:server-eb1a040a-a7df-c129-e4e8-667ec8e1d3c1 Address:eb1a040a-a7df-c129-e4e8-667ec8e1d3c1}]" +2025-01-25T14:40:41.982Z [INFO] server-645c0379-3bf7-78cc-e178-92387681180a: entering follower state: follower="Node at 645c0379-3bf7-78cc-e178-92387681180a [Follower]" leader-address= leader-id= +2025-01-25T14:40:41.982Z [INFO] server-eb1a040a-a7df-c129-e4e8-667ec8e1d3c1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6b32230b-c1b5-c06b-0863-a3d0aeba301d Address:6b32230b-c1b5-c06b-0863-a3d0aeba301d} {Suffrage:Voter ID:server-645c0379-3bf7-78cc-e178-92387681180a Address:645c0379-3bf7-78cc-e178-92387681180a} {Suffrage:Voter ID:server-eb1a040a-a7df-c129-e4e8-667ec8e1d3c1 Address:eb1a040a-a7df-c129-e4e8-667ec8e1d3c1}]" +2025-01-25T14:40:41.982Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:41.982Z [INFO] server-eb1a040a-a7df-c129-e4e8-667ec8e1d3c1: entering follower state: follower="Node at eb1a040a-a7df-c129-e4e8-667ec8e1d3c1 [Follower]" leader-address= leader-id= +2025-01-25T14:40:42.042Z [WARN] server-6b32230b-c1b5-c06b-0863-a3d0aeba301d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:42.042Z [INFO] server-6b32230b-c1b5-c06b-0863-a3d0aeba301d: entering candidate state: node="Node at 6b32230b-c1b5-c06b-0863-a3d0aeba301d [Candidate]" term=2 +2025-01-25T14:40:42.043Z [INFO] server-6b32230b-c1b5-c06b-0863-a3d0aeba301d: election won: term=2 tally=2 +2025-01-25T14:40:42.043Z [INFO] server-6b32230b-c1b5-c06b-0863-a3d0aeba301d: entering leader state: leader="Node at 6b32230b-c1b5-c06b-0863-a3d0aeba301d [Leader]" +2025-01-25T14:40:42.043Z [INFO] server-6b32230b-c1b5-c06b-0863-a3d0aeba301d: added peer, starting replication: peer=server-645c0379-3bf7-78cc-e178-92387681180a +2025-01-25T14:40:42.043Z [INFO] server-6b32230b-c1b5-c06b-0863-a3d0aeba301d: added peer, starting replication: peer=server-eb1a040a-a7df-c129-e4e8-667ec8e1d3c1 +2025-01-25T14:40:42.045Z [INFO] server-6b32230b-c1b5-c06b-0863-a3d0aeba301d: pipelining replication: peer="{Voter server-eb1a040a-a7df-c129-e4e8-667ec8e1d3c1 eb1a040a-a7df-c129-e4e8-667ec8e1d3c1}" +2025-01-25T14:40:42.045Z [INFO] server-6b32230b-c1b5-c06b-0863-a3d0aeba301d: pipelining replication: peer="{Voter server-645c0379-3bf7-78cc-e178-92387681180a 645c0379-3bf7-78cc-e178-92387681180a}" +2025-01-25T14:40:42.150Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:42.044855279 +0000 UTC m=+111.090694596 (1 nodes), 62.159139ms from start of poll, 63.871811ms from cluster start. Timeout at 2025-01-25 14:40:42.149915946 +0000 UTC m=+111.195754972, 105.060376ms after stability +2025-01-25T14:40:42.150Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:40:42.256Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:40:42.150805129 +0000 UTC m=+111.196644446 (2 nodes), 4.666µs from start of poll, 169.821661ms from cluster start. Timeout at 2025-01-25 14:40:42.256028547 +0000 UTC m=+111.301867864, 105.223418ms after stability +2025-01-25T14:40:46.427Z [INFO] server-6b32230b-c1b5-c06b-0863-a3d0aeba301d: aborting pipeline replication: peer="{Voter server-eb1a040a-a7df-c129-e4e8-667ec8e1d3c1 eb1a040a-a7df-c129-e4e8-667ec8e1d3c1}" +2025-01-25T14:40:46.427Z [INFO] server-6b32230b-c1b5-c06b-0863-a3d0aeba301d: aborting pipeline replication: peer="{Voter server-645c0379-3bf7-78cc-e178-92387681180a 645c0379-3bf7-78cc-e178-92387681180a}" +2025-01-25T14:40:46.926Z [ERROR] server-6b32230b-c1b5-c06b-0863-a3d0aeba301d: failed to heartbeat to: peer=645c0379-3bf7-78cc-e178-92387681180a backoff time=10ms error="command timed out" +2025-01-25T14:40:46.928Z [ERROR] server-6b32230b-c1b5-c06b-0863-a3d0aeba301d: failed to heartbeat to: peer=eb1a040a-a7df-c129-e4e8-667ec8e1d3c1 backoff time=10ms error="command timed out" +--- PASS: TestRaft_LeadershipTransferLeaderReplicationTimeout (4.95s) === RUN TestRaft_LeadershipTransferIgnoresNonvoters -2025-01-25T14:29:56.207Z [INFO] server-cb4e076e-a78b-e0fb-3804-7ea5fddf3287: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cb4e076e-a78b-e0fb-3804-7ea5fddf3287 Address:cb4e076e-a78b-e0fb-3804-7ea5fddf3287} {Suffrage:Voter ID:server-3b500cf7-47d8-babe-6855-2ef8dd28a6da Address:3b500cf7-47d8-babe-6855-2ef8dd28a6da}]" -2025-01-25T14:29:56.207Z [INFO] server-cb4e076e-a78b-e0fb-3804-7ea5fddf3287: entering follower state: follower="Node at cb4e076e-a78b-e0fb-3804-7ea5fddf3287 [Follower]" leader-address= leader-id= -2025-01-25T14:29:56.207Z [INFO] server-3b500cf7-47d8-babe-6855-2ef8dd28a6da: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cb4e076e-a78b-e0fb-3804-7ea5fddf3287 Address:cb4e076e-a78b-e0fb-3804-7ea5fddf3287} {Suffrage:Voter ID:server-3b500cf7-47d8-babe-6855-2ef8dd28a6da Address:3b500cf7-47d8-babe-6855-2ef8dd28a6da}]" -2025-01-25T14:29:56.207Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:29:56.207Z [INFO] server-3b500cf7-47d8-babe-6855-2ef8dd28a6da: entering follower state: follower="Node at 3b500cf7-47d8-babe-6855-2ef8dd28a6da [Follower]" leader-address= leader-id= -2025-01-25T14:29:56.277Z [WARN] server-3b500cf7-47d8-babe-6855-2ef8dd28a6da: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:56.278Z [INFO] server-3b500cf7-47d8-babe-6855-2ef8dd28a6da: entering candidate state: node="Node at 3b500cf7-47d8-babe-6855-2ef8dd28a6da [Candidate]" term=2 -2025-01-25T14:29:56.278Z [INFO] server-3b500cf7-47d8-babe-6855-2ef8dd28a6da: election won: term=2 tally=2 -2025-01-25T14:29:56.278Z [INFO] server-3b500cf7-47d8-babe-6855-2ef8dd28a6da: entering leader state: leader="Node at 3b500cf7-47d8-babe-6855-2ef8dd28a6da [Leader]" -2025-01-25T14:29:56.278Z [INFO] server-3b500cf7-47d8-babe-6855-2ef8dd28a6da: added peer, starting replication: peer=server-cb4e076e-a78b-e0fb-3804-7ea5fddf3287 -2025-01-25T14:29:56.278Z [INFO] server-3b500cf7-47d8-babe-6855-2ef8dd28a6da: pipelining replication: peer="{Voter server-cb4e076e-a78b-e0fb-3804-7ea5fddf3287 cb4e076e-a78b-e0fb-3804-7ea5fddf3287}" -2025-01-25T14:29:56.383Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:29:56.278334723 +0000 UTC m=+106.132659705 (1 nodes), 70.751494ms from start of poll, 71.315394ms from cluster start. Timeout at 2025-01-25 14:29:56.383337883 +0000 UTC m=+106.237662825, 105.00312ms after stability -2025-01-25T14:29:56.383Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:56.489Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:56.383859325 +0000 UTC m=+106.238184307 (1 nodes), 1.48µs from start of poll, 176.839996ms from cluster start. Timeout at 2025-01-25 14:29:56.489096036 +0000 UTC m=+106.343420978, 105.236671ms after stability -2025-01-25T14:29:56.490Z [INFO] server-3b500cf7-47d8-babe-6855-2ef8dd28a6da: updating configuration: command=DemoteVoter server-id=server-cb4e076e-a78b-e0fb-3804-7ea5fddf3287 server-addr= servers="[{Suffrage:Nonvoter ID:server-cb4e076e-a78b-e0fb-3804-7ea5fddf3287 Address:cb4e076e-a78b-e0fb-3804-7ea5fddf3287} {Suffrage:Voter ID:server-3b500cf7-47d8-babe-6855-2ef8dd28a6da Address:3b500cf7-47d8-babe-6855-2ef8dd28a6da}]" -2025-01-25T14:29:56.490Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:56.596Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:56.490416771 +0000 UTC m=+106.344741713 (1 nodes), 1.24µs from start of poll, 283.397402ms from cluster start. Timeout at 2025-01-25 14:29:56.59573696 +0000 UTC m=+106.450061902, 105.320189ms after stability -2025-01-25T14:29:56.597Z [INFO] server-3b500cf7-47d8-babe-6855-2ef8dd28a6da: aborting pipeline replication: peer="{Voter server-cb4e076e-a78b-e0fb-3804-7ea5fddf3287 cb4e076e-a78b-e0fb-3804-7ea5fddf3287}" ---- PASS: TestRaft_LeadershipTransferIgnoresNonvoters (0.39s) +2025-01-25T14:40:46.935Z [INFO] server-0f3999e1-bfab-0ccc-190a-27ae87029032: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0f3999e1-bfab-0ccc-190a-27ae87029032 Address:0f3999e1-bfab-0ccc-190a-27ae87029032} {Suffrage:Voter ID:server-eee113a8-3b4e-429a-2919-21c68bc52422 Address:eee113a8-3b4e-429a-2919-21c68bc52422}]" +2025-01-25T14:40:46.936Z [INFO] server-0f3999e1-bfab-0ccc-190a-27ae87029032: entering follower state: follower="Node at 0f3999e1-bfab-0ccc-190a-27ae87029032 [Follower]" leader-address= leader-id= +2025-01-25T14:40:46.936Z [INFO] server-eee113a8-3b4e-429a-2919-21c68bc52422: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0f3999e1-bfab-0ccc-190a-27ae87029032 Address:0f3999e1-bfab-0ccc-190a-27ae87029032} {Suffrage:Voter ID:server-eee113a8-3b4e-429a-2919-21c68bc52422 Address:eee113a8-3b4e-429a-2919-21c68bc52422}]" +2025-01-25T14:40:46.936Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:40:46.936Z [INFO] server-eee113a8-3b4e-429a-2919-21c68bc52422: entering follower state: follower="Node at eee113a8-3b4e-429a-2919-21c68bc52422 [Follower]" leader-address= leader-id= +2025-01-25T14:40:46.999Z [WARN] server-0f3999e1-bfab-0ccc-190a-27ae87029032: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:46.999Z [INFO] server-0f3999e1-bfab-0ccc-190a-27ae87029032: entering candidate state: node="Node at 0f3999e1-bfab-0ccc-190a-27ae87029032 [Candidate]" term=2 +2025-01-25T14:40:47.017Z [WARN] server-eee113a8-3b4e-429a-2919-21c68bc52422: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:47.017Z [INFO] server-eee113a8-3b4e-429a-2919-21c68bc52422: entering candidate state: node="Node at eee113a8-3b4e-429a-2919-21c68bc52422 [Candidate]" term=2 +2025-01-25T14:40:47.017Z [INFO] server-eee113a8-3b4e-429a-2919-21c68bc52422: duplicate requestVote for same term: term=2 +2025-01-25T14:40:47.022Z [INFO] server-0f3999e1-bfab-0ccc-190a-27ae87029032: duplicate requestVote for same term: term=2 +2025-01-25T14:40:47.083Z [WARN] server-0f3999e1-bfab-0ccc-190a-27ae87029032: Election timeout reached, restarting election +2025-01-25T14:40:47.083Z [INFO] server-0f3999e1-bfab-0ccc-190a-27ae87029032: entering candidate state: node="Node at 0f3999e1-bfab-0ccc-190a-27ae87029032 [Candidate]" term=3 +2025-01-25T14:40:47.083Z [INFO] server-eee113a8-3b4e-429a-2919-21c68bc52422: entering follower state: follower="Node at eee113a8-3b4e-429a-2919-21c68bc52422 [Follower]" leader-address= leader-id= +2025-01-25T14:40:47.083Z [INFO] server-0f3999e1-bfab-0ccc-190a-27ae87029032: election won: term=3 tally=2 +2025-01-25T14:40:47.084Z [INFO] server-0f3999e1-bfab-0ccc-190a-27ae87029032: entering leader state: leader="Node at 0f3999e1-bfab-0ccc-190a-27ae87029032 [Leader]" +2025-01-25T14:40:47.084Z [INFO] server-0f3999e1-bfab-0ccc-190a-27ae87029032: added peer, starting replication: peer=server-eee113a8-3b4e-429a-2919-21c68bc52422 +2025-01-25T14:40:47.084Z [INFO] server-0f3999e1-bfab-0ccc-190a-27ae87029032: pipelining replication: peer="{Voter server-eee113a8-3b4e-429a-2919-21c68bc52422 eee113a8-3b4e-429a-2919-21c68bc52422}" +2025-01-25T14:40:47.189Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:40:47.084114282 +0000 UTC m=+116.129953600 (1 nodes), 147.416679ms from start of poll, 148.629141ms from cluster start. Timeout at 2025-01-25 14:40:47.189122157 +0000 UTC m=+116.234961474, 105.007874ms after stability +2025-01-25T14:40:47.189Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:47.295Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:47.189938424 +0000 UTC m=+116.235777741 (1 nodes), 4.666µs from start of poll, 254.453282ms from cluster start. Timeout at 2025-01-25 14:40:47.295109049 +0000 UTC m=+116.340948658, 105.170917ms after stability +2025-01-25T14:40:47.295Z [INFO] server-0f3999e1-bfab-0ccc-190a-27ae87029032: updating configuration: command=DemoteVoter server-id=server-eee113a8-3b4e-429a-2919-21c68bc52422 server-addr= servers="[{Suffrage:Voter ID:server-0f3999e1-bfab-0ccc-190a-27ae87029032 Address:0f3999e1-bfab-0ccc-190a-27ae87029032} {Suffrage:Nonvoter ID:server-eee113a8-3b4e-429a-2919-21c68bc52422 Address:eee113a8-3b4e-429a-2919-21c68bc52422}]" +2025-01-25T14:40:47.296Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:47.402Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:47.296598443 +0000 UTC m=+116.342437760 (1 nodes), 4.958µs from start of poll, 361.113301ms from cluster start. Timeout at 2025-01-25 14:40:47.40176761 +0000 UTC m=+116.447606927, 105.169167ms after stability +2025-01-25T14:40:47.418Z [INFO] server-0f3999e1-bfab-0ccc-190a-27ae87029032: aborting pipeline replication: peer="{Voter server-eee113a8-3b4e-429a-2919-21c68bc52422 eee113a8-3b4e-429a-2919-21c68bc52422}" +--- PASS: TestRaft_LeadershipTransferIgnoresNonvoters (0.50s) === RUN TestRaft_LeadershipTransferStopRightAway --- PASS: TestRaft_LeadershipTransferStopRightAway (0.00s) === RUN TestRaft_GetConfigurationNoBootstrap -2025-01-25T14:29:56.599Z [INFO] server-f27a6ea9-eaf4-4af9-ed74-33118fe1544a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f27a6ea9-eaf4-4af9-ed74-33118fe1544a Address:f27a6ea9-eaf4-4af9-ed74-33118fe1544a} {Suffrage:Voter ID:server-c431c6c0-2334-02c8-cb62-9b03ddf0b899 Address:c431c6c0-2334-02c8-cb62-9b03ddf0b899}]" -2025-01-25T14:29:56.600Z [INFO] server-c431c6c0-2334-02c8-cb62-9b03ddf0b899: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f27a6ea9-eaf4-4af9-ed74-33118fe1544a Address:f27a6ea9-eaf4-4af9-ed74-33118fe1544a} {Suffrage:Voter ID:server-c431c6c0-2334-02c8-cb62-9b03ddf0b899 Address:c431c6c0-2334-02c8-cb62-9b03ddf0b899}]" -2025-01-25T14:29:56.600Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:29:56.600Z [INFO] server-f27a6ea9-eaf4-4af9-ed74-33118fe1544a: entering follower state: follower="Node at f27a6ea9-eaf4-4af9-ed74-33118fe1544a [Follower]" leader-address= leader-id= -2025-01-25T14:29:56.600Z [INFO] server-c431c6c0-2334-02c8-cb62-9b03ddf0b899: entering follower state: follower="Node at c431c6c0-2334-02c8-cb62-9b03ddf0b899 [Follower]" leader-address= leader-id= -2025-01-25T14:29:56.698Z [WARN] server-c431c6c0-2334-02c8-cb62-9b03ddf0b899: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:56.698Z [INFO] server-c431c6c0-2334-02c8-cb62-9b03ddf0b899: entering candidate state: node="Node at c431c6c0-2334-02c8-cb62-9b03ddf0b899 [Candidate]" term=2 -2025-01-25T14:29:56.698Z [INFO] server-c431c6c0-2334-02c8-cb62-9b03ddf0b899: election won: term=2 tally=2 -2025-01-25T14:29:56.698Z [INFO] server-c431c6c0-2334-02c8-cb62-9b03ddf0b899: entering leader state: leader="Node at c431c6c0-2334-02c8-cb62-9b03ddf0b899 [Leader]" -2025-01-25T14:29:56.698Z [INFO] server-c431c6c0-2334-02c8-cb62-9b03ddf0b899: added peer, starting replication: peer=server-f27a6ea9-eaf4-4af9-ed74-33118fe1544a -2025-01-25T14:29:56.698Z [INFO] server-c431c6c0-2334-02c8-cb62-9b03ddf0b899: pipelining replication: peer="{Voter server-f27a6ea9-eaf4-4af9-ed74-33118fe1544a f27a6ea9-eaf4-4af9-ed74-33118fe1544a}" -2025-01-25T14:29:56.803Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:29:56.698518361 +0000 UTC m=+106.552843343 (1 nodes), 98.34514ms from start of poll, 98.865522ms from cluster start. Timeout at 2025-01-25 14:29:56.803520041 +0000 UTC m=+106.657845023, 105.00168ms after stability -2025-01-25T14:29:56.803Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:56.909Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:56.803716896 +0000 UTC m=+106.658041878 (1 nodes), 1.28µs from start of poll, 204.064057ms from cluster start. Timeout at 2025-01-25 14:29:56.908948208 +0000 UTC m=+106.763273190, 105.231312ms after stability -2025-01-25T14:29:56.921Z [INFO] server-c431c6c0-2334-02c8-cb62-9b03ddf0b899: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f27a6ea9-eaf4-4af9-ed74-33118fe1544a Address:f27a6ea9-eaf4-4af9-ed74-33118fe1544a} {Suffrage:Voter ID:server-c431c6c0-2334-02c8-cb62-9b03ddf0b899 Address:c431c6c0-2334-02c8-cb62-9b03ddf0b899}]" -2025-01-25T14:29:56.921Z [INFO] server-c431c6c0-2334-02c8-cb62-9b03ddf0b899: aborting pipeline replication: peer="{Voter server-f27a6ea9-eaf4-4af9-ed74-33118fe1544a f27a6ea9-eaf4-4af9-ed74-33118fe1544a}" ---- PASS: TestRaft_GetConfigurationNoBootstrap (0.32s) +2025-01-25T14:40:47.436Z [INFO] server-4c252920-9109-9e1a-970e-da5176ec3327: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4c252920-9109-9e1a-970e-da5176ec3327 Address:4c252920-9109-9e1a-970e-da5176ec3327} {Suffrage:Voter ID:server-9869aefc-66b1-d52d-b555-7bce7f7dab3a Address:9869aefc-66b1-d52d-b555-7bce7f7dab3a}]" +2025-01-25T14:40:47.436Z [INFO] server-4c252920-9109-9e1a-970e-da5176ec3327: entering follower state: follower="Node at 4c252920-9109-9e1a-970e-da5176ec3327 [Follower]" leader-address= leader-id= +2025-01-25T14:40:47.437Z [INFO] server-9869aefc-66b1-d52d-b555-7bce7f7dab3a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4c252920-9109-9e1a-970e-da5176ec3327 Address:4c252920-9109-9e1a-970e-da5176ec3327} {Suffrage:Voter ID:server-9869aefc-66b1-d52d-b555-7bce7f7dab3a Address:9869aefc-66b1-d52d-b555-7bce7f7dab3a}]" +2025-01-25T14:40:47.437Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:40:47.437Z [INFO] server-9869aefc-66b1-d52d-b555-7bce7f7dab3a: entering follower state: follower="Node at 9869aefc-66b1-d52d-b555-7bce7f7dab3a [Follower]" leader-address= leader-id= +2025-01-25T14:40:47.509Z [WARN] server-4c252920-9109-9e1a-970e-da5176ec3327: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:47.509Z [INFO] server-4c252920-9109-9e1a-970e-da5176ec3327: entering candidate state: node="Node at 4c252920-9109-9e1a-970e-da5176ec3327 [Candidate]" term=2 +2025-01-25T14:40:47.509Z [INFO] server-4c252920-9109-9e1a-970e-da5176ec3327: election won: term=2 tally=2 +2025-01-25T14:40:47.509Z [INFO] server-4c252920-9109-9e1a-970e-da5176ec3327: entering leader state: leader="Node at 4c252920-9109-9e1a-970e-da5176ec3327 [Leader]" +2025-01-25T14:40:47.509Z [INFO] server-4c252920-9109-9e1a-970e-da5176ec3327: added peer, starting replication: peer=server-9869aefc-66b1-d52d-b555-7bce7f7dab3a +2025-01-25T14:40:47.510Z [INFO] server-4c252920-9109-9e1a-970e-da5176ec3327: pipelining replication: peer="{Voter server-9869aefc-66b1-d52d-b555-7bce7f7dab3a 9869aefc-66b1-d52d-b555-7bce7f7dab3a}" +2025-01-25T14:40:47.615Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:40:47.509838605 +0000 UTC m=+116.555677923 (1 nodes), 72.437711ms from start of poll, 73.630631ms from cluster start. Timeout at 2025-01-25 14:40:47.614847064 +0000 UTC m=+116.660686381, 105.008458ms after stability +2025-01-25T14:40:47.615Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:47.721Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:47.615740413 +0000 UTC m=+116.661579731 (1 nodes), 4.083µs from start of poll, 179.532439ms from cluster start. Timeout at 2025-01-25 14:40:47.720957122 +0000 UTC m=+116.766796439, 105.216708ms after stability +2025-01-25T14:40:47.728Z [INFO] server-9869aefc-66b1-d52d-b555-7bce7f7dab3a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4c252920-9109-9e1a-970e-da5176ec3327 Address:4c252920-9109-9e1a-970e-da5176ec3327} {Suffrage:Voter ID:server-9869aefc-66b1-d52d-b555-7bce7f7dab3a Address:9869aefc-66b1-d52d-b555-7bce7f7dab3a}]" +2025-01-25T14:40:47.729Z [INFO] server-4c252920-9109-9e1a-970e-da5176ec3327: aborting pipeline replication: peer="{Voter server-9869aefc-66b1-d52d-b555-7bce7f7dab3a 9869aefc-66b1-d52d-b555-7bce7f7dab3a}" +--- PASS: TestRaft_GetConfigurationNoBootstrap (0.30s) === RUN TestRaft_LogStoreIsMonotonic -2025-01-25T14:29:56.923Z [INFO] server-66710fe2-7ffd-67d0-6cdc-b6e8a57fb895: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-66710fe2-7ffd-67d0-6cdc-b6e8a57fb895 Address:66710fe2-7ffd-67d0-6cdc-b6e8a57fb895}]" -2025-01-25T14:29:56.923Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:56.923Z [INFO] server-66710fe2-7ffd-67d0-6cdc-b6e8a57fb895: entering follower state: follower="Node at 66710fe2-7ffd-67d0-6cdc-b6e8a57fb895 [Follower]" leader-address= leader-id= -2025-01-25T14:29:57.009Z [WARN] server-66710fe2-7ffd-67d0-6cdc-b6e8a57fb895: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:57.010Z [INFO] server-66710fe2-7ffd-67d0-6cdc-b6e8a57fb895: entering candidate state: node="Node at 66710fe2-7ffd-67d0-6cdc-b6e8a57fb895 [Candidate]" term=2 -2025-01-25T14:29:57.010Z [INFO] server-66710fe2-7ffd-67d0-6cdc-b6e8a57fb895: election won: term=2 tally=1 -2025-01-25T14:29:57.010Z [INFO] server-66710fe2-7ffd-67d0-6cdc-b6e8a57fb895: entering leader state: leader="Node at 66710fe2-7ffd-67d0-6cdc-b6e8a57fb895 [Leader]" -2025-01-25T14:29:57.115Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:57.010241103 +0000 UTC m=+106.864566085 (1 nodes), 87.020748ms from start of poll, 87.310258ms from cluster start. Timeout at 2025-01-25 14:29:57.115248743 +0000 UTC m=+106.969573725, 105.00764ms after stability ---- PASS: TestRaft_LogStoreIsMonotonic (0.19s) +2025-01-25T14:40:47.732Z [INFO] server-bf3f3278-a40f-c4ef-d65d-f7d89e1e4cef: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bf3f3278-a40f-c4ef-d65d-f7d89e1e4cef Address:bf3f3278-a40f-c4ef-d65d-f7d89e1e4cef}]" +2025-01-25T14:40:47.732Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:47.733Z [INFO] server-bf3f3278-a40f-c4ef-d65d-f7d89e1e4cef: entering follower state: follower="Node at bf3f3278-a40f-c4ef-d65d-f7d89e1e4cef [Follower]" leader-address= leader-id= +2025-01-25T14:40:47.793Z [WARN] server-bf3f3278-a40f-c4ef-d65d-f7d89e1e4cef: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:47.793Z [INFO] server-bf3f3278-a40f-c4ef-d65d-f7d89e1e4cef: entering candidate state: node="Node at bf3f3278-a40f-c4ef-d65d-f7d89e1e4cef [Candidate]" term=2 +2025-01-25T14:40:47.793Z [INFO] server-bf3f3278-a40f-c4ef-d65d-f7d89e1e4cef: election won: term=2 tally=1 +2025-01-25T14:40:47.793Z [INFO] server-bf3f3278-a40f-c4ef-d65d-f7d89e1e4cef: entering leader state: leader="Node at bf3f3278-a40f-c4ef-d65d-f7d89e1e4cef [Leader]" +2025-01-25T14:40:47.909Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:47.80458588 +0000 UTC m=+116.850425489 (1 nodes), 71.56275ms from start of poll, 72.308252ms from cluster start. Timeout at 2025-01-25 14:40:47.909607754 +0000 UTC m=+116.955447072, 105.021583ms after stability +--- PASS: TestRaft_LogStoreIsMonotonic (0.18s) === RUN TestRaft_CacheLogWithStoreError -2025-01-25T14:29:57.118Z [INFO] server-dd96b427-3c80-ef12-ac75-22a3c69d7d06: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dd96b427-3c80-ef12-ac75-22a3c69d7d06 Address:dd96b427-3c80-ef12-ac75-22a3c69d7d06} {Suffrage:Voter ID:server-fbbfa707-236e-d103-b6ef-0e4c13712e9d Address:fbbfa707-236e-d103-b6ef-0e4c13712e9d}]" -2025-01-25T14:29:57.118Z [INFO] server-fbbfa707-236e-d103-b6ef-0e4c13712e9d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dd96b427-3c80-ef12-ac75-22a3c69d7d06 Address:dd96b427-3c80-ef12-ac75-22a3c69d7d06} {Suffrage:Voter ID:server-fbbfa707-236e-d103-b6ef-0e4c13712e9d Address:fbbfa707-236e-d103-b6ef-0e4c13712e9d}]" -2025-01-25T14:29:57.118Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:29:57.118Z [INFO] server-dd96b427-3c80-ef12-ac75-22a3c69d7d06: entering follower state: follower="Node at dd96b427-3c80-ef12-ac75-22a3c69d7d06 [Follower]" leader-address= leader-id= -2025-01-25T14:29:57.119Z [INFO] server-fbbfa707-236e-d103-b6ef-0e4c13712e9d: entering follower state: follower="Node at fbbfa707-236e-d103-b6ef-0e4c13712e9d [Follower]" leader-address= leader-id= -2025-01-25T14:29:57.171Z [WARN] server-dd96b427-3c80-ef12-ac75-22a3c69d7d06: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:57.171Z [INFO] server-dd96b427-3c80-ef12-ac75-22a3c69d7d06: entering candidate state: node="Node at dd96b427-3c80-ef12-ac75-22a3c69d7d06 [Candidate]" term=2 -2025-01-25T14:29:57.171Z [INFO] server-dd96b427-3c80-ef12-ac75-22a3c69d7d06: election won: term=2 tally=2 -2025-01-25T14:29:57.171Z [INFO] server-dd96b427-3c80-ef12-ac75-22a3c69d7d06: entering leader state: leader="Node at dd96b427-3c80-ef12-ac75-22a3c69d7d06 [Leader]" -2025-01-25T14:29:57.171Z [INFO] server-dd96b427-3c80-ef12-ac75-22a3c69d7d06: added peer, starting replication: peer=server-fbbfa707-236e-d103-b6ef-0e4c13712e9d -2025-01-25T14:29:57.172Z [INFO] server-dd96b427-3c80-ef12-ac75-22a3c69d7d06: pipelining replication: peer="{Voter server-fbbfa707-236e-d103-b6ef-0e4c13712e9d fbbfa707-236e-d103-b6ef-0e4c13712e9d}" -2025-01-25T14:29:57.277Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:29:57.171961123 +0000 UTC m=+107.026286105 (1 nodes), 53.405438ms from start of poll, 53.925459ms from cluster start. Timeout at 2025-01-25 14:29:57.276966163 +0000 UTC m=+107.131291145, 105.00504ms after stability -2025-01-25T14:29:57.277Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:57.383Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:57.277624681 +0000 UTC m=+107.131949663 (1 nodes), 1.64µs from start of poll, 159.589017ms from cluster start. Timeout at 2025-01-25 14:29:57.38290703 +0000 UTC m=+107.237232012, 105.282349ms after stability -2025-01-25T14:29:57.384Z [ERROR] server-fbbfa707-236e-d103-b6ef-0e4c13712e9d: failed to append to logs: error="unable to store logs within log store, err: "some error"" -2025-01-25T14:29:57.385Z [WARN] server-fbbfa707-236e-d103-b6ef-0e4c13712e9d: failed to get previous log: previous-index=9 last-index=7 error="log not found" -2025-01-25T14:29:57.385Z [INFO] server-dd96b427-3c80-ef12-ac75-22a3c69d7d06: aborting pipeline replication: peer="{Voter server-fbbfa707-236e-d103-b6ef-0e4c13712e9d fbbfa707-236e-d103-b6ef-0e4c13712e9d}" -2025-01-25T14:29:57.394Z [INFO] server-dd96b427-3c80-ef12-ac75-22a3c69d7d06: pipelining replication: peer="{Voter server-fbbfa707-236e-d103-b6ef-0e4c13712e9d fbbfa707-236e-d103-b6ef-0e4c13712e9d}" -2025-01-25T14:29:57.395Z [INFO] server-fbbfa707-236e-d103-b6ef-0e4c13712e9d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dd96b427-3c80-ef12-ac75-22a3c69d7d06 Address:dd96b427-3c80-ef12-ac75-22a3c69d7d06} {Suffrage:Voter ID:server-fbbfa707-236e-d103-b6ef-0e4c13712e9d Address:fbbfa707-236e-d103-b6ef-0e4c13712e9d}]" -2025-01-25T14:29:57.396Z [INFO] server-dd96b427-3c80-ef12-ac75-22a3c69d7d06: aborting pipeline replication: peer="{Voter server-fbbfa707-236e-d103-b6ef-0e4c13712e9d fbbfa707-236e-d103-b6ef-0e4c13712e9d}" ---- PASS: TestRaft_CacheLogWithStoreError (0.28s) +2025-01-25T14:40:47.915Z [INFO] server-79bdba1c-c406-3e87-6794-41b2a0aa7bf1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-79bdba1c-c406-3e87-6794-41b2a0aa7bf1 Address:79bdba1c-c406-3e87-6794-41b2a0aa7bf1} {Suffrage:Voter ID:server-cd537bb0-fc2d-c336-e312-6ae56407e732 Address:cd537bb0-fc2d-c336-e312-6ae56407e732}]" +2025-01-25T14:40:47.915Z [INFO] server-79bdba1c-c406-3e87-6794-41b2a0aa7bf1: entering follower state: follower="Node at 79bdba1c-c406-3e87-6794-41b2a0aa7bf1 [Follower]" leader-address= leader-id= +2025-01-25T14:40:47.915Z [INFO] server-cd537bb0-fc2d-c336-e312-6ae56407e732: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-79bdba1c-c406-3e87-6794-41b2a0aa7bf1 Address:79bdba1c-c406-3e87-6794-41b2a0aa7bf1} {Suffrage:Voter ID:server-cd537bb0-fc2d-c336-e312-6ae56407e732 Address:cd537bb0-fc2d-c336-e312-6ae56407e732}]" +2025-01-25T14:40:47.916Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:40:47.916Z [INFO] server-cd537bb0-fc2d-c336-e312-6ae56407e732: entering follower state: follower="Node at cd537bb0-fc2d-c336-e312-6ae56407e732 [Follower]" leader-address= leader-id= +2025-01-25T14:40:48.003Z [WARN] server-79bdba1c-c406-3e87-6794-41b2a0aa7bf1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:48.003Z [INFO] server-79bdba1c-c406-3e87-6794-41b2a0aa7bf1: entering candidate state: node="Node at 79bdba1c-c406-3e87-6794-41b2a0aa7bf1 [Candidate]" term=2 +2025-01-25T14:40:48.003Z [INFO] server-79bdba1c-c406-3e87-6794-41b2a0aa7bf1: election won: term=2 tally=2 +2025-01-25T14:40:48.003Z [INFO] server-79bdba1c-c406-3e87-6794-41b2a0aa7bf1: entering leader state: leader="Node at 79bdba1c-c406-3e87-6794-41b2a0aa7bf1 [Leader]" +2025-01-25T14:40:48.003Z [INFO] server-79bdba1c-c406-3e87-6794-41b2a0aa7bf1: added peer, starting replication: peer=server-cd537bb0-fc2d-c336-e312-6ae56407e732 +2025-01-25T14:40:48.004Z [INFO] server-79bdba1c-c406-3e87-6794-41b2a0aa7bf1: pipelining replication: peer="{Voter server-cd537bb0-fc2d-c336-e312-6ae56407e732 cd537bb0-fc2d-c336-e312-6ae56407e732}" +2025-01-25T14:40:48.109Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:40:48.003997002 +0000 UTC m=+117.049836320 (1 nodes), 87.843548ms from start of poll, 89.450927ms from cluster start. Timeout at 2025-01-25 14:40:48.109018003 +0000 UTC m=+117.154857320, 105.021ms after stability +2025-01-25T14:40:48.109Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:48.235Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:48.109512976 +0000 UTC m=+117.155352293 (1 nodes), 4.375µs from start of poll, 194.9669ms from cluster start. Timeout at 2025-01-25 14:40:48.214747768 +0000 UTC m=+117.260587085, 105.234792ms after stability +2025-01-25T14:40:48.240Z [ERROR] server-cd537bb0-fc2d-c336-e312-6ae56407e732: failed to append to logs: error="unable to store logs within log store, err: "some error"" +2025-01-25T14:40:48.240Z [WARN] server-cd537bb0-fc2d-c336-e312-6ae56407e732: failed to get previous log: previous-index=9 last-index=7 error="log not found" +2025-01-25T14:40:48.241Z [INFO] server-79bdba1c-c406-3e87-6794-41b2a0aa7bf1: aborting pipeline replication: peer="{Voter server-cd537bb0-fc2d-c336-e312-6ae56407e732 cd537bb0-fc2d-c336-e312-6ae56407e732}" +2025-01-25T14:40:48.247Z [INFO] server-79bdba1c-c406-3e87-6794-41b2a0aa7bf1: pipelining replication: peer="{Voter server-cd537bb0-fc2d-c336-e312-6ae56407e732 cd537bb0-fc2d-c336-e312-6ae56407e732}" +2025-01-25T14:40:48.248Z [INFO] server-cd537bb0-fc2d-c336-e312-6ae56407e732: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-79bdba1c-c406-3e87-6794-41b2a0aa7bf1 Address:79bdba1c-c406-3e87-6794-41b2a0aa7bf1} {Suffrage:Voter ID:server-cd537bb0-fc2d-c336-e312-6ae56407e732 Address:cd537bb0-fc2d-c336-e312-6ae56407e732}]" +2025-01-25T14:40:48.249Z [INFO] server-79bdba1c-c406-3e87-6794-41b2a0aa7bf1: aborting pipeline replication: peer="{Voter server-cd537bb0-fc2d-c336-e312-6ae56407e732 cd537bb0-fc2d-c336-e312-6ae56407e732}" +--- PASS: TestRaft_CacheLogWithStoreError (0.34s) === RUN TestRaft_ReloadConfig -2025-01-25T14:29:57.397Z [INFO] server-e2808c35-767c-9e30-fb74-bf3df4ce3505: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e2808c35-767c-9e30-fb74-bf3df4ce3505 Address:e2808c35-767c-9e30-fb74-bf3df4ce3505}]" +2025-01-25T14:40:48.252Z [INFO] server-39251505-a05f-9f21-8e8e-d6416fe9cae9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-39251505-a05f-9f21-8e8e-d6416fe9cae9 Address:39251505-a05f-9f21-8e8e-d6416fe9cae9}]" +2025-01-25T14:40:48.252Z [INFO] server-39251505-a05f-9f21-8e8e-d6416fe9cae9: entering follower state: follower="Node at 39251505-a05f-9f21-8e8e-d6416fe9cae9 [Shutdown]" leader-address= leader-id= --- PASS: TestRaft_ReloadConfig (0.00s) === RUN TestRaft_ReloadConfigValidates -2025-01-25T14:29:57.398Z [INFO] server-037f4600-4e4a-125a-a7c3-a26b9121a9fb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-037f4600-4e4a-125a-a7c3-a26b9121a9fb Address:037f4600-4e4a-125a-a7c3-a26b9121a9fb}]" +2025-01-25T14:40:48.255Z [INFO] server-ffd6cae5-bc6a-a58d-6ece-a2784a0d4d5e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ffd6cae5-bc6a-a58d-6ece-a2784a0d4d5e Address:ffd6cae5-bc6a-a58d-6ece-a2784a0d4d5e}]" --- PASS: TestRaft_ReloadConfigValidates (0.00s) === RUN TestRaft_InstallSnapshot_InvalidPeers -2025-01-25T14:29:57.401Z [ERROR] failed to install snapshot: error="failed to decode peers: codec.decoder: Only encoded map or array can be decoded into a slice. (valueType: 2)" +2025-01-25T14:40:48.257Z [ERROR] failed to install snapshot: error="failed to decode peers: codec.decoder: Only encoded map or array can be decoded into a slice. (valueType: 2)" --- PASS: TestRaft_InstallSnapshot_InvalidPeers (0.00s) === RUN TestRaft_VoteNotGranted_WhenNodeNotInCluster -2025-01-25T14:29:57.402Z [INFO] server-db30f276-75c4-c6c3-032d-31803158d704: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-db30f276-75c4-c6c3-032d-31803158d704 Address:db30f276-75c4-c6c3-032d-31803158d704} {Suffrage:Voter ID:server-4f82702b-2dac-21b8-1eac-da805577c8d4 Address:4f82702b-2dac-21b8-1eac-da805577c8d4} {Suffrage:Voter ID:server-0575ecae-99f8-0ded-5a67-999ef60160c9 Address:0575ecae-99f8-0ded-5a67-999ef60160c9}]" -2025-01-25T14:29:57.402Z [INFO] server-4f82702b-2dac-21b8-1eac-da805577c8d4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-db30f276-75c4-c6c3-032d-31803158d704 Address:db30f276-75c4-c6c3-032d-31803158d704} {Suffrage:Voter ID:server-4f82702b-2dac-21b8-1eac-da805577c8d4 Address:4f82702b-2dac-21b8-1eac-da805577c8d4} {Suffrage:Voter ID:server-0575ecae-99f8-0ded-5a67-999ef60160c9 Address:0575ecae-99f8-0ded-5a67-999ef60160c9}]" -2025-01-25T14:29:57.402Z [INFO] server-0575ecae-99f8-0ded-5a67-999ef60160c9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-db30f276-75c4-c6c3-032d-31803158d704 Address:db30f276-75c4-c6c3-032d-31803158d704} {Suffrage:Voter ID:server-4f82702b-2dac-21b8-1eac-da805577c8d4 Address:4f82702b-2dac-21b8-1eac-da805577c8d4} {Suffrage:Voter ID:server-0575ecae-99f8-0ded-5a67-999ef60160c9 Address:0575ecae-99f8-0ded-5a67-999ef60160c9}]" -2025-01-25T14:29:57.403Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:57.403Z [INFO] server-4f82702b-2dac-21b8-1eac-da805577c8d4: entering follower state: follower="Node at 4f82702b-2dac-21b8-1eac-da805577c8d4 [Follower]" leader-address= leader-id= -2025-01-25T14:29:57.403Z [INFO] server-db30f276-75c4-c6c3-032d-31803158d704: entering follower state: follower="Node at db30f276-75c4-c6c3-032d-31803158d704 [Follower]" leader-address= leader-id= -2025-01-25T14:29:57.403Z [INFO] server-0575ecae-99f8-0ded-5a67-999ef60160c9: entering follower state: follower="Node at 0575ecae-99f8-0ded-5a67-999ef60160c9 [Follower]" leader-address= leader-id= -2025-01-25T14:29:57.469Z [WARN] server-4f82702b-2dac-21b8-1eac-da805577c8d4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:57.469Z [INFO] server-4f82702b-2dac-21b8-1eac-da805577c8d4: entering candidate state: node="Node at 4f82702b-2dac-21b8-1eac-da805577c8d4 [Candidate]" term=2 -2025-01-25T14:29:57.470Z [INFO] server-4f82702b-2dac-21b8-1eac-da805577c8d4: election won: term=2 tally=2 -2025-01-25T14:29:57.470Z [INFO] server-4f82702b-2dac-21b8-1eac-da805577c8d4: entering leader state: leader="Node at 4f82702b-2dac-21b8-1eac-da805577c8d4 [Leader]" -2025-01-25T14:29:57.470Z [INFO] server-4f82702b-2dac-21b8-1eac-da805577c8d4: added peer, starting replication: peer=server-db30f276-75c4-c6c3-032d-31803158d704 -2025-01-25T14:29:57.470Z [INFO] server-4f82702b-2dac-21b8-1eac-da805577c8d4: added peer, starting replication: peer=server-0575ecae-99f8-0ded-5a67-999ef60160c9 -2025-01-25T14:29:57.470Z [WARN] server-0575ecae-99f8-0ded-5a67-999ef60160c9: rejecting vote request since our last term is greater: candidate=4f82702b-2dac-21b8-1eac-da805577c8d4 last-term=2 last-candidate-term=1 -2025-01-25T14:29:57.470Z [INFO] server-4f82702b-2dac-21b8-1eac-da805577c8d4: pipelining replication: peer="{Voter server-db30f276-75c4-c6c3-032d-31803158d704 db30f276-75c4-c6c3-032d-31803158d704}" -2025-01-25T14:29:57.470Z [INFO] server-4f82702b-2dac-21b8-1eac-da805577c8d4: pipelining replication: peer="{Voter server-0575ecae-99f8-0ded-5a67-999ef60160c9 0575ecae-99f8-0ded-5a67-999ef60160c9}" -2025-01-25T14:29:57.575Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:57.470599216 +0000 UTC m=+107.324924198 (1 nodes), 66.803596ms from start of poll, 68.205305ms from cluster start. Timeout at 2025-01-25 14:29:57.575609336 +0000 UTC m=+107.429934398, 105.0102ms after stability -2025-01-25T14:29:57.581Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:29:57.687Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:29:57.581459787 +0000 UTC m=+107.435784729 (2 nodes), 2.56µs from start of poll, 179.065836ms from cluster start. Timeout at 2025-01-25 14:29:57.686678019 +0000 UTC m=+107.541002961, 105.218232ms after stability -2025-01-25T14:29:57.687Z [INFO] server-4f82702b-2dac-21b8-1eac-da805577c8d4: updating configuration: command=RemoveServer server-id=server-db30f276-75c4-c6c3-032d-31803158d704 server-addr= servers="[{Suffrage:Voter ID:server-4f82702b-2dac-21b8-1eac-da805577c8d4 Address:4f82702b-2dac-21b8-1eac-da805577c8d4} {Suffrage:Voter ID:server-0575ecae-99f8-0ded-5a67-999ef60160c9 Address:0575ecae-99f8-0ded-5a67-999ef60160c9}]" -2025-01-25T14:29:57.687Z [INFO] server-4f82702b-2dac-21b8-1eac-da805577c8d4: removed peer, stopping replication: peer=server-db30f276-75c4-c6c3-032d-31803158d704 last-index=3 -2025-01-25T14:29:57.687Z [INFO] server-4f82702b-2dac-21b8-1eac-da805577c8d4: aborting pipeline replication: peer="{Voter server-db30f276-75c4-c6c3-032d-31803158d704 db30f276-75c4-c6c3-032d-31803158d704}" -2025-01-25T14:29:57.743Z [WARN] server-db30f276-75c4-c6c3-032d-31803158d704: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election -2025-01-25T14:29:57.807Z [ERROR] server-4f82702b-2dac-21b8-1eac-da805577c8d4: failed to heartbeat to: peer=0575ecae-99f8-0ded-5a67-999ef60160c9 backoff time=10ms error="failed to connect to peer: 0575ecae-99f8-0ded-5a67-999ef60160c9" -2025-01-25T14:29:57.810Z [ERROR] server-4f82702b-2dac-21b8-1eac-da805577c8d4: failed to pipeline appendEntries: peer="{Voter server-0575ecae-99f8-0ded-5a67-999ef60160c9 0575ecae-99f8-0ded-5a67-999ef60160c9}" error="append pipeline closed" -2025-01-25T14:29:57.811Z [INFO] server-4f82702b-2dac-21b8-1eac-da805577c8d4: aborting pipeline replication: peer="{Voter server-0575ecae-99f8-0ded-5a67-999ef60160c9 0575ecae-99f8-0ded-5a67-999ef60160c9}" -2025-01-25T14:29:57.817Z [ERROR] server-4f82702b-2dac-21b8-1eac-da805577c8d4: failed to appendEntries to: peer="{Voter server-0575ecae-99f8-0ded-5a67-999ef60160c9 0575ecae-99f8-0ded-5a67-999ef60160c9}" error="failed to connect to peer: 0575ecae-99f8-0ded-5a67-999ef60160c9" -2025-01-25T14:29:57.827Z [ERROR] server-4f82702b-2dac-21b8-1eac-da805577c8d4: failed to heartbeat to: peer=0575ecae-99f8-0ded-5a67-999ef60160c9 backoff time=10ms error="failed to connect to peer: 0575ecae-99f8-0ded-5a67-999ef60160c9" -2025-01-25T14:29:57.839Z [ERROR] server-4f82702b-2dac-21b8-1eac-da805577c8d4: failed to appendEntries to: peer="{Voter server-0575ecae-99f8-0ded-5a67-999ef60160c9 0575ecae-99f8-0ded-5a67-999ef60160c9}" error="failed to connect to peer: 0575ecae-99f8-0ded-5a67-999ef60160c9" -2025-01-25T14:29:57.848Z [ERROR] server-4f82702b-2dac-21b8-1eac-da805577c8d4: failed to heartbeat to: peer=0575ecae-99f8-0ded-5a67-999ef60160c9 backoff time=10ms error="failed to connect to peer: 0575ecae-99f8-0ded-5a67-999ef60160c9" -2025-01-25T14:29:57.853Z [WARN] server-4f82702b-2dac-21b8-1eac-da805577c8d4: failed to contact: server-id=server-0575ecae-99f8-0ded-5a67-999ef60160c9 time=51.099201ms -2025-01-25T14:29:57.853Z [WARN] server-4f82702b-2dac-21b8-1eac-da805577c8d4: failed to contact quorum of nodes, stepping down -2025-01-25T14:29:57.853Z [INFO] server-4f82702b-2dac-21b8-1eac-da805577c8d4: entering follower state: follower="Node at 4f82702b-2dac-21b8-1eac-da805577c8d4 [Follower]" leader-address= leader-id= -2025-01-25T14:29:57.855Z [ERROR] server-4f82702b-2dac-21b8-1eac-da805577c8d4: failed to appendEntries to: peer="{Voter server-0575ecae-99f8-0ded-5a67-999ef60160c9 0575ecae-99f8-0ded-5a67-999ef60160c9}" error="failed to connect to peer: 0575ecae-99f8-0ded-5a67-999ef60160c9" -2025-01-25T14:29:57.915Z [WARN] server-0575ecae-99f8-0ded-5a67-999ef60160c9: heartbeat timeout reached, starting election: last-leader-addr=4f82702b-2dac-21b8-1eac-da805577c8d4 last-leader-id=server-4f82702b-2dac-21b8-1eac-da805577c8d4 -2025-01-25T14:29:57.915Z [INFO] server-0575ecae-99f8-0ded-5a67-999ef60160c9: entering candidate state: node="Node at 0575ecae-99f8-0ded-5a67-999ef60160c9 [Candidate]" term=3 -2025-01-25T14:29:57.915Z [ERROR] server-0575ecae-99f8-0ded-5a67-999ef60160c9: failed to make requestVote RPC: target="{Voter server-4f82702b-2dac-21b8-1eac-da805577c8d4 4f82702b-2dac-21b8-1eac-da805577c8d4}" error="failed to connect to peer: 4f82702b-2dac-21b8-1eac-da805577c8d4" term=3 -2025-01-25T14:29:57.917Z [WARN] server-0575ecae-99f8-0ded-5a67-999ef60160c9: rejecting vote request since node is not in configuration: from=db30f276-75c4-c6c3-032d-31803158d704 ---- PASS: TestRaft_VoteNotGranted_WhenNodeNotInCluster (0.52s) +2025-01-25T14:40:48.260Z [INFO] server-ab8911e8-013d-4c9a-95d4-348df2d389d0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ab8911e8-013d-4c9a-95d4-348df2d389d0 Address:ab8911e8-013d-4c9a-95d4-348df2d389d0} {Suffrage:Voter ID:server-7f8830fa-57d7-6279-dbf7-008d76d8d140 Address:7f8830fa-57d7-6279-dbf7-008d76d8d140} {Suffrage:Voter ID:server-d3e934a2-7f5d-0167-9c46-45db9df9ab60 Address:d3e934a2-7f5d-0167-9c46-45db9df9ab60}]" +2025-01-25T14:40:48.261Z [INFO] server-ab8911e8-013d-4c9a-95d4-348df2d389d0: entering follower state: follower="Node at ab8911e8-013d-4c9a-95d4-348df2d389d0 [Follower]" leader-address= leader-id= +2025-01-25T14:40:48.261Z [INFO] server-7f8830fa-57d7-6279-dbf7-008d76d8d140: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ab8911e8-013d-4c9a-95d4-348df2d389d0 Address:ab8911e8-013d-4c9a-95d4-348df2d389d0} {Suffrage:Voter ID:server-7f8830fa-57d7-6279-dbf7-008d76d8d140 Address:7f8830fa-57d7-6279-dbf7-008d76d8d140} {Suffrage:Voter ID:server-d3e934a2-7f5d-0167-9c46-45db9df9ab60 Address:d3e934a2-7f5d-0167-9c46-45db9df9ab60}]" +2025-01-25T14:40:48.261Z [INFO] server-7f8830fa-57d7-6279-dbf7-008d76d8d140: entering follower state: follower="Node at 7f8830fa-57d7-6279-dbf7-008d76d8d140 [Follower]" leader-address= leader-id= +2025-01-25T14:40:48.261Z [INFO] server-d3e934a2-7f5d-0167-9c46-45db9df9ab60: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ab8911e8-013d-4c9a-95d4-348df2d389d0 Address:ab8911e8-013d-4c9a-95d4-348df2d389d0} {Suffrage:Voter ID:server-7f8830fa-57d7-6279-dbf7-008d76d8d140 Address:7f8830fa-57d7-6279-dbf7-008d76d8d140} {Suffrage:Voter ID:server-d3e934a2-7f5d-0167-9c46-45db9df9ab60 Address:d3e934a2-7f5d-0167-9c46-45db9df9ab60}]" +2025-01-25T14:40:48.262Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:48.262Z [INFO] server-d3e934a2-7f5d-0167-9c46-45db9df9ab60: entering follower state: follower="Node at d3e934a2-7f5d-0167-9c46-45db9df9ab60 [Follower]" leader-address= leader-id= +2025-01-25T14:40:48.347Z [WARN] server-d3e934a2-7f5d-0167-9c46-45db9df9ab60: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:48.347Z [INFO] server-d3e934a2-7f5d-0167-9c46-45db9df9ab60: entering candidate state: node="Node at d3e934a2-7f5d-0167-9c46-45db9df9ab60 [Candidate]" term=2 +2025-01-25T14:40:48.348Z [INFO] server-d3e934a2-7f5d-0167-9c46-45db9df9ab60: election won: term=2 tally=2 +2025-01-25T14:40:48.348Z [INFO] server-d3e934a2-7f5d-0167-9c46-45db9df9ab60: entering leader state: leader="Node at d3e934a2-7f5d-0167-9c46-45db9df9ab60 [Leader]" +2025-01-25T14:40:48.348Z [INFO] server-d3e934a2-7f5d-0167-9c46-45db9df9ab60: added peer, starting replication: peer=server-ab8911e8-013d-4c9a-95d4-348df2d389d0 +2025-01-25T14:40:48.348Z [INFO] server-d3e934a2-7f5d-0167-9c46-45db9df9ab60: added peer, starting replication: peer=server-7f8830fa-57d7-6279-dbf7-008d76d8d140 +2025-01-25T14:40:48.348Z [INFO] server-d3e934a2-7f5d-0167-9c46-45db9df9ab60: pipelining replication: peer="{Voter server-ab8911e8-013d-4c9a-95d4-348df2d389d0 ab8911e8-013d-4c9a-95d4-348df2d389d0}" +2025-01-25T14:40:48.349Z [INFO] server-d3e934a2-7f5d-0167-9c46-45db9df9ab60: pipelining replication: peer="{Voter server-7f8830fa-57d7-6279-dbf7-008d76d8d140 7f8830fa-57d7-6279-dbf7-008d76d8d140}" +2025-01-25T14:40:48.453Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:48.348287296 +0000 UTC m=+117.394126614 (1 nodes), 86.088584ms from start of poll, 87.830131ms from cluster start. Timeout at 2025-01-25 14:40:48.453301879 +0000 UTC m=+117.499141197, 105.014583ms after stability +2025-01-25T14:40:48.459Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:40:48.565Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:40:48.459388119 +0000 UTC m=+117.505227437 (2 nodes), 6.708µs from start of poll, 198.930954ms from cluster start. Timeout at 2025-01-25 14:40:48.564572161 +0000 UTC m=+117.610411479, 105.184042ms after stability +2025-01-25T14:40:48.565Z [INFO] server-d3e934a2-7f5d-0167-9c46-45db9df9ab60: updating configuration: command=RemoveServer server-id=server-ab8911e8-013d-4c9a-95d4-348df2d389d0 server-addr= servers="[{Suffrage:Voter ID:server-7f8830fa-57d7-6279-dbf7-008d76d8d140 Address:7f8830fa-57d7-6279-dbf7-008d76d8d140} {Suffrage:Voter ID:server-d3e934a2-7f5d-0167-9c46-45db9df9ab60 Address:d3e934a2-7f5d-0167-9c46-45db9df9ab60}]" +2025-01-25T14:40:48.566Z [INFO] server-d3e934a2-7f5d-0167-9c46-45db9df9ab60: removed peer, stopping replication: peer=server-ab8911e8-013d-4c9a-95d4-348df2d389d0 last-index=3 +2025-01-25T14:40:48.566Z [INFO] server-d3e934a2-7f5d-0167-9c46-45db9df9ab60: aborting pipeline replication: peer="{Voter server-ab8911e8-013d-4c9a-95d4-348df2d389d0 ab8911e8-013d-4c9a-95d4-348df2d389d0}" +2025-01-25T14:40:48.618Z [WARN] server-ab8911e8-013d-4c9a-95d4-348df2d389d0: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +2025-01-25T14:40:48.672Z [ERROR] server-d3e934a2-7f5d-0167-9c46-45db9df9ab60: failed to pipeline appendEntries: peer="{Voter server-7f8830fa-57d7-6279-dbf7-008d76d8d140 7f8830fa-57d7-6279-dbf7-008d76d8d140}" error="append pipeline closed" +2025-01-25T14:40:48.672Z [INFO] server-d3e934a2-7f5d-0167-9c46-45db9df9ab60: aborting pipeline replication: peer="{Voter server-7f8830fa-57d7-6279-dbf7-008d76d8d140 7f8830fa-57d7-6279-dbf7-008d76d8d140}" +2025-01-25T14:40:48.675Z [ERROR] server-d3e934a2-7f5d-0167-9c46-45db9df9ab60: failed to heartbeat to: peer=7f8830fa-57d7-6279-dbf7-008d76d8d140 backoff time=10ms error="failed to connect to peer: 7f8830fa-57d7-6279-dbf7-008d76d8d140" +2025-01-25T14:40:48.679Z [ERROR] server-d3e934a2-7f5d-0167-9c46-45db9df9ab60: failed to appendEntries to: peer="{Voter server-7f8830fa-57d7-6279-dbf7-008d76d8d140 7f8830fa-57d7-6279-dbf7-008d76d8d140}" error="failed to connect to peer: 7f8830fa-57d7-6279-dbf7-008d76d8d140" +2025-01-25T14:40:48.694Z [ERROR] server-d3e934a2-7f5d-0167-9c46-45db9df9ab60: failed to heartbeat to: peer=7f8830fa-57d7-6279-dbf7-008d76d8d140 backoff time=10ms error="failed to connect to peer: 7f8830fa-57d7-6279-dbf7-008d76d8d140" +2025-01-25T14:40:48.701Z [ERROR] server-d3e934a2-7f5d-0167-9c46-45db9df9ab60: failed to appendEntries to: peer="{Voter server-7f8830fa-57d7-6279-dbf7-008d76d8d140 7f8830fa-57d7-6279-dbf7-008d76d8d140}" error="failed to connect to peer: 7f8830fa-57d7-6279-dbf7-008d76d8d140" +2025-01-25T14:40:48.712Z [ERROR] server-d3e934a2-7f5d-0167-9c46-45db9df9ab60: failed to heartbeat to: peer=7f8830fa-57d7-6279-dbf7-008d76d8d140 backoff time=10ms error="failed to connect to peer: 7f8830fa-57d7-6279-dbf7-008d76d8d140" +2025-01-25T14:40:48.718Z [WARN] server-d3e934a2-7f5d-0167-9c46-45db9df9ab60: failed to contact: server-id=server-7f8830fa-57d7-6279-dbf7-008d76d8d140 time=50.272272ms +2025-01-25T14:40:48.718Z [WARN] server-d3e934a2-7f5d-0167-9c46-45db9df9ab60: failed to contact quorum of nodes, stepping down +2025-01-25T14:40:48.718Z [INFO] server-d3e934a2-7f5d-0167-9c46-45db9df9ab60: entering follower state: follower="Node at d3e934a2-7f5d-0167-9c46-45db9df9ab60 [Follower]" leader-address= leader-id= +2025-01-25T14:40:48.719Z [ERROR] server-d3e934a2-7f5d-0167-9c46-45db9df9ab60: failed to appendEntries to: peer="{Voter server-7f8830fa-57d7-6279-dbf7-008d76d8d140 7f8830fa-57d7-6279-dbf7-008d76d8d140}" error="failed to connect to peer: 7f8830fa-57d7-6279-dbf7-008d76d8d140" +2025-01-25T14:40:48.785Z [WARN] server-d3e934a2-7f5d-0167-9c46-45db9df9ab60: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:48.785Z [INFO] server-d3e934a2-7f5d-0167-9c46-45db9df9ab60: entering candidate state: node="Node at d3e934a2-7f5d-0167-9c46-45db9df9ab60 [Candidate]" term=3 +2025-01-25T14:40:48.786Z [ERROR] server-d3e934a2-7f5d-0167-9c46-45db9df9ab60: failed to make requestVote RPC: target="{Voter server-7f8830fa-57d7-6279-dbf7-008d76d8d140 7f8830fa-57d7-6279-dbf7-008d76d8d140}" error="failed to connect to peer: 7f8830fa-57d7-6279-dbf7-008d76d8d140" term=3 +2025-01-25T14:40:48.803Z [WARN] server-7f8830fa-57d7-6279-dbf7-008d76d8d140: heartbeat timeout reached, starting election: last-leader-addr=d3e934a2-7f5d-0167-9c46-45db9df9ab60 last-leader-id=server-d3e934a2-7f5d-0167-9c46-45db9df9ab60 +2025-01-25T14:40:48.804Z [INFO] server-7f8830fa-57d7-6279-dbf7-008d76d8d140: entering candidate state: node="Node at 7f8830fa-57d7-6279-dbf7-008d76d8d140 [Candidate]" term=3 +2025-01-25T14:40:48.804Z [ERROR] server-7f8830fa-57d7-6279-dbf7-008d76d8d140: failed to make requestVote RPC: target="{Voter server-d3e934a2-7f5d-0167-9c46-45db9df9ab60 d3e934a2-7f5d-0167-9c46-45db9df9ab60}" error="failed to connect to peer: d3e934a2-7f5d-0167-9c46-45db9df9ab60" term=3 +2025-01-25T14:40:48.805Z [WARN] server-7f8830fa-57d7-6279-dbf7-008d76d8d140: rejecting vote request since node is not in configuration: from=ab8911e8-013d-4c9a-95d4-348df2d389d0 +--- PASS: TestRaft_VoteNotGranted_WhenNodeNotInCluster (0.55s) === RUN TestRaft_ClusterCanRegainStability_WhenNonVoterWithHigherTermJoin -2025-01-25T14:29:57.920Z [INFO] server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 Address:a66f840e-ffe7-9536-7fc6-7f7da39d6cb3} {Suffrage:Voter ID:server-05bd9c38-1487-3899-8245-14972d6639ff Address:05bd9c38-1487-3899-8245-14972d6639ff} {Suffrage:Voter ID:server-96992678-6d6d-da45-643c-8131ddef872a Address:96992678-6d6d-da45-643c-8131ddef872a}]" -2025-01-25T14:29:57.920Z [INFO] server-05bd9c38-1487-3899-8245-14972d6639ff: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 Address:a66f840e-ffe7-9536-7fc6-7f7da39d6cb3} {Suffrage:Voter ID:server-05bd9c38-1487-3899-8245-14972d6639ff Address:05bd9c38-1487-3899-8245-14972d6639ff} {Suffrage:Voter ID:server-96992678-6d6d-da45-643c-8131ddef872a Address:96992678-6d6d-da45-643c-8131ddef872a}]" -2025-01-25T14:29:57.920Z [INFO] server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3: entering follower state: follower="Node at a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 [Follower]" leader-address= leader-id= -2025-01-25T14:29:57.920Z [INFO] server-96992678-6d6d-da45-643c-8131ddef872a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 Address:a66f840e-ffe7-9536-7fc6-7f7da39d6cb3} {Suffrage:Voter ID:server-05bd9c38-1487-3899-8245-14972d6639ff Address:05bd9c38-1487-3899-8245-14972d6639ff} {Suffrage:Voter ID:server-96992678-6d6d-da45-643c-8131ddef872a Address:96992678-6d6d-da45-643c-8131ddef872a}]" -2025-01-25T14:29:57.920Z [INFO] server-05bd9c38-1487-3899-8245-14972d6639ff: entering follower state: follower="Node at 05bd9c38-1487-3899-8245-14972d6639ff [Follower]" leader-address= leader-id= -2025-01-25T14:29:57.920Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:57.920Z [INFO] server-96992678-6d6d-da45-643c-8131ddef872a: entering follower state: follower="Node at 96992678-6d6d-da45-643c-8131ddef872a [Follower]" leader-address= leader-id= -2025-01-25T14:29:57.979Z [WARN] server-96992678-6d6d-da45-643c-8131ddef872a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:57.979Z [INFO] server-96992678-6d6d-da45-643c-8131ddef872a: entering candidate state: node="Node at 96992678-6d6d-da45-643c-8131ddef872a [Candidate]" term=2 -2025-01-25T14:29:57.979Z [INFO] server-96992678-6d6d-da45-643c-8131ddef872a: election won: term=2 tally=2 -2025-01-25T14:29:57.979Z [INFO] server-96992678-6d6d-da45-643c-8131ddef872a: entering leader state: leader="Node at 96992678-6d6d-da45-643c-8131ddef872a [Leader]" -2025-01-25T14:29:57.979Z [INFO] server-96992678-6d6d-da45-643c-8131ddef872a: added peer, starting replication: peer=server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 -2025-01-25T14:29:57.979Z [INFO] server-96992678-6d6d-da45-643c-8131ddef872a: added peer, starting replication: peer=server-05bd9c38-1487-3899-8245-14972d6639ff -2025-01-25T14:29:57.979Z [INFO] server-96992678-6d6d-da45-643c-8131ddef872a: pipelining replication: peer="{Voter server-05bd9c38-1487-3899-8245-14972d6639ff 05bd9c38-1487-3899-8245-14972d6639ff}" -2025-01-25T14:29:57.980Z [INFO] server-96992678-6d6d-da45-643c-8131ddef872a: pipelining replication: peer="{Voter server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 a66f840e-ffe7-9536-7fc6-7f7da39d6cb3}" -2025-01-25T14:29:58.086Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:57.979859049 +0000 UTC m=+107.834183991 (1 nodes), 58.935159ms from start of poll, 59.720331ms from cluster start. Timeout at 2025-01-25 14:29:58.084867249 +0000 UTC m=+107.939192231, 105.00824ms after stability -2025-01-25T14:29:58.091Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:29:58.214Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:29:58.091901377 +0000 UTC m=+107.946226359 (2 nodes), 2.48µs from start of poll, 171.762699ms from cluster start. Timeout at 2025-01-25 14:29:58.197279764 +0000 UTC m=+108.051604746, 105.378387ms after stability -2025-01-25T14:29:58.220Z [ERROR] server-96992678-6d6d-da45-643c-8131ddef872a: failed to pipeline appendEntries: peer="{Voter server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 a66f840e-ffe7-9536-7fc6-7f7da39d6cb3}" error="append pipeline closed" -2025-01-25T14:29:58.220Z [INFO] server-96992678-6d6d-da45-643c-8131ddef872a: aborting pipeline replication: peer="{Voter server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 a66f840e-ffe7-9536-7fc6-7f7da39d6cb3}" -2025-01-25T14:29:58.222Z [ERROR] server-96992678-6d6d-da45-643c-8131ddef872a: failed to heartbeat to: peer=a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 backoff time=10ms error="failed to connect to peer: a66f840e-ffe7-9536-7fc6-7f7da39d6cb3" -2025-01-25T14:29:58.227Z [ERROR] server-96992678-6d6d-da45-643c-8131ddef872a: failed to appendEntries to: peer="{Voter server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 a66f840e-ffe7-9536-7fc6-7f7da39d6cb3}" error="failed to connect to peer: a66f840e-ffe7-9536-7fc6-7f7da39d6cb3" -2025-01-25T14:29:58.240Z [ERROR] server-96992678-6d6d-da45-643c-8131ddef872a: failed to heartbeat to: peer=a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 backoff time=10ms error="failed to connect to peer: a66f840e-ffe7-9536-7fc6-7f7da39d6cb3" -2025-01-25T14:29:58.244Z [ERROR] server-96992678-6d6d-da45-643c-8131ddef872a: failed to appendEntries to: peer="{Voter server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 a66f840e-ffe7-9536-7fc6-7f7da39d6cb3}" error="failed to connect to peer: a66f840e-ffe7-9536-7fc6-7f7da39d6cb3" -2025-01-25T14:29:58.259Z [ERROR] server-96992678-6d6d-da45-643c-8131ddef872a: failed to heartbeat to: peer=a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 backoff time=10ms error="failed to connect to peer: a66f840e-ffe7-9536-7fc6-7f7da39d6cb3" -2025-01-25T14:29:58.262Z [ERROR] server-96992678-6d6d-da45-643c-8131ddef872a: failed to appendEntries to: peer="{Voter server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 a66f840e-ffe7-9536-7fc6-7f7da39d6cb3}" error="failed to connect to peer: a66f840e-ffe7-9536-7fc6-7f7da39d6cb3" -2025-01-25T14:29:58.275Z [WARN] server-96992678-6d6d-da45-643c-8131ddef872a: failed to contact: server-id=server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 time=60.891249ms -2025-01-25T14:29:58.277Z [ERROR] server-96992678-6d6d-da45-643c-8131ddef872a: failed to heartbeat to: peer=a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 backoff time=20ms error="failed to connect to peer: a66f840e-ffe7-9536-7fc6-7f7da39d6cb3" -2025-01-25T14:29:58.292Z [ERROR] server-96992678-6d6d-da45-643c-8131ddef872a: failed to appendEntries to: peer="{Voter server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 a66f840e-ffe7-9536-7fc6-7f7da39d6cb3}" error="failed to connect to peer: a66f840e-ffe7-9536-7fc6-7f7da39d6cb3" -2025-01-25T14:29:58.308Z [ERROR] server-96992678-6d6d-da45-643c-8131ddef872a: failed to heartbeat to: peer=a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 backoff time=25ms error="failed to connect to peer: a66f840e-ffe7-9536-7fc6-7f7da39d6cb3" -2025-01-25T14:29:58.319Z [INFO] server-96992678-6d6d-da45-643c-8131ddef872a: updating configuration: command=RemoveServer server-id=server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 server-addr= servers="[{Suffrage:Voter ID:server-05bd9c38-1487-3899-8245-14972d6639ff Address:05bd9c38-1487-3899-8245-14972d6639ff} {Suffrage:Voter ID:server-96992678-6d6d-da45-643c-8131ddef872a Address:96992678-6d6d-da45-643c-8131ddef872a}]" -2025-01-25T14:29:58.320Z [INFO] server-96992678-6d6d-da45-643c-8131ddef872a: removed peer, stopping replication: peer=server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 last-index=3 -2025-01-25T14:29:58.320Z [INFO] server-96992678-6d6d-da45-643c-8131ddef872a: updating configuration: command=AddNonvoter server-id=server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 server-addr=a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 servers="[{Suffrage:Voter ID:server-05bd9c38-1487-3899-8245-14972d6639ff Address:05bd9c38-1487-3899-8245-14972d6639ff} {Suffrage:Voter ID:server-96992678-6d6d-da45-643c-8131ddef872a Address:96992678-6d6d-da45-643c-8131ddef872a} {Suffrage:Nonvoter ID:server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 Address:a66f840e-ffe7-9536-7fc6-7f7da39d6cb3}]" -2025-01-25T14:29:58.320Z [INFO] server-96992678-6d6d-da45-643c-8131ddef872a: added peer, starting replication: peer=server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 -2025-01-25T14:29:58.320Z [ERROR] server-96992678-6d6d-da45-643c-8131ddef872a: failed to appendEntries to: peer="{Nonvoter server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 a66f840e-ffe7-9536-7fc6-7f7da39d6cb3}" error="failed to connect to peer: a66f840e-ffe7-9536-7fc6-7f7da39d6cb3" -2025-01-25T14:29:58.321Z [WARN] server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3: heartbeat timeout reached, starting election: last-leader-addr=96992678-6d6d-da45-643c-8131ddef872a last-leader-id=server-96992678-6d6d-da45-643c-8131ddef872a -2025-01-25T14:29:58.321Z [INFO] server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3: entering candidate state: node="Node at a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 [Candidate]" term=23 -2025-01-25T14:29:58.321Z [WARN] server-96992678-6d6d-da45-643c-8131ddef872a: rejecting vote request since we have a leader: from=a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 leader=96992678-6d6d-da45-643c-8131ddef872a leader-id=server-96992678-6d6d-da45-643c-8131ddef872a -2025-01-25T14:29:58.321Z [WARN] server-05bd9c38-1487-3899-8245-14972d6639ff: rejecting vote request since we have a leader: from=a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 leader=96992678-6d6d-da45-643c-8131ddef872a leader-id=server-96992678-6d6d-da45-643c-8131ddef872a -2025-01-25T14:29:58.340Z [ERROR] server-96992678-6d6d-da45-643c-8131ddef872a: peer has newer term, stopping replication: peer="{Nonvoter server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 a66f840e-ffe7-9536-7fc6-7f7da39d6cb3}" -2025-01-25T14:29:58.340Z [INFO] server-96992678-6d6d-da45-643c-8131ddef872a: entering follower state: follower="Node at 96992678-6d6d-da45-643c-8131ddef872a [Follower]" leader-address= leader-id= -2025-01-25T14:29:58.340Z [INFO] server-96992678-6d6d-da45-643c-8131ddef872a: aborting pipeline replication: peer="{Voter server-05bd9c38-1487-3899-8245-14972d6639ff 05bd9c38-1487-3899-8245-14972d6639ff}" -2025-01-25T14:29:58.343Z [ERROR] server-96992678-6d6d-da45-643c-8131ddef872a: peer has newer term, stopping replication: peer="{Voter server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 a66f840e-ffe7-9536-7fc6-7f7da39d6cb3}" -2025-01-25T14:29:58.406Z [WARN] server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3: Election timeout reached, restarting election -2025-01-25T14:29:58.407Z [INFO] server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3: entering candidate state: node="Node at a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 [Candidate]" term=24 -2025-01-25T14:29:58.407Z [WARN] server-96992678-6d6d-da45-643c-8131ddef872a: rejecting vote request since node is not a voter: from=a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 -2025-01-25T14:29:58.407Z [WARN] server-05bd9c38-1487-3899-8245-14972d6639ff: rejecting vote request since we have a leader: from=a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 leader=96992678-6d6d-da45-643c-8131ddef872a leader-id=server-96992678-6d6d-da45-643c-8131ddef872a -2025-01-25T14:29:58.426Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:58.434Z [WARN] server-96992678-6d6d-da45-643c-8131ddef872a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:58.434Z [INFO] server-96992678-6d6d-da45-643c-8131ddef872a: entering candidate state: node="Node at 96992678-6d6d-da45-643c-8131ddef872a [Candidate]" term=25 -2025-01-25T14:29:58.434Z [INFO] server-96992678-6d6d-da45-643c-8131ddef872a: election won: term=25 tally=2 -2025-01-25T14:29:58.434Z [INFO] server-96992678-6d6d-da45-643c-8131ddef872a: entering leader state: leader="Node at 96992678-6d6d-da45-643c-8131ddef872a [Leader]" -2025-01-25T14:29:58.434Z [INFO] server-96992678-6d6d-da45-643c-8131ddef872a: added peer, starting replication: peer=server-05bd9c38-1487-3899-8245-14972d6639ff -2025-01-25T14:29:58.434Z [INFO] server-96992678-6d6d-da45-643c-8131ddef872a: added peer, starting replication: peer=server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 -2025-01-25T14:29:58.435Z [WARN] server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3: failed to get previous log: previous-index=4 last-index=2 error="log not found" -2025-01-25T14:29:58.435Z [INFO] server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3: entering follower state: follower="Node at a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 [Follower]" leader-address=96992678-6d6d-da45-643c-8131ddef872a leader-id=server-96992678-6d6d-da45-643c-8131ddef872a -2025-01-25T14:29:58.435Z [WARN] server-96992678-6d6d-da45-643c-8131ddef872a: appendEntries rejected, sending older logs: peer="{Nonvoter server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 a66f840e-ffe7-9536-7fc6-7f7da39d6cb3}" next=3 -2025-01-25T14:29:58.435Z [INFO] server-96992678-6d6d-da45-643c-8131ddef872a: pipelining replication: peer="{Voter server-05bd9c38-1487-3899-8245-14972d6639ff 05bd9c38-1487-3899-8245-14972d6639ff}" -2025-01-25T14:29:58.436Z [INFO] server-96992678-6d6d-da45-643c-8131ddef872a: pipelining replication: peer="{Nonvoter server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 a66f840e-ffe7-9536-7fc6-7f7da39d6cb3}" -2025-01-25T14:29:58.543Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:58.435502092 +0000 UTC m=+108.289827074 (1 nodes), 8.9168ms from start of poll, 515.363414ms from cluster start. Timeout at 2025-01-25 14:29:58.54054785 +0000 UTC m=+108.394872832, 105.045758ms after stability -2025-01-25T14:29:58.613Z [INFO] server-96992678-6d6d-da45-643c-8131ddef872a: updating configuration: command=RemoveServer server-id=server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 server-addr= servers="[{Suffrage:Voter ID:server-05bd9c38-1487-3899-8245-14972d6639ff Address:05bd9c38-1487-3899-8245-14972d6639ff} {Suffrage:Voter ID:server-96992678-6d6d-da45-643c-8131ddef872a Address:96992678-6d6d-da45-643c-8131ddef872a}]" -2025-01-25T14:29:58.613Z [INFO] server-96992678-6d6d-da45-643c-8131ddef872a: removed peer, stopping replication: peer=server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 last-index=106 -2025-01-25T14:29:58.613Z [INFO] server-96992678-6d6d-da45-643c-8131ddef872a: aborting pipeline replication: peer="{Nonvoter server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 a66f840e-ffe7-9536-7fc6-7f7da39d6cb3}" -2025-01-25T14:29:58.613Z [INFO] server-96992678-6d6d-da45-643c-8131ddef872a: updating configuration: command=AddVoter server-id=server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 server-addr=a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 servers="[{Suffrage:Voter ID:server-05bd9c38-1487-3899-8245-14972d6639ff Address:05bd9c38-1487-3899-8245-14972d6639ff} {Suffrage:Voter ID:server-96992678-6d6d-da45-643c-8131ddef872a Address:96992678-6d6d-da45-643c-8131ddef872a} {Suffrage:Voter ID:server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 Address:a66f840e-ffe7-9536-7fc6-7f7da39d6cb3}]" -2025-01-25T14:29:58.613Z [INFO] server-96992678-6d6d-da45-643c-8131ddef872a: added peer, starting replication: peer=server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 -2025-01-25T14:29:58.614Z [WARN] server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3: failed to get previous log: previous-index=107 last-index=106 error="log not found" -2025-01-25T14:29:58.614Z [WARN] server-96992678-6d6d-da45-643c-8131ddef872a: appendEntries rejected, sending older logs: peer="{Voter server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 a66f840e-ffe7-9536-7fc6-7f7da39d6cb3}" next=107 -2025-01-25T14:29:58.614Z [INFO] server-96992678-6d6d-da45-643c-8131ddef872a: pipelining replication: peer="{Voter server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 a66f840e-ffe7-9536-7fc6-7f7da39d6cb3}" -2025-01-25T14:29:59.728Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:59.833Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:59.728071735 +0000 UTC m=+109.582396717 (1 nodes), 1.76µs from start of poll, 1.807933057s from cluster start. Timeout at 2025-01-25 14:29:59.833370444 +0000 UTC m=+109.687695426, 105.298709ms after stability -2025-01-25T14:29:59.833Z [INFO] server-96992678-6d6d-da45-643c-8131ddef872a: aborting pipeline replication: peer="{Voter server-a66f840e-ffe7-9536-7fc6-7f7da39d6cb3 a66f840e-ffe7-9536-7fc6-7f7da39d6cb3}" -2025-01-25T14:29:59.833Z [INFO] server-96992678-6d6d-da45-643c-8131ddef872a: aborting pipeline replication: peer="{Voter server-05bd9c38-1487-3899-8245-14972d6639ff 05bd9c38-1487-3899-8245-14972d6639ff}" ---- PASS: TestRaft_ClusterCanRegainStability_WhenNonVoterWithHigherTermJoin (1.92s) +2025-01-25T14:40:48.813Z [INFO] server-6955b61c-3383-bea1-9aff-3e9ad83265f3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6955b61c-3383-bea1-9aff-3e9ad83265f3 Address:6955b61c-3383-bea1-9aff-3e9ad83265f3} {Suffrage:Voter ID:server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c Address:ed5a78d2-2480-8d22-f126-2d2c9f0da76c} {Suffrage:Voter ID:server-fcedee9c-3bdb-e70b-71b0-d0caea61b59c Address:fcedee9c-3bdb-e70b-71b0-d0caea61b59c}]" +2025-01-25T14:40:48.814Z [INFO] server-6955b61c-3383-bea1-9aff-3e9ad83265f3: entering follower state: follower="Node at 6955b61c-3383-bea1-9aff-3e9ad83265f3 [Follower]" leader-address= leader-id= +2025-01-25T14:40:48.814Z [INFO] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6955b61c-3383-bea1-9aff-3e9ad83265f3 Address:6955b61c-3383-bea1-9aff-3e9ad83265f3} {Suffrage:Voter ID:server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c Address:ed5a78d2-2480-8d22-f126-2d2c9f0da76c} {Suffrage:Voter ID:server-fcedee9c-3bdb-e70b-71b0-d0caea61b59c Address:fcedee9c-3bdb-e70b-71b0-d0caea61b59c}]" +2025-01-25T14:40:48.815Z [INFO] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: entering follower state: follower="Node at ed5a78d2-2480-8d22-f126-2d2c9f0da76c [Follower]" leader-address= leader-id= +2025-01-25T14:40:48.815Z [INFO] server-fcedee9c-3bdb-e70b-71b0-d0caea61b59c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6955b61c-3383-bea1-9aff-3e9ad83265f3 Address:6955b61c-3383-bea1-9aff-3e9ad83265f3} {Suffrage:Voter ID:server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c Address:ed5a78d2-2480-8d22-f126-2d2c9f0da76c} {Suffrage:Voter ID:server-fcedee9c-3bdb-e70b-71b0-d0caea61b59c Address:fcedee9c-3bdb-e70b-71b0-d0caea61b59c}]" +2025-01-25T14:40:48.815Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:48.816Z [INFO] server-fcedee9c-3bdb-e70b-71b0-d0caea61b59c: entering follower state: follower="Node at fcedee9c-3bdb-e70b-71b0-d0caea61b59c [Follower]" leader-address= leader-id= +2025-01-25T14:40:48.867Z [WARN] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:48.867Z [INFO] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: entering candidate state: node="Node at ed5a78d2-2480-8d22-f126-2d2c9f0da76c [Candidate]" term=2 +2025-01-25T14:40:48.868Z [INFO] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: election won: term=2 tally=2 +2025-01-25T14:40:48.868Z [INFO] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: entering leader state: leader="Node at ed5a78d2-2480-8d22-f126-2d2c9f0da76c [Leader]" +2025-01-25T14:40:48.868Z [INFO] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: added peer, starting replication: peer=server-6955b61c-3383-bea1-9aff-3e9ad83265f3 +2025-01-25T14:40:48.868Z [INFO] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: added peer, starting replication: peer=server-fcedee9c-3bdb-e70b-71b0-d0caea61b59c +2025-01-25T14:40:48.869Z [INFO] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: pipelining replication: peer="{Voter server-6955b61c-3383-bea1-9aff-3e9ad83265f3 6955b61c-3383-bea1-9aff-3e9ad83265f3}" +2025-01-25T14:40:48.869Z [INFO] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: pipelining replication: peer="{Voter server-fcedee9c-3bdb-e70b-71b0-d0caea61b59c fcedee9c-3bdb-e70b-71b0-d0caea61b59c}" +2025-01-25T14:40:48.973Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:48.868449644 +0000 UTC m=+117.914288961 (1 nodes), 52.431027ms from start of poll, 55.152869ms from cluster start. Timeout at 2025-01-25 14:40:48.973470644 +0000 UTC m=+118.019309962, 105.021001ms after stability +2025-01-25T14:40:48.993Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:40:49.100Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:40:48.994084093 +0000 UTC m=+118.039923410 (2 nodes), 6.708µs from start of poll, 180.787318ms from cluster start. Timeout at 2025-01-25 14:40:49.09933376 +0000 UTC m=+118.145173078, 105.249668ms after stability +2025-01-25T14:40:49.103Z [ERROR] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: failed to pipeline appendEntries: peer="{Voter server-6955b61c-3383-bea1-9aff-3e9ad83265f3 6955b61c-3383-bea1-9aff-3e9ad83265f3}" error="append pipeline closed" +2025-01-25T14:40:49.103Z [INFO] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: aborting pipeline replication: peer="{Voter server-6955b61c-3383-bea1-9aff-3e9ad83265f3 6955b61c-3383-bea1-9aff-3e9ad83265f3}" +2025-01-25T14:40:49.105Z [ERROR] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: failed to heartbeat to: peer=6955b61c-3383-bea1-9aff-3e9ad83265f3 backoff time=10ms error="failed to connect to peer: 6955b61c-3383-bea1-9aff-3e9ad83265f3" +2025-01-25T14:40:49.111Z [ERROR] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: failed to appendEntries to: peer="{Voter server-6955b61c-3383-bea1-9aff-3e9ad83265f3 6955b61c-3383-bea1-9aff-3e9ad83265f3}" error="failed to connect to peer: 6955b61c-3383-bea1-9aff-3e9ad83265f3" +2025-01-25T14:40:49.123Z [ERROR] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: failed to heartbeat to: peer=6955b61c-3383-bea1-9aff-3e9ad83265f3 backoff time=10ms error="failed to connect to peer: 6955b61c-3383-bea1-9aff-3e9ad83265f3" +2025-01-25T14:40:49.129Z [ERROR] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: failed to appendEntries to: peer="{Voter server-6955b61c-3383-bea1-9aff-3e9ad83265f3 6955b61c-3383-bea1-9aff-3e9ad83265f3}" error="failed to connect to peer: 6955b61c-3383-bea1-9aff-3e9ad83265f3" +2025-01-25T14:40:49.139Z [ERROR] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: failed to heartbeat to: peer=6955b61c-3383-bea1-9aff-3e9ad83265f3 backoff time=10ms error="failed to connect to peer: 6955b61c-3383-bea1-9aff-3e9ad83265f3" +2025-01-25T14:40:49.146Z [ERROR] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: failed to appendEntries to: peer="{Voter server-6955b61c-3383-bea1-9aff-3e9ad83265f3 6955b61c-3383-bea1-9aff-3e9ad83265f3}" error="failed to connect to peer: 6955b61c-3383-bea1-9aff-3e9ad83265f3" +2025-01-25T14:40:49.154Z [WARN] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: failed to contact: server-id=server-6955b61c-3383-bea1-9aff-3e9ad83265f3 time=57.2415ms +2025-01-25T14:40:49.156Z [ERROR] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: failed to heartbeat to: peer=6955b61c-3383-bea1-9aff-3e9ad83265f3 backoff time=20ms error="failed to connect to peer: 6955b61c-3383-bea1-9aff-3e9ad83265f3" +2025-01-25T14:40:49.173Z [ERROR] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: failed to appendEntries to: peer="{Voter server-6955b61c-3383-bea1-9aff-3e9ad83265f3 6955b61c-3383-bea1-9aff-3e9ad83265f3}" error="failed to connect to peer: 6955b61c-3383-bea1-9aff-3e9ad83265f3" +2025-01-25T14:40:49.186Z [ERROR] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: failed to heartbeat to: peer=6955b61c-3383-bea1-9aff-3e9ad83265f3 backoff time=25ms error="failed to connect to peer: 6955b61c-3383-bea1-9aff-3e9ad83265f3" +2025-01-25T14:40:49.202Z [WARN] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: failed to contact: server-id=server-6955b61c-3383-bea1-9aff-3e9ad83265f3 time=105.16364ms +2025-01-25T14:40:49.206Z [INFO] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: updating configuration: command=RemoveServer server-id=server-6955b61c-3383-bea1-9aff-3e9ad83265f3 server-addr= servers="[{Suffrage:Voter ID:server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c Address:ed5a78d2-2480-8d22-f126-2d2c9f0da76c} {Suffrage:Voter ID:server-fcedee9c-3bdb-e70b-71b0-d0caea61b59c Address:fcedee9c-3bdb-e70b-71b0-d0caea61b59c}]" +2025-01-25T14:40:49.206Z [INFO] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: removed peer, stopping replication: peer=server-6955b61c-3383-bea1-9aff-3e9ad83265f3 last-index=3 +2025-01-25T14:40:49.207Z [INFO] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: updating configuration: command=AddNonvoter server-id=server-6955b61c-3383-bea1-9aff-3e9ad83265f3 server-addr=6955b61c-3383-bea1-9aff-3e9ad83265f3 servers="[{Suffrage:Voter ID:server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c Address:ed5a78d2-2480-8d22-f126-2d2c9f0da76c} {Suffrage:Voter ID:server-fcedee9c-3bdb-e70b-71b0-d0caea61b59c Address:fcedee9c-3bdb-e70b-71b0-d0caea61b59c} {Suffrage:Nonvoter ID:server-6955b61c-3383-bea1-9aff-3e9ad83265f3 Address:6955b61c-3383-bea1-9aff-3e9ad83265f3}]" +2025-01-25T14:40:49.208Z [INFO] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: added peer, starting replication: peer=server-6955b61c-3383-bea1-9aff-3e9ad83265f3 +2025-01-25T14:40:49.208Z [ERROR] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: failed to appendEntries to: peer="{Nonvoter server-6955b61c-3383-bea1-9aff-3e9ad83265f3 6955b61c-3383-bea1-9aff-3e9ad83265f3}" error="failed to connect to peer: 6955b61c-3383-bea1-9aff-3e9ad83265f3" +2025-01-25T14:40:49.220Z [ERROR] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: peer has newer term, stopping replication: peer="{Voter server-6955b61c-3383-bea1-9aff-3e9ad83265f3 6955b61c-3383-bea1-9aff-3e9ad83265f3}" +2025-01-25T14:40:49.221Z [INFO] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: entering follower state: follower="Node at ed5a78d2-2480-8d22-f126-2d2c9f0da76c [Follower]" leader-address= leader-id= +2025-01-25T14:40:49.221Z [INFO] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: aborting pipeline replication: peer="{Voter server-fcedee9c-3bdb-e70b-71b0-d0caea61b59c fcedee9c-3bdb-e70b-71b0-d0caea61b59c}" +2025-01-25T14:40:49.225Z [ERROR] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: peer has newer term, stopping replication: peer="{Nonvoter server-6955b61c-3383-bea1-9aff-3e9ad83265f3 6955b61c-3383-bea1-9aff-3e9ad83265f3}" +2025-01-25T14:40:49.231Z [WARN] server-6955b61c-3383-bea1-9aff-3e9ad83265f3: heartbeat timeout reached, starting election: last-leader-addr=ed5a78d2-2480-8d22-f126-2d2c9f0da76c last-leader-id=server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c +2025-01-25T14:40:49.231Z [INFO] server-6955b61c-3383-bea1-9aff-3e9ad83265f3: entering candidate state: node="Node at 6955b61c-3383-bea1-9aff-3e9ad83265f3 [Candidate]" term=23 +2025-01-25T14:40:49.231Z [WARN] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: rejecting vote request since node is not a voter: from=6955b61c-3383-bea1-9aff-3e9ad83265f3 +2025-01-25T14:40:49.231Z [WARN] server-fcedee9c-3bdb-e70b-71b0-d0caea61b59c: rejecting vote request since we have a leader: from=6955b61c-3383-bea1-9aff-3e9ad83265f3 leader=ed5a78d2-2480-8d22-f126-2d2c9f0da76c leader-id=server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c +2025-01-25T14:40:49.274Z [WARN] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:49.274Z [INFO] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: entering candidate state: node="Node at ed5a78d2-2480-8d22-f126-2d2c9f0da76c [Candidate]" term=24 +2025-01-25T14:40:49.275Z [INFO] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: election won: term=24 tally=2 +2025-01-25T14:40:49.275Z [INFO] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: entering leader state: leader="Node at ed5a78d2-2480-8d22-f126-2d2c9f0da76c [Leader]" +2025-01-25T14:40:49.275Z [INFO] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: added peer, starting replication: peer=server-fcedee9c-3bdb-e70b-71b0-d0caea61b59c +2025-01-25T14:40:49.275Z [INFO] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: added peer, starting replication: peer=server-6955b61c-3383-bea1-9aff-3e9ad83265f3 +2025-01-25T14:40:49.275Z [WARN] server-6955b61c-3383-bea1-9aff-3e9ad83265f3: failed to get previous log: previous-index=4 last-index=2 error="log not found" +2025-01-25T14:40:49.275Z [INFO] server-6955b61c-3383-bea1-9aff-3e9ad83265f3: entering follower state: follower="Node at 6955b61c-3383-bea1-9aff-3e9ad83265f3 [Follower]" leader-address=ed5a78d2-2480-8d22-f126-2d2c9f0da76c leader-id=server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c +2025-01-25T14:40:49.275Z [WARN] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: appendEntries rejected, sending older logs: peer="{Nonvoter server-6955b61c-3383-bea1-9aff-3e9ad83265f3 6955b61c-3383-bea1-9aff-3e9ad83265f3}" next=3 +2025-01-25T14:40:49.276Z [INFO] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: pipelining replication: peer="{Voter server-fcedee9c-3bdb-e70b-71b0-d0caea61b59c fcedee9c-3bdb-e70b-71b0-d0caea61b59c}" +2025-01-25T14:40:49.276Z [INFO] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: pipelining replication: peer="{Nonvoter server-6955b61c-3383-bea1-9aff-3e9ad83265f3 6955b61c-3383-bea1-9aff-3e9ad83265f3}" +2025-01-25T14:40:49.315Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:49.421Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:49.315713321 +0000 UTC m=+118.361552638 (1 nodes), 543.085µs from start of poll, 502.416546ms from cluster start. Timeout at 2025-01-25 14:40:49.420970571 +0000 UTC m=+118.466809889, 105.257251ms after stability +2025-01-25T14:40:49.521Z [INFO] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: updating configuration: command=RemoveServer server-id=server-6955b61c-3383-bea1-9aff-3e9ad83265f3 server-addr= servers="[{Suffrage:Voter ID:server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c Address:ed5a78d2-2480-8d22-f126-2d2c9f0da76c} {Suffrage:Voter ID:server-fcedee9c-3bdb-e70b-71b0-d0caea61b59c Address:fcedee9c-3bdb-e70b-71b0-d0caea61b59c}]" +2025-01-25T14:40:49.521Z [INFO] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: removed peer, stopping replication: peer=server-6955b61c-3383-bea1-9aff-3e9ad83265f3 last-index=106 +2025-01-25T14:40:49.521Z [INFO] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: aborting pipeline replication: peer="{Nonvoter server-6955b61c-3383-bea1-9aff-3e9ad83265f3 6955b61c-3383-bea1-9aff-3e9ad83265f3}" +2025-01-25T14:40:49.522Z [INFO] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: updating configuration: command=AddVoter server-id=server-6955b61c-3383-bea1-9aff-3e9ad83265f3 server-addr=6955b61c-3383-bea1-9aff-3e9ad83265f3 servers="[{Suffrage:Voter ID:server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c Address:ed5a78d2-2480-8d22-f126-2d2c9f0da76c} {Suffrage:Voter ID:server-fcedee9c-3bdb-e70b-71b0-d0caea61b59c Address:fcedee9c-3bdb-e70b-71b0-d0caea61b59c} {Suffrage:Voter ID:server-6955b61c-3383-bea1-9aff-3e9ad83265f3 Address:6955b61c-3383-bea1-9aff-3e9ad83265f3}]" +2025-01-25T14:40:49.522Z [INFO] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: added peer, starting replication: peer=server-6955b61c-3383-bea1-9aff-3e9ad83265f3 +2025-01-25T14:40:49.523Z [WARN] server-6955b61c-3383-bea1-9aff-3e9ad83265f3: failed to get previous log: previous-index=107 last-index=106 error="log not found" +2025-01-25T14:40:49.523Z [WARN] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: appendEntries rejected, sending older logs: peer="{Voter server-6955b61c-3383-bea1-9aff-3e9ad83265f3 6955b61c-3383-bea1-9aff-3e9ad83265f3}" next=107 +2025-01-25T14:40:49.524Z [INFO] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: pipelining replication: peer="{Voter server-6955b61c-3383-bea1-9aff-3e9ad83265f3 6955b61c-3383-bea1-9aff-3e9ad83265f3}" +2025-01-25T14:40:50.501Z [WARN] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: failed to contact: server-id=server-fcedee9c-3bdb-e70b-71b0-d0caea61b59c time=53.459447ms +2025-01-25T14:40:50.659Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:50.765Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:50.659507649 +0000 UTC m=+119.705346967 (1 nodes), 3.792µs from start of poll, 1.846210875s from cluster start. Timeout at 2025-01-25 14:40:50.764783567 +0000 UTC m=+119.810622884, 105.275917ms after stability +2025-01-25T14:40:50.785Z [INFO] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: aborting pipeline replication: peer="{Voter server-6955b61c-3383-bea1-9aff-3e9ad83265f3 6955b61c-3383-bea1-9aff-3e9ad83265f3}" +2025-01-25T14:40:50.796Z [INFO] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: aborting pipeline replication: peer="{Voter server-fcedee9c-3bdb-e70b-71b0-d0caea61b59c fcedee9c-3bdb-e70b-71b0-d0caea61b59c}" +2025-01-25T14:40:51.277Z [ERROR] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: failed to heartbeat to: peer=6955b61c-3383-bea1-9aff-3e9ad83265f3 backoff time=10ms error="command timed out" +2025-01-25T14:40:51.277Z [ERROR] server-ed5a78d2-2480-8d22-f126-2d2c9f0da76c: failed to heartbeat to: peer=fcedee9c-3bdb-e70b-71b0-d0caea61b59c backoff time=10ms error="command timed out" +--- PASS: TestRaft_ClusterCanRegainStability_WhenNonVoterWithHigherTermJoin (2.47s) === RUN TestRaft_FollowerRemovalNoElection -2025-01-25T14:29:59.836Z [INFO] server-2a23e391-6f61-7ee9-2d06-7913f83a7468: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2a23e391-6f61-7ee9-2d06-7913f83a7468 Address:2a23e391-6f61-7ee9-2d06-7913f83a7468} {Suffrage:Voter ID:server-da86c863-436d-7519-d7f0-91d1a582634b Address:da86c863-436d-7519-d7f0-91d1a582634b} {Suffrage:Voter ID:server-9c2edd62-05a0-2906-f869-9b6a134db5cf Address:9c2edd62-05a0-2906-f869-9b6a134db5cf}]" -2025-01-25T14:29:59.836Z [INFO] server-2a23e391-6f61-7ee9-2d06-7913f83a7468: entering follower state: follower="Node at 2a23e391-6f61-7ee9-2d06-7913f83a7468 [Follower]" leader-address= leader-id= -2025-01-25T14:29:59.836Z [INFO] server-da86c863-436d-7519-d7f0-91d1a582634b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2a23e391-6f61-7ee9-2d06-7913f83a7468 Address:2a23e391-6f61-7ee9-2d06-7913f83a7468} {Suffrage:Voter ID:server-da86c863-436d-7519-d7f0-91d1a582634b Address:da86c863-436d-7519-d7f0-91d1a582634b} {Suffrage:Voter ID:server-9c2edd62-05a0-2906-f869-9b6a134db5cf Address:9c2edd62-05a0-2906-f869-9b6a134db5cf}]" -2025-01-25T14:29:59.836Z [INFO] server-da86c863-436d-7519-d7f0-91d1a582634b: entering follower state: follower="Node at da86c863-436d-7519-d7f0-91d1a582634b [Follower]" leader-address= leader-id= -2025-01-25T14:29:59.836Z [INFO] server-9c2edd62-05a0-2906-f869-9b6a134db5cf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2a23e391-6f61-7ee9-2d06-7913f83a7468 Address:2a23e391-6f61-7ee9-2d06-7913f83a7468} {Suffrage:Voter ID:server-da86c863-436d-7519-d7f0-91d1a582634b Address:da86c863-436d-7519-d7f0-91d1a582634b} {Suffrage:Voter ID:server-9c2edd62-05a0-2906-f869-9b6a134db5cf Address:9c2edd62-05a0-2906-f869-9b6a134db5cf}]" -2025-01-25T14:29:59.837Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:29:59.837Z [INFO] server-9c2edd62-05a0-2906-f869-9b6a134db5cf: entering follower state: follower="Node at 9c2edd62-05a0-2906-f869-9b6a134db5cf [Follower]" leader-address= leader-id= -2025-01-25T14:29:59.985Z [WARN] server-2a23e391-6f61-7ee9-2d06-7913f83a7468: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:59.985Z [INFO] server-2a23e391-6f61-7ee9-2d06-7913f83a7468: entering candidate state: node="Node at 2a23e391-6f61-7ee9-2d06-7913f83a7468 [Candidate]" term=2 -2025-01-25T14:29:59.984Z [WARN] server-9c2edd62-05a0-2906-f869-9b6a134db5cf: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:29:59.986Z [INFO] server-9c2edd62-05a0-2906-f869-9b6a134db5cf: entering candidate state: node="Node at 9c2edd62-05a0-2906-f869-9b6a134db5cf [Candidate]" term=2 -2025-01-25T14:29:59.987Z [INFO] server-9c2edd62-05a0-2906-f869-9b6a134db5cf: duplicate requestVote for same term: term=2 -2025-01-25T14:29:59.988Z [INFO] server-2a23e391-6f61-7ee9-2d06-7913f83a7468: election won: term=2 tally=2 -2025-01-25T14:29:59.988Z [INFO] server-2a23e391-6f61-7ee9-2d06-7913f83a7468: entering leader state: leader="Node at 2a23e391-6f61-7ee9-2d06-7913f83a7468 [Leader]" -2025-01-25T14:29:59.988Z [INFO] server-2a23e391-6f61-7ee9-2d06-7913f83a7468: added peer, starting replication: peer=server-da86c863-436d-7519-d7f0-91d1a582634b -2025-01-25T14:29:59.988Z [INFO] server-2a23e391-6f61-7ee9-2d06-7913f83a7468: added peer, starting replication: peer=server-9c2edd62-05a0-2906-f869-9b6a134db5cf -2025-01-25T14:29:59.988Z [INFO] server-9c2edd62-05a0-2906-f869-9b6a134db5cf: entering follower state: follower="Node at 9c2edd62-05a0-2906-f869-9b6a134db5cf [Follower]" leader-address=2a23e391-6f61-7ee9-2d06-7913f83a7468 leader-id=server-2a23e391-6f61-7ee9-2d06-7913f83a7468 -2025-01-25T14:29:59.988Z [INFO] server-2a23e391-6f61-7ee9-2d06-7913f83a7468: pipelining replication: peer="{Voter server-9c2edd62-05a0-2906-f869-9b6a134db5cf 9c2edd62-05a0-2906-f869-9b6a134db5cf}" -2025-01-25T14:29:59.988Z [INFO] server-da86c863-436d-7519-d7f0-91d1a582634b: duplicate requestVote for same term: term=2 -2025-01-25T14:29:59.988Z [INFO] server-2a23e391-6f61-7ee9-2d06-7913f83a7468: pipelining replication: peer="{Voter server-da86c863-436d-7519-d7f0-91d1a582634b da86c863-436d-7519-d7f0-91d1a582634b}" -2025-01-25T14:29:59.988Z [WARN] server-2a23e391-6f61-7ee9-2d06-7913f83a7468: rejecting vote request since we have a leader: from=9c2edd62-05a0-2906-f869-9b6a134db5cf leader=2a23e391-6f61-7ee9-2d06-7913f83a7468 leader-id=server-2a23e391-6f61-7ee9-2d06-7913f83a7468 -2025-01-25T14:30:00.195Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:29:59.98886903 +0000 UTC m=+109.843193972 (1 nodes), 151.812017ms from start of poll, 152.746543ms from cluster start. Timeout at 2025-01-25 14:30:00.19387291 +0000 UTC m=+110.048197892, 205.00392ms after stability -2025-01-25T14:30:00.195Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:30:00.401Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:30:00.195201004 +0000 UTC m=+110.049525946 (1 nodes), 2.52µs from start of poll, 359.078517ms from cluster start. Timeout at 2025-01-25 14:30:00.400370758 +0000 UTC m=+110.254695780, 205.169834ms after stability -2025-01-25T14:30:00.407Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:30:00.612Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:30:00.40733233 +0000 UTC m=+110.261657312 (2 nodes), 3.28µs from start of poll, 571.209883ms from cluster start. Timeout at 2025-01-25 14:30:00.61259488 +0000 UTC m=+110.466919902, 205.26259ms after stability -2025-01-25T14:30:00.612Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:30:00.818Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:30:00.612767177 +0000 UTC m=+110.467092159 (1 nodes), 1.8µs from start of poll, 776.64473ms from cluster start. Timeout at 2025-01-25 14:30:00.818001489 +0000 UTC m=+110.672326431, 205.234272ms after stability - raft_test.go:2961: [INFO] restarting Node at da86c863-436d-7519-d7f0-91d1a582634b [Follower] -2025-01-25T14:30:00.818Z [INFO] server-da86c863-436d-7519-d7f0-91d1a582634b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2a23e391-6f61-7ee9-2d06-7913f83a7468 Address:2a23e391-6f61-7ee9-2d06-7913f83a7468} {Suffrage:Voter ID:server-da86c863-436d-7519-d7f0-91d1a582634b Address:da86c863-436d-7519-d7f0-91d1a582634b} {Suffrage:Voter ID:server-9c2edd62-05a0-2906-f869-9b6a134db5cf Address:9c2edd62-05a0-2906-f869-9b6a134db5cf}]" -2025-01-25T14:30:00.819Z [INFO] server-da86c863-436d-7519-d7f0-91d1a582634b: entering follower state: follower="Node at da86c863-436d-7519-d7f0-91d1a582634b [Follower]" leader-address= leader-id= -2025-01-25T14:30:01.069Z [INFO] server-2a23e391-6f61-7ee9-2d06-7913f83a7468: aborting pipeline replication: peer="{Voter server-9c2edd62-05a0-2906-f869-9b6a134db5cf 9c2edd62-05a0-2906-f869-9b6a134db5cf}" -2025-01-25T14:30:01.452Z [ERROR] server-2a23e391-6f61-7ee9-2d06-7913f83a7468: failed to pipeline appendEntries: peer="{Voter server-da86c863-436d-7519-d7f0-91d1a582634b da86c863-436d-7519-d7f0-91d1a582634b}" error="command enqueue timeout" -2025-01-25T14:30:01.453Z [INFO] server-2a23e391-6f61-7ee9-2d06-7913f83a7468: aborting pipeline replication: peer="{Voter server-da86c863-436d-7519-d7f0-91d1a582634b da86c863-436d-7519-d7f0-91d1a582634b}" -2025-01-25T14:30:01.578Z [ERROR] server-2a23e391-6f61-7ee9-2d06-7913f83a7468: failed to heartbeat to: peer=da86c863-436d-7519-d7f0-91d1a582634b backoff time=10ms error="command timed out" ---- PASS: TestRaft_FollowerRemovalNoElection (1.75s) +2025-01-25T14:40:51.283Z [INFO] server-cf9f77e0-3dde-716c-b548-23531bd861ac: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cf9f77e0-3dde-716c-b548-23531bd861ac Address:cf9f77e0-3dde-716c-b548-23531bd861ac} {Suffrage:Voter ID:server-c2168b86-87f5-aebd-d42d-3fb8207cf290 Address:c2168b86-87f5-aebd-d42d-3fb8207cf290} {Suffrage:Voter ID:server-5acd58d3-f2dd-e49f-2911-66202fdb91ee Address:5acd58d3-f2dd-e49f-2911-66202fdb91ee}]" +2025-01-25T14:40:51.283Z [INFO] server-cf9f77e0-3dde-716c-b548-23531bd861ac: entering follower state: follower="Node at cf9f77e0-3dde-716c-b548-23531bd861ac [Follower]" leader-address= leader-id= +2025-01-25T14:40:51.284Z [INFO] server-c2168b86-87f5-aebd-d42d-3fb8207cf290: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cf9f77e0-3dde-716c-b548-23531bd861ac Address:cf9f77e0-3dde-716c-b548-23531bd861ac} {Suffrage:Voter ID:server-c2168b86-87f5-aebd-d42d-3fb8207cf290 Address:c2168b86-87f5-aebd-d42d-3fb8207cf290} {Suffrage:Voter ID:server-5acd58d3-f2dd-e49f-2911-66202fdb91ee Address:5acd58d3-f2dd-e49f-2911-66202fdb91ee}]" +2025-01-25T14:40:51.284Z [INFO] server-c2168b86-87f5-aebd-d42d-3fb8207cf290: entering follower state: follower="Node at c2168b86-87f5-aebd-d42d-3fb8207cf290 [Follower]" leader-address= leader-id= +2025-01-25T14:40:51.284Z [INFO] server-5acd58d3-f2dd-e49f-2911-66202fdb91ee: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cf9f77e0-3dde-716c-b548-23531bd861ac Address:cf9f77e0-3dde-716c-b548-23531bd861ac} {Suffrage:Voter ID:server-c2168b86-87f5-aebd-d42d-3fb8207cf290 Address:c2168b86-87f5-aebd-d42d-3fb8207cf290} {Suffrage:Voter ID:server-5acd58d3-f2dd-e49f-2911-66202fdb91ee Address:5acd58d3-f2dd-e49f-2911-66202fdb91ee}]" +2025-01-25T14:40:51.284Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:51.284Z [INFO] server-5acd58d3-f2dd-e49f-2911-66202fdb91ee: entering follower state: follower="Node at 5acd58d3-f2dd-e49f-2911-66202fdb91ee [Follower]" leader-address= leader-id= +2025-01-25T14:40:51.462Z [WARN] server-c2168b86-87f5-aebd-d42d-3fb8207cf290: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:51.462Z [INFO] server-c2168b86-87f5-aebd-d42d-3fb8207cf290: entering candidate state: node="Node at c2168b86-87f5-aebd-d42d-3fb8207cf290 [Candidate]" term=2 +2025-01-25T14:40:51.462Z [WARN] server-5acd58d3-f2dd-e49f-2911-66202fdb91ee: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:51.462Z [INFO] server-5acd58d3-f2dd-e49f-2911-66202fdb91ee: entering candidate state: node="Node at 5acd58d3-f2dd-e49f-2911-66202fdb91ee [Candidate]" term=2 +2025-01-25T14:40:51.463Z [INFO] server-c2168b86-87f5-aebd-d42d-3fb8207cf290: election won: term=2 tally=2 +2025-01-25T14:40:51.463Z [INFO] server-5acd58d3-f2dd-e49f-2911-66202fdb91ee: duplicate requestVote for same term: term=2 +2025-01-25T14:40:51.463Z [INFO] server-c2168b86-87f5-aebd-d42d-3fb8207cf290: entering leader state: leader="Node at c2168b86-87f5-aebd-d42d-3fb8207cf290 [Leader]" +2025-01-25T14:40:51.463Z [INFO] server-cf9f77e0-3dde-716c-b548-23531bd861ac: duplicate requestVote for same term: term=2 +2025-01-25T14:40:51.463Z [INFO] server-c2168b86-87f5-aebd-d42d-3fb8207cf290: added peer, starting replication: peer=server-cf9f77e0-3dde-716c-b548-23531bd861ac +2025-01-25T14:40:51.463Z [INFO] server-c2168b86-87f5-aebd-d42d-3fb8207cf290: added peer, starting replication: peer=server-5acd58d3-f2dd-e49f-2911-66202fdb91ee +2025-01-25T14:40:51.463Z [WARN] server-c2168b86-87f5-aebd-d42d-3fb8207cf290: rejecting vote request since we have a leader: from=5acd58d3-f2dd-e49f-2911-66202fdb91ee leader=c2168b86-87f5-aebd-d42d-3fb8207cf290 leader-id=server-c2168b86-87f5-aebd-d42d-3fb8207cf290 +2025-01-25T14:40:51.463Z [INFO] server-5acd58d3-f2dd-e49f-2911-66202fdb91ee: entering follower state: follower="Node at 5acd58d3-f2dd-e49f-2911-66202fdb91ee [Follower]" leader-address=c2168b86-87f5-aebd-d42d-3fb8207cf290 leader-id=server-c2168b86-87f5-aebd-d42d-3fb8207cf290 +2025-01-25T14:40:51.463Z [INFO] server-c2168b86-87f5-aebd-d42d-3fb8207cf290: pipelining replication: peer="{Voter server-cf9f77e0-3dde-716c-b548-23531bd861ac cf9f77e0-3dde-716c-b548-23531bd861ac}" +2025-01-25T14:40:51.464Z [INFO] server-c2168b86-87f5-aebd-d42d-3fb8207cf290: pipelining replication: peer="{Voter server-5acd58d3-f2dd-e49f-2911-66202fdb91ee 5acd58d3-f2dd-e49f-2911-66202fdb91ee}" +2025-01-25T14:40:51.669Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:51.463925116 +0000 UTC m=+120.509764434 (1 nodes), 179.145105ms from start of poll, 181.009444ms from cluster start. Timeout at 2025-01-25 14:40:51.668940283 +0000 UTC m=+120.714779601, 205.015167ms after stability +2025-01-25T14:40:51.669Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:51.875Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:51.669418006 +0000 UTC m=+120.715257324 (1 nodes), 3.5µs from start of poll, 386.502334ms from cluster start. Timeout at 2025-01-25 14:40:51.87472834 +0000 UTC m=+120.920567658, 205.310334ms after stability +2025-01-25T14:40:51.881Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:40:52.087Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:40:51.881800541 +0000 UTC m=+120.927639859 (2 nodes), 6.417µs from start of poll, 598.884869ms from cluster start. Timeout at 2025-01-25 14:40:52.086988959 +0000 UTC m=+121.132828276, 205.188417ms after stability +2025-01-25T14:40:52.087Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:52.293Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:52.087419764 +0000 UTC m=+121.133259082 (1 nodes), 5.542µs from start of poll, 804.504092ms from cluster start. Timeout at 2025-01-25 14:40:52.292599139 +0000 UTC m=+121.338438457, 205.179375ms after stability + raft_test.go:2961: [INFO] restarting Node at cf9f77e0-3dde-716c-b548-23531bd861ac [Follower] +2025-01-25T14:40:52.294Z [INFO] server-cf9f77e0-3dde-716c-b548-23531bd861ac: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cf9f77e0-3dde-716c-b548-23531bd861ac Address:cf9f77e0-3dde-716c-b548-23531bd861ac} {Suffrage:Voter ID:server-c2168b86-87f5-aebd-d42d-3fb8207cf290 Address:c2168b86-87f5-aebd-d42d-3fb8207cf290} {Suffrage:Voter ID:server-5acd58d3-f2dd-e49f-2911-66202fdb91ee Address:5acd58d3-f2dd-e49f-2911-66202fdb91ee}]" +2025-01-25T14:40:52.294Z [INFO] server-cf9f77e0-3dde-716c-b548-23531bd861ac: entering follower state: follower="Node at cf9f77e0-3dde-716c-b548-23531bd861ac [Follower]" leader-address= leader-id= +2025-01-25T14:40:52.545Z [INFO] server-c2168b86-87f5-aebd-d42d-3fb8207cf290: aborting pipeline replication: peer="{Voter server-5acd58d3-f2dd-e49f-2911-66202fdb91ee 5acd58d3-f2dd-e49f-2911-66202fdb91ee}" +2025-01-25T14:40:52.932Z [ERROR] server-c2168b86-87f5-aebd-d42d-3fb8207cf290: failed to pipeline appendEntries: peer="{Voter server-cf9f77e0-3dde-716c-b548-23531bd861ac cf9f77e0-3dde-716c-b548-23531bd861ac}" error="command enqueue timeout" +2025-01-25T14:40:52.933Z [INFO] server-c2168b86-87f5-aebd-d42d-3fb8207cf290: aborting pipeline replication: peer="{Voter server-cf9f77e0-3dde-716c-b548-23531bd861ac cf9f77e0-3dde-716c-b548-23531bd861ac}" +2025-01-25T14:40:53.060Z [ERROR] server-c2168b86-87f5-aebd-d42d-3fb8207cf290: failed to heartbeat to: peer=cf9f77e0-3dde-716c-b548-23531bd861ac backoff time=10ms error="command timed out" +--- PASS: TestRaft_FollowerRemovalNoElection (1.78s) === RUN TestRaft_VoteWithNoIDNoAddr -2025-01-25T14:30:01.581Z [INFO] server-4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1 Address:4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1} {Suffrage:Voter ID:server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1 Address:1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1} {Suffrage:Voter ID:server-a142183c-0374-2555-f5ca-8cd872388a26 Address:a142183c-0374-2555-f5ca-8cd872388a26}]" -2025-01-25T14:30:01.582Z [INFO] server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1 Address:4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1} {Suffrage:Voter ID:server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1 Address:1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1} {Suffrage:Voter ID:server-a142183c-0374-2555-f5ca-8cd872388a26 Address:a142183c-0374-2555-f5ca-8cd872388a26}]" -2025-01-25T14:30:01.582Z [INFO] server-4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1: entering follower state: follower="Node at 4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1 [Follower]" leader-address= leader-id= -2025-01-25T14:30:01.582Z [INFO] server-a142183c-0374-2555-f5ca-8cd872388a26: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1 Address:4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1} {Suffrage:Voter ID:server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1 Address:1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1} {Suffrage:Voter ID:server-a142183c-0374-2555-f5ca-8cd872388a26 Address:a142183c-0374-2555-f5ca-8cd872388a26}]" -2025-01-25T14:30:01.582Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:30:01.582Z [INFO] server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1: entering follower state: follower="Node at 1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1 [Follower]" leader-address= leader-id= -2025-01-25T14:30:01.582Z [INFO] server-a142183c-0374-2555-f5ca-8cd872388a26: entering follower state: follower="Node at a142183c-0374-2555-f5ca-8cd872388a26 [Follower]" leader-address= leader-id= -2025-01-25T14:30:01.640Z [WARN] server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-25T14:30:01.640Z [INFO] server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1: entering candidate state: node="Node at 1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1 [Candidate]" term=2 -2025-01-25T14:30:01.640Z [INFO] server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1: election won: term=2 tally=2 -2025-01-25T14:30:01.640Z [INFO] server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1: entering leader state: leader="Node at 1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1 [Leader]" -2025-01-25T14:30:01.640Z [INFO] server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1: added peer, starting replication: peer=server-4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1 -2025-01-25T14:30:01.640Z [INFO] server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1: added peer, starting replication: peer=server-a142183c-0374-2555-f5ca-8cd872388a26 -2025-01-25T14:30:01.640Z [INFO] server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1: pipelining replication: peer="{Voter server-4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1 4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1}" -2025-01-25T14:30:01.640Z [INFO] server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1: pipelining replication: peer="{Voter server-a142183c-0374-2555-f5ca-8cd872388a26 a142183c-0374-2555-f5ca-8cd872388a26}" -2025-01-25T14:30:01.746Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:30:01.640504151 +0000 UTC m=+111.494829093 (1 nodes), 58.147747ms from start of poll, 58.876281ms from cluster start. Timeout at 2025-01-25 14:30:01.745508631 +0000 UTC m=+111.599833613, 105.00452ms after stability -2025-01-25T14:30:01.746Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-25T14:30:01.852Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:30:01.746799526 +0000 UTC m=+111.601124508 (1 nodes), 1.52µs from start of poll, 165.171696ms from cluster start. Timeout at 2025-01-25 14:30:01.852002318 +0000 UTC m=+111.706327300, 105.202792ms after stability -2025-01-25T14:30:01.857Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-25T14:30:01.973Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:30:01.85756422 +0000 UTC m=+111.711889202 (2 nodes), 2.36µs from start of poll, 275.93639ms from cluster start. Timeout at 2025-01-25 14:30:01.962809572 +0000 UTC m=+111.817134514, 105.245312ms after stability -2025-01-25T14:30:01.974Z [ERROR] server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1: failed to pipeline appendEntries: peer="{Voter server-a142183c-0374-2555-f5ca-8cd872388a26 a142183c-0374-2555-f5ca-8cd872388a26}" error="append pipeline closed" -2025-01-25T14:30:01.974Z [INFO] server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1: aborting pipeline replication: peer="{Voter server-a142183c-0374-2555-f5ca-8cd872388a26 a142183c-0374-2555-f5ca-8cd872388a26}" -2025-01-25T14:30:01.982Z [ERROR] server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1: failed to heartbeat to: peer=4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1 backoff time=10ms error="failed to connect to peer: 4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1" -2025-01-25T14:30:01.983Z [ERROR] server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1: failed to pipeline appendEntries: peer="{Voter server-4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1 4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1}" error="append pipeline closed" -2025-01-25T14:30:01.983Z [INFO] server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1: aborting pipeline replication: peer="{Voter server-4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1 4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1}" -2025-01-25T14:30:01.983Z [ERROR] server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1: failed to heartbeat to: peer=a142183c-0374-2555-f5ca-8cd872388a26 backoff time=10ms error="failed to connect to peer: a142183c-0374-2555-f5ca-8cd872388a26" -2025-01-25T14:30:01.983Z [ERROR] server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1: failed to appendEntries to: peer="{Voter server-a142183c-0374-2555-f5ca-8cd872388a26 a142183c-0374-2555-f5ca-8cd872388a26}" error="failed to connect to peer: a142183c-0374-2555-f5ca-8cd872388a26" -2025-01-25T14:30:01.995Z [ERROR] server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1: failed to appendEntries to: peer="{Voter server-4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1 4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1}" error="failed to connect to peer: 4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1" -2025-01-25T14:30:02.001Z [ERROR] server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1: failed to heartbeat to: peer=a142183c-0374-2555-f5ca-8cd872388a26 backoff time=10ms error="failed to connect to peer: a142183c-0374-2555-f5ca-8cd872388a26" -2025-01-25T14:30:02.004Z [ERROR] server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1: failed to heartbeat to: peer=4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1 backoff time=10ms error="failed to connect to peer: 4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1" -2025-01-25T14:30:02.006Z [ERROR] server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1: failed to appendEntries to: peer="{Voter server-a142183c-0374-2555-f5ca-8cd872388a26 a142183c-0374-2555-f5ca-8cd872388a26}" error="failed to connect to peer: a142183c-0374-2555-f5ca-8cd872388a26" -2025-01-25T14:30:02.015Z [ERROR] server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1: failed to appendEntries to: peer="{Voter server-4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1 4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1}" error="failed to connect to peer: 4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1" -2025-01-25T14:30:02.021Z [ERROR] server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1: failed to heartbeat to: peer=a142183c-0374-2555-f5ca-8cd872388a26 backoff time=10ms error="failed to connect to peer: a142183c-0374-2555-f5ca-8cd872388a26" -2025-01-25T14:30:02.021Z [ERROR] server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1: failed to heartbeat to: peer=4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1 backoff time=10ms error="failed to connect to peer: 4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1" -2025-01-25T14:30:02.027Z [WARN] server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1: failed to contact: server-id=server-4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1 time=53.462316ms -2025-01-25T14:30:02.027Z [WARN] server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1: failed to contact: server-id=server-a142183c-0374-2555-f5ca-8cd872388a26 time=52.444473ms -2025-01-25T14:30:02.027Z [WARN] server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1: failed to contact quorum of nodes, stepping down -2025-01-25T14:30:02.027Z [INFO] server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1: entering follower state: follower="Node at 1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1 [Follower]" leader-address= leader-id= -2025-01-25T14:30:02.027Z [ERROR] server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1: failed to appendEntries to: peer="{Voter server-a142183c-0374-2555-f5ca-8cd872388a26 a142183c-0374-2555-f5ca-8cd872388a26}" error="failed to connect to peer: a142183c-0374-2555-f5ca-8cd872388a26" -2025-01-25T14:30:02.031Z [WARN] server-a142183c-0374-2555-f5ca-8cd872388a26: heartbeat timeout reached, starting election: last-leader-addr=1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1 last-leader-id=server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1 -2025-01-25T14:30:02.031Z [INFO] server-a142183c-0374-2555-f5ca-8cd872388a26: entering candidate state: node="Node at a142183c-0374-2555-f5ca-8cd872388a26 [Candidate]" term=3 -2025-01-25T14:30:02.031Z [ERROR] server-a142183c-0374-2555-f5ca-8cd872388a26: failed to make requestVote RPC: target="{Voter server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1 1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1}" error="failed to connect to peer: 1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1" term=3 -2025-01-25T14:30:02.031Z [WARN] server-4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1: rejecting vote request since we have a leader: from=a142183c-0374-2555-f5ca-8cd872388a26 leader=1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1 leader-id=server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1 -2025-01-25T14:30:02.032Z [ERROR] server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1: failed to appendEntries to: peer="{Voter server-4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1 4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1}" error="failed to connect to peer: 4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1" -2025-01-25T14:30:02.040Z [WARN] server-4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1: heartbeat timeout reached, starting election: last-leader-addr=1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1 last-leader-id=server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1 -2025-01-25T14:30:02.040Z [INFO] server-4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1: entering candidate state: node="Node at 4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1 [Candidate]" term=3 -2025-01-25T14:30:02.041Z [ERROR] server-4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1: failed to make requestVote RPC: target="{Voter server-1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1 1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1}" error="failed to connect to peer: 1b9e86cf-9de8-ee67-75de-9cf5baa2e8e1" term=3 -2025-01-25T14:30:02.041Z [INFO] server-a142183c-0374-2555-f5ca-8cd872388a26: duplicate requestVote for same term: term=3 -2025-01-25T14:30:02.079Z [INFO] server-4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1: entering follower state: follower="Node at 4e4b88b5-d8e4-b7a0-8df0-362ecdf2a6c1 [Follower]" leader-address= leader-id= ---- PASS: TestRaft_VoteWithNoIDNoAddr (0.50s) +2025-01-25T14:40:53.065Z [INFO] server-c55cced2-7611-d6a7-3c1c-607c4bca828e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c55cced2-7611-d6a7-3c1c-607c4bca828e Address:c55cced2-7611-d6a7-3c1c-607c4bca828e} {Suffrage:Voter ID:server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719 Address:c99b7fde-34bd-bbcc-7ebd-06cfe5914719} {Suffrage:Voter ID:server-cee08b61-1b4e-0c4e-a81e-406be6ea2c7b Address:cee08b61-1b4e-0c4e-a81e-406be6ea2c7b}]" +2025-01-25T14:40:53.065Z [INFO] server-c55cced2-7611-d6a7-3c1c-607c4bca828e: entering follower state: follower="Node at c55cced2-7611-d6a7-3c1c-607c4bca828e [Follower]" leader-address= leader-id= +2025-01-25T14:40:53.066Z [INFO] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c55cced2-7611-d6a7-3c1c-607c4bca828e Address:c55cced2-7611-d6a7-3c1c-607c4bca828e} {Suffrage:Voter ID:server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719 Address:c99b7fde-34bd-bbcc-7ebd-06cfe5914719} {Suffrage:Voter ID:server-cee08b61-1b4e-0c4e-a81e-406be6ea2c7b Address:cee08b61-1b4e-0c4e-a81e-406be6ea2c7b}]" +2025-01-25T14:40:53.066Z [INFO] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: entering follower state: follower="Node at c99b7fde-34bd-bbcc-7ebd-06cfe5914719 [Follower]" leader-address= leader-id= +2025-01-25T14:40:53.066Z [INFO] server-cee08b61-1b4e-0c4e-a81e-406be6ea2c7b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c55cced2-7611-d6a7-3c1c-607c4bca828e Address:c55cced2-7611-d6a7-3c1c-607c4bca828e} {Suffrage:Voter ID:server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719 Address:c99b7fde-34bd-bbcc-7ebd-06cfe5914719} {Suffrage:Voter ID:server-cee08b61-1b4e-0c4e-a81e-406be6ea2c7b Address:cee08b61-1b4e-0c4e-a81e-406be6ea2c7b}]" +2025-01-25T14:40:53.066Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:53.066Z [INFO] server-cee08b61-1b4e-0c4e-a81e-406be6ea2c7b: entering follower state: follower="Node at cee08b61-1b4e-0c4e-a81e-406be6ea2c7b [Follower]" leader-address= leader-id= +2025-01-25T14:40:53.143Z [WARN] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:53.143Z [INFO] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: entering candidate state: node="Node at c99b7fde-34bd-bbcc-7ebd-06cfe5914719 [Candidate]" term=2 +2025-01-25T14:40:53.143Z [INFO] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: election won: term=2 tally=2 +2025-01-25T14:40:53.143Z [INFO] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: entering leader state: leader="Node at c99b7fde-34bd-bbcc-7ebd-06cfe5914719 [Leader]" +2025-01-25T14:40:53.143Z [INFO] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: added peer, starting replication: peer=server-c55cced2-7611-d6a7-3c1c-607c4bca828e +2025-01-25T14:40:53.143Z [INFO] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: added peer, starting replication: peer=server-cee08b61-1b4e-0c4e-a81e-406be6ea2c7b +2025-01-25T14:40:53.144Z [INFO] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: pipelining replication: peer="{Voter server-cee08b61-1b4e-0c4e-a81e-406be6ea2c7b cee08b61-1b4e-0c4e-a81e-406be6ea2c7b}" +2025-01-25T14:40:53.144Z [INFO] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: pipelining replication: peer="{Voter server-c55cced2-7611-d6a7-3c1c-607c4bca828e c55cced2-7611-d6a7-3c1c-607c4bca828e}" +2025-01-25T14:40:53.249Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:53.14379934 +0000 UTC m=+122.189638657 (1 nodes), 76.805015ms from start of poll, 78.527895ms from cluster start. Timeout at 2025-01-25 14:40:53.248915131 +0000 UTC m=+122.294754449, 105.115792ms after stability +2025-01-25T14:40:53.249Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-25T14:40:53.354Z [INFO] cluster: stable state for Leader reached at 2025-01-25 14:40:53.249484772 +0000 UTC m=+122.295324090 (1 nodes), 6.125µs from start of poll, 184.213328ms from cluster start. Timeout at 2025-01-25 14:40:53.354711106 +0000 UTC m=+122.400550424, 105.226334ms after stability +2025-01-25T14:40:53.360Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-25T14:40:53.466Z [INFO] cluster: stable state for Follower reached at 2025-01-25 14:40:53.36041922 +0000 UTC m=+122.406258538 (2 nodes), 5.542µs from start of poll, 295.147776ms from cluster start. Timeout at 2025-01-25 14:40:53.465612304 +0000 UTC m=+122.511451621, 105.193083ms after stability +2025-01-25T14:40:53.473Z [ERROR] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: failed to pipeline appendEntries: peer="{Voter server-c55cced2-7611-d6a7-3c1c-607c4bca828e c55cced2-7611-d6a7-3c1c-607c4bca828e}" error="append pipeline closed" +2025-01-25T14:40:53.474Z [INFO] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: aborting pipeline replication: peer="{Voter server-c55cced2-7611-d6a7-3c1c-607c4bca828e c55cced2-7611-d6a7-3c1c-607c4bca828e}" +2025-01-25T14:40:53.473Z [ERROR] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: failed to heartbeat to: peer=cee08b61-1b4e-0c4e-a81e-406be6ea2c7b backoff time=10ms error="failed to connect to peer: cee08b61-1b4e-0c4e-a81e-406be6ea2c7b" +2025-01-25T14:40:53.476Z [ERROR] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: failed to pipeline appendEntries: peer="{Voter server-cee08b61-1b4e-0c4e-a81e-406be6ea2c7b cee08b61-1b4e-0c4e-a81e-406be6ea2c7b}" error="append pipeline closed" +2025-01-25T14:40:53.476Z [INFO] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: aborting pipeline replication: peer="{Voter server-cee08b61-1b4e-0c4e-a81e-406be6ea2c7b cee08b61-1b4e-0c4e-a81e-406be6ea2c7b}" +2025-01-25T14:40:53.485Z [ERROR] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: failed to appendEntries to: peer="{Voter server-c55cced2-7611-d6a7-3c1c-607c4bca828e c55cced2-7611-d6a7-3c1c-607c4bca828e}" error="failed to connect to peer: c55cced2-7611-d6a7-3c1c-607c4bca828e" +2025-01-25T14:40:53.485Z [ERROR] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: failed to appendEntries to: peer="{Voter server-cee08b61-1b4e-0c4e-a81e-406be6ea2c7b cee08b61-1b4e-0c4e-a81e-406be6ea2c7b}" error="failed to connect to peer: cee08b61-1b4e-0c4e-a81e-406be6ea2c7b" +2025-01-25T14:40:53.485Z [ERROR] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: failed to heartbeat to: peer=c55cced2-7611-d6a7-3c1c-607c4bca828e backoff time=10ms error="failed to connect to peer: c55cced2-7611-d6a7-3c1c-607c4bca828e" +2025-01-25T14:40:53.493Z [ERROR] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: failed to heartbeat to: peer=cee08b61-1b4e-0c4e-a81e-406be6ea2c7b backoff time=10ms error="failed to connect to peer: cee08b61-1b4e-0c4e-a81e-406be6ea2c7b" +2025-01-25T14:40:53.502Z [ERROR] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: failed to appendEntries to: peer="{Voter server-cee08b61-1b4e-0c4e-a81e-406be6ea2c7b cee08b61-1b4e-0c4e-a81e-406be6ea2c7b}" error="failed to connect to peer: cee08b61-1b4e-0c4e-a81e-406be6ea2c7b" +2025-01-25T14:40:53.502Z [ERROR] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: failed to heartbeat to: peer=c55cced2-7611-d6a7-3c1c-607c4bca828e backoff time=10ms error="failed to connect to peer: c55cced2-7611-d6a7-3c1c-607c4bca828e" +2025-01-25T14:40:53.503Z [ERROR] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: failed to appendEntries to: peer="{Voter server-c55cced2-7611-d6a7-3c1c-607c4bca828e c55cced2-7611-d6a7-3c1c-607c4bca828e}" error="failed to connect to peer: c55cced2-7611-d6a7-3c1c-607c4bca828e" +2025-01-25T14:40:53.510Z [ERROR] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: failed to heartbeat to: peer=cee08b61-1b4e-0c4e-a81e-406be6ea2c7b backoff time=10ms error="failed to connect to peer: cee08b61-1b4e-0c4e-a81e-406be6ea2c7b" +2025-01-25T14:40:53.518Z [ERROR] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: failed to appendEntries to: peer="{Voter server-cee08b61-1b4e-0c4e-a81e-406be6ea2c7b cee08b61-1b4e-0c4e-a81e-406be6ea2c7b}" error="failed to connect to peer: cee08b61-1b4e-0c4e-a81e-406be6ea2c7b" +2025-01-25T14:40:53.518Z [WARN] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: failed to contact: server-id=server-cee08b61-1b4e-0c4e-a81e-406be6ea2c7b time=50.28873ms +2025-01-25T14:40:53.520Z [ERROR] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: failed to heartbeat to: peer=c55cced2-7611-d6a7-3c1c-607c4bca828e backoff time=10ms error="failed to connect to peer: c55cced2-7611-d6a7-3c1c-607c4bca828e" +2025-01-25T14:40:53.520Z [ERROR] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: failed to appendEntries to: peer="{Voter server-c55cced2-7611-d6a7-3c1c-607c4bca828e c55cced2-7611-d6a7-3c1c-607c4bca828e}" error="failed to connect to peer: c55cced2-7611-d6a7-3c1c-607c4bca828e" +2025-01-25T14:40:53.529Z [WARN] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: failed to contact: server-id=server-c55cced2-7611-d6a7-3c1c-607c4bca828e time=53.662282ms +2025-01-25T14:40:53.529Z [WARN] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: failed to contact: server-id=server-cee08b61-1b4e-0c4e-a81e-406be6ea2c7b time=61.210054ms +2025-01-25T14:40:53.529Z [WARN] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: failed to contact quorum of nodes, stepping down +2025-01-25T14:40:53.529Z [INFO] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: entering follower state: follower="Node at c99b7fde-34bd-bbcc-7ebd-06cfe5914719 [Follower]" leader-address= leader-id= +2025-01-25T14:40:53.530Z [ERROR] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: failed to heartbeat to: peer=cee08b61-1b4e-0c4e-a81e-406be6ea2c7b backoff time=20ms error="failed to connect to peer: cee08b61-1b4e-0c4e-a81e-406be6ea2c7b" +2025-01-25T14:40:53.538Z [ERROR] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: failed to heartbeat to: peer=c55cced2-7611-d6a7-3c1c-607c4bca828e backoff time=20ms error="failed to connect to peer: c55cced2-7611-d6a7-3c1c-607c4bca828e" +2025-01-25T14:40:53.547Z [ERROR] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: failed to appendEntries to: peer="{Voter server-c55cced2-7611-d6a7-3c1c-607c4bca828e c55cced2-7611-d6a7-3c1c-607c4bca828e}" error="failed to connect to peer: c55cced2-7611-d6a7-3c1c-607c4bca828e" +2025-01-25T14:40:53.549Z [ERROR] server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719: failed to appendEntries to: peer="{Voter server-cee08b61-1b4e-0c4e-a81e-406be6ea2c7b cee08b61-1b4e-0c4e-a81e-406be6ea2c7b}" error="failed to connect to peer: cee08b61-1b4e-0c4e-a81e-406be6ea2c7b" +2025-01-25T14:40:53.558Z [WARN] server-cee08b61-1b4e-0c4e-a81e-406be6ea2c7b: heartbeat timeout reached, starting election: last-leader-addr=c99b7fde-34bd-bbcc-7ebd-06cfe5914719 last-leader-id=server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719 +2025-01-25T14:40:53.558Z [INFO] server-cee08b61-1b4e-0c4e-a81e-406be6ea2c7b: entering candidate state: node="Node at cee08b61-1b4e-0c4e-a81e-406be6ea2c7b [Candidate]" term=3 +2025-01-25T14:40:53.558Z [WARN] server-c55cced2-7611-d6a7-3c1c-607c4bca828e: rejecting vote request since we have a leader: from=cee08b61-1b4e-0c4e-a81e-406be6ea2c7b leader=c99b7fde-34bd-bbcc-7ebd-06cfe5914719 leader-id=server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719 +2025-01-25T14:40:53.558Z [ERROR] server-cee08b61-1b4e-0c4e-a81e-406be6ea2c7b: failed to make requestVote RPC: target="{Voter server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719 c99b7fde-34bd-bbcc-7ebd-06cfe5914719}" error="failed to connect to peer: c99b7fde-34bd-bbcc-7ebd-06cfe5914719" term=3 +2025-01-25T14:40:53.581Z [WARN] server-c55cced2-7611-d6a7-3c1c-607c4bca828e: heartbeat timeout reached, starting election: last-leader-addr=c99b7fde-34bd-bbcc-7ebd-06cfe5914719 last-leader-id=server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719 +2025-01-25T14:40:53.581Z [INFO] server-c55cced2-7611-d6a7-3c1c-607c4bca828e: entering candidate state: node="Node at c55cced2-7611-d6a7-3c1c-607c4bca828e [Candidate]" term=3 +2025-01-25T14:40:53.581Z [ERROR] server-c55cced2-7611-d6a7-3c1c-607c4bca828e: failed to make requestVote RPC: target="{Voter server-c99b7fde-34bd-bbcc-7ebd-06cfe5914719 c99b7fde-34bd-bbcc-7ebd-06cfe5914719}" error="failed to connect to peer: c99b7fde-34bd-bbcc-7ebd-06cfe5914719" term=3 +2025-01-25T14:40:53.581Z [INFO] server-c55cced2-7611-d6a7-3c1c-607c4bca828e: entering follower state: follower="Node at c55cced2-7611-d6a7-3c1c-607c4bca828e [Shutdown]" leader-address= leader-id= +2025-01-25T14:40:54.082Z [ERROR] server-c55cced2-7611-d6a7-3c1c-607c4bca828e: failed to make requestVote RPC: target="{Voter server-cee08b61-1b4e-0c4e-a81e-406be6ea2c7b cee08b61-1b4e-0c4e-a81e-406be6ea2c7b}" error="command timed out" term=3 +--- PASS: TestRaft_VoteWithNoIDNoAddr (1.02s) === RUN TestRaft_runFollower_State_Transition === RUN TestRaft_runFollower_State_Transition/NonVoter -2025-01-25T14:30:02.082Z [INFO] 127.0.0.1:42887 :: starting node: addr=127.0.0.1:42887 -2025-01-25T14:30:02.082Z [INFO] 127.0.0.1:42887 :: initial configuration: index=0 servers=[] -2025-01-25T14:30:02.082Z [INFO] 127.0.0.1:42887 :: entering follower state: follower="Node at 127.0.0.1:42887 [Follower]" leader-address= leader-id= -2025-01-25T14:30:02.145Z [WARN] 127.0.0.1:42887 :: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +2025-01-25T14:40:54.086Z [INFO] 127.0.0.1:33579 :: starting node: addr=127.0.0.1:33579 +2025-01-25T14:40:54.086Z [INFO] 127.0.0.1:33579 :: initial configuration: index=0 servers=[] +2025-01-25T14:40:54.087Z [INFO] 127.0.0.1:33579 :: entering follower state: follower="Node at 127.0.0.1:33579 [Follower]" leader-address= leader-id= +2025-01-25T14:40:54.180Z [WARN] 127.0.0.1:33579 :: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election === RUN TestRaft_runFollower_State_Transition/Voter -2025-01-25T14:30:02.233Z [INFO] 127.0.0.1:41847 :: starting node: addr=127.0.0.1:41847 -2025-01-25T14:30:02.233Z [INFO] 127.0.0.1:41847 :: initial configuration: index=0 servers=[] -2025-01-25T14:30:02.233Z [INFO] 127.0.0.1:41847 :: entering follower state: follower="Node at 127.0.0.1:41847 [Follower]" leader-address= leader-id= -2025-01-25T14:30:02.320Z [WARN] 127.0.0.1:41847 :: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-25T14:40:54.239Z [INFO] 127.0.0.1:40985 :: starting node: addr=127.0.0.1:40985 +2025-01-25T14:40:54.239Z [INFO] 127.0.0.1:40985 :: initial configuration: index=0 servers=[] +2025-01-25T14:40:54.239Z [INFO] 127.0.0.1:40985 :: entering follower state: follower="Node at 127.0.0.1:40985 [Follower]" leader-address= leader-id= +2025-01-25T14:40:54.327Z [WARN] 127.0.0.1:40985 :: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= === RUN TestRaft_runFollower_State_Transition/Not_in_Config -2025-01-25T14:30:02.384Z [INFO] 127.0.0.1:44491 :: starting node: addr=127.0.0.1:44491 -2025-01-25T14:30:02.384Z [INFO] 127.0.0.1:44491 :: initial configuration: index=0 servers=[] -2025-01-25T14:30:02.384Z [INFO] 127.0.0.1:44491 :: entering follower state: follower="Node at 127.0.0.1:44491 [Follower]" leader-address= leader-id= -2025-01-25T14:30:02.442Z [WARN] 127.0.0.1:44491 :: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election ---- PASS: TestRaft_runFollower_State_Transition (0.45s) +2025-01-25T14:40:54.392Z [INFO] 127.0.0.1:34649 :: starting node: addr=127.0.0.1:34649 +2025-01-25T14:40:54.392Z [INFO] 127.0.0.1:34649 :: initial configuration: index=0 servers=[] +2025-01-25T14:40:54.392Z [INFO] 127.0.0.1:34649 :: entering follower state: follower="Node at 127.0.0.1:34649 [Follower]" leader-address= leader-id= +2025-01-25T14:40:54.489Z [WARN] 127.0.0.1:34649 :: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +--- PASS: TestRaft_runFollower_State_Transition (0.46s) --- PASS: TestRaft_runFollower_State_Transition/NonVoter (0.15s) --- PASS: TestRaft_runFollower_State_Transition/Voter (0.15s) --- PASS: TestRaft_runFollower_State_Transition/Not_in_Config (0.15s) === RUN TestRaft_runFollower_ReloadTimeoutConfigs -2025-01-25T14:30:02.536Z [INFO] 127.0.0.1:39553 :: starting node: addr=127.0.0.1:39553 -2025-01-25T14:30:02.536Z [INFO] 127.0.0.1:39553 :: initial configuration: index=0 servers=[] -2025-01-25T14:30:02.536Z [INFO] 127.0.0.1:39553 :: entering follower state: follower="Node at 127.0.0.1:39553 [Follower]" leader-address= leader-id= -2025-01-25T14:30:02.536Z [WARN] 127.0.0.1:39553 :: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= ---- PASS: TestRaft_runFollower_ReloadTimeoutConfigs (0.16s) +2025-01-25T14:40:54.544Z [INFO] 127.0.0.1:37959 :: starting node: addr=127.0.0.1:37959 +2025-01-25T14:40:54.545Z [INFO] 127.0.0.1:37959 :: initial configuration: index=0 servers=[] +2025-01-25T14:40:54.545Z [INFO] 127.0.0.1:37959 :: entering follower state: follower="Node at 127.0.0.1:37959 [Follower]" leader-address= leader-id= +2025-01-25T14:40:54.545Z [WARN] 127.0.0.1:37959 :: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +--- PASS: TestRaft_runFollower_ReloadTimeoutConfigs (0.15s) === RUN TestSaturationMetric === RUN TestSaturationMetric/without_smoothing --- PASS: TestSaturationMetric (0.00s) @@ -4227,7 +4296,7 @@ === RUN TestRandomTimeout --- PASS: TestRandomTimeout (0.00s) === RUN TestNewSeed ---- PASS: TestNewSeed (0.00s) +--- PASS: TestNewSeed (0.01s) === RUN TestRandomTimeout_NoTime --- PASS: TestRandomTimeout_NoTime (0.00s) === RUN TestMin @@ -4241,7 +4310,7 @@ === RUN TestOverrideNotifyBool --- PASS: TestOverrideNotifyBool (0.00s) PASS -ok github.com/hashicorp/raft 112.578s +ok github.com/hashicorp/raft 123.796s make[1]: Leaving directory '/build/reproducible-path/golang-github-hashicorp-raft-1.5.0' create-stamp debian/debhelper-build-stamp dh_testroot -O--buildsystem=golang @@ -4269,12 +4338,14 @@ dpkg-buildpackage: info: binary-only upload (no source included) dpkg-genchanges: info: including full source code in upload I: copying local configuration +I: user script /srv/workspace/pbuilder/29299/tmp/hooks/B01_cleanup starting +I: user script /srv/workspace/pbuilder/29299/tmp/hooks/B01_cleanup finished I: unmounting dev/ptmx filesystem I: unmounting dev/pts filesystem I: unmounting dev/shm filesystem I: unmounting proc filesystem I: unmounting sys filesystem I: cleaning the build env -I: removing directory /srv/workspace/pbuilder/22464 and its subdirectories -I: Current time: Sat Jan 25 02:30:16 -12 2025 -I: pbuilder-time-stamp: 1737815416 +I: removing directory /srv/workspace/pbuilder/29299 and its subdirectories +I: Current time: Sun Jan 26 04:41:24 +14 2025 +I: pbuilder-time-stamp: 1737816084