Diff of the two buildlogs: -- --- b1/build.log 2025-01-28 14:49:42.289492612 +0000 +++ b2/build.log 2025-01-28 14:56:18.007651183 +0000 @@ -1,6 +1,6 @@ I: pbuilder: network access will be disabled during build -I: Current time: Tue Jan 28 02:38:43 -12 2025 -I: pbuilder-time-stamp: 1738075123 +I: Current time: Wed Jan 29 04:49:51 +14 2025 +I: pbuilder-time-stamp: 1738075791 I: Building the build Environment I: extracting base tarball [/var/cache/pbuilder/unstable-reproducible-base.tgz] I: copying local configuration @@ -24,52 +24,84 @@ dpkg-source: info: applying 0001-Skip-unreliable-tests.patch I: Not using root during the build. I: Installing the build-deps -I: user script /srv/workspace/pbuilder/5020/tmp/hooks/D02_print_environment starting +I: user script /srv/workspace/pbuilder/31365/tmp/hooks/D01_modify_environment starting +debug: Running on virt64a. +I: Changing host+domainname to test build reproducibility +I: Adding a custom variable just for the fun of it... +I: Changing /bin/sh to bash +'/bin/sh' -> '/bin/bash' +lrwxrwxrwx 1 root root 9 Jan 28 14:50 /bin/sh -> /bin/bash +I: Setting pbuilder2's login shell to /bin/bash +I: Setting pbuilder2's GECOS to second user,second room,second work-phone,second home-phone,second other +I: user script /srv/workspace/pbuilder/31365/tmp/hooks/D01_modify_environment finished +I: user script /srv/workspace/pbuilder/31365/tmp/hooks/D02_print_environment starting I: set - BUILDDIR='/build/reproducible-path' - BUILDUSERGECOS='first user,first room,first work-phone,first home-phone,first other' - BUILDUSERNAME='pbuilder1' - BUILD_ARCH='armhf' - DEBIAN_FRONTEND='noninteractive' - DEB_BUILD_OPTIONS='buildinfo=+all reproducible=+all parallel=3 ' - DISTRIBUTION='unstable' - HOME='/root' - HOST_ARCH='armhf' + BASH=/bin/sh + BASHOPTS=checkwinsize:cmdhist:complete_fullquote:extquote:force_fignore:globasciiranges:globskipdots:hostcomplete:interactive_comments:patsub_replacement:progcomp:promptvars:sourcepath + BASH_ALIASES=() + BASH_ARGC=() + BASH_ARGV=() + BASH_CMDS=() + BASH_LINENO=([0]="12" [1]="0") + BASH_LOADABLES_PATH=/usr/local/lib/bash:/usr/lib/bash:/opt/local/lib/bash:/usr/pkg/lib/bash:/opt/pkg/lib/bash:. + BASH_SOURCE=([0]="/tmp/hooks/D02_print_environment" [1]="/tmp/hooks/D02_print_environment") + BASH_VERSINFO=([0]="5" [1]="2" [2]="37" [3]="1" [4]="release" [5]="arm-unknown-linux-gnueabihf") + BASH_VERSION='5.2.37(1)-release' + BUILDDIR=/build/reproducible-path + BUILDUSERGECOS='second user,second room,second work-phone,second home-phone,second other' + BUILDUSERNAME=pbuilder2 + BUILD_ARCH=armhf + DEBIAN_FRONTEND=noninteractive + DEB_BUILD_OPTIONS='buildinfo=+all reproducible=+all parallel=4 ' + DIRSTACK=() + DISTRIBUTION=unstable + EUID=0 + FUNCNAME=([0]="Echo" [1]="main") + GROUPS=() + HOME=/root + HOSTNAME=i-capture-the-hostname + HOSTTYPE=arm + HOST_ARCH=armhf IFS=' ' - INVOCATION_ID='28571caa5af849b4bffd30c66d7e80c7' - LANG='C' - LANGUAGE='en_US:en' - LC_ALL='C' - MAIL='/var/mail/root' - OPTIND='1' - PATH='/usr/sbin:/usr/bin:/sbin:/bin:/usr/games' - PBCURRENTCOMMANDLINEOPERATION='build' - PBUILDER_OPERATION='build' - PBUILDER_PKGDATADIR='/usr/share/pbuilder' - PBUILDER_PKGLIBDIR='/usr/lib/pbuilder' - PBUILDER_SYSCONFDIR='/etc' - PPID='5020' - PS1='# ' - PS2='> ' + INVOCATION_ID=1f1135c6ed76417ab154bb54404cfc95 + LANG=C + LANGUAGE=it_CH:it + LC_ALL=C + MACHTYPE=arm-unknown-linux-gnueabihf + MAIL=/var/mail/root + OPTERR=1 + OPTIND=1 + OSTYPE=linux-gnueabihf + PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path + PBCURRENTCOMMANDLINEOPERATION=build + PBUILDER_OPERATION=build + PBUILDER_PKGDATADIR=/usr/share/pbuilder + PBUILDER_PKGLIBDIR=/usr/lib/pbuilder + PBUILDER_SYSCONFDIR=/etc + PIPESTATUS=([0]="0") + POSIXLY_CORRECT=y + PPID=31365 PS4='+ ' - PWD='/' - SHELL='/bin/bash' - SHLVL='2' - SUDO_COMMAND='/usr/bin/timeout -k 18.1h 18h /usr/bin/ionice -c 3 /usr/bin/nice /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/r-b-build.mPRYFda3/pbuilderrc_mml7 --distribution unstable --hookdir /etc/pbuilder/first-build-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/unstable-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/r-b-build.mPRYFda3/b1 --logfile b1/build.log golang-github-hashicorp-raft_1.5.0-1.dsc' - SUDO_GID='113' - SUDO_UID='107' - SUDO_USER='jenkins' - TERM='unknown' - TZ='/usr/share/zoneinfo/Etc/GMT+12' - USER='root' - _='/usr/bin/systemd-run' - http_proxy='http://10.0.0.15:3142/' + PWD=/ + SHELL=/bin/bash + SHELLOPTS=braceexpand:errexit:hashall:interactive-comments:posix + SHLVL=3 + SUDO_COMMAND='/usr/bin/timeout -k 24.1h 24h /usr/bin/ionice -c 3 /usr/bin/nice -n 11 /usr/bin/unshare --uts -- /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/r-b-build.mPRYFda3/pbuilderrc_yuu9 --distribution unstable --hookdir /etc/pbuilder/rebuild-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/unstable-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/r-b-build.mPRYFda3/b2 --logfile b2/build.log golang-github-hashicorp-raft_1.5.0-1.dsc' + SUDO_GID=114 + SUDO_UID=108 + SUDO_USER=jenkins + TERM=unknown + TZ=/usr/share/zoneinfo/Etc/GMT-14 + UID=0 + USER=root + _='I: set' + http_proxy=http://10.0.0.15:3142/ I: uname -a - Linux wbq0 6.1.0-30-armmp #1 SMP Debian 6.1.124-1 (2025-01-12) armv7l GNU/Linux + Linux i-capture-the-hostname 6.1.0-30-arm64 #1 SMP Debian 6.1.124-1 (2025-01-12) aarch64 GNU/Linux I: ls -l /bin lrwxrwxrwx 1 root root 7 Nov 22 14:40 /bin -> usr/bin -I: user script /srv/workspace/pbuilder/5020/tmp/hooks/D02_print_environment finished +I: user script /srv/workspace/pbuilder/31365/tmp/hooks/D02_print_environment finished -> Attempting to satisfy build-dependencies -> Creating pbuilder-satisfydepends-dummy package Package: pbuilder-satisfydepends-dummy @@ -223,7 +255,7 @@ Get: 104 http://deb.debian.org/debian unstable/main armhf golang-gopkg-check.v1-dev all 0.0+git20200902.038fdea-1 [32.1 kB] Get: 105 http://deb.debian.org/debian unstable/main armhf golang-gopkg-mgo.v2-dev all 2016.08.01-7 [316 kB] Get: 106 http://deb.debian.org/debian unstable/main armhf golang-github-hashicorp-go-msgpack-dev all 0.5.5-1 [43.3 kB] -Fetched 123 MB in 13s (9639 kB/s) +Fetched 123 MB in 24s (5136 kB/s) Preconfiguring packages ... Selecting previously unselected package sensible-utils. (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 19568 files and directories currently installed.) @@ -662,7 +694,11 @@ Building tag database... -> Finished parsing the build-deps I: Building the package -I: Running cd /build/reproducible-path/golang-github-hashicorp-raft-1.5.0/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" HOME="/nonexistent/first-build" dpkg-buildpackage -us -uc -b && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" HOME="/nonexistent/first-build" dpkg-genchanges -S > ../golang-github-hashicorp-raft_1.5.0-1_source.changes +I: user script /srv/workspace/pbuilder/31365/tmp/hooks/A99_set_merged_usr starting +Not re-configuring usrmerge for unstable +I: user script /srv/workspace/pbuilder/31365/tmp/hooks/A99_set_merged_usr finished +hostname: Name or service not known +I: Running cd /build/reproducible-path/golang-github-hashicorp-raft-1.5.0/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" HOME="/nonexistent/second-build" dpkg-buildpackage -us -uc -b && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" HOME="/nonexistent/second-build" dpkg-genchanges -S > ../golang-github-hashicorp-raft_1.5.0-1_source.changes dpkg-buildpackage: info: source package golang-github-hashicorp-raft dpkg-buildpackage: info: source version 1.5.0-1 dpkg-buildpackage: info: source distribution unstable @@ -680,17 +716,17 @@ dh_autoreconf -O--buildsystem=golang dh_auto_configure -O--buildsystem=golang dh_auto_build -O--buildsystem=golang - cd obj-arm-linux-gnueabihf && go install -trimpath -v -p 3 github.com/hashicorp/raft github.com/hashicorp/raft/bench + cd obj-arm-linux-gnueabihf && go install -trimpath -v -p 4 github.com/hashicorp/raft github.com/hashicorp/raft/bench internal/unsafeheader +internal/byteorder internal/goarch internal/cpu -internal/byteorder +internal/coverage/rtcov internal/abi internal/chacha8rand -internal/bytealg -internal/coverage/rtcov internal/godebugs internal/goexperiment +internal/bytealg internal/goos internal/profilerecord internal/runtime/atomic @@ -698,158 +734,158 @@ internal/stringslite runtime/internal/math runtime/internal/sys -internal/runtime/exithook internal/race sync/atomic unicode +internal/runtime/exithook unicode/utf8 container/list -runtime internal/asan internal/itoa internal/msan math/bits -math crypto/internal/alias crypto/internal/boring/sig cmp +math encoding unicode/utf16 log/internal internal/nettrace +runtime vendor/golang.org/x/crypto/cryptobyte/asn1 vendor/golang.org/x/crypto/internal/alias +iter crypto/subtle internal/reflectlite sync -iter slices internal/weak maps -internal/testlog internal/bisect +internal/testlog internal/singleflight +runtime/cgo errors sort io internal/godebug internal/oserror -syscall bytes strings +syscall hash strconv bufio crypto/cipher -crypto/internal/randutil crypto +crypto/internal/randutil reflect crypto/internal/boring path -math/rand time internal/syscall/unix internal/syscall/execenv +math/rand github.com/hashicorp/golang-lru/simplelru github.com/hashicorp/go-immutable-radix vendor/golang.org/x/net/dns/dnsmessage +math/rand/v2 context io/fs internal/poll internal/filepathlite -math/rand/v2 -os internal/concurrent -unique -internal/fmtsort -encoding/binary -net/netip -encoding/base64 -runtime/cgo hash/crc32 -fmt +unique crypto/aes +net/netip crypto/des +os crypto/internal/edwards25519/field crypto/internal/nistec/fiat +internal/fmtsort +encoding/binary +crypto/sha512 +fmt +encoding/base64 +net +crypto/internal/edwards25519 +crypto/hmac +vendor/golang.org/x/crypto/chacha20 +vendor/golang.org/x/crypto/internal/poly1305 math/big encoding/json -crypto/rand log -net compress/flate crypto/internal/nistec compress/gzip +vendor/golang.org/x/crypto/chacha20poly1305 +crypto/rand crypto/internal/bigmod crypto/internal/boring/bbig -crypto/sha512 +encoding/asn1 crypto/ecdh crypto/elliptic -encoding/asn1 -crypto/internal/edwards25519 -vendor/golang.org/x/crypto/cryptobyte crypto/ed25519 -crypto/hmac -vendor/golang.org/x/crypto/chacha20 -crypto/ecdsa -vendor/golang.org/x/crypto/internal/poly1305 -vendor/golang.org/x/crypto/chacha20poly1305 vendor/golang.org/x/crypto/hkdf -vendor/golang.org/x/sys/cpu crypto/internal/hpke -vendor/golang.org/x/crypto/sha3 +vendor/golang.org/x/crypto/cryptobyte +vendor/golang.org/x/sys/cpu crypto/md5 +vendor/golang.org/x/crypto/sha3 crypto/rc4 crypto/rsa +crypto/ecdsa crypto/internal/mlkem768 crypto/sha1 crypto/sha256 crypto/dsa encoding/hex encoding/pem -crypto/x509/pkix net/url +crypto/x509/pkix path/filepath vendor/golang.org/x/text/transform vendor/golang.org/x/text/unicode/bidi vendor/golang.org/x/text/unicode/norm -vendor/golang.org/x/text/secure/bidirule vendor/golang.org/x/net/http2/hpack mime -vendor/golang.org/x/net/idna +vendor/golang.org/x/text/secure/bidirule mime/quotedprintable net/http/internal net/http/internal/ascii os/signal golang.org/x/sys/unix +vendor/golang.org/x/net/idna +io/ioutil +regexp/syntax +internal/saferio +encoding/gob +regexp crypto/x509 net/textproto -vendor/golang.org/x/net/http/httpguts vendor/golang.org/x/net/http/httpproxy +vendor/golang.org/x/net/http/httpguts mime/multipart -io/ioutil -regexp/syntax -crypto/tls -regexp github.com/mattn/go-isatty github.com/mattn/go-colorable github.com/fatih/color -internal/saferio github.com/hashicorp/go-hclog -encoding/gob go/token html text/template/parse hash/crc64 flag -text/template +crypto/tls internal/sysinfo runtime/debug -net/http/httptrace runtime/trace -net/http testing +text/template html/template +net/http/httptrace +net/http net/rpc github.com/armon/go-metrics github.com/hashicorp/go-msgpack/codec @@ -858,7 +894,7 @@ debian/rules override_dh_auto_test make[1]: Entering directory '/build/reproducible-path/golang-github-hashicorp-raft-1.5.0' dh_auto_test - cd obj-arm-linux-gnueabihf && go test -vet=off -v -p 3 github.com/hashicorp/raft github.com/hashicorp/raft/bench + cd obj-arm-linux-gnueabihf && go test -vet=off -v -p 4 github.com/hashicorp/raft github.com/hashicorp/raft/bench ? github.com/hashicorp/raft/bench [no test files] === RUN TestCommitment_setVoters --- PASS: TestCommitment_setVoters (0.00s) @@ -901,33 +937,33 @@ === RUN TestFileSnapshotSinkImpl --- PASS: TestFileSnapshotSinkImpl (0.00s) === RUN TestFileSS_CreateSnapshotMissingParentDir -2025-01-28T14:46:56.937Z [INFO] creating new snapshot: path=/tmp/raft283189632/raft1113880266/snapshots/3-10-1738075616937.tmp ---- PASS: TestFileSS_CreateSnapshotMissingParentDir (0.01s) +2025-01-28T14:53:34.332Z [INFO] creating new snapshot: path=/tmp/raft1018138551/raft3254672211/snapshots/3-10-1738076014332.tmp +--- PASS: TestFileSS_CreateSnapshotMissingParentDir (23.05s) === RUN TestFileSS_CreateSnapshot -2025-01-28T14:46:56.948Z [INFO] creating new snapshot: path=/tmp/raft1913927195/snapshots/3-10-1738075616948.tmp -2025-01-28T14:46:56.952Z [WARN] found temporary snapshot: name=3-10-1738075616948.tmp ---- PASS: TestFileSS_CreateSnapshot (0.01s) +2025-01-28T14:53:57.378Z [INFO] creating new snapshot: path=/tmp/raft1768655196/snapshots/3-10-1738076037378.tmp +2025-01-28T14:53:57.465Z [WARN] found temporary snapshot: name=3-10-1738076037378.tmp +--- PASS: TestFileSS_CreateSnapshot (0.13s) === RUN TestFileSS_CancelSnapshot -2025-01-28T14:46:56.962Z [INFO] creating new snapshot: path=/tmp/raft220914051/snapshots/3-10-1738075616962.tmp ---- PASS: TestFileSS_CancelSnapshot (0.01s) +2025-01-28T14:53:57.505Z [INFO] creating new snapshot: path=/tmp/raft220298489/snapshots/3-10-1738076037505.tmp +--- PASS: TestFileSS_CancelSnapshot (0.02s) === RUN TestFileSS_Retention -2025-01-28T14:46:56.970Z [INFO] creating new snapshot: path=/tmp/raft1508563252/snapshots/3-10-1738075616970.tmp -2025-01-28T14:46:56.979Z [INFO] creating new snapshot: path=/tmp/raft1508563252/snapshots/3-11-1738075616979.tmp -2025-01-28T14:46:56.990Z [INFO] creating new snapshot: path=/tmp/raft1508563252/snapshots/3-12-1738075616989.tmp -2025-01-28T14:46:57.000Z [INFO] reaping snapshot: path=/tmp/raft1508563252/snapshots/3-10-1738075616970 -2025-01-28T14:46:57.001Z [INFO] creating new snapshot: path=/tmp/raft1508563252/snapshots/3-13-1738075617001.tmp -2025-01-28T14:46:57.011Z [INFO] reaping snapshot: path=/tmp/raft1508563252/snapshots/3-11-1738075616979 -2025-01-28T14:46:57.013Z [INFO] creating new snapshot: path=/tmp/raft1508563252/snapshots/3-14-1738075617013.tmp -2025-01-28T14:46:57.023Z [INFO] reaping snapshot: path=/tmp/raft1508563252/snapshots/3-12-1738075616989 ---- PASS: TestFileSS_Retention (0.06s) +2025-01-28T14:53:57.529Z [INFO] creating new snapshot: path=/tmp/raft3110082571/snapshots/3-10-1738076037529.tmp +2025-01-28T14:53:57.584Z [INFO] creating new snapshot: path=/tmp/raft3110082571/snapshots/3-11-1738076037584.tmp +2025-01-28T14:53:57.612Z [INFO] creating new snapshot: path=/tmp/raft3110082571/snapshots/3-12-1738076037612.tmp +2025-01-28T14:53:57.651Z [INFO] reaping snapshot: path=/tmp/raft3110082571/snapshots/3-10-1738076037529 +2025-01-28T14:53:57.652Z [INFO] creating new snapshot: path=/tmp/raft3110082571/snapshots/3-13-1738076037652.tmp +2025-01-28T14:53:59.683Z [INFO] reaping snapshot: path=/tmp/raft3110082571/snapshots/3-11-1738076037584 +2025-01-28T14:53:59.684Z [INFO] creating new snapshot: path=/tmp/raft3110082571/snapshots/3-14-1738076039684.tmp +2025-01-28T14:53:59.703Z [INFO] reaping snapshot: path=/tmp/raft3110082571/snapshots/3-12-1738076037612 +--- PASS: TestFileSS_Retention (2.18s) === RUN TestFileSS_BadPerm --- PASS: TestFileSS_BadPerm (0.00s) === RUN TestFileSS_MissingParentDir --- PASS: TestFileSS_MissingParentDir (0.00s) === RUN TestFileSS_Ordering -2025-01-28T14:46:57.036Z [INFO] creating new snapshot: path=/tmp/raft967178168/snapshots/5-130350-1738075617036.tmp -2025-01-28T14:46:57.046Z [INFO] creating new snapshot: path=/tmp/raft967178168/snapshots/36-204917-1738075617046.tmp ---- PASS: TestFileSS_Ordering (0.02s) +2025-01-28T14:53:59.709Z [INFO] creating new snapshot: path=/tmp/raft787323634/snapshots/5-130350-1738076039709.tmp +2025-01-28T14:53:59.732Z [INFO] creating new snapshot: path=/tmp/raft787323634/snapshots/36-204917-1738076039731.tmp +--- PASS: TestFileSS_Ordering (0.04s) === RUN TestDeferFutureSuccess --- PASS: TestDeferFutureSuccess (0.00s) === RUN TestDeferFutureError @@ -963,62 +999,62 @@ === RUN TestEmitsLogStoreMetrics --- PASS: TestEmitsLogStoreMetrics (0.01s) === RUN TestNetworkTransport_CloseStreams ---- PASS: TestNetworkTransport_CloseStreams (0.02s) +--- PASS: TestNetworkTransport_CloseStreams (0.01s) === RUN TestNetworkTransport_StartStop --- PASS: TestNetworkTransport_StartStop (0.00s) === RUN TestNetworkTransport_Heartbeat_FastPath --- PASS: TestNetworkTransport_Heartbeat_FastPath (0.00s) === RUN TestNetworkTransport_AppendEntries ---- PASS: TestNetworkTransport_AppendEntries (0.01s) +--- PASS: TestNetworkTransport_AppendEntries (0.00s) === RUN TestNetworkTransport_AppendEntriesPipeline ---- PASS: TestNetworkTransport_AppendEntriesPipeline (0.02s) +--- PASS: TestNetworkTransport_AppendEntriesPipeline (0.01s) === RUN TestNetworkTransport_AppendEntriesPipeline_CloseStreams ---- PASS: TestNetworkTransport_AppendEntriesPipeline_CloseStreams (0.06s) +--- PASS: TestNetworkTransport_AppendEntriesPipeline_CloseStreams (0.03s) === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=0 +2025-01-28T14:53:59.879Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=1 -2025-01-28T14:46:57.258Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=2 -2025-01-28T14:46:57.318Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" +2025-01-28T14:53:59.932Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=10 -2025-01-28T14:46:57.381Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" ---- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight (0.18s) +2025-01-28T14:53:59.986Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" +--- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight (0.16s) --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=0 (0.05s) --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=1 (0.00s) - --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=2 (0.06s) - --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=10 (0.06s) + --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=2 (0.05s) + --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=10 (0.05s) === RUN TestNetworkTransport_RequestVote ---- PASS: TestNetworkTransport_RequestVote (0.01s) +--- PASS: TestNetworkTransport_RequestVote (0.00s) === RUN TestNetworkTransport_InstallSnapshot ---- PASS: TestNetworkTransport_InstallSnapshot (0.02s) +--- PASS: TestNetworkTransport_InstallSnapshot (0.00s) === RUN TestNetworkTransport_EncodeDecode --- PASS: TestNetworkTransport_EncodeDecode (0.00s) === RUN TestNetworkTransport_EncodeDecode_AddressProvider --- PASS: TestNetworkTransport_EncodeDecode_AddressProvider (0.00s) === RUN TestNetworkTransport_PooledConn ---- PASS: TestNetworkTransport_PooledConn (0.03s) +--- PASS: TestNetworkTransport_PooledConn (0.00s) === RUN TestNetworkTransport_ListenBackoff - net_transport_test.go:839: countingWriter: 2025-01-28T14:46:57.443Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-01-28T14:54:00.000Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-01-28T14:46:57.449Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-01-28T14:54:00.005Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-01-28T14:46:57.460Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-01-28T14:54:00.016Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-01-28T14:46:57.481Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-01-28T14:54:00.036Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-01-28T14:46:57.522Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-01-28T14:54:00.077Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-01-28T14:46:57.603Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-01-28T14:54:00.158Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-01-28T14:46:57.764Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-01-28T14:54:00.319Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-01-28T14:46:58.085Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-01-28T14:54:00.640Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-01-28T14:46:58.726Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-01-28T14:54:01.281Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-01-28T14:46:59.730Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-01-28T14:54:02.284Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-01-28T14:47:00.733Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2025-01-28T14:54:03.287Z [ERROR] test: failed to accept connection: error="intentional error in test" --- PASS: TestNetworkTransport_ListenBackoff (4.00s) === RUN TestPeersJSON_BadConfiguration @@ -1028,2595 +1064,2556 @@ === RUN TestPeersJSON_ReadConfigJSON --- PASS: TestPeersJSON_ReadConfigJSON (0.00s) === RUN TestRaft_StartStop -2025-01-28T14:47:01.463Z [INFO] server-3358659b-b3cf-31b1-e880-6378dbcd77c4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3358659b-b3cf-31b1-e880-6378dbcd77c4 Address:3358659b-b3cf-31b1-e880-6378dbcd77c4}]" ---- PASS: TestRaft_StartStop (0.01s) +2025-01-28T14:54:04.006Z [INFO] server-1a660542-3c61-e666-67c9-c3101f2eedb6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1a660542-3c61-e666-67c9-c3101f2eedb6 Address:1a660542-3c61-e666-67c9-c3101f2eedb6}]" +--- PASS: TestRaft_StartStop (0.00s) === RUN TestRaft_AfterShutdown -2025-01-28T14:47:01.471Z [INFO] server-a67335e7-2352-3a79-b424-4492a7a4a57e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a67335e7-2352-3a79-b424-4492a7a4a57e Address:a67335e7-2352-3a79-b424-4492a7a4a57e}]" ---- PASS: TestRaft_AfterShutdown (0.01s) +2025-01-28T14:54:04.008Z [INFO] server-dfb916f7-f85a-0916-eaa0-325d42f3738f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dfb916f7-f85a-0916-eaa0-325d42f3738f Address:dfb916f7-f85a-0916-eaa0-325d42f3738f}]" +--- PASS: TestRaft_AfterShutdown (0.00s) === RUN TestRaft_LiveBootstrap -2025-01-28T14:47:01.485Z [INFO] server-79546aa9-537c-b229-10aa-db3e354499e8: initial configuration: index=0 servers=[] -2025-01-28T14:47:01.486Z [INFO] server-7724a847-9ed4-aa13-63c6-6094d6b5e8d6: initial configuration: index=0 servers=[] -2025-01-28T14:47:01.486Z [INFO] server-79546aa9-537c-b229-10aa-db3e354499e8: entering follower state: follower="Node at 79546aa9-537c-b229-10aa-db3e354499e8 [Follower]" leader-address= leader-id= -2025-01-28T14:47:01.487Z [INFO] server-7724a847-9ed4-aa13-63c6-6094d6b5e8d6: entering follower state: follower="Node at 7724a847-9ed4-aa13-63c6-6094d6b5e8d6 [Follower]" leader-address= leader-id= -2025-01-28T14:47:01.487Z [INFO] server-45feca13-291e-9a5e-0004-3a482b181ca4: initial configuration: index=0 servers=[] -2025-01-28T14:47:01.487Z [INFO] server-45feca13-291e-9a5e-0004-3a482b181ca4: entering follower state: follower="Node at 45feca13-291e-9a5e-0004-3a482b181ca4 [Follower]" leader-address= leader-id= -2025-01-28T14:47:01.488Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:47:01.541Z [WARN] server-45feca13-291e-9a5e-0004-3a482b181ca4: no known peers, aborting election -2025-01-28T14:47:01.550Z [WARN] server-79546aa9-537c-b229-10aa-db3e354499e8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:01.550Z [INFO] server-79546aa9-537c-b229-10aa-db3e354499e8: entering candidate state: node="Node at 79546aa9-537c-b229-10aa-db3e354499e8 [Candidate]" term=2 -2025-01-28T14:47:01.551Z [INFO] server-79546aa9-537c-b229-10aa-db3e354499e8: election won: term=2 tally=2 -2025-01-28T14:47:01.551Z [INFO] server-79546aa9-537c-b229-10aa-db3e354499e8: entering leader state: leader="Node at 79546aa9-537c-b229-10aa-db3e354499e8 [Leader]" -2025-01-28T14:47:01.552Z [INFO] server-79546aa9-537c-b229-10aa-db3e354499e8: added peer, starting replication: peer=server-7724a847-9ed4-aa13-63c6-6094d6b5e8d6 -2025-01-28T14:47:01.552Z [INFO] server-79546aa9-537c-b229-10aa-db3e354499e8: added peer, starting replication: peer=server-45feca13-291e-9a5e-0004-3a482b181ca4 -2025-01-28T14:47:01.552Z [WARN] server-7724a847-9ed4-aa13-63c6-6094d6b5e8d6: failed to get previous log: previous-index=1 last-index=0 error="log not found" -2025-01-28T14:47:01.553Z [WARN] server-45feca13-291e-9a5e-0004-3a482b181ca4: failed to get previous log: previous-index=1 last-index=0 error="log not found" -2025-01-28T14:47:01.553Z [WARN] server-79546aa9-537c-b229-10aa-db3e354499e8: appendEntries rejected, sending older logs: peer="{Voter server-7724a847-9ed4-aa13-63c6-6094d6b5e8d6 7724a847-9ed4-aa13-63c6-6094d6b5e8d6}" next=1 -2025-01-28T14:47:01.554Z [WARN] server-79546aa9-537c-b229-10aa-db3e354499e8: appendEntries rejected, sending older logs: peer="{Voter server-45feca13-291e-9a5e-0004-3a482b181ca4 45feca13-291e-9a5e-0004-3a482b181ca4}" next=1 -2025-01-28T14:47:01.555Z [INFO] server-79546aa9-537c-b229-10aa-db3e354499e8: pipelining replication: peer="{Voter server-7724a847-9ed4-aa13-63c6-6094d6b5e8d6 7724a847-9ed4-aa13-63c6-6094d6b5e8d6}" -2025-01-28T14:47:01.555Z [INFO] server-79546aa9-537c-b229-10aa-db3e354499e8: pipelining replication: peer="{Voter server-45feca13-291e-9a5e-0004-3a482b181ca4 45feca13-291e-9a5e-0004-3a482b181ca4}" -2025-01-28T14:47:01.657Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:47:01.551970922 +0000 UTC m=+4.635377280 (2 nodes), 63.063205ms from start of poll, 66.794983ms from cluster start. Timeout at 2025-01-28 14:47:01.65699259 +0000 UTC m=+4.740398615, 105.021335ms after stability -2025-01-28T14:47:01.658Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:01.763Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:01.658408749 +0000 UTC m=+4.741814440 (1 nodes), 12.334µs from start of poll, 173.232143ms from cluster start. Timeout at 2025-01-28 14:47:01.763607755 +0000 UTC m=+4.847014779, 105.200339ms after stability -2025-01-28T14:47:01.778Z [INFO] server-79546aa9-537c-b229-10aa-db3e354499e8: aborting pipeline replication: peer="{Voter server-45feca13-291e-9a5e-0004-3a482b181ca4 45feca13-291e-9a5e-0004-3a482b181ca4}" -2025-01-28T14:47:01.778Z [INFO] server-79546aa9-537c-b229-10aa-db3e354499e8: aborting pipeline replication: peer="{Voter server-7724a847-9ed4-aa13-63c6-6094d6b5e8d6 7724a847-9ed4-aa13-63c6-6094d6b5e8d6}" ---- PASS: TestRaft_LiveBootstrap (0.31s) +2025-01-28T14:54:04.011Z [INFO] server-ab0ff5c1-30b6-d579-ae24-70044e9c067e: initial configuration: index=0 servers=[] +2025-01-28T14:54:04.011Z [INFO] server-ba9b9407-a8bd-3c6d-65f8-4754fac01422: initial configuration: index=0 servers=[] +2025-01-28T14:54:04.011Z [INFO] server-ab0ff5c1-30b6-d579-ae24-70044e9c067e: entering follower state: follower="Node at ab0ff5c1-30b6-d579-ae24-70044e9c067e [Follower]" leader-address= leader-id= +2025-01-28T14:54:04.011Z [INFO] server-ba9b9407-a8bd-3c6d-65f8-4754fac01422: entering follower state: follower="Node at ba9b9407-a8bd-3c6d-65f8-4754fac01422 [Follower]" leader-address= leader-id= +2025-01-28T14:54:04.011Z [INFO] server-3b972842-c6f4-61c6-b7ab-7b09a3b1c352: initial configuration: index=0 servers=[] +2025-01-28T14:54:04.012Z [INFO] server-3b972842-c6f4-61c6-b7ab-7b09a3b1c352: entering follower state: follower="Node at 3b972842-c6f4-61c6-b7ab-7b09a3b1c352 [Follower]" leader-address= leader-id= +2025-01-28T14:54:04.012Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:54:04.068Z [WARN] server-3b972842-c6f4-61c6-b7ab-7b09a3b1c352: no known peers, aborting election +2025-01-28T14:54:04.083Z [WARN] server-ab0ff5c1-30b6-d579-ae24-70044e9c067e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:04.083Z [INFO] server-ab0ff5c1-30b6-d579-ae24-70044e9c067e: entering candidate state: node="Node at ab0ff5c1-30b6-d579-ae24-70044e9c067e [Candidate]" term=2 +2025-01-28T14:54:04.083Z [INFO] server-ab0ff5c1-30b6-d579-ae24-70044e9c067e: election won: term=2 tally=2 +2025-01-28T14:54:04.083Z [INFO] server-ab0ff5c1-30b6-d579-ae24-70044e9c067e: entering leader state: leader="Node at ab0ff5c1-30b6-d579-ae24-70044e9c067e [Leader]" +2025-01-28T14:54:04.083Z [INFO] server-ab0ff5c1-30b6-d579-ae24-70044e9c067e: added peer, starting replication: peer=server-ba9b9407-a8bd-3c6d-65f8-4754fac01422 +2025-01-28T14:54:04.083Z [INFO] server-ab0ff5c1-30b6-d579-ae24-70044e9c067e: added peer, starting replication: peer=server-3b972842-c6f4-61c6-b7ab-7b09a3b1c352 +2025-01-28T14:54:04.084Z [WARN] server-3b972842-c6f4-61c6-b7ab-7b09a3b1c352: failed to get previous log: previous-index=1 last-index=0 error="log not found" +2025-01-28T14:54:04.084Z [WARN] server-ba9b9407-a8bd-3c6d-65f8-4754fac01422: failed to get previous log: previous-index=1 last-index=0 error="log not found" +2025-01-28T14:54:04.084Z [WARN] server-ab0ff5c1-30b6-d579-ae24-70044e9c067e: appendEntries rejected, sending older logs: peer="{Voter server-3b972842-c6f4-61c6-b7ab-7b09a3b1c352 3b972842-c6f4-61c6-b7ab-7b09a3b1c352}" next=1 +2025-01-28T14:54:04.084Z [WARN] server-ab0ff5c1-30b6-d579-ae24-70044e9c067e: appendEntries rejected, sending older logs: peer="{Voter server-ba9b9407-a8bd-3c6d-65f8-4754fac01422 ba9b9407-a8bd-3c6d-65f8-4754fac01422}" next=1 +2025-01-28T14:54:04.084Z [INFO] server-ab0ff5c1-30b6-d579-ae24-70044e9c067e: pipelining replication: peer="{Voter server-3b972842-c6f4-61c6-b7ab-7b09a3b1c352 3b972842-c6f4-61c6-b7ab-7b09a3b1c352}" +2025-01-28T14:54:04.084Z [INFO] server-ab0ff5c1-30b6-d579-ae24-70044e9c067e: pipelining replication: peer="{Voter server-ba9b9407-a8bd-3c6d-65f8-4754fac01422 ba9b9407-a8bd-3c6d-65f8-4754fac01422}" +2025-01-28T14:54:04.189Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:54:04.08387253 +0000 UTC m=+29.758362530 (2 nodes), 71.416523ms from start of poll, 72.365033ms from cluster start. Timeout at 2025-01-28 14:54:04.188881609 +0000 UTC m=+29.863371589, 105.009059ms after stability +2025-01-28T14:54:04.189Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:04.295Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:04.189903007 +0000 UTC m=+29.864392967 (1 nodes), 3.3µs from start of poll, 178.39547ms from cluster start. Timeout at 2025-01-28 14:54:04.294917286 +0000 UTC m=+29.969407286, 105.014319ms after stability +2025-01-28T14:54:04.306Z [INFO] server-ab0ff5c1-30b6-d579-ae24-70044e9c067e: aborting pipeline replication: peer="{Voter server-3b972842-c6f4-61c6-b7ab-7b09a3b1c352 3b972842-c6f4-61c6-b7ab-7b09a3b1c352}" +2025-01-28T14:54:04.306Z [INFO] server-ab0ff5c1-30b6-d579-ae24-70044e9c067e: aborting pipeline replication: peer="{Voter server-ba9b9407-a8bd-3c6d-65f8-4754fac01422 ba9b9407-a8bd-3c6d-65f8-4754fac01422}" +--- PASS: TestRaft_LiveBootstrap (0.30s) === RUN TestRaft_LiveBootstrap_From_NonVoter -2025-01-28T14:47:01.793Z [INFO] server-48bb254e-ac56-6b6c-0499-21b5f200f092: initial configuration: index=0 servers=[] -2025-01-28T14:47:01.794Z [INFO] server-25fdb754-f485-ac62-c7bb-c30efa6ef312: initial configuration: index=0 servers=[] -2025-01-28T14:47:01.793Z [INFO] server-48bb254e-ac56-6b6c-0499-21b5f200f092: entering follower state: follower="Node at 48bb254e-ac56-6b6c-0499-21b5f200f092 [Follower]" leader-address= leader-id= -2025-01-28T14:47:01.794Z [INFO] server-25fdb754-f485-ac62-c7bb-c30efa6ef312: entering follower state: follower="Node at 25fdb754-f485-ac62-c7bb-c30efa6ef312 [Follower]" leader-address= leader-id= ---- PASS: TestRaft_LiveBootstrap_From_NonVoter (0.01s) +2025-01-28T14:54:04.310Z [INFO] server-825aaf67-c1b2-23e0-0f42-c87d2160602f: initial configuration: index=0 servers=[] +2025-01-28T14:54:04.310Z [INFO] server-a2c4c3c5-b8ee-1f10-36a5-a6e5f29bcfb7: initial configuration: index=0 servers=[] +2025-01-28T14:54:04.310Z [INFO] server-825aaf67-c1b2-23e0-0f42-c87d2160602f: entering follower state: follower="Node at 825aaf67-c1b2-23e0-0f42-c87d2160602f [Follower]" leader-address= leader-id= +2025-01-28T14:54:04.310Z [INFO] server-a2c4c3c5-b8ee-1f10-36a5-a6e5f29bcfb7: entering follower state: follower="Node at a2c4c3c5-b8ee-1f10-36a5-a6e5f29bcfb7 [Follower]" leader-address= leader-id= +--- PASS: TestRaft_LiveBootstrap_From_NonVoter (0.00s) === RUN TestRaft_RecoverCluster_NoState -2025-01-28T14:47:01.804Z [INFO] server-23e37d06-0887-185c-d158-99d3aed2a00c: initial configuration: index=0 servers=[] -2025-01-28T14:47:01.804Z [INFO] server-23e37d06-0887-185c-d158-99d3aed2a00c: entering follower state: follower="Node at 23e37d06-0887-185c-d158-99d3aed2a00c [Shutdown]" leader-address= leader-id= ---- PASS: TestRaft_RecoverCluster_NoState (0.01s) +2025-01-28T14:54:04.312Z [INFO] server-cf112137-089b-ebab-fcab-6a5d19f879f3: initial configuration: index=0 servers=[] +--- PASS: TestRaft_RecoverCluster_NoState (0.00s) === RUN TestRaft_RecoverCluster === RUN TestRaft_RecoverCluster/no_snapshot,_no_trailing_logs -2025-01-28T14:47:01.818Z [INFO] server-df6943aa-1ab1-b097-3f72-549d08243b9e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-df6943aa-1ab1-b097-3f72-549d08243b9e Address:df6943aa-1ab1-b097-3f72-549d08243b9e} {Suffrage:Voter ID:server-1852e1aa-4cc1-a195-8079-5984b0db113a Address:1852e1aa-4cc1-a195-8079-5984b0db113a} {Suffrage:Voter ID:server-fe8c31c3-3ad4-ff83-d85a-35a0960b6de1 Address:fe8c31c3-3ad4-ff83-d85a-35a0960b6de1}]" -2025-01-28T14:47:01.819Z [INFO] server-df6943aa-1ab1-b097-3f72-549d08243b9e: entering follower state: follower="Node at df6943aa-1ab1-b097-3f72-549d08243b9e [Follower]" leader-address= leader-id= -2025-01-28T14:47:01.820Z [INFO] server-1852e1aa-4cc1-a195-8079-5984b0db113a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-df6943aa-1ab1-b097-3f72-549d08243b9e Address:df6943aa-1ab1-b097-3f72-549d08243b9e} {Suffrage:Voter ID:server-1852e1aa-4cc1-a195-8079-5984b0db113a Address:1852e1aa-4cc1-a195-8079-5984b0db113a} {Suffrage:Voter ID:server-fe8c31c3-3ad4-ff83-d85a-35a0960b6de1 Address:fe8c31c3-3ad4-ff83-d85a-35a0960b6de1}]" -2025-01-28T14:47:01.820Z [INFO] server-1852e1aa-4cc1-a195-8079-5984b0db113a: entering follower state: follower="Node at 1852e1aa-4cc1-a195-8079-5984b0db113a [Follower]" leader-address= leader-id= -2025-01-28T14:47:01.821Z [INFO] server-fe8c31c3-3ad4-ff83-d85a-35a0960b6de1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-df6943aa-1ab1-b097-3f72-549d08243b9e Address:df6943aa-1ab1-b097-3f72-549d08243b9e} {Suffrage:Voter ID:server-1852e1aa-4cc1-a195-8079-5984b0db113a Address:1852e1aa-4cc1-a195-8079-5984b0db113a} {Suffrage:Voter ID:server-fe8c31c3-3ad4-ff83-d85a-35a0960b6de1 Address:fe8c31c3-3ad4-ff83-d85a-35a0960b6de1}]" -2025-01-28T14:47:01.822Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:01.822Z [INFO] server-fe8c31c3-3ad4-ff83-d85a-35a0960b6de1: entering follower state: follower="Node at fe8c31c3-3ad4-ff83-d85a-35a0960b6de1 [Follower]" leader-address= leader-id= -2025-01-28T14:47:01.874Z [WARN] server-df6943aa-1ab1-b097-3f72-549d08243b9e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:01.874Z [INFO] server-df6943aa-1ab1-b097-3f72-549d08243b9e: entering candidate state: node="Node at df6943aa-1ab1-b097-3f72-549d08243b9e [Candidate]" term=2 -2025-01-28T14:47:01.875Z [INFO] server-df6943aa-1ab1-b097-3f72-549d08243b9e: election won: term=2 tally=2 -2025-01-28T14:47:01.875Z [INFO] server-df6943aa-1ab1-b097-3f72-549d08243b9e: entering leader state: leader="Node at df6943aa-1ab1-b097-3f72-549d08243b9e [Leader]" -2025-01-28T14:47:01.875Z [INFO] server-df6943aa-1ab1-b097-3f72-549d08243b9e: added peer, starting replication: peer=server-1852e1aa-4cc1-a195-8079-5984b0db113a -2025-01-28T14:47:01.876Z [INFO] server-df6943aa-1ab1-b097-3f72-549d08243b9e: added peer, starting replication: peer=server-fe8c31c3-3ad4-ff83-d85a-35a0960b6de1 -2025-01-28T14:47:01.877Z [INFO] server-df6943aa-1ab1-b097-3f72-549d08243b9e: pipelining replication: peer="{Voter server-fe8c31c3-3ad4-ff83-d85a-35a0960b6de1 fe8c31c3-3ad4-ff83-d85a-35a0960b6de1}" -2025-01-28T14:47:01.879Z [INFO] server-df6943aa-1ab1-b097-3f72-549d08243b9e: pipelining replication: peer="{Voter server-1852e1aa-4cc1-a195-8079-5984b0db113a 1852e1aa-4cc1-a195-8079-5984b0db113a}" -2025-01-28T14:47:01.983Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:01.877683259 +0000 UTC m=+4.961089616 (1 nodes), 55.357976ms from start of poll, 60.373792ms from cluster start. Timeout at 2025-01-28 14:47:01.982718593 +0000 UTC m=+5.066124951, 105.035335ms after stability -2025-01-28T14:47:01.983Z [INFO] server-df6943aa-1ab1-b097-3f72-549d08243b9e: aborting pipeline replication: peer="{Voter server-1852e1aa-4cc1-a195-8079-5984b0db113a 1852e1aa-4cc1-a195-8079-5984b0db113a}" -2025-01-28T14:47:01.983Z [INFO] server-df6943aa-1ab1-b097-3f72-549d08243b9e: aborting pipeline replication: peer="{Voter server-fe8c31c3-3ad4-ff83-d85a-35a0960b6de1 fe8c31c3-3ad4-ff83-d85a-35a0960b6de1}" -2025-01-28T14:47:01.985Z [INFO] creating new snapshot: path=/tmp/raft3361722972/snapshots/2-2-1738075621984.tmp -2025-01-28T14:47:01.993Z [INFO] server-df6943aa-1ab1-b097-3f72-549d08243b9e: starting restore from snapshot: id=2-2-1738075621984 last-index=2 last-term=2 size-in-bytes=1 -2025-01-28T14:47:01.995Z [INFO] server-df6943aa-1ab1-b097-3f72-549d08243b9e: snapshot restore progress: id=2-2-1738075621984 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% -2025-01-28T14:47:01.995Z [INFO] server-df6943aa-1ab1-b097-3f72-549d08243b9e: restored from snapshot: id=2-2-1738075621984 last-index=2 last-term=2 size-in-bytes=1 -2025-01-28T14:47:01.996Z [INFO] server-df6943aa-1ab1-b097-3f72-549d08243b9e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-df6943aa-1ab1-b097-3f72-549d08243b9e Address:df6943aa-1ab1-b097-3f72-549d08243b9e} {Suffrage:Voter ID:server-1852e1aa-4cc1-a195-8079-5984b0db113a Address:1852e1aa-4cc1-a195-8079-5984b0db113a} {Suffrage:Voter ID:server-fe8c31c3-3ad4-ff83-d85a-35a0960b6de1 Address:fe8c31c3-3ad4-ff83-d85a-35a0960b6de1}]" -2025-01-28T14:47:01.996Z [INFO] server-df6943aa-1ab1-b097-3f72-549d08243b9e: entering follower state: follower="Node at df6943aa-1ab1-b097-3f72-549d08243b9e [Follower]" leader-address= leader-id= -2025-01-28T14:47:01.997Z [INFO] creating new snapshot: path=/tmp/raft2582477310/snapshots/2-2-1738075621996.tmp -2025-01-28T14:47:02.004Z [INFO] server-1852e1aa-4cc1-a195-8079-5984b0db113a: starting restore from snapshot: id=2-2-1738075621996 last-index=2 last-term=2 size-in-bytes=1 -2025-01-28T14:47:02.006Z [INFO] server-1852e1aa-4cc1-a195-8079-5984b0db113a: snapshot restore progress: id=2-2-1738075621996 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% -2025-01-28T14:47:02.007Z [INFO] server-1852e1aa-4cc1-a195-8079-5984b0db113a: restored from snapshot: id=2-2-1738075621996 last-index=2 last-term=2 size-in-bytes=1 -2025-01-28T14:47:02.007Z [INFO] server-1852e1aa-4cc1-a195-8079-5984b0db113a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-df6943aa-1ab1-b097-3f72-549d08243b9e Address:df6943aa-1ab1-b097-3f72-549d08243b9e} {Suffrage:Voter ID:server-1852e1aa-4cc1-a195-8079-5984b0db113a Address:1852e1aa-4cc1-a195-8079-5984b0db113a} {Suffrage:Voter ID:server-fe8c31c3-3ad4-ff83-d85a-35a0960b6de1 Address:fe8c31c3-3ad4-ff83-d85a-35a0960b6de1}]" -2025-01-28T14:47:02.007Z [INFO] server-1852e1aa-4cc1-a195-8079-5984b0db113a: entering follower state: follower="Node at 1852e1aa-4cc1-a195-8079-5984b0db113a [Follower]" leader-address= leader-id= -2025-01-28T14:47:02.008Z [INFO] creating new snapshot: path=/tmp/raft1264297467/snapshots/2-2-1738075622008.tmp -2025-01-28T14:47:02.016Z [INFO] server-fe8c31c3-3ad4-ff83-d85a-35a0960b6de1: starting restore from snapshot: id=2-2-1738075622008 last-index=2 last-term=2 size-in-bytes=1 -2025-01-28T14:47:02.018Z [INFO] server-fe8c31c3-3ad4-ff83-d85a-35a0960b6de1: snapshot restore progress: id=2-2-1738075622008 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% -2025-01-28T14:47:02.018Z [INFO] server-fe8c31c3-3ad4-ff83-d85a-35a0960b6de1: restored from snapshot: id=2-2-1738075622008 last-index=2 last-term=2 size-in-bytes=1 -2025-01-28T14:47:02.018Z [INFO] server-fe8c31c3-3ad4-ff83-d85a-35a0960b6de1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-df6943aa-1ab1-b097-3f72-549d08243b9e Address:df6943aa-1ab1-b097-3f72-549d08243b9e} {Suffrage:Voter ID:server-1852e1aa-4cc1-a195-8079-5984b0db113a Address:1852e1aa-4cc1-a195-8079-5984b0db113a} {Suffrage:Voter ID:server-fe8c31c3-3ad4-ff83-d85a-35a0960b6de1 Address:fe8c31c3-3ad4-ff83-d85a-35a0960b6de1}]" -2025-01-28T14:47:02.019Z [INFO] server-fe8c31c3-3ad4-ff83-d85a-35a0960b6de1: entering follower state: follower="Node at fe8c31c3-3ad4-ff83-d85a-35a0960b6de1 [Follower]" leader-address= leader-id= -2025-01-28T14:47:02.074Z [WARN] server-df6943aa-1ab1-b097-3f72-549d08243b9e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:02.074Z [INFO] server-df6943aa-1ab1-b097-3f72-549d08243b9e: entering candidate state: node="Node at df6943aa-1ab1-b097-3f72-549d08243b9e [Candidate]" term=3 -2025-01-28T14:47:02.075Z [WARN] server-fe8c31c3-3ad4-ff83-d85a-35a0960b6de1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:02.075Z [INFO] server-fe8c31c3-3ad4-ff83-d85a-35a0960b6de1: entering candidate state: node="Node at fe8c31c3-3ad4-ff83-d85a-35a0960b6de1 [Candidate]" term=3 -2025-01-28T14:47:02.076Z [INFO] server-df6943aa-1ab1-b097-3f72-549d08243b9e: election won: term=3 tally=2 -2025-01-28T14:47:02.076Z [INFO] server-df6943aa-1ab1-b097-3f72-549d08243b9e: entering leader state: leader="Node at df6943aa-1ab1-b097-3f72-549d08243b9e [Leader]" -2025-01-28T14:47:02.076Z [INFO] server-fe8c31c3-3ad4-ff83-d85a-35a0960b6de1: duplicate requestVote for same term: term=3 -2025-01-28T14:47:02.076Z [INFO] server-df6943aa-1ab1-b097-3f72-549d08243b9e: added peer, starting replication: peer=server-1852e1aa-4cc1-a195-8079-5984b0db113a -2025-01-28T14:47:02.076Z [INFO] server-df6943aa-1ab1-b097-3f72-549d08243b9e: added peer, starting replication: peer=server-fe8c31c3-3ad4-ff83-d85a-35a0960b6de1 -2025-01-28T14:47:02.076Z [INFO] server-1852e1aa-4cc1-a195-8079-5984b0db113a: duplicate requestVote for same term: term=3 -2025-01-28T14:47:02.077Z [WARN] server-df6943aa-1ab1-b097-3f72-549d08243b9e: rejecting vote request since we have a leader: from=fe8c31c3-3ad4-ff83-d85a-35a0960b6de1 leader=df6943aa-1ab1-b097-3f72-549d08243b9e leader-id=server-df6943aa-1ab1-b097-3f72-549d08243b9e -2025-01-28T14:47:02.078Z [INFO] server-df6943aa-1ab1-b097-3f72-549d08243b9e: pipelining replication: peer="{Voter server-1852e1aa-4cc1-a195-8079-5984b0db113a 1852e1aa-4cc1-a195-8079-5984b0db113a}" -2025-01-28T14:47:02.078Z [INFO] server-fe8c31c3-3ad4-ff83-d85a-35a0960b6de1: entering follower state: follower="Node at fe8c31c3-3ad4-ff83-d85a-35a0960b6de1 [Follower]" leader-address=df6943aa-1ab1-b097-3f72-549d08243b9e leader-id=server-df6943aa-1ab1-b097-3f72-549d08243b9e -2025-01-28T14:47:02.078Z [INFO] server-df6943aa-1ab1-b097-3f72-549d08243b9e: pipelining replication: peer="{Voter server-fe8c31c3-3ad4-ff83-d85a-35a0960b6de1 fe8c31c3-3ad4-ff83-d85a-35a0960b6de1}" -2025-01-28T14:47:02.334Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:02.441Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:02.335187841 +0000 UTC m=+5.418594199 (1 nodes), 435.013µs from start of poll, 517.878375ms from cluster start. Timeout at 2025-01-28 14:47:02.440277509 +0000 UTC m=+5.523683867, 105.089668ms after stability -2025-01-28T14:47:02.442Z [INFO] server-df6943aa-1ab1-b097-3f72-549d08243b9e: aborting pipeline replication: peer="{Voter server-1852e1aa-4cc1-a195-8079-5984b0db113a 1852e1aa-4cc1-a195-8079-5984b0db113a}" -2025-01-28T14:47:02.442Z [INFO] server-df6943aa-1ab1-b097-3f72-549d08243b9e: aborting pipeline replication: peer="{Voter server-fe8c31c3-3ad4-ff83-d85a-35a0960b6de1 fe8c31c3-3ad4-ff83-d85a-35a0960b6de1}" -2025-01-28T14:47:02.942Z [ERROR] server-df6943aa-1ab1-b097-3f72-549d08243b9e: failed to heartbeat to: peer=1852e1aa-4cc1-a195-8079-5984b0db113a backoff time=10ms error="command timed out" +2025-01-28T14:54:04.314Z [INFO] server-139f096b-28b6-854a-86b1-b8c9c56fe6d0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-139f096b-28b6-854a-86b1-b8c9c56fe6d0 Address:139f096b-28b6-854a-86b1-b8c9c56fe6d0} {Suffrage:Voter ID:server-1321883d-4cf3-7286-8743-a5a9a9aef940 Address:1321883d-4cf3-7286-8743-a5a9a9aef940} {Suffrage:Voter ID:server-415f37d2-c85f-1819-e4e8-bb29df644aba Address:415f37d2-c85f-1819-e4e8-bb29df644aba}]" +2025-01-28T14:54:04.315Z [INFO] server-139f096b-28b6-854a-86b1-b8c9c56fe6d0: entering follower state: follower="Node at 139f096b-28b6-854a-86b1-b8c9c56fe6d0 [Follower]" leader-address= leader-id= +2025-01-28T14:54:04.315Z [INFO] server-1321883d-4cf3-7286-8743-a5a9a9aef940: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-139f096b-28b6-854a-86b1-b8c9c56fe6d0 Address:139f096b-28b6-854a-86b1-b8c9c56fe6d0} {Suffrage:Voter ID:server-1321883d-4cf3-7286-8743-a5a9a9aef940 Address:1321883d-4cf3-7286-8743-a5a9a9aef940} {Suffrage:Voter ID:server-415f37d2-c85f-1819-e4e8-bb29df644aba Address:415f37d2-c85f-1819-e4e8-bb29df644aba}]" +2025-01-28T14:54:04.315Z [INFO] server-1321883d-4cf3-7286-8743-a5a9a9aef940: entering follower state: follower="Node at 1321883d-4cf3-7286-8743-a5a9a9aef940 [Follower]" leader-address= leader-id= +2025-01-28T14:54:04.315Z [INFO] server-415f37d2-c85f-1819-e4e8-bb29df644aba: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-139f096b-28b6-854a-86b1-b8c9c56fe6d0 Address:139f096b-28b6-854a-86b1-b8c9c56fe6d0} {Suffrage:Voter ID:server-1321883d-4cf3-7286-8743-a5a9a9aef940 Address:1321883d-4cf3-7286-8743-a5a9a9aef940} {Suffrage:Voter ID:server-415f37d2-c85f-1819-e4e8-bb29df644aba Address:415f37d2-c85f-1819-e4e8-bb29df644aba}]" +2025-01-28T14:54:04.315Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:04.315Z [INFO] server-415f37d2-c85f-1819-e4e8-bb29df644aba: entering follower state: follower="Node at 415f37d2-c85f-1819-e4e8-bb29df644aba [Follower]" leader-address= leader-id= +2025-01-28T14:54:04.391Z [WARN] server-415f37d2-c85f-1819-e4e8-bb29df644aba: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:04.391Z [INFO] server-415f37d2-c85f-1819-e4e8-bb29df644aba: entering candidate state: node="Node at 415f37d2-c85f-1819-e4e8-bb29df644aba [Candidate]" term=2 +2025-01-28T14:54:04.392Z [INFO] server-415f37d2-c85f-1819-e4e8-bb29df644aba: election won: term=2 tally=2 +2025-01-28T14:54:04.392Z [INFO] server-415f37d2-c85f-1819-e4e8-bb29df644aba: entering leader state: leader="Node at 415f37d2-c85f-1819-e4e8-bb29df644aba [Leader]" +2025-01-28T14:54:04.392Z [INFO] server-415f37d2-c85f-1819-e4e8-bb29df644aba: added peer, starting replication: peer=server-139f096b-28b6-854a-86b1-b8c9c56fe6d0 +2025-01-28T14:54:04.392Z [INFO] server-415f37d2-c85f-1819-e4e8-bb29df644aba: added peer, starting replication: peer=server-1321883d-4cf3-7286-8743-a5a9a9aef940 +2025-01-28T14:54:04.392Z [INFO] server-415f37d2-c85f-1819-e4e8-bb29df644aba: pipelining replication: peer="{Voter server-139f096b-28b6-854a-86b1-b8c9c56fe6d0 139f096b-28b6-854a-86b1-b8c9c56fe6d0}" +2025-01-28T14:54:04.392Z [INFO] server-415f37d2-c85f-1819-e4e8-bb29df644aba: pipelining replication: peer="{Voter server-1321883d-4cf3-7286-8743-a5a9a9aef940 1321883d-4cf3-7286-8743-a5a9a9aef940}" +2025-01-28T14:54:04.497Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:04.392304953 +0000 UTC m=+30.066794933 (1 nodes), 76.561318ms from start of poll, 77.562023ms from cluster start. Timeout at 2025-01-28 14:54:04.497418062 +0000 UTC m=+30.171908142, 105.113209ms after stability +2025-01-28T14:54:04.498Z [INFO] server-415f37d2-c85f-1819-e4e8-bb29df644aba: aborting pipeline replication: peer="{Voter server-1321883d-4cf3-7286-8743-a5a9a9aef940 1321883d-4cf3-7286-8743-a5a9a9aef940}" +2025-01-28T14:54:04.498Z [INFO] server-415f37d2-c85f-1819-e4e8-bb29df644aba: aborting pipeline replication: peer="{Voter server-139f096b-28b6-854a-86b1-b8c9c56fe6d0 139f096b-28b6-854a-86b1-b8c9c56fe6d0}" +2025-01-28T14:54:04.498Z [INFO] creating new snapshot: path=/tmp/raft2030302248/snapshots/2-2-1738076044498.tmp +2025-01-28T14:54:04.501Z [INFO] server-139f096b-28b6-854a-86b1-b8c9c56fe6d0: starting restore from snapshot: id=2-2-1738076044498 last-index=2 last-term=2 size-in-bytes=1 +2025-01-28T14:54:04.501Z [INFO] server-139f096b-28b6-854a-86b1-b8c9c56fe6d0: snapshot restore progress: id=2-2-1738076044498 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% +2025-01-28T14:54:04.501Z [INFO] server-139f096b-28b6-854a-86b1-b8c9c56fe6d0: restored from snapshot: id=2-2-1738076044498 last-index=2 last-term=2 size-in-bytes=1 +2025-01-28T14:54:04.501Z [INFO] server-139f096b-28b6-854a-86b1-b8c9c56fe6d0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-139f096b-28b6-854a-86b1-b8c9c56fe6d0 Address:139f096b-28b6-854a-86b1-b8c9c56fe6d0} {Suffrage:Voter ID:server-1321883d-4cf3-7286-8743-a5a9a9aef940 Address:1321883d-4cf3-7286-8743-a5a9a9aef940} {Suffrage:Voter ID:server-415f37d2-c85f-1819-e4e8-bb29df644aba Address:415f37d2-c85f-1819-e4e8-bb29df644aba}]" +2025-01-28T14:54:04.501Z [INFO] server-139f096b-28b6-854a-86b1-b8c9c56fe6d0: entering follower state: follower="Node at 139f096b-28b6-854a-86b1-b8c9c56fe6d0 [Follower]" leader-address= leader-id= +2025-01-28T14:54:04.501Z [INFO] creating new snapshot: path=/tmp/raft462386264/snapshots/2-2-1738076044501.tmp +2025-01-28T14:54:04.504Z [INFO] server-1321883d-4cf3-7286-8743-a5a9a9aef940: starting restore from snapshot: id=2-2-1738076044501 last-index=2 last-term=2 size-in-bytes=1 +2025-01-28T14:54:04.504Z [INFO] server-1321883d-4cf3-7286-8743-a5a9a9aef940: snapshot restore progress: id=2-2-1738076044501 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% +2025-01-28T14:54:04.504Z [INFO] server-1321883d-4cf3-7286-8743-a5a9a9aef940: restored from snapshot: id=2-2-1738076044501 last-index=2 last-term=2 size-in-bytes=1 +2025-01-28T14:54:04.504Z [INFO] server-1321883d-4cf3-7286-8743-a5a9a9aef940: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-139f096b-28b6-854a-86b1-b8c9c56fe6d0 Address:139f096b-28b6-854a-86b1-b8c9c56fe6d0} {Suffrage:Voter ID:server-1321883d-4cf3-7286-8743-a5a9a9aef940 Address:1321883d-4cf3-7286-8743-a5a9a9aef940} {Suffrage:Voter ID:server-415f37d2-c85f-1819-e4e8-bb29df644aba Address:415f37d2-c85f-1819-e4e8-bb29df644aba}]" +2025-01-28T14:54:04.504Z [INFO] server-1321883d-4cf3-7286-8743-a5a9a9aef940: entering follower state: follower="Node at 1321883d-4cf3-7286-8743-a5a9a9aef940 [Follower]" leader-address= leader-id= +2025-01-28T14:54:04.505Z [INFO] creating new snapshot: path=/tmp/raft1841362757/snapshots/2-2-1738076044505.tmp +2025-01-28T14:54:04.506Z [INFO] server-415f37d2-c85f-1819-e4e8-bb29df644aba: starting restore from snapshot: id=2-2-1738076044505 last-index=2 last-term=2 size-in-bytes=1 +2025-01-28T14:54:04.507Z [INFO] server-415f37d2-c85f-1819-e4e8-bb29df644aba: snapshot restore progress: id=2-2-1738076044505 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% +2025-01-28T14:54:04.507Z [INFO] server-415f37d2-c85f-1819-e4e8-bb29df644aba: restored from snapshot: id=2-2-1738076044505 last-index=2 last-term=2 size-in-bytes=1 +2025-01-28T14:54:04.507Z [INFO] server-415f37d2-c85f-1819-e4e8-bb29df644aba: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-139f096b-28b6-854a-86b1-b8c9c56fe6d0 Address:139f096b-28b6-854a-86b1-b8c9c56fe6d0} {Suffrage:Voter ID:server-1321883d-4cf3-7286-8743-a5a9a9aef940 Address:1321883d-4cf3-7286-8743-a5a9a9aef940} {Suffrage:Voter ID:server-415f37d2-c85f-1819-e4e8-bb29df644aba Address:415f37d2-c85f-1819-e4e8-bb29df644aba}]" +2025-01-28T14:54:04.507Z [INFO] server-415f37d2-c85f-1819-e4e8-bb29df644aba: entering follower state: follower="Node at 415f37d2-c85f-1819-e4e8-bb29df644aba [Follower]" leader-address= leader-id= +2025-01-28T14:54:04.560Z [WARN] server-415f37d2-c85f-1819-e4e8-bb29df644aba: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:04.560Z [INFO] server-415f37d2-c85f-1819-e4e8-bb29df644aba: entering candidate state: node="Node at 415f37d2-c85f-1819-e4e8-bb29df644aba [Candidate]" term=3 +2025-01-28T14:54:04.560Z [INFO] server-415f37d2-c85f-1819-e4e8-bb29df644aba: election won: term=3 tally=2 +2025-01-28T14:54:04.560Z [INFO] server-415f37d2-c85f-1819-e4e8-bb29df644aba: entering leader state: leader="Node at 415f37d2-c85f-1819-e4e8-bb29df644aba [Leader]" +2025-01-28T14:54:04.560Z [INFO] server-415f37d2-c85f-1819-e4e8-bb29df644aba: added peer, starting replication: peer=server-139f096b-28b6-854a-86b1-b8c9c56fe6d0 +2025-01-28T14:54:04.560Z [INFO] server-415f37d2-c85f-1819-e4e8-bb29df644aba: added peer, starting replication: peer=server-1321883d-4cf3-7286-8743-a5a9a9aef940 +2025-01-28T14:54:04.561Z [INFO] server-415f37d2-c85f-1819-e4e8-bb29df644aba: pipelining replication: peer="{Voter server-1321883d-4cf3-7286-8743-a5a9a9aef940 1321883d-4cf3-7286-8743-a5a9a9aef940}" +2025-01-28T14:54:04.561Z [INFO] server-415f37d2-c85f-1819-e4e8-bb29df644aba: pipelining replication: peer="{Voter server-139f096b-28b6-854a-86b1-b8c9c56fe6d0 139f096b-28b6-854a-86b1-b8c9c56fe6d0}" +2025-01-28T14:54:04.822Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:04.929Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:04.82311999 +0000 UTC m=+30.497609970 (1 nodes), 77.433µs from start of poll, 508.37706ms from cluster start. Timeout at 2025-01-28 14:54:04.928151047 +0000 UTC m=+30.602641027, 105.031057ms after stability +2025-01-28T14:54:04.929Z [INFO] server-415f37d2-c85f-1819-e4e8-bb29df644aba: aborting pipeline replication: peer="{Voter server-1321883d-4cf3-7286-8743-a5a9a9aef940 1321883d-4cf3-7286-8743-a5a9a9aef940}" +2025-01-28T14:54:04.929Z [INFO] server-415f37d2-c85f-1819-e4e8-bb29df644aba: aborting pipeline replication: peer="{Voter server-139f096b-28b6-854a-86b1-b8c9c56fe6d0 139f096b-28b6-854a-86b1-b8c9c56fe6d0}" +2025-01-28T14:54:05.430Z [ERROR] server-415f37d2-c85f-1819-e4e8-bb29df644aba: failed to heartbeat to: peer=1321883d-4cf3-7286-8743-a5a9a9aef940 backoff time=10ms error="command timed out" === RUN TestRaft_RecoverCluster/no_snapshot,_some_trailing_logs -2025-01-28T14:47:02.965Z [INFO] server-94d9935c-4fa0-7c31-b12c-73ad442e6428: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-94d9935c-4fa0-7c31-b12c-73ad442e6428 Address:94d9935c-4fa0-7c31-b12c-73ad442e6428} {Suffrage:Voter ID:server-97c5019d-853d-2af6-38ab-20c5409bbc12 Address:97c5019d-853d-2af6-38ab-20c5409bbc12} {Suffrage:Voter ID:server-1a0e1a4d-9257-48ca-5915-cf97777ef904 Address:1a0e1a4d-9257-48ca-5915-cf97777ef904}]" -2025-01-28T14:47:02.966Z [INFO] server-94d9935c-4fa0-7c31-b12c-73ad442e6428: entering follower state: follower="Node at 94d9935c-4fa0-7c31-b12c-73ad442e6428 [Follower]" leader-address= leader-id= -2025-01-28T14:47:02.967Z [INFO] server-97c5019d-853d-2af6-38ab-20c5409bbc12: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-94d9935c-4fa0-7c31-b12c-73ad442e6428 Address:94d9935c-4fa0-7c31-b12c-73ad442e6428} {Suffrage:Voter ID:server-97c5019d-853d-2af6-38ab-20c5409bbc12 Address:97c5019d-853d-2af6-38ab-20c5409bbc12} {Suffrage:Voter ID:server-1a0e1a4d-9257-48ca-5915-cf97777ef904 Address:1a0e1a4d-9257-48ca-5915-cf97777ef904}]" -2025-01-28T14:47:02.968Z [INFO] server-97c5019d-853d-2af6-38ab-20c5409bbc12: entering follower state: follower="Node at 97c5019d-853d-2af6-38ab-20c5409bbc12 [Follower]" leader-address= leader-id= -2025-01-28T14:47:02.969Z [INFO] server-1a0e1a4d-9257-48ca-5915-cf97777ef904: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-94d9935c-4fa0-7c31-b12c-73ad442e6428 Address:94d9935c-4fa0-7c31-b12c-73ad442e6428} {Suffrage:Voter ID:server-97c5019d-853d-2af6-38ab-20c5409bbc12 Address:97c5019d-853d-2af6-38ab-20c5409bbc12} {Suffrage:Voter ID:server-1a0e1a4d-9257-48ca-5915-cf97777ef904 Address:1a0e1a4d-9257-48ca-5915-cf97777ef904}]" -2025-01-28T14:47:02.969Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:02.969Z [INFO] server-1a0e1a4d-9257-48ca-5915-cf97777ef904: entering follower state: follower="Node at 1a0e1a4d-9257-48ca-5915-cf97777ef904 [Follower]" leader-address= leader-id= -2025-01-28T14:47:03.022Z [WARN] server-1a0e1a4d-9257-48ca-5915-cf97777ef904: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:03.022Z [INFO] server-1a0e1a4d-9257-48ca-5915-cf97777ef904: entering candidate state: node="Node at 1a0e1a4d-9257-48ca-5915-cf97777ef904 [Candidate]" term=2 -2025-01-28T14:47:03.023Z [INFO] server-1a0e1a4d-9257-48ca-5915-cf97777ef904: election won: term=2 tally=2 -2025-01-28T14:47:03.023Z [INFO] server-1a0e1a4d-9257-48ca-5915-cf97777ef904: entering leader state: leader="Node at 1a0e1a4d-9257-48ca-5915-cf97777ef904 [Leader]" -2025-01-28T14:47:03.024Z [INFO] server-1a0e1a4d-9257-48ca-5915-cf97777ef904: added peer, starting replication: peer=server-94d9935c-4fa0-7c31-b12c-73ad442e6428 -2025-01-28T14:47:03.024Z [INFO] server-1a0e1a4d-9257-48ca-5915-cf97777ef904: added peer, starting replication: peer=server-97c5019d-853d-2af6-38ab-20c5409bbc12 -2025-01-28T14:47:03.025Z [INFO] server-1a0e1a4d-9257-48ca-5915-cf97777ef904: pipelining replication: peer="{Voter server-94d9935c-4fa0-7c31-b12c-73ad442e6428 94d9935c-4fa0-7c31-b12c-73ad442e6428}" -2025-01-28T14:47:03.026Z [INFO] server-1a0e1a4d-9257-48ca-5915-cf97777ef904: pipelining replication: peer="{Voter server-97c5019d-853d-2af6-38ab-20c5409bbc12 97c5019d-853d-2af6-38ab-20c5409bbc12}" -2025-01-28T14:47:03.130Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:03.024219297 +0000 UTC m=+6.107626654 (1 nodes), 54.620288ms from start of poll, 59.478765ms from cluster start. Timeout at 2025-01-28 14:47:03.129263298 +0000 UTC m=+6.212669322, 105.042668ms after stability -2025-01-28T14:47:03.138Z [INFO] server-1a0e1a4d-9257-48ca-5915-cf97777ef904: aborting pipeline replication: peer="{Voter server-97c5019d-853d-2af6-38ab-20c5409bbc12 97c5019d-853d-2af6-38ab-20c5409bbc12}" -2025-01-28T14:47:03.139Z [INFO] server-1a0e1a4d-9257-48ca-5915-cf97777ef904: aborting pipeline replication: peer="{Voter server-94d9935c-4fa0-7c31-b12c-73ad442e6428 94d9935c-4fa0-7c31-b12c-73ad442e6428}" -2025-01-28T14:47:03.140Z [INFO] creating new snapshot: path=/tmp/raft2035946799/snapshots/2-6-1738075623140.tmp -2025-01-28T14:47:03.148Z [INFO] server-94d9935c-4fa0-7c31-b12c-73ad442e6428: starting restore from snapshot: id=2-6-1738075623140 last-index=6 last-term=2 size-in-bytes=25 -2025-01-28T14:47:03.150Z [INFO] server-94d9935c-4fa0-7c31-b12c-73ad442e6428: snapshot restore progress: id=2-6-1738075623140 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% -2025-01-28T14:47:03.151Z [INFO] server-94d9935c-4fa0-7c31-b12c-73ad442e6428: restored from snapshot: id=2-6-1738075623140 last-index=6 last-term=2 size-in-bytes=25 -2025-01-28T14:47:03.151Z [INFO] server-94d9935c-4fa0-7c31-b12c-73ad442e6428: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-94d9935c-4fa0-7c31-b12c-73ad442e6428 Address:94d9935c-4fa0-7c31-b12c-73ad442e6428} {Suffrage:Voter ID:server-97c5019d-853d-2af6-38ab-20c5409bbc12 Address:97c5019d-853d-2af6-38ab-20c5409bbc12} {Suffrage:Voter ID:server-1a0e1a4d-9257-48ca-5915-cf97777ef904 Address:1a0e1a4d-9257-48ca-5915-cf97777ef904}]" -2025-01-28T14:47:03.151Z [INFO] server-94d9935c-4fa0-7c31-b12c-73ad442e6428: entering follower state: follower="Node at 94d9935c-4fa0-7c31-b12c-73ad442e6428 [Follower]" leader-address= leader-id= -2025-01-28T14:47:03.152Z [INFO] creating new snapshot: path=/tmp/raft3191791572/snapshots/2-6-1738075623152.tmp -2025-01-28T14:47:03.160Z [INFO] server-97c5019d-853d-2af6-38ab-20c5409bbc12: starting restore from snapshot: id=2-6-1738075623152 last-index=6 last-term=2 size-in-bytes=25 -2025-01-28T14:47:03.161Z [INFO] server-97c5019d-853d-2af6-38ab-20c5409bbc12: snapshot restore progress: id=2-6-1738075623152 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% -2025-01-28T14:47:03.162Z [INFO] server-97c5019d-853d-2af6-38ab-20c5409bbc12: restored from snapshot: id=2-6-1738075623152 last-index=6 last-term=2 size-in-bytes=25 -2025-01-28T14:47:03.162Z [INFO] server-97c5019d-853d-2af6-38ab-20c5409bbc12: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-94d9935c-4fa0-7c31-b12c-73ad442e6428 Address:94d9935c-4fa0-7c31-b12c-73ad442e6428} {Suffrage:Voter ID:server-97c5019d-853d-2af6-38ab-20c5409bbc12 Address:97c5019d-853d-2af6-38ab-20c5409bbc12} {Suffrage:Voter ID:server-1a0e1a4d-9257-48ca-5915-cf97777ef904 Address:1a0e1a4d-9257-48ca-5915-cf97777ef904}]" -2025-01-28T14:47:03.163Z [INFO] server-97c5019d-853d-2af6-38ab-20c5409bbc12: entering follower state: follower="Node at 97c5019d-853d-2af6-38ab-20c5409bbc12 [Follower]" leader-address= leader-id= -2025-01-28T14:47:03.163Z [INFO] creating new snapshot: path=/tmp/raft2635543855/snapshots/2-6-1738075623163.tmp -2025-01-28T14:47:03.171Z [INFO] server-1a0e1a4d-9257-48ca-5915-cf97777ef904: starting restore from snapshot: id=2-6-1738075623163 last-index=6 last-term=2 size-in-bytes=25 -2025-01-28T14:47:03.173Z [INFO] server-1a0e1a4d-9257-48ca-5915-cf97777ef904: snapshot restore progress: id=2-6-1738075623163 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% -2025-01-28T14:47:03.173Z [INFO] server-1a0e1a4d-9257-48ca-5915-cf97777ef904: restored from snapshot: id=2-6-1738075623163 last-index=6 last-term=2 size-in-bytes=25 -2025-01-28T14:47:03.173Z [INFO] server-1a0e1a4d-9257-48ca-5915-cf97777ef904: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-94d9935c-4fa0-7c31-b12c-73ad442e6428 Address:94d9935c-4fa0-7c31-b12c-73ad442e6428} {Suffrage:Voter ID:server-97c5019d-853d-2af6-38ab-20c5409bbc12 Address:97c5019d-853d-2af6-38ab-20c5409bbc12} {Suffrage:Voter ID:server-1a0e1a4d-9257-48ca-5915-cf97777ef904 Address:1a0e1a4d-9257-48ca-5915-cf97777ef904}]" -2025-01-28T14:47:03.174Z [INFO] server-1a0e1a4d-9257-48ca-5915-cf97777ef904: entering follower state: follower="Node at 1a0e1a4d-9257-48ca-5915-cf97777ef904 [Follower]" leader-address= leader-id= -2025-01-28T14:47:03.226Z [WARN] server-1a0e1a4d-9257-48ca-5915-cf97777ef904: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:03.226Z [INFO] server-1a0e1a4d-9257-48ca-5915-cf97777ef904: entering candidate state: node="Node at 1a0e1a4d-9257-48ca-5915-cf97777ef904 [Candidate]" term=3 -2025-01-28T14:47:03.227Z [INFO] server-1a0e1a4d-9257-48ca-5915-cf97777ef904: election won: term=3 tally=2 -2025-01-28T14:47:03.227Z [INFO] server-1a0e1a4d-9257-48ca-5915-cf97777ef904: entering leader state: leader="Node at 1a0e1a4d-9257-48ca-5915-cf97777ef904 [Leader]" -2025-01-28T14:47:03.228Z [INFO] server-1a0e1a4d-9257-48ca-5915-cf97777ef904: added peer, starting replication: peer=server-94d9935c-4fa0-7c31-b12c-73ad442e6428 -2025-01-28T14:47:03.228Z [INFO] server-1a0e1a4d-9257-48ca-5915-cf97777ef904: added peer, starting replication: peer=server-97c5019d-853d-2af6-38ab-20c5409bbc12 -2025-01-28T14:47:03.229Z [INFO] server-1a0e1a4d-9257-48ca-5915-cf97777ef904: pipelining replication: peer="{Voter server-94d9935c-4fa0-7c31-b12c-73ad442e6428 94d9935c-4fa0-7c31-b12c-73ad442e6428}" -2025-01-28T14:47:03.229Z [INFO] server-1a0e1a4d-9257-48ca-5915-cf97777ef904: pipelining replication: peer="{Voter server-97c5019d-853d-2af6-38ab-20c5409bbc12 97c5019d-853d-2af6-38ab-20c5409bbc12}" -2025-01-28T14:47:03.489Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:03.597Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:03.490645811 +0000 UTC m=+6.574052168 (1 nodes), 421.679µs from start of poll, 525.904279ms from cluster start. Timeout at 2025-01-28 14:47:03.595681811 +0000 UTC m=+6.679088169, 105.036001ms after stability -2025-01-28T14:47:03.598Z [INFO] server-1a0e1a4d-9257-48ca-5915-cf97777ef904: aborting pipeline replication: peer="{Voter server-97c5019d-853d-2af6-38ab-20c5409bbc12 97c5019d-853d-2af6-38ab-20c5409bbc12}" -2025-01-28T14:47:03.598Z [INFO] server-1a0e1a4d-9257-48ca-5915-cf97777ef904: aborting pipeline replication: peer="{Voter server-94d9935c-4fa0-7c31-b12c-73ad442e6428 94d9935c-4fa0-7c31-b12c-73ad442e6428}" +2025-01-28T14:54:05.436Z [INFO] server-0cb1976e-18c2-6d6f-cfe2-0864af19caa2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0cb1976e-18c2-6d6f-cfe2-0864af19caa2 Address:0cb1976e-18c2-6d6f-cfe2-0864af19caa2} {Suffrage:Voter ID:server-b9b1cf77-fae2-b389-93bb-9f6313fcaff9 Address:b9b1cf77-fae2-b389-93bb-9f6313fcaff9} {Suffrage:Voter ID:server-ecaea33d-0b8c-e99b-c96d-e9c72cd518dd Address:ecaea33d-0b8c-e99b-c96d-e9c72cd518dd}]" +2025-01-28T14:54:05.436Z [INFO] server-0cb1976e-18c2-6d6f-cfe2-0864af19caa2: entering follower state: follower="Node at 0cb1976e-18c2-6d6f-cfe2-0864af19caa2 [Follower]" leader-address= leader-id= +2025-01-28T14:54:05.436Z [INFO] server-b9b1cf77-fae2-b389-93bb-9f6313fcaff9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0cb1976e-18c2-6d6f-cfe2-0864af19caa2 Address:0cb1976e-18c2-6d6f-cfe2-0864af19caa2} {Suffrage:Voter ID:server-b9b1cf77-fae2-b389-93bb-9f6313fcaff9 Address:b9b1cf77-fae2-b389-93bb-9f6313fcaff9} {Suffrage:Voter ID:server-ecaea33d-0b8c-e99b-c96d-e9c72cd518dd Address:ecaea33d-0b8c-e99b-c96d-e9c72cd518dd}]" +2025-01-28T14:54:05.436Z [INFO] server-b9b1cf77-fae2-b389-93bb-9f6313fcaff9: entering follower state: follower="Node at b9b1cf77-fae2-b389-93bb-9f6313fcaff9 [Follower]" leader-address= leader-id= +2025-01-28T14:54:05.436Z [INFO] server-ecaea33d-0b8c-e99b-c96d-e9c72cd518dd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0cb1976e-18c2-6d6f-cfe2-0864af19caa2 Address:0cb1976e-18c2-6d6f-cfe2-0864af19caa2} {Suffrage:Voter ID:server-b9b1cf77-fae2-b389-93bb-9f6313fcaff9 Address:b9b1cf77-fae2-b389-93bb-9f6313fcaff9} {Suffrage:Voter ID:server-ecaea33d-0b8c-e99b-c96d-e9c72cd518dd Address:ecaea33d-0b8c-e99b-c96d-e9c72cd518dd}]" +2025-01-28T14:54:05.436Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:05.436Z [INFO] server-ecaea33d-0b8c-e99b-c96d-e9c72cd518dd: entering follower state: follower="Node at ecaea33d-0b8c-e99b-c96d-e9c72cd518dd [Follower]" leader-address= leader-id= +2025-01-28T14:54:05.490Z [WARN] server-b9b1cf77-fae2-b389-93bb-9f6313fcaff9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:05.490Z [INFO] server-b9b1cf77-fae2-b389-93bb-9f6313fcaff9: entering candidate state: node="Node at b9b1cf77-fae2-b389-93bb-9f6313fcaff9 [Candidate]" term=2 +2025-01-28T14:54:05.490Z [INFO] server-b9b1cf77-fae2-b389-93bb-9f6313fcaff9: election won: term=2 tally=2 +2025-01-28T14:54:05.490Z [INFO] server-b9b1cf77-fae2-b389-93bb-9f6313fcaff9: entering leader state: leader="Node at b9b1cf77-fae2-b389-93bb-9f6313fcaff9 [Leader]" +2025-01-28T14:54:05.490Z [INFO] server-b9b1cf77-fae2-b389-93bb-9f6313fcaff9: added peer, starting replication: peer=server-0cb1976e-18c2-6d6f-cfe2-0864af19caa2 +2025-01-28T14:54:05.490Z [INFO] server-b9b1cf77-fae2-b389-93bb-9f6313fcaff9: added peer, starting replication: peer=server-ecaea33d-0b8c-e99b-c96d-e9c72cd518dd +2025-01-28T14:54:05.491Z [INFO] server-b9b1cf77-fae2-b389-93bb-9f6313fcaff9: pipelining replication: peer="{Voter server-ecaea33d-0b8c-e99b-c96d-e9c72cd518dd ecaea33d-0b8c-e99b-c96d-e9c72cd518dd}" +2025-01-28T14:54:05.491Z [INFO] server-b9b1cf77-fae2-b389-93bb-9f6313fcaff9: pipelining replication: peer="{Voter server-0cb1976e-18c2-6d6f-cfe2-0864af19caa2 0cb1976e-18c2-6d6f-cfe2-0864af19caa2}" +2025-01-28T14:54:05.596Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:05.490978704 +0000 UTC m=+31.165468664 (1 nodes), 54.000925ms from start of poll, 55.093362ms from cluster start. Timeout at 2025-01-28 14:54:05.595984143 +0000 UTC m=+31.270474183, 105.005519ms after stability +2025-01-28T14:54:05.598Z [INFO] server-b9b1cf77-fae2-b389-93bb-9f6313fcaff9: aborting pipeline replication: peer="{Voter server-ecaea33d-0b8c-e99b-c96d-e9c72cd518dd ecaea33d-0b8c-e99b-c96d-e9c72cd518dd}" +2025-01-28T14:54:05.598Z [INFO] server-b9b1cf77-fae2-b389-93bb-9f6313fcaff9: aborting pipeline replication: peer="{Voter server-0cb1976e-18c2-6d6f-cfe2-0864af19caa2 0cb1976e-18c2-6d6f-cfe2-0864af19caa2}" +2025-01-28T14:54:05.598Z [INFO] creating new snapshot: path=/tmp/raft1183262515/snapshots/2-6-1738076045598.tmp +2025-01-28T14:54:05.600Z [INFO] server-0cb1976e-18c2-6d6f-cfe2-0864af19caa2: starting restore from snapshot: id=2-6-1738076045598 last-index=6 last-term=2 size-in-bytes=25 +2025-01-28T14:54:05.601Z [INFO] server-0cb1976e-18c2-6d6f-cfe2-0864af19caa2: snapshot restore progress: id=2-6-1738076045598 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% +2025-01-28T14:54:05.601Z [INFO] server-0cb1976e-18c2-6d6f-cfe2-0864af19caa2: restored from snapshot: id=2-6-1738076045598 last-index=6 last-term=2 size-in-bytes=25 +2025-01-28T14:54:05.601Z [INFO] server-0cb1976e-18c2-6d6f-cfe2-0864af19caa2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0cb1976e-18c2-6d6f-cfe2-0864af19caa2 Address:0cb1976e-18c2-6d6f-cfe2-0864af19caa2} {Suffrage:Voter ID:server-b9b1cf77-fae2-b389-93bb-9f6313fcaff9 Address:b9b1cf77-fae2-b389-93bb-9f6313fcaff9} {Suffrage:Voter ID:server-ecaea33d-0b8c-e99b-c96d-e9c72cd518dd Address:ecaea33d-0b8c-e99b-c96d-e9c72cd518dd}]" +2025-01-28T14:54:05.601Z [INFO] server-0cb1976e-18c2-6d6f-cfe2-0864af19caa2: entering follower state: follower="Node at 0cb1976e-18c2-6d6f-cfe2-0864af19caa2 [Follower]" leader-address= leader-id= +2025-01-28T14:54:05.601Z [INFO] creating new snapshot: path=/tmp/raft675510915/snapshots/2-6-1738076045601.tmp +2025-01-28T14:54:05.603Z [INFO] server-b9b1cf77-fae2-b389-93bb-9f6313fcaff9: starting restore from snapshot: id=2-6-1738076045601 last-index=6 last-term=2 size-in-bytes=25 +2025-01-28T14:54:05.603Z [INFO] server-b9b1cf77-fae2-b389-93bb-9f6313fcaff9: snapshot restore progress: id=2-6-1738076045601 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% +2025-01-28T14:54:05.604Z [INFO] server-b9b1cf77-fae2-b389-93bb-9f6313fcaff9: restored from snapshot: id=2-6-1738076045601 last-index=6 last-term=2 size-in-bytes=25 +2025-01-28T14:54:05.604Z [INFO] server-b9b1cf77-fae2-b389-93bb-9f6313fcaff9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0cb1976e-18c2-6d6f-cfe2-0864af19caa2 Address:0cb1976e-18c2-6d6f-cfe2-0864af19caa2} {Suffrage:Voter ID:server-b9b1cf77-fae2-b389-93bb-9f6313fcaff9 Address:b9b1cf77-fae2-b389-93bb-9f6313fcaff9} {Suffrage:Voter ID:server-ecaea33d-0b8c-e99b-c96d-e9c72cd518dd Address:ecaea33d-0b8c-e99b-c96d-e9c72cd518dd}]" +2025-01-28T14:54:05.604Z [INFO] server-b9b1cf77-fae2-b389-93bb-9f6313fcaff9: entering follower state: follower="Node at b9b1cf77-fae2-b389-93bb-9f6313fcaff9 [Follower]" leader-address= leader-id= +2025-01-28T14:54:05.604Z [INFO] creating new snapshot: path=/tmp/raft3113940077/snapshots/2-6-1738076045604.tmp +2025-01-28T14:54:05.605Z [INFO] server-ecaea33d-0b8c-e99b-c96d-e9c72cd518dd: starting restore from snapshot: id=2-6-1738076045604 last-index=6 last-term=2 size-in-bytes=25 +2025-01-28T14:54:05.606Z [INFO] server-ecaea33d-0b8c-e99b-c96d-e9c72cd518dd: snapshot restore progress: id=2-6-1738076045604 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% +2025-01-28T14:54:05.606Z [INFO] server-ecaea33d-0b8c-e99b-c96d-e9c72cd518dd: restored from snapshot: id=2-6-1738076045604 last-index=6 last-term=2 size-in-bytes=25 +2025-01-28T14:54:05.606Z [INFO] server-ecaea33d-0b8c-e99b-c96d-e9c72cd518dd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0cb1976e-18c2-6d6f-cfe2-0864af19caa2 Address:0cb1976e-18c2-6d6f-cfe2-0864af19caa2} {Suffrage:Voter ID:server-b9b1cf77-fae2-b389-93bb-9f6313fcaff9 Address:b9b1cf77-fae2-b389-93bb-9f6313fcaff9} {Suffrage:Voter ID:server-ecaea33d-0b8c-e99b-c96d-e9c72cd518dd Address:ecaea33d-0b8c-e99b-c96d-e9c72cd518dd}]" +2025-01-28T14:54:05.606Z [INFO] server-ecaea33d-0b8c-e99b-c96d-e9c72cd518dd: entering follower state: follower="Node at ecaea33d-0b8c-e99b-c96d-e9c72cd518dd [Follower]" leader-address= leader-id= +2025-01-28T14:54:05.685Z [WARN] server-b9b1cf77-fae2-b389-93bb-9f6313fcaff9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:05.685Z [INFO] server-b9b1cf77-fae2-b389-93bb-9f6313fcaff9: entering candidate state: node="Node at b9b1cf77-fae2-b389-93bb-9f6313fcaff9 [Candidate]" term=3 +2025-01-28T14:54:05.685Z [INFO] server-b9b1cf77-fae2-b389-93bb-9f6313fcaff9: election won: term=3 tally=2 +2025-01-28T14:54:05.685Z [INFO] server-b9b1cf77-fae2-b389-93bb-9f6313fcaff9: entering leader state: leader="Node at b9b1cf77-fae2-b389-93bb-9f6313fcaff9 [Leader]" +2025-01-28T14:54:05.685Z [INFO] server-b9b1cf77-fae2-b389-93bb-9f6313fcaff9: added peer, starting replication: peer=server-0cb1976e-18c2-6d6f-cfe2-0864af19caa2 +2025-01-28T14:54:05.685Z [INFO] server-b9b1cf77-fae2-b389-93bb-9f6313fcaff9: added peer, starting replication: peer=server-ecaea33d-0b8c-e99b-c96d-e9c72cd518dd +2025-01-28T14:54:05.686Z [INFO] server-b9b1cf77-fae2-b389-93bb-9f6313fcaff9: pipelining replication: peer="{Voter server-ecaea33d-0b8c-e99b-c96d-e9c72cd518dd ecaea33d-0b8c-e99b-c96d-e9c72cd518dd}" +2025-01-28T14:54:05.686Z [INFO] server-b9b1cf77-fae2-b389-93bb-9f6313fcaff9: pipelining replication: peer="{Voter server-0cb1976e-18c2-6d6f-cfe2-0864af19caa2 0cb1976e-18c2-6d6f-cfe2-0864af19caa2}" +2025-01-28T14:54:05.922Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:06.030Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:05.925042195 +0000 UTC m=+31.599532175 (1 nodes), 96.051µs from start of poll, 489.156873ms from cluster start. Timeout at 2025-01-28 14:54:06.030066333 +0000 UTC m=+31.704556333, 105.024158ms after stability +2025-01-28T14:54:06.030Z [INFO] server-b9b1cf77-fae2-b389-93bb-9f6313fcaff9: aborting pipeline replication: peer="{Voter server-ecaea33d-0b8c-e99b-c96d-e9c72cd518dd ecaea33d-0b8c-e99b-c96d-e9c72cd518dd}" +2025-01-28T14:54:06.030Z [INFO] server-b9b1cf77-fae2-b389-93bb-9f6313fcaff9: aborting pipeline replication: peer="{Voter server-0cb1976e-18c2-6d6f-cfe2-0864af19caa2 0cb1976e-18c2-6d6f-cfe2-0864af19caa2}" +2025-01-28T14:54:06.531Z [ERROR] server-b9b1cf77-fae2-b389-93bb-9f6313fcaff9: failed to heartbeat to: peer=ecaea33d-0b8c-e99b-c96d-e9c72cd518dd backoff time=10ms error="command timed out" === RUN TestRaft_RecoverCluster/snapshot,_with_trailing_logs -2025-01-28T14:47:03.621Z [INFO] server-f7b027f1-68b8-2216-8157-cedc103a184e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f7b027f1-68b8-2216-8157-cedc103a184e Address:f7b027f1-68b8-2216-8157-cedc103a184e} {Suffrage:Voter ID:server-bbc63f83-1cb3-4bcf-2919-31b9c55552e9 Address:bbc63f83-1cb3-4bcf-2919-31b9c55552e9} {Suffrage:Voter ID:server-cc998fce-243f-d7c9-9c87-63072b683c42 Address:cc998fce-243f-d7c9-9c87-63072b683c42}]" -2025-01-28T14:47:03.622Z [INFO] server-f7b027f1-68b8-2216-8157-cedc103a184e: entering follower state: follower="Node at f7b027f1-68b8-2216-8157-cedc103a184e [Follower]" leader-address= leader-id= -2025-01-28T14:47:03.623Z [INFO] server-bbc63f83-1cb3-4bcf-2919-31b9c55552e9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f7b027f1-68b8-2216-8157-cedc103a184e Address:f7b027f1-68b8-2216-8157-cedc103a184e} {Suffrage:Voter ID:server-bbc63f83-1cb3-4bcf-2919-31b9c55552e9 Address:bbc63f83-1cb3-4bcf-2919-31b9c55552e9} {Suffrage:Voter ID:server-cc998fce-243f-d7c9-9c87-63072b683c42 Address:cc998fce-243f-d7c9-9c87-63072b683c42}]" -2025-01-28T14:47:03.623Z [INFO] server-bbc63f83-1cb3-4bcf-2919-31b9c55552e9: entering follower state: follower="Node at bbc63f83-1cb3-4bcf-2919-31b9c55552e9 [Follower]" leader-address= leader-id= -2025-01-28T14:47:03.624Z [INFO] server-cc998fce-243f-d7c9-9c87-63072b683c42: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f7b027f1-68b8-2216-8157-cedc103a184e Address:f7b027f1-68b8-2216-8157-cedc103a184e} {Suffrage:Voter ID:server-bbc63f83-1cb3-4bcf-2919-31b9c55552e9 Address:bbc63f83-1cb3-4bcf-2919-31b9c55552e9} {Suffrage:Voter ID:server-cc998fce-243f-d7c9-9c87-63072b683c42 Address:cc998fce-243f-d7c9-9c87-63072b683c42}]" -2025-01-28T14:47:03.625Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:03.625Z [INFO] server-cc998fce-243f-d7c9-9c87-63072b683c42: entering follower state: follower="Node at cc998fce-243f-d7c9-9c87-63072b683c42 [Follower]" leader-address= leader-id= -2025-01-28T14:47:03.685Z [WARN] server-cc998fce-243f-d7c9-9c87-63072b683c42: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:03.685Z [INFO] server-cc998fce-243f-d7c9-9c87-63072b683c42: entering candidate state: node="Node at cc998fce-243f-d7c9-9c87-63072b683c42 [Candidate]" term=2 -2025-01-28T14:47:03.686Z [INFO] server-cc998fce-243f-d7c9-9c87-63072b683c42: election won: term=2 tally=2 -2025-01-28T14:47:03.686Z [INFO] server-cc998fce-243f-d7c9-9c87-63072b683c42: entering leader state: leader="Node at cc998fce-243f-d7c9-9c87-63072b683c42 [Leader]" -2025-01-28T14:47:03.686Z [INFO] server-cc998fce-243f-d7c9-9c87-63072b683c42: added peer, starting replication: peer=server-f7b027f1-68b8-2216-8157-cedc103a184e -2025-01-28T14:47:03.687Z [INFO] server-cc998fce-243f-d7c9-9c87-63072b683c42: added peer, starting replication: peer=server-bbc63f83-1cb3-4bcf-2919-31b9c55552e9 -2025-01-28T14:47:03.688Z [INFO] server-cc998fce-243f-d7c9-9c87-63072b683c42: pipelining replication: peer="{Voter server-f7b027f1-68b8-2216-8157-cedc103a184e f7b027f1-68b8-2216-8157-cedc103a184e}" -2025-01-28T14:47:03.688Z [INFO] server-cc998fce-243f-d7c9-9c87-63072b683c42: pipelining replication: peer="{Voter server-bbc63f83-1cb3-4bcf-2919-31b9c55552e9 bbc63f83-1cb3-4bcf-2919-31b9c55552e9}" -2025-01-28T14:47:03.792Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:03.687126977 +0000 UTC m=+6.770533001 (1 nodes), 61.932505ms from start of poll, 66.840984ms from cluster start. Timeout at 2025-01-28 14:47:03.792146644 +0000 UTC m=+6.875553668, 105.020667ms after stability -2025-01-28T14:47:03.845Z [INFO] server-cc998fce-243f-d7c9-9c87-63072b683c42: aborting pipeline replication: peer="{Voter server-bbc63f83-1cb3-4bcf-2919-31b9c55552e9 bbc63f83-1cb3-4bcf-2919-31b9c55552e9}" -2025-01-28T14:47:03.845Z [INFO] server-cc998fce-243f-d7c9-9c87-63072b683c42: aborting pipeline replication: peer="{Voter server-f7b027f1-68b8-2216-8157-cedc103a184e f7b027f1-68b8-2216-8157-cedc103a184e}" -2025-01-28T14:47:03.846Z [INFO] creating new snapshot: path=/tmp/raft3439626179/snapshots/2-27-1738075623846.tmp -2025-01-28T14:47:03.855Z [INFO] server-f7b027f1-68b8-2216-8157-cedc103a184e: starting restore from snapshot: id=2-27-1738075623846 last-index=27 last-term=2 size-in-bytes=168 -2025-01-28T14:47:03.858Z [INFO] server-f7b027f1-68b8-2216-8157-cedc103a184e: snapshot restore progress: id=2-27-1738075623846 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% -2025-01-28T14:47:03.858Z [INFO] server-f7b027f1-68b8-2216-8157-cedc103a184e: restored from snapshot: id=2-27-1738075623846 last-index=27 last-term=2 size-in-bytes=168 -2025-01-28T14:47:03.858Z [INFO] server-f7b027f1-68b8-2216-8157-cedc103a184e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f7b027f1-68b8-2216-8157-cedc103a184e Address:f7b027f1-68b8-2216-8157-cedc103a184e} {Suffrage:Voter ID:server-bbc63f83-1cb3-4bcf-2919-31b9c55552e9 Address:bbc63f83-1cb3-4bcf-2919-31b9c55552e9} {Suffrage:Voter ID:server-cc998fce-243f-d7c9-9c87-63072b683c42 Address:cc998fce-243f-d7c9-9c87-63072b683c42}]" -2025-01-28T14:47:03.859Z [INFO] server-f7b027f1-68b8-2216-8157-cedc103a184e: entering follower state: follower="Node at f7b027f1-68b8-2216-8157-cedc103a184e [Follower]" leader-address= leader-id= -2025-01-28T14:47:03.860Z [INFO] creating new snapshot: path=/tmp/raft1996937746/snapshots/2-27-1738075623860.tmp -2025-01-28T14:47:03.869Z [INFO] server-bbc63f83-1cb3-4bcf-2919-31b9c55552e9: starting restore from snapshot: id=2-27-1738075623860 last-index=27 last-term=2 size-in-bytes=168 -2025-01-28T14:47:03.871Z [INFO] server-bbc63f83-1cb3-4bcf-2919-31b9c55552e9: snapshot restore progress: id=2-27-1738075623860 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% -2025-01-28T14:47:03.872Z [INFO] server-bbc63f83-1cb3-4bcf-2919-31b9c55552e9: restored from snapshot: id=2-27-1738075623860 last-index=27 last-term=2 size-in-bytes=168 -2025-01-28T14:47:03.872Z [INFO] server-bbc63f83-1cb3-4bcf-2919-31b9c55552e9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f7b027f1-68b8-2216-8157-cedc103a184e Address:f7b027f1-68b8-2216-8157-cedc103a184e} {Suffrage:Voter ID:server-bbc63f83-1cb3-4bcf-2919-31b9c55552e9 Address:bbc63f83-1cb3-4bcf-2919-31b9c55552e9} {Suffrage:Voter ID:server-cc998fce-243f-d7c9-9c87-63072b683c42 Address:cc998fce-243f-d7c9-9c87-63072b683c42}]" -2025-01-28T14:47:03.873Z [INFO] server-bbc63f83-1cb3-4bcf-2919-31b9c55552e9: entering follower state: follower="Node at bbc63f83-1cb3-4bcf-2919-31b9c55552e9 [Follower]" leader-address= leader-id= -2025-01-28T14:47:03.874Z [INFO] creating new snapshot: path=/tmp/raft2051273926/snapshots/2-27-1738075623874.tmp -2025-01-28T14:47:03.881Z [INFO] server-cc998fce-243f-d7c9-9c87-63072b683c42: starting restore from snapshot: id=2-27-1738075623874 last-index=27 last-term=2 size-in-bytes=168 -2025-01-28T14:47:03.883Z [INFO] server-cc998fce-243f-d7c9-9c87-63072b683c42: snapshot restore progress: id=2-27-1738075623874 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% -2025-01-28T14:47:03.883Z [INFO] server-cc998fce-243f-d7c9-9c87-63072b683c42: restored from snapshot: id=2-27-1738075623874 last-index=27 last-term=2 size-in-bytes=168 -2025-01-28T14:47:03.883Z [INFO] server-cc998fce-243f-d7c9-9c87-63072b683c42: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f7b027f1-68b8-2216-8157-cedc103a184e Address:f7b027f1-68b8-2216-8157-cedc103a184e} {Suffrage:Voter ID:server-bbc63f83-1cb3-4bcf-2919-31b9c55552e9 Address:bbc63f83-1cb3-4bcf-2919-31b9c55552e9} {Suffrage:Voter ID:server-cc998fce-243f-d7c9-9c87-63072b683c42 Address:cc998fce-243f-d7c9-9c87-63072b683c42}]" -2025-01-28T14:47:03.883Z [INFO] server-cc998fce-243f-d7c9-9c87-63072b683c42: entering follower state: follower="Node at cc998fce-243f-d7c9-9c87-63072b683c42 [Follower]" leader-address= leader-id= -2025-01-28T14:47:03.923Z [WARN] server-f7b027f1-68b8-2216-8157-cedc103a184e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:03.923Z [INFO] server-f7b027f1-68b8-2216-8157-cedc103a184e: entering candidate state: node="Node at f7b027f1-68b8-2216-8157-cedc103a184e [Candidate]" term=3 -2025-01-28T14:47:03.923Z [INFO] server-f7b027f1-68b8-2216-8157-cedc103a184e: election won: term=3 tally=2 -2025-01-28T14:47:03.924Z [INFO] server-f7b027f1-68b8-2216-8157-cedc103a184e: entering leader state: leader="Node at f7b027f1-68b8-2216-8157-cedc103a184e [Leader]" -2025-01-28T14:47:03.924Z [INFO] server-f7b027f1-68b8-2216-8157-cedc103a184e: added peer, starting replication: peer=server-bbc63f83-1cb3-4bcf-2919-31b9c55552e9 -2025-01-28T14:47:03.924Z [INFO] server-f7b027f1-68b8-2216-8157-cedc103a184e: added peer, starting replication: peer=server-cc998fce-243f-d7c9-9c87-63072b683c42 -2025-01-28T14:47:03.925Z [INFO] server-f7b027f1-68b8-2216-8157-cedc103a184e: pipelining replication: peer="{Voter server-bbc63f83-1cb3-4bcf-2919-31b9c55552e9 bbc63f83-1cb3-4bcf-2919-31b9c55552e9}" -2025-01-28T14:47:03.925Z [INFO] server-f7b027f1-68b8-2216-8157-cedc103a184e: pipelining replication: peer="{Voter server-cc998fce-243f-d7c9-9c87-63072b683c42 cc998fce-243f-d7c9-9c87-63072b683c42}" -2025-01-28T14:47:04.198Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:04.305Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:04.199727528 +0000 UTC m=+7.283134219 (1 nodes), 443.679µs from start of poll, 579.442202ms from cluster start. Timeout at 2025-01-28 14:47:04.304887199 +0000 UTC m=+7.388293224, 105.159005ms after stability -2025-01-28T14:47:04.306Z [INFO] server-f7b027f1-68b8-2216-8157-cedc103a184e: aborting pipeline replication: peer="{Voter server-cc998fce-243f-d7c9-9c87-63072b683c42 cc998fce-243f-d7c9-9c87-63072b683c42}" -2025-01-28T14:47:04.306Z [INFO] server-f7b027f1-68b8-2216-8157-cedc103a184e: aborting pipeline replication: peer="{Voter server-bbc63f83-1cb3-4bcf-2919-31b9c55552e9 bbc63f83-1cb3-4bcf-2919-31b9c55552e9}" -2025-01-28T14:47:04.807Z [ERROR] server-f7b027f1-68b8-2216-8157-cedc103a184e: failed to heartbeat to: peer=bbc63f83-1cb3-4bcf-2919-31b9c55552e9 backoff time=10ms error="command timed out" ---- PASS: TestRaft_RecoverCluster (3.01s) - --- PASS: TestRaft_RecoverCluster/no_snapshot,_no_trailing_logs (1.15s) - --- PASS: TestRaft_RecoverCluster/no_snapshot,_some_trailing_logs (0.66s) - --- PASS: TestRaft_RecoverCluster/snapshot,_with_trailing_logs (1.21s) +2025-01-28T14:54:06.537Z [INFO] server-31607b25-9e8c-51af-26f5-7ccfc34cad4a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31607b25-9e8c-51af-26f5-7ccfc34cad4a Address:31607b25-9e8c-51af-26f5-7ccfc34cad4a} {Suffrage:Voter ID:server-893a3b65-5cc9-bccf-a8fa-92de7626fc78 Address:893a3b65-5cc9-bccf-a8fa-92de7626fc78} {Suffrage:Voter ID:server-b22db518-672f-70eb-3290-9cba5bc1cf78 Address:b22db518-672f-70eb-3290-9cba5bc1cf78}]" +2025-01-28T14:54:06.537Z [INFO] server-31607b25-9e8c-51af-26f5-7ccfc34cad4a: entering follower state: follower="Node at 31607b25-9e8c-51af-26f5-7ccfc34cad4a [Follower]" leader-address= leader-id= +2025-01-28T14:54:06.537Z [INFO] server-893a3b65-5cc9-bccf-a8fa-92de7626fc78: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31607b25-9e8c-51af-26f5-7ccfc34cad4a Address:31607b25-9e8c-51af-26f5-7ccfc34cad4a} {Suffrage:Voter ID:server-893a3b65-5cc9-bccf-a8fa-92de7626fc78 Address:893a3b65-5cc9-bccf-a8fa-92de7626fc78} {Suffrage:Voter ID:server-b22db518-672f-70eb-3290-9cba5bc1cf78 Address:b22db518-672f-70eb-3290-9cba5bc1cf78}]" +2025-01-28T14:54:06.537Z [INFO] server-893a3b65-5cc9-bccf-a8fa-92de7626fc78: entering follower state: follower="Node at 893a3b65-5cc9-bccf-a8fa-92de7626fc78 [Follower]" leader-address= leader-id= +2025-01-28T14:54:06.538Z [INFO] server-b22db518-672f-70eb-3290-9cba5bc1cf78: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31607b25-9e8c-51af-26f5-7ccfc34cad4a Address:31607b25-9e8c-51af-26f5-7ccfc34cad4a} {Suffrage:Voter ID:server-893a3b65-5cc9-bccf-a8fa-92de7626fc78 Address:893a3b65-5cc9-bccf-a8fa-92de7626fc78} {Suffrage:Voter ID:server-b22db518-672f-70eb-3290-9cba5bc1cf78 Address:b22db518-672f-70eb-3290-9cba5bc1cf78}]" +2025-01-28T14:54:06.538Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:06.538Z [INFO] server-b22db518-672f-70eb-3290-9cba5bc1cf78: entering follower state: follower="Node at b22db518-672f-70eb-3290-9cba5bc1cf78 [Follower]" leader-address= leader-id= +2025-01-28T14:54:06.597Z [WARN] server-31607b25-9e8c-51af-26f5-7ccfc34cad4a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:06.597Z [INFO] server-31607b25-9e8c-51af-26f5-7ccfc34cad4a: entering candidate state: node="Node at 31607b25-9e8c-51af-26f5-7ccfc34cad4a [Candidate]" term=2 +2025-01-28T14:54:06.597Z [INFO] server-31607b25-9e8c-51af-26f5-7ccfc34cad4a: election won: term=2 tally=2 +2025-01-28T14:54:06.597Z [INFO] server-31607b25-9e8c-51af-26f5-7ccfc34cad4a: entering leader state: leader="Node at 31607b25-9e8c-51af-26f5-7ccfc34cad4a [Leader]" +2025-01-28T14:54:06.597Z [INFO] server-31607b25-9e8c-51af-26f5-7ccfc34cad4a: added peer, starting replication: peer=server-893a3b65-5cc9-bccf-a8fa-92de7626fc78 +2025-01-28T14:54:06.597Z [INFO] server-31607b25-9e8c-51af-26f5-7ccfc34cad4a: added peer, starting replication: peer=server-b22db518-672f-70eb-3290-9cba5bc1cf78 +2025-01-28T14:54:06.597Z [INFO] server-31607b25-9e8c-51af-26f5-7ccfc34cad4a: pipelining replication: peer="{Voter server-893a3b65-5cc9-bccf-a8fa-92de7626fc78 893a3b65-5cc9-bccf-a8fa-92de7626fc78}" +2025-01-28T14:54:06.597Z [INFO] server-31607b25-9e8c-51af-26f5-7ccfc34cad4a: pipelining replication: peer="{Voter server-b22db518-672f-70eb-3290-9cba5bc1cf78 b22db518-672f-70eb-3290-9cba5bc1cf78}" +2025-01-28T14:54:06.702Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:06.597510734 +0000 UTC m=+32.272000734 (1 nodes), 59.235211ms from start of poll, 60.31077ms from cluster start. Timeout at 2025-01-28 14:54:06.702522573 +0000 UTC m=+32.377012553, 105.011819ms after stability +2025-01-28T14:54:06.710Z [INFO] server-31607b25-9e8c-51af-26f5-7ccfc34cad4a: aborting pipeline replication: peer="{Voter server-b22db518-672f-70eb-3290-9cba5bc1cf78 b22db518-672f-70eb-3290-9cba5bc1cf78}" +2025-01-28T14:54:06.710Z [INFO] server-31607b25-9e8c-51af-26f5-7ccfc34cad4a: aborting pipeline replication: peer="{Voter server-893a3b65-5cc9-bccf-a8fa-92de7626fc78 893a3b65-5cc9-bccf-a8fa-92de7626fc78}" +2025-01-28T14:54:06.710Z [INFO] creating new snapshot: path=/tmp/raft3604028932/snapshots/2-27-1738076046710.tmp +2025-01-28T14:54:06.713Z [INFO] server-31607b25-9e8c-51af-26f5-7ccfc34cad4a: starting restore from snapshot: id=2-27-1738076046710 last-index=27 last-term=2 size-in-bytes=168 +2025-01-28T14:54:06.713Z [INFO] server-31607b25-9e8c-51af-26f5-7ccfc34cad4a: snapshot restore progress: id=2-27-1738076046710 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% +2025-01-28T14:54:06.713Z [INFO] server-31607b25-9e8c-51af-26f5-7ccfc34cad4a: restored from snapshot: id=2-27-1738076046710 last-index=27 last-term=2 size-in-bytes=168 +2025-01-28T14:54:06.713Z [INFO] server-31607b25-9e8c-51af-26f5-7ccfc34cad4a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31607b25-9e8c-51af-26f5-7ccfc34cad4a Address:31607b25-9e8c-51af-26f5-7ccfc34cad4a} {Suffrage:Voter ID:server-893a3b65-5cc9-bccf-a8fa-92de7626fc78 Address:893a3b65-5cc9-bccf-a8fa-92de7626fc78} {Suffrage:Voter ID:server-b22db518-672f-70eb-3290-9cba5bc1cf78 Address:b22db518-672f-70eb-3290-9cba5bc1cf78}]" +2025-01-28T14:54:06.713Z [INFO] server-31607b25-9e8c-51af-26f5-7ccfc34cad4a: entering follower state: follower="Node at 31607b25-9e8c-51af-26f5-7ccfc34cad4a [Follower]" leader-address= leader-id= +2025-01-28T14:54:06.713Z [INFO] creating new snapshot: path=/tmp/raft3703374330/snapshots/2-27-1738076046713.tmp +2025-01-28T14:54:06.716Z [INFO] server-893a3b65-5cc9-bccf-a8fa-92de7626fc78: starting restore from snapshot: id=2-27-1738076046713 last-index=27 last-term=2 size-in-bytes=168 +2025-01-28T14:54:06.716Z [INFO] server-893a3b65-5cc9-bccf-a8fa-92de7626fc78: snapshot restore progress: id=2-27-1738076046713 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% +2025-01-28T14:54:06.716Z [INFO] server-893a3b65-5cc9-bccf-a8fa-92de7626fc78: restored from snapshot: id=2-27-1738076046713 last-index=27 last-term=2 size-in-bytes=168 +2025-01-28T14:54:06.716Z [INFO] server-893a3b65-5cc9-bccf-a8fa-92de7626fc78: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31607b25-9e8c-51af-26f5-7ccfc34cad4a Address:31607b25-9e8c-51af-26f5-7ccfc34cad4a} {Suffrage:Voter ID:server-893a3b65-5cc9-bccf-a8fa-92de7626fc78 Address:893a3b65-5cc9-bccf-a8fa-92de7626fc78} {Suffrage:Voter ID:server-b22db518-672f-70eb-3290-9cba5bc1cf78 Address:b22db518-672f-70eb-3290-9cba5bc1cf78}]" +2025-01-28T14:54:06.716Z [INFO] server-893a3b65-5cc9-bccf-a8fa-92de7626fc78: entering follower state: follower="Node at 893a3b65-5cc9-bccf-a8fa-92de7626fc78 [Follower]" leader-address= leader-id= +2025-01-28T14:54:06.716Z [INFO] creating new snapshot: path=/tmp/raft1485038946/snapshots/2-27-1738076046716.tmp +2025-01-28T14:54:06.718Z [INFO] server-b22db518-672f-70eb-3290-9cba5bc1cf78: starting restore from snapshot: id=2-27-1738076046716 last-index=27 last-term=2 size-in-bytes=168 +2025-01-28T14:54:06.719Z [INFO] server-b22db518-672f-70eb-3290-9cba5bc1cf78: snapshot restore progress: id=2-27-1738076046716 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% +2025-01-28T14:54:06.719Z [INFO] server-b22db518-672f-70eb-3290-9cba5bc1cf78: restored from snapshot: id=2-27-1738076046716 last-index=27 last-term=2 size-in-bytes=168 +2025-01-28T14:54:06.719Z [INFO] server-b22db518-672f-70eb-3290-9cba5bc1cf78: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31607b25-9e8c-51af-26f5-7ccfc34cad4a Address:31607b25-9e8c-51af-26f5-7ccfc34cad4a} {Suffrage:Voter ID:server-893a3b65-5cc9-bccf-a8fa-92de7626fc78 Address:893a3b65-5cc9-bccf-a8fa-92de7626fc78} {Suffrage:Voter ID:server-b22db518-672f-70eb-3290-9cba5bc1cf78 Address:b22db518-672f-70eb-3290-9cba5bc1cf78}]" +2025-01-28T14:54:06.719Z [INFO] server-b22db518-672f-70eb-3290-9cba5bc1cf78: entering follower state: follower="Node at b22db518-672f-70eb-3290-9cba5bc1cf78 [Follower]" leader-address= leader-id= +2025-01-28T14:54:06.772Z [WARN] server-b22db518-672f-70eb-3290-9cba5bc1cf78: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:06.772Z [INFO] server-b22db518-672f-70eb-3290-9cba5bc1cf78: entering candidate state: node="Node at b22db518-672f-70eb-3290-9cba5bc1cf78 [Candidate]" term=3 +2025-01-28T14:54:06.772Z [INFO] server-b22db518-672f-70eb-3290-9cba5bc1cf78: election won: term=3 tally=2 +2025-01-28T14:54:06.772Z [INFO] server-b22db518-672f-70eb-3290-9cba5bc1cf78: entering leader state: leader="Node at b22db518-672f-70eb-3290-9cba5bc1cf78 [Leader]" +2025-01-28T14:54:06.772Z [INFO] server-b22db518-672f-70eb-3290-9cba5bc1cf78: added peer, starting replication: peer=server-31607b25-9e8c-51af-26f5-7ccfc34cad4a +2025-01-28T14:54:06.772Z [INFO] server-b22db518-672f-70eb-3290-9cba5bc1cf78: added peer, starting replication: peer=server-893a3b65-5cc9-bccf-a8fa-92de7626fc78 +2025-01-28T14:54:06.773Z [INFO] server-b22db518-672f-70eb-3290-9cba5bc1cf78: pipelining replication: peer="{Voter server-31607b25-9e8c-51af-26f5-7ccfc34cad4a 31607b25-9e8c-51af-26f5-7ccfc34cad4a}" +2025-01-28T14:54:06.773Z [INFO] server-b22db518-672f-70eb-3290-9cba5bc1cf78: pipelining replication: peer="{Voter server-893a3b65-5cc9-bccf-a8fa-92de7626fc78 893a3b65-5cc9-bccf-a8fa-92de7626fc78}" +2025-01-28T14:54:07.035Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:07.141Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:07.035633082 +0000 UTC m=+32.710123042 (1 nodes), 66.854µs from start of poll, 498.433078ms from cluster start. Timeout at 2025-01-28 14:54:07.14065896 +0000 UTC m=+32.815148980, 105.025938ms after stability +2025-01-28T14:54:07.141Z [INFO] server-b22db518-672f-70eb-3290-9cba5bc1cf78: aborting pipeline replication: peer="{Voter server-893a3b65-5cc9-bccf-a8fa-92de7626fc78 893a3b65-5cc9-bccf-a8fa-92de7626fc78}" +2025-01-28T14:54:07.141Z [INFO] server-b22db518-672f-70eb-3290-9cba5bc1cf78: aborting pipeline replication: peer="{Voter server-31607b25-9e8c-51af-26f5-7ccfc34cad4a 31607b25-9e8c-51af-26f5-7ccfc34cad4a}" +--- PASS: TestRaft_RecoverCluster (2.83s) + --- PASS: TestRaft_RecoverCluster/no_snapshot,_no_trailing_logs (1.12s) + --- PASS: TestRaft_RecoverCluster/no_snapshot,_some_trailing_logs (1.10s) + --- PASS: TestRaft_RecoverCluster/snapshot,_with_trailing_logs (0.61s) === RUN TestRaft_HasExistingState raft_test.go:258: Skipping unreliable test --- SKIP: TestRaft_HasExistingState (0.00s) === RUN TestRaft_SingleNode -2025-01-28T14:47:04.825Z [INFO] server-1e288a7f-7f12-3fc8-e3fd-cdd537b4d129: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1e288a7f-7f12-3fc8-e3fd-cdd537b4d129 Address:1e288a7f-7f12-3fc8-e3fd-cdd537b4d129}]" -2025-01-28T14:47:04.826Z [INFO] server-1e288a7f-7f12-3fc8-e3fd-cdd537b4d129: entering follower state: follower="Node at 1e288a7f-7f12-3fc8-e3fd-cdd537b4d129 [Follower]" leader-address= leader-id= -2025-01-28T14:47:04.906Z [WARN] server-1e288a7f-7f12-3fc8-e3fd-cdd537b4d129: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:04.906Z [INFO] server-1e288a7f-7f12-3fc8-e3fd-cdd537b4d129: entering candidate state: node="Node at 1e288a7f-7f12-3fc8-e3fd-cdd537b4d129 [Candidate]" term=2 -2025-01-28T14:47:04.907Z [INFO] server-1e288a7f-7f12-3fc8-e3fd-cdd537b4d129: election won: term=2 tally=1 -2025-01-28T14:47:04.907Z [INFO] server-1e288a7f-7f12-3fc8-e3fd-cdd537b4d129: entering leader state: leader="Node at 1e288a7f-7f12-3fc8-e3fd-cdd537b4d129 [Leader]" +2025-01-28T14:54:07.145Z [INFO] server-2c2f2f1c-87df-4c24-4c68-16a980495fc5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2c2f2f1c-87df-4c24-4c68-16a980495fc5 Address:2c2f2f1c-87df-4c24-4c68-16a980495fc5}]" +2025-01-28T14:54:07.145Z [INFO] server-2c2f2f1c-87df-4c24-4c68-16a980495fc5: entering follower state: follower="Node at 2c2f2f1c-87df-4c24-4c68-16a980495fc5 [Follower]" leader-address= leader-id= +2025-01-28T14:54:07.235Z [WARN] server-2c2f2f1c-87df-4c24-4c68-16a980495fc5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:07.235Z [INFO] server-2c2f2f1c-87df-4c24-4c68-16a980495fc5: entering candidate state: node="Node at 2c2f2f1c-87df-4c24-4c68-16a980495fc5 [Candidate]" term=2 +2025-01-28T14:54:07.235Z [INFO] server-2c2f2f1c-87df-4c24-4c68-16a980495fc5: election won: term=2 tally=1 +2025-01-28T14:54:07.235Z [INFO] server-2c2f2f1c-87df-4c24-4c68-16a980495fc5: entering leader state: leader="Node at 2c2f2f1c-87df-4c24-4c68-16a980495fc5 [Leader]" --- PASS: TestRaft_SingleNode (0.09s) === RUN TestRaft_TripleNode -2025-01-28T14:47:04.922Z [INFO] server-54ffa07a-76e2-8807-cd00-76ea5a156cd5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-54ffa07a-76e2-8807-cd00-76ea5a156cd5 Address:54ffa07a-76e2-8807-cd00-76ea5a156cd5} {Suffrage:Voter ID:server-b8987b37-93de-fc75-3558-7f04ffd4418d Address:b8987b37-93de-fc75-3558-7f04ffd4418d} {Suffrage:Voter ID:server-d1070395-f174-c0ce-f8cb-cf3543699be1 Address:d1070395-f174-c0ce-f8cb-cf3543699be1}]" -2025-01-28T14:47:04.923Z [INFO] server-54ffa07a-76e2-8807-cd00-76ea5a156cd5: entering follower state: follower="Node at 54ffa07a-76e2-8807-cd00-76ea5a156cd5 [Follower]" leader-address= leader-id= -2025-01-28T14:47:04.924Z [INFO] server-b8987b37-93de-fc75-3558-7f04ffd4418d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-54ffa07a-76e2-8807-cd00-76ea5a156cd5 Address:54ffa07a-76e2-8807-cd00-76ea5a156cd5} {Suffrage:Voter ID:server-b8987b37-93de-fc75-3558-7f04ffd4418d Address:b8987b37-93de-fc75-3558-7f04ffd4418d} {Suffrage:Voter ID:server-d1070395-f174-c0ce-f8cb-cf3543699be1 Address:d1070395-f174-c0ce-f8cb-cf3543699be1}]" -2025-01-28T14:47:04.924Z [INFO] server-b8987b37-93de-fc75-3558-7f04ffd4418d: entering follower state: follower="Node at b8987b37-93de-fc75-3558-7f04ffd4418d [Follower]" leader-address= leader-id= -2025-01-28T14:47:04.925Z [INFO] server-d1070395-f174-c0ce-f8cb-cf3543699be1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-54ffa07a-76e2-8807-cd00-76ea5a156cd5 Address:54ffa07a-76e2-8807-cd00-76ea5a156cd5} {Suffrage:Voter ID:server-b8987b37-93de-fc75-3558-7f04ffd4418d Address:b8987b37-93de-fc75-3558-7f04ffd4418d} {Suffrage:Voter ID:server-d1070395-f174-c0ce-f8cb-cf3543699be1 Address:d1070395-f174-c0ce-f8cb-cf3543699be1}]" -2025-01-28T14:47:04.925Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:47:04.925Z [INFO] server-d1070395-f174-c0ce-f8cb-cf3543699be1: entering follower state: follower="Node at d1070395-f174-c0ce-f8cb-cf3543699be1 [Follower]" leader-address= leader-id= -2025-01-28T14:47:04.983Z [WARN] server-d1070395-f174-c0ce-f8cb-cf3543699be1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:04.983Z [INFO] server-d1070395-f174-c0ce-f8cb-cf3543699be1: entering candidate state: node="Node at d1070395-f174-c0ce-f8cb-cf3543699be1 [Candidate]" term=2 -2025-01-28T14:47:04.984Z [INFO] server-d1070395-f174-c0ce-f8cb-cf3543699be1: election won: term=2 tally=2 -2025-01-28T14:47:04.984Z [INFO] server-d1070395-f174-c0ce-f8cb-cf3543699be1: entering leader state: leader="Node at d1070395-f174-c0ce-f8cb-cf3543699be1 [Leader]" -2025-01-28T14:47:04.985Z [INFO] server-d1070395-f174-c0ce-f8cb-cf3543699be1: added peer, starting replication: peer=server-54ffa07a-76e2-8807-cd00-76ea5a156cd5 -2025-01-28T14:47:04.985Z [INFO] server-d1070395-f174-c0ce-f8cb-cf3543699be1: added peer, starting replication: peer=server-b8987b37-93de-fc75-3558-7f04ffd4418d -2025-01-28T14:47:04.986Z [INFO] server-d1070395-f174-c0ce-f8cb-cf3543699be1: pipelining replication: peer="{Voter server-54ffa07a-76e2-8807-cd00-76ea5a156cd5 54ffa07a-76e2-8807-cd00-76ea5a156cd5}" -2025-01-28T14:47:04.987Z [INFO] server-d1070395-f174-c0ce-f8cb-cf3543699be1: pipelining replication: peer="{Voter server-b8987b37-93de-fc75-3558-7f04ffd4418d b8987b37-93de-fc75-3558-7f04ffd4418d}" -2025-01-28T14:47:05.090Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:47:04.98517618 +0000 UTC m=+8.068582871 (2 nodes), 59.114422ms from start of poll, 63.945898ms from cluster start. Timeout at 2025-01-28 14:47:05.090215514 +0000 UTC m=+8.173621539, 105.038668ms after stability -2025-01-28T14:47:05.091Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:05.197Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:05.091284663 +0000 UTC m=+8.174690354 (1 nodes), 12.667µs from start of poll, 170.053381ms from cluster start. Timeout at 2025-01-28 14:47:05.196366333 +0000 UTC m=+8.279772691, 105.082337ms after stability -2025-01-28T14:47:05.212Z [INFO] server-d1070395-f174-c0ce-f8cb-cf3543699be1: aborting pipeline replication: peer="{Voter server-b8987b37-93de-fc75-3558-7f04ffd4418d b8987b37-93de-fc75-3558-7f04ffd4418d}" -2025-01-28T14:47:05.212Z [INFO] server-d1070395-f174-c0ce-f8cb-cf3543699be1: aborting pipeline replication: peer="{Voter server-54ffa07a-76e2-8807-cd00-76ea5a156cd5 54ffa07a-76e2-8807-cd00-76ea5a156cd5}" -2025-01-28T14:47:05.714Z [ERROR] server-d1070395-f174-c0ce-f8cb-cf3543699be1: failed to heartbeat to: peer=b8987b37-93de-fc75-3558-7f04ffd4418d backoff time=10ms error="command timed out" ---- PASS: TestRaft_TripleNode (0.81s) +2025-01-28T14:54:07.239Z [INFO] server-4176ae00-01fe-9053-9737-d187d7c52821: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4176ae00-01fe-9053-9737-d187d7c52821 Address:4176ae00-01fe-9053-9737-d187d7c52821} {Suffrage:Voter ID:server-78e757d8-8d82-847e-a62a-2541af981efc Address:78e757d8-8d82-847e-a62a-2541af981efc} {Suffrage:Voter ID:server-19964c54-a948-f5f4-f109-0505c07c6eff Address:19964c54-a948-f5f4-f109-0505c07c6eff}]" +2025-01-28T14:54:07.239Z [INFO] server-4176ae00-01fe-9053-9737-d187d7c52821: entering follower state: follower="Node at 4176ae00-01fe-9053-9737-d187d7c52821 [Follower]" leader-address= leader-id= +2025-01-28T14:54:07.239Z [INFO] server-78e757d8-8d82-847e-a62a-2541af981efc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4176ae00-01fe-9053-9737-d187d7c52821 Address:4176ae00-01fe-9053-9737-d187d7c52821} {Suffrage:Voter ID:server-78e757d8-8d82-847e-a62a-2541af981efc Address:78e757d8-8d82-847e-a62a-2541af981efc} {Suffrage:Voter ID:server-19964c54-a948-f5f4-f109-0505c07c6eff Address:19964c54-a948-f5f4-f109-0505c07c6eff}]" +2025-01-28T14:54:07.239Z [INFO] server-78e757d8-8d82-847e-a62a-2541af981efc: entering follower state: follower="Node at 78e757d8-8d82-847e-a62a-2541af981efc [Follower]" leader-address= leader-id= +2025-01-28T14:54:07.239Z [INFO] server-19964c54-a948-f5f4-f109-0505c07c6eff: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4176ae00-01fe-9053-9737-d187d7c52821 Address:4176ae00-01fe-9053-9737-d187d7c52821} {Suffrage:Voter ID:server-78e757d8-8d82-847e-a62a-2541af981efc Address:78e757d8-8d82-847e-a62a-2541af981efc} {Suffrage:Voter ID:server-19964c54-a948-f5f4-f109-0505c07c6eff Address:19964c54-a948-f5f4-f109-0505c07c6eff}]" +2025-01-28T14:54:07.239Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:54:07.239Z [INFO] server-19964c54-a948-f5f4-f109-0505c07c6eff: entering follower state: follower="Node at 19964c54-a948-f5f4-f109-0505c07c6eff [Follower]" leader-address= leader-id= +2025-01-28T14:54:07.318Z [WARN] server-19964c54-a948-f5f4-f109-0505c07c6eff: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:07.318Z [INFO] server-19964c54-a948-f5f4-f109-0505c07c6eff: entering candidate state: node="Node at 19964c54-a948-f5f4-f109-0505c07c6eff [Candidate]" term=2 +2025-01-28T14:54:07.318Z [INFO] server-19964c54-a948-f5f4-f109-0505c07c6eff: election won: term=2 tally=2 +2025-01-28T14:54:07.318Z [INFO] server-19964c54-a948-f5f4-f109-0505c07c6eff: entering leader state: leader="Node at 19964c54-a948-f5f4-f109-0505c07c6eff [Leader]" +2025-01-28T14:54:07.318Z [INFO] server-19964c54-a948-f5f4-f109-0505c07c6eff: added peer, starting replication: peer=server-4176ae00-01fe-9053-9737-d187d7c52821 +2025-01-28T14:54:07.318Z [INFO] server-19964c54-a948-f5f4-f109-0505c07c6eff: added peer, starting replication: peer=server-78e757d8-8d82-847e-a62a-2541af981efc +2025-01-28T14:54:07.319Z [INFO] server-19964c54-a948-f5f4-f109-0505c07c6eff: pipelining replication: peer="{Voter server-4176ae00-01fe-9053-9737-d187d7c52821 4176ae00-01fe-9053-9737-d187d7c52821}" +2025-01-28T14:54:07.319Z [INFO] server-19964c54-a948-f5f4-f109-0505c07c6eff: pipelining replication: peer="{Voter server-78e757d8-8d82-847e-a62a-2541af981efc 78e757d8-8d82-847e-a62a-2541af981efc}" +2025-01-28T14:54:07.424Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:54:07.318928156 +0000 UTC m=+32.993418136 (2 nodes), 78.998167ms from start of poll, 80.176136ms from cluster start. Timeout at 2025-01-28 14:54:07.423934635 +0000 UTC m=+33.098424695, 105.006559ms after stability +2025-01-28T14:54:07.424Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:07.529Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:07.424766651 +0000 UTC m=+33.099256631 (1 nodes), 1.999µs from start of poll, 186.014631ms from cluster start. Timeout at 2025-01-28 14:54:07.529795448 +0000 UTC m=+33.204285508, 105.028877ms after stability +2025-01-28T14:54:07.541Z [INFO] server-19964c54-a948-f5f4-f109-0505c07c6eff: aborting pipeline replication: peer="{Voter server-78e757d8-8d82-847e-a62a-2541af981efc 78e757d8-8d82-847e-a62a-2541af981efc}" +2025-01-28T14:54:07.541Z [INFO] server-19964c54-a948-f5f4-f109-0505c07c6eff: aborting pipeline replication: peer="{Voter server-4176ae00-01fe-9053-9737-d187d7c52821 4176ae00-01fe-9053-9737-d187d7c52821}" +--- PASS: TestRaft_TripleNode (0.31s) === RUN TestRaft_LeaderFail -2025-01-28T14:47:05.732Z [INFO] server-e86384bf-8c6b-4166-35b1-83d62a5346db: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e86384bf-8c6b-4166-35b1-83d62a5346db Address:e86384bf-8c6b-4166-35b1-83d62a5346db} {Suffrage:Voter ID:server-a4321908-d7fe-45f3-5b12-771cfcb77ecf Address:a4321908-d7fe-45f3-5b12-771cfcb77ecf} {Suffrage:Voter ID:server-fcf8ed41-29dd-91bd-1de3-270377a51fba Address:fcf8ed41-29dd-91bd-1de3-270377a51fba}]" -2025-01-28T14:47:05.733Z [INFO] server-e86384bf-8c6b-4166-35b1-83d62a5346db: entering follower state: follower="Node at e86384bf-8c6b-4166-35b1-83d62a5346db [Follower]" leader-address= leader-id= -2025-01-28T14:47:05.734Z [INFO] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e86384bf-8c6b-4166-35b1-83d62a5346db Address:e86384bf-8c6b-4166-35b1-83d62a5346db} {Suffrage:Voter ID:server-a4321908-d7fe-45f3-5b12-771cfcb77ecf Address:a4321908-d7fe-45f3-5b12-771cfcb77ecf} {Suffrage:Voter ID:server-fcf8ed41-29dd-91bd-1de3-270377a51fba Address:fcf8ed41-29dd-91bd-1de3-270377a51fba}]" -2025-01-28T14:47:05.734Z [INFO] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: entering follower state: follower="Node at a4321908-d7fe-45f3-5b12-771cfcb77ecf [Follower]" leader-address= leader-id= -2025-01-28T14:47:05.735Z [INFO] server-fcf8ed41-29dd-91bd-1de3-270377a51fba: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e86384bf-8c6b-4166-35b1-83d62a5346db Address:e86384bf-8c6b-4166-35b1-83d62a5346db} {Suffrage:Voter ID:server-a4321908-d7fe-45f3-5b12-771cfcb77ecf Address:a4321908-d7fe-45f3-5b12-771cfcb77ecf} {Suffrage:Voter ID:server-fcf8ed41-29dd-91bd-1de3-270377a51fba Address:fcf8ed41-29dd-91bd-1de3-270377a51fba}]" -2025-01-28T14:47:05.735Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:47:05.736Z [INFO] server-fcf8ed41-29dd-91bd-1de3-270377a51fba: entering follower state: follower="Node at fcf8ed41-29dd-91bd-1de3-270377a51fba [Follower]" leader-address= leader-id= -2025-01-28T14:47:05.797Z [WARN] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:05.797Z [INFO] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: entering candidate state: node="Node at a4321908-d7fe-45f3-5b12-771cfcb77ecf [Candidate]" term=2 -2025-01-28T14:47:05.798Z [INFO] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: election won: term=2 tally=2 -2025-01-28T14:47:05.798Z [INFO] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: entering leader state: leader="Node at a4321908-d7fe-45f3-5b12-771cfcb77ecf [Leader]" -2025-01-28T14:47:05.798Z [INFO] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: added peer, starting replication: peer=server-e86384bf-8c6b-4166-35b1-83d62a5346db -2025-01-28T14:47:05.799Z [INFO] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: added peer, starting replication: peer=server-fcf8ed41-29dd-91bd-1de3-270377a51fba -2025-01-28T14:47:05.800Z [INFO] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: pipelining replication: peer="{Voter server-e86384bf-8c6b-4166-35b1-83d62a5346db e86384bf-8c6b-4166-35b1-83d62a5346db}" -2025-01-28T14:47:05.801Z [INFO] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: pipelining replication: peer="{Voter server-fcf8ed41-29dd-91bd-1de3-270377a51fba fcf8ed41-29dd-91bd-1de3-270377a51fba}" -2025-01-28T14:47:05.904Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:47:05.798934339 +0000 UTC m=+8.882340697 (2 nodes), 62.799532ms from start of poll, 67.630675ms from cluster start. Timeout at 2025-01-28 14:47:05.903956339 +0000 UTC m=+8.987362030, 105.021333ms after stability -2025-01-28T14:47:05.904Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:06.010Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:05.90485315 +0000 UTC m=+8.988258841 (1 nodes), 11µs from start of poll, 173.548819ms from cluster start. Timeout at 2025-01-28 14:47:06.009907151 +0000 UTC m=+9.093313509, 105.054668ms after stability - raft_test.go:378: [INFO] Disconnecting Node at a4321908-d7fe-45f3-5b12-771cfcb77ecf [Leader] -2025-01-28T14:47:06.029Z [ERROR] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: failed to pipeline appendEntries: peer="{Voter server-fcf8ed41-29dd-91bd-1de3-270377a51fba fcf8ed41-29dd-91bd-1de3-270377a51fba}" error="append pipeline closed" -2025-01-28T14:47:06.029Z [ERROR] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: failed to pipeline appendEntries: peer="{Voter server-e86384bf-8c6b-4166-35b1-83d62a5346db e86384bf-8c6b-4166-35b1-83d62a5346db}" error="append pipeline closed" -2025-01-28T14:47:06.029Z [INFO] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: aborting pipeline replication: peer="{Voter server-e86384bf-8c6b-4166-35b1-83d62a5346db e86384bf-8c6b-4166-35b1-83d62a5346db}" -2025-01-28T14:47:06.030Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:06.029Z [INFO] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: aborting pipeline replication: peer="{Voter server-fcf8ed41-29dd-91bd-1de3-270377a51fba fcf8ed41-29dd-91bd-1de3-270377a51fba}" -2025-01-28T14:47:06.030Z [ERROR] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: failed to heartbeat to: peer=fcf8ed41-29dd-91bd-1de3-270377a51fba backoff time=10ms error="failed to connect to peer: fcf8ed41-29dd-91bd-1de3-270377a51fba" -2025-01-28T14:47:06.033Z [ERROR] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: failed to heartbeat to: peer=e86384bf-8c6b-4166-35b1-83d62a5346db backoff time=10ms error="failed to connect to peer: e86384bf-8c6b-4166-35b1-83d62a5346db" -2025-01-28T14:47:06.037Z [ERROR] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: failed to appendEntries to: peer="{Voter server-e86384bf-8c6b-4166-35b1-83d62a5346db e86384bf-8c6b-4166-35b1-83d62a5346db}" error="failed to connect to peer: e86384bf-8c6b-4166-35b1-83d62a5346db" -2025-01-28T14:47:06.041Z [ERROR] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: failed to appendEntries to: peer="{Voter server-fcf8ed41-29dd-91bd-1de3-270377a51fba fcf8ed41-29dd-91bd-1de3-270377a51fba}" error="failed to connect to peer: fcf8ed41-29dd-91bd-1de3-270377a51fba" -2025-01-28T14:47:06.050Z [ERROR] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: failed to heartbeat to: peer=fcf8ed41-29dd-91bd-1de3-270377a51fba backoff time=10ms error="failed to connect to peer: fcf8ed41-29dd-91bd-1de3-270377a51fba" -2025-01-28T14:47:06.053Z [ERROR] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: failed to heartbeat to: peer=e86384bf-8c6b-4166-35b1-83d62a5346db backoff time=10ms error="failed to connect to peer: e86384bf-8c6b-4166-35b1-83d62a5346db" -2025-01-28T14:47:06.055Z [ERROR] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: failed to appendEntries to: peer="{Voter server-e86384bf-8c6b-4166-35b1-83d62a5346db e86384bf-8c6b-4166-35b1-83d62a5346db}" error="failed to connect to peer: e86384bf-8c6b-4166-35b1-83d62a5346db" -2025-01-28T14:47:06.061Z [ERROR] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: failed to appendEntries to: peer="{Voter server-fcf8ed41-29dd-91bd-1de3-270377a51fba fcf8ed41-29dd-91bd-1de3-270377a51fba}" error="failed to connect to peer: fcf8ed41-29dd-91bd-1de3-270377a51fba" -2025-01-28T14:47:06.069Z [ERROR] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: failed to heartbeat to: peer=fcf8ed41-29dd-91bd-1de3-270377a51fba backoff time=10ms error="failed to connect to peer: fcf8ed41-29dd-91bd-1de3-270377a51fba" -2025-01-28T14:47:06.073Z [ERROR] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: failed to heartbeat to: peer=e86384bf-8c6b-4166-35b1-83d62a5346db backoff time=10ms error="failed to connect to peer: e86384bf-8c6b-4166-35b1-83d62a5346db" -2025-01-28T14:47:06.073Z [ERROR] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: failed to appendEntries to: peer="{Voter server-e86384bf-8c6b-4166-35b1-83d62a5346db e86384bf-8c6b-4166-35b1-83d62a5346db}" error="failed to connect to peer: e86384bf-8c6b-4166-35b1-83d62a5346db" -2025-01-28T14:47:06.074Z [WARN] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: failed to contact: server-id=server-e86384bf-8c6b-4166-35b1-83d62a5346db time=50.701172ms -2025-01-28T14:47:06.074Z [WARN] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: failed to contact: server-id=server-fcf8ed41-29dd-91bd-1de3-270377a51fba time=51.33019ms -2025-01-28T14:47:06.075Z [WARN] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: failed to contact quorum of nodes, stepping down -2025-01-28T14:47:06.075Z [INFO] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: entering follower state: follower="Node at a4321908-d7fe-45f3-5b12-771cfcb77ecf [Follower]" leader-address= leader-id= -2025-01-28T14:47:06.080Z [ERROR] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: failed to appendEntries to: peer="{Voter server-fcf8ed41-29dd-91bd-1de3-270377a51fba fcf8ed41-29dd-91bd-1de3-270377a51fba}" error="failed to connect to peer: fcf8ed41-29dd-91bd-1de3-270377a51fba" -2025-01-28T14:47:06.109Z [WARN] server-e86384bf-8c6b-4166-35b1-83d62a5346db: heartbeat timeout reached, starting election: last-leader-addr=a4321908-d7fe-45f3-5b12-771cfcb77ecf last-leader-id=server-a4321908-d7fe-45f3-5b12-771cfcb77ecf -2025-01-28T14:47:06.109Z [INFO] server-e86384bf-8c6b-4166-35b1-83d62a5346db: entering candidate state: node="Node at e86384bf-8c6b-4166-35b1-83d62a5346db [Candidate]" term=3 -2025-01-28T14:47:06.110Z [WARN] server-fcf8ed41-29dd-91bd-1de3-270377a51fba: rejecting vote request since we have a leader: from=e86384bf-8c6b-4166-35b1-83d62a5346db leader=a4321908-d7fe-45f3-5b12-771cfcb77ecf leader-id=server-a4321908-d7fe-45f3-5b12-771cfcb77ecf -2025-01-28T14:47:06.110Z [ERROR] server-e86384bf-8c6b-4166-35b1-83d62a5346db: failed to make requestVote RPC: target="{Voter server-a4321908-d7fe-45f3-5b12-771cfcb77ecf a4321908-d7fe-45f3-5b12-771cfcb77ecf}" error="failed to connect to peer: a4321908-d7fe-45f3-5b12-771cfcb77ecf" term=3 -2025-01-28T14:47:06.142Z [WARN] server-fcf8ed41-29dd-91bd-1de3-270377a51fba: heartbeat timeout reached, starting election: last-leader-addr=a4321908-d7fe-45f3-5b12-771cfcb77ecf last-leader-id=server-a4321908-d7fe-45f3-5b12-771cfcb77ecf -2025-01-28T14:47:06.142Z [INFO] server-fcf8ed41-29dd-91bd-1de3-270377a51fba: entering candidate state: node="Node at fcf8ed41-29dd-91bd-1de3-270377a51fba [Candidate]" term=3 -2025-01-28T14:47:06.142Z [INFO] server-e86384bf-8c6b-4166-35b1-83d62a5346db: duplicate requestVote for same term: term=3 -2025-01-28T14:47:06.142Z [ERROR] server-fcf8ed41-29dd-91bd-1de3-270377a51fba: failed to make requestVote RPC: target="{Voter server-a4321908-d7fe-45f3-5b12-771cfcb77ecf a4321908-d7fe-45f3-5b12-771cfcb77ecf}" error="failed to connect to peer: a4321908-d7fe-45f3-5b12-771cfcb77ecf" term=3 -2025-01-28T14:47:06.166Z [WARN] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:06.166Z [INFO] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: entering candidate state: node="Node at a4321908-d7fe-45f3-5b12-771cfcb77ecf [Candidate]" term=3 -2025-01-28T14:47:06.167Z [ERROR] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: failed to make requestVote RPC: target="{Voter server-fcf8ed41-29dd-91bd-1de3-270377a51fba fcf8ed41-29dd-91bd-1de3-270377a51fba}" error="failed to connect to peer: fcf8ed41-29dd-91bd-1de3-270377a51fba" term=3 -2025-01-28T14:47:06.167Z [ERROR] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: failed to make requestVote RPC: target="{Voter server-e86384bf-8c6b-4166-35b1-83d62a5346db e86384bf-8c6b-4166-35b1-83d62a5346db}" error="failed to connect to peer: e86384bf-8c6b-4166-35b1-83d62a5346db" term=3 -2025-01-28T14:47:06.183Z [WARN] server-e86384bf-8c6b-4166-35b1-83d62a5346db: Election timeout reached, restarting election -2025-01-28T14:47:06.183Z [INFO] server-e86384bf-8c6b-4166-35b1-83d62a5346db: entering candidate state: node="Node at e86384bf-8c6b-4166-35b1-83d62a5346db [Candidate]" term=4 -2025-01-28T14:47:06.183Z [ERROR] server-e86384bf-8c6b-4166-35b1-83d62a5346db: failed to make requestVote RPC: target="{Voter server-a4321908-d7fe-45f3-5b12-771cfcb77ecf a4321908-d7fe-45f3-5b12-771cfcb77ecf}" error="failed to connect to peer: a4321908-d7fe-45f3-5b12-771cfcb77ecf" term=4 -2025-01-28T14:47:06.183Z [INFO] server-fcf8ed41-29dd-91bd-1de3-270377a51fba: entering follower state: follower="Node at fcf8ed41-29dd-91bd-1de3-270377a51fba [Follower]" leader-address= leader-id= -2025-01-28T14:47:06.183Z [INFO] server-e86384bf-8c6b-4166-35b1-83d62a5346db: election won: term=4 tally=2 -2025-01-28T14:47:06.183Z [INFO] server-e86384bf-8c6b-4166-35b1-83d62a5346db: entering leader state: leader="Node at e86384bf-8c6b-4166-35b1-83d62a5346db [Leader]" -2025-01-28T14:47:06.183Z [INFO] server-e86384bf-8c6b-4166-35b1-83d62a5346db: added peer, starting replication: peer=server-a4321908-d7fe-45f3-5b12-771cfcb77ecf -2025-01-28T14:47:06.184Z [INFO] server-e86384bf-8c6b-4166-35b1-83d62a5346db: added peer, starting replication: peer=server-fcf8ed41-29dd-91bd-1de3-270377a51fba -2025-01-28T14:47:06.184Z [ERROR] server-e86384bf-8c6b-4166-35b1-83d62a5346db: failed to appendEntries to: peer="{Voter server-a4321908-d7fe-45f3-5b12-771cfcb77ecf a4321908-d7fe-45f3-5b12-771cfcb77ecf}" error="failed to connect to peer: a4321908-d7fe-45f3-5b12-771cfcb77ecf" -2025-01-28T14:47:06.184Z [INFO] server-e86384bf-8c6b-4166-35b1-83d62a5346db: pipelining replication: peer="{Voter server-fcf8ed41-29dd-91bd-1de3-270377a51fba fcf8ed41-29dd-91bd-1de3-270377a51fba}" -2025-01-28T14:47:06.192Z [ERROR] server-e86384bf-8c6b-4166-35b1-83d62a5346db: failed to heartbeat to: peer=a4321908-d7fe-45f3-5b12-771cfcb77ecf backoff time=10ms error="failed to connect to peer: a4321908-d7fe-45f3-5b12-771cfcb77ecf" -2025-01-28T14:47:06.194Z [ERROR] server-e86384bf-8c6b-4166-35b1-83d62a5346db: failed to appendEntries to: peer="{Voter server-a4321908-d7fe-45f3-5b12-771cfcb77ecf a4321908-d7fe-45f3-5b12-771cfcb77ecf}" error="failed to connect to peer: a4321908-d7fe-45f3-5b12-771cfcb77ecf" -2025-01-28T14:47:06.211Z [ERROR] server-e86384bf-8c6b-4166-35b1-83d62a5346db: failed to heartbeat to: peer=a4321908-d7fe-45f3-5b12-771cfcb77ecf backoff time=10ms error="failed to connect to peer: a4321908-d7fe-45f3-5b12-771cfcb77ecf" -2025-01-28T14:47:06.213Z [ERROR] server-e86384bf-8c6b-4166-35b1-83d62a5346db: failed to appendEntries to: peer="{Voter server-a4321908-d7fe-45f3-5b12-771cfcb77ecf a4321908-d7fe-45f3-5b12-771cfcb77ecf}" error="failed to connect to peer: a4321908-d7fe-45f3-5b12-771cfcb77ecf" -2025-01-28T14:47:06.232Z [ERROR] server-e86384bf-8c6b-4166-35b1-83d62a5346db: failed to heartbeat to: peer=a4321908-d7fe-45f3-5b12-771cfcb77ecf backoff time=10ms error="failed to connect to peer: a4321908-d7fe-45f3-5b12-771cfcb77ecf" -2025-01-28T14:47:06.234Z [WARN] server-e86384bf-8c6b-4166-35b1-83d62a5346db: failed to contact: server-id=server-a4321908-d7fe-45f3-5b12-771cfcb77ecf time=50.907845ms -2025-01-28T14:47:06.241Z [ERROR] server-e86384bf-8c6b-4166-35b1-83d62a5346db: failed to appendEntries to: peer="{Voter server-a4321908-d7fe-45f3-5b12-771cfcb77ecf a4321908-d7fe-45f3-5b12-771cfcb77ecf}" error="failed to connect to peer: a4321908-d7fe-45f3-5b12-771cfcb77ecf" -2025-01-28T14:47:06.245Z [WARN] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: Election timeout reached, restarting election -2025-01-28T14:47:06.245Z [INFO] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: entering candidate state: node="Node at a4321908-d7fe-45f3-5b12-771cfcb77ecf [Candidate]" term=4 -2025-01-28T14:47:06.246Z [ERROR] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: failed to make requestVote RPC: target="{Voter server-e86384bf-8c6b-4166-35b1-83d62a5346db e86384bf-8c6b-4166-35b1-83d62a5346db}" error="failed to connect to peer: e86384bf-8c6b-4166-35b1-83d62a5346db" term=4 -2025-01-28T14:47:06.246Z [ERROR] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: failed to make requestVote RPC: target="{Voter server-fcf8ed41-29dd-91bd-1de3-270377a51fba fcf8ed41-29dd-91bd-1de3-270377a51fba}" error="failed to connect to peer: fcf8ed41-29dd-91bd-1de3-270377a51fba" term=4 -2025-01-28T14:47:06.250Z [ERROR] server-e86384bf-8c6b-4166-35b1-83d62a5346db: failed to heartbeat to: peer=a4321908-d7fe-45f3-5b12-771cfcb77ecf backoff time=20ms error="failed to connect to peer: a4321908-d7fe-45f3-5b12-771cfcb77ecf" -2025-01-28T14:47:06.279Z [ERROR] server-e86384bf-8c6b-4166-35b1-83d62a5346db: failed to heartbeat to: peer=a4321908-d7fe-45f3-5b12-771cfcb77ecf backoff time=25ms error="failed to connect to peer: a4321908-d7fe-45f3-5b12-771cfcb77ecf" -2025-01-28T14:47:06.281Z [WARN] server-e86384bf-8c6b-4166-35b1-83d62a5346db: failed to contact: server-id=server-a4321908-d7fe-45f3-5b12-771cfcb77ecf time=97.76957ms -2025-01-28T14:47:06.289Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:06.183963436 +0000 UTC m=+9.267368461 (1 nodes), 153.731564ms from start of poll, 452.658439ms from cluster start. Timeout at 2025-01-28 14:47:06.28898277 +0000 UTC m=+9.372388461, 105.02ms after stability -2025-01-28T14:47:06.291Z [ERROR] server-e86384bf-8c6b-4166-35b1-83d62a5346db: failed to appendEntries to: peer="{Voter server-a4321908-d7fe-45f3-5b12-771cfcb77ecf a4321908-d7fe-45f3-5b12-771cfcb77ecf}" error="failed to connect to peer: a4321908-d7fe-45f3-5b12-771cfcb77ecf" - raft_test.go:413: [INFO] Reconnecting Node at a4321908-d7fe-45f3-5b12-771cfcb77ecf [Candidate] -2025-01-28T14:47:06.312Z [INFO] server-a4321908-d7fe-45f3-5b12-771cfcb77ecf: entering follower state: follower="Node at a4321908-d7fe-45f3-5b12-771cfcb77ecf [Follower]" leader-address=e86384bf-8c6b-4166-35b1-83d62a5346db leader-id=server-e86384bf-8c6b-4166-35b1-83d62a5346db -2025-01-28T14:47:06.372Z [INFO] server-e86384bf-8c6b-4166-35b1-83d62a5346db: pipelining replication: peer="{Voter server-a4321908-d7fe-45f3-5b12-771cfcb77ecf a4321908-d7fe-45f3-5b12-771cfcb77ecf}" -2025-01-28T14:47:06.378Z [INFO] server-e86384bf-8c6b-4166-35b1-83d62a5346db: aborting pipeline replication: peer="{Voter server-fcf8ed41-29dd-91bd-1de3-270377a51fba fcf8ed41-29dd-91bd-1de3-270377a51fba}" -2025-01-28T14:47:06.378Z [INFO] server-e86384bf-8c6b-4166-35b1-83d62a5346db: aborting pipeline replication: peer="{Voter server-a4321908-d7fe-45f3-5b12-771cfcb77ecf a4321908-d7fe-45f3-5b12-771cfcb77ecf}" ---- PASS: TestRaft_LeaderFail (0.66s) +2025-01-28T14:54:07.545Z [INFO] server-95200575-5c4b-73ff-91a6-38f0a636df08: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-95200575-5c4b-73ff-91a6-38f0a636df08 Address:95200575-5c4b-73ff-91a6-38f0a636df08} {Suffrage:Voter ID:server-ca8f3652-28d9-7f14-6bd4-c7e032de850c Address:ca8f3652-28d9-7f14-6bd4-c7e032de850c} {Suffrage:Voter ID:server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01 Address:c85ee916-e3a7-2f3d-e0a7-266d2eb04f01}]" +2025-01-28T14:54:07.545Z [INFO] server-95200575-5c4b-73ff-91a6-38f0a636df08: entering follower state: follower="Node at 95200575-5c4b-73ff-91a6-38f0a636df08 [Follower]" leader-address= leader-id= +2025-01-28T14:54:07.545Z [INFO] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-95200575-5c4b-73ff-91a6-38f0a636df08 Address:95200575-5c4b-73ff-91a6-38f0a636df08} {Suffrage:Voter ID:server-ca8f3652-28d9-7f14-6bd4-c7e032de850c Address:ca8f3652-28d9-7f14-6bd4-c7e032de850c} {Suffrage:Voter ID:server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01 Address:c85ee916-e3a7-2f3d-e0a7-266d2eb04f01}]" +2025-01-28T14:54:07.546Z [INFO] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: entering follower state: follower="Node at ca8f3652-28d9-7f14-6bd4-c7e032de850c [Follower]" leader-address= leader-id= +2025-01-28T14:54:07.546Z [INFO] server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-95200575-5c4b-73ff-91a6-38f0a636df08 Address:95200575-5c4b-73ff-91a6-38f0a636df08} {Suffrage:Voter ID:server-ca8f3652-28d9-7f14-6bd4-c7e032de850c Address:ca8f3652-28d9-7f14-6bd4-c7e032de850c} {Suffrage:Voter ID:server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01 Address:c85ee916-e3a7-2f3d-e0a7-266d2eb04f01}]" +2025-01-28T14:54:07.546Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:54:07.546Z [INFO] server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01: entering follower state: follower="Node at c85ee916-e3a7-2f3d-e0a7-266d2eb04f01 [Follower]" leader-address= leader-id= +2025-01-28T14:54:07.600Z [WARN] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:07.600Z [INFO] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: entering candidate state: node="Node at ca8f3652-28d9-7f14-6bd4-c7e032de850c [Candidate]" term=2 +2025-01-28T14:54:07.600Z [INFO] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: election won: term=2 tally=2 +2025-01-28T14:54:07.600Z [INFO] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: entering leader state: leader="Node at ca8f3652-28d9-7f14-6bd4-c7e032de850c [Leader]" +2025-01-28T14:54:07.600Z [INFO] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: added peer, starting replication: peer=server-95200575-5c4b-73ff-91a6-38f0a636df08 +2025-01-28T14:54:07.600Z [INFO] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: added peer, starting replication: peer=server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01 +2025-01-28T14:54:07.601Z [INFO] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: pipelining replication: peer="{Voter server-95200575-5c4b-73ff-91a6-38f0a636df08 95200575-5c4b-73ff-91a6-38f0a636df08}" +2025-01-28T14:54:07.601Z [INFO] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: pipelining replication: peer="{Voter server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01 c85ee916-e3a7-2f3d-e0a7-266d2eb04f01}" +2025-01-28T14:54:07.706Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:54:07.601015604 +0000 UTC m=+33.275505584 (2 nodes), 54.614208ms from start of poll, 55.696505ms from cluster start. Timeout at 2025-01-28 14:54:07.706026423 +0000 UTC m=+33.380516403, 105.010819ms after stability +2025-01-28T14:54:07.706Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:07.812Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:07.706403822 +0000 UTC m=+33.380893802 (1 nodes), 2.1µs from start of poll, 161.084723ms from cluster start. Timeout at 2025-01-28 14:54:07.81142778 +0000 UTC m=+33.485917820, 105.024018ms after stability + raft_test.go:378: [INFO] Disconnecting Node at ca8f3652-28d9-7f14-6bd4-c7e032de850c [Leader] +2025-01-28T14:54:07.828Z [ERROR] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: failed to pipeline appendEntries: peer="{Voter server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01 c85ee916-e3a7-2f3d-e0a7-266d2eb04f01}" error="append pipeline closed" +2025-01-28T14:54:07.828Z [INFO] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: aborting pipeline replication: peer="{Voter server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01 c85ee916-e3a7-2f3d-e0a7-266d2eb04f01}" +2025-01-28T14:54:07.829Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:07.830Z [ERROR] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: failed to pipeline appendEntries: peer="{Voter server-95200575-5c4b-73ff-91a6-38f0a636df08 95200575-5c4b-73ff-91a6-38f0a636df08}" error="append pipeline closed" +2025-01-28T14:54:07.830Z [INFO] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: aborting pipeline replication: peer="{Voter server-95200575-5c4b-73ff-91a6-38f0a636df08 95200575-5c4b-73ff-91a6-38f0a636df08}" +2025-01-28T14:54:07.830Z [ERROR] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: failed to heartbeat to: peer=95200575-5c4b-73ff-91a6-38f0a636df08 backoff time=10ms error="failed to connect to peer: 95200575-5c4b-73ff-91a6-38f0a636df08" +2025-01-28T14:54:07.832Z [ERROR] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: failed to heartbeat to: peer=c85ee916-e3a7-2f3d-e0a7-266d2eb04f01 backoff time=10ms error="failed to connect to peer: c85ee916-e3a7-2f3d-e0a7-266d2eb04f01" +2025-01-28T14:54:07.835Z [ERROR] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: failed to appendEntries to: peer="{Voter server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01 c85ee916-e3a7-2f3d-e0a7-266d2eb04f01}" error="failed to connect to peer: c85ee916-e3a7-2f3d-e0a7-266d2eb04f01" +2025-01-28T14:54:07.839Z [ERROR] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: failed to appendEntries to: peer="{Voter server-95200575-5c4b-73ff-91a6-38f0a636df08 95200575-5c4b-73ff-91a6-38f0a636df08}" error="failed to connect to peer: 95200575-5c4b-73ff-91a6-38f0a636df08" +2025-01-28T14:54:07.848Z [ERROR] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: failed to heartbeat to: peer=95200575-5c4b-73ff-91a6-38f0a636df08 backoff time=10ms error="failed to connect to peer: 95200575-5c4b-73ff-91a6-38f0a636df08" +2025-01-28T14:54:07.852Z [ERROR] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: failed to heartbeat to: peer=c85ee916-e3a7-2f3d-e0a7-266d2eb04f01 backoff time=10ms error="failed to connect to peer: c85ee916-e3a7-2f3d-e0a7-266d2eb04f01" +2025-01-28T14:54:07.854Z [ERROR] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: failed to appendEntries to: peer="{Voter server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01 c85ee916-e3a7-2f3d-e0a7-266d2eb04f01}" error="failed to connect to peer: c85ee916-e3a7-2f3d-e0a7-266d2eb04f01" +2025-01-28T14:54:07.858Z [ERROR] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: failed to appendEntries to: peer="{Voter server-95200575-5c4b-73ff-91a6-38f0a636df08 95200575-5c4b-73ff-91a6-38f0a636df08}" error="failed to connect to peer: 95200575-5c4b-73ff-91a6-38f0a636df08" +2025-01-28T14:54:07.868Z [ERROR] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: failed to heartbeat to: peer=95200575-5c4b-73ff-91a6-38f0a636df08 backoff time=10ms error="failed to connect to peer: 95200575-5c4b-73ff-91a6-38f0a636df08" +2025-01-28T14:54:07.871Z [ERROR] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: failed to heartbeat to: peer=c85ee916-e3a7-2f3d-e0a7-266d2eb04f01 backoff time=10ms error="failed to connect to peer: c85ee916-e3a7-2f3d-e0a7-266d2eb04f01" +2025-01-28T14:54:07.872Z [ERROR] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: failed to appendEntries to: peer="{Voter server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01 c85ee916-e3a7-2f3d-e0a7-266d2eb04f01}" error="failed to connect to peer: c85ee916-e3a7-2f3d-e0a7-266d2eb04f01" +2025-01-28T14:54:07.874Z [WARN] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: failed to contact: server-id=server-95200575-5c4b-73ff-91a6-38f0a636df08 time=51.023827ms +2025-01-28T14:54:07.875Z [WARN] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: failed to contact: server-id=server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01 time=50.940775ms +2025-01-28T14:54:07.875Z [WARN] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: failed to contact quorum of nodes, stepping down +2025-01-28T14:54:07.875Z [INFO] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: entering follower state: follower="Node at ca8f3652-28d9-7f14-6bd4-c7e032de850c [Follower]" leader-address= leader-id= +2025-01-28T14:54:07.877Z [ERROR] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: failed to appendEntries to: peer="{Voter server-95200575-5c4b-73ff-91a6-38f0a636df08 95200575-5c4b-73ff-91a6-38f0a636df08}" error="failed to connect to peer: 95200575-5c4b-73ff-91a6-38f0a636df08" +2025-01-28T14:54:07.906Z [WARN] server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01: heartbeat timeout reached, starting election: last-leader-addr=ca8f3652-28d9-7f14-6bd4-c7e032de850c last-leader-id=server-ca8f3652-28d9-7f14-6bd4-c7e032de850c +2025-01-28T14:54:07.906Z [INFO] server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01: entering candidate state: node="Node at c85ee916-e3a7-2f3d-e0a7-266d2eb04f01 [Candidate]" term=3 +2025-01-28T14:54:07.906Z [ERROR] server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01: failed to make requestVote RPC: target="{Voter server-ca8f3652-28d9-7f14-6bd4-c7e032de850c ca8f3652-28d9-7f14-6bd4-c7e032de850c}" error="failed to connect to peer: ca8f3652-28d9-7f14-6bd4-c7e032de850c" term=3 +2025-01-28T14:54:07.906Z [WARN] server-95200575-5c4b-73ff-91a6-38f0a636df08: rejecting vote request since we have a leader: from=c85ee916-e3a7-2f3d-e0a7-266d2eb04f01 leader=ca8f3652-28d9-7f14-6bd4-c7e032de850c leader-id=server-ca8f3652-28d9-7f14-6bd4-c7e032de850c +2025-01-28T14:54:07.937Z [WARN] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:07.937Z [INFO] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: entering candidate state: node="Node at ca8f3652-28d9-7f14-6bd4-c7e032de850c [Candidate]" term=3 +2025-01-28T14:54:07.937Z [ERROR] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: failed to make requestVote RPC: target="{Voter server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01 c85ee916-e3a7-2f3d-e0a7-266d2eb04f01}" error="failed to connect to peer: c85ee916-e3a7-2f3d-e0a7-266d2eb04f01" term=3 +2025-01-28T14:54:07.937Z [ERROR] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: failed to make requestVote RPC: target="{Voter server-95200575-5c4b-73ff-91a6-38f0a636df08 95200575-5c4b-73ff-91a6-38f0a636df08}" error="failed to connect to peer: 95200575-5c4b-73ff-91a6-38f0a636df08" term=3 +2025-01-28T14:54:07.951Z [WARN] server-95200575-5c4b-73ff-91a6-38f0a636df08: heartbeat timeout reached, starting election: last-leader-addr=ca8f3652-28d9-7f14-6bd4-c7e032de850c last-leader-id=server-ca8f3652-28d9-7f14-6bd4-c7e032de850c +2025-01-28T14:54:07.951Z [INFO] server-95200575-5c4b-73ff-91a6-38f0a636df08: entering candidate state: node="Node at 95200575-5c4b-73ff-91a6-38f0a636df08 [Candidate]" term=3 +2025-01-28T14:54:07.951Z [INFO] server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01: duplicate requestVote for same term: term=3 +2025-01-28T14:54:07.951Z [ERROR] server-95200575-5c4b-73ff-91a6-38f0a636df08: failed to make requestVote RPC: target="{Voter server-ca8f3652-28d9-7f14-6bd4-c7e032de850c ca8f3652-28d9-7f14-6bd4-c7e032de850c}" error="failed to connect to peer: ca8f3652-28d9-7f14-6bd4-c7e032de850c" term=3 +2025-01-28T14:54:08.002Z [WARN] server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01: Election timeout reached, restarting election +2025-01-28T14:54:08.003Z [INFO] server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01: entering candidate state: node="Node at c85ee916-e3a7-2f3d-e0a7-266d2eb04f01 [Candidate]" term=4 +2025-01-28T14:54:08.003Z [ERROR] server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01: failed to make requestVote RPC: target="{Voter server-ca8f3652-28d9-7f14-6bd4-c7e032de850c ca8f3652-28d9-7f14-6bd4-c7e032de850c}" error="failed to connect to peer: ca8f3652-28d9-7f14-6bd4-c7e032de850c" term=4 +2025-01-28T14:54:08.003Z [INFO] server-95200575-5c4b-73ff-91a6-38f0a636df08: entering follower state: follower="Node at 95200575-5c4b-73ff-91a6-38f0a636df08 [Follower]" leader-address= leader-id= +2025-01-28T14:54:08.003Z [INFO] server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01: election won: term=4 tally=2 +2025-01-28T14:54:08.003Z [INFO] server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01: entering leader state: leader="Node at c85ee916-e3a7-2f3d-e0a7-266d2eb04f01 [Leader]" +2025-01-28T14:54:08.003Z [INFO] server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01: added peer, starting replication: peer=server-95200575-5c4b-73ff-91a6-38f0a636df08 +2025-01-28T14:54:08.003Z [INFO] server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01: added peer, starting replication: peer=server-ca8f3652-28d9-7f14-6bd4-c7e032de850c +2025-01-28T14:54:08.003Z [ERROR] server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01: failed to appendEntries to: peer="{Voter server-ca8f3652-28d9-7f14-6bd4-c7e032de850c ca8f3652-28d9-7f14-6bd4-c7e032de850c}" error="failed to connect to peer: ca8f3652-28d9-7f14-6bd4-c7e032de850c" +2025-01-28T14:54:08.004Z [INFO] server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01: pipelining replication: peer="{Voter server-95200575-5c4b-73ff-91a6-38f0a636df08 95200575-5c4b-73ff-91a6-38f0a636df08}" +2025-01-28T14:54:08.014Z [ERROR] server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01: failed to heartbeat to: peer=ca8f3652-28d9-7f14-6bd4-c7e032de850c backoff time=10ms error="failed to connect to peer: ca8f3652-28d9-7f14-6bd4-c7e032de850c" +2025-01-28T14:54:08.019Z [WARN] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: Election timeout reached, restarting election +2025-01-28T14:54:08.019Z [INFO] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: entering candidate state: node="Node at ca8f3652-28d9-7f14-6bd4-c7e032de850c [Candidate]" term=4 +2025-01-28T14:54:08.019Z [ERROR] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: failed to make requestVote RPC: target="{Voter server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01 c85ee916-e3a7-2f3d-e0a7-266d2eb04f01}" error="failed to connect to peer: c85ee916-e3a7-2f3d-e0a7-266d2eb04f01" term=4 +2025-01-28T14:54:08.019Z [ERROR] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: failed to make requestVote RPC: target="{Voter server-95200575-5c4b-73ff-91a6-38f0a636df08 95200575-5c4b-73ff-91a6-38f0a636df08}" error="failed to connect to peer: 95200575-5c4b-73ff-91a6-38f0a636df08" term=4 +2025-01-28T14:54:08.020Z [ERROR] server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01: failed to appendEntries to: peer="{Voter server-ca8f3652-28d9-7f14-6bd4-c7e032de850c ca8f3652-28d9-7f14-6bd4-c7e032de850c}" error="failed to connect to peer: ca8f3652-28d9-7f14-6bd4-c7e032de850c" +2025-01-28T14:54:08.033Z [ERROR] server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01: failed to heartbeat to: peer=ca8f3652-28d9-7f14-6bd4-c7e032de850c backoff time=10ms error="failed to connect to peer: ca8f3652-28d9-7f14-6bd4-c7e032de850c" +2025-01-28T14:54:08.038Z [ERROR] server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01: failed to appendEntries to: peer="{Voter server-ca8f3652-28d9-7f14-6bd4-c7e032de850c ca8f3652-28d9-7f14-6bd4-c7e032de850c}" error="failed to connect to peer: ca8f3652-28d9-7f14-6bd4-c7e032de850c" +2025-01-28T14:54:08.053Z [ERROR] server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01: failed to heartbeat to: peer=ca8f3652-28d9-7f14-6bd4-c7e032de850c backoff time=10ms error="failed to connect to peer: ca8f3652-28d9-7f14-6bd4-c7e032de850c" +2025-01-28T14:54:08.054Z [WARN] server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01: failed to contact: server-id=server-ca8f3652-28d9-7f14-6bd4-c7e032de850c time=51.050384ms +2025-01-28T14:54:08.067Z [ERROR] server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01: failed to appendEntries to: peer="{Voter server-ca8f3652-28d9-7f14-6bd4-c7e032de850c ca8f3652-28d9-7f14-6bd4-c7e032de850c}" error="failed to connect to peer: ca8f3652-28d9-7f14-6bd4-c7e032de850c" +2025-01-28T14:54:08.072Z [ERROR] server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01: failed to heartbeat to: peer=ca8f3652-28d9-7f14-6bd4-c7e032de850c backoff time=20ms error="failed to connect to peer: ca8f3652-28d9-7f14-6bd4-c7e032de850c" +2025-01-28T14:54:08.076Z [WARN] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: Election timeout reached, restarting election +2025-01-28T14:54:08.076Z [INFO] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: entering candidate state: node="Node at ca8f3652-28d9-7f14-6bd4-c7e032de850c [Candidate]" term=5 +2025-01-28T14:54:08.076Z [ERROR] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: failed to make requestVote RPC: target="{Voter server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01 c85ee916-e3a7-2f3d-e0a7-266d2eb04f01}" error="failed to connect to peer: c85ee916-e3a7-2f3d-e0a7-266d2eb04f01" term=5 +2025-01-28T14:54:08.076Z [ERROR] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: failed to make requestVote RPC: target="{Voter server-95200575-5c4b-73ff-91a6-38f0a636df08 95200575-5c4b-73ff-91a6-38f0a636df08}" error="failed to connect to peer: 95200575-5c4b-73ff-91a6-38f0a636df08" term=5 +2025-01-28T14:54:08.102Z [ERROR] server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01: failed to heartbeat to: peer=ca8f3652-28d9-7f14-6bd4-c7e032de850c backoff time=25ms error="failed to connect to peer: ca8f3652-28d9-7f14-6bd4-c7e032de850c" +2025-01-28T14:54:08.104Z [WARN] server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01: failed to contact: server-id=server-ca8f3652-28d9-7f14-6bd4-c7e032de850c time=100.732677ms +2025-01-28T14:54:08.109Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:08.00375221 +0000 UTC m=+33.678242210 (1 nodes), 174.278138ms from start of poll, 458.433131ms from cluster start. Timeout at 2025-01-28 14:54:08.10875665 +0000 UTC m=+33.783247230, 105.00502ms after stability + raft_test.go:413: [INFO] Reconnecting Node at ca8f3652-28d9-7f14-6bd4-c7e032de850c [Candidate] +2025-01-28T14:54:08.114Z [ERROR] server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01: peer has newer term, stopping replication: peer="{Voter server-ca8f3652-28d9-7f14-6bd4-c7e032de850c ca8f3652-28d9-7f14-6bd4-c7e032de850c}" +2025-01-28T14:54:08.115Z [INFO] server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01: entering follower state: follower="Node at c85ee916-e3a7-2f3d-e0a7-266d2eb04f01 [Follower]" leader-address= leader-id= +2025-01-28T14:54:08.115Z [INFO] server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01: aborting pipeline replication: peer="{Voter server-95200575-5c4b-73ff-91a6-38f0a636df08 95200575-5c4b-73ff-91a6-38f0a636df08}" +2025-01-28T14:54:08.162Z [WARN] server-95200575-5c4b-73ff-91a6-38f0a636df08: heartbeat timeout reached, starting election: last-leader-addr=c85ee916-e3a7-2f3d-e0a7-266d2eb04f01 last-leader-id=server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01 +2025-01-28T14:54:08.162Z [INFO] server-95200575-5c4b-73ff-91a6-38f0a636df08: entering candidate state: node="Node at 95200575-5c4b-73ff-91a6-38f0a636df08 [Candidate]" term=5 +2025-01-28T14:54:08.162Z [INFO] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: duplicate requestVote for same term: term=5 +2025-01-28T14:54:08.162Z [INFO] server-95200575-5c4b-73ff-91a6-38f0a636df08: election won: term=5 tally=2 +2025-01-28T14:54:08.162Z [INFO] server-95200575-5c4b-73ff-91a6-38f0a636df08: entering leader state: leader="Node at 95200575-5c4b-73ff-91a6-38f0a636df08 [Leader]" +2025-01-28T14:54:08.162Z [INFO] server-95200575-5c4b-73ff-91a6-38f0a636df08: added peer, starting replication: peer=server-ca8f3652-28d9-7f14-6bd4-c7e032de850c +2025-01-28T14:54:08.162Z [INFO] server-95200575-5c4b-73ff-91a6-38f0a636df08: added peer, starting replication: peer=server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01 +2025-01-28T14:54:08.162Z [WARN] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: failed to get previous log: previous-index=5 last-index=3 error="log not found" +2025-01-28T14:54:08.162Z [INFO] server-ca8f3652-28d9-7f14-6bd4-c7e032de850c: entering follower state: follower="Node at ca8f3652-28d9-7f14-6bd4-c7e032de850c [Follower]" leader-address=95200575-5c4b-73ff-91a6-38f0a636df08 leader-id=server-95200575-5c4b-73ff-91a6-38f0a636df08 +2025-01-28T14:54:08.162Z [INFO] server-95200575-5c4b-73ff-91a6-38f0a636df08: pipelining replication: peer="{Voter server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01 c85ee916-e3a7-2f3d-e0a7-266d2eb04f01}" +2025-01-28T14:54:08.162Z [WARN] server-95200575-5c4b-73ff-91a6-38f0a636df08: appendEntries rejected, sending older logs: peer="{Voter server-ca8f3652-28d9-7f14-6bd4-c7e032de850c ca8f3652-28d9-7f14-6bd4-c7e032de850c}" next=4 +2025-01-28T14:54:08.163Z [INFO] server-95200575-5c4b-73ff-91a6-38f0a636df08: pipelining replication: peer="{Voter server-ca8f3652-28d9-7f14-6bd4-c7e032de850c ca8f3652-28d9-7f14-6bd4-c7e032de850c}" +2025-01-28T14:54:08.168Z [INFO] server-95200575-5c4b-73ff-91a6-38f0a636df08: aborting pipeline replication: peer="{Voter server-ca8f3652-28d9-7f14-6bd4-c7e032de850c ca8f3652-28d9-7f14-6bd4-c7e032de850c}" +2025-01-28T14:54:08.168Z [INFO] server-95200575-5c4b-73ff-91a6-38f0a636df08: aborting pipeline replication: peer="{Voter server-c85ee916-e3a7-2f3d-e0a7-266d2eb04f01 c85ee916-e3a7-2f3d-e0a7-266d2eb04f01}" +--- PASS: TestRaft_LeaderFail (0.63s) === RUN TestRaft_BehindFollower -2025-01-28T14:47:06.397Z [INFO] server-27452489-e8d3-7a9f-381e-dd328430c8dc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-27452489-e8d3-7a9f-381e-dd328430c8dc Address:27452489-e8d3-7a9f-381e-dd328430c8dc} {Suffrage:Voter ID:server-84aa2645-417b-90ca-0165-0e53534a0f9b Address:84aa2645-417b-90ca-0165-0e53534a0f9b} {Suffrage:Voter ID:server-8230fc43-42c1-42b0-0f4e-84c48f032580 Address:8230fc43-42c1-42b0-0f4e-84c48f032580}]" -2025-01-28T14:47:06.397Z [INFO] server-27452489-e8d3-7a9f-381e-dd328430c8dc: entering follower state: follower="Node at 27452489-e8d3-7a9f-381e-dd328430c8dc [Follower]" leader-address= leader-id= -2025-01-28T14:47:06.398Z [INFO] server-84aa2645-417b-90ca-0165-0e53534a0f9b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-27452489-e8d3-7a9f-381e-dd328430c8dc Address:27452489-e8d3-7a9f-381e-dd328430c8dc} {Suffrage:Voter ID:server-84aa2645-417b-90ca-0165-0e53534a0f9b Address:84aa2645-417b-90ca-0165-0e53534a0f9b} {Suffrage:Voter ID:server-8230fc43-42c1-42b0-0f4e-84c48f032580 Address:8230fc43-42c1-42b0-0f4e-84c48f032580}]" -2025-01-28T14:47:06.399Z [INFO] server-84aa2645-417b-90ca-0165-0e53534a0f9b: entering follower state: follower="Node at 84aa2645-417b-90ca-0165-0e53534a0f9b [Follower]" leader-address= leader-id= -2025-01-28T14:47:06.400Z [INFO] server-8230fc43-42c1-42b0-0f4e-84c48f032580: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-27452489-e8d3-7a9f-381e-dd328430c8dc Address:27452489-e8d3-7a9f-381e-dd328430c8dc} {Suffrage:Voter ID:server-84aa2645-417b-90ca-0165-0e53534a0f9b Address:84aa2645-417b-90ca-0165-0e53534a0f9b} {Suffrage:Voter ID:server-8230fc43-42c1-42b0-0f4e-84c48f032580 Address:8230fc43-42c1-42b0-0f4e-84c48f032580}]" -2025-01-28T14:47:06.400Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:06.401Z [INFO] server-8230fc43-42c1-42b0-0f4e-84c48f032580: entering follower state: follower="Node at 8230fc43-42c1-42b0-0f4e-84c48f032580 [Follower]" leader-address= leader-id= -2025-01-28T14:47:06.454Z [WARN] server-27452489-e8d3-7a9f-381e-dd328430c8dc: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:06.454Z [INFO] server-27452489-e8d3-7a9f-381e-dd328430c8dc: entering candidate state: node="Node at 27452489-e8d3-7a9f-381e-dd328430c8dc [Candidate]" term=2 -2025-01-28T14:47:06.455Z [INFO] server-27452489-e8d3-7a9f-381e-dd328430c8dc: election won: term=2 tally=2 -2025-01-28T14:47:06.455Z [INFO] server-27452489-e8d3-7a9f-381e-dd328430c8dc: entering leader state: leader="Node at 27452489-e8d3-7a9f-381e-dd328430c8dc [Leader]" -2025-01-28T14:47:06.456Z [INFO] server-27452489-e8d3-7a9f-381e-dd328430c8dc: added peer, starting replication: peer=server-84aa2645-417b-90ca-0165-0e53534a0f9b -2025-01-28T14:47:06.456Z [INFO] server-27452489-e8d3-7a9f-381e-dd328430c8dc: added peer, starting replication: peer=server-8230fc43-42c1-42b0-0f4e-84c48f032580 -2025-01-28T14:47:06.457Z [INFO] server-27452489-e8d3-7a9f-381e-dd328430c8dc: pipelining replication: peer="{Voter server-84aa2645-417b-90ca-0165-0e53534a0f9b 84aa2645-417b-90ca-0165-0e53534a0f9b}" -2025-01-28T14:47:06.457Z [INFO] server-27452489-e8d3-7a9f-381e-dd328430c8dc: pipelining replication: peer="{Voter server-8230fc43-42c1-42b0-0f4e-84c48f032580 8230fc43-42c1-42b0-0f4e-84c48f032580}" -2025-01-28T14:47:06.562Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:06.456151517 +0000 UTC m=+9.539558208 (1 nodes), 55.098969ms from start of poll, 59.97678ms from cluster start. Timeout at 2025-01-28 14:47:06.561190852 +0000 UTC m=+9.644596543, 105.038335ms after stability -2025-01-28T14:47:06.562Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:47:06.668Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:47:06.562911353 +0000 UTC m=+9.646317044 (2 nodes), 13.667µs from start of poll, 166.735616ms from cluster start. Timeout at 2025-01-28 14:47:06.667997355 +0000 UTC m=+9.751403380, 105.086336ms after stability -2025-01-28T14:47:06.669Z [ERROR] server-27452489-e8d3-7a9f-381e-dd328430c8dc: failed to pipeline appendEntries: peer="{Voter server-84aa2645-417b-90ca-0165-0e53534a0f9b 84aa2645-417b-90ca-0165-0e53534a0f9b}" error="append pipeline closed" -2025-01-28T14:47:06.669Z [INFO] server-27452489-e8d3-7a9f-381e-dd328430c8dc: aborting pipeline replication: peer="{Voter server-84aa2645-417b-90ca-0165-0e53534a0f9b 84aa2645-417b-90ca-0165-0e53534a0f9b}" -2025-01-28T14:47:06.670Z [ERROR] server-27452489-e8d3-7a9f-381e-dd328430c8dc: failed to appendEntries to: peer="{Voter server-84aa2645-417b-90ca-0165-0e53534a0f9b 84aa2645-417b-90ca-0165-0e53534a0f9b}" error="failed to connect to peer: 84aa2645-417b-90ca-0165-0e53534a0f9b" -2025-01-28T14:47:06.673Z [ERROR] server-27452489-e8d3-7a9f-381e-dd328430c8dc: failed to heartbeat to: peer=84aa2645-417b-90ca-0165-0e53534a0f9b backoff time=10ms error="failed to connect to peer: 84aa2645-417b-90ca-0165-0e53534a0f9b" -2025-01-28T14:47:06.688Z [ERROR] server-27452489-e8d3-7a9f-381e-dd328430c8dc: failed to appendEntries to: peer="{Voter server-84aa2645-417b-90ca-0165-0e53534a0f9b 84aa2645-417b-90ca-0165-0e53534a0f9b}" error="failed to connect to peer: 84aa2645-417b-90ca-0165-0e53534a0f9b" -2025-01-28T14:47:06.698Z [ERROR] server-27452489-e8d3-7a9f-381e-dd328430c8dc: failed to heartbeat to: peer=84aa2645-417b-90ca-0165-0e53534a0f9b backoff time=10ms error="failed to connect to peer: 84aa2645-417b-90ca-0165-0e53534a0f9b" -2025-01-28T14:47:06.699Z [ERROR] server-27452489-e8d3-7a9f-381e-dd328430c8dc: failed to appendEntries to: peer="{Voter server-84aa2645-417b-90ca-0165-0e53534a0f9b 84aa2645-417b-90ca-0165-0e53534a0f9b}" error="failed to connect to peer: 84aa2645-417b-90ca-0165-0e53534a0f9b" -2025-01-28T14:47:06.716Z [ERROR] server-27452489-e8d3-7a9f-381e-dd328430c8dc: failed to heartbeat to: peer=84aa2645-417b-90ca-0165-0e53534a0f9b backoff time=10ms error="failed to connect to peer: 84aa2645-417b-90ca-0165-0e53534a0f9b" -2025-01-28T14:47:06.718Z [WARN] server-27452489-e8d3-7a9f-381e-dd328430c8dc: failed to contact: server-id=server-84aa2645-417b-90ca-0165-0e53534a0f9b time=50.244492ms -2025-01-28T14:47:06.720Z [ERROR] server-27452489-e8d3-7a9f-381e-dd328430c8dc: failed to appendEntries to: peer="{Voter server-84aa2645-417b-90ca-0165-0e53534a0f9b 84aa2645-417b-90ca-0165-0e53534a0f9b}" error="failed to connect to peer: 84aa2645-417b-90ca-0165-0e53534a0f9b" +2025-01-28T14:54:08.172Z [INFO] server-259c2814-2c0e-8740-7c64-7fcdd0bb018f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-259c2814-2c0e-8740-7c64-7fcdd0bb018f Address:259c2814-2c0e-8740-7c64-7fcdd0bb018f} {Suffrage:Voter ID:server-cb79b771-b354-3c34-6925-3cfa59b3eab5 Address:cb79b771-b354-3c34-6925-3cfa59b3eab5} {Suffrage:Voter ID:server-d9e430cf-1cbc-2b83-e0b7-8e63354295bf Address:d9e430cf-1cbc-2b83-e0b7-8e63354295bf}]" +2025-01-28T14:54:08.172Z [INFO] server-259c2814-2c0e-8740-7c64-7fcdd0bb018f: entering follower state: follower="Node at 259c2814-2c0e-8740-7c64-7fcdd0bb018f [Follower]" leader-address= leader-id= +2025-01-28T14:54:08.172Z [INFO] server-cb79b771-b354-3c34-6925-3cfa59b3eab5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-259c2814-2c0e-8740-7c64-7fcdd0bb018f Address:259c2814-2c0e-8740-7c64-7fcdd0bb018f} {Suffrage:Voter ID:server-cb79b771-b354-3c34-6925-3cfa59b3eab5 Address:cb79b771-b354-3c34-6925-3cfa59b3eab5} {Suffrage:Voter ID:server-d9e430cf-1cbc-2b83-e0b7-8e63354295bf Address:d9e430cf-1cbc-2b83-e0b7-8e63354295bf}]" +2025-01-28T14:54:08.172Z [INFO] server-cb79b771-b354-3c34-6925-3cfa59b3eab5: entering follower state: follower="Node at cb79b771-b354-3c34-6925-3cfa59b3eab5 [Follower]" leader-address= leader-id= +2025-01-28T14:54:08.172Z [INFO] server-d9e430cf-1cbc-2b83-e0b7-8e63354295bf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-259c2814-2c0e-8740-7c64-7fcdd0bb018f Address:259c2814-2c0e-8740-7c64-7fcdd0bb018f} {Suffrage:Voter ID:server-cb79b771-b354-3c34-6925-3cfa59b3eab5 Address:cb79b771-b354-3c34-6925-3cfa59b3eab5} {Suffrage:Voter ID:server-d9e430cf-1cbc-2b83-e0b7-8e63354295bf Address:d9e430cf-1cbc-2b83-e0b7-8e63354295bf}]" +2025-01-28T14:54:08.172Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:08.172Z [INFO] server-d9e430cf-1cbc-2b83-e0b7-8e63354295bf: entering follower state: follower="Node at d9e430cf-1cbc-2b83-e0b7-8e63354295bf [Follower]" leader-address= leader-id= +2025-01-28T14:54:08.227Z [WARN] server-259c2814-2c0e-8740-7c64-7fcdd0bb018f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:08.227Z [INFO] server-259c2814-2c0e-8740-7c64-7fcdd0bb018f: entering candidate state: node="Node at 259c2814-2c0e-8740-7c64-7fcdd0bb018f [Candidate]" term=2 +2025-01-28T14:54:08.227Z [INFO] server-259c2814-2c0e-8740-7c64-7fcdd0bb018f: election won: term=2 tally=2 +2025-01-28T14:54:08.227Z [INFO] server-259c2814-2c0e-8740-7c64-7fcdd0bb018f: entering leader state: leader="Node at 259c2814-2c0e-8740-7c64-7fcdd0bb018f [Leader]" +2025-01-28T14:54:08.227Z [INFO] server-259c2814-2c0e-8740-7c64-7fcdd0bb018f: added peer, starting replication: peer=server-cb79b771-b354-3c34-6925-3cfa59b3eab5 +2025-01-28T14:54:08.227Z [INFO] server-259c2814-2c0e-8740-7c64-7fcdd0bb018f: added peer, starting replication: peer=server-d9e430cf-1cbc-2b83-e0b7-8e63354295bf +2025-01-28T14:54:08.227Z [INFO] server-259c2814-2c0e-8740-7c64-7fcdd0bb018f: pipelining replication: peer="{Voter server-cb79b771-b354-3c34-6925-3cfa59b3eab5 cb79b771-b354-3c34-6925-3cfa59b3eab5}" +2025-01-28T14:54:08.228Z [INFO] server-259c2814-2c0e-8740-7c64-7fcdd0bb018f: pipelining replication: peer="{Voter server-d9e430cf-1cbc-2b83-e0b7-8e63354295bf d9e430cf-1cbc-2b83-e0b7-8e63354295bf}" +2025-01-28T14:54:08.333Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:08.227765297 +0000 UTC m=+33.902255397 (1 nodes), 54.840166ms from start of poll, 55.856351ms from cluster start. Timeout at 2025-01-28 14:54:08.332770216 +0000 UTC m=+34.007260276, 105.004879ms after stability +2025-01-28T14:54:08.334Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:54:08.439Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:54:08.334122683 +0000 UTC m=+34.008612663 (2 nodes), 2.8µs from start of poll, 162.213617ms from cluster start. Timeout at 2025-01-28 14:54:08.439142161 +0000 UTC m=+34.113632261, 105.019598ms after stability +2025-01-28T14:54:08.439Z [ERROR] server-259c2814-2c0e-8740-7c64-7fcdd0bb018f: failed to pipeline appendEntries: peer="{Voter server-cb79b771-b354-3c34-6925-3cfa59b3eab5 cb79b771-b354-3c34-6925-3cfa59b3eab5}" error="append pipeline closed" +2025-01-28T14:54:08.439Z [INFO] server-259c2814-2c0e-8740-7c64-7fcdd0bb018f: aborting pipeline replication: peer="{Voter server-cb79b771-b354-3c34-6925-3cfa59b3eab5 cb79b771-b354-3c34-6925-3cfa59b3eab5}" +2025-01-28T14:54:08.439Z [ERROR] server-259c2814-2c0e-8740-7c64-7fcdd0bb018f: failed to appendEntries to: peer="{Voter server-cb79b771-b354-3c34-6925-3cfa59b3eab5 cb79b771-b354-3c34-6925-3cfa59b3eab5}" error="failed to connect to peer: cb79b771-b354-3c34-6925-3cfa59b3eab5" +2025-01-28T14:54:08.444Z [ERROR] server-259c2814-2c0e-8740-7c64-7fcdd0bb018f: failed to heartbeat to: peer=cb79b771-b354-3c34-6925-3cfa59b3eab5 backoff time=10ms error="failed to connect to peer: cb79b771-b354-3c34-6925-3cfa59b3eab5" raft_test.go:459: [INFO] Finished apply without behind follower -2025-01-28T14:47:06.764Z [INFO] server-27452489-e8d3-7a9f-381e-dd328430c8dc: pipelining replication: peer="{Voter server-84aa2645-417b-90ca-0165-0e53534a0f9b 84aa2645-417b-90ca-0165-0e53534a0f9b}" -2025-01-28T14:47:06.773Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:06.880Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:06.774070955 +0000 UTC m=+9.857476647 (1 nodes), 12.334µs from start of poll, 377.895219ms from cluster start. Timeout at 2025-01-28 14:47:06.879167291 +0000 UTC m=+9.962572982, 105.096335ms after stability -2025-01-28T14:47:06.880Z [INFO] server-27452489-e8d3-7a9f-381e-dd328430c8dc: aborting pipeline replication: peer="{Voter server-8230fc43-42c1-42b0-0f4e-84c48f032580 8230fc43-42c1-42b0-0f4e-84c48f032580}" -2025-01-28T14:47:06.881Z [INFO] server-27452489-e8d3-7a9f-381e-dd328430c8dc: aborting pipeline replication: peer="{Voter server-84aa2645-417b-90ca-0165-0e53534a0f9b 84aa2645-417b-90ca-0165-0e53534a0f9b}" -2025-01-28T14:47:07.381Z [ERROR] server-27452489-e8d3-7a9f-381e-dd328430c8dc: failed to heartbeat to: peer=84aa2645-417b-90ca-0165-0e53534a0f9b backoff time=10ms error="command timed out" ---- PASS: TestRaft_BehindFollower (1.00s) +2025-01-28T14:54:08.450Z [INFO] server-259c2814-2c0e-8740-7c64-7fcdd0bb018f: pipelining replication: peer="{Voter server-cb79b771-b354-3c34-6925-3cfa59b3eab5 cb79b771-b354-3c34-6925-3cfa59b3eab5}" +2025-01-28T14:54:08.463Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:08.568Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:08.463628286 +0000 UTC m=+34.138118266 (1 nodes), 2.36µs from start of poll, 291.71922ms from cluster start. Timeout at 2025-01-28 14:54:08.568663243 +0000 UTC m=+34.243153283, 105.035017ms after stability +2025-01-28T14:54:08.569Z [INFO] server-259c2814-2c0e-8740-7c64-7fcdd0bb018f: aborting pipeline replication: peer="{Voter server-cb79b771-b354-3c34-6925-3cfa59b3eab5 cb79b771-b354-3c34-6925-3cfa59b3eab5}" +2025-01-28T14:54:08.569Z [INFO] server-259c2814-2c0e-8740-7c64-7fcdd0bb018f: aborting pipeline replication: peer="{Voter server-d9e430cf-1cbc-2b83-e0b7-8e63354295bf d9e430cf-1cbc-2b83-e0b7-8e63354295bf}" +--- PASS: TestRaft_BehindFollower (0.40s) === RUN TestRaft_ApplyNonLeader -2025-01-28T14:47:07.400Z [INFO] server-70acce05-c77b-6d47-99c8-d16b2b1b214b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-70acce05-c77b-6d47-99c8-d16b2b1b214b Address:70acce05-c77b-6d47-99c8-d16b2b1b214b} {Suffrage:Voter ID:server-dfa1fdab-b4a7-68ad-6a37-aaac9a70bb79 Address:dfa1fdab-b4a7-68ad-6a37-aaac9a70bb79} {Suffrage:Voter ID:server-c1998568-e227-8e8f-dd1d-64639753dbd5 Address:c1998568-e227-8e8f-dd1d-64639753dbd5}]" -2025-01-28T14:47:07.401Z [INFO] server-70acce05-c77b-6d47-99c8-d16b2b1b214b: entering follower state: follower="Node at 70acce05-c77b-6d47-99c8-d16b2b1b214b [Follower]" leader-address= leader-id= -2025-01-28T14:47:07.402Z [INFO] server-dfa1fdab-b4a7-68ad-6a37-aaac9a70bb79: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-70acce05-c77b-6d47-99c8-d16b2b1b214b Address:70acce05-c77b-6d47-99c8-d16b2b1b214b} {Suffrage:Voter ID:server-dfa1fdab-b4a7-68ad-6a37-aaac9a70bb79 Address:dfa1fdab-b4a7-68ad-6a37-aaac9a70bb79} {Suffrage:Voter ID:server-c1998568-e227-8e8f-dd1d-64639753dbd5 Address:c1998568-e227-8e8f-dd1d-64639753dbd5}]" -2025-01-28T14:47:07.402Z [INFO] server-dfa1fdab-b4a7-68ad-6a37-aaac9a70bb79: entering follower state: follower="Node at dfa1fdab-b4a7-68ad-6a37-aaac9a70bb79 [Follower]" leader-address= leader-id= -2025-01-28T14:47:07.403Z [INFO] server-c1998568-e227-8e8f-dd1d-64639753dbd5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-70acce05-c77b-6d47-99c8-d16b2b1b214b Address:70acce05-c77b-6d47-99c8-d16b2b1b214b} {Suffrage:Voter ID:server-dfa1fdab-b4a7-68ad-6a37-aaac9a70bb79 Address:dfa1fdab-b4a7-68ad-6a37-aaac9a70bb79} {Suffrage:Voter ID:server-c1998568-e227-8e8f-dd1d-64639753dbd5 Address:c1998568-e227-8e8f-dd1d-64639753dbd5}]" -2025-01-28T14:47:07.404Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:07.404Z [INFO] server-c1998568-e227-8e8f-dd1d-64639753dbd5: entering follower state: follower="Node at c1998568-e227-8e8f-dd1d-64639753dbd5 [Follower]" leader-address= leader-id= -2025-01-28T14:47:07.478Z [WARN] server-70acce05-c77b-6d47-99c8-d16b2b1b214b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:07.478Z [INFO] server-70acce05-c77b-6d47-99c8-d16b2b1b214b: entering candidate state: node="Node at 70acce05-c77b-6d47-99c8-d16b2b1b214b [Candidate]" term=2 -2025-01-28T14:47:07.479Z [INFO] server-70acce05-c77b-6d47-99c8-d16b2b1b214b: election won: term=2 tally=2 -2025-01-28T14:47:07.479Z [INFO] server-70acce05-c77b-6d47-99c8-d16b2b1b214b: entering leader state: leader="Node at 70acce05-c77b-6d47-99c8-d16b2b1b214b [Leader]" -2025-01-28T14:47:07.480Z [INFO] server-70acce05-c77b-6d47-99c8-d16b2b1b214b: added peer, starting replication: peer=server-dfa1fdab-b4a7-68ad-6a37-aaac9a70bb79 -2025-01-28T14:47:07.480Z [INFO] server-70acce05-c77b-6d47-99c8-d16b2b1b214b: added peer, starting replication: peer=server-c1998568-e227-8e8f-dd1d-64639753dbd5 -2025-01-28T14:47:07.481Z [INFO] server-70acce05-c77b-6d47-99c8-d16b2b1b214b: pipelining replication: peer="{Voter server-dfa1fdab-b4a7-68ad-6a37-aaac9a70bb79 dfa1fdab-b4a7-68ad-6a37-aaac9a70bb79}" -2025-01-28T14:47:07.482Z [INFO] server-70acce05-c77b-6d47-99c8-d16b2b1b214b: pipelining replication: peer="{Voter server-c1998568-e227-8e8f-dd1d-64639753dbd5 c1998568-e227-8e8f-dd1d-64639753dbd5}" -2025-01-28T14:47:07.586Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:07.479986246 +0000 UTC m=+10.563392604 (1 nodes), 75.636246ms from start of poll, 80.539058ms from cluster start. Timeout at 2025-01-28 14:47:07.585026581 +0000 UTC m=+10.668432605, 105.040001ms after stability -2025-01-28T14:47:07.586Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:47:07.692Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:47:07.586637412 +0000 UTC m=+10.670043103 (2 nodes), 14.667µs from start of poll, 187.189557ms from cluster start. Timeout at 2025-01-28 14:47:07.691714748 +0000 UTC m=+10.775120772, 105.077669ms after stability -2025-01-28T14:47:07.692Z [INFO] server-70acce05-c77b-6d47-99c8-d16b2b1b214b: aborting pipeline replication: peer="{Voter server-c1998568-e227-8e8f-dd1d-64639753dbd5 c1998568-e227-8e8f-dd1d-64639753dbd5}" -2025-01-28T14:47:07.692Z [INFO] server-70acce05-c77b-6d47-99c8-d16b2b1b214b: aborting pipeline replication: peer="{Voter server-dfa1fdab-b4a7-68ad-6a37-aaac9a70bb79 dfa1fdab-b4a7-68ad-6a37-aaac9a70bb79}" ---- PASS: TestRaft_ApplyNonLeader (0.31s) +2025-01-28T14:54:08.573Z [INFO] server-97b26d48-6271-3c33-0bf4-f49dd176cbcf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-97b26d48-6271-3c33-0bf4-f49dd176cbcf Address:97b26d48-6271-3c33-0bf4-f49dd176cbcf} {Suffrage:Voter ID:server-a0ca82cd-07d1-ff92-338c-434d892ac522 Address:a0ca82cd-07d1-ff92-338c-434d892ac522} {Suffrage:Voter ID:server-6ae0868c-4dbc-e09d-2375-337216515fe1 Address:6ae0868c-4dbc-e09d-2375-337216515fe1}]" +2025-01-28T14:54:08.573Z [INFO] server-97b26d48-6271-3c33-0bf4-f49dd176cbcf: entering follower state: follower="Node at 97b26d48-6271-3c33-0bf4-f49dd176cbcf [Follower]" leader-address= leader-id= +2025-01-28T14:54:08.573Z [INFO] server-a0ca82cd-07d1-ff92-338c-434d892ac522: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-97b26d48-6271-3c33-0bf4-f49dd176cbcf Address:97b26d48-6271-3c33-0bf4-f49dd176cbcf} {Suffrage:Voter ID:server-a0ca82cd-07d1-ff92-338c-434d892ac522 Address:a0ca82cd-07d1-ff92-338c-434d892ac522} {Suffrage:Voter ID:server-6ae0868c-4dbc-e09d-2375-337216515fe1 Address:6ae0868c-4dbc-e09d-2375-337216515fe1}]" +2025-01-28T14:54:08.573Z [INFO] server-a0ca82cd-07d1-ff92-338c-434d892ac522: entering follower state: follower="Node at a0ca82cd-07d1-ff92-338c-434d892ac522 [Follower]" leader-address= leader-id= +2025-01-28T14:54:08.573Z [INFO] server-6ae0868c-4dbc-e09d-2375-337216515fe1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-97b26d48-6271-3c33-0bf4-f49dd176cbcf Address:97b26d48-6271-3c33-0bf4-f49dd176cbcf} {Suffrage:Voter ID:server-a0ca82cd-07d1-ff92-338c-434d892ac522 Address:a0ca82cd-07d1-ff92-338c-434d892ac522} {Suffrage:Voter ID:server-6ae0868c-4dbc-e09d-2375-337216515fe1 Address:6ae0868c-4dbc-e09d-2375-337216515fe1}]" +2025-01-28T14:54:08.574Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:08.574Z [INFO] server-6ae0868c-4dbc-e09d-2375-337216515fe1: entering follower state: follower="Node at 6ae0868c-4dbc-e09d-2375-337216515fe1 [Follower]" leader-address= leader-id= +2025-01-28T14:54:08.628Z [WARN] server-6ae0868c-4dbc-e09d-2375-337216515fe1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:08.628Z [INFO] server-6ae0868c-4dbc-e09d-2375-337216515fe1: entering candidate state: node="Node at 6ae0868c-4dbc-e09d-2375-337216515fe1 [Candidate]" term=2 +2025-01-28T14:54:08.628Z [WARN] server-a0ca82cd-07d1-ff92-338c-434d892ac522: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:08.629Z [INFO] server-a0ca82cd-07d1-ff92-338c-434d892ac522: entering candidate state: node="Node at a0ca82cd-07d1-ff92-338c-434d892ac522 [Candidate]" term=2 +2025-01-28T14:54:08.629Z [INFO] server-6ae0868c-4dbc-e09d-2375-337216515fe1: election won: term=2 tally=2 +2025-01-28T14:54:08.629Z [INFO] server-a0ca82cd-07d1-ff92-338c-434d892ac522: duplicate requestVote for same term: term=2 +2025-01-28T14:54:08.629Z [INFO] server-6ae0868c-4dbc-e09d-2375-337216515fe1: entering leader state: leader="Node at 6ae0868c-4dbc-e09d-2375-337216515fe1 [Leader]" +2025-01-28T14:54:08.629Z [INFO] server-6ae0868c-4dbc-e09d-2375-337216515fe1: added peer, starting replication: peer=server-97b26d48-6271-3c33-0bf4-f49dd176cbcf +2025-01-28T14:54:08.629Z [INFO] server-97b26d48-6271-3c33-0bf4-f49dd176cbcf: duplicate requestVote for same term: term=2 +2025-01-28T14:54:08.629Z [INFO] server-6ae0868c-4dbc-e09d-2375-337216515fe1: added peer, starting replication: peer=server-a0ca82cd-07d1-ff92-338c-434d892ac522 +2025-01-28T14:54:08.629Z [WARN] server-6ae0868c-4dbc-e09d-2375-337216515fe1: rejecting vote request since we have a leader: from=a0ca82cd-07d1-ff92-338c-434d892ac522 leader=6ae0868c-4dbc-e09d-2375-337216515fe1 leader-id=server-6ae0868c-4dbc-e09d-2375-337216515fe1 +2025-01-28T14:54:08.629Z [INFO] server-a0ca82cd-07d1-ff92-338c-434d892ac522: entering follower state: follower="Node at a0ca82cd-07d1-ff92-338c-434d892ac522 [Follower]" leader-address=6ae0868c-4dbc-e09d-2375-337216515fe1 leader-id=server-6ae0868c-4dbc-e09d-2375-337216515fe1 +2025-01-28T14:54:08.629Z [INFO] server-6ae0868c-4dbc-e09d-2375-337216515fe1: pipelining replication: peer="{Voter server-97b26d48-6271-3c33-0bf4-f49dd176cbcf 97b26d48-6271-3c33-0bf4-f49dd176cbcf}" +2025-01-28T14:54:08.629Z [INFO] server-6ae0868c-4dbc-e09d-2375-337216515fe1: pipelining replication: peer="{Voter server-a0ca82cd-07d1-ff92-338c-434d892ac522 a0ca82cd-07d1-ff92-338c-434d892ac522}" +2025-01-28T14:54:08.734Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:08.62947374 +0000 UTC m=+34.303963720 (1 nodes), 55.43329ms from start of poll, 56.49447ms from cluster start. Timeout at 2025-01-28 14:54:08.734477499 +0000 UTC m=+34.408967619, 105.003899ms after stability +2025-01-28T14:54:08.734Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:54:08.840Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:54:08.734823341 +0000 UTC m=+34.409313321 (2 nodes), 2.679µs from start of poll, 161.844071ms from cluster start. Timeout at 2025-01-28 14:54:08.839858918 +0000 UTC m=+34.514348978, 105.035657ms after stability +2025-01-28T14:54:08.840Z [INFO] server-6ae0868c-4dbc-e09d-2375-337216515fe1: aborting pipeline replication: peer="{Voter server-a0ca82cd-07d1-ff92-338c-434d892ac522 a0ca82cd-07d1-ff92-338c-434d892ac522}" +2025-01-28T14:54:08.840Z [INFO] server-6ae0868c-4dbc-e09d-2375-337216515fe1: aborting pipeline replication: peer="{Voter server-97b26d48-6271-3c33-0bf4-f49dd176cbcf 97b26d48-6271-3c33-0bf4-f49dd176cbcf}" +2025-01-28T14:54:09.341Z [ERROR] server-6ae0868c-4dbc-e09d-2375-337216515fe1: failed to heartbeat to: peer=a0ca82cd-07d1-ff92-338c-434d892ac522 backoff time=10ms error="command timed out" +--- PASS: TestRaft_ApplyNonLeader (0.77s) === RUN TestRaft_ApplyConcurrent -2025-01-28T14:47:07.711Z [INFO] server-e776d1bb-0d70-30d8-28fb-dd78713fadcd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e776d1bb-0d70-30d8-28fb-dd78713fadcd Address:e776d1bb-0d70-30d8-28fb-dd78713fadcd} {Suffrage:Voter ID:server-e035fc49-bd0e-2ae8-910b-ad3a5fc9b747 Address:e035fc49-bd0e-2ae8-910b-ad3a5fc9b747} {Suffrage:Voter ID:server-b18dffe5-c0fa-955e-2a3a-d5368a14682e Address:b18dffe5-c0fa-955e-2a3a-d5368a14682e}]" -2025-01-28T14:47:07.712Z [INFO] server-e776d1bb-0d70-30d8-28fb-dd78713fadcd: entering follower state: follower="Node at e776d1bb-0d70-30d8-28fb-dd78713fadcd [Follower]" leader-address= leader-id= -2025-01-28T14:47:07.713Z [INFO] server-e035fc49-bd0e-2ae8-910b-ad3a5fc9b747: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e776d1bb-0d70-30d8-28fb-dd78713fadcd Address:e776d1bb-0d70-30d8-28fb-dd78713fadcd} {Suffrage:Voter ID:server-e035fc49-bd0e-2ae8-910b-ad3a5fc9b747 Address:e035fc49-bd0e-2ae8-910b-ad3a5fc9b747} {Suffrage:Voter ID:server-b18dffe5-c0fa-955e-2a3a-d5368a14682e Address:b18dffe5-c0fa-955e-2a3a-d5368a14682e}]" -2025-01-28T14:47:07.713Z [INFO] server-e035fc49-bd0e-2ae8-910b-ad3a5fc9b747: entering follower state: follower="Node at e035fc49-bd0e-2ae8-910b-ad3a5fc9b747 [Follower]" leader-address= leader-id= -2025-01-28T14:47:07.714Z [INFO] server-b18dffe5-c0fa-955e-2a3a-d5368a14682e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e776d1bb-0d70-30d8-28fb-dd78713fadcd Address:e776d1bb-0d70-30d8-28fb-dd78713fadcd} {Suffrage:Voter ID:server-e035fc49-bd0e-2ae8-910b-ad3a5fc9b747 Address:e035fc49-bd0e-2ae8-910b-ad3a5fc9b747} {Suffrage:Voter ID:server-b18dffe5-c0fa-955e-2a3a-d5368a14682e Address:b18dffe5-c0fa-955e-2a3a-d5368a14682e}]" -2025-01-28T14:47:07.715Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:07.715Z [INFO] server-b18dffe5-c0fa-955e-2a3a-d5368a14682e: entering follower state: follower="Node at b18dffe5-c0fa-955e-2a3a-d5368a14682e [Follower]" leader-address= leader-id= -2025-01-28T14:47:07.826Z [WARN] server-b18dffe5-c0fa-955e-2a3a-d5368a14682e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:07.827Z [INFO] server-b18dffe5-c0fa-955e-2a3a-d5368a14682e: entering candidate state: node="Node at b18dffe5-c0fa-955e-2a3a-d5368a14682e [Candidate]" term=2 -2025-01-28T14:47:07.828Z [INFO] server-b18dffe5-c0fa-955e-2a3a-d5368a14682e: election won: term=2 tally=2 -2025-01-28T14:47:07.828Z [INFO] server-b18dffe5-c0fa-955e-2a3a-d5368a14682e: entering leader state: leader="Node at b18dffe5-c0fa-955e-2a3a-d5368a14682e [Leader]" -2025-01-28T14:47:07.828Z [INFO] server-b18dffe5-c0fa-955e-2a3a-d5368a14682e: added peer, starting replication: peer=server-e776d1bb-0d70-30d8-28fb-dd78713fadcd -2025-01-28T14:47:07.829Z [INFO] server-b18dffe5-c0fa-955e-2a3a-d5368a14682e: added peer, starting replication: peer=server-e035fc49-bd0e-2ae8-910b-ad3a5fc9b747 -2025-01-28T14:47:07.830Z [INFO] server-b18dffe5-c0fa-955e-2a3a-d5368a14682e: pipelining replication: peer="{Voter server-e776d1bb-0d70-30d8-28fb-dd78713fadcd e776d1bb-0d70-30d8-28fb-dd78713fadcd}" -2025-01-28T14:47:07.830Z [INFO] server-b18dffe5-c0fa-955e-2a3a-d5368a14682e: pipelining replication: peer="{Voter server-e035fc49-bd0e-2ae8-910b-ad3a5fc9b747 e035fc49-bd0e-2ae8-910b-ad3a5fc9b747}" -2025-01-28T14:47:08.034Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:07.828632263 +0000 UTC m=+10.912038621 (1 nodes), 113.279363ms from start of poll, 117.931501ms from cluster start. Timeout at 2025-01-28 14:47:08.033671931 +0000 UTC m=+11.117077955, 205.039334ms after stability -2025-01-28T14:47:08.068Z [INFO] server-b18dffe5-c0fa-955e-2a3a-d5368a14682e: aborting pipeline replication: peer="{Voter server-e035fc49-bd0e-2ae8-910b-ad3a5fc9b747 e035fc49-bd0e-2ae8-910b-ad3a5fc9b747}" -2025-01-28T14:47:08.068Z [INFO] server-b18dffe5-c0fa-955e-2a3a-d5368a14682e: aborting pipeline replication: peer="{Voter server-e776d1bb-0d70-30d8-28fb-dd78713fadcd e776d1bb-0d70-30d8-28fb-dd78713fadcd}" ---- PASS: TestRaft_ApplyConcurrent (0.38s) +2025-01-28T14:54:09.345Z [INFO] server-6e008763-cf5d-9965-aeef-6013999d31e9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6e008763-cf5d-9965-aeef-6013999d31e9 Address:6e008763-cf5d-9965-aeef-6013999d31e9} {Suffrage:Voter ID:server-f33119c5-e798-2b25-c714-1cf09faf7f84 Address:f33119c5-e798-2b25-c714-1cf09faf7f84} {Suffrage:Voter ID:server-d4deb887-edd7-f23c-c0b7-fdf1e8164b5b Address:d4deb887-edd7-f23c-c0b7-fdf1e8164b5b}]" +2025-01-28T14:54:09.345Z [INFO] server-6e008763-cf5d-9965-aeef-6013999d31e9: entering follower state: follower="Node at 6e008763-cf5d-9965-aeef-6013999d31e9 [Follower]" leader-address= leader-id= +2025-01-28T14:54:09.345Z [INFO] server-f33119c5-e798-2b25-c714-1cf09faf7f84: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6e008763-cf5d-9965-aeef-6013999d31e9 Address:6e008763-cf5d-9965-aeef-6013999d31e9} {Suffrage:Voter ID:server-f33119c5-e798-2b25-c714-1cf09faf7f84 Address:f33119c5-e798-2b25-c714-1cf09faf7f84} {Suffrage:Voter ID:server-d4deb887-edd7-f23c-c0b7-fdf1e8164b5b Address:d4deb887-edd7-f23c-c0b7-fdf1e8164b5b}]" +2025-01-28T14:54:09.345Z [INFO] server-f33119c5-e798-2b25-c714-1cf09faf7f84: entering follower state: follower="Node at f33119c5-e798-2b25-c714-1cf09faf7f84 [Follower]" leader-address= leader-id= +2025-01-28T14:54:09.346Z [INFO] server-d4deb887-edd7-f23c-c0b7-fdf1e8164b5b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6e008763-cf5d-9965-aeef-6013999d31e9 Address:6e008763-cf5d-9965-aeef-6013999d31e9} {Suffrage:Voter ID:server-f33119c5-e798-2b25-c714-1cf09faf7f84 Address:f33119c5-e798-2b25-c714-1cf09faf7f84} {Suffrage:Voter ID:server-d4deb887-edd7-f23c-c0b7-fdf1e8164b5b Address:d4deb887-edd7-f23c-c0b7-fdf1e8164b5b}]" +2025-01-28T14:54:09.346Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:09.346Z [INFO] server-d4deb887-edd7-f23c-c0b7-fdf1e8164b5b: entering follower state: follower="Node at d4deb887-edd7-f23c-c0b7-fdf1e8164b5b [Follower]" leader-address= leader-id= +2025-01-28T14:54:09.476Z [WARN] server-6e008763-cf5d-9965-aeef-6013999d31e9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:09.476Z [INFO] server-6e008763-cf5d-9965-aeef-6013999d31e9: entering candidate state: node="Node at 6e008763-cf5d-9965-aeef-6013999d31e9 [Candidate]" term=2 +2025-01-28T14:54:09.476Z [INFO] server-6e008763-cf5d-9965-aeef-6013999d31e9: election won: term=2 tally=2 +2025-01-28T14:54:09.476Z [INFO] server-6e008763-cf5d-9965-aeef-6013999d31e9: entering leader state: leader="Node at 6e008763-cf5d-9965-aeef-6013999d31e9 [Leader]" +2025-01-28T14:54:09.476Z [INFO] server-6e008763-cf5d-9965-aeef-6013999d31e9: added peer, starting replication: peer=server-f33119c5-e798-2b25-c714-1cf09faf7f84 +2025-01-28T14:54:09.476Z [INFO] server-6e008763-cf5d-9965-aeef-6013999d31e9: added peer, starting replication: peer=server-d4deb887-edd7-f23c-c0b7-fdf1e8164b5b +2025-01-28T14:54:09.477Z [INFO] server-6e008763-cf5d-9965-aeef-6013999d31e9: pipelining replication: peer="{Voter server-d4deb887-edd7-f23c-c0b7-fdf1e8164b5b d4deb887-edd7-f23c-c0b7-fdf1e8164b5b}" +2025-01-28T14:54:09.477Z [INFO] server-6e008763-cf5d-9965-aeef-6013999d31e9: pipelining replication: peer="{Voter server-f33119c5-e798-2b25-c714-1cf09faf7f84 f33119c5-e798-2b25-c714-1cf09faf7f84}" +2025-01-28T14:54:09.682Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:09.476873077 +0000 UTC m=+35.151363057 (1 nodes), 130.578562ms from start of poll, 131.628403ms from cluster start. Timeout at 2025-01-28 14:54:09.681879916 +0000 UTC m=+35.356369956, 205.006899ms after stability +2025-01-28T14:54:09.697Z [INFO] server-6e008763-cf5d-9965-aeef-6013999d31e9: aborting pipeline replication: peer="{Voter server-d4deb887-edd7-f23c-c0b7-fdf1e8164b5b d4deb887-edd7-f23c-c0b7-fdf1e8164b5b}" +2025-01-28T14:54:09.697Z [INFO] server-6e008763-cf5d-9965-aeef-6013999d31e9: aborting pipeline replication: peer="{Voter server-f33119c5-e798-2b25-c714-1cf09faf7f84 f33119c5-e798-2b25-c714-1cf09faf7f84}" +--- PASS: TestRaft_ApplyConcurrent (0.36s) === RUN TestRaft_ApplyConcurrent_Timeout -2025-01-28T14:47:08.089Z [INFO] server-3a03d5bc-14c7-bd88-eb45-6c53a2c0baa2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3a03d5bc-14c7-bd88-eb45-6c53a2c0baa2 Address:3a03d5bc-14c7-bd88-eb45-6c53a2c0baa2}]" -2025-01-28T14:47:08.090Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:08.090Z [INFO] server-3a03d5bc-14c7-bd88-eb45-6c53a2c0baa2: entering follower state: follower="Node at 3a03d5bc-14c7-bd88-eb45-6c53a2c0baa2 [Follower]" leader-address= leader-id= -2025-01-28T14:47:08.277Z [WARN] server-3a03d5bc-14c7-bd88-eb45-6c53a2c0baa2: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:08.278Z [INFO] server-3a03d5bc-14c7-bd88-eb45-6c53a2c0baa2: entering candidate state: node="Node at 3a03d5bc-14c7-bd88-eb45-6c53a2c0baa2 [Candidate]" term=2 -2025-01-28T14:47:08.278Z [INFO] server-3a03d5bc-14c7-bd88-eb45-6c53a2c0baa2: election won: term=2 tally=1 -2025-01-28T14:47:08.278Z [INFO] server-3a03d5bc-14c7-bd88-eb45-6c53a2c0baa2: entering leader state: leader="Node at 3a03d5bc-14c7-bd88-eb45-6c53a2c0baa2 [Leader]" -2025-01-28T14:47:08.481Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:08.279074636 +0000 UTC m=+11.362480661 (1 nodes), 188.431594ms from start of poll, 190.327651ms from cluster start. Timeout at 2025-01-28 14:47:08.480140972 +0000 UTC m=+11.563546996, 201.066335ms after stability +2025-01-28T14:54:09.699Z [INFO] server-15df5da1-5489-9190-9dbc-5ab5efaffb73: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-15df5da1-5489-9190-9dbc-5ab5efaffb73 Address:15df5da1-5489-9190-9dbc-5ab5efaffb73}]" +2025-01-28T14:54:09.700Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:09.700Z [INFO] server-15df5da1-5489-9190-9dbc-5ab5efaffb73: entering follower state: follower="Node at 15df5da1-5489-9190-9dbc-5ab5efaffb73 [Follower]" leader-address= leader-id= +2025-01-28T14:54:09.891Z [WARN] server-15df5da1-5489-9190-9dbc-5ab5efaffb73: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:09.891Z [INFO] server-15df5da1-5489-9190-9dbc-5ab5efaffb73: entering candidate state: node="Node at 15df5da1-5489-9190-9dbc-5ab5efaffb73 [Candidate]" term=2 +2025-01-28T14:54:09.891Z [INFO] server-15df5da1-5489-9190-9dbc-5ab5efaffb73: election won: term=2 tally=1 +2025-01-28T14:54:09.891Z [INFO] server-15df5da1-5489-9190-9dbc-5ab5efaffb73: entering leader state: leader="Node at 15df5da1-5489-9190-9dbc-5ab5efaffb73 [Leader]" +2025-01-28T14:54:10.093Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:09.891974176 +0000 UTC m=+35.566464156 (1 nodes), 191.927373ms from start of poll, 192.346874ms from cluster start. Timeout at 2025-01-28 14:54:10.092981015 +0000 UTC m=+35.767471135, 201.006979ms after stability --- PASS: TestRaft_ApplyConcurrent_Timeout (0.40s) === RUN TestRaft_JoinNode -2025-01-28T14:47:08.493Z [INFO] server-b5e086a3-a5d6-b7e9-110c-cb4a8866a1bb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b5e086a3-a5d6-b7e9-110c-cb4a8866a1bb Address:b5e086a3-a5d6-b7e9-110c-cb4a8866a1bb} {Suffrage:Voter ID:server-1de2e149-53ee-70ae-41bc-5cb5868bca3b Address:1de2e149-53ee-70ae-41bc-5cb5868bca3b}]" -2025-01-28T14:47:08.493Z [INFO] server-b5e086a3-a5d6-b7e9-110c-cb4a8866a1bb: entering follower state: follower="Node at b5e086a3-a5d6-b7e9-110c-cb4a8866a1bb [Follower]" leader-address= leader-id= -2025-01-28T14:47:08.494Z [INFO] server-1de2e149-53ee-70ae-41bc-5cb5868bca3b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b5e086a3-a5d6-b7e9-110c-cb4a8866a1bb Address:b5e086a3-a5d6-b7e9-110c-cb4a8866a1bb} {Suffrage:Voter ID:server-1de2e149-53ee-70ae-41bc-5cb5868bca3b Address:1de2e149-53ee-70ae-41bc-5cb5868bca3b}]" -2025-01-28T14:47:08.495Z [INFO] server-1de2e149-53ee-70ae-41bc-5cb5868bca3b: entering follower state: follower="Node at 1de2e149-53ee-70ae-41bc-5cb5868bca3b [Follower]" leader-address= leader-id= -2025-01-28T14:47:08.498Z [INFO] server-6bef4d51-4f51-ac84-ad3d-f283191476da: initial configuration: index=0 servers=[] -2025-01-28T14:47:08.498Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:08.499Z [INFO] server-6bef4d51-4f51-ac84-ad3d-f283191476da: entering follower state: follower="Node at 6bef4d51-4f51-ac84-ad3d-f283191476da [Follower]" leader-address= leader-id= -2025-01-28T14:47:08.569Z [WARN] server-1de2e149-53ee-70ae-41bc-5cb5868bca3b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:08.569Z [INFO] server-1de2e149-53ee-70ae-41bc-5cb5868bca3b: entering candidate state: node="Node at 1de2e149-53ee-70ae-41bc-5cb5868bca3b [Candidate]" term=2 -2025-01-28T14:47:08.570Z [INFO] server-1de2e149-53ee-70ae-41bc-5cb5868bca3b: election won: term=2 tally=2 -2025-01-28T14:47:08.570Z [INFO] server-1de2e149-53ee-70ae-41bc-5cb5868bca3b: entering leader state: leader="Node at 1de2e149-53ee-70ae-41bc-5cb5868bca3b [Leader]" -2025-01-28T14:47:08.571Z [INFO] server-1de2e149-53ee-70ae-41bc-5cb5868bca3b: added peer, starting replication: peer=server-b5e086a3-a5d6-b7e9-110c-cb4a8866a1bb -2025-01-28T14:47:08.572Z [INFO] server-1de2e149-53ee-70ae-41bc-5cb5868bca3b: pipelining replication: peer="{Voter server-b5e086a3-a5d6-b7e9-110c-cb4a8866a1bb b5e086a3-a5d6-b7e9-110c-cb4a8866a1bb}" -2025-01-28T14:47:08.592Z [WARN] server-6bef4d51-4f51-ac84-ad3d-f283191476da: no known peers, aborting election -2025-01-28T14:47:08.676Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:08.571304979 +0000 UTC m=+11.654711337 (1 nodes), 72.354815ms from start of poll, 79.278021ms from cluster start. Timeout at 2025-01-28 14:47:08.676341979 +0000 UTC m=+11.759748004, 105.036667ms after stability -2025-01-28T14:47:08.677Z [INFO] server-1de2e149-53ee-70ae-41bc-5cb5868bca3b: updating configuration: command=AddVoter server-id=server-6bef4d51-4f51-ac84-ad3d-f283191476da server-addr=6bef4d51-4f51-ac84-ad3d-f283191476da servers="[{Suffrage:Voter ID:server-b5e086a3-a5d6-b7e9-110c-cb4a8866a1bb Address:b5e086a3-a5d6-b7e9-110c-cb4a8866a1bb} {Suffrage:Voter ID:server-1de2e149-53ee-70ae-41bc-5cb5868bca3b Address:1de2e149-53ee-70ae-41bc-5cb5868bca3b} {Suffrage:Voter ID:server-6bef4d51-4f51-ac84-ad3d-f283191476da Address:6bef4d51-4f51-ac84-ad3d-f283191476da}]" -2025-01-28T14:47:08.678Z [INFO] server-1de2e149-53ee-70ae-41bc-5cb5868bca3b: added peer, starting replication: peer=server-6bef4d51-4f51-ac84-ad3d-f283191476da -2025-01-28T14:47:08.678Z [WARN] server-6bef4d51-4f51-ac84-ad3d-f283191476da: failed to get previous log: previous-index=3 last-index=0 error="log not found" -2025-01-28T14:47:08.679Z [WARN] server-1de2e149-53ee-70ae-41bc-5cb5868bca3b: appendEntries rejected, sending older logs: peer="{Voter server-6bef4d51-4f51-ac84-ad3d-f283191476da 6bef4d51-4f51-ac84-ad3d-f283191476da}" next=1 -2025-01-28T14:47:08.680Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:08.681Z [INFO] server-1de2e149-53ee-70ae-41bc-5cb5868bca3b: pipelining replication: peer="{Voter server-6bef4d51-4f51-ac84-ad3d-f283191476da 6bef4d51-4f51-ac84-ad3d-f283191476da}" -2025-01-28T14:47:08.785Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:08.680373884 +0000 UTC m=+11.763779575 (1 nodes), 11.667µs from start of poll, 188.346259ms from cluster start. Timeout at 2025-01-28 14:47:08.785443885 +0000 UTC m=+11.868849910, 105.070335ms after stability -2025-01-28T14:47:08.786Z [INFO] server-1de2e149-53ee-70ae-41bc-5cb5868bca3b: aborting pipeline replication: peer="{Voter server-6bef4d51-4f51-ac84-ad3d-f283191476da 6bef4d51-4f51-ac84-ad3d-f283191476da}" -2025-01-28T14:47:08.786Z [INFO] server-1de2e149-53ee-70ae-41bc-5cb5868bca3b: aborting pipeline replication: peer="{Voter server-b5e086a3-a5d6-b7e9-110c-cb4a8866a1bb b5e086a3-a5d6-b7e9-110c-cb4a8866a1bb}" ---- PASS: TestRaft_JoinNode (0.31s) +2025-01-28T14:54:10.096Z [INFO] server-327182ef-baab-9b93-33be-9143202b1706: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-327182ef-baab-9b93-33be-9143202b1706 Address:327182ef-baab-9b93-33be-9143202b1706} {Suffrage:Voter ID:server-50cc7076-8741-7987-c24e-f8a12069a615 Address:50cc7076-8741-7987-c24e-f8a12069a615}]" +2025-01-28T14:54:10.096Z [INFO] server-327182ef-baab-9b93-33be-9143202b1706: entering follower state: follower="Node at 327182ef-baab-9b93-33be-9143202b1706 [Follower]" leader-address= leader-id= +2025-01-28T14:54:10.097Z [INFO] server-50cc7076-8741-7987-c24e-f8a12069a615: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-327182ef-baab-9b93-33be-9143202b1706 Address:327182ef-baab-9b93-33be-9143202b1706} {Suffrage:Voter ID:server-50cc7076-8741-7987-c24e-f8a12069a615 Address:50cc7076-8741-7987-c24e-f8a12069a615}]" +2025-01-28T14:54:10.097Z [INFO] server-50cc7076-8741-7987-c24e-f8a12069a615: entering follower state: follower="Node at 50cc7076-8741-7987-c24e-f8a12069a615 [Follower]" leader-address= leader-id= +2025-01-28T14:54:10.097Z [INFO] server-fdc951f2-46d3-6012-50e8-e739433a6060: initial configuration: index=0 servers=[] +2025-01-28T14:54:10.097Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:10.097Z [INFO] server-fdc951f2-46d3-6012-50e8-e739433a6060: entering follower state: follower="Node at fdc951f2-46d3-6012-50e8-e739433a6060 [Follower]" leader-address= leader-id= +2025-01-28T14:54:10.171Z [WARN] server-fdc951f2-46d3-6012-50e8-e739433a6060: no known peers, aborting election +2025-01-28T14:54:10.177Z [WARN] server-50cc7076-8741-7987-c24e-f8a12069a615: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:10.177Z [INFO] server-50cc7076-8741-7987-c24e-f8a12069a615: entering candidate state: node="Node at 50cc7076-8741-7987-c24e-f8a12069a615 [Candidate]" term=2 +2025-01-28T14:54:10.177Z [INFO] server-50cc7076-8741-7987-c24e-f8a12069a615: election won: term=2 tally=2 +2025-01-28T14:54:10.178Z [INFO] server-50cc7076-8741-7987-c24e-f8a12069a615: entering leader state: leader="Node at 50cc7076-8741-7987-c24e-f8a12069a615 [Leader]" +2025-01-28T14:54:10.178Z [INFO] server-50cc7076-8741-7987-c24e-f8a12069a615: added peer, starting replication: peer=server-327182ef-baab-9b93-33be-9143202b1706 +2025-01-28T14:54:10.178Z [INFO] server-50cc7076-8741-7987-c24e-f8a12069a615: pipelining replication: peer="{Voter server-327182ef-baab-9b93-33be-9143202b1706 327182ef-baab-9b93-33be-9143202b1706}" +2025-01-28T14:54:10.284Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:10.178178316 +0000 UTC m=+35.852668316 (1 nodes), 80.201314ms from start of poll, 81.698253ms from cluster start. Timeout at 2025-01-28 14:54:10.283185676 +0000 UTC m=+35.957675716, 105.0074ms after stability +2025-01-28T14:54:10.284Z [INFO] server-50cc7076-8741-7987-c24e-f8a12069a615: updating configuration: command=AddVoter server-id=server-fdc951f2-46d3-6012-50e8-e739433a6060 server-addr=fdc951f2-46d3-6012-50e8-e739433a6060 servers="[{Suffrage:Voter ID:server-327182ef-baab-9b93-33be-9143202b1706 Address:327182ef-baab-9b93-33be-9143202b1706} {Suffrage:Voter ID:server-50cc7076-8741-7987-c24e-f8a12069a615 Address:50cc7076-8741-7987-c24e-f8a12069a615} {Suffrage:Voter ID:server-fdc951f2-46d3-6012-50e8-e739433a6060 Address:fdc951f2-46d3-6012-50e8-e739433a6060}]" +2025-01-28T14:54:10.284Z [INFO] server-50cc7076-8741-7987-c24e-f8a12069a615: added peer, starting replication: peer=server-fdc951f2-46d3-6012-50e8-e739433a6060 +2025-01-28T14:54:10.284Z [WARN] server-fdc951f2-46d3-6012-50e8-e739433a6060: failed to get previous log: previous-index=3 last-index=0 error="log not found" +2025-01-28T14:54:10.284Z [WARN] server-50cc7076-8741-7987-c24e-f8a12069a615: appendEntries rejected, sending older logs: peer="{Voter server-fdc951f2-46d3-6012-50e8-e739433a6060 fdc951f2-46d3-6012-50e8-e739433a6060}" next=1 +2025-01-28T14:54:10.285Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:10.285Z [INFO] server-50cc7076-8741-7987-c24e-f8a12069a615: pipelining replication: peer="{Voter server-fdc951f2-46d3-6012-50e8-e739433a6060 fdc951f2-46d3-6012-50e8-e739433a6060}" +2025-01-28T14:54:10.390Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:10.285206859 +0000 UTC m=+35.959696879 (1 nodes), 1.86µs from start of poll, 188.726816ms from cluster start. Timeout at 2025-01-28 14:54:10.390230617 +0000 UTC m=+36.064720597, 105.023718ms after stability +2025-01-28T14:54:10.391Z [INFO] server-50cc7076-8741-7987-c24e-f8a12069a615: aborting pipeline replication: peer="{Voter server-fdc951f2-46d3-6012-50e8-e739433a6060 fdc951f2-46d3-6012-50e8-e739433a6060}" +2025-01-28T14:54:10.391Z [INFO] server-50cc7076-8741-7987-c24e-f8a12069a615: aborting pipeline replication: peer="{Voter server-327182ef-baab-9b93-33be-9143202b1706 327182ef-baab-9b93-33be-9143202b1706}" +--- PASS: TestRaft_JoinNode (0.30s) === RUN TestRaft_JoinNode_ConfigStore -2025-01-28T14:47:08.798Z [INFO] server-c9ae9745-3747-ad0d-8d0e-981f4739124a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c9ae9745-3747-ad0d-8d0e-981f4739124a Address:c9ae9745-3747-ad0d-8d0e-981f4739124a}]" -2025-01-28T14:47:08.799Z [INFO] server-c9ae9745-3747-ad0d-8d0e-981f4739124a: entering follower state: follower="Node at c9ae9745-3747-ad0d-8d0e-981f4739124a [Follower]" leader-address= leader-id= -2025-01-28T14:47:08.802Z [INFO] server-27322f22-a8f6-ebbc-2383-913b2318d7b3: initial configuration: index=0 servers=[] -2025-01-28T14:47:08.802Z [INFO] server-27322f22-a8f6-ebbc-2383-913b2318d7b3: entering follower state: follower="Node at 27322f22-a8f6-ebbc-2383-913b2318d7b3 [Follower]" leader-address= leader-id= -2025-01-28T14:47:08.805Z [INFO] server-68bb3633-f9f3-bd67-ee3d-a9f6ca0940fe: initial configuration: index=0 servers=[] -2025-01-28T14:47:08.806Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:08.806Z [INFO] server-68bb3633-f9f3-bd67-ee3d-a9f6ca0940fe: entering follower state: follower="Node at 68bb3633-f9f3-bd67-ee3d-a9f6ca0940fe [Follower]" leader-address= leader-id= -2025-01-28T14:47:08.870Z [WARN] server-68bb3633-f9f3-bd67-ee3d-a9f6ca0940fe: no known peers, aborting election -2025-01-28T14:47:08.870Z [WARN] server-c9ae9745-3747-ad0d-8d0e-981f4739124a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:08.871Z [INFO] server-c9ae9745-3747-ad0d-8d0e-981f4739124a: entering candidate state: node="Node at c9ae9745-3747-ad0d-8d0e-981f4739124a [Candidate]" term=2 -2025-01-28T14:47:08.871Z [INFO] server-c9ae9745-3747-ad0d-8d0e-981f4739124a: election won: term=2 tally=1 -2025-01-28T14:47:08.871Z [INFO] server-c9ae9745-3747-ad0d-8d0e-981f4739124a: entering leader state: leader="Node at c9ae9745-3747-ad0d-8d0e-981f4739124a [Leader]" -2025-01-28T14:47:08.873Z [WARN] server-27322f22-a8f6-ebbc-2383-913b2318d7b3: no known peers, aborting election -2025-01-28T14:47:08.977Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:08.872019573 +0000 UTC m=+11.955425931 (1 nodes), 65.532946ms from start of poll, 74.42321ms from cluster start. Timeout at 2025-01-28 14:47:08.977056574 +0000 UTC m=+12.060462932, 105.037001ms after stability -2025-01-28T14:47:08.978Z [INFO] server-c9ae9745-3747-ad0d-8d0e-981f4739124a: updating configuration: command=AddVoter server-id=server-27322f22-a8f6-ebbc-2383-913b2318d7b3 server-addr=27322f22-a8f6-ebbc-2383-913b2318d7b3 servers="[{Suffrage:Voter ID:server-c9ae9745-3747-ad0d-8d0e-981f4739124a Address:c9ae9745-3747-ad0d-8d0e-981f4739124a} {Suffrage:Voter ID:server-27322f22-a8f6-ebbc-2383-913b2318d7b3 Address:27322f22-a8f6-ebbc-2383-913b2318d7b3}]" -2025-01-28T14:47:08.979Z [INFO] server-c9ae9745-3747-ad0d-8d0e-981f4739124a: added peer, starting replication: peer=server-27322f22-a8f6-ebbc-2383-913b2318d7b3 -2025-01-28T14:47:08.980Z [WARN] server-27322f22-a8f6-ebbc-2383-913b2318d7b3: failed to get previous log: previous-index=3 last-index=0 error="log not found" -2025-01-28T14:47:08.980Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:08.981Z [WARN] server-c9ae9745-3747-ad0d-8d0e-981f4739124a: appendEntries rejected, sending older logs: peer="{Voter server-27322f22-a8f6-ebbc-2383-913b2318d7b3 27322f22-a8f6-ebbc-2383-913b2318d7b3}" next=1 -2025-01-28T14:47:08.983Z [INFO] server-c9ae9745-3747-ad0d-8d0e-981f4739124a: pipelining replication: peer="{Voter server-27322f22-a8f6-ebbc-2383-913b2318d7b3 27322f22-a8f6-ebbc-2383-913b2318d7b3}" -2025-01-28T14:47:09.086Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:08.980580796 +0000 UTC m=+12.063986821 (1 nodes), 12.001µs from start of poll, 182.9841ms from cluster start. Timeout at 2025-01-28 14:47:09.085657131 +0000 UTC m=+12.169063156, 105.076335ms after stability -2025-01-28T14:47:09.086Z [INFO] server-c9ae9745-3747-ad0d-8d0e-981f4739124a: updating configuration: command=AddVoter server-id=server-68bb3633-f9f3-bd67-ee3d-a9f6ca0940fe server-addr=68bb3633-f9f3-bd67-ee3d-a9f6ca0940fe servers="[{Suffrage:Voter ID:server-c9ae9745-3747-ad0d-8d0e-981f4739124a Address:c9ae9745-3747-ad0d-8d0e-981f4739124a} {Suffrage:Voter ID:server-27322f22-a8f6-ebbc-2383-913b2318d7b3 Address:27322f22-a8f6-ebbc-2383-913b2318d7b3} {Suffrage:Voter ID:server-68bb3633-f9f3-bd67-ee3d-a9f6ca0940fe Address:68bb3633-f9f3-bd67-ee3d-a9f6ca0940fe}]" -2025-01-28T14:47:09.087Z [INFO] server-c9ae9745-3747-ad0d-8d0e-981f4739124a: added peer, starting replication: peer=server-68bb3633-f9f3-bd67-ee3d-a9f6ca0940fe -2025-01-28T14:47:09.088Z [WARN] server-68bb3633-f9f3-bd67-ee3d-a9f6ca0940fe: failed to get previous log: previous-index=4 last-index=0 error="log not found" -2025-01-28T14:47:09.089Z [WARN] server-c9ae9745-3747-ad0d-8d0e-981f4739124a: appendEntries rejected, sending older logs: peer="{Voter server-68bb3633-f9f3-bd67-ee3d-a9f6ca0940fe 68bb3633-f9f3-bd67-ee3d-a9f6ca0940fe}" next=1 -2025-01-28T14:47:09.090Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:09.090Z [INFO] server-c9ae9745-3747-ad0d-8d0e-981f4739124a: pipelining replication: peer="{Voter server-68bb3633-f9f3-bd67-ee3d-a9f6ca0940fe 68bb3633-f9f3-bd67-ee3d-a9f6ca0940fe}" -2025-01-28T14:47:09.196Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:09.090504393 +0000 UTC m=+12.173910084 (1 nodes), 12.334µs from start of poll, 292.907363ms from cluster start. Timeout at 2025-01-28 14:47:09.195558728 +0000 UTC m=+12.278964752, 105.054668ms after stability -2025-01-28T14:47:09.196Z [INFO] server-c9ae9745-3747-ad0d-8d0e-981f4739124a: aborting pipeline replication: peer="{Voter server-27322f22-a8f6-ebbc-2383-913b2318d7b3 27322f22-a8f6-ebbc-2383-913b2318d7b3}" -2025-01-28T14:47:09.196Z [INFO] server-c9ae9745-3747-ad0d-8d0e-981f4739124a: aborting pipeline replication: peer="{Voter server-68bb3633-f9f3-bd67-ee3d-a9f6ca0940fe 68bb3633-f9f3-bd67-ee3d-a9f6ca0940fe}" ---- PASS: TestRaft_JoinNode_ConfigStore (0.41s) +2025-01-28T14:54:10.393Z [INFO] server-183828d8-410c-72b5-37b5-b094d1b7c6fa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-183828d8-410c-72b5-37b5-b094d1b7c6fa Address:183828d8-410c-72b5-37b5-b094d1b7c6fa}]" +2025-01-28T14:54:10.393Z [INFO] server-183828d8-410c-72b5-37b5-b094d1b7c6fa: entering follower state: follower="Node at 183828d8-410c-72b5-37b5-b094d1b7c6fa [Follower]" leader-address= leader-id= +2025-01-28T14:54:10.394Z [INFO] server-c14a7aa7-240d-742f-6ff7-cea5db30bb7f: initial configuration: index=0 servers=[] +2025-01-28T14:54:10.394Z [INFO] server-c14a7aa7-240d-742f-6ff7-cea5db30bb7f: entering follower state: follower="Node at c14a7aa7-240d-742f-6ff7-cea5db30bb7f [Follower]" leader-address= leader-id= +2025-01-28T14:54:10.395Z [INFO] server-bed2a870-29c5-4ddb-d2d9-36c311fa1a1b: initial configuration: index=0 servers=[] +2025-01-28T14:54:10.395Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:10.395Z [INFO] server-bed2a870-29c5-4ddb-d2d9-36c311fa1a1b: entering follower state: follower="Node at bed2a870-29c5-4ddb-d2d9-36c311fa1a1b [Follower]" leader-address= leader-id= +2025-01-28T14:54:10.474Z [WARN] server-bed2a870-29c5-4ddb-d2d9-36c311fa1a1b: no known peers, aborting election +2025-01-28T14:54:10.486Z [WARN] server-c14a7aa7-240d-742f-6ff7-cea5db30bb7f: no known peers, aborting election +2025-01-28T14:54:10.492Z [WARN] server-183828d8-410c-72b5-37b5-b094d1b7c6fa: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:10.492Z [INFO] server-183828d8-410c-72b5-37b5-b094d1b7c6fa: entering candidate state: node="Node at 183828d8-410c-72b5-37b5-b094d1b7c6fa [Candidate]" term=2 +2025-01-28T14:54:10.492Z [INFO] server-183828d8-410c-72b5-37b5-b094d1b7c6fa: election won: term=2 tally=1 +2025-01-28T14:54:10.493Z [INFO] server-183828d8-410c-72b5-37b5-b094d1b7c6fa: entering leader state: leader="Node at 183828d8-410c-72b5-37b5-b094d1b7c6fa [Leader]" +2025-01-28T14:54:10.598Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:10.493109526 +0000 UTC m=+36.167599965 (1 nodes), 97.711361ms from start of poll, 99.609682ms from cluster start. Timeout at 2025-01-28 14:54:10.598116584 +0000 UTC m=+36.272606624, 105.006659ms after stability +2025-01-28T14:54:10.598Z [INFO] server-183828d8-410c-72b5-37b5-b094d1b7c6fa: updating configuration: command=AddVoter server-id=server-c14a7aa7-240d-742f-6ff7-cea5db30bb7f server-addr=c14a7aa7-240d-742f-6ff7-cea5db30bb7f servers="[{Suffrage:Voter ID:server-183828d8-410c-72b5-37b5-b094d1b7c6fa Address:183828d8-410c-72b5-37b5-b094d1b7c6fa} {Suffrage:Voter ID:server-c14a7aa7-240d-742f-6ff7-cea5db30bb7f Address:c14a7aa7-240d-742f-6ff7-cea5db30bb7f}]" +2025-01-28T14:54:10.598Z [INFO] server-183828d8-410c-72b5-37b5-b094d1b7c6fa: added peer, starting replication: peer=server-c14a7aa7-240d-742f-6ff7-cea5db30bb7f +2025-01-28T14:54:10.598Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:10.599Z [WARN] server-c14a7aa7-240d-742f-6ff7-cea5db30bb7f: failed to get previous log: previous-index=3 last-index=0 error="log not found" +2025-01-28T14:54:10.599Z [WARN] server-183828d8-410c-72b5-37b5-b094d1b7c6fa: appendEntries rejected, sending older logs: peer="{Voter server-c14a7aa7-240d-742f-6ff7-cea5db30bb7f c14a7aa7-240d-742f-6ff7-cea5db30bb7f}" next=1 +2025-01-28T14:54:10.599Z [INFO] server-183828d8-410c-72b5-37b5-b094d1b7c6fa: pipelining replication: peer="{Voter server-c14a7aa7-240d-742f-6ff7-cea5db30bb7f c14a7aa7-240d-742f-6ff7-cea5db30bb7f}" +2025-01-28T14:54:10.704Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:10.599008855 +0000 UTC m=+36.273498815 (1 nodes), 2.54µs from start of poll, 205.508532ms from cluster start. Timeout at 2025-01-28 14:54:10.704032953 +0000 UTC m=+36.378522933, 105.024118ms after stability +2025-01-28T14:54:10.704Z [INFO] server-183828d8-410c-72b5-37b5-b094d1b7c6fa: updating configuration: command=AddVoter server-id=server-bed2a870-29c5-4ddb-d2d9-36c311fa1a1b server-addr=bed2a870-29c5-4ddb-d2d9-36c311fa1a1b servers="[{Suffrage:Voter ID:server-183828d8-410c-72b5-37b5-b094d1b7c6fa Address:183828d8-410c-72b5-37b5-b094d1b7c6fa} {Suffrage:Voter ID:server-c14a7aa7-240d-742f-6ff7-cea5db30bb7f Address:c14a7aa7-240d-742f-6ff7-cea5db30bb7f} {Suffrage:Voter ID:server-bed2a870-29c5-4ddb-d2d9-36c311fa1a1b Address:bed2a870-29c5-4ddb-d2d9-36c311fa1a1b}]" +2025-01-28T14:54:10.704Z [INFO] server-183828d8-410c-72b5-37b5-b094d1b7c6fa: added peer, starting replication: peer=server-bed2a870-29c5-4ddb-d2d9-36c311fa1a1b +2025-01-28T14:54:10.704Z [WARN] server-bed2a870-29c5-4ddb-d2d9-36c311fa1a1b: failed to get previous log: previous-index=4 last-index=0 error="log not found" +2025-01-28T14:54:10.704Z [WARN] server-183828d8-410c-72b5-37b5-b094d1b7c6fa: appendEntries rejected, sending older logs: peer="{Voter server-bed2a870-29c5-4ddb-d2d9-36c311fa1a1b bed2a870-29c5-4ddb-d2d9-36c311fa1a1b}" next=1 +2025-01-28T14:54:10.705Z [INFO] server-183828d8-410c-72b5-37b5-b094d1b7c6fa: pipelining replication: peer="{Voter server-bed2a870-29c5-4ddb-d2d9-36c311fa1a1b bed2a870-29c5-4ddb-d2d9-36c311fa1a1b}" +2025-01-28T14:54:10.705Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:10.810Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:10.705254292 +0000 UTC m=+36.379744272 (1 nodes), 2.02µs from start of poll, 311.753989ms from cluster start. Timeout at 2025-01-28 14:54:10.81027775 +0000 UTC m=+36.484767890, 105.023618ms after stability +2025-01-28T14:54:10.810Z [INFO] server-183828d8-410c-72b5-37b5-b094d1b7c6fa: aborting pipeline replication: peer="{Voter server-bed2a870-29c5-4ddb-d2d9-36c311fa1a1b bed2a870-29c5-4ddb-d2d9-36c311fa1a1b}" +2025-01-28T14:54:10.810Z [INFO] server-183828d8-410c-72b5-37b5-b094d1b7c6fa: aborting pipeline replication: peer="{Voter server-c14a7aa7-240d-742f-6ff7-cea5db30bb7f c14a7aa7-240d-742f-6ff7-cea5db30bb7f}" +--- PASS: TestRaft_JoinNode_ConfigStore (0.42s) === RUN TestRaft_RemoveFollower -2025-01-28T14:47:09.207Z [INFO] server-23fab774-ab30-2a89-1897-b1298a7e24d7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-23fab774-ab30-2a89-1897-b1298a7e24d7 Address:23fab774-ab30-2a89-1897-b1298a7e24d7} {Suffrage:Voter ID:server-2c71e324-1188-437b-3808-ae43c76baf5c Address:2c71e324-1188-437b-3808-ae43c76baf5c} {Suffrage:Voter ID:server-9a9b8701-2e2e-8656-7935-597f1257e969 Address:9a9b8701-2e2e-8656-7935-597f1257e969}]" -2025-01-28T14:47:09.207Z [INFO] server-23fab774-ab30-2a89-1897-b1298a7e24d7: entering follower state: follower="Node at 23fab774-ab30-2a89-1897-b1298a7e24d7 [Follower]" leader-address= leader-id= -2025-01-28T14:47:09.207Z [INFO] server-2c71e324-1188-437b-3808-ae43c76baf5c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-23fab774-ab30-2a89-1897-b1298a7e24d7 Address:23fab774-ab30-2a89-1897-b1298a7e24d7} {Suffrage:Voter ID:server-2c71e324-1188-437b-3808-ae43c76baf5c Address:2c71e324-1188-437b-3808-ae43c76baf5c} {Suffrage:Voter ID:server-9a9b8701-2e2e-8656-7935-597f1257e969 Address:9a9b8701-2e2e-8656-7935-597f1257e969}]" -2025-01-28T14:47:09.208Z [INFO] server-2c71e324-1188-437b-3808-ae43c76baf5c: entering follower state: follower="Node at 2c71e324-1188-437b-3808-ae43c76baf5c [Follower]" leader-address= leader-id= -2025-01-28T14:47:09.209Z [INFO] server-9a9b8701-2e2e-8656-7935-597f1257e969: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-23fab774-ab30-2a89-1897-b1298a7e24d7 Address:23fab774-ab30-2a89-1897-b1298a7e24d7} {Suffrage:Voter ID:server-2c71e324-1188-437b-3808-ae43c76baf5c Address:2c71e324-1188-437b-3808-ae43c76baf5c} {Suffrage:Voter ID:server-9a9b8701-2e2e-8656-7935-597f1257e969 Address:9a9b8701-2e2e-8656-7935-597f1257e969}]" -2025-01-28T14:47:09.209Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:09.209Z [INFO] server-9a9b8701-2e2e-8656-7935-597f1257e969: entering follower state: follower="Node at 9a9b8701-2e2e-8656-7935-597f1257e969 [Follower]" leader-address= leader-id= -2025-01-28T14:47:09.276Z [WARN] server-23fab774-ab30-2a89-1897-b1298a7e24d7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:09.276Z [INFO] server-23fab774-ab30-2a89-1897-b1298a7e24d7: entering candidate state: node="Node at 23fab774-ab30-2a89-1897-b1298a7e24d7 [Candidate]" term=2 -2025-01-28T14:47:09.277Z [INFO] server-23fab774-ab30-2a89-1897-b1298a7e24d7: election won: term=2 tally=2 -2025-01-28T14:47:09.277Z [INFO] server-23fab774-ab30-2a89-1897-b1298a7e24d7: entering leader state: leader="Node at 23fab774-ab30-2a89-1897-b1298a7e24d7 [Leader]" -2025-01-28T14:47:09.278Z [INFO] server-23fab774-ab30-2a89-1897-b1298a7e24d7: added peer, starting replication: peer=server-2c71e324-1188-437b-3808-ae43c76baf5c -2025-01-28T14:47:09.278Z [INFO] server-23fab774-ab30-2a89-1897-b1298a7e24d7: added peer, starting replication: peer=server-9a9b8701-2e2e-8656-7935-597f1257e969 -2025-01-28T14:47:09.279Z [INFO] server-23fab774-ab30-2a89-1897-b1298a7e24d7: pipelining replication: peer="{Voter server-2c71e324-1188-437b-3808-ae43c76baf5c 2c71e324-1188-437b-3808-ae43c76baf5c}" -2025-01-28T14:47:09.280Z [INFO] server-23fab774-ab30-2a89-1897-b1298a7e24d7: pipelining replication: peer="{Voter server-9a9b8701-2e2e-8656-7935-597f1257e969 9a9b8701-2e2e-8656-7935-597f1257e969}" -2025-01-28T14:47:09.384Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:09.278105963 +0000 UTC m=+12.361512654 (1 nodes), 68.087022ms from start of poll, 71.599126ms from cluster start. Timeout at 2025-01-28 14:47:09.383126297 +0000 UTC m=+12.466531988, 105.019334ms after stability -2025-01-28T14:47:09.390Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:47:09.495Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:47:09.390386629 +0000 UTC m=+12.473792320 (2 nodes), 16.333µs from start of poll, 183.878792ms from cluster start. Timeout at 2025-01-28 14:47:09.495482299 +0000 UTC m=+12.578888323, 105.096003ms after stability -2025-01-28T14:47:09.496Z [INFO] server-23fab774-ab30-2a89-1897-b1298a7e24d7: updating configuration: command=RemoveServer server-id=server-2c71e324-1188-437b-3808-ae43c76baf5c server-addr= servers="[{Suffrage:Voter ID:server-23fab774-ab30-2a89-1897-b1298a7e24d7 Address:23fab774-ab30-2a89-1897-b1298a7e24d7} {Suffrage:Voter ID:server-9a9b8701-2e2e-8656-7935-597f1257e969 Address:9a9b8701-2e2e-8656-7935-597f1257e969}]" -2025-01-28T14:47:09.497Z [INFO] server-23fab774-ab30-2a89-1897-b1298a7e24d7: removed peer, stopping replication: peer=server-2c71e324-1188-437b-3808-ae43c76baf5c last-index=3 -2025-01-28T14:47:09.497Z [INFO] server-23fab774-ab30-2a89-1897-b1298a7e24d7: aborting pipeline replication: peer="{Voter server-2c71e324-1188-437b-3808-ae43c76baf5c 2c71e324-1188-437b-3808-ae43c76baf5c}" -2025-01-28T14:47:09.587Z [WARN] server-2c71e324-1188-437b-3808-ae43c76baf5c: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election -2025-01-28T14:47:09.604Z [INFO] server-23fab774-ab30-2a89-1897-b1298a7e24d7: aborting pipeline replication: peer="{Voter server-9a9b8701-2e2e-8656-7935-597f1257e969 9a9b8701-2e2e-8656-7935-597f1257e969}" ---- PASS: TestRaft_RemoveFollower (0.41s) +2025-01-28T14:54:10.816Z [INFO] server-12f0ae44-c32d-27c7-ddc8-30c3f3b876c5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-12f0ae44-c32d-27c7-ddc8-30c3f3b876c5 Address:12f0ae44-c32d-27c7-ddc8-30c3f3b876c5} {Suffrage:Voter ID:server-59c54d84-49d3-c527-e7ed-304a18561415 Address:59c54d84-49d3-c527-e7ed-304a18561415} {Suffrage:Voter ID:server-18f60f93-92a5-bd66-fbd4-a4e4fe3a30df Address:18f60f93-92a5-bd66-fbd4-a4e4fe3a30df}]" +2025-01-28T14:54:10.817Z [INFO] server-12f0ae44-c32d-27c7-ddc8-30c3f3b876c5: entering follower state: follower="Node at 12f0ae44-c32d-27c7-ddc8-30c3f3b876c5 [Follower]" leader-address= leader-id= +2025-01-28T14:54:10.817Z [INFO] server-59c54d84-49d3-c527-e7ed-304a18561415: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-12f0ae44-c32d-27c7-ddc8-30c3f3b876c5 Address:12f0ae44-c32d-27c7-ddc8-30c3f3b876c5} {Suffrage:Voter ID:server-59c54d84-49d3-c527-e7ed-304a18561415 Address:59c54d84-49d3-c527-e7ed-304a18561415} {Suffrage:Voter ID:server-18f60f93-92a5-bd66-fbd4-a4e4fe3a30df Address:18f60f93-92a5-bd66-fbd4-a4e4fe3a30df}]" +2025-01-28T14:54:10.817Z [INFO] server-59c54d84-49d3-c527-e7ed-304a18561415: entering follower state: follower="Node at 59c54d84-49d3-c527-e7ed-304a18561415 [Follower]" leader-address= leader-id= +2025-01-28T14:54:10.817Z [INFO] server-18f60f93-92a5-bd66-fbd4-a4e4fe3a30df: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-12f0ae44-c32d-27c7-ddc8-30c3f3b876c5 Address:12f0ae44-c32d-27c7-ddc8-30c3f3b876c5} {Suffrage:Voter ID:server-59c54d84-49d3-c527-e7ed-304a18561415 Address:59c54d84-49d3-c527-e7ed-304a18561415} {Suffrage:Voter ID:server-18f60f93-92a5-bd66-fbd4-a4e4fe3a30df Address:18f60f93-92a5-bd66-fbd4-a4e4fe3a30df}]" +2025-01-28T14:54:10.817Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:10.817Z [INFO] server-18f60f93-92a5-bd66-fbd4-a4e4fe3a30df: entering follower state: follower="Node at 18f60f93-92a5-bd66-fbd4-a4e4fe3a30df [Follower]" leader-address= leader-id= +2025-01-28T14:54:10.872Z [WARN] server-18f60f93-92a5-bd66-fbd4-a4e4fe3a30df: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:10.872Z [INFO] server-18f60f93-92a5-bd66-fbd4-a4e4fe3a30df: entering candidate state: node="Node at 18f60f93-92a5-bd66-fbd4-a4e4fe3a30df [Candidate]" term=2 +2025-01-28T14:54:10.872Z [INFO] server-18f60f93-92a5-bd66-fbd4-a4e4fe3a30df: election won: term=2 tally=2 +2025-01-28T14:54:10.872Z [INFO] server-18f60f93-92a5-bd66-fbd4-a4e4fe3a30df: entering leader state: leader="Node at 18f60f93-92a5-bd66-fbd4-a4e4fe3a30df [Leader]" +2025-01-28T14:54:10.873Z [INFO] server-18f60f93-92a5-bd66-fbd4-a4e4fe3a30df: added peer, starting replication: peer=server-12f0ae44-c32d-27c7-ddc8-30c3f3b876c5 +2025-01-28T14:54:10.873Z [INFO] server-18f60f93-92a5-bd66-fbd4-a4e4fe3a30df: added peer, starting replication: peer=server-59c54d84-49d3-c527-e7ed-304a18561415 +2025-01-28T14:54:10.873Z [INFO] server-18f60f93-92a5-bd66-fbd4-a4e4fe3a30df: pipelining replication: peer="{Voter server-12f0ae44-c32d-27c7-ddc8-30c3f3b876c5 12f0ae44-c32d-27c7-ddc8-30c3f3b876c5}" +2025-01-28T14:54:10.873Z [INFO] server-18f60f93-92a5-bd66-fbd4-a4e4fe3a30df: pipelining replication: peer="{Voter server-59c54d84-49d3-c527-e7ed-304a18561415 59c54d84-49d3-c527-e7ed-304a18561415}" +2025-01-28T14:54:10.978Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:10.872984828 +0000 UTC m=+36.547474808 (1 nodes), 55.201692ms from start of poll, 56.470572ms from cluster start. Timeout at 2025-01-28 14:54:10.977991708 +0000 UTC m=+36.652481768, 105.00696ms after stability +2025-01-28T14:54:10.983Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:54:11.089Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:54:10.983770777 +0000 UTC m=+36.658260757 (2 nodes), 3.1µs from start of poll, 167.256521ms from cluster start. Timeout at 2025-01-28 14:54:11.088801574 +0000 UTC m=+36.763291614, 105.030857ms after stability +2025-01-28T14:54:11.089Z [INFO] server-18f60f93-92a5-bd66-fbd4-a4e4fe3a30df: updating configuration: command=RemoveServer server-id=server-12f0ae44-c32d-27c7-ddc8-30c3f3b876c5 server-addr= servers="[{Suffrage:Voter ID:server-59c54d84-49d3-c527-e7ed-304a18561415 Address:59c54d84-49d3-c527-e7ed-304a18561415} {Suffrage:Voter ID:server-18f60f93-92a5-bd66-fbd4-a4e4fe3a30df Address:18f60f93-92a5-bd66-fbd4-a4e4fe3a30df}]" +2025-01-28T14:54:11.089Z [INFO] server-18f60f93-92a5-bd66-fbd4-a4e4fe3a30df: removed peer, stopping replication: peer=server-12f0ae44-c32d-27c7-ddc8-30c3f3b876c5 last-index=3 +2025-01-28T14:54:11.089Z [INFO] server-18f60f93-92a5-bd66-fbd4-a4e4fe3a30df: aborting pipeline replication: peer="{Voter server-12f0ae44-c32d-27c7-ddc8-30c3f3b876c5 12f0ae44-c32d-27c7-ddc8-30c3f3b876c5}" +2025-01-28T14:54:11.195Z [INFO] server-18f60f93-92a5-bd66-fbd4-a4e4fe3a30df: aborting pipeline replication: peer="{Voter server-59c54d84-49d3-c527-e7ed-304a18561415 59c54d84-49d3-c527-e7ed-304a18561415}" +--- PASS: TestRaft_RemoveFollower (0.38s) === RUN TestRaft_RemoveLeader -2025-01-28T14:47:09.622Z [INFO] server-0bb198d2-3144-9050-708d-ba597a89098c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0bb198d2-3144-9050-708d-ba597a89098c Address:0bb198d2-3144-9050-708d-ba597a89098c} {Suffrage:Voter ID:server-3800ac0c-3deb-159b-bef8-f7ea1c026bbe Address:3800ac0c-3deb-159b-bef8-f7ea1c026bbe} {Suffrage:Voter ID:server-20938b53-c978-b131-c99d-33b8cb97c052 Address:20938b53-c978-b131-c99d-33b8cb97c052}]" -2025-01-28T14:47:09.622Z [INFO] server-0bb198d2-3144-9050-708d-ba597a89098c: entering follower state: follower="Node at 0bb198d2-3144-9050-708d-ba597a89098c [Follower]" leader-address= leader-id= -2025-01-28T14:47:09.624Z [INFO] server-3800ac0c-3deb-159b-bef8-f7ea1c026bbe: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0bb198d2-3144-9050-708d-ba597a89098c Address:0bb198d2-3144-9050-708d-ba597a89098c} {Suffrage:Voter ID:server-3800ac0c-3deb-159b-bef8-f7ea1c026bbe Address:3800ac0c-3deb-159b-bef8-f7ea1c026bbe} {Suffrage:Voter ID:server-20938b53-c978-b131-c99d-33b8cb97c052 Address:20938b53-c978-b131-c99d-33b8cb97c052}]" -2025-01-28T14:47:09.624Z [INFO] server-3800ac0c-3deb-159b-bef8-f7ea1c026bbe: entering follower state: follower="Node at 3800ac0c-3deb-159b-bef8-f7ea1c026bbe [Follower]" leader-address= leader-id= -2025-01-28T14:47:09.625Z [INFO] server-20938b53-c978-b131-c99d-33b8cb97c052: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0bb198d2-3144-9050-708d-ba597a89098c Address:0bb198d2-3144-9050-708d-ba597a89098c} {Suffrage:Voter ID:server-3800ac0c-3deb-159b-bef8-f7ea1c026bbe Address:3800ac0c-3deb-159b-bef8-f7ea1c026bbe} {Suffrage:Voter ID:server-20938b53-c978-b131-c99d-33b8cb97c052 Address:20938b53-c978-b131-c99d-33b8cb97c052}]" -2025-01-28T14:47:09.625Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:09.626Z [INFO] server-20938b53-c978-b131-c99d-33b8cb97c052: entering follower state: follower="Node at 20938b53-c978-b131-c99d-33b8cb97c052 [Follower]" leader-address= leader-id= -2025-01-28T14:47:09.689Z [WARN] server-20938b53-c978-b131-c99d-33b8cb97c052: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:09.690Z [WARN] server-0bb198d2-3144-9050-708d-ba597a89098c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:09.690Z [INFO] server-20938b53-c978-b131-c99d-33b8cb97c052: entering candidate state: node="Node at 20938b53-c978-b131-c99d-33b8cb97c052 [Candidate]" term=2 -2025-01-28T14:47:09.690Z [INFO] server-0bb198d2-3144-9050-708d-ba597a89098c: entering candidate state: node="Node at 0bb198d2-3144-9050-708d-ba597a89098c [Candidate]" term=2 -2025-01-28T14:47:09.691Z [INFO] server-20938b53-c978-b131-c99d-33b8cb97c052: duplicate requestVote for same term: term=2 -2025-01-28T14:47:09.691Z [INFO] server-0bb198d2-3144-9050-708d-ba597a89098c: duplicate requestVote for same term: term=2 -2025-01-28T14:47:09.691Z [INFO] server-3800ac0c-3deb-159b-bef8-f7ea1c026bbe: duplicate requestVote for same term: term=2 -2025-01-28T14:47:09.691Z [INFO] server-20938b53-c978-b131-c99d-33b8cb97c052: election won: term=2 tally=2 -2025-01-28T14:47:09.691Z [INFO] server-20938b53-c978-b131-c99d-33b8cb97c052: entering leader state: leader="Node at 20938b53-c978-b131-c99d-33b8cb97c052 [Leader]" -2025-01-28T14:47:09.691Z [INFO] server-20938b53-c978-b131-c99d-33b8cb97c052: added peer, starting replication: peer=server-0bb198d2-3144-9050-708d-ba597a89098c -2025-01-28T14:47:09.691Z [INFO] server-20938b53-c978-b131-c99d-33b8cb97c052: added peer, starting replication: peer=server-3800ac0c-3deb-159b-bef8-f7ea1c026bbe -2025-01-28T14:47:09.692Z [INFO] server-0bb198d2-3144-9050-708d-ba597a89098c: entering follower state: follower="Node at 0bb198d2-3144-9050-708d-ba597a89098c [Follower]" leader-address=20938b53-c978-b131-c99d-33b8cb97c052 leader-id=server-20938b53-c978-b131-c99d-33b8cb97c052 -2025-01-28T14:47:09.693Z [INFO] server-20938b53-c978-b131-c99d-33b8cb97c052: pipelining replication: peer="{Voter server-0bb198d2-3144-9050-708d-ba597a89098c 0bb198d2-3144-9050-708d-ba597a89098c}" -2025-01-28T14:47:09.693Z [INFO] server-20938b53-c978-b131-c99d-33b8cb97c052: pipelining replication: peer="{Voter server-3800ac0c-3deb-159b-bef8-f7ea1c026bbe 3800ac0c-3deb-159b-bef8-f7ea1c026bbe}" -2025-01-28T14:47:09.798Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:09.692929611 +0000 UTC m=+12.776335969 (1 nodes), 66.818984ms from start of poll, 71.74013ms from cluster start. Timeout at 2025-01-28 14:47:09.797950279 +0000 UTC m=+12.881356303, 105.020334ms after stability -2025-01-28T14:47:09.803Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:47:09.909Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:47:09.804017909 +0000 UTC m=+12.887423600 (2 nodes), 15µs from start of poll, 182.827761ms from cluster start. Timeout at 2025-01-28 14:47:09.909115912 +0000 UTC m=+12.992521936, 105.098336ms after stability -2025-01-28T14:47:09.909Z [INFO] server-20938b53-c978-b131-c99d-33b8cb97c052: updating configuration: command=RemoveServer server-id=server-20938b53-c978-b131-c99d-33b8cb97c052 server-addr= servers="[{Suffrage:Voter ID:server-0bb198d2-3144-9050-708d-ba597a89098c Address:0bb198d2-3144-9050-708d-ba597a89098c} {Suffrage:Voter ID:server-3800ac0c-3deb-159b-bef8-f7ea1c026bbe Address:3800ac0c-3deb-159b-bef8-f7ea1c026bbe}]" -2025-01-28T14:47:09.912Z [INFO] server-20938b53-c978-b131-c99d-33b8cb97c052: removed ourself, shutting down -2025-01-28T14:47:09.912Z [INFO] server-20938b53-c978-b131-c99d-33b8cb97c052: aborting pipeline replication: peer="{Voter server-3800ac0c-3deb-159b-bef8-f7ea1c026bbe 3800ac0c-3deb-159b-bef8-f7ea1c026bbe}" -2025-01-28T14:47:09.912Z [INFO] server-20938b53-c978-b131-c99d-33b8cb97c052: aborting pipeline replication: peer="{Voter server-0bb198d2-3144-9050-708d-ba597a89098c 0bb198d2-3144-9050-708d-ba597a89098c}" -2025-01-28T14:47:09.976Z [WARN] server-3800ac0c-3deb-159b-bef8-f7ea1c026bbe: heartbeat timeout reached, starting election: last-leader-addr=20938b53-c978-b131-c99d-33b8cb97c052 last-leader-id=server-20938b53-c978-b131-c99d-33b8cb97c052 -2025-01-28T14:47:09.976Z [INFO] server-3800ac0c-3deb-159b-bef8-f7ea1c026bbe: entering candidate state: node="Node at 3800ac0c-3deb-159b-bef8-f7ea1c026bbe [Candidate]" term=3 -2025-01-28T14:47:09.977Z [WARN] server-0bb198d2-3144-9050-708d-ba597a89098c: rejecting vote request since we have a leader: from=3800ac0c-3deb-159b-bef8-f7ea1c026bbe leader=20938b53-c978-b131-c99d-33b8cb97c052 leader-id=server-20938b53-c978-b131-c99d-33b8cb97c052 -2025-01-28T14:47:10.003Z [WARN] server-0bb198d2-3144-9050-708d-ba597a89098c: heartbeat timeout reached, starting election: last-leader-addr=20938b53-c978-b131-c99d-33b8cb97c052 last-leader-id=server-20938b53-c978-b131-c99d-33b8cb97c052 -2025-01-28T14:47:10.004Z [INFO] server-0bb198d2-3144-9050-708d-ba597a89098c: entering candidate state: node="Node at 0bb198d2-3144-9050-708d-ba597a89098c [Candidate]" term=3 -2025-01-28T14:47:10.004Z [INFO] server-3800ac0c-3deb-159b-bef8-f7ea1c026bbe: duplicate requestVote for same term: term=3 -2025-01-28T14:47:10.074Z [WARN] server-3800ac0c-3deb-159b-bef8-f7ea1c026bbe: Election timeout reached, restarting election -2025-01-28T14:47:10.075Z [INFO] server-3800ac0c-3deb-159b-bef8-f7ea1c026bbe: entering candidate state: node="Node at 3800ac0c-3deb-159b-bef8-f7ea1c026bbe [Candidate]" term=4 -2025-01-28T14:47:10.075Z [INFO] server-0bb198d2-3144-9050-708d-ba597a89098c: entering follower state: follower="Node at 0bb198d2-3144-9050-708d-ba597a89098c [Follower]" leader-address= leader-id= -2025-01-28T14:47:10.076Z [INFO] server-3800ac0c-3deb-159b-bef8-f7ea1c026bbe: election won: term=4 tally=2 -2025-01-28T14:47:10.076Z [INFO] server-3800ac0c-3deb-159b-bef8-f7ea1c026bbe: entering leader state: leader="Node at 3800ac0c-3deb-159b-bef8-f7ea1c026bbe [Leader]" -2025-01-28T14:47:10.076Z [INFO] server-3800ac0c-3deb-159b-bef8-f7ea1c026bbe: added peer, starting replication: peer=server-0bb198d2-3144-9050-708d-ba597a89098c -2025-01-28T14:47:10.078Z [INFO] server-3800ac0c-3deb-159b-bef8-f7ea1c026bbe: pipelining replication: peer="{Voter server-0bb198d2-3144-9050-708d-ba597a89098c 0bb198d2-3144-9050-708d-ba597a89098c}" -2025-01-28T14:47:10.124Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:10.231Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:10.125987468 +0000 UTC m=+13.209393826 (1 nodes), 960.695µs from start of poll, 504.797987ms from cluster start. Timeout at 2025-01-28 14:47:10.231064804 +0000 UTC m=+13.314471162, 105.077336ms after stability -2025-01-28T14:47:10.232Z [INFO] server-3800ac0c-3deb-159b-bef8-f7ea1c026bbe: aborting pipeline replication: peer="{Voter server-0bb198d2-3144-9050-708d-ba597a89098c 0bb198d2-3144-9050-708d-ba597a89098c}" ---- PASS: TestRaft_RemoveLeader (0.63s) +2025-01-28T14:54:11.199Z [INFO] server-05ce62bb-e9c2-814a-bfa7-4004b680a037: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-05ce62bb-e9c2-814a-bfa7-4004b680a037 Address:05ce62bb-e9c2-814a-bfa7-4004b680a037} {Suffrage:Voter ID:server-87e7f958-5e84-1f2c-4bf8-e7f6dc3784e7 Address:87e7f958-5e84-1f2c-4bf8-e7f6dc3784e7} {Suffrage:Voter ID:server-d74b3a6a-b3b2-3935-bc09-493fe8e760a6 Address:d74b3a6a-b3b2-3935-bc09-493fe8e760a6}]" +2025-01-28T14:54:11.199Z [INFO] server-05ce62bb-e9c2-814a-bfa7-4004b680a037: entering follower state: follower="Node at 05ce62bb-e9c2-814a-bfa7-4004b680a037 [Follower]" leader-address= leader-id= +2025-01-28T14:54:11.199Z [INFO] server-87e7f958-5e84-1f2c-4bf8-e7f6dc3784e7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-05ce62bb-e9c2-814a-bfa7-4004b680a037 Address:05ce62bb-e9c2-814a-bfa7-4004b680a037} {Suffrage:Voter ID:server-87e7f958-5e84-1f2c-4bf8-e7f6dc3784e7 Address:87e7f958-5e84-1f2c-4bf8-e7f6dc3784e7} {Suffrage:Voter ID:server-d74b3a6a-b3b2-3935-bc09-493fe8e760a6 Address:d74b3a6a-b3b2-3935-bc09-493fe8e760a6}]" +2025-01-28T14:54:11.199Z [INFO] server-87e7f958-5e84-1f2c-4bf8-e7f6dc3784e7: entering follower state: follower="Node at 87e7f958-5e84-1f2c-4bf8-e7f6dc3784e7 [Follower]" leader-address= leader-id= +2025-01-28T14:54:11.199Z [INFO] server-d74b3a6a-b3b2-3935-bc09-493fe8e760a6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-05ce62bb-e9c2-814a-bfa7-4004b680a037 Address:05ce62bb-e9c2-814a-bfa7-4004b680a037} {Suffrage:Voter ID:server-87e7f958-5e84-1f2c-4bf8-e7f6dc3784e7 Address:87e7f958-5e84-1f2c-4bf8-e7f6dc3784e7} {Suffrage:Voter ID:server-d74b3a6a-b3b2-3935-bc09-493fe8e760a6 Address:d74b3a6a-b3b2-3935-bc09-493fe8e760a6}]" +2025-01-28T14:54:11.200Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:11.200Z [INFO] server-d74b3a6a-b3b2-3935-bc09-493fe8e760a6: entering follower state: follower="Node at d74b3a6a-b3b2-3935-bc09-493fe8e760a6 [Follower]" leader-address= leader-id= +2025-01-28T14:54:11.250Z [WARN] server-05ce62bb-e9c2-814a-bfa7-4004b680a037: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:11.250Z [INFO] server-05ce62bb-e9c2-814a-bfa7-4004b680a037: entering candidate state: node="Node at 05ce62bb-e9c2-814a-bfa7-4004b680a037 [Candidate]" term=2 +2025-01-28T14:54:11.250Z [INFO] server-05ce62bb-e9c2-814a-bfa7-4004b680a037: election won: term=2 tally=2 +2025-01-28T14:54:11.250Z [INFO] server-05ce62bb-e9c2-814a-bfa7-4004b680a037: entering leader state: leader="Node at 05ce62bb-e9c2-814a-bfa7-4004b680a037 [Leader]" +2025-01-28T14:54:11.251Z [INFO] server-05ce62bb-e9c2-814a-bfa7-4004b680a037: added peer, starting replication: peer=server-87e7f958-5e84-1f2c-4bf8-e7f6dc3784e7 +2025-01-28T14:54:11.251Z [INFO] server-05ce62bb-e9c2-814a-bfa7-4004b680a037: added peer, starting replication: peer=server-d74b3a6a-b3b2-3935-bc09-493fe8e760a6 +2025-01-28T14:54:11.251Z [INFO] server-05ce62bb-e9c2-814a-bfa7-4004b680a037: pipelining replication: peer="{Voter server-87e7f958-5e84-1f2c-4bf8-e7f6dc3784e7 87e7f958-5e84-1f2c-4bf8-e7f6dc3784e7}" +2025-01-28T14:54:11.251Z [INFO] server-05ce62bb-e9c2-814a-bfa7-4004b680a037: pipelining replication: peer="{Voter server-d74b3a6a-b3b2-3935-bc09-493fe8e760a6 d74b3a6a-b3b2-3935-bc09-493fe8e760a6}" +2025-01-28T14:54:11.357Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:11.251040723 +0000 UTC m=+36.925530683 (1 nodes), 50.963792ms from start of poll, 52.098865ms from cluster start. Timeout at 2025-01-28 14:54:11.356044362 +0000 UTC m=+37.030534382, 105.003699ms after stability +2025-01-28T14:54:11.362Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:54:11.469Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:54:11.362926768 +0000 UTC m=+37.037416748 (2 nodes), 2.86µs from start of poll, 163.98493ms from cluster start. Timeout at 2025-01-28 14:54:11.467952985 +0000 UTC m=+37.142443025, 105.026277ms after stability +2025-01-28T14:54:11.469Z [INFO] server-05ce62bb-e9c2-814a-bfa7-4004b680a037: updating configuration: command=RemoveServer server-id=server-05ce62bb-e9c2-814a-bfa7-4004b680a037 server-addr= servers="[{Suffrage:Voter ID:server-87e7f958-5e84-1f2c-4bf8-e7f6dc3784e7 Address:87e7f958-5e84-1f2c-4bf8-e7f6dc3784e7} {Suffrage:Voter ID:server-d74b3a6a-b3b2-3935-bc09-493fe8e760a6 Address:d74b3a6a-b3b2-3935-bc09-493fe8e760a6}]" +2025-01-28T14:54:11.469Z [INFO] server-05ce62bb-e9c2-814a-bfa7-4004b680a037: removed ourself, shutting down +2025-01-28T14:54:11.469Z [INFO] server-05ce62bb-e9c2-814a-bfa7-4004b680a037: aborting pipeline replication: peer="{Voter server-87e7f958-5e84-1f2c-4bf8-e7f6dc3784e7 87e7f958-5e84-1f2c-4bf8-e7f6dc3784e7}" +2025-01-28T14:54:11.469Z [INFO] server-05ce62bb-e9c2-814a-bfa7-4004b680a037: aborting pipeline replication: peer="{Voter server-d74b3a6a-b3b2-3935-bc09-493fe8e760a6 d74b3a6a-b3b2-3935-bc09-493fe8e760a6}" +2025-01-28T14:54:11.553Z [WARN] server-87e7f958-5e84-1f2c-4bf8-e7f6dc3784e7: heartbeat timeout reached, starting election: last-leader-addr=05ce62bb-e9c2-814a-bfa7-4004b680a037 last-leader-id=server-05ce62bb-e9c2-814a-bfa7-4004b680a037 +2025-01-28T14:54:11.553Z [INFO] server-87e7f958-5e84-1f2c-4bf8-e7f6dc3784e7: entering candidate state: node="Node at 87e7f958-5e84-1f2c-4bf8-e7f6dc3784e7 [Candidate]" term=3 +2025-01-28T14:54:11.553Z [WARN] server-d74b3a6a-b3b2-3935-bc09-493fe8e760a6: rejecting vote request since we have a leader: from=87e7f958-5e84-1f2c-4bf8-e7f6dc3784e7 leader=05ce62bb-e9c2-814a-bfa7-4004b680a037 leader-id=server-05ce62bb-e9c2-814a-bfa7-4004b680a037 +2025-01-28T14:54:11.585Z [WARN] server-d74b3a6a-b3b2-3935-bc09-493fe8e760a6: heartbeat timeout reached, starting election: last-leader-addr=05ce62bb-e9c2-814a-bfa7-4004b680a037 last-leader-id=server-05ce62bb-e9c2-814a-bfa7-4004b680a037 +2025-01-28T14:54:11.585Z [INFO] server-d74b3a6a-b3b2-3935-bc09-493fe8e760a6: entering candidate state: node="Node at d74b3a6a-b3b2-3935-bc09-493fe8e760a6 [Candidate]" term=3 +2025-01-28T14:54:11.585Z [INFO] server-87e7f958-5e84-1f2c-4bf8-e7f6dc3784e7: duplicate requestVote for same term: term=3 +2025-01-28T14:54:11.618Z [WARN] server-87e7f958-5e84-1f2c-4bf8-e7f6dc3784e7: Election timeout reached, restarting election +2025-01-28T14:54:11.618Z [INFO] server-87e7f958-5e84-1f2c-4bf8-e7f6dc3784e7: entering candidate state: node="Node at 87e7f958-5e84-1f2c-4bf8-e7f6dc3784e7 [Candidate]" term=4 +2025-01-28T14:54:11.618Z [INFO] server-d74b3a6a-b3b2-3935-bc09-493fe8e760a6: entering follower state: follower="Node at d74b3a6a-b3b2-3935-bc09-493fe8e760a6 [Follower]" leader-address= leader-id= +2025-01-28T14:54:11.618Z [INFO] server-87e7f958-5e84-1f2c-4bf8-e7f6dc3784e7: election won: term=4 tally=2 +2025-01-28T14:54:11.618Z [INFO] server-87e7f958-5e84-1f2c-4bf8-e7f6dc3784e7: entering leader state: leader="Node at 87e7f958-5e84-1f2c-4bf8-e7f6dc3784e7 [Leader]" +2025-01-28T14:54:11.618Z [INFO] server-87e7f958-5e84-1f2c-4bf8-e7f6dc3784e7: added peer, starting replication: peer=server-d74b3a6a-b3b2-3935-bc09-493fe8e760a6 +2025-01-28T14:54:11.619Z [INFO] server-87e7f958-5e84-1f2c-4bf8-e7f6dc3784e7: pipelining replication: peer="{Voter server-d74b3a6a-b3b2-3935-bc09-493fe8e760a6 d74b3a6a-b3b2-3935-bc09-493fe8e760a6}" +2025-01-28T14:54:11.681Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:11.786Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:11.681293493 +0000 UTC m=+37.355783473 (1 nodes), 152.766µs from start of poll, 482.351655ms from cluster start. Timeout at 2025-01-28 14:54:11.786396063 +0000 UTC m=+37.460886103, 105.10263ms after stability +2025-01-28T14:54:11.786Z [INFO] server-87e7f958-5e84-1f2c-4bf8-e7f6dc3784e7: aborting pipeline replication: peer="{Voter server-d74b3a6a-b3b2-3935-bc09-493fe8e760a6 d74b3a6a-b3b2-3935-bc09-493fe8e760a6}" +--- PASS: TestRaft_RemoveLeader (0.59s) === RUN TestRaft_RemoveLeader_NoShutdown -2025-01-28T14:47:10.249Z [INFO] server-03944325-2272-53cd-dabd-1e894a3c7765: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-03944325-2272-53cd-dabd-1e894a3c7765 Address:03944325-2272-53cd-dabd-1e894a3c7765} {Suffrage:Voter ID:server-e6d775a2-4723-0609-3a27-5fe05c4fd77d Address:e6d775a2-4723-0609-3a27-5fe05c4fd77d} {Suffrage:Voter ID:server-519e4f44-a153-9c68-28fd-d67ac519194f Address:519e4f44-a153-9c68-28fd-d67ac519194f}]" -2025-01-28T14:47:10.250Z [INFO] server-03944325-2272-53cd-dabd-1e894a3c7765: entering follower state: follower="Node at 03944325-2272-53cd-dabd-1e894a3c7765 [Follower]" leader-address= leader-id= -2025-01-28T14:47:10.251Z [INFO] server-e6d775a2-4723-0609-3a27-5fe05c4fd77d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-03944325-2272-53cd-dabd-1e894a3c7765 Address:03944325-2272-53cd-dabd-1e894a3c7765} {Suffrage:Voter ID:server-e6d775a2-4723-0609-3a27-5fe05c4fd77d Address:e6d775a2-4723-0609-3a27-5fe05c4fd77d} {Suffrage:Voter ID:server-519e4f44-a153-9c68-28fd-d67ac519194f Address:519e4f44-a153-9c68-28fd-d67ac519194f}]" -2025-01-28T14:47:10.252Z [INFO] server-e6d775a2-4723-0609-3a27-5fe05c4fd77d: entering follower state: follower="Node at e6d775a2-4723-0609-3a27-5fe05c4fd77d [Follower]" leader-address= leader-id= -2025-01-28T14:47:10.253Z [INFO] server-519e4f44-a153-9c68-28fd-d67ac519194f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-03944325-2272-53cd-dabd-1e894a3c7765 Address:03944325-2272-53cd-dabd-1e894a3c7765} {Suffrage:Voter ID:server-e6d775a2-4723-0609-3a27-5fe05c4fd77d Address:e6d775a2-4723-0609-3a27-5fe05c4fd77d} {Suffrage:Voter ID:server-519e4f44-a153-9c68-28fd-d67ac519194f Address:519e4f44-a153-9c68-28fd-d67ac519194f}]" -2025-01-28T14:47:10.253Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:47:10.253Z [INFO] server-519e4f44-a153-9c68-28fd-d67ac519194f: entering follower state: follower="Node at 519e4f44-a153-9c68-28fd-d67ac519194f [Follower]" leader-address= leader-id= -2025-01-28T14:47:10.309Z [WARN] server-e6d775a2-4723-0609-3a27-5fe05c4fd77d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:10.310Z [INFO] server-e6d775a2-4723-0609-3a27-5fe05c4fd77d: entering candidate state: node="Node at e6d775a2-4723-0609-3a27-5fe05c4fd77d [Candidate]" term=2 -2025-01-28T14:47:10.311Z [INFO] server-e6d775a2-4723-0609-3a27-5fe05c4fd77d: election won: term=2 tally=2 -2025-01-28T14:47:10.311Z [INFO] server-e6d775a2-4723-0609-3a27-5fe05c4fd77d: entering leader state: leader="Node at e6d775a2-4723-0609-3a27-5fe05c4fd77d [Leader]" -2025-01-28T14:47:10.311Z [INFO] server-e6d775a2-4723-0609-3a27-5fe05c4fd77d: added peer, starting replication: peer=server-03944325-2272-53cd-dabd-1e894a3c7765 -2025-01-28T14:47:10.311Z [INFO] server-e6d775a2-4723-0609-3a27-5fe05c4fd77d: added peer, starting replication: peer=server-519e4f44-a153-9c68-28fd-d67ac519194f -2025-01-28T14:47:10.312Z [INFO] server-e6d775a2-4723-0609-3a27-5fe05c4fd77d: pipelining replication: peer="{Voter server-519e4f44-a153-9c68-28fd-d67ac519194f 519e4f44-a153-9c68-28fd-d67ac519194f}" -2025-01-28T14:47:10.313Z [INFO] server-e6d775a2-4723-0609-3a27-5fe05c4fd77d: pipelining replication: peer="{Voter server-03944325-2272-53cd-dabd-1e894a3c7765 03944325-2272-53cd-dabd-1e894a3c7765}" -2025-01-28T14:47:10.416Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:47:10.311385639 +0000 UTC m=+13.394792330 (2 nodes), 57.298701ms from start of poll, 62.620859ms from cluster start. Timeout at 2025-01-28 14:47:10.416431641 +0000 UTC m=+13.499837665, 105.045335ms after stability -2025-01-28T14:47:10.417Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:10.523Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:10.417200447 +0000 UTC m=+13.500606472 (1 nodes), 12.334µs from start of poll, 168.435001ms from cluster start. Timeout at 2025-01-28 14:47:10.522280116 +0000 UTC m=+13.605686140, 105.079668ms after stability -2025-01-28T14:47:10.564Z [INFO] server-e6d775a2-4723-0609-3a27-5fe05c4fd77d: updating configuration: command=RemoveServer server-id=server-e6d775a2-4723-0609-3a27-5fe05c4fd77d server-addr= servers="[{Suffrage:Voter ID:server-03944325-2272-53cd-dabd-1e894a3c7765 Address:03944325-2272-53cd-dabd-1e894a3c7765} {Suffrage:Voter ID:server-519e4f44-a153-9c68-28fd-d67ac519194f Address:519e4f44-a153-9c68-28fd-d67ac519194f}]" -2025-01-28T14:47:10.566Z [INFO] server-e6d775a2-4723-0609-3a27-5fe05c4fd77d: removed ourself, transitioning to follower -2025-01-28T14:47:10.566Z [INFO] server-e6d775a2-4723-0609-3a27-5fe05c4fd77d: entering follower state: follower="Node at e6d775a2-4723-0609-3a27-5fe05c4fd77d [Follower]" leader-address= leader-id= -2025-01-28T14:47:10.567Z [INFO] server-e6d775a2-4723-0609-3a27-5fe05c4fd77d: aborting pipeline replication: peer="{Voter server-03944325-2272-53cd-dabd-1e894a3c7765 03944325-2272-53cd-dabd-1e894a3c7765}" -2025-01-28T14:47:10.567Z [INFO] server-e6d775a2-4723-0609-3a27-5fe05c4fd77d: aborting pipeline replication: peer="{Voter server-519e4f44-a153-9c68-28fd-d67ac519194f 519e4f44-a153-9c68-28fd-d67ac519194f}" -2025-01-28T14:47:10.635Z [WARN] server-519e4f44-a153-9c68-28fd-d67ac519194f: heartbeat timeout reached, starting election: last-leader-addr=e6d775a2-4723-0609-3a27-5fe05c4fd77d last-leader-id=server-e6d775a2-4723-0609-3a27-5fe05c4fd77d -2025-01-28T14:47:10.636Z [INFO] server-519e4f44-a153-9c68-28fd-d67ac519194f: entering candidate state: node="Node at 519e4f44-a153-9c68-28fd-d67ac519194f [Candidate]" term=3 -2025-01-28T14:47:10.636Z [WARN] server-03944325-2272-53cd-dabd-1e894a3c7765: rejecting vote request since we have a leader: from=519e4f44-a153-9c68-28fd-d67ac519194f leader=e6d775a2-4723-0609-3a27-5fe05c4fd77d leader-id=server-e6d775a2-4723-0609-3a27-5fe05c4fd77d -2025-01-28T14:47:10.647Z [WARN] server-03944325-2272-53cd-dabd-1e894a3c7765: heartbeat timeout reached, starting election: last-leader-addr=e6d775a2-4723-0609-3a27-5fe05c4fd77d last-leader-id=server-e6d775a2-4723-0609-3a27-5fe05c4fd77d -2025-01-28T14:47:10.647Z [INFO] server-03944325-2272-53cd-dabd-1e894a3c7765: entering candidate state: node="Node at 03944325-2272-53cd-dabd-1e894a3c7765 [Candidate]" term=3 -2025-01-28T14:47:10.648Z [INFO] server-519e4f44-a153-9c68-28fd-d67ac519194f: duplicate requestVote for same term: term=3 -2025-01-28T14:47:10.661Z [WARN] server-e6d775a2-4723-0609-3a27-5fe05c4fd77d: not part of stable configuration, aborting election -2025-01-28T14:47:10.676Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:10.706Z [WARN] server-519e4f44-a153-9c68-28fd-d67ac519194f: Election timeout reached, restarting election -2025-01-28T14:47:10.707Z [INFO] server-519e4f44-a153-9c68-28fd-d67ac519194f: entering candidate state: node="Node at 519e4f44-a153-9c68-28fd-d67ac519194f [Candidate]" term=4 -2025-01-28T14:47:10.707Z [INFO] server-03944325-2272-53cd-dabd-1e894a3c7765: entering follower state: follower="Node at 03944325-2272-53cd-dabd-1e894a3c7765 [Follower]" leader-address= leader-id= -2025-01-28T14:47:10.708Z [INFO] server-519e4f44-a153-9c68-28fd-d67ac519194f: election won: term=4 tally=2 -2025-01-28T14:47:10.708Z [INFO] server-519e4f44-a153-9c68-28fd-d67ac519194f: entering leader state: leader="Node at 519e4f44-a153-9c68-28fd-d67ac519194f [Leader]" -2025-01-28T14:47:10.708Z [INFO] server-519e4f44-a153-9c68-28fd-d67ac519194f: added peer, starting replication: peer=server-03944325-2272-53cd-dabd-1e894a3c7765 -2025-01-28T14:47:10.710Z [INFO] server-519e4f44-a153-9c68-28fd-d67ac519194f: pipelining replication: peer="{Voter server-03944325-2272-53cd-dabd-1e894a3c7765 03944325-2272-53cd-dabd-1e894a3c7765}" -2025-01-28T14:47:10.814Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:10.709024111 +0000 UTC m=+13.792430469 (1 nodes), 31.853946ms from start of poll, 460.258998ms from cluster start. Timeout at 2025-01-28 14:47:10.814045446 +0000 UTC m=+13.897451137, 105.020668ms after stability -2025-01-28T14:47:10.920Z [INFO] server-519e4f44-a153-9c68-28fd-d67ac519194f: aborting pipeline replication: peer="{Voter server-03944325-2272-53cd-dabd-1e894a3c7765 03944325-2272-53cd-dabd-1e894a3c7765}" ---- PASS: TestRaft_RemoveLeader_NoShutdown (0.69s) +2025-01-28T14:54:11.790Z [INFO] server-0c484a7d-1415-40ba-fe0c-dcafa3dc2e1c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0c484a7d-1415-40ba-fe0c-dcafa3dc2e1c Address:0c484a7d-1415-40ba-fe0c-dcafa3dc2e1c} {Suffrage:Voter ID:server-05fc388a-219d-e8b2-8b93-f540449453f4 Address:05fc388a-219d-e8b2-8b93-f540449453f4} {Suffrage:Voter ID:server-4c6e348c-2d58-77df-adde-6f86d27fa231 Address:4c6e348c-2d58-77df-adde-6f86d27fa231}]" +2025-01-28T14:54:11.790Z [INFO] server-0c484a7d-1415-40ba-fe0c-dcafa3dc2e1c: entering follower state: follower="Node at 0c484a7d-1415-40ba-fe0c-dcafa3dc2e1c [Follower]" leader-address= leader-id= +2025-01-28T14:54:11.790Z [INFO] server-05fc388a-219d-e8b2-8b93-f540449453f4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0c484a7d-1415-40ba-fe0c-dcafa3dc2e1c Address:0c484a7d-1415-40ba-fe0c-dcafa3dc2e1c} {Suffrage:Voter ID:server-05fc388a-219d-e8b2-8b93-f540449453f4 Address:05fc388a-219d-e8b2-8b93-f540449453f4} {Suffrage:Voter ID:server-4c6e348c-2d58-77df-adde-6f86d27fa231 Address:4c6e348c-2d58-77df-adde-6f86d27fa231}]" +2025-01-28T14:54:11.790Z [INFO] server-05fc388a-219d-e8b2-8b93-f540449453f4: entering follower state: follower="Node at 05fc388a-219d-e8b2-8b93-f540449453f4 [Follower]" leader-address= leader-id= +2025-01-28T14:54:11.791Z [INFO] server-4c6e348c-2d58-77df-adde-6f86d27fa231: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0c484a7d-1415-40ba-fe0c-dcafa3dc2e1c Address:0c484a7d-1415-40ba-fe0c-dcafa3dc2e1c} {Suffrage:Voter ID:server-05fc388a-219d-e8b2-8b93-f540449453f4 Address:05fc388a-219d-e8b2-8b93-f540449453f4} {Suffrage:Voter ID:server-4c6e348c-2d58-77df-adde-6f86d27fa231 Address:4c6e348c-2d58-77df-adde-6f86d27fa231}]" +2025-01-28T14:54:11.791Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:54:11.791Z [INFO] server-4c6e348c-2d58-77df-adde-6f86d27fa231: entering follower state: follower="Node at 4c6e348c-2d58-77df-adde-6f86d27fa231 [Follower]" leader-address= leader-id= +2025-01-28T14:54:11.859Z [WARN] server-05fc388a-219d-e8b2-8b93-f540449453f4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:11.859Z [INFO] server-05fc388a-219d-e8b2-8b93-f540449453f4: entering candidate state: node="Node at 05fc388a-219d-e8b2-8b93-f540449453f4 [Candidate]" term=2 +2025-01-28T14:54:11.859Z [INFO] server-05fc388a-219d-e8b2-8b93-f540449453f4: election won: term=2 tally=2 +2025-01-28T14:54:11.859Z [INFO] server-05fc388a-219d-e8b2-8b93-f540449453f4: entering leader state: leader="Node at 05fc388a-219d-e8b2-8b93-f540449453f4 [Leader]" +2025-01-28T14:54:11.859Z [INFO] server-05fc388a-219d-e8b2-8b93-f540449453f4: added peer, starting replication: peer=server-0c484a7d-1415-40ba-fe0c-dcafa3dc2e1c +2025-01-28T14:54:11.859Z [INFO] server-05fc388a-219d-e8b2-8b93-f540449453f4: added peer, starting replication: peer=server-4c6e348c-2d58-77df-adde-6f86d27fa231 +2025-01-28T14:54:11.860Z [INFO] server-05fc388a-219d-e8b2-8b93-f540449453f4: pipelining replication: peer="{Voter server-0c484a7d-1415-40ba-fe0c-dcafa3dc2e1c 0c484a7d-1415-40ba-fe0c-dcafa3dc2e1c}" +2025-01-28T14:54:11.860Z [INFO] server-05fc388a-219d-e8b2-8b93-f540449453f4: pipelining replication: peer="{Voter server-4c6e348c-2d58-77df-adde-6f86d27fa231 4c6e348c-2d58-77df-adde-6f86d27fa231}" +2025-01-28T14:54:11.965Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:54:11.85982807 +0000 UTC m=+37.534318050 (2 nodes), 68.629645ms from start of poll, 69.697145ms from cluster start. Timeout at 2025-01-28 14:54:11.964833049 +0000 UTC m=+37.639323089, 105.005039ms after stability +2025-01-28T14:54:11.965Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:12.070Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:11.965812072 +0000 UTC m=+37.640302072 (1 nodes), 2.46µs from start of poll, 175.681167ms from cluster start. Timeout at 2025-01-28 14:54:12.07083607 +0000 UTC m=+37.745326070, 105.023998ms after stability +2025-01-28T14:54:12.077Z [INFO] server-05fc388a-219d-e8b2-8b93-f540449453f4: updating configuration: command=RemoveServer server-id=server-05fc388a-219d-e8b2-8b93-f540449453f4 server-addr= servers="[{Suffrage:Voter ID:server-0c484a7d-1415-40ba-fe0c-dcafa3dc2e1c Address:0c484a7d-1415-40ba-fe0c-dcafa3dc2e1c} {Suffrage:Voter ID:server-4c6e348c-2d58-77df-adde-6f86d27fa231 Address:4c6e348c-2d58-77df-adde-6f86d27fa231}]" +2025-01-28T14:54:12.077Z [INFO] server-05fc388a-219d-e8b2-8b93-f540449453f4: removed ourself, transitioning to follower +2025-01-28T14:54:12.077Z [INFO] server-05fc388a-219d-e8b2-8b93-f540449453f4: entering follower state: follower="Node at 05fc388a-219d-e8b2-8b93-f540449453f4 [Follower]" leader-address= leader-id= +2025-01-28T14:54:12.077Z [INFO] server-05fc388a-219d-e8b2-8b93-f540449453f4: aborting pipeline replication: peer="{Voter server-4c6e348c-2d58-77df-adde-6f86d27fa231 4c6e348c-2d58-77df-adde-6f86d27fa231}" +2025-01-28T14:54:12.077Z [INFO] server-05fc388a-219d-e8b2-8b93-f540449453f4: aborting pipeline replication: peer="{Voter server-0c484a7d-1415-40ba-fe0c-dcafa3dc2e1c 0c484a7d-1415-40ba-fe0c-dcafa3dc2e1c}" +2025-01-28T14:54:12.141Z [WARN] server-05fc388a-219d-e8b2-8b93-f540449453f4: not part of stable configuration, aborting election +2025-01-28T14:54:12.152Z [WARN] server-4c6e348c-2d58-77df-adde-6f86d27fa231: heartbeat timeout reached, starting election: last-leader-addr=05fc388a-219d-e8b2-8b93-f540449453f4 last-leader-id=server-05fc388a-219d-e8b2-8b93-f540449453f4 +2025-01-28T14:54:12.152Z [INFO] server-4c6e348c-2d58-77df-adde-6f86d27fa231: entering candidate state: node="Node at 4c6e348c-2d58-77df-adde-6f86d27fa231 [Candidate]" term=3 +2025-01-28T14:54:12.152Z [WARN] server-0c484a7d-1415-40ba-fe0c-dcafa3dc2e1c: rejecting vote request since we have a leader: from=4c6e348c-2d58-77df-adde-6f86d27fa231 leader=05fc388a-219d-e8b2-8b93-f540449453f4 leader-id=server-05fc388a-219d-e8b2-8b93-f540449453f4 +2025-01-28T14:54:12.180Z [WARN] server-0c484a7d-1415-40ba-fe0c-dcafa3dc2e1c: heartbeat timeout reached, starting election: last-leader-addr=05fc388a-219d-e8b2-8b93-f540449453f4 last-leader-id=server-05fc388a-219d-e8b2-8b93-f540449453f4 +2025-01-28T14:54:12.180Z [INFO] server-0c484a7d-1415-40ba-fe0c-dcafa3dc2e1c: entering candidate state: node="Node at 0c484a7d-1415-40ba-fe0c-dcafa3dc2e1c [Candidate]" term=3 +2025-01-28T14:54:12.181Z [INFO] server-4c6e348c-2d58-77df-adde-6f86d27fa231: duplicate requestVote for same term: term=3 +2025-01-28T14:54:12.184Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:12.238Z [WARN] server-0c484a7d-1415-40ba-fe0c-dcafa3dc2e1c: Election timeout reached, restarting election +2025-01-28T14:54:12.238Z [INFO] server-0c484a7d-1415-40ba-fe0c-dcafa3dc2e1c: entering candidate state: node="Node at 0c484a7d-1415-40ba-fe0c-dcafa3dc2e1c [Candidate]" term=4 +2025-01-28T14:54:12.238Z [INFO] server-4c6e348c-2d58-77df-adde-6f86d27fa231: entering follower state: follower="Node at 4c6e348c-2d58-77df-adde-6f86d27fa231 [Follower]" leader-address= leader-id= +2025-01-28T14:54:12.239Z [INFO] server-0c484a7d-1415-40ba-fe0c-dcafa3dc2e1c: election won: term=4 tally=2 +2025-01-28T14:54:12.239Z [INFO] server-0c484a7d-1415-40ba-fe0c-dcafa3dc2e1c: entering leader state: leader="Node at 0c484a7d-1415-40ba-fe0c-dcafa3dc2e1c [Leader]" +2025-01-28T14:54:12.239Z [INFO] server-0c484a7d-1415-40ba-fe0c-dcafa3dc2e1c: added peer, starting replication: peer=server-4c6e348c-2d58-77df-adde-6f86d27fa231 +2025-01-28T14:54:12.239Z [INFO] server-0c484a7d-1415-40ba-fe0c-dcafa3dc2e1c: pipelining replication: peer="{Voter server-4c6e348c-2d58-77df-adde-6f86d27fa231 4c6e348c-2d58-77df-adde-6f86d27fa231}" +2025-01-28T14:54:12.345Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:12.239061574 +0000 UTC m=+37.913551534 (1 nodes), 54.925039ms from start of poll, 448.930629ms from cluster start. Timeout at 2025-01-28 14:54:12.344068994 +0000 UTC m=+38.018559034, 105.0075ms after stability +2025-01-28T14:54:12.450Z [INFO] server-0c484a7d-1415-40ba-fe0c-dcafa3dc2e1c: aborting pipeline replication: peer="{Voter server-4c6e348c-2d58-77df-adde-6f86d27fa231 4c6e348c-2d58-77df-adde-6f86d27fa231}" +--- PASS: TestRaft_RemoveLeader_NoShutdown (0.66s) === RUN TestRaft_RemoveFollower_SplitCluster -2025-01-28T14:47:10.940Z [INFO] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91 Address:936c0f23-73e1-513b-2c9e-6ce7e97dbc91} {Suffrage:Voter ID:server-c7a92434-7a92-4fd8-b335-7567fef318aa Address:c7a92434-7a92-4fd8-b335-7567fef318aa} {Suffrage:Voter ID:server-fc3157a1-e38e-1410-4693-3b06129397c4 Address:fc3157a1-e38e-1410-4693-3b06129397c4} {Suffrage:Voter ID:server-6b6349ac-b54a-04f0-d1ad-b695097b35c5 Address:6b6349ac-b54a-04f0-d1ad-b695097b35c5}]" -2025-01-28T14:47:10.941Z [INFO] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: entering follower state: follower="Node at 936c0f23-73e1-513b-2c9e-6ce7e97dbc91 [Follower]" leader-address= leader-id= -2025-01-28T14:47:10.942Z [INFO] server-c7a92434-7a92-4fd8-b335-7567fef318aa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91 Address:936c0f23-73e1-513b-2c9e-6ce7e97dbc91} {Suffrage:Voter ID:server-c7a92434-7a92-4fd8-b335-7567fef318aa Address:c7a92434-7a92-4fd8-b335-7567fef318aa} {Suffrage:Voter ID:server-fc3157a1-e38e-1410-4693-3b06129397c4 Address:fc3157a1-e38e-1410-4693-3b06129397c4} {Suffrage:Voter ID:server-6b6349ac-b54a-04f0-d1ad-b695097b35c5 Address:6b6349ac-b54a-04f0-d1ad-b695097b35c5}]" -2025-01-28T14:47:10.943Z [INFO] server-c7a92434-7a92-4fd8-b335-7567fef318aa: entering follower state: follower="Node at c7a92434-7a92-4fd8-b335-7567fef318aa [Follower]" leader-address= leader-id= -2025-01-28T14:47:10.943Z [INFO] server-fc3157a1-e38e-1410-4693-3b06129397c4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91 Address:936c0f23-73e1-513b-2c9e-6ce7e97dbc91} {Suffrage:Voter ID:server-c7a92434-7a92-4fd8-b335-7567fef318aa Address:c7a92434-7a92-4fd8-b335-7567fef318aa} {Suffrage:Voter ID:server-fc3157a1-e38e-1410-4693-3b06129397c4 Address:fc3157a1-e38e-1410-4693-3b06129397c4} {Suffrage:Voter ID:server-6b6349ac-b54a-04f0-d1ad-b695097b35c5 Address:6b6349ac-b54a-04f0-d1ad-b695097b35c5}]" -2025-01-28T14:47:10.944Z [INFO] server-fc3157a1-e38e-1410-4693-3b06129397c4: entering follower state: follower="Node at fc3157a1-e38e-1410-4693-3b06129397c4 [Follower]" leader-address= leader-id= -2025-01-28T14:47:10.945Z [INFO] server-6b6349ac-b54a-04f0-d1ad-b695097b35c5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91 Address:936c0f23-73e1-513b-2c9e-6ce7e97dbc91} {Suffrage:Voter ID:server-c7a92434-7a92-4fd8-b335-7567fef318aa Address:c7a92434-7a92-4fd8-b335-7567fef318aa} {Suffrage:Voter ID:server-fc3157a1-e38e-1410-4693-3b06129397c4 Address:fc3157a1-e38e-1410-4693-3b06129397c4} {Suffrage:Voter ID:server-6b6349ac-b54a-04f0-d1ad-b695097b35c5 Address:6b6349ac-b54a-04f0-d1ad-b695097b35c5}]" -2025-01-28T14:47:10.946Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:10.946Z [INFO] server-6b6349ac-b54a-04f0-d1ad-b695097b35c5: entering follower state: follower="Node at 6b6349ac-b54a-04f0-d1ad-b695097b35c5 [Follower]" leader-address= leader-id= -2025-01-28T14:47:11.003Z [WARN] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:11.004Z [INFO] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: entering candidate state: node="Node at 936c0f23-73e1-513b-2c9e-6ce7e97dbc91 [Candidate]" term=2 -2025-01-28T14:47:11.005Z [INFO] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: election won: term=2 tally=3 -2025-01-28T14:47:11.005Z [INFO] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: entering leader state: leader="Node at 936c0f23-73e1-513b-2c9e-6ce7e97dbc91 [Leader]" -2025-01-28T14:47:11.005Z [INFO] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: added peer, starting replication: peer=server-c7a92434-7a92-4fd8-b335-7567fef318aa -2025-01-28T14:47:11.005Z [INFO] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: added peer, starting replication: peer=server-fc3157a1-e38e-1410-4693-3b06129397c4 -2025-01-28T14:47:11.006Z [INFO] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: added peer, starting replication: peer=server-6b6349ac-b54a-04f0-d1ad-b695097b35c5 -2025-01-28T14:47:11.007Z [INFO] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: pipelining replication: peer="{Voter server-c7a92434-7a92-4fd8-b335-7567fef318aa c7a92434-7a92-4fd8-b335-7567fef318aa}" -2025-01-28T14:47:11.007Z [INFO] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: pipelining replication: peer="{Voter server-fc3157a1-e38e-1410-4693-3b06129397c4 fc3157a1-e38e-1410-4693-3b06129397c4}" -2025-01-28T14:47:11.008Z [INFO] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: pipelining replication: peer="{Voter server-6b6349ac-b54a-04f0-d1ad-b695097b35c5 6b6349ac-b54a-04f0-d1ad-b695097b35c5}" -2025-01-28T14:47:11.112Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:11.005818922 +0000 UTC m=+14.089225280 (1 nodes), 59.534434ms from start of poll, 66.233966ms from cluster start. Timeout at 2025-01-28 14:47:11.110839923 +0000 UTC m=+14.194245947, 105.020667ms after stability -2025-01-28T14:47:11.217Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:47:11.323Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:47:11.218204228 +0000 UTC m=+14.301610252 (3 nodes), 18µs from start of poll, 278.618938ms from cluster start. Timeout at 2025-01-28 14:47:11.323309564 +0000 UTC m=+14.406715589, 105.105337ms after stability -2025-01-28T14:47:11.324Z [INFO] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: updating configuration: command=RemoveServer server-id=server-6b6349ac-b54a-04f0-d1ad-b695097b35c5 server-addr= servers="[{Suffrage:Voter ID:server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91 Address:936c0f23-73e1-513b-2c9e-6ce7e97dbc91} {Suffrage:Voter ID:server-c7a92434-7a92-4fd8-b335-7567fef318aa Address:c7a92434-7a92-4fd8-b335-7567fef318aa} {Suffrage:Voter ID:server-fc3157a1-e38e-1410-4693-3b06129397c4 Address:fc3157a1-e38e-1410-4693-3b06129397c4}]" -2025-01-28T14:47:11.325Z [ERROR] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: failed to pipeline appendEntries: peer="{Voter server-fc3157a1-e38e-1410-4693-3b06129397c4 fc3157a1-e38e-1410-4693-3b06129397c4}" error="append pipeline closed" -2025-01-28T14:47:11.325Z [INFO] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: removed peer, stopping replication: peer=server-6b6349ac-b54a-04f0-d1ad-b695097b35c5 last-index=3 -2025-01-28T14:47:11.325Z [ERROR] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: failed to pipeline appendEntries: peer="{Voter server-c7a92434-7a92-4fd8-b335-7567fef318aa c7a92434-7a92-4fd8-b335-7567fef318aa}" error="append pipeline closed" -2025-01-28T14:47:11.325Z [INFO] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: aborting pipeline replication: peer="{Voter server-c7a92434-7a92-4fd8-b335-7567fef318aa c7a92434-7a92-4fd8-b335-7567fef318aa}" -2025-01-28T14:47:11.325Z [INFO] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: aborting pipeline replication: peer="{Voter server-6b6349ac-b54a-04f0-d1ad-b695097b35c5 6b6349ac-b54a-04f0-d1ad-b695097b35c5}" -2025-01-28T14:47:11.325Z [INFO] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: aborting pipeline replication: peer="{Voter server-fc3157a1-e38e-1410-4693-3b06129397c4 fc3157a1-e38e-1410-4693-3b06129397c4}" -2025-01-28T14:47:11.326Z [ERROR] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: failed to appendEntries to: peer="{Voter server-fc3157a1-e38e-1410-4693-3b06129397c4 fc3157a1-e38e-1410-4693-3b06129397c4}" error="failed to connect to peer: fc3157a1-e38e-1410-4693-3b06129397c4" -2025-01-28T14:47:11.328Z [ERROR] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: failed to heartbeat to: peer=fc3157a1-e38e-1410-4693-3b06129397c4 backoff time=10ms error="failed to connect to peer: fc3157a1-e38e-1410-4693-3b06129397c4" -2025-01-28T14:47:11.333Z [ERROR] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: failed to heartbeat to: peer=c7a92434-7a92-4fd8-b335-7567fef318aa backoff time=10ms error="failed to connect to peer: c7a92434-7a92-4fd8-b335-7567fef318aa" -2025-01-28T14:47:11.335Z [ERROR] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: failed to appendEntries to: peer="{Voter server-c7a92434-7a92-4fd8-b335-7567fef318aa c7a92434-7a92-4fd8-b335-7567fef318aa}" error="failed to connect to peer: c7a92434-7a92-4fd8-b335-7567fef318aa" -2025-01-28T14:47:11.343Z [ERROR] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: failed to appendEntries to: peer="{Voter server-fc3157a1-e38e-1410-4693-3b06129397c4 fc3157a1-e38e-1410-4693-3b06129397c4}" error="failed to connect to peer: fc3157a1-e38e-1410-4693-3b06129397c4" -2025-01-28T14:47:11.345Z [ERROR] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: failed to heartbeat to: peer=fc3157a1-e38e-1410-4693-3b06129397c4 backoff time=10ms error="failed to connect to peer: fc3157a1-e38e-1410-4693-3b06129397c4" -2025-01-28T14:47:11.351Z [ERROR] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: failed to heartbeat to: peer=c7a92434-7a92-4fd8-b335-7567fef318aa backoff time=10ms error="failed to connect to peer: c7a92434-7a92-4fd8-b335-7567fef318aa" -2025-01-28T14:47:11.354Z [ERROR] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: failed to appendEntries to: peer="{Voter server-c7a92434-7a92-4fd8-b335-7567fef318aa c7a92434-7a92-4fd8-b335-7567fef318aa}" error="failed to connect to peer: c7a92434-7a92-4fd8-b335-7567fef318aa" -2025-01-28T14:47:11.360Z [ERROR] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: failed to appendEntries to: peer="{Voter server-fc3157a1-e38e-1410-4693-3b06129397c4 fc3157a1-e38e-1410-4693-3b06129397c4}" error="failed to connect to peer: fc3157a1-e38e-1410-4693-3b06129397c4" -2025-01-28T14:47:11.362Z [ERROR] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: failed to heartbeat to: peer=fc3157a1-e38e-1410-4693-3b06129397c4 backoff time=10ms error="failed to connect to peer: fc3157a1-e38e-1410-4693-3b06129397c4" -2025-01-28T14:47:11.370Z [WARN] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: failed to contact: server-id=server-fc3157a1-e38e-1410-4693-3b06129397c4 time=51.244855ms -2025-01-28T14:47:11.371Z [ERROR] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: failed to appendEntries to: peer="{Voter server-c7a92434-7a92-4fd8-b335-7567fef318aa c7a92434-7a92-4fd8-b335-7567fef318aa}" error="failed to connect to peer: c7a92434-7a92-4fd8-b335-7567fef318aa" -2025-01-28T14:47:11.373Z [ERROR] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: failed to heartbeat to: peer=c7a92434-7a92-4fd8-b335-7567fef318aa backoff time=10ms error="failed to connect to peer: c7a92434-7a92-4fd8-b335-7567fef318aa" -2025-01-28T14:47:11.378Z [WARN] server-c7a92434-7a92-4fd8-b335-7567fef318aa: heartbeat timeout reached, starting election: last-leader-addr=936c0f23-73e1-513b-2c9e-6ce7e97dbc91 last-leader-id=server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91 -2025-01-28T14:47:11.378Z [INFO] server-c7a92434-7a92-4fd8-b335-7567fef318aa: entering candidate state: node="Node at c7a92434-7a92-4fd8-b335-7567fef318aa [Candidate]" term=3 -2025-01-28T14:47:11.378Z [ERROR] server-c7a92434-7a92-4fd8-b335-7567fef318aa: failed to make requestVote RPC: target="{Voter server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91 936c0f23-73e1-513b-2c9e-6ce7e97dbc91}" error="failed to connect to peer: 936c0f23-73e1-513b-2c9e-6ce7e97dbc91" term=3 -2025-01-28T14:47:11.379Z [WARN] server-fc3157a1-e38e-1410-4693-3b06129397c4: rejecting vote request since we have a leader: from=c7a92434-7a92-4fd8-b335-7567fef318aa leader=936c0f23-73e1-513b-2c9e-6ce7e97dbc91 leader-id=server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91 -2025-01-28T14:47:11.378Z [ERROR] server-c7a92434-7a92-4fd8-b335-7567fef318aa: failed to make requestVote RPC: target="{Voter server-6b6349ac-b54a-04f0-d1ad-b695097b35c5 6b6349ac-b54a-04f0-d1ad-b695097b35c5}" error="failed to connect to peer: 6b6349ac-b54a-04f0-d1ad-b695097b35c5" term=3 -2025-01-28T14:47:11.381Z [WARN] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: failed to contact: server-id=server-c7a92434-7a92-4fd8-b335-7567fef318aa time=57.708047ms -2025-01-28T14:47:11.381Z [WARN] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: failed to contact: server-id=server-fc3157a1-e38e-1410-4693-3b06129397c4 time=61.863837ms -2025-01-28T14:47:11.381Z [WARN] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: failed to contact quorum of nodes, stepping down -2025-01-28T14:47:11.382Z [INFO] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: entering follower state: follower="Node at 936c0f23-73e1-513b-2c9e-6ce7e97dbc91 [Follower]" leader-address= leader-id= -2025-01-28T14:47:11.382Z [ERROR] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: failed to heartbeat to: peer=fc3157a1-e38e-1410-4693-3b06129397c4 backoff time=20ms error="failed to connect to peer: fc3157a1-e38e-1410-4693-3b06129397c4" -2025-01-28T14:47:11.382Z [ERROR] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: failed to appendEntries to: peer="{Voter server-c7a92434-7a92-4fd8-b335-7567fef318aa c7a92434-7a92-4fd8-b335-7567fef318aa}" error="failed to connect to peer: c7a92434-7a92-4fd8-b335-7567fef318aa" -2025-01-28T14:47:11.382Z [ERROR] server-936c0f23-73e1-513b-2c9e-6ce7e97dbc91: failed to appendEntries to: peer="{Voter server-fc3157a1-e38e-1410-4693-3b06129397c4 fc3157a1-e38e-1410-4693-3b06129397c4}" error="failed to connect to peer: fc3157a1-e38e-1410-4693-3b06129397c4" ---- PASS: TestRaft_RemoveFollower_SplitCluster (0.46s) +2025-01-28T14:54:12.456Z [INFO] server-42825a80-bd55-5310-334b-d02866c2b71a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-42825a80-bd55-5310-334b-d02866c2b71a Address:42825a80-bd55-5310-334b-d02866c2b71a} {Suffrage:Voter ID:server-e516185a-dd82-97e2-fdae-73975d6c1ed0 Address:e516185a-dd82-97e2-fdae-73975d6c1ed0} {Suffrage:Voter ID:server-e27b9610-e5d8-7241-0ebf-ec53c025a26e Address:e27b9610-e5d8-7241-0ebf-ec53c025a26e} {Suffrage:Voter ID:server-d44ee3a7-d67d-8f31-b14c-1bb4d7a5b927 Address:d44ee3a7-d67d-8f31-b14c-1bb4d7a5b927}]" +2025-01-28T14:54:12.456Z [INFO] server-42825a80-bd55-5310-334b-d02866c2b71a: entering follower state: follower="Node at 42825a80-bd55-5310-334b-d02866c2b71a [Follower]" leader-address= leader-id= +2025-01-28T14:54:12.456Z [INFO] server-e516185a-dd82-97e2-fdae-73975d6c1ed0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-42825a80-bd55-5310-334b-d02866c2b71a Address:42825a80-bd55-5310-334b-d02866c2b71a} {Suffrage:Voter ID:server-e516185a-dd82-97e2-fdae-73975d6c1ed0 Address:e516185a-dd82-97e2-fdae-73975d6c1ed0} {Suffrage:Voter ID:server-e27b9610-e5d8-7241-0ebf-ec53c025a26e Address:e27b9610-e5d8-7241-0ebf-ec53c025a26e} {Suffrage:Voter ID:server-d44ee3a7-d67d-8f31-b14c-1bb4d7a5b927 Address:d44ee3a7-d67d-8f31-b14c-1bb4d7a5b927}]" +2025-01-28T14:54:12.456Z [INFO] server-e27b9610-e5d8-7241-0ebf-ec53c025a26e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-42825a80-bd55-5310-334b-d02866c2b71a Address:42825a80-bd55-5310-334b-d02866c2b71a} {Suffrage:Voter ID:server-e516185a-dd82-97e2-fdae-73975d6c1ed0 Address:e516185a-dd82-97e2-fdae-73975d6c1ed0} {Suffrage:Voter ID:server-e27b9610-e5d8-7241-0ebf-ec53c025a26e Address:e27b9610-e5d8-7241-0ebf-ec53c025a26e} {Suffrage:Voter ID:server-d44ee3a7-d67d-8f31-b14c-1bb4d7a5b927 Address:d44ee3a7-d67d-8f31-b14c-1bb4d7a5b927}]" +2025-01-28T14:54:12.457Z [INFO] server-d44ee3a7-d67d-8f31-b14c-1bb4d7a5b927: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-42825a80-bd55-5310-334b-d02866c2b71a Address:42825a80-bd55-5310-334b-d02866c2b71a} {Suffrage:Voter ID:server-e516185a-dd82-97e2-fdae-73975d6c1ed0 Address:e516185a-dd82-97e2-fdae-73975d6c1ed0} {Suffrage:Voter ID:server-e27b9610-e5d8-7241-0ebf-ec53c025a26e Address:e27b9610-e5d8-7241-0ebf-ec53c025a26e} {Suffrage:Voter ID:server-d44ee3a7-d67d-8f31-b14c-1bb4d7a5b927 Address:d44ee3a7-d67d-8f31-b14c-1bb4d7a5b927}]" +2025-01-28T14:54:12.457Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:12.457Z [INFO] server-e516185a-dd82-97e2-fdae-73975d6c1ed0: entering follower state: follower="Node at e516185a-dd82-97e2-fdae-73975d6c1ed0 [Follower]" leader-address= leader-id= +2025-01-28T14:54:12.457Z [INFO] server-e27b9610-e5d8-7241-0ebf-ec53c025a26e: entering follower state: follower="Node at e27b9610-e5d8-7241-0ebf-ec53c025a26e [Follower]" leader-address= leader-id= +2025-01-28T14:54:12.457Z [INFO] server-d44ee3a7-d67d-8f31-b14c-1bb4d7a5b927: entering follower state: follower="Node at d44ee3a7-d67d-8f31-b14c-1bb4d7a5b927 [Follower]" leader-address= leader-id= +2025-01-28T14:54:12.515Z [WARN] server-42825a80-bd55-5310-334b-d02866c2b71a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:12.515Z [INFO] server-42825a80-bd55-5310-334b-d02866c2b71a: entering candidate state: node="Node at 42825a80-bd55-5310-334b-d02866c2b71a [Candidate]" term=2 +2025-01-28T14:54:12.516Z [INFO] server-42825a80-bd55-5310-334b-d02866c2b71a: election won: term=2 tally=3 +2025-01-28T14:54:12.516Z [INFO] server-42825a80-bd55-5310-334b-d02866c2b71a: entering leader state: leader="Node at 42825a80-bd55-5310-334b-d02866c2b71a [Leader]" +2025-01-28T14:54:12.516Z [INFO] server-42825a80-bd55-5310-334b-d02866c2b71a: added peer, starting replication: peer=server-e516185a-dd82-97e2-fdae-73975d6c1ed0 +2025-01-28T14:54:12.516Z [INFO] server-42825a80-bd55-5310-334b-d02866c2b71a: added peer, starting replication: peer=server-e27b9610-e5d8-7241-0ebf-ec53c025a26e +2025-01-28T14:54:12.516Z [INFO] server-42825a80-bd55-5310-334b-d02866c2b71a: added peer, starting replication: peer=server-d44ee3a7-d67d-8f31-b14c-1bb4d7a5b927 +2025-01-28T14:54:12.516Z [INFO] server-42825a80-bd55-5310-334b-d02866c2b71a: pipelining replication: peer="{Voter server-e516185a-dd82-97e2-fdae-73975d6c1ed0 e516185a-dd82-97e2-fdae-73975d6c1ed0}" +2025-01-28T14:54:12.516Z [INFO] server-42825a80-bd55-5310-334b-d02866c2b71a: pipelining replication: peer="{Voter server-e27b9610-e5d8-7241-0ebf-ec53c025a26e e27b9610-e5d8-7241-0ebf-ec53c025a26e}" +2025-01-28T14:54:12.516Z [INFO] server-42825a80-bd55-5310-334b-d02866c2b71a: pipelining replication: peer="{Voter server-d44ee3a7-d67d-8f31-b14c-1bb4d7a5b927 d44ee3a7-d67d-8f31-b14c-1bb4d7a5b927}" +2025-01-28T14:54:12.621Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:12.516322237 +0000 UTC m=+38.190812217 (1 nodes), 59.163099ms from start of poll, 60.479894ms from cluster start. Timeout at 2025-01-28 14:54:12.621335176 +0000 UTC m=+38.295825156, 105.012939ms after stability +2025-01-28T14:54:12.727Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:54:12.832Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:54:12.727272536 +0000 UTC m=+38.401762516 (3 nodes), 2.9µs from start of poll, 271.430193ms from cluster start. Timeout at 2025-01-28 14:54:12.832321711 +0000 UTC m=+38.506811711, 105.049195ms after stability +2025-01-28T14:54:12.832Z [INFO] server-42825a80-bd55-5310-334b-d02866c2b71a: updating configuration: command=RemoveServer server-id=server-d44ee3a7-d67d-8f31-b14c-1bb4d7a5b927 server-addr= servers="[{Suffrage:Voter ID:server-42825a80-bd55-5310-334b-d02866c2b71a Address:42825a80-bd55-5310-334b-d02866c2b71a} {Suffrage:Voter ID:server-e516185a-dd82-97e2-fdae-73975d6c1ed0 Address:e516185a-dd82-97e2-fdae-73975d6c1ed0} {Suffrage:Voter ID:server-e27b9610-e5d8-7241-0ebf-ec53c025a26e Address:e27b9610-e5d8-7241-0ebf-ec53c025a26e}]" +2025-01-28T14:54:12.832Z [INFO] server-42825a80-bd55-5310-334b-d02866c2b71a: removed peer, stopping replication: peer=server-d44ee3a7-d67d-8f31-b14c-1bb4d7a5b927 last-index=3 +2025-01-28T14:54:12.833Z [INFO] server-42825a80-bd55-5310-334b-d02866c2b71a: aborting pipeline replication: peer="{Voter server-d44ee3a7-d67d-8f31-b14c-1bb4d7a5b927 d44ee3a7-d67d-8f31-b14c-1bb4d7a5b927}" +2025-01-28T14:54:12.832Z [ERROR] server-42825a80-bd55-5310-334b-d02866c2b71a: failed to pipeline appendEntries: peer="{Voter server-e516185a-dd82-97e2-fdae-73975d6c1ed0 e516185a-dd82-97e2-fdae-73975d6c1ed0}" error="append pipeline closed" +2025-01-28T14:54:12.833Z [INFO] server-42825a80-bd55-5310-334b-d02866c2b71a: aborting pipeline replication: peer="{Voter server-e516185a-dd82-97e2-fdae-73975d6c1ed0 e516185a-dd82-97e2-fdae-73975d6c1ed0}" +2025-01-28T14:54:12.833Z [ERROR] server-42825a80-bd55-5310-334b-d02866c2b71a: failed to pipeline appendEntries: peer="{Voter server-e27b9610-e5d8-7241-0ebf-ec53c025a26e e27b9610-e5d8-7241-0ebf-ec53c025a26e}" error="append pipeline closed" +2025-01-28T14:54:12.833Z [INFO] server-42825a80-bd55-5310-334b-d02866c2b71a: aborting pipeline replication: peer="{Voter server-e27b9610-e5d8-7241-0ebf-ec53c025a26e e27b9610-e5d8-7241-0ebf-ec53c025a26e}" +2025-01-28T14:54:12.833Z [ERROR] server-42825a80-bd55-5310-334b-d02866c2b71a: failed to appendEntries to: peer="{Voter server-e516185a-dd82-97e2-fdae-73975d6c1ed0 e516185a-dd82-97e2-fdae-73975d6c1ed0}" error="failed to connect to peer: e516185a-dd82-97e2-fdae-73975d6c1ed0" +2025-01-28T14:54:12.837Z [ERROR] server-42825a80-bd55-5310-334b-d02866c2b71a: failed to heartbeat to: peer=e516185a-dd82-97e2-fdae-73975d6c1ed0 backoff time=10ms error="failed to connect to peer: e516185a-dd82-97e2-fdae-73975d6c1ed0" +2025-01-28T14:54:12.841Z [ERROR] server-42825a80-bd55-5310-334b-d02866c2b71a: failed to heartbeat to: peer=e27b9610-e5d8-7241-0ebf-ec53c025a26e backoff time=10ms error="failed to connect to peer: e27b9610-e5d8-7241-0ebf-ec53c025a26e" +2025-01-28T14:54:12.843Z [ERROR] server-42825a80-bd55-5310-334b-d02866c2b71a: failed to appendEntries to: peer="{Voter server-e27b9610-e5d8-7241-0ebf-ec53c025a26e e27b9610-e5d8-7241-0ebf-ec53c025a26e}" error="failed to connect to peer: e27b9610-e5d8-7241-0ebf-ec53c025a26e" +2025-01-28T14:54:12.849Z [ERROR] server-42825a80-bd55-5310-334b-d02866c2b71a: failed to appendEntries to: peer="{Voter server-e516185a-dd82-97e2-fdae-73975d6c1ed0 e516185a-dd82-97e2-fdae-73975d6c1ed0}" error="failed to connect to peer: e516185a-dd82-97e2-fdae-73975d6c1ed0" +2025-01-28T14:54:12.856Z [ERROR] server-42825a80-bd55-5310-334b-d02866c2b71a: failed to heartbeat to: peer=e516185a-dd82-97e2-fdae-73975d6c1ed0 backoff time=10ms error="failed to connect to peer: e516185a-dd82-97e2-fdae-73975d6c1ed0" +2025-01-28T14:54:12.859Z [ERROR] server-42825a80-bd55-5310-334b-d02866c2b71a: failed to heartbeat to: peer=e27b9610-e5d8-7241-0ebf-ec53c025a26e backoff time=10ms error="failed to connect to peer: e27b9610-e5d8-7241-0ebf-ec53c025a26e" +2025-01-28T14:54:12.861Z [ERROR] server-42825a80-bd55-5310-334b-d02866c2b71a: failed to appendEntries to: peer="{Voter server-e27b9610-e5d8-7241-0ebf-ec53c025a26e e27b9610-e5d8-7241-0ebf-ec53c025a26e}" error="failed to connect to peer: e27b9610-e5d8-7241-0ebf-ec53c025a26e" +2025-01-28T14:54:12.869Z [ERROR] server-42825a80-bd55-5310-334b-d02866c2b71a: failed to appendEntries to: peer="{Voter server-e516185a-dd82-97e2-fdae-73975d6c1ed0 e516185a-dd82-97e2-fdae-73975d6c1ed0}" error="failed to connect to peer: e516185a-dd82-97e2-fdae-73975d6c1ed0" +2025-01-28T14:54:12.876Z [ERROR] server-42825a80-bd55-5310-334b-d02866c2b71a: failed to heartbeat to: peer=e516185a-dd82-97e2-fdae-73975d6c1ed0 backoff time=10ms error="failed to connect to peer: e516185a-dd82-97e2-fdae-73975d6c1ed0" +2025-01-28T14:54:12.879Z [ERROR] server-42825a80-bd55-5310-334b-d02866c2b71a: failed to appendEntries to: peer="{Voter server-e27b9610-e5d8-7241-0ebf-ec53c025a26e e27b9610-e5d8-7241-0ebf-ec53c025a26e}" error="failed to connect to peer: e27b9610-e5d8-7241-0ebf-ec53c025a26e" +2025-01-28T14:54:12.879Z [ERROR] server-42825a80-bd55-5310-334b-d02866c2b71a: failed to heartbeat to: peer=e27b9610-e5d8-7241-0ebf-ec53c025a26e backoff time=10ms error="failed to connect to peer: e27b9610-e5d8-7241-0ebf-ec53c025a26e" +2025-01-28T14:54:12.879Z [WARN] server-42825a80-bd55-5310-334b-d02866c2b71a: failed to contact: server-id=server-e516185a-dd82-97e2-fdae-73975d6c1ed0 time=50.759012ms +2025-01-28T14:54:12.890Z [WARN] server-42825a80-bd55-5310-334b-d02866c2b71a: failed to contact: server-id=server-e516185a-dd82-97e2-fdae-73975d6c1ed0 time=61.382769ms +2025-01-28T14:54:12.890Z [WARN] server-42825a80-bd55-5310-334b-d02866c2b71a: failed to contact: server-id=server-e27b9610-e5d8-7241-0ebf-ec53c025a26e time=57.962771ms +2025-01-28T14:54:12.890Z [WARN] server-42825a80-bd55-5310-334b-d02866c2b71a: failed to contact quorum of nodes, stepping down +2025-01-28T14:54:12.890Z [INFO] server-42825a80-bd55-5310-334b-d02866c2b71a: entering follower state: follower="Node at 42825a80-bd55-5310-334b-d02866c2b71a [Follower]" leader-address= leader-id= +2025-01-28T14:54:12.890Z [ERROR] server-42825a80-bd55-5310-334b-d02866c2b71a: failed to appendEntries to: peer="{Voter server-e516185a-dd82-97e2-fdae-73975d6c1ed0 e516185a-dd82-97e2-fdae-73975d6c1ed0}" error="failed to connect to peer: e516185a-dd82-97e2-fdae-73975d6c1ed0" +2025-01-28T14:54:12.890Z [ERROR] server-42825a80-bd55-5310-334b-d02866c2b71a: failed to appendEntries to: peer="{Voter server-e27b9610-e5d8-7241-0ebf-ec53c025a26e e27b9610-e5d8-7241-0ebf-ec53c025a26e}" error="failed to connect to peer: e27b9610-e5d8-7241-0ebf-ec53c025a26e" +--- PASS: TestRaft_RemoveFollower_SplitCluster (0.44s) === RUN TestRaft_AddKnownPeer -2025-01-28T14:47:11.403Z [INFO] server-590c519e-4232-2b34-c578-3ea3353245d7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-590c519e-4232-2b34-c578-3ea3353245d7 Address:590c519e-4232-2b34-c578-3ea3353245d7} {Suffrage:Voter ID:server-7a3c82da-b7dc-927b-f82b-1658e9cdb1d1 Address:7a3c82da-b7dc-927b-f82b-1658e9cdb1d1} {Suffrage:Voter ID:server-43254534-c596-2eea-1096-1a56d7141b20 Address:43254534-c596-2eea-1096-1a56d7141b20}]" -2025-01-28T14:47:11.403Z [INFO] server-590c519e-4232-2b34-c578-3ea3353245d7: entering follower state: follower="Node at 590c519e-4232-2b34-c578-3ea3353245d7 [Follower]" leader-address= leader-id= -2025-01-28T14:47:11.404Z [INFO] server-7a3c82da-b7dc-927b-f82b-1658e9cdb1d1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-590c519e-4232-2b34-c578-3ea3353245d7 Address:590c519e-4232-2b34-c578-3ea3353245d7} {Suffrage:Voter ID:server-7a3c82da-b7dc-927b-f82b-1658e9cdb1d1 Address:7a3c82da-b7dc-927b-f82b-1658e9cdb1d1} {Suffrage:Voter ID:server-43254534-c596-2eea-1096-1a56d7141b20 Address:43254534-c596-2eea-1096-1a56d7141b20}]" -2025-01-28T14:47:11.405Z [INFO] server-7a3c82da-b7dc-927b-f82b-1658e9cdb1d1: entering follower state: follower="Node at 7a3c82da-b7dc-927b-f82b-1658e9cdb1d1 [Follower]" leader-address= leader-id= -2025-01-28T14:47:11.406Z [INFO] server-43254534-c596-2eea-1096-1a56d7141b20: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-590c519e-4232-2b34-c578-3ea3353245d7 Address:590c519e-4232-2b34-c578-3ea3353245d7} {Suffrage:Voter ID:server-7a3c82da-b7dc-927b-f82b-1658e9cdb1d1 Address:7a3c82da-b7dc-927b-f82b-1658e9cdb1d1} {Suffrage:Voter ID:server-43254534-c596-2eea-1096-1a56d7141b20 Address:43254534-c596-2eea-1096-1a56d7141b20}]" -2025-01-28T14:47:11.406Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:11.407Z [INFO] server-43254534-c596-2eea-1096-1a56d7141b20: entering follower state: follower="Node at 43254534-c596-2eea-1096-1a56d7141b20 [Follower]" leader-address= leader-id= -2025-01-28T14:47:11.461Z [WARN] server-590c519e-4232-2b34-c578-3ea3353245d7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:11.461Z [INFO] server-590c519e-4232-2b34-c578-3ea3353245d7: entering candidate state: node="Node at 590c519e-4232-2b34-c578-3ea3353245d7 [Candidate]" term=2 -2025-01-28T14:47:11.462Z [INFO] server-590c519e-4232-2b34-c578-3ea3353245d7: election won: term=2 tally=2 -2025-01-28T14:47:11.462Z [INFO] server-590c519e-4232-2b34-c578-3ea3353245d7: entering leader state: leader="Node at 590c519e-4232-2b34-c578-3ea3353245d7 [Leader]" -2025-01-28T14:47:11.462Z [INFO] server-590c519e-4232-2b34-c578-3ea3353245d7: added peer, starting replication: peer=server-7a3c82da-b7dc-927b-f82b-1658e9cdb1d1 -2025-01-28T14:47:11.463Z [INFO] server-590c519e-4232-2b34-c578-3ea3353245d7: added peer, starting replication: peer=server-43254534-c596-2eea-1096-1a56d7141b20 -2025-01-28T14:47:11.464Z [INFO] server-590c519e-4232-2b34-c578-3ea3353245d7: pipelining replication: peer="{Voter server-7a3c82da-b7dc-927b-f82b-1658e9cdb1d1 7a3c82da-b7dc-927b-f82b-1658e9cdb1d1}" -2025-01-28T14:47:11.464Z [INFO] server-590c519e-4232-2b34-c578-3ea3353245d7: pipelining replication: peer="{Voter server-43254534-c596-2eea-1096-1a56d7141b20 43254534-c596-2eea-1096-1a56d7141b20}" -2025-01-28T14:47:11.569Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:11.463016829 +0000 UTC m=+14.546423187 (1 nodes), 55.979995ms from start of poll, 61.002478ms from cluster start. Timeout at 2025-01-28 14:47:11.568036497 +0000 UTC m=+14.651442522, 105.019335ms after stability -2025-01-28T14:47:11.569Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:47:11.675Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:47:11.569973005 +0000 UTC m=+14.653379363 (2 nodes), 16.668µs from start of poll, 167.958654ms from cluster start. Timeout at 2025-01-28 14:47:11.675066674 +0000 UTC m=+14.758472698, 105.093335ms after stability -2025-01-28T14:47:11.675Z [INFO] server-590c519e-4232-2b34-c578-3ea3353245d7: updating configuration: command=AddVoter server-id=server-7a3c82da-b7dc-927b-f82b-1658e9cdb1d1 server-addr=7a3c82da-b7dc-927b-f82b-1658e9cdb1d1 servers="[{Suffrage:Voter ID:server-590c519e-4232-2b34-c578-3ea3353245d7 Address:590c519e-4232-2b34-c578-3ea3353245d7} {Suffrage:Voter ID:server-7a3c82da-b7dc-927b-f82b-1658e9cdb1d1 Address:7a3c82da-b7dc-927b-f82b-1658e9cdb1d1} {Suffrage:Voter ID:server-43254534-c596-2eea-1096-1a56d7141b20 Address:43254534-c596-2eea-1096-1a56d7141b20}]" -2025-01-28T14:47:11.680Z [INFO] server-590c519e-4232-2b34-c578-3ea3353245d7: aborting pipeline replication: peer="{Voter server-43254534-c596-2eea-1096-1a56d7141b20 43254534-c596-2eea-1096-1a56d7141b20}" -2025-01-28T14:47:11.680Z [INFO] server-590c519e-4232-2b34-c578-3ea3353245d7: aborting pipeline replication: peer="{Voter server-7a3c82da-b7dc-927b-f82b-1658e9cdb1d1 7a3c82da-b7dc-927b-f82b-1658e9cdb1d1}" +2025-01-28T14:54:12.894Z [INFO] server-cd3e7900-30db-3d90-8214-e7a0b136665e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cd3e7900-30db-3d90-8214-e7a0b136665e Address:cd3e7900-30db-3d90-8214-e7a0b136665e} {Suffrage:Voter ID:server-457801f5-520b-ec19-26b0-b71b98e0a840 Address:457801f5-520b-ec19-26b0-b71b98e0a840} {Suffrage:Voter ID:server-486b8bd7-7d5d-b434-93d8-ab4c8a737ff8 Address:486b8bd7-7d5d-b434-93d8-ab4c8a737ff8}]" +2025-01-28T14:54:12.894Z [INFO] server-cd3e7900-30db-3d90-8214-e7a0b136665e: entering follower state: follower="Node at cd3e7900-30db-3d90-8214-e7a0b136665e [Follower]" leader-address= leader-id= +2025-01-28T14:54:12.895Z [INFO] server-457801f5-520b-ec19-26b0-b71b98e0a840: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cd3e7900-30db-3d90-8214-e7a0b136665e Address:cd3e7900-30db-3d90-8214-e7a0b136665e} {Suffrage:Voter ID:server-457801f5-520b-ec19-26b0-b71b98e0a840 Address:457801f5-520b-ec19-26b0-b71b98e0a840} {Suffrage:Voter ID:server-486b8bd7-7d5d-b434-93d8-ab4c8a737ff8 Address:486b8bd7-7d5d-b434-93d8-ab4c8a737ff8}]" +2025-01-28T14:54:12.895Z [INFO] server-457801f5-520b-ec19-26b0-b71b98e0a840: entering follower state: follower="Node at 457801f5-520b-ec19-26b0-b71b98e0a840 [Follower]" leader-address= leader-id= +2025-01-28T14:54:12.895Z [INFO] server-486b8bd7-7d5d-b434-93d8-ab4c8a737ff8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cd3e7900-30db-3d90-8214-e7a0b136665e Address:cd3e7900-30db-3d90-8214-e7a0b136665e} {Suffrage:Voter ID:server-457801f5-520b-ec19-26b0-b71b98e0a840 Address:457801f5-520b-ec19-26b0-b71b98e0a840} {Suffrage:Voter ID:server-486b8bd7-7d5d-b434-93d8-ab4c8a737ff8 Address:486b8bd7-7d5d-b434-93d8-ab4c8a737ff8}]" +2025-01-28T14:54:12.895Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:12.895Z [INFO] server-486b8bd7-7d5d-b434-93d8-ab4c8a737ff8: entering follower state: follower="Node at 486b8bd7-7d5d-b434-93d8-ab4c8a737ff8 [Follower]" leader-address= leader-id= +2025-01-28T14:54:12.967Z [WARN] server-457801f5-520b-ec19-26b0-b71b98e0a840: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:12.967Z [INFO] server-457801f5-520b-ec19-26b0-b71b98e0a840: entering candidate state: node="Node at 457801f5-520b-ec19-26b0-b71b98e0a840 [Candidate]" term=2 +2025-01-28T14:54:12.968Z [INFO] server-457801f5-520b-ec19-26b0-b71b98e0a840: election won: term=2 tally=2 +2025-01-28T14:54:12.968Z [INFO] server-457801f5-520b-ec19-26b0-b71b98e0a840: entering leader state: leader="Node at 457801f5-520b-ec19-26b0-b71b98e0a840 [Leader]" +2025-01-28T14:54:12.968Z [INFO] server-457801f5-520b-ec19-26b0-b71b98e0a840: added peer, starting replication: peer=server-cd3e7900-30db-3d90-8214-e7a0b136665e +2025-01-28T14:54:12.968Z [INFO] server-457801f5-520b-ec19-26b0-b71b98e0a840: added peer, starting replication: peer=server-486b8bd7-7d5d-b434-93d8-ab4c8a737ff8 +2025-01-28T14:54:12.968Z [INFO] server-457801f5-520b-ec19-26b0-b71b98e0a840: pipelining replication: peer="{Voter server-cd3e7900-30db-3d90-8214-e7a0b136665e cd3e7900-30db-3d90-8214-e7a0b136665e}" +2025-01-28T14:54:12.968Z [INFO] server-457801f5-520b-ec19-26b0-b71b98e0a840: pipelining replication: peer="{Voter server-486b8bd7-7d5d-b434-93d8-ab4c8a737ff8 486b8bd7-7d5d-b434-93d8-ab4c8a737ff8}" +2025-01-28T14:54:13.074Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:12.968327775 +0000 UTC m=+38.642817755 (1 nodes), 72.730979ms from start of poll, 73.832735ms from cluster start. Timeout at 2025-01-28 14:54:13.073332295 +0000 UTC m=+38.747822295, 105.00454ms after stability +2025-01-28T14:54:13.074Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:54:13.179Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:54:13.074316536 +0000 UTC m=+38.748806496 (2 nodes), 2.86µs from start of poll, 179.821476ms from cluster start. Timeout at 2025-01-28 14:54:13.179449464 +0000 UTC m=+38.853939444, 105.132948ms after stability +2025-01-28T14:54:13.179Z [INFO] server-457801f5-520b-ec19-26b0-b71b98e0a840: updating configuration: command=AddVoter server-id=server-cd3e7900-30db-3d90-8214-e7a0b136665e server-addr=cd3e7900-30db-3d90-8214-e7a0b136665e servers="[{Suffrage:Voter ID:server-cd3e7900-30db-3d90-8214-e7a0b136665e Address:cd3e7900-30db-3d90-8214-e7a0b136665e} {Suffrage:Voter ID:server-457801f5-520b-ec19-26b0-b71b98e0a840 Address:457801f5-520b-ec19-26b0-b71b98e0a840} {Suffrage:Voter ID:server-486b8bd7-7d5d-b434-93d8-ab4c8a737ff8 Address:486b8bd7-7d5d-b434-93d8-ab4c8a737ff8}]" +2025-01-28T14:54:13.180Z [INFO] server-457801f5-520b-ec19-26b0-b71b98e0a840: aborting pipeline replication: peer="{Voter server-cd3e7900-30db-3d90-8214-e7a0b136665e cd3e7900-30db-3d90-8214-e7a0b136665e}" +2025-01-28T14:54:13.180Z [INFO] server-457801f5-520b-ec19-26b0-b71b98e0a840: aborting pipeline replication: peer="{Voter server-486b8bd7-7d5d-b434-93d8-ab4c8a737ff8 486b8bd7-7d5d-b434-93d8-ab4c8a737ff8}" --- PASS: TestRaft_AddKnownPeer (0.29s) === RUN TestRaft_RemoveUnknownPeer -2025-01-28T14:47:11.697Z [INFO] server-62c0d1ef-75d6-2746-3250-de3d8ace4f16: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-62c0d1ef-75d6-2746-3250-de3d8ace4f16 Address:62c0d1ef-75d6-2746-3250-de3d8ace4f16} {Suffrage:Voter ID:server-971de209-6a92-8bfc-97e5-fa010cd91795 Address:971de209-6a92-8bfc-97e5-fa010cd91795} {Suffrage:Voter ID:server-0e079425-9518-c376-3564-dd199eec0866 Address:0e079425-9518-c376-3564-dd199eec0866}]" -2025-01-28T14:47:11.698Z [INFO] server-62c0d1ef-75d6-2746-3250-de3d8ace4f16: entering follower state: follower="Node at 62c0d1ef-75d6-2746-3250-de3d8ace4f16 [Follower]" leader-address= leader-id= -2025-01-28T14:47:11.699Z [INFO] server-971de209-6a92-8bfc-97e5-fa010cd91795: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-62c0d1ef-75d6-2746-3250-de3d8ace4f16 Address:62c0d1ef-75d6-2746-3250-de3d8ace4f16} {Suffrage:Voter ID:server-971de209-6a92-8bfc-97e5-fa010cd91795 Address:971de209-6a92-8bfc-97e5-fa010cd91795} {Suffrage:Voter ID:server-0e079425-9518-c376-3564-dd199eec0866 Address:0e079425-9518-c376-3564-dd199eec0866}]" -2025-01-28T14:47:11.700Z [INFO] server-971de209-6a92-8bfc-97e5-fa010cd91795: entering follower state: follower="Node at 971de209-6a92-8bfc-97e5-fa010cd91795 [Follower]" leader-address= leader-id= -2025-01-28T14:47:11.701Z [INFO] server-0e079425-9518-c376-3564-dd199eec0866: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-62c0d1ef-75d6-2746-3250-de3d8ace4f16 Address:62c0d1ef-75d6-2746-3250-de3d8ace4f16} {Suffrage:Voter ID:server-971de209-6a92-8bfc-97e5-fa010cd91795 Address:971de209-6a92-8bfc-97e5-fa010cd91795} {Suffrage:Voter ID:server-0e079425-9518-c376-3564-dd199eec0866 Address:0e079425-9518-c376-3564-dd199eec0866}]" -2025-01-28T14:47:11.702Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:11.702Z [INFO] server-0e079425-9518-c376-3564-dd199eec0866: entering follower state: follower="Node at 0e079425-9518-c376-3564-dd199eec0866 [Follower]" leader-address= leader-id= -2025-01-28T14:47:11.758Z [WARN] server-971de209-6a92-8bfc-97e5-fa010cd91795: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:11.758Z [INFO] server-971de209-6a92-8bfc-97e5-fa010cd91795: entering candidate state: node="Node at 971de209-6a92-8bfc-97e5-fa010cd91795 [Candidate]" term=2 -2025-01-28T14:47:11.759Z [INFO] server-971de209-6a92-8bfc-97e5-fa010cd91795: election won: term=2 tally=2 -2025-01-28T14:47:11.759Z [INFO] server-971de209-6a92-8bfc-97e5-fa010cd91795: entering leader state: leader="Node at 971de209-6a92-8bfc-97e5-fa010cd91795 [Leader]" -2025-01-28T14:47:11.760Z [INFO] server-971de209-6a92-8bfc-97e5-fa010cd91795: added peer, starting replication: peer=server-62c0d1ef-75d6-2746-3250-de3d8ace4f16 -2025-01-28T14:47:11.760Z [INFO] server-971de209-6a92-8bfc-97e5-fa010cd91795: added peer, starting replication: peer=server-0e079425-9518-c376-3564-dd199eec0866 -2025-01-28T14:47:11.761Z [INFO] server-971de209-6a92-8bfc-97e5-fa010cd91795: pipelining replication: peer="{Voter server-62c0d1ef-75d6-2746-3250-de3d8ace4f16 62c0d1ef-75d6-2746-3250-de3d8ace4f16}" -2025-01-28T14:47:11.761Z [INFO] server-971de209-6a92-8bfc-97e5-fa010cd91795: pipelining replication: peer="{Voter server-0e079425-9518-c376-3564-dd199eec0866 0e079425-9518-c376-3564-dd199eec0866}" -2025-01-28T14:47:11.865Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:11.759980312 +0000 UTC m=+14.843386670 (1 nodes), 57.667045ms from start of poll, 63.222877ms from cluster start. Timeout at 2025-01-28 14:47:11.865017314 +0000 UTC m=+14.948423672, 105.037002ms after stability -2025-01-28T14:47:11.865Z [INFO] server-971de209-6a92-8bfc-97e5-fa010cd91795: updating configuration: command=RemoveServer server-id=eb2a8234-0b01-7cfb-eea6-d505480a7285 server-addr= servers="[{Suffrage:Voter ID:server-62c0d1ef-75d6-2746-3250-de3d8ace4f16 Address:62c0d1ef-75d6-2746-3250-de3d8ace4f16} {Suffrage:Voter ID:server-971de209-6a92-8bfc-97e5-fa010cd91795 Address:971de209-6a92-8bfc-97e5-fa010cd91795} {Suffrage:Voter ID:server-0e079425-9518-c376-3564-dd199eec0866 Address:0e079425-9518-c376-3564-dd199eec0866}]" -2025-01-28T14:47:11.868Z [INFO] server-971de209-6a92-8bfc-97e5-fa010cd91795: aborting pipeline replication: peer="{Voter server-0e079425-9518-c376-3564-dd199eec0866 0e079425-9518-c376-3564-dd199eec0866}" -2025-01-28T14:47:11.868Z [INFO] server-971de209-6a92-8bfc-97e5-fa010cd91795: aborting pipeline replication: peer="{Voter server-62c0d1ef-75d6-2746-3250-de3d8ace4f16 62c0d1ef-75d6-2746-3250-de3d8ace4f16}" ---- PASS: TestRaft_RemoveUnknownPeer (0.19s) +2025-01-28T14:54:13.184Z [INFO] server-f493538c-403e-5872-b349-66e762fa4c28: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f493538c-403e-5872-b349-66e762fa4c28 Address:f493538c-403e-5872-b349-66e762fa4c28} {Suffrage:Voter ID:server-f1617c35-8568-b033-701c-5644b234f32d Address:f1617c35-8568-b033-701c-5644b234f32d} {Suffrage:Voter ID:server-8e3a5de3-a10f-45e1-ea67-0ed109937eda Address:8e3a5de3-a10f-45e1-ea67-0ed109937eda}]" +2025-01-28T14:54:13.184Z [INFO] server-f493538c-403e-5872-b349-66e762fa4c28: entering follower state: follower="Node at f493538c-403e-5872-b349-66e762fa4c28 [Follower]" leader-address= leader-id= +2025-01-28T14:54:13.184Z [INFO] server-f1617c35-8568-b033-701c-5644b234f32d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f493538c-403e-5872-b349-66e762fa4c28 Address:f493538c-403e-5872-b349-66e762fa4c28} {Suffrage:Voter ID:server-f1617c35-8568-b033-701c-5644b234f32d Address:f1617c35-8568-b033-701c-5644b234f32d} {Suffrage:Voter ID:server-8e3a5de3-a10f-45e1-ea67-0ed109937eda Address:8e3a5de3-a10f-45e1-ea67-0ed109937eda}]" +2025-01-28T14:54:13.185Z [INFO] server-f1617c35-8568-b033-701c-5644b234f32d: entering follower state: follower="Node at f1617c35-8568-b033-701c-5644b234f32d [Follower]" leader-address= leader-id= +2025-01-28T14:54:13.185Z [INFO] server-8e3a5de3-a10f-45e1-ea67-0ed109937eda: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f493538c-403e-5872-b349-66e762fa4c28 Address:f493538c-403e-5872-b349-66e762fa4c28} {Suffrage:Voter ID:server-f1617c35-8568-b033-701c-5644b234f32d Address:f1617c35-8568-b033-701c-5644b234f32d} {Suffrage:Voter ID:server-8e3a5de3-a10f-45e1-ea67-0ed109937eda Address:8e3a5de3-a10f-45e1-ea67-0ed109937eda}]" +2025-01-28T14:54:13.185Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:13.185Z [INFO] server-8e3a5de3-a10f-45e1-ea67-0ed109937eda: entering follower state: follower="Node at 8e3a5de3-a10f-45e1-ea67-0ed109937eda [Follower]" leader-address= leader-id= +2025-01-28T14:54:13.253Z [WARN] server-f1617c35-8568-b033-701c-5644b234f32d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:13.253Z [INFO] server-f1617c35-8568-b033-701c-5644b234f32d: entering candidate state: node="Node at f1617c35-8568-b033-701c-5644b234f32d [Candidate]" term=2 +2025-01-28T14:54:13.254Z [INFO] server-f1617c35-8568-b033-701c-5644b234f32d: election won: term=2 tally=2 +2025-01-28T14:54:13.254Z [INFO] server-f1617c35-8568-b033-701c-5644b234f32d: entering leader state: leader="Node at f1617c35-8568-b033-701c-5644b234f32d [Leader]" +2025-01-28T14:54:13.254Z [INFO] server-f1617c35-8568-b033-701c-5644b234f32d: added peer, starting replication: peer=server-f493538c-403e-5872-b349-66e762fa4c28 +2025-01-28T14:54:13.254Z [INFO] server-f1617c35-8568-b033-701c-5644b234f32d: added peer, starting replication: peer=server-8e3a5de3-a10f-45e1-ea67-0ed109937eda +2025-01-28T14:54:13.254Z [INFO] server-f1617c35-8568-b033-701c-5644b234f32d: pipelining replication: peer="{Voter server-f493538c-403e-5872-b349-66e762fa4c28 f493538c-403e-5872-b349-66e762fa4c28}" +2025-01-28T14:54:13.254Z [INFO] server-f1617c35-8568-b033-701c-5644b234f32d: pipelining replication: peer="{Voter server-8e3a5de3-a10f-45e1-ea67-0ed109937eda 8e3a5de3-a10f-45e1-ea67-0ed109937eda}" +2025-01-28T14:54:13.360Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:13.254300437 +0000 UTC m=+38.928790417 (1 nodes), 68.954715ms from start of poll, 70.028934ms from cluster start. Timeout at 2025-01-28 14:54:13.359308936 +0000 UTC m=+39.033798996, 105.008579ms after stability +2025-01-28T14:54:13.360Z [INFO] server-f1617c35-8568-b033-701c-5644b234f32d: updating configuration: command=RemoveServer server-id=f348d836-aee0-ef8c-a33c-c2217a91085f server-addr= servers="[{Suffrage:Voter ID:server-f493538c-403e-5872-b349-66e762fa4c28 Address:f493538c-403e-5872-b349-66e762fa4c28} {Suffrage:Voter ID:server-f1617c35-8568-b033-701c-5644b234f32d Address:f1617c35-8568-b033-701c-5644b234f32d} {Suffrage:Voter ID:server-8e3a5de3-a10f-45e1-ea67-0ed109937eda Address:8e3a5de3-a10f-45e1-ea67-0ed109937eda}]" +2025-01-28T14:54:13.361Z [INFO] server-f1617c35-8568-b033-701c-5644b234f32d: aborting pipeline replication: peer="{Voter server-8e3a5de3-a10f-45e1-ea67-0ed109937eda 8e3a5de3-a10f-45e1-ea67-0ed109937eda}" +2025-01-28T14:54:13.361Z [INFO] server-f1617c35-8568-b033-701c-5644b234f32d: aborting pipeline replication: peer="{Voter server-f493538c-403e-5872-b349-66e762fa4c28 f493538c-403e-5872-b349-66e762fa4c28}" +--- PASS: TestRaft_RemoveUnknownPeer (0.18s) === RUN TestRaft_SnapshotRestore -2025-01-28T14:47:11.881Z [INFO] server-884a97c4-a05f-e5ae-65f2-e50401e8f4a1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-884a97c4-a05f-e5ae-65f2-e50401e8f4a1 Address:884a97c4-a05f-e5ae-65f2-e50401e8f4a1}]" -2025-01-28T14:47:11.881Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:11.882Z [INFO] server-884a97c4-a05f-e5ae-65f2-e50401e8f4a1: entering follower state: follower="Node at 884a97c4-a05f-e5ae-65f2-e50401e8f4a1 [Follower]" leader-address= leader-id= -2025-01-28T14:47:11.963Z [WARN] server-884a97c4-a05f-e5ae-65f2-e50401e8f4a1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:11.963Z [INFO] server-884a97c4-a05f-e5ae-65f2-e50401e8f4a1: entering candidate state: node="Node at 884a97c4-a05f-e5ae-65f2-e50401e8f4a1 [Candidate]" term=2 -2025-01-28T14:47:11.964Z [INFO] server-884a97c4-a05f-e5ae-65f2-e50401e8f4a1: election won: term=2 tally=1 -2025-01-28T14:47:11.964Z [INFO] server-884a97c4-a05f-e5ae-65f2-e50401e8f4a1: entering leader state: leader="Node at 884a97c4-a05f-e5ae-65f2-e50401e8f4a1 [Leader]" -2025-01-28T14:47:12.071Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:11.964751392 +0000 UTC m=+15.048157750 (1 nodes), 82.689122ms from start of poll, 84.348838ms from cluster start. Timeout at 2025-01-28 14:47:12.069825728 +0000 UTC m=+15.153231752, 105.074002ms after stability -2025-01-28T14:47:12.122Z [INFO] server-884a97c4-a05f-e5ae-65f2-e50401e8f4a1: starting snapshot up to: index=102 -2025-01-28T14:47:12.122Z [INFO] creating new snapshot: path=/tmp/raft2293657919/snapshots/2-102-1738075632122.tmp -2025-01-28T14:47:12.129Z [INFO] server-884a97c4-a05f-e5ae-65f2-e50401e8f4a1: compacting logs: from=1 to=92 -2025-01-28T14:47:12.129Z [INFO] server-884a97c4-a05f-e5ae-65f2-e50401e8f4a1: snapshot complete up to: index=102 -2025-01-28T14:47:12.132Z [INFO] server-884a97c4-a05f-e5ae-65f2-e50401e8f4a1: starting restore from snapshot: id=2-102-1738075632122 last-index=102 last-term=2 size-in-bytes=693 -2025-01-28T14:47:12.135Z [INFO] server-884a97c4-a05f-e5ae-65f2-e50401e8f4a1: snapshot restore progress: id=2-102-1738075632122 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% -2025-01-28T14:47:12.135Z [INFO] server-884a97c4-a05f-e5ae-65f2-e50401e8f4a1: restored from snapshot: id=2-102-1738075632122 last-index=102 last-term=2 size-in-bytes=693 -2025-01-28T14:47:12.135Z [INFO] server-884a97c4-a05f-e5ae-65f2-e50401e8f4a1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-884a97c4-a05f-e5ae-65f2-e50401e8f4a1 Address:884a97c4-a05f-e5ae-65f2-e50401e8f4a1}]" ---- PASS: TestRaft_SnapshotRestore (0.26s) +2025-01-28T14:54:13.363Z [INFO] server-696bd993-5836-9d5f-7b58-3fcf8cf0b2f5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-696bd993-5836-9d5f-7b58-3fcf8cf0b2f5 Address:696bd993-5836-9d5f-7b58-3fcf8cf0b2f5}]" +2025-01-28T14:54:13.364Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:13.364Z [INFO] server-696bd993-5836-9d5f-7b58-3fcf8cf0b2f5: entering follower state: follower="Node at 696bd993-5836-9d5f-7b58-3fcf8cf0b2f5 [Follower]" leader-address= leader-id= +2025-01-28T14:54:13.429Z [WARN] server-696bd993-5836-9d5f-7b58-3fcf8cf0b2f5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:13.429Z [INFO] server-696bd993-5836-9d5f-7b58-3fcf8cf0b2f5: entering candidate state: node="Node at 696bd993-5836-9d5f-7b58-3fcf8cf0b2f5 [Candidate]" term=2 +2025-01-28T14:54:13.429Z [INFO] server-696bd993-5836-9d5f-7b58-3fcf8cf0b2f5: election won: term=2 tally=1 +2025-01-28T14:54:13.429Z [INFO] server-696bd993-5836-9d5f-7b58-3fcf8cf0b2f5: entering leader state: leader="Node at 696bd993-5836-9d5f-7b58-3fcf8cf0b2f5 [Leader]" +2025-01-28T14:54:13.535Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:13.429873413 +0000 UTC m=+39.104363393 (1 nodes), 65.788313ms from start of poll, 66.185136ms from cluster start. Timeout at 2025-01-28 14:54:13.534877733 +0000 UTC m=+39.209367753, 105.00436ms after stability +2025-01-28T14:54:13.543Z [INFO] server-696bd993-5836-9d5f-7b58-3fcf8cf0b2f5: starting snapshot up to: index=102 +2025-01-28T14:54:13.544Z [INFO] creating new snapshot: path=/tmp/raft4231758816/snapshots/2-102-1738076053543.tmp +2025-01-28T14:54:13.545Z [INFO] server-696bd993-5836-9d5f-7b58-3fcf8cf0b2f5: compacting logs: from=1 to=92 +2025-01-28T14:54:13.545Z [INFO] server-696bd993-5836-9d5f-7b58-3fcf8cf0b2f5: snapshot complete up to: index=102 +2025-01-28T14:54:13.546Z [INFO] server-696bd993-5836-9d5f-7b58-3fcf8cf0b2f5: starting restore from snapshot: id=2-102-1738076053543 last-index=102 last-term=2 size-in-bytes=693 +2025-01-28T14:54:13.547Z [INFO] server-696bd993-5836-9d5f-7b58-3fcf8cf0b2f5: snapshot restore progress: id=2-102-1738076053543 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% +2025-01-28T14:54:13.547Z [INFO] server-696bd993-5836-9d5f-7b58-3fcf8cf0b2f5: restored from snapshot: id=2-102-1738076053543 last-index=102 last-term=2 size-in-bytes=693 +2025-01-28T14:54:13.547Z [INFO] server-696bd993-5836-9d5f-7b58-3fcf8cf0b2f5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-696bd993-5836-9d5f-7b58-3fcf8cf0b2f5 Address:696bd993-5836-9d5f-7b58-3fcf8cf0b2f5}]" +--- PASS: TestRaft_SnapshotRestore (0.19s) === RUN TestRaft_RestoreSnapshotOnStartup_Monotonic -2025-01-28T14:47:12.144Z [INFO] server-62d76cd7-062f-60f2-a993-deb148e8cb66: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-62d76cd7-062f-60f2-a993-deb148e8cb66 Address:62d76cd7-062f-60f2-a993-deb148e8cb66}]" -2025-01-28T14:47:12.145Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:12.145Z [INFO] server-62d76cd7-062f-60f2-a993-deb148e8cb66: entering follower state: follower="Node at 62d76cd7-062f-60f2-a993-deb148e8cb66 [Follower]" leader-address= leader-id= -2025-01-28T14:47:12.214Z [WARN] server-62d76cd7-062f-60f2-a993-deb148e8cb66: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:12.214Z [INFO] server-62d76cd7-062f-60f2-a993-deb148e8cb66: entering candidate state: node="Node at 62d76cd7-062f-60f2-a993-deb148e8cb66 [Candidate]" term=2 -2025-01-28T14:47:12.215Z [INFO] server-62d76cd7-062f-60f2-a993-deb148e8cb66: election won: term=2 tally=1 -2025-01-28T14:47:12.215Z [INFO] server-62d76cd7-062f-60f2-a993-deb148e8cb66: entering leader state: leader="Node at 62d76cd7-062f-60f2-a993-deb148e8cb66 [Leader]" -2025-01-28T14:47:12.321Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:12.215670508 +0000 UTC m=+15.299076866 (1 nodes), 70.329422ms from start of poll, 71.827466ms from cluster start. Timeout at 2025-01-28 14:47:12.320729177 +0000 UTC m=+15.404135201, 105.058335ms after stability -2025-01-28T14:47:12.368Z [INFO] server-62d76cd7-062f-60f2-a993-deb148e8cb66: starting snapshot up to: index=102 -2025-01-28T14:47:12.368Z [INFO] creating new snapshot: path=/tmp/raft1978134631/snapshots/2-102-1738075632368.tmp -2025-01-28T14:47:12.374Z [INFO] server-62d76cd7-062f-60f2-a993-deb148e8cb66: compacting logs: from=1 to=92 -2025-01-28T14:47:12.375Z [INFO] server-62d76cd7-062f-60f2-a993-deb148e8cb66: snapshot complete up to: index=102 -2025-01-28T14:47:12.377Z [INFO] server-62d76cd7-062f-60f2-a993-deb148e8cb66: starting restore from snapshot: id=2-102-1738075632368 last-index=102 last-term=2 size-in-bytes=693 -2025-01-28T14:47:12.380Z [INFO] server-62d76cd7-062f-60f2-a993-deb148e8cb66: snapshot restore progress: id=2-102-1738075632368 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% -2025-01-28T14:47:12.380Z [INFO] server-62d76cd7-062f-60f2-a993-deb148e8cb66: restored from snapshot: id=2-102-1738075632368 last-index=102 last-term=2 size-in-bytes=693 -2025-01-28T14:47:12.380Z [INFO] server-62d76cd7-062f-60f2-a993-deb148e8cb66: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-62d76cd7-062f-60f2-a993-deb148e8cb66 Address:62d76cd7-062f-60f2-a993-deb148e8cb66}]" -2025-01-28T14:47:12.380Z [INFO] server-62d76cd7-062f-60f2-a993-deb148e8cb66: entering follower state: follower="Node at 62d76cd7-062f-60f2-a993-deb148e8cb66 [Shutdown]" leader-address= leader-id= ---- PASS: TestRaft_RestoreSnapshotOnStartup_Monotonic (0.24s) +2025-01-28T14:54:13.549Z [INFO] server-e0e2ca0f-500c-d72e-e3a0-e673bae1c0d9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e0e2ca0f-500c-d72e-e3a0-e673bae1c0d9 Address:e0e2ca0f-500c-d72e-e3a0-e673bae1c0d9}]" +2025-01-28T14:54:13.549Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:13.549Z [INFO] server-e0e2ca0f-500c-d72e-e3a0-e673bae1c0d9: entering follower state: follower="Node at e0e2ca0f-500c-d72e-e3a0-e673bae1c0d9 [Follower]" leader-address= leader-id= +2025-01-28T14:54:13.621Z [WARN] server-e0e2ca0f-500c-d72e-e3a0-e673bae1c0d9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:13.621Z [INFO] server-e0e2ca0f-500c-d72e-e3a0-e673bae1c0d9: entering candidate state: node="Node at e0e2ca0f-500c-d72e-e3a0-e673bae1c0d9 [Candidate]" term=2 +2025-01-28T14:54:13.622Z [INFO] server-e0e2ca0f-500c-d72e-e3a0-e673bae1c0d9: election won: term=2 tally=1 +2025-01-28T14:54:13.622Z [INFO] server-e0e2ca0f-500c-d72e-e3a0-e673bae1c0d9: entering leader state: leader="Node at e0e2ca0f-500c-d72e-e3a0-e673bae1c0d9 [Leader]" +2025-01-28T14:54:13.727Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:13.622115237 +0000 UTC m=+39.296605237 (1 nodes), 72.593911ms from start of poll, 72.970076ms from cluster start. Timeout at 2025-01-28 14:54:13.727122316 +0000 UTC m=+39.401612436, 105.007199ms after stability +2025-01-28T14:54:13.733Z [INFO] server-e0e2ca0f-500c-d72e-e3a0-e673bae1c0d9: starting snapshot up to: index=102 +2025-01-28T14:54:13.733Z [INFO] creating new snapshot: path=/tmp/raft3532354257/snapshots/2-102-1738076053733.tmp +2025-01-28T14:54:13.735Z [INFO] server-e0e2ca0f-500c-d72e-e3a0-e673bae1c0d9: compacting logs: from=1 to=92 +2025-01-28T14:54:13.736Z [INFO] server-e0e2ca0f-500c-d72e-e3a0-e673bae1c0d9: snapshot complete up to: index=102 +2025-01-28T14:54:13.736Z [INFO] server-e0e2ca0f-500c-d72e-e3a0-e673bae1c0d9: starting restore from snapshot: id=2-102-1738076053733 last-index=102 last-term=2 size-in-bytes=693 +2025-01-28T14:54:13.737Z [INFO] server-e0e2ca0f-500c-d72e-e3a0-e673bae1c0d9: snapshot restore progress: id=2-102-1738076053733 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% +2025-01-28T14:54:13.737Z [INFO] server-e0e2ca0f-500c-d72e-e3a0-e673bae1c0d9: restored from snapshot: id=2-102-1738076053733 last-index=102 last-term=2 size-in-bytes=693 +2025-01-28T14:54:13.737Z [INFO] server-e0e2ca0f-500c-d72e-e3a0-e673bae1c0d9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e0e2ca0f-500c-d72e-e3a0-e673bae1c0d9 Address:e0e2ca0f-500c-d72e-e3a0-e673bae1c0d9}]" +--- PASS: TestRaft_RestoreSnapshotOnStartup_Monotonic (0.19s) === RUN TestRaft_SnapshotRestore_Progress -2025-01-28T14:47:12.390Z [INFO] server-26c267a5-1274-601c-7df4-affc5dde90d7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-26c267a5-1274-601c-7df4-affc5dde90d7 Address:26c267a5-1274-601c-7df4-affc5dde90d7}]" -2025-01-28T14:47:12.390Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:12.390Z [INFO] server-26c267a5-1274-601c-7df4-affc5dde90d7: entering follower state: follower="Node at 26c267a5-1274-601c-7df4-affc5dde90d7 [Follower]" leader-address= leader-id= -2025-01-28T14:47:12.492Z [WARN] server-26c267a5-1274-601c-7df4-affc5dde90d7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:12.492Z [INFO] server-26c267a5-1274-601c-7df4-affc5dde90d7: entering candidate state: node="Node at 26c267a5-1274-601c-7df4-affc5dde90d7 [Candidate]" term=2 -2025-01-28T14:47:12.492Z [INFO] server-26c267a5-1274-601c-7df4-affc5dde90d7: election won: term=2 tally=1 -2025-01-28T14:47:12.493Z [INFO] server-26c267a5-1274-601c-7df4-affc5dde90d7: entering leader state: leader="Node at 26c267a5-1274-601c-7df4-affc5dde90d7 [Leader]" -2025-01-28T14:47:12.599Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:12.493309751 +0000 UTC m=+15.576716108 (1 nodes), 102.635713ms from start of poll, 104.156425ms from cluster start. Timeout at 2025-01-28 14:47:12.598344753 +0000 UTC m=+15.681750777, 105.034669ms after stability -2025-01-28T14:47:12.647Z [INFO] server-26c267a5-1274-601c-7df4-affc5dde90d7: starting snapshot up to: index=102 -2025-01-28T14:47:12.648Z [INFO] creating new snapshot: path=/tmp/raft3979978216/snapshots/2-102-1738075632647.tmp -2025-01-28T14:47:12.655Z [INFO] server-26c267a5-1274-601c-7df4-affc5dde90d7: compacting logs: from=1 to=92 -2025-01-28T14:47:12.655Z [INFO] server-26c267a5-1274-601c-7df4-affc5dde90d7: snapshot complete up to: index=102 ---- PASS: TestRaft_SnapshotRestore_Progress (0.28s) +2025-01-28T14:54:13.740Z [INFO] server-e1fa01d8-1b90-d514-f0c5-886c3913a8f0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e1fa01d8-1b90-d514-f0c5-886c3913a8f0 Address:e1fa01d8-1b90-d514-f0c5-886c3913a8f0}]" +2025-01-28T14:54:13.740Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:13.740Z [INFO] server-e1fa01d8-1b90-d514-f0c5-886c3913a8f0: entering follower state: follower="Node at e1fa01d8-1b90-d514-f0c5-886c3913a8f0 [Follower]" leader-address= leader-id= +2025-01-28T14:54:13.800Z [WARN] server-e1fa01d8-1b90-d514-f0c5-886c3913a8f0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:13.801Z [INFO] server-e1fa01d8-1b90-d514-f0c5-886c3913a8f0: entering candidate state: node="Node at e1fa01d8-1b90-d514-f0c5-886c3913a8f0 [Candidate]" term=2 +2025-01-28T14:54:13.801Z [INFO] server-e1fa01d8-1b90-d514-f0c5-886c3913a8f0: election won: term=2 tally=1 +2025-01-28T14:54:13.801Z [INFO] server-e1fa01d8-1b90-d514-f0c5-886c3913a8f0: entering leader state: leader="Node at e1fa01d8-1b90-d514-f0c5-886c3913a8f0 [Leader]" +2025-01-28T14:54:13.907Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:13.801256057 +0000 UTC m=+39.475746037 (1 nodes), 60.539209ms from start of poll, 60.941451ms from cluster start. Timeout at 2025-01-28 14:54:13.906264337 +0000 UTC m=+39.580754377, 105.00834ms after stability +2025-01-28T14:54:13.914Z [INFO] server-e1fa01d8-1b90-d514-f0c5-886c3913a8f0: starting snapshot up to: index=102 +2025-01-28T14:54:13.914Z [INFO] creating new snapshot: path=/tmp/raft2116801761/snapshots/2-102-1738076053914.tmp +2025-01-28T14:54:13.916Z [INFO] server-e1fa01d8-1b90-d514-f0c5-886c3913a8f0: compacting logs: from=1 to=92 +2025-01-28T14:54:13.916Z [INFO] server-e1fa01d8-1b90-d514-f0c5-886c3913a8f0: snapshot complete up to: index=102 +--- PASS: TestRaft_SnapshotRestore_Progress (0.18s) === RUN TestRaft_NoRestoreOnStart -2025-01-28T14:47:12.672Z [INFO] server-ca8f3287-0908-181a-20c5-650ee85ec296: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ca8f3287-0908-181a-20c5-650ee85ec296 Address:ca8f3287-0908-181a-20c5-650ee85ec296}]" -2025-01-28T14:47:12.673Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:12.673Z [INFO] server-ca8f3287-0908-181a-20c5-650ee85ec296: entering follower state: follower="Node at ca8f3287-0908-181a-20c5-650ee85ec296 [Follower]" leader-address= leader-id= -2025-01-28T14:47:12.735Z [WARN] server-ca8f3287-0908-181a-20c5-650ee85ec296: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:12.735Z [INFO] server-ca8f3287-0908-181a-20c5-650ee85ec296: entering candidate state: node="Node at ca8f3287-0908-181a-20c5-650ee85ec296 [Candidate]" term=2 -2025-01-28T14:47:12.735Z [INFO] server-ca8f3287-0908-181a-20c5-650ee85ec296: election won: term=2 tally=1 -2025-01-28T14:47:12.736Z [INFO] server-ca8f3287-0908-181a-20c5-650ee85ec296: entering leader state: leader="Node at ca8f3287-0908-181a-20c5-650ee85ec296 [Leader]" -2025-01-28T14:47:12.843Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:12.737152657 +0000 UTC m=+15.820559015 (1 nodes), 63.772227ms from start of poll, 65.260605ms from cluster start. Timeout at 2025-01-28 14:47:12.842193992 +0000 UTC m=+15.925600016, 105.041001ms after stability -2025-01-28T14:47:12.899Z [INFO] server-ca8f3287-0908-181a-20c5-650ee85ec296: starting snapshot up to: index=102 -2025-01-28T14:47:12.899Z [INFO] creating new snapshot: path=/tmp/raft1739837881/snapshots/2-102-1738075632899.tmp -2025-01-28T14:47:12.906Z [INFO] server-ca8f3287-0908-181a-20c5-650ee85ec296: compacting logs: from=1 to=92 -2025-01-28T14:47:12.906Z [INFO] server-ca8f3287-0908-181a-20c5-650ee85ec296: snapshot complete up to: index=102 -2025-01-28T14:47:12.909Z [INFO] server-ca8f3287-0908-181a-20c5-650ee85ec296: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ca8f3287-0908-181a-20c5-650ee85ec296 Address:ca8f3287-0908-181a-20c5-650ee85ec296}]" ---- PASS: TestRaft_NoRestoreOnStart (0.24s) -2025-01-28T14:47:12.909Z [INFO] server-ca8f3287-0908-181a-20c5-650ee85ec296: entering follower state: follower="Node at ca8f3287-0908-181a-20c5-650ee85ec296 [Follower]" leader-address= leader-id= +2025-01-28T14:54:13.920Z [INFO] server-835fea57-0626-ba4b-02c0-4957702425ed: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-835fea57-0626-ba4b-02c0-4957702425ed Address:835fea57-0626-ba4b-02c0-4957702425ed}]" +2025-01-28T14:54:13.920Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:13.920Z [INFO] server-835fea57-0626-ba4b-02c0-4957702425ed: entering follower state: follower="Node at 835fea57-0626-ba4b-02c0-4957702425ed [Follower]" leader-address= leader-id= +2025-01-28T14:54:14.011Z [WARN] server-835fea57-0626-ba4b-02c0-4957702425ed: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:14.012Z [INFO] server-835fea57-0626-ba4b-02c0-4957702425ed: entering candidate state: node="Node at 835fea57-0626-ba4b-02c0-4957702425ed [Candidate]" term=2 +2025-01-28T14:54:14.012Z [INFO] server-835fea57-0626-ba4b-02c0-4957702425ed: election won: term=2 tally=1 +2025-01-28T14:54:14.012Z [INFO] server-835fea57-0626-ba4b-02c0-4957702425ed: entering leader state: leader="Node at 835fea57-0626-ba4b-02c0-4957702425ed [Leader]" +2025-01-28T14:54:14.118Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:14.012417556 +0000 UTC m=+39.686907556 (1 nodes), 91.89435ms from start of poll, 92.283694ms from cluster start. Timeout at 2025-01-28 14:54:14.117424175 +0000 UTC m=+39.791914155, 105.006599ms after stability +2025-01-28T14:54:14.124Z [INFO] server-835fea57-0626-ba4b-02c0-4957702425ed: starting snapshot up to: index=102 +2025-01-28T14:54:14.124Z [INFO] creating new snapshot: path=/tmp/raft1326688817/snapshots/2-102-1738076054124.tmp +2025-01-28T14:54:14.126Z [INFO] server-835fea57-0626-ba4b-02c0-4957702425ed: compacting logs: from=1 to=92 +2025-01-28T14:54:14.126Z [INFO] server-835fea57-0626-ba4b-02c0-4957702425ed: snapshot complete up to: index=102 +2025-01-28T14:54:14.126Z [INFO] server-835fea57-0626-ba4b-02c0-4957702425ed: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-835fea57-0626-ba4b-02c0-4957702425ed Address:835fea57-0626-ba4b-02c0-4957702425ed}]" +--- PASS: TestRaft_NoRestoreOnStart (0.21s) === RUN TestRaft_SnapshotRestore_PeerChange +2025-01-28T14:54:14.126Z [INFO] server-835fea57-0626-ba4b-02c0-4957702425ed: entering follower state: follower="Node at 835fea57-0626-ba4b-02c0-4957702425ed [Follower]" leader-address= leader-id= raft_test.go:1266: Skipping unreliable test --- SKIP: TestRaft_SnapshotRestore_PeerChange (0.00s) === RUN TestRaft_AutoSnapshot -2025-01-28T14:47:12.915Z [INFO] server-57196a83-cc55-e28e-15f3-812732bdaa86: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-57196a83-cc55-e28e-15f3-812732bdaa86 Address:57196a83-cc55-e28e-15f3-812732bdaa86}]" -2025-01-28T14:47:12.916Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:12.916Z [INFO] server-57196a83-cc55-e28e-15f3-812732bdaa86: entering follower state: follower="Node at 57196a83-cc55-e28e-15f3-812732bdaa86 [Follower]" leader-address= leader-id= -2025-01-28T14:47:12.964Z [WARN] server-ca8f3287-0908-181a-20c5-650ee85ec296: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:12.964Z [INFO] server-ca8f3287-0908-181a-20c5-650ee85ec296: entering candidate state: node="Node at ca8f3287-0908-181a-20c5-650ee85ec296 [Candidate]" term=3 -2025-01-28T14:47:12.965Z [INFO] server-ca8f3287-0908-181a-20c5-650ee85ec296: election won: term=3 tally=1 -2025-01-28T14:47:12.965Z [INFO] server-ca8f3287-0908-181a-20c5-650ee85ec296: entering leader state: leader="Node at ca8f3287-0908-181a-20c5-650ee85ec296 [Leader]" -2025-01-28T14:47:12.987Z [WARN] server-57196a83-cc55-e28e-15f3-812732bdaa86: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:12.987Z [INFO] server-57196a83-cc55-e28e-15f3-812732bdaa86: entering candidate state: node="Node at 57196a83-cc55-e28e-15f3-812732bdaa86 [Candidate]" term=2 -2025-01-28T14:47:12.988Z [INFO] server-57196a83-cc55-e28e-15f3-812732bdaa86: election won: term=2 tally=1 -2025-01-28T14:47:12.988Z [INFO] server-57196a83-cc55-e28e-15f3-812732bdaa86: entering leader state: leader="Node at 57196a83-cc55-e28e-15f3-812732bdaa86 [Leader]" -2025-01-28T14:47:13.095Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:12.988784794 +0000 UTC m=+16.072190819 (1 nodes), 72.542154ms from start of poll, 74.008198ms from cluster start. Timeout at 2025-01-28 14:47:13.093822795 +0000 UTC m=+16.177228820, 105.038001ms after stability -2025-01-28T14:47:13.125Z [INFO] server-57196a83-cc55-e28e-15f3-812732bdaa86: starting snapshot up to: index=63 -2025-01-28T14:47:13.125Z [INFO] creating new snapshot: path=/tmp/raft713996494/snapshots/2-63-1738075633125.tmp -2025-01-28T14:47:13.134Z [INFO] server-57196a83-cc55-e28e-15f3-812732bdaa86: compacting logs: from=1 to=63 -2025-01-28T14:47:13.134Z [INFO] server-57196a83-cc55-e28e-15f3-812732bdaa86: snapshot complete up to: index=63 ---- PASS: TestRaft_AutoSnapshot (0.34s) +2025-01-28T14:54:14.127Z [INFO] server-54d22736-1d2a-770d-74e7-06ef75daf5fa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-54d22736-1d2a-770d-74e7-06ef75daf5fa Address:54d22736-1d2a-770d-74e7-06ef75daf5fa}]" +2025-01-28T14:54:14.127Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:14.127Z [INFO] server-54d22736-1d2a-770d-74e7-06ef75daf5fa: entering follower state: follower="Node at 54d22736-1d2a-770d-74e7-06ef75daf5fa [Follower]" leader-address= leader-id= +2025-01-28T14:54:14.193Z [WARN] server-54d22736-1d2a-770d-74e7-06ef75daf5fa: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:14.193Z [INFO] server-54d22736-1d2a-770d-74e7-06ef75daf5fa: entering candidate state: node="Node at 54d22736-1d2a-770d-74e7-06ef75daf5fa [Candidate]" term=2 +2025-01-28T14:54:14.194Z [INFO] server-54d22736-1d2a-770d-74e7-06ef75daf5fa: election won: term=2 tally=1 +2025-01-28T14:54:14.194Z [INFO] server-54d22736-1d2a-770d-74e7-06ef75daf5fa: entering leader state: leader="Node at 54d22736-1d2a-770d-74e7-06ef75daf5fa [Leader]" +2025-01-28T14:54:14.219Z [WARN] server-835fea57-0626-ba4b-02c0-4957702425ed: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:14.219Z [INFO] server-835fea57-0626-ba4b-02c0-4957702425ed: entering candidate state: node="Node at 835fea57-0626-ba4b-02c0-4957702425ed [Candidate]" term=3 +2025-01-28T14:54:14.219Z [INFO] server-835fea57-0626-ba4b-02c0-4957702425ed: election won: term=3 tally=1 +2025-01-28T14:54:14.219Z [INFO] server-835fea57-0626-ba4b-02c0-4957702425ed: entering leader state: leader="Node at 835fea57-0626-ba4b-02c0-4957702425ed [Leader]" +2025-01-28T14:54:14.299Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:14.194240203 +0000 UTC m=+39.868730183 (1 nodes), 66.295185ms from start of poll, 66.622035ms from cluster start. Timeout at 2025-01-28 14:54:14.299295737 +0000 UTC m=+39.973785797, 105.055614ms after stability +2025-01-28T14:54:14.315Z [INFO] server-54d22736-1d2a-770d-74e7-06ef75daf5fa: starting snapshot up to: index=102 +2025-01-28T14:54:14.315Z [INFO] creating new snapshot: path=/tmp/raft4012878319/snapshots/2-102-1738076054315.tmp +2025-01-28T14:54:14.317Z [INFO] server-54d22736-1d2a-770d-74e7-06ef75daf5fa: compacting logs: from=1 to=92 +2025-01-28T14:54:14.317Z [INFO] server-54d22736-1d2a-770d-74e7-06ef75daf5fa: snapshot complete up to: index=102 +--- PASS: TestRaft_AutoSnapshot (0.29s) === RUN TestRaft_UserSnapshot -2025-01-28T14:47:13.252Z [INFO] server-51b56a40-9b2b-2990-8cfb-737ad7096eac: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-51b56a40-9b2b-2990-8cfb-737ad7096eac Address:51b56a40-9b2b-2990-8cfb-737ad7096eac}]" -2025-01-28T14:47:13.252Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:13.253Z [INFO] server-51b56a40-9b2b-2990-8cfb-737ad7096eac: entering follower state: follower="Node at 51b56a40-9b2b-2990-8cfb-737ad7096eac [Follower]" leader-address= leader-id= -2025-01-28T14:47:13.305Z [WARN] server-51b56a40-9b2b-2990-8cfb-737ad7096eac: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:13.305Z [INFO] server-51b56a40-9b2b-2990-8cfb-737ad7096eac: entering candidate state: node="Node at 51b56a40-9b2b-2990-8cfb-737ad7096eac [Candidate]" term=2 -2025-01-28T14:47:13.306Z [INFO] server-51b56a40-9b2b-2990-8cfb-737ad7096eac: election won: term=2 tally=1 -2025-01-28T14:47:13.306Z [INFO] server-51b56a40-9b2b-2990-8cfb-737ad7096eac: entering leader state: leader="Node at 51b56a40-9b2b-2990-8cfb-737ad7096eac [Leader]" -2025-01-28T14:47:13.413Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:13.307035909 +0000 UTC m=+16.390442267 (1 nodes), 53.948935ms from start of poll, 55.491647ms from cluster start. Timeout at 2025-01-28 14:47:13.412072243 +0000 UTC m=+16.495478601, 105.036334ms after stability -2025-01-28T14:47:13.414Z [ERROR] server-51b56a40-9b2b-2990-8cfb-737ad7096eac: failed to take snapshot: error="nothing new to snapshot" -2025-01-28T14:47:13.419Z [INFO] server-51b56a40-9b2b-2990-8cfb-737ad7096eac: starting snapshot up to: index=12 -2025-01-28T14:47:13.420Z [INFO] creating new snapshot: path=/tmp/raft2817216302/snapshots/2-12-1738075633420.tmp -2025-01-28T14:47:13.426Z [INFO] server-51b56a40-9b2b-2990-8cfb-737ad7096eac: compacting logs: from=1 to=2 -2025-01-28T14:47:13.426Z [INFO] server-51b56a40-9b2b-2990-8cfb-737ad7096eac: snapshot complete up to: index=12 ---- PASS: TestRaft_UserSnapshot (0.18s) +2025-01-28T14:54:14.414Z [INFO] server-c7464b7d-5c64-7fe9-2afc-668011747a55: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c7464b7d-5c64-7fe9-2afc-668011747a55 Address:c7464b7d-5c64-7fe9-2afc-668011747a55}]" +2025-01-28T14:54:14.414Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:14.414Z [INFO] server-c7464b7d-5c64-7fe9-2afc-668011747a55: entering follower state: follower="Node at c7464b7d-5c64-7fe9-2afc-668011747a55 [Follower]" leader-address= leader-id= +2025-01-28T14:54:14.510Z [WARN] server-c7464b7d-5c64-7fe9-2afc-668011747a55: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:14.510Z [INFO] server-c7464b7d-5c64-7fe9-2afc-668011747a55: entering candidate state: node="Node at c7464b7d-5c64-7fe9-2afc-668011747a55 [Candidate]" term=2 +2025-01-28T14:54:14.510Z [INFO] server-c7464b7d-5c64-7fe9-2afc-668011747a55: election won: term=2 tally=1 +2025-01-28T14:54:14.510Z [INFO] server-c7464b7d-5c64-7fe9-2afc-668011747a55: entering leader state: leader="Node at c7464b7d-5c64-7fe9-2afc-668011747a55 [Leader]" +2025-01-28T14:54:14.616Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:14.510508647 +0000 UTC m=+40.184998627 (1 nodes), 95.551446ms from start of poll, 95.947648ms from cluster start. Timeout at 2025-01-28 14:54:14.615516746 +0000 UTC m=+40.290006786, 105.008159ms after stability +2025-01-28T14:54:14.617Z [ERROR] server-c7464b7d-5c64-7fe9-2afc-668011747a55: failed to take snapshot: error="nothing new to snapshot" +2025-01-28T14:54:14.618Z [INFO] server-c7464b7d-5c64-7fe9-2afc-668011747a55: starting snapshot up to: index=12 +2025-01-28T14:54:14.618Z [INFO] creating new snapshot: path=/tmp/raft2041857284/snapshots/2-12-1738076054618.tmp +2025-01-28T14:54:14.619Z [INFO] server-c7464b7d-5c64-7fe9-2afc-668011747a55: compacting logs: from=1 to=2 +2025-01-28T14:54:14.619Z [INFO] server-c7464b7d-5c64-7fe9-2afc-668011747a55: snapshot complete up to: index=12 +--- PASS: TestRaft_UserSnapshot (0.21s) === RUN TestRaft_UserRestore === RUN TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_false -2025-01-28T14:47:13.443Z [INFO] server-1226ab04-35b0-2be9-5be4-78b115ecd48a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1226ab04-35b0-2be9-5be4-78b115ecd48a Address:1226ab04-35b0-2be9-5be4-78b115ecd48a} {Suffrage:Voter ID:server-84b8fbe0-13b4-b759-c8dc-2380bb74f023 Address:84b8fbe0-13b4-b759-c8dc-2380bb74f023} {Suffrage:Voter ID:server-1dfb61d6-ddb4-bc22-0bf3-78241ca9782c Address:1dfb61d6-ddb4-bc22-0bf3-78241ca9782c}]" -2025-01-28T14:47:13.443Z [INFO] server-1226ab04-35b0-2be9-5be4-78b115ecd48a: entering follower state: follower="Node at 1226ab04-35b0-2be9-5be4-78b115ecd48a [Follower]" leader-address= leader-id= -2025-01-28T14:47:13.444Z [INFO] server-84b8fbe0-13b4-b759-c8dc-2380bb74f023: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1226ab04-35b0-2be9-5be4-78b115ecd48a Address:1226ab04-35b0-2be9-5be4-78b115ecd48a} {Suffrage:Voter ID:server-84b8fbe0-13b4-b759-c8dc-2380bb74f023 Address:84b8fbe0-13b4-b759-c8dc-2380bb74f023} {Suffrage:Voter ID:server-1dfb61d6-ddb4-bc22-0bf3-78241ca9782c Address:1dfb61d6-ddb4-bc22-0bf3-78241ca9782c}]" -2025-01-28T14:47:13.445Z [INFO] server-84b8fbe0-13b4-b759-c8dc-2380bb74f023: entering follower state: follower="Node at 84b8fbe0-13b4-b759-c8dc-2380bb74f023 [Follower]" leader-address= leader-id= -2025-01-28T14:47:13.446Z [INFO] server-1dfb61d6-ddb4-bc22-0bf3-78241ca9782c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1226ab04-35b0-2be9-5be4-78b115ecd48a Address:1226ab04-35b0-2be9-5be4-78b115ecd48a} {Suffrage:Voter ID:server-84b8fbe0-13b4-b759-c8dc-2380bb74f023 Address:84b8fbe0-13b4-b759-c8dc-2380bb74f023} {Suffrage:Voter ID:server-1dfb61d6-ddb4-bc22-0bf3-78241ca9782c Address:1dfb61d6-ddb4-bc22-0bf3-78241ca9782c}]" -2025-01-28T14:47:13.446Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:13.446Z [INFO] server-1dfb61d6-ddb4-bc22-0bf3-78241ca9782c: entering follower state: follower="Node at 1dfb61d6-ddb4-bc22-0bf3-78241ca9782c [Follower]" leader-address= leader-id= -2025-01-28T14:47:13.970Z [WARN] server-1dfb61d6-ddb4-bc22-0bf3-78241ca9782c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:13.970Z [INFO] server-1dfb61d6-ddb4-bc22-0bf3-78241ca9782c: entering candidate state: node="Node at 1dfb61d6-ddb4-bc22-0bf3-78241ca9782c [Candidate]" term=2 -2025-01-28T14:47:13.971Z [INFO] server-1dfb61d6-ddb4-bc22-0bf3-78241ca9782c: election won: term=2 tally=2 -2025-01-28T14:47:13.971Z [INFO] server-1dfb61d6-ddb4-bc22-0bf3-78241ca9782c: entering leader state: leader="Node at 1dfb61d6-ddb4-bc22-0bf3-78241ca9782c [Leader]" -2025-01-28T14:47:13.971Z [INFO] server-1dfb61d6-ddb4-bc22-0bf3-78241ca9782c: added peer, starting replication: peer=server-1226ab04-35b0-2be9-5be4-78b115ecd48a -2025-01-28T14:47:13.972Z [INFO] server-1dfb61d6-ddb4-bc22-0bf3-78241ca9782c: added peer, starting replication: peer=server-84b8fbe0-13b4-b759-c8dc-2380bb74f023 -2025-01-28T14:47:13.973Z [INFO] server-1dfb61d6-ddb4-bc22-0bf3-78241ca9782c: pipelining replication: peer="{Voter server-84b8fbe0-13b4-b759-c8dc-2380bb74f023 84b8fbe0-13b4-b759-c8dc-2380bb74f023}" -2025-01-28T14:47:13.976Z [INFO] server-1dfb61d6-ddb4-bc22-0bf3-78241ca9782c: pipelining replication: peer="{Voter server-1226ab04-35b0-2be9-5be4-78b115ecd48a 1226ab04-35b0-2be9-5be4-78b115ecd48a}" -2025-01-28T14:47:14.980Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:13.975182746 +0000 UTC m=+17.058589437 (1 nodes), 528.206015ms from start of poll, 532.988157ms from cluster start. Timeout at 2025-01-28 14:47:14.980205079 +0000 UTC m=+18.063611104, 1.005021667s after stability -2025-01-28T14:47:14.987Z [INFO] server-1dfb61d6-ddb4-bc22-0bf3-78241ca9782c: starting snapshot up to: index=12 -2025-01-28T14:47:14.987Z [INFO] creating new snapshot: path=/tmp/raft172538491/snapshots/2-12-1738075634987.tmp -2025-01-28T14:47:14.994Z [INFO] server-1dfb61d6-ddb4-bc22-0bf3-78241ca9782c: snapshot complete up to: index=12 -2025-01-28T14:47:15.004Z [INFO] creating new snapshot: path=/tmp/raft172538491/snapshots/2-23-1738075635004.tmp -2025-01-28T14:47:15.011Z [INFO] server-1dfb61d6-ddb4-bc22-0bf3-78241ca9782c: copied to local snapshot: bytes=71 -2025-01-28T14:47:15.013Z [INFO] server-1dfb61d6-ddb4-bc22-0bf3-78241ca9782c: snapshot restore progress: id=2-23-1738075635004 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:15.013Z [INFO] server-1dfb61d6-ddb4-bc22-0bf3-78241ca9782c: restored user snapshot: index=23 -2025-01-28T14:47:15.014Z [ERROR] server-1dfb61d6-ddb4-bc22-0bf3-78241ca9782c: failed to get log: index=23 error="log not found" -2025-01-28T14:47:15.014Z [ERROR] server-1dfb61d6-ddb4-bc22-0bf3-78241ca9782c: failed to get log: index=23 error="log not found" -2025-01-28T14:47:15.014Z [INFO] server-1dfb61d6-ddb4-bc22-0bf3-78241ca9782c: aborting pipeline replication: peer="{Voter server-84b8fbe0-13b4-b759-c8dc-2380bb74f023 84b8fbe0-13b4-b759-c8dc-2380bb74f023}" -2025-01-28T14:47:15.014Z [INFO] server-1dfb61d6-ddb4-bc22-0bf3-78241ca9782c: aborting pipeline replication: peer="{Voter server-1226ab04-35b0-2be9-5be4-78b115ecd48a 1226ab04-35b0-2be9-5be4-78b115ecd48a}" -2025-01-28T14:47:15.022Z [ERROR] server-1dfb61d6-ddb4-bc22-0bf3-78241ca9782c: failed to get log: index=23 error="log not found" -2025-01-28T14:47:15.025Z [ERROR] server-1dfb61d6-ddb4-bc22-0bf3-78241ca9782c: failed to get log: index=23 error="log not found" -2025-01-28T14:47:15.027Z [INFO] creating new snapshot: path=/tmp/raft1680502720/snapshots/2-23-1738075635026.tmp -2025-01-28T14:47:15.034Z [INFO] creating new snapshot: path=/tmp/raft4173689595/snapshots/2-23-1738075635034.tmp -2025-01-28T14:47:15.035Z [INFO] server-1226ab04-35b0-2be9-5be4-78b115ecd48a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:15.038Z [INFO] server-84b8fbe0-13b4-b759-c8dc-2380bb74f023: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:15.043Z [INFO] server-1226ab04-35b0-2be9-5be4-78b115ecd48a: copied to local snapshot: bytes=71 -2025-01-28T14:47:15.046Z [INFO] server-84b8fbe0-13b4-b759-c8dc-2380bb74f023: copied to local snapshot: bytes=71 -2025-01-28T14:47:15.046Z [INFO] server-1226ab04-35b0-2be9-5be4-78b115ecd48a: snapshot restore progress: id=2-23-1738075635026 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:15.047Z [INFO] server-1226ab04-35b0-2be9-5be4-78b115ecd48a: Installed remote snapshot -2025-01-28T14:47:15.048Z [INFO] server-84b8fbe0-13b4-b759-c8dc-2380bb74f023: snapshot restore progress: id=2-23-1738075635034 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:15.049Z [INFO] server-84b8fbe0-13b4-b759-c8dc-2380bb74f023: Installed remote snapshot -2025-01-28T14:47:15.049Z [INFO] server-1dfb61d6-ddb4-bc22-0bf3-78241ca9782c: pipelining replication: peer="{Voter server-1226ab04-35b0-2be9-5be4-78b115ecd48a 1226ab04-35b0-2be9-5be4-78b115ecd48a}" -2025-01-28T14:47:15.049Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:47:15.051Z [INFO] server-1dfb61d6-ddb4-bc22-0bf3-78241ca9782c: pipelining replication: peer="{Voter server-84b8fbe0-13b4-b759-c8dc-2380bb74f023 84b8fbe0-13b4-b759-c8dc-2380bb74f023}" -2025-01-28T14:47:16.056Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:47:15.050115993 +0000 UTC m=+18.133522017 (2 nodes), 15.667µs from start of poll, 1.607920737s from cluster start. Timeout at 2025-01-28 14:47:16.055211329 +0000 UTC m=+19.138617686, 1.005095669s after stability -2025-01-28T14:47:16.075Z [INFO] server-1dfb61d6-ddb4-bc22-0bf3-78241ca9782c: aborting pipeline replication: peer="{Voter server-84b8fbe0-13b4-b759-c8dc-2380bb74f023 84b8fbe0-13b4-b759-c8dc-2380bb74f023}" -2025-01-28T14:47:16.075Z [INFO] server-1dfb61d6-ddb4-bc22-0bf3-78241ca9782c: aborting pipeline replication: peer="{Voter server-1226ab04-35b0-2be9-5be4-78b115ecd48a 1226ab04-35b0-2be9-5be4-78b115ecd48a}" +2025-01-28T14:54:14.623Z [INFO] server-0640f2fe-9fcd-00cf-0dbf-653e06ad6761: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0640f2fe-9fcd-00cf-0dbf-653e06ad6761 Address:0640f2fe-9fcd-00cf-0dbf-653e06ad6761} {Suffrage:Voter ID:server-8729d1b3-48b0-0b5c-9099-236c1b1a99f2 Address:8729d1b3-48b0-0b5c-9099-236c1b1a99f2} {Suffrage:Voter ID:server-5401f093-b02f-401e-a07e-c348c61f98f7 Address:5401f093-b02f-401e-a07e-c348c61f98f7}]" +2025-01-28T14:54:14.624Z [INFO] server-0640f2fe-9fcd-00cf-0dbf-653e06ad6761: entering follower state: follower="Node at 0640f2fe-9fcd-00cf-0dbf-653e06ad6761 [Follower]" leader-address= leader-id= +2025-01-28T14:54:14.624Z [INFO] server-8729d1b3-48b0-0b5c-9099-236c1b1a99f2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0640f2fe-9fcd-00cf-0dbf-653e06ad6761 Address:0640f2fe-9fcd-00cf-0dbf-653e06ad6761} {Suffrage:Voter ID:server-8729d1b3-48b0-0b5c-9099-236c1b1a99f2 Address:8729d1b3-48b0-0b5c-9099-236c1b1a99f2} {Suffrage:Voter ID:server-5401f093-b02f-401e-a07e-c348c61f98f7 Address:5401f093-b02f-401e-a07e-c348c61f98f7}]" +2025-01-28T14:54:14.624Z [INFO] server-8729d1b3-48b0-0b5c-9099-236c1b1a99f2: entering follower state: follower="Node at 8729d1b3-48b0-0b5c-9099-236c1b1a99f2 [Follower]" leader-address= leader-id= +2025-01-28T14:54:14.624Z [INFO] server-5401f093-b02f-401e-a07e-c348c61f98f7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0640f2fe-9fcd-00cf-0dbf-653e06ad6761 Address:0640f2fe-9fcd-00cf-0dbf-653e06ad6761} {Suffrage:Voter ID:server-8729d1b3-48b0-0b5c-9099-236c1b1a99f2 Address:8729d1b3-48b0-0b5c-9099-236c1b1a99f2} {Suffrage:Voter ID:server-5401f093-b02f-401e-a07e-c348c61f98f7 Address:5401f093-b02f-401e-a07e-c348c61f98f7}]" +2025-01-28T14:54:14.624Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:14.624Z [INFO] server-5401f093-b02f-401e-a07e-c348c61f98f7: entering follower state: follower="Node at 5401f093-b02f-401e-a07e-c348c61f98f7 [Follower]" leader-address= leader-id= +2025-01-28T14:54:15.155Z [WARN] server-0640f2fe-9fcd-00cf-0dbf-653e06ad6761: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:15.156Z [INFO] server-0640f2fe-9fcd-00cf-0dbf-653e06ad6761: entering candidate state: node="Node at 0640f2fe-9fcd-00cf-0dbf-653e06ad6761 [Candidate]" term=2 +2025-01-28T14:54:15.156Z [INFO] server-0640f2fe-9fcd-00cf-0dbf-653e06ad6761: election won: term=2 tally=2 +2025-01-28T14:54:15.156Z [INFO] server-0640f2fe-9fcd-00cf-0dbf-653e06ad6761: entering leader state: leader="Node at 0640f2fe-9fcd-00cf-0dbf-653e06ad6761 [Leader]" +2025-01-28T14:54:15.156Z [INFO] server-0640f2fe-9fcd-00cf-0dbf-653e06ad6761: added peer, starting replication: peer=server-8729d1b3-48b0-0b5c-9099-236c1b1a99f2 +2025-01-28T14:54:15.156Z [INFO] server-0640f2fe-9fcd-00cf-0dbf-653e06ad6761: added peer, starting replication: peer=server-5401f093-b02f-401e-a07e-c348c61f98f7 +2025-01-28T14:54:15.156Z [INFO] server-0640f2fe-9fcd-00cf-0dbf-653e06ad6761: pipelining replication: peer="{Voter server-8729d1b3-48b0-0b5c-9099-236c1b1a99f2 8729d1b3-48b0-0b5c-9099-236c1b1a99f2}" +2025-01-28T14:54:15.156Z [INFO] server-0640f2fe-9fcd-00cf-0dbf-653e06ad6761: pipelining replication: peer="{Voter server-5401f093-b02f-401e-a07e-c348c61f98f7 5401f093-b02f-401e-a07e-c348c61f98f7}" +2025-01-28T14:54:16.162Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:15.156483126 +0000 UTC m=+40.830973106 (1 nodes), 531.851625ms from start of poll, 533.014376ms from cluster start. Timeout at 2025-01-28 14:54:16.161496145 +0000 UTC m=+41.835986185, 1.005013079s after stability +2025-01-28T14:54:16.163Z [INFO] server-0640f2fe-9fcd-00cf-0dbf-653e06ad6761: starting snapshot up to: index=12 +2025-01-28T14:54:16.163Z [INFO] creating new snapshot: path=/tmp/raft3069554211/snapshots/2-12-1738076056163.tmp +2025-01-28T14:54:16.165Z [INFO] server-0640f2fe-9fcd-00cf-0dbf-653e06ad6761: snapshot complete up to: index=12 +2025-01-28T14:54:16.166Z [INFO] creating new snapshot: path=/tmp/raft3069554211/snapshots/2-23-1738076056166.tmp +2025-01-28T14:54:16.168Z [INFO] server-0640f2fe-9fcd-00cf-0dbf-653e06ad6761: copied to local snapshot: bytes=71 +2025-01-28T14:54:16.169Z [INFO] server-0640f2fe-9fcd-00cf-0dbf-653e06ad6761: snapshot restore progress: id=2-23-1738076056166 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:16.169Z [INFO] server-0640f2fe-9fcd-00cf-0dbf-653e06ad6761: restored user snapshot: index=23 +2025-01-28T14:54:16.169Z [ERROR] server-0640f2fe-9fcd-00cf-0dbf-653e06ad6761: failed to get log: index=23 error="log not found" +2025-01-28T14:54:16.169Z [ERROR] server-0640f2fe-9fcd-00cf-0dbf-653e06ad6761: failed to get log: index=23 error="log not found" +2025-01-28T14:54:16.169Z [INFO] server-0640f2fe-9fcd-00cf-0dbf-653e06ad6761: aborting pipeline replication: peer="{Voter server-8729d1b3-48b0-0b5c-9099-236c1b1a99f2 8729d1b3-48b0-0b5c-9099-236c1b1a99f2}" +2025-01-28T14:54:16.169Z [INFO] server-0640f2fe-9fcd-00cf-0dbf-653e06ad6761: aborting pipeline replication: peer="{Voter server-5401f093-b02f-401e-a07e-c348c61f98f7 5401f093-b02f-401e-a07e-c348c61f98f7}" +2025-01-28T14:54:16.175Z [ERROR] server-0640f2fe-9fcd-00cf-0dbf-653e06ad6761: failed to get log: index=23 error="log not found" +2025-01-28T14:54:16.175Z [ERROR] server-0640f2fe-9fcd-00cf-0dbf-653e06ad6761: failed to get log: index=23 error="log not found" +2025-01-28T14:54:16.176Z [INFO] creating new snapshot: path=/tmp/raft1826080371/snapshots/2-23-1738076056176.tmp +2025-01-28T14:54:16.176Z [INFO] creating new snapshot: path=/tmp/raft1059235554/snapshots/2-23-1738076056176.tmp +2025-01-28T14:54:16.177Z [INFO] server-8729d1b3-48b0-0b5c-9099-236c1b1a99f2: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:16.177Z [INFO] server-5401f093-b02f-401e-a07e-c348c61f98f7: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:16.178Z [INFO] server-5401f093-b02f-401e-a07e-c348c61f98f7: copied to local snapshot: bytes=71 +2025-01-28T14:54:16.178Z [INFO] server-8729d1b3-48b0-0b5c-9099-236c1b1a99f2: copied to local snapshot: bytes=71 +2025-01-28T14:54:16.178Z [INFO] server-5401f093-b02f-401e-a07e-c348c61f98f7: snapshot restore progress: id=2-23-1738076056176 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:16.178Z [INFO] server-8729d1b3-48b0-0b5c-9099-236c1b1a99f2: snapshot restore progress: id=2-23-1738076056176 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:16.178Z [INFO] server-5401f093-b02f-401e-a07e-c348c61f98f7: Installed remote snapshot +2025-01-28T14:54:16.178Z [INFO] server-8729d1b3-48b0-0b5c-9099-236c1b1a99f2: Installed remote snapshot +2025-01-28T14:54:16.179Z [WARN] server-5401f093-b02f-401e-a07e-c348c61f98f7: skipping application of old log: index=22 +2025-01-28T14:54:16.179Z [WARN] server-8729d1b3-48b0-0b5c-9099-236c1b1a99f2: skipping application of old log: index=22 +2025-01-28T14:54:16.179Z [INFO] server-0640f2fe-9fcd-00cf-0dbf-653e06ad6761: pipelining replication: peer="{Voter server-5401f093-b02f-401e-a07e-c348c61f98f7 5401f093-b02f-401e-a07e-c348c61f98f7}" +2025-01-28T14:54:16.179Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:54:16.179Z [INFO] server-0640f2fe-9fcd-00cf-0dbf-653e06ad6761: pipelining replication: peer="{Voter server-8729d1b3-48b0-0b5c-9099-236c1b1a99f2 8729d1b3-48b0-0b5c-9099-236c1b1a99f2}" +2025-01-28T14:54:17.185Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:54:16.179393266 +0000 UTC m=+41.853883246 (2 nodes), 2.78µs from start of poll, 1.555924516s from cluster start. Timeout at 2025-01-28 14:54:17.184409164 +0000 UTC m=+42.858899164, 1.005015918s after stability +2025-01-28T14:54:17.198Z [INFO] server-0640f2fe-9fcd-00cf-0dbf-653e06ad6761: aborting pipeline replication: peer="{Voter server-5401f093-b02f-401e-a07e-c348c61f98f7 5401f093-b02f-401e-a07e-c348c61f98f7}" +2025-01-28T14:54:17.198Z [INFO] server-0640f2fe-9fcd-00cf-0dbf-653e06ad6761: aborting pipeline replication: peer="{Voter server-8729d1b3-48b0-0b5c-9099-236c1b1a99f2 8729d1b3-48b0-0b5c-9099-236c1b1a99f2}" === RUN TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_false -2025-01-28T14:47:16.100Z [INFO] server-b48fead8-5972-2638-6f3c-ccb7a1f78a17: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b48fead8-5972-2638-6f3c-ccb7a1f78a17 Address:b48fead8-5972-2638-6f3c-ccb7a1f78a17} {Suffrage:Voter ID:server-ccf56c4c-764e-ee95-f90c-3c19dffcac3a Address:ccf56c4c-764e-ee95-f90c-3c19dffcac3a} {Suffrage:Voter ID:server-17035c84-a8b1-8e78-0330-ee95b2f35c9d Address:17035c84-a8b1-8e78-0330-ee95b2f35c9d}]" -2025-01-28T14:47:16.101Z [INFO] server-b48fead8-5972-2638-6f3c-ccb7a1f78a17: entering follower state: follower="Node at b48fead8-5972-2638-6f3c-ccb7a1f78a17 [Follower]" leader-address= leader-id= -2025-01-28T14:47:16.102Z [INFO] server-ccf56c4c-764e-ee95-f90c-3c19dffcac3a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b48fead8-5972-2638-6f3c-ccb7a1f78a17 Address:b48fead8-5972-2638-6f3c-ccb7a1f78a17} {Suffrage:Voter ID:server-ccf56c4c-764e-ee95-f90c-3c19dffcac3a Address:ccf56c4c-764e-ee95-f90c-3c19dffcac3a} {Suffrage:Voter ID:server-17035c84-a8b1-8e78-0330-ee95b2f35c9d Address:17035c84-a8b1-8e78-0330-ee95b2f35c9d}]" -2025-01-28T14:47:16.102Z [INFO] server-ccf56c4c-764e-ee95-f90c-3c19dffcac3a: entering follower state: follower="Node at ccf56c4c-764e-ee95-f90c-3c19dffcac3a [Follower]" leader-address= leader-id= -2025-01-28T14:47:16.103Z [INFO] server-17035c84-a8b1-8e78-0330-ee95b2f35c9d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b48fead8-5972-2638-6f3c-ccb7a1f78a17 Address:b48fead8-5972-2638-6f3c-ccb7a1f78a17} {Suffrage:Voter ID:server-ccf56c4c-764e-ee95-f90c-3c19dffcac3a Address:ccf56c4c-764e-ee95-f90c-3c19dffcac3a} {Suffrage:Voter ID:server-17035c84-a8b1-8e78-0330-ee95b2f35c9d Address:17035c84-a8b1-8e78-0330-ee95b2f35c9d}]" -2025-01-28T14:47:16.103Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:16.104Z [INFO] server-17035c84-a8b1-8e78-0330-ee95b2f35c9d: entering follower state: follower="Node at 17035c84-a8b1-8e78-0330-ee95b2f35c9d [Follower]" leader-address= leader-id= -2025-01-28T14:47:16.703Z [WARN] server-17035c84-a8b1-8e78-0330-ee95b2f35c9d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:16.703Z [INFO] server-17035c84-a8b1-8e78-0330-ee95b2f35c9d: entering candidate state: node="Node at 17035c84-a8b1-8e78-0330-ee95b2f35c9d [Candidate]" term=2 -2025-01-28T14:47:16.704Z [INFO] server-17035c84-a8b1-8e78-0330-ee95b2f35c9d: election won: term=2 tally=2 -2025-01-28T14:47:16.704Z [INFO] server-17035c84-a8b1-8e78-0330-ee95b2f35c9d: entering leader state: leader="Node at 17035c84-a8b1-8e78-0330-ee95b2f35c9d [Leader]" -2025-01-28T14:47:16.705Z [INFO] server-17035c84-a8b1-8e78-0330-ee95b2f35c9d: added peer, starting replication: peer=server-b48fead8-5972-2638-6f3c-ccb7a1f78a17 -2025-01-28T14:47:16.705Z [INFO] server-17035c84-a8b1-8e78-0330-ee95b2f35c9d: added peer, starting replication: peer=server-ccf56c4c-764e-ee95-f90c-3c19dffcac3a -2025-01-28T14:47:16.706Z [INFO] server-17035c84-a8b1-8e78-0330-ee95b2f35c9d: pipelining replication: peer="{Voter server-b48fead8-5972-2638-6f3c-ccb7a1f78a17 b48fead8-5972-2638-6f3c-ccb7a1f78a17}" -2025-01-28T14:47:16.707Z [INFO] server-17035c84-a8b1-8e78-0330-ee95b2f35c9d: pipelining replication: peer="{Voter server-ccf56c4c-764e-ee95-f90c-3c19dffcac3a ccf56c4c-764e-ee95-f90c-3c19dffcac3a}" -2025-01-28T14:47:17.711Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:16.705240132 +0000 UTC m=+19.788646156 (1 nodes), 601.087512ms from start of poll, 605.761984ms from cluster start. Timeout at 2025-01-28 14:47:17.710279466 +0000 UTC m=+20.793685490, 1.005039334s after stability -2025-01-28T14:47:17.721Z [INFO] server-17035c84-a8b1-8e78-0330-ee95b2f35c9d: starting snapshot up to: index=12 -2025-01-28T14:47:17.721Z [INFO] creating new snapshot: path=/tmp/raft2278412720/snapshots/2-12-1738075637721.tmp -2025-01-28T14:47:17.728Z [INFO] server-17035c84-a8b1-8e78-0330-ee95b2f35c9d: snapshot complete up to: index=12 -2025-01-28T14:47:17.737Z [INFO] creating new snapshot: path=/tmp/raft2278412720/snapshots/2-23-1738075637737.tmp -2025-01-28T14:47:17.744Z [INFO] server-17035c84-a8b1-8e78-0330-ee95b2f35c9d: copied to local snapshot: bytes=71 -2025-01-28T14:47:17.746Z [INFO] server-17035c84-a8b1-8e78-0330-ee95b2f35c9d: snapshot restore progress: id=2-23-1738075637737 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:17.747Z [INFO] server-17035c84-a8b1-8e78-0330-ee95b2f35c9d: removing all old logs from log store -2025-01-28T14:47:17.747Z [INFO] server-17035c84-a8b1-8e78-0330-ee95b2f35c9d: compacting logs: from=1 to=22 -2025-01-28T14:47:17.747Z [INFO] server-17035c84-a8b1-8e78-0330-ee95b2f35c9d: restored user snapshot: index=23 -2025-01-28T14:47:17.747Z [ERROR] server-17035c84-a8b1-8e78-0330-ee95b2f35c9d: failed to get log: index=22 error="log not found" -2025-01-28T14:47:17.748Z [INFO] server-17035c84-a8b1-8e78-0330-ee95b2f35c9d: aborting pipeline replication: peer="{Voter server-ccf56c4c-764e-ee95-f90c-3c19dffcac3a ccf56c4c-764e-ee95-f90c-3c19dffcac3a}" -2025-01-28T14:47:17.748Z [ERROR] server-17035c84-a8b1-8e78-0330-ee95b2f35c9d: failed to get log: index=22 error="log not found" -2025-01-28T14:47:17.748Z [ERROR] server-17035c84-a8b1-8e78-0330-ee95b2f35c9d: failed to get log: index=22 error="log not found" -2025-01-28T14:47:17.748Z [INFO] server-17035c84-a8b1-8e78-0330-ee95b2f35c9d: aborting pipeline replication: peer="{Voter server-b48fead8-5972-2638-6f3c-ccb7a1f78a17 b48fead8-5972-2638-6f3c-ccb7a1f78a17}" -2025-01-28T14:47:17.752Z [INFO] creating new snapshot: path=/tmp/raft2291532567/snapshots/2-23-1738075637752.tmp -2025-01-28T14:47:17.755Z [INFO] server-ccf56c4c-764e-ee95-f90c-3c19dffcac3a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:17.756Z [ERROR] server-17035c84-a8b1-8e78-0330-ee95b2f35c9d: failed to get log: index=22 error="log not found" -2025-01-28T14:47:17.759Z [INFO] server-ccf56c4c-764e-ee95-f90c-3c19dffcac3a: copied to local snapshot: bytes=71 -2025-01-28T14:47:17.761Z [INFO] creating new snapshot: path=/tmp/raft1291913436/snapshots/2-23-1738075637761.tmp -2025-01-28T14:47:17.762Z [INFO] server-ccf56c4c-764e-ee95-f90c-3c19dffcac3a: snapshot restore progress: id=2-23-1738075637752 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:17.762Z [INFO] server-ccf56c4c-764e-ee95-f90c-3c19dffcac3a: removing all old logs from log store -2025-01-28T14:47:17.762Z [INFO] server-ccf56c4c-764e-ee95-f90c-3c19dffcac3a: compacting logs: from=1 to=22 -2025-01-28T14:47:17.763Z [INFO] server-ccf56c4c-764e-ee95-f90c-3c19dffcac3a: Installed remote snapshot -2025-01-28T14:47:17.764Z [INFO] server-17035c84-a8b1-8e78-0330-ee95b2f35c9d: pipelining replication: peer="{Voter server-ccf56c4c-764e-ee95-f90c-3c19dffcac3a ccf56c4c-764e-ee95-f90c-3c19dffcac3a}" -2025-01-28T14:47:17.764Z [INFO] server-b48fead8-5972-2638-6f3c-ccb7a1f78a17: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:17.765Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:47:17.768Z [INFO] server-b48fead8-5972-2638-6f3c-ccb7a1f78a17: copied to local snapshot: bytes=71 -2025-01-28T14:47:17.771Z [INFO] server-b48fead8-5972-2638-6f3c-ccb7a1f78a17: snapshot restore progress: id=2-23-1738075637761 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:17.771Z [INFO] server-b48fead8-5972-2638-6f3c-ccb7a1f78a17: removing all old logs from log store -2025-01-28T14:47:17.771Z [INFO] server-b48fead8-5972-2638-6f3c-ccb7a1f78a17: compacting logs: from=1 to=22 -2025-01-28T14:47:17.772Z [INFO] server-b48fead8-5972-2638-6f3c-ccb7a1f78a17: Installed remote snapshot -2025-01-28T14:47:17.773Z [INFO] server-17035c84-a8b1-8e78-0330-ee95b2f35c9d: pipelining replication: peer="{Voter server-b48fead8-5972-2638-6f3c-ccb7a1f78a17 b48fead8-5972-2638-6f3c-ccb7a1f78a17}" -2025-01-28T14:47:18.771Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:47:17.765522278 +0000 UTC m=+20.848927969 (2 nodes), 16.334µs from start of poll, 1.666043797s from cluster start. Timeout at 2025-01-28 14:47:18.77061428 +0000 UTC m=+21.854020304, 1.005092335s after stability -2025-01-28T14:47:18.790Z [INFO] server-17035c84-a8b1-8e78-0330-ee95b2f35c9d: aborting pipeline replication: peer="{Voter server-b48fead8-5972-2638-6f3c-ccb7a1f78a17 b48fead8-5972-2638-6f3c-ccb7a1f78a17}" -2025-01-28T14:47:18.790Z [INFO] server-17035c84-a8b1-8e78-0330-ee95b2f35c9d: aborting pipeline replication: peer="{Voter server-ccf56c4c-764e-ee95-f90c-3c19dffcac3a ccf56c4c-764e-ee95-f90c-3c19dffcac3a}" +2025-01-28T14:54:17.204Z [INFO] server-9cf0ae6c-d9fe-370f-2c7c-54d3af84533b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9cf0ae6c-d9fe-370f-2c7c-54d3af84533b Address:9cf0ae6c-d9fe-370f-2c7c-54d3af84533b} {Suffrage:Voter ID:server-c040e7ec-138c-ac03-79b0-8c7b0bc18403 Address:c040e7ec-138c-ac03-79b0-8c7b0bc18403} {Suffrage:Voter ID:server-9c609124-1208-e526-d005-97b61985b125 Address:9c609124-1208-e526-d005-97b61985b125}]" +2025-01-28T14:54:17.204Z [INFO] server-9cf0ae6c-d9fe-370f-2c7c-54d3af84533b: entering follower state: follower="Node at 9cf0ae6c-d9fe-370f-2c7c-54d3af84533b [Follower]" leader-address= leader-id= +2025-01-28T14:54:17.205Z [INFO] server-c040e7ec-138c-ac03-79b0-8c7b0bc18403: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9cf0ae6c-d9fe-370f-2c7c-54d3af84533b Address:9cf0ae6c-d9fe-370f-2c7c-54d3af84533b} {Suffrage:Voter ID:server-c040e7ec-138c-ac03-79b0-8c7b0bc18403 Address:c040e7ec-138c-ac03-79b0-8c7b0bc18403} {Suffrage:Voter ID:server-9c609124-1208-e526-d005-97b61985b125 Address:9c609124-1208-e526-d005-97b61985b125}]" +2025-01-28T14:54:17.206Z [INFO] server-c040e7ec-138c-ac03-79b0-8c7b0bc18403: entering follower state: follower="Node at c040e7ec-138c-ac03-79b0-8c7b0bc18403 [Follower]" leader-address= leader-id= +2025-01-28T14:54:17.206Z [INFO] server-9c609124-1208-e526-d005-97b61985b125: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9cf0ae6c-d9fe-370f-2c7c-54d3af84533b Address:9cf0ae6c-d9fe-370f-2c7c-54d3af84533b} {Suffrage:Voter ID:server-c040e7ec-138c-ac03-79b0-8c7b0bc18403 Address:c040e7ec-138c-ac03-79b0-8c7b0bc18403} {Suffrage:Voter ID:server-9c609124-1208-e526-d005-97b61985b125 Address:9c609124-1208-e526-d005-97b61985b125}]" +2025-01-28T14:54:17.206Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:17.206Z [INFO] server-9c609124-1208-e526-d005-97b61985b125: entering follower state: follower="Node at 9c609124-1208-e526-d005-97b61985b125 [Follower]" leader-address= leader-id= +2025-01-28T14:54:17.904Z [WARN] server-c040e7ec-138c-ac03-79b0-8c7b0bc18403: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:17.904Z [INFO] server-c040e7ec-138c-ac03-79b0-8c7b0bc18403: entering candidate state: node="Node at c040e7ec-138c-ac03-79b0-8c7b0bc18403 [Candidate]" term=2 +2025-01-28T14:54:17.905Z [INFO] server-c040e7ec-138c-ac03-79b0-8c7b0bc18403: election won: term=2 tally=2 +2025-01-28T14:54:17.905Z [INFO] server-c040e7ec-138c-ac03-79b0-8c7b0bc18403: entering leader state: leader="Node at c040e7ec-138c-ac03-79b0-8c7b0bc18403 [Leader]" +2025-01-28T14:54:17.905Z [INFO] server-c040e7ec-138c-ac03-79b0-8c7b0bc18403: added peer, starting replication: peer=server-9cf0ae6c-d9fe-370f-2c7c-54d3af84533b +2025-01-28T14:54:17.905Z [INFO] server-c040e7ec-138c-ac03-79b0-8c7b0bc18403: added peer, starting replication: peer=server-9c609124-1208-e526-d005-97b61985b125 +2025-01-28T14:54:17.905Z [INFO] server-c040e7ec-138c-ac03-79b0-8c7b0bc18403: pipelining replication: peer="{Voter server-9cf0ae6c-d9fe-370f-2c7c-54d3af84533b 9cf0ae6c-d9fe-370f-2c7c-54d3af84533b}" +2025-01-28T14:54:17.905Z [INFO] server-c040e7ec-138c-ac03-79b0-8c7b0bc18403: pipelining replication: peer="{Voter server-9c609124-1208-e526-d005-97b61985b125 9c609124-1208-e526-d005-97b61985b125}" +2025-01-28T14:54:18.910Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:17.905109743 +0000 UTC m=+43.579599703 (1 nodes), 698.64739ms from start of poll, 701.248545ms from cluster start. Timeout at 2025-01-28 14:54:18.910117182 +0000 UTC m=+44.584607242, 1.005007539s after stability +2025-01-28T14:54:18.911Z [INFO] server-c040e7ec-138c-ac03-79b0-8c7b0bc18403: starting snapshot up to: index=12 +2025-01-28T14:54:18.911Z [INFO] creating new snapshot: path=/tmp/raft2401974711/snapshots/2-12-1738076058911.tmp +2025-01-28T14:54:18.913Z [INFO] server-c040e7ec-138c-ac03-79b0-8c7b0bc18403: snapshot complete up to: index=12 +2025-01-28T14:54:18.915Z [INFO] creating new snapshot: path=/tmp/raft2401974711/snapshots/2-23-1738076058915.tmp +2025-01-28T14:54:18.916Z [INFO] server-c040e7ec-138c-ac03-79b0-8c7b0bc18403: copied to local snapshot: bytes=71 +2025-01-28T14:54:18.917Z [INFO] server-c040e7ec-138c-ac03-79b0-8c7b0bc18403: snapshot restore progress: id=2-23-1738076058915 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:18.917Z [INFO] server-c040e7ec-138c-ac03-79b0-8c7b0bc18403: removing all old logs from log store +2025-01-28T14:54:18.917Z [INFO] server-c040e7ec-138c-ac03-79b0-8c7b0bc18403: compacting logs: from=1 to=22 +2025-01-28T14:54:18.917Z [INFO] server-c040e7ec-138c-ac03-79b0-8c7b0bc18403: restored user snapshot: index=23 +2025-01-28T14:54:18.917Z [ERROR] server-c040e7ec-138c-ac03-79b0-8c7b0bc18403: failed to get log: index=22 error="log not found" +2025-01-28T14:54:18.917Z [INFO] server-c040e7ec-138c-ac03-79b0-8c7b0bc18403: aborting pipeline replication: peer="{Voter server-9c609124-1208-e526-d005-97b61985b125 9c609124-1208-e526-d005-97b61985b125}" +2025-01-28T14:54:18.917Z [ERROR] server-c040e7ec-138c-ac03-79b0-8c7b0bc18403: failed to get log: index=22 error="log not found" +2025-01-28T14:54:18.917Z [INFO] server-c040e7ec-138c-ac03-79b0-8c7b0bc18403: aborting pipeline replication: peer="{Voter server-9cf0ae6c-d9fe-370f-2c7c-54d3af84533b 9cf0ae6c-d9fe-370f-2c7c-54d3af84533b}" +2025-01-28T14:54:18.924Z [ERROR] server-c040e7ec-138c-ac03-79b0-8c7b0bc18403: failed to get log: index=22 error="log not found" +2025-01-28T14:54:18.925Z [INFO] creating new snapshot: path=/tmp/raft3936816475/snapshots/2-23-1738076058925.tmp +2025-01-28T14:54:18.926Z [INFO] server-9cf0ae6c-d9fe-370f-2c7c-54d3af84533b: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:18.927Z [INFO] server-9cf0ae6c-d9fe-370f-2c7c-54d3af84533b: copied to local snapshot: bytes=71 +2025-01-28T14:54:18.927Z [ERROR] server-c040e7ec-138c-ac03-79b0-8c7b0bc18403: failed to get log: index=22 error="log not found" +2025-01-28T14:54:18.927Z [INFO] server-9cf0ae6c-d9fe-370f-2c7c-54d3af84533b: snapshot restore progress: id=2-23-1738076058925 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:18.927Z [INFO] server-9cf0ae6c-d9fe-370f-2c7c-54d3af84533b: removing all old logs from log store +2025-01-28T14:54:18.927Z [INFO] server-9cf0ae6c-d9fe-370f-2c7c-54d3af84533b: compacting logs: from=1 to=22 +2025-01-28T14:54:18.927Z [INFO] server-9cf0ae6c-d9fe-370f-2c7c-54d3af84533b: Installed remote snapshot +2025-01-28T14:54:18.928Z [WARN] server-9cf0ae6c-d9fe-370f-2c7c-54d3af84533b: skipping application of old log: index=22 +2025-01-28T14:54:18.928Z [INFO] creating new snapshot: path=/tmp/raft2466843163/snapshots/2-23-1738076058928.tmp +2025-01-28T14:54:18.928Z [INFO] server-c040e7ec-138c-ac03-79b0-8c7b0bc18403: pipelining replication: peer="{Voter server-9cf0ae6c-d9fe-370f-2c7c-54d3af84533b 9cf0ae6c-d9fe-370f-2c7c-54d3af84533b}" +2025-01-28T14:54:18.928Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:54:18.928Z [INFO] server-9c609124-1208-e526-d005-97b61985b125: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:18.929Z [INFO] server-9c609124-1208-e526-d005-97b61985b125: copied to local snapshot: bytes=71 +2025-01-28T14:54:18.930Z [INFO] server-9c609124-1208-e526-d005-97b61985b125: snapshot restore progress: id=2-23-1738076058928 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:18.930Z [INFO] server-9c609124-1208-e526-d005-97b61985b125: removing all old logs from log store +2025-01-28T14:54:18.930Z [INFO] server-9c609124-1208-e526-d005-97b61985b125: compacting logs: from=1 to=22 +2025-01-28T14:54:18.930Z [INFO] server-9c609124-1208-e526-d005-97b61985b125: Installed remote snapshot +2025-01-28T14:54:18.930Z [INFO] server-c040e7ec-138c-ac03-79b0-8c7b0bc18403: pipelining replication: peer="{Voter server-9c609124-1208-e526-d005-97b61985b125 9c609124-1208-e526-d005-97b61985b125}" +2025-01-28T14:54:19.934Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:54:18.928462981 +0000 UTC m=+44.602953001 (2 nodes), 3.44µs from start of poll, 1.724601843s from cluster start. Timeout at 2025-01-28 14:54:19.933497578 +0000 UTC m=+45.607987558, 1.005034557s after stability +2025-01-28T14:54:19.947Z [INFO] server-c040e7ec-138c-ac03-79b0-8c7b0bc18403: aborting pipeline replication: peer="{Voter server-9c609124-1208-e526-d005-97b61985b125 9c609124-1208-e526-d005-97b61985b125}" +2025-01-28T14:54:19.947Z [INFO] server-c040e7ec-138c-ac03-79b0-8c7b0bc18403: aborting pipeline replication: peer="{Voter server-9cf0ae6c-d9fe-370f-2c7c-54d3af84533b 9cf0ae6c-d9fe-370f-2c7c-54d3af84533b}" === RUN TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_true -2025-01-28T14:47:18.815Z [INFO] server-4ff75592-9024-530c-833a-dc93f2cf222f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4ff75592-9024-530c-833a-dc93f2cf222f Address:4ff75592-9024-530c-833a-dc93f2cf222f} {Suffrage:Voter ID:server-d9d3529d-9515-358b-f54a-9e5fd80525d8 Address:d9d3529d-9515-358b-f54a-9e5fd80525d8} {Suffrage:Voter ID:server-677a9cf7-1e63-5934-ed6a-b93205f9e737 Address:677a9cf7-1e63-5934-ed6a-b93205f9e737}]" -2025-01-28T14:47:18.816Z [INFO] server-4ff75592-9024-530c-833a-dc93f2cf222f: entering follower state: follower="Node at 4ff75592-9024-530c-833a-dc93f2cf222f [Follower]" leader-address= leader-id= -2025-01-28T14:47:18.816Z [INFO] server-d9d3529d-9515-358b-f54a-9e5fd80525d8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4ff75592-9024-530c-833a-dc93f2cf222f Address:4ff75592-9024-530c-833a-dc93f2cf222f} {Suffrage:Voter ID:server-d9d3529d-9515-358b-f54a-9e5fd80525d8 Address:d9d3529d-9515-358b-f54a-9e5fd80525d8} {Suffrage:Voter ID:server-677a9cf7-1e63-5934-ed6a-b93205f9e737 Address:677a9cf7-1e63-5934-ed6a-b93205f9e737}]" -2025-01-28T14:47:18.817Z [INFO] server-d9d3529d-9515-358b-f54a-9e5fd80525d8: entering follower state: follower="Node at d9d3529d-9515-358b-f54a-9e5fd80525d8 [Follower]" leader-address= leader-id= -2025-01-28T14:47:18.818Z [INFO] server-677a9cf7-1e63-5934-ed6a-b93205f9e737: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4ff75592-9024-530c-833a-dc93f2cf222f Address:4ff75592-9024-530c-833a-dc93f2cf222f} {Suffrage:Voter ID:server-d9d3529d-9515-358b-f54a-9e5fd80525d8 Address:d9d3529d-9515-358b-f54a-9e5fd80525d8} {Suffrage:Voter ID:server-677a9cf7-1e63-5934-ed6a-b93205f9e737 Address:677a9cf7-1e63-5934-ed6a-b93205f9e737}]" -2025-01-28T14:47:18.818Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:18.819Z [INFO] server-677a9cf7-1e63-5934-ed6a-b93205f9e737: entering follower state: follower="Node at 677a9cf7-1e63-5934-ed6a-b93205f9e737 [Follower]" leader-address= leader-id= -2025-01-28T14:47:19.444Z [WARN] server-677a9cf7-1e63-5934-ed6a-b93205f9e737: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:19.444Z [INFO] server-677a9cf7-1e63-5934-ed6a-b93205f9e737: entering candidate state: node="Node at 677a9cf7-1e63-5934-ed6a-b93205f9e737 [Candidate]" term=2 -2025-01-28T14:47:19.446Z [INFO] server-677a9cf7-1e63-5934-ed6a-b93205f9e737: election won: term=2 tally=2 -2025-01-28T14:47:19.446Z [INFO] server-677a9cf7-1e63-5934-ed6a-b93205f9e737: entering leader state: leader="Node at 677a9cf7-1e63-5934-ed6a-b93205f9e737 [Leader]" -2025-01-28T14:47:19.446Z [INFO] server-677a9cf7-1e63-5934-ed6a-b93205f9e737: added peer, starting replication: peer=server-4ff75592-9024-530c-833a-dc93f2cf222f -2025-01-28T14:47:19.446Z [INFO] server-677a9cf7-1e63-5934-ed6a-b93205f9e737: added peer, starting replication: peer=server-d9d3529d-9515-358b-f54a-9e5fd80525d8 -2025-01-28T14:47:19.447Z [INFO] server-677a9cf7-1e63-5934-ed6a-b93205f9e737: pipelining replication: peer="{Voter server-4ff75592-9024-530c-833a-dc93f2cf222f 4ff75592-9024-530c-833a-dc93f2cf222f}" -2025-01-28T14:47:19.447Z [INFO] server-677a9cf7-1e63-5934-ed6a-b93205f9e737: pipelining replication: peer="{Voter server-d9d3529d-9515-358b-f54a-9e5fd80525d8 d9d3529d-9515-358b-f54a-9e5fd80525d8}" -2025-01-28T14:47:20.452Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:19.44656252 +0000 UTC m=+22.529967878 (1 nodes), 627.413628ms from start of poll, 632.306107ms from cluster start. Timeout at 2025-01-28 14:47:20.451580854 +0000 UTC m=+23.534986879, 1.005019001s after stability -2025-01-28T14:47:20.460Z [INFO] server-677a9cf7-1e63-5934-ed6a-b93205f9e737: starting snapshot up to: index=12 -2025-01-28T14:47:20.460Z [INFO] creating new snapshot: path=/tmp/raft2604533988/snapshots/2-12-1738075640460.tmp -2025-01-28T14:47:20.467Z [INFO] server-677a9cf7-1e63-5934-ed6a-b93205f9e737: snapshot complete up to: index=12 -2025-01-28T14:47:20.485Z [INFO] server-a0f0a4a8-93b0-0591-dbfc-a0ece30e1aaa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a0f0a4a8-93b0-0591-dbfc-a0ece30e1aaa Address:a0f0a4a8-93b0-0591-dbfc-a0ece30e1aaa} {Suffrage:Voter ID:server-f507ec73-74b1-74b3-f0b0-8e1e844cb235 Address:f507ec73-74b1-74b3-f0b0-8e1e844cb235} {Suffrage:Voter ID:server-7508b3ae-4f6c-1bd2-1d2e-8000a5d2cb78 Address:7508b3ae-4f6c-1bd2-1d2e-8000a5d2cb78}]" -2025-01-28T14:47:20.486Z [INFO] server-a0f0a4a8-93b0-0591-dbfc-a0ece30e1aaa: entering follower state: follower="Node at a0f0a4a8-93b0-0591-dbfc-a0ece30e1aaa [Follower]" leader-address= leader-id= -2025-01-28T14:47:20.487Z [INFO] server-f507ec73-74b1-74b3-f0b0-8e1e844cb235: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a0f0a4a8-93b0-0591-dbfc-a0ece30e1aaa Address:a0f0a4a8-93b0-0591-dbfc-a0ece30e1aaa} {Suffrage:Voter ID:server-f507ec73-74b1-74b3-f0b0-8e1e844cb235 Address:f507ec73-74b1-74b3-f0b0-8e1e844cb235} {Suffrage:Voter ID:server-7508b3ae-4f6c-1bd2-1d2e-8000a5d2cb78 Address:7508b3ae-4f6c-1bd2-1d2e-8000a5d2cb78}]" -2025-01-28T14:47:20.488Z [INFO] server-f507ec73-74b1-74b3-f0b0-8e1e844cb235: entering follower state: follower="Node at f507ec73-74b1-74b3-f0b0-8e1e844cb235 [Follower]" leader-address= leader-id= -2025-01-28T14:47:20.488Z [INFO] server-7508b3ae-4f6c-1bd2-1d2e-8000a5d2cb78: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a0f0a4a8-93b0-0591-dbfc-a0ece30e1aaa Address:a0f0a4a8-93b0-0591-dbfc-a0ece30e1aaa} {Suffrage:Voter ID:server-f507ec73-74b1-74b3-f0b0-8e1e844cb235 Address:f507ec73-74b1-74b3-f0b0-8e1e844cb235} {Suffrage:Voter ID:server-7508b3ae-4f6c-1bd2-1d2e-8000a5d2cb78 Address:7508b3ae-4f6c-1bd2-1d2e-8000a5d2cb78}]" -2025-01-28T14:47:20.489Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:20.489Z [INFO] server-7508b3ae-4f6c-1bd2-1d2e-8000a5d2cb78: entering follower state: follower="Node at 7508b3ae-4f6c-1bd2-1d2e-8000a5d2cb78 [Follower]" leader-address= leader-id= -2025-01-28T14:47:21.104Z [WARN] server-f507ec73-74b1-74b3-f0b0-8e1e844cb235: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:21.104Z [INFO] server-f507ec73-74b1-74b3-f0b0-8e1e844cb235: entering candidate state: node="Node at f507ec73-74b1-74b3-f0b0-8e1e844cb235 [Candidate]" term=2 -2025-01-28T14:47:21.105Z [INFO] server-f507ec73-74b1-74b3-f0b0-8e1e844cb235: election won: term=2 tally=2 -2025-01-28T14:47:21.106Z [INFO] server-f507ec73-74b1-74b3-f0b0-8e1e844cb235: entering leader state: leader="Node at f507ec73-74b1-74b3-f0b0-8e1e844cb235 [Leader]" -2025-01-28T14:47:21.106Z [INFO] server-f507ec73-74b1-74b3-f0b0-8e1e844cb235: added peer, starting replication: peer=server-a0f0a4a8-93b0-0591-dbfc-a0ece30e1aaa -2025-01-28T14:47:21.106Z [INFO] server-f507ec73-74b1-74b3-f0b0-8e1e844cb235: added peer, starting replication: peer=server-7508b3ae-4f6c-1bd2-1d2e-8000a5d2cb78 -2025-01-28T14:47:21.108Z [INFO] server-f507ec73-74b1-74b3-f0b0-8e1e844cb235: pipelining replication: peer="{Voter server-a0f0a4a8-93b0-0591-dbfc-a0ece30e1aaa a0f0a4a8-93b0-0591-dbfc-a0ece30e1aaa}" -2025-01-28T14:47:21.108Z [INFO] server-f507ec73-74b1-74b3-f0b0-8e1e844cb235: pipelining replication: peer="{Voter server-7508b3ae-4f6c-1bd2-1d2e-8000a5d2cb78 7508b3ae-4f6c-1bd2-1d2e-8000a5d2cb78}" -2025-01-28T14:47:22.111Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:21.106248795 +0000 UTC m=+24.189655486 (1 nodes), 616.747977ms from start of poll, 621.601121ms from cluster start. Timeout at 2025-01-28 14:47:22.111291463 +0000 UTC m=+25.194697821, 1.005042335s after stability -2025-01-28T14:47:22.114Z [INFO] creating new snapshot: path=/tmp/raft656545943/snapshots/2-13-1738075642114.tmp -2025-01-28T14:47:22.121Z [INFO] server-f507ec73-74b1-74b3-f0b0-8e1e844cb235: copied to local snapshot: bytes=71 -2025-01-28T14:47:22.123Z [INFO] server-f507ec73-74b1-74b3-f0b0-8e1e844cb235: snapshot restore progress: id=2-13-1738075642114 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:22.124Z [INFO] server-f507ec73-74b1-74b3-f0b0-8e1e844cb235: restored user snapshot: index=13 -2025-01-28T14:47:22.125Z [ERROR] server-f507ec73-74b1-74b3-f0b0-8e1e844cb235: failed to get log: index=3 error="log not found" -2025-01-28T14:47:22.125Z [INFO] server-f507ec73-74b1-74b3-f0b0-8e1e844cb235: aborting pipeline replication: peer="{Voter server-a0f0a4a8-93b0-0591-dbfc-a0ece30e1aaa a0f0a4a8-93b0-0591-dbfc-a0ece30e1aaa}" -2025-01-28T14:47:22.125Z [ERROR] server-f507ec73-74b1-74b3-f0b0-8e1e844cb235: failed to get log: index=3 error="log not found" -2025-01-28T14:47:22.126Z [INFO] server-f507ec73-74b1-74b3-f0b0-8e1e844cb235: aborting pipeline replication: peer="{Voter server-7508b3ae-4f6c-1bd2-1d2e-8000a5d2cb78 7508b3ae-4f6c-1bd2-1d2e-8000a5d2cb78}" -2025-01-28T14:47:22.133Z [ERROR] server-f507ec73-74b1-74b3-f0b0-8e1e844cb235: failed to get log: index=3 error="log not found" -2025-01-28T14:47:22.135Z [ERROR] server-f507ec73-74b1-74b3-f0b0-8e1e844cb235: failed to get log: index=3 error="log not found" -2025-01-28T14:47:22.137Z [INFO] creating new snapshot: path=/tmp/raft1878919715/snapshots/2-13-1738075642137.tmp -2025-01-28T14:47:22.140Z [INFO] creating new snapshot: path=/tmp/raft2425637237/snapshots/2-13-1738075642140.tmp -2025-01-28T14:47:22.141Z [INFO] server-a0f0a4a8-93b0-0591-dbfc-a0ece30e1aaa: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:22.144Z [INFO] server-7508b3ae-4f6c-1bd2-1d2e-8000a5d2cb78: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:22.145Z [INFO] server-a0f0a4a8-93b0-0591-dbfc-a0ece30e1aaa: copied to local snapshot: bytes=71 -2025-01-28T14:47:22.153Z [INFO] server-7508b3ae-4f6c-1bd2-1d2e-8000a5d2cb78: copied to local snapshot: bytes=71 -2025-01-28T14:47:22.156Z [INFO] server-a0f0a4a8-93b0-0591-dbfc-a0ece30e1aaa: snapshot restore progress: id=2-13-1738075642137 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:22.157Z [INFO] server-a0f0a4a8-93b0-0591-dbfc-a0ece30e1aaa: Installed remote snapshot -2025-01-28T14:47:22.163Z [INFO] server-7508b3ae-4f6c-1bd2-1d2e-8000a5d2cb78: snapshot restore progress: id=2-13-1738075642140 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:22.163Z [INFO] server-f507ec73-74b1-74b3-f0b0-8e1e844cb235: pipelining replication: peer="{Voter server-a0f0a4a8-93b0-0591-dbfc-a0ece30e1aaa a0f0a4a8-93b0-0591-dbfc-a0ece30e1aaa}" -2025-01-28T14:47:22.163Z [INFO] server-7508b3ae-4f6c-1bd2-1d2e-8000a5d2cb78: Installed remote snapshot -2025-01-28T14:47:22.164Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:47:22.164Z [INFO] server-f507ec73-74b1-74b3-f0b0-8e1e844cb235: pipelining replication: peer="{Voter server-7508b3ae-4f6c-1bd2-1d2e-8000a5d2cb78 7508b3ae-4f6c-1bd2-1d2e-8000a5d2cb78}" -2025-01-28T14:47:23.169Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:47:22.164148204 +0000 UTC m=+25.247553228 (2 nodes), 9.667µs from start of poll, 1.679498863s from cluster start. Timeout at 2025-01-28 14:47:23.169332876 +0000 UTC m=+26.252738901, 1.005185673s after stability -2025-01-28T14:47:23.186Z [INFO] server-677a9cf7-1e63-5934-ed6a-b93205f9e737: aborting pipeline replication: peer="{Voter server-4ff75592-9024-530c-833a-dc93f2cf222f 4ff75592-9024-530c-833a-dc93f2cf222f}" -2025-01-28T14:47:23.186Z [INFO] server-677a9cf7-1e63-5934-ed6a-b93205f9e737: aborting pipeline replication: peer="{Voter server-d9d3529d-9515-358b-f54a-9e5fd80525d8 d9d3529d-9515-358b-f54a-9e5fd80525d8}" +2025-01-28T14:54:19.952Z [INFO] server-1ef7e83a-bb69-a369-9bb8-6ca81262f962: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1ef7e83a-bb69-a369-9bb8-6ca81262f962 Address:1ef7e83a-bb69-a369-9bb8-6ca81262f962} {Suffrage:Voter ID:server-9c0e008d-5edd-34cd-4663-902eb315721b Address:9c0e008d-5edd-34cd-4663-902eb315721b} {Suffrage:Voter ID:server-9e3fd7b9-09db-d49d-37e3-57a86907dd99 Address:9e3fd7b9-09db-d49d-37e3-57a86907dd99}]" +2025-01-28T14:54:19.953Z [INFO] server-1ef7e83a-bb69-a369-9bb8-6ca81262f962: entering follower state: follower="Node at 1ef7e83a-bb69-a369-9bb8-6ca81262f962 [Follower]" leader-address= leader-id= +2025-01-28T14:54:19.953Z [INFO] server-9c0e008d-5edd-34cd-4663-902eb315721b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1ef7e83a-bb69-a369-9bb8-6ca81262f962 Address:1ef7e83a-bb69-a369-9bb8-6ca81262f962} {Suffrage:Voter ID:server-9c0e008d-5edd-34cd-4663-902eb315721b Address:9c0e008d-5edd-34cd-4663-902eb315721b} {Suffrage:Voter ID:server-9e3fd7b9-09db-d49d-37e3-57a86907dd99 Address:9e3fd7b9-09db-d49d-37e3-57a86907dd99}]" +2025-01-28T14:54:19.953Z [INFO] server-9c0e008d-5edd-34cd-4663-902eb315721b: entering follower state: follower="Node at 9c0e008d-5edd-34cd-4663-902eb315721b [Follower]" leader-address= leader-id= +2025-01-28T14:54:19.953Z [INFO] server-9e3fd7b9-09db-d49d-37e3-57a86907dd99: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1ef7e83a-bb69-a369-9bb8-6ca81262f962 Address:1ef7e83a-bb69-a369-9bb8-6ca81262f962} {Suffrage:Voter ID:server-9c0e008d-5edd-34cd-4663-902eb315721b Address:9c0e008d-5edd-34cd-4663-902eb315721b} {Suffrage:Voter ID:server-9e3fd7b9-09db-d49d-37e3-57a86907dd99 Address:9e3fd7b9-09db-d49d-37e3-57a86907dd99}]" +2025-01-28T14:54:19.953Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:19.953Z [INFO] server-9e3fd7b9-09db-d49d-37e3-57a86907dd99: entering follower state: follower="Node at 9e3fd7b9-09db-d49d-37e3-57a86907dd99 [Follower]" leader-address= leader-id= +2025-01-28T14:54:20.484Z [WARN] server-9c0e008d-5edd-34cd-4663-902eb315721b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:20.484Z [INFO] server-9c0e008d-5edd-34cd-4663-902eb315721b: entering candidate state: node="Node at 9c0e008d-5edd-34cd-4663-902eb315721b [Candidate]" term=2 +2025-01-28T14:54:20.485Z [INFO] server-9c0e008d-5edd-34cd-4663-902eb315721b: election won: term=2 tally=2 +2025-01-28T14:54:20.485Z [INFO] server-9c0e008d-5edd-34cd-4663-902eb315721b: entering leader state: leader="Node at 9c0e008d-5edd-34cd-4663-902eb315721b [Leader]" +2025-01-28T14:54:20.485Z [INFO] server-9c0e008d-5edd-34cd-4663-902eb315721b: added peer, starting replication: peer=server-1ef7e83a-bb69-a369-9bb8-6ca81262f962 +2025-01-28T14:54:20.485Z [INFO] server-9c0e008d-5edd-34cd-4663-902eb315721b: added peer, starting replication: peer=server-9e3fd7b9-09db-d49d-37e3-57a86907dd99 +2025-01-28T14:54:20.485Z [INFO] server-9c0e008d-5edd-34cd-4663-902eb315721b: pipelining replication: peer="{Voter server-9e3fd7b9-09db-d49d-37e3-57a86907dd99 9e3fd7b9-09db-d49d-37e3-57a86907dd99}" +2025-01-28T14:54:20.485Z [INFO] server-9c0e008d-5edd-34cd-4663-902eb315721b: pipelining replication: peer="{Voter server-1ef7e83a-bb69-a369-9bb8-6ca81262f962 1ef7e83a-bb69-a369-9bb8-6ca81262f962}" +2025-01-28T14:54:21.490Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:20.485216618 +0000 UTC m=+46.159706598 (1 nodes), 531.612528ms from start of poll, 532.722783ms from cluster start. Timeout at 2025-01-28 14:54:21.490224318 +0000 UTC m=+47.164714418, 1.00500782s after stability +2025-01-28T14:54:21.492Z [INFO] server-9c0e008d-5edd-34cd-4663-902eb315721b: starting snapshot up to: index=12 +2025-01-28T14:54:21.492Z [INFO] creating new snapshot: path=/tmp/raft924787358/snapshots/2-12-1738076061492.tmp +2025-01-28T14:54:21.669Z [INFO] server-9c0e008d-5edd-34cd-4663-902eb315721b: snapshot complete up to: index=12 +2025-01-28T14:54:21.673Z [INFO] server-796717a3-8be9-6a90-c3ff-babcfc941ad1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-796717a3-8be9-6a90-c3ff-babcfc941ad1 Address:796717a3-8be9-6a90-c3ff-babcfc941ad1} {Suffrage:Voter ID:server-6e60e5aa-c036-8918-49be-ab3c8b222b91 Address:6e60e5aa-c036-8918-49be-ab3c8b222b91} {Suffrage:Voter ID:server-d1afff6d-4fc0-3302-952c-38c375c42ee7 Address:d1afff6d-4fc0-3302-952c-38c375c42ee7}]" +2025-01-28T14:54:21.673Z [INFO] server-796717a3-8be9-6a90-c3ff-babcfc941ad1: entering follower state: follower="Node at 796717a3-8be9-6a90-c3ff-babcfc941ad1 [Follower]" leader-address= leader-id= +2025-01-28T14:54:21.673Z [INFO] server-6e60e5aa-c036-8918-49be-ab3c8b222b91: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-796717a3-8be9-6a90-c3ff-babcfc941ad1 Address:796717a3-8be9-6a90-c3ff-babcfc941ad1} {Suffrage:Voter ID:server-6e60e5aa-c036-8918-49be-ab3c8b222b91 Address:6e60e5aa-c036-8918-49be-ab3c8b222b91} {Suffrage:Voter ID:server-d1afff6d-4fc0-3302-952c-38c375c42ee7 Address:d1afff6d-4fc0-3302-952c-38c375c42ee7}]" +2025-01-28T14:54:21.673Z [INFO] server-6e60e5aa-c036-8918-49be-ab3c8b222b91: entering follower state: follower="Node at 6e60e5aa-c036-8918-49be-ab3c8b222b91 [Follower]" leader-address= leader-id= +2025-01-28T14:54:21.673Z [INFO] server-d1afff6d-4fc0-3302-952c-38c375c42ee7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-796717a3-8be9-6a90-c3ff-babcfc941ad1 Address:796717a3-8be9-6a90-c3ff-babcfc941ad1} {Suffrage:Voter ID:server-6e60e5aa-c036-8918-49be-ab3c8b222b91 Address:6e60e5aa-c036-8918-49be-ab3c8b222b91} {Suffrage:Voter ID:server-d1afff6d-4fc0-3302-952c-38c375c42ee7 Address:d1afff6d-4fc0-3302-952c-38c375c42ee7}]" +2025-01-28T14:54:21.673Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:21.673Z [INFO] server-d1afff6d-4fc0-3302-952c-38c375c42ee7: entering follower state: follower="Node at d1afff6d-4fc0-3302-952c-38c375c42ee7 [Follower]" leader-address= leader-id= +2025-01-28T14:54:22.181Z [WARN] server-796717a3-8be9-6a90-c3ff-babcfc941ad1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:22.181Z [INFO] server-796717a3-8be9-6a90-c3ff-babcfc941ad1: entering candidate state: node="Node at 796717a3-8be9-6a90-c3ff-babcfc941ad1 [Candidate]" term=2 +2025-01-28T14:54:22.181Z [WARN] server-d1afff6d-4fc0-3302-952c-38c375c42ee7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:22.181Z [INFO] server-d1afff6d-4fc0-3302-952c-38c375c42ee7: entering candidate state: node="Node at d1afff6d-4fc0-3302-952c-38c375c42ee7 [Candidate]" term=2 +2025-01-28T14:54:22.181Z [INFO] server-796717a3-8be9-6a90-c3ff-babcfc941ad1: election won: term=2 tally=2 +2025-01-28T14:54:22.181Z [INFO] server-d1afff6d-4fc0-3302-952c-38c375c42ee7: duplicate requestVote for same term: term=2 +2025-01-28T14:54:22.181Z [INFO] server-796717a3-8be9-6a90-c3ff-babcfc941ad1: entering leader state: leader="Node at 796717a3-8be9-6a90-c3ff-babcfc941ad1 [Leader]" +2025-01-28T14:54:22.181Z [INFO] server-796717a3-8be9-6a90-c3ff-babcfc941ad1: added peer, starting replication: peer=server-6e60e5aa-c036-8918-49be-ab3c8b222b91 +2025-01-28T14:54:22.181Z [INFO] server-796717a3-8be9-6a90-c3ff-babcfc941ad1: added peer, starting replication: peer=server-d1afff6d-4fc0-3302-952c-38c375c42ee7 +2025-01-28T14:54:22.181Z [INFO] server-6e60e5aa-c036-8918-49be-ab3c8b222b91: duplicate requestVote for same term: term=2 +2025-01-28T14:54:22.181Z [WARN] server-796717a3-8be9-6a90-c3ff-babcfc941ad1: rejecting vote request since we have a leader: from=d1afff6d-4fc0-3302-952c-38c375c42ee7 leader=796717a3-8be9-6a90-c3ff-babcfc941ad1 leader-id=server-796717a3-8be9-6a90-c3ff-babcfc941ad1 +2025-01-28T14:54:22.181Z [INFO] server-796717a3-8be9-6a90-c3ff-babcfc941ad1: pipelining replication: peer="{Voter server-6e60e5aa-c036-8918-49be-ab3c8b222b91 6e60e5aa-c036-8918-49be-ab3c8b222b91}" +2025-01-28T14:54:22.181Z [INFO] server-d1afff6d-4fc0-3302-952c-38c375c42ee7: entering follower state: follower="Node at d1afff6d-4fc0-3302-952c-38c375c42ee7 [Follower]" leader-address=796717a3-8be9-6a90-c3ff-babcfc941ad1 leader-id=server-796717a3-8be9-6a90-c3ff-babcfc941ad1 +2025-01-28T14:54:22.182Z [INFO] server-796717a3-8be9-6a90-c3ff-babcfc941ad1: pipelining replication: peer="{Voter server-d1afff6d-4fc0-3302-952c-38c375c42ee7 d1afff6d-4fc0-3302-952c-38c375c42ee7}" +2025-01-28T14:54:23.187Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:22.181825242 +0000 UTC m=+47.856315222 (1 nodes), 507.973998ms from start of poll, 509.070595ms from cluster start. Timeout at 2025-01-28 14:54:23.186832821 +0000 UTC m=+48.861322941, 1.005007719s after stability +2025-01-28T14:54:23.188Z [INFO] creating new snapshot: path=/tmp/raft4007420181/snapshots/2-13-1738076063188.tmp +2025-01-28T14:54:23.189Z [INFO] server-796717a3-8be9-6a90-c3ff-babcfc941ad1: copied to local snapshot: bytes=71 +2025-01-28T14:54:23.190Z [INFO] server-796717a3-8be9-6a90-c3ff-babcfc941ad1: snapshot restore progress: id=2-13-1738076063188 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:23.190Z [INFO] server-796717a3-8be9-6a90-c3ff-babcfc941ad1: restored user snapshot: index=13 +2025-01-28T14:54:23.190Z [ERROR] server-796717a3-8be9-6a90-c3ff-babcfc941ad1: failed to get log: index=3 error="log not found" +2025-01-28T14:54:23.191Z [INFO] server-796717a3-8be9-6a90-c3ff-babcfc941ad1: aborting pipeline replication: peer="{Voter server-d1afff6d-4fc0-3302-952c-38c375c42ee7 d1afff6d-4fc0-3302-952c-38c375c42ee7}" +2025-01-28T14:54:23.190Z [ERROR] server-796717a3-8be9-6a90-c3ff-babcfc941ad1: failed to get log: index=3 error="log not found" +2025-01-28T14:54:23.191Z [INFO] server-796717a3-8be9-6a90-c3ff-babcfc941ad1: aborting pipeline replication: peer="{Voter server-6e60e5aa-c036-8918-49be-ab3c8b222b91 6e60e5aa-c036-8918-49be-ab3c8b222b91}" +2025-01-28T14:54:23.199Z [ERROR] server-796717a3-8be9-6a90-c3ff-babcfc941ad1: failed to get log: index=3 error="log not found" +2025-01-28T14:54:23.200Z [INFO] creating new snapshot: path=/tmp/raft1823198774/snapshots/2-13-1738076063200.tmp +2025-01-28T14:54:23.200Z [ERROR] server-796717a3-8be9-6a90-c3ff-babcfc941ad1: failed to get log: index=3 error="log not found" +2025-01-28T14:54:23.201Z [INFO] creating new snapshot: path=/tmp/raft171869902/snapshots/2-13-1738076063201.tmp +2025-01-28T14:54:23.201Z [INFO] server-d1afff6d-4fc0-3302-952c-38c375c42ee7: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:23.201Z [INFO] server-6e60e5aa-c036-8918-49be-ab3c8b222b91: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:23.202Z [INFO] server-d1afff6d-4fc0-3302-952c-38c375c42ee7: copied to local snapshot: bytes=71 +2025-01-28T14:54:23.202Z [INFO] server-6e60e5aa-c036-8918-49be-ab3c8b222b91: copied to local snapshot: bytes=71 +2025-01-28T14:54:23.202Z [INFO] server-d1afff6d-4fc0-3302-952c-38c375c42ee7: snapshot restore progress: id=2-13-1738076063200 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:23.202Z [INFO] server-d1afff6d-4fc0-3302-952c-38c375c42ee7: Installed remote snapshot +2025-01-28T14:54:23.203Z [INFO] server-6e60e5aa-c036-8918-49be-ab3c8b222b91: snapshot restore progress: id=2-13-1738076063201 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:23.203Z [INFO] server-6e60e5aa-c036-8918-49be-ab3c8b222b91: Installed remote snapshot +2025-01-28T14:54:23.203Z [INFO] server-796717a3-8be9-6a90-c3ff-babcfc941ad1: pipelining replication: peer="{Voter server-d1afff6d-4fc0-3302-952c-38c375c42ee7 d1afff6d-4fc0-3302-952c-38c375c42ee7}" +2025-01-28T14:54:23.203Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:54:23.203Z [INFO] server-796717a3-8be9-6a90-c3ff-babcfc941ad1: pipelining replication: peer="{Voter server-6e60e5aa-c036-8918-49be-ab3c8b222b91 6e60e5aa-c036-8918-49be-ab3c8b222b91}" +2025-01-28T14:54:24.208Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:54:23.203659603 +0000 UTC m=+48.878149583 (2 nodes), 2.28µs from start of poll, 1.530904956s from cluster start. Timeout at 2025-01-28 14:54:24.208699619 +0000 UTC m=+49.883189739, 1.005040156s after stability +2025-01-28T14:54:24.222Z [INFO] server-9c0e008d-5edd-34cd-4663-902eb315721b: aborting pipeline replication: peer="{Voter server-9e3fd7b9-09db-d49d-37e3-57a86907dd99 9e3fd7b9-09db-d49d-37e3-57a86907dd99}" +2025-01-28T14:54:24.222Z [INFO] server-9c0e008d-5edd-34cd-4663-902eb315721b: aborting pipeline replication: peer="{Voter server-1ef7e83a-bb69-a369-9bb8-6ca81262f962 1ef7e83a-bb69-a369-9bb8-6ca81262f962}" === RUN TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_true -2025-01-28T14:47:23.207Z [INFO] server-0fe07390-0968-ea5e-60d1-4c2f880ef77b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0fe07390-0968-ea5e-60d1-4c2f880ef77b Address:0fe07390-0968-ea5e-60d1-4c2f880ef77b} {Suffrage:Voter ID:server-f17939a7-fb26-6f31-6b75-88ca50ed48d1 Address:f17939a7-fb26-6f31-6b75-88ca50ed48d1} {Suffrage:Voter ID:server-dc182dbf-004c-763d-20be-9957a307222e Address:dc182dbf-004c-763d-20be-9957a307222e}]" -2025-01-28T14:47:23.207Z [INFO] server-0fe07390-0968-ea5e-60d1-4c2f880ef77b: entering follower state: follower="Node at 0fe07390-0968-ea5e-60d1-4c2f880ef77b [Follower]" leader-address= leader-id= -2025-01-28T14:47:23.208Z [INFO] server-f17939a7-fb26-6f31-6b75-88ca50ed48d1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0fe07390-0968-ea5e-60d1-4c2f880ef77b Address:0fe07390-0968-ea5e-60d1-4c2f880ef77b} {Suffrage:Voter ID:server-f17939a7-fb26-6f31-6b75-88ca50ed48d1 Address:f17939a7-fb26-6f31-6b75-88ca50ed48d1} {Suffrage:Voter ID:server-dc182dbf-004c-763d-20be-9957a307222e Address:dc182dbf-004c-763d-20be-9957a307222e}]" -2025-01-28T14:47:23.209Z [INFO] server-f17939a7-fb26-6f31-6b75-88ca50ed48d1: entering follower state: follower="Node at f17939a7-fb26-6f31-6b75-88ca50ed48d1 [Follower]" leader-address= leader-id= -2025-01-28T14:47:23.210Z [INFO] server-dc182dbf-004c-763d-20be-9957a307222e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0fe07390-0968-ea5e-60d1-4c2f880ef77b Address:0fe07390-0968-ea5e-60d1-4c2f880ef77b} {Suffrage:Voter ID:server-f17939a7-fb26-6f31-6b75-88ca50ed48d1 Address:f17939a7-fb26-6f31-6b75-88ca50ed48d1} {Suffrage:Voter ID:server-dc182dbf-004c-763d-20be-9957a307222e Address:dc182dbf-004c-763d-20be-9957a307222e}]" -2025-01-28T14:47:23.210Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:23.210Z [INFO] server-dc182dbf-004c-763d-20be-9957a307222e: entering follower state: follower="Node at dc182dbf-004c-763d-20be-9957a307222e [Follower]" leader-address= leader-id= -2025-01-28T14:47:23.714Z [WARN] server-f17939a7-fb26-6f31-6b75-88ca50ed48d1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:23.714Z [INFO] server-f17939a7-fb26-6f31-6b75-88ca50ed48d1: entering candidate state: node="Node at f17939a7-fb26-6f31-6b75-88ca50ed48d1 [Candidate]" term=2 -2025-01-28T14:47:23.715Z [INFO] server-f17939a7-fb26-6f31-6b75-88ca50ed48d1: election won: term=2 tally=2 -2025-01-28T14:47:23.716Z [INFO] server-f17939a7-fb26-6f31-6b75-88ca50ed48d1: entering leader state: leader="Node at f17939a7-fb26-6f31-6b75-88ca50ed48d1 [Leader]" -2025-01-28T14:47:23.716Z [INFO] server-f17939a7-fb26-6f31-6b75-88ca50ed48d1: added peer, starting replication: peer=server-0fe07390-0968-ea5e-60d1-4c2f880ef77b -2025-01-28T14:47:23.716Z [INFO] server-f17939a7-fb26-6f31-6b75-88ca50ed48d1: added peer, starting replication: peer=server-dc182dbf-004c-763d-20be-9957a307222e -2025-01-28T14:47:23.717Z [INFO] server-f17939a7-fb26-6f31-6b75-88ca50ed48d1: pipelining replication: peer="{Voter server-0fe07390-0968-ea5e-60d1-4c2f880ef77b 0fe07390-0968-ea5e-60d1-4c2f880ef77b}" -2025-01-28T14:47:23.718Z [INFO] server-f17939a7-fb26-6f31-6b75-88ca50ed48d1: pipelining replication: peer="{Voter server-dc182dbf-004c-763d-20be-9957a307222e dc182dbf-004c-763d-20be-9957a307222e}" -2025-01-28T14:47:24.722Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:23.716378956 +0000 UTC m=+26.799785314 (1 nodes), 505.56801ms from start of poll, 510.853834ms from cluster start. Timeout at 2025-01-28 14:47:24.721419957 +0000 UTC m=+27.804825982, 1.005040668s after stability -2025-01-28T14:47:24.733Z [INFO] server-f17939a7-fb26-6f31-6b75-88ca50ed48d1: starting snapshot up to: index=12 -2025-01-28T14:47:24.733Z [INFO] creating new snapshot: path=/tmp/raft2364932551/snapshots/2-12-1738075644733.tmp -2025-01-28T14:47:24.740Z [INFO] server-f17939a7-fb26-6f31-6b75-88ca50ed48d1: snapshot complete up to: index=12 -2025-01-28T14:47:24.771Z [INFO] server-6bae03fd-423d-baff-c73a-4c44b86b68e5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6bae03fd-423d-baff-c73a-4c44b86b68e5 Address:6bae03fd-423d-baff-c73a-4c44b86b68e5} {Suffrage:Voter ID:server-018cc126-6d81-d83e-7e22-6c832322804d Address:018cc126-6d81-d83e-7e22-6c832322804d} {Suffrage:Voter ID:server-f86d04aa-4db5-551f-bd4e-29adf720afd7 Address:f86d04aa-4db5-551f-bd4e-29adf720afd7}]" -2025-01-28T14:47:24.771Z [INFO] server-6bae03fd-423d-baff-c73a-4c44b86b68e5: entering follower state: follower="Node at 6bae03fd-423d-baff-c73a-4c44b86b68e5 [Follower]" leader-address= leader-id= -2025-01-28T14:47:24.773Z [INFO] server-018cc126-6d81-d83e-7e22-6c832322804d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6bae03fd-423d-baff-c73a-4c44b86b68e5 Address:6bae03fd-423d-baff-c73a-4c44b86b68e5} {Suffrage:Voter ID:server-018cc126-6d81-d83e-7e22-6c832322804d Address:018cc126-6d81-d83e-7e22-6c832322804d} {Suffrage:Voter ID:server-f86d04aa-4db5-551f-bd4e-29adf720afd7 Address:f86d04aa-4db5-551f-bd4e-29adf720afd7}]" -2025-01-28T14:47:24.773Z [INFO] server-018cc126-6d81-d83e-7e22-6c832322804d: entering follower state: follower="Node at 018cc126-6d81-d83e-7e22-6c832322804d [Follower]" leader-address= leader-id= -2025-01-28T14:47:24.774Z [INFO] server-f86d04aa-4db5-551f-bd4e-29adf720afd7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6bae03fd-423d-baff-c73a-4c44b86b68e5 Address:6bae03fd-423d-baff-c73a-4c44b86b68e5} {Suffrage:Voter ID:server-018cc126-6d81-d83e-7e22-6c832322804d Address:018cc126-6d81-d83e-7e22-6c832322804d} {Suffrage:Voter ID:server-f86d04aa-4db5-551f-bd4e-29adf720afd7 Address:f86d04aa-4db5-551f-bd4e-29adf720afd7}]" -2025-01-28T14:47:24.774Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:24.774Z [INFO] server-f86d04aa-4db5-551f-bd4e-29adf720afd7: entering follower state: follower="Node at f86d04aa-4db5-551f-bd4e-29adf720afd7 [Follower]" leader-address= leader-id= -2025-01-28T14:47:25.354Z [WARN] server-f86d04aa-4db5-551f-bd4e-29adf720afd7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:25.355Z [INFO] server-f86d04aa-4db5-551f-bd4e-29adf720afd7: entering candidate state: node="Node at f86d04aa-4db5-551f-bd4e-29adf720afd7 [Candidate]" term=2 -2025-01-28T14:47:25.356Z [INFO] server-f86d04aa-4db5-551f-bd4e-29adf720afd7: election won: term=2 tally=2 -2025-01-28T14:47:25.356Z [INFO] server-f86d04aa-4db5-551f-bd4e-29adf720afd7: entering leader state: leader="Node at f86d04aa-4db5-551f-bd4e-29adf720afd7 [Leader]" -2025-01-28T14:47:25.356Z [INFO] server-f86d04aa-4db5-551f-bd4e-29adf720afd7: added peer, starting replication: peer=server-6bae03fd-423d-baff-c73a-4c44b86b68e5 -2025-01-28T14:47:25.356Z [INFO] server-f86d04aa-4db5-551f-bd4e-29adf720afd7: added peer, starting replication: peer=server-018cc126-6d81-d83e-7e22-6c832322804d -2025-01-28T14:47:25.358Z [INFO] server-f86d04aa-4db5-551f-bd4e-29adf720afd7: pipelining replication: peer="{Voter server-018cc126-6d81-d83e-7e22-6c832322804d 018cc126-6d81-d83e-7e22-6c832322804d}" -2025-01-28T14:47:25.359Z [WARN] server-6bae03fd-423d-baff-c73a-4c44b86b68e5: rejecting vote request since our last term is greater: candidate=f86d04aa-4db5-551f-bd4e-29adf720afd7 last-term=2 last-candidate-term=1 -2025-01-28T14:47:25.360Z [INFO] server-f86d04aa-4db5-551f-bd4e-29adf720afd7: pipelining replication: peer="{Voter server-6bae03fd-423d-baff-c73a-4c44b86b68e5 6bae03fd-423d-baff-c73a-4c44b86b68e5}" -2025-01-28T14:47:26.365Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:25.360013422 +0000 UTC m=+28.443419780 (1 nodes), 585.074371ms from start of poll, 590.229191ms from cluster start. Timeout at 2025-01-28 14:47:26.365111759 +0000 UTC m=+29.448518450, 1.00509867s after stability -2025-01-28T14:47:26.368Z [INFO] creating new snapshot: path=/tmp/raft1136064849/snapshots/2-13-1738075646368.tmp -2025-01-28T14:47:26.375Z [INFO] server-f86d04aa-4db5-551f-bd4e-29adf720afd7: copied to local snapshot: bytes=71 -2025-01-28T14:47:26.376Z [INFO] server-f86d04aa-4db5-551f-bd4e-29adf720afd7: snapshot restore progress: id=2-13-1738075646368 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:26.377Z [INFO] server-f86d04aa-4db5-551f-bd4e-29adf720afd7: removing all old logs from log store -2025-01-28T14:47:26.377Z [INFO] server-f86d04aa-4db5-551f-bd4e-29adf720afd7: compacting logs: from=1 to=2 -2025-01-28T14:47:26.377Z [INFO] server-f86d04aa-4db5-551f-bd4e-29adf720afd7: restored user snapshot: index=13 -2025-01-28T14:47:26.377Z [ERROR] server-f86d04aa-4db5-551f-bd4e-29adf720afd7: failed to get log: index=2 error="log not found" -2025-01-28T14:47:26.377Z [INFO] server-f86d04aa-4db5-551f-bd4e-29adf720afd7: aborting pipeline replication: peer="{Voter server-6bae03fd-423d-baff-c73a-4c44b86b68e5 6bae03fd-423d-baff-c73a-4c44b86b68e5}" -2025-01-28T14:47:26.377Z [ERROR] server-f86d04aa-4db5-551f-bd4e-29adf720afd7: failed to get log: index=2 error="log not found" -2025-01-28T14:47:26.378Z [INFO] server-f86d04aa-4db5-551f-bd4e-29adf720afd7: aborting pipeline replication: peer="{Voter server-018cc126-6d81-d83e-7e22-6c832322804d 018cc126-6d81-d83e-7e22-6c832322804d}" -2025-01-28T14:47:26.383Z [ERROR] server-f86d04aa-4db5-551f-bd4e-29adf720afd7: failed to get log: index=2 error="log not found" -2025-01-28T14:47:26.385Z [ERROR] server-f86d04aa-4db5-551f-bd4e-29adf720afd7: failed to get log: index=2 error="log not found" -2025-01-28T14:47:26.388Z [INFO] creating new snapshot: path=/tmp/raft3671420036/snapshots/2-13-1738075646388.tmp -2025-01-28T14:47:26.390Z [INFO] creating new snapshot: path=/tmp/raft2647387347/snapshots/2-13-1738075646390.tmp -2025-01-28T14:47:26.392Z [INFO] server-018cc126-6d81-d83e-7e22-6c832322804d: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:26.394Z [INFO] server-6bae03fd-423d-baff-c73a-4c44b86b68e5: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:26.396Z [INFO] server-018cc126-6d81-d83e-7e22-6c832322804d: copied to local snapshot: bytes=71 -2025-01-28T14:47:26.399Z [INFO] server-6bae03fd-423d-baff-c73a-4c44b86b68e5: copied to local snapshot: bytes=71 -2025-01-28T14:47:26.400Z [INFO] server-018cc126-6d81-d83e-7e22-6c832322804d: snapshot restore progress: id=2-13-1738075646388 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:26.400Z [INFO] server-018cc126-6d81-d83e-7e22-6c832322804d: removing all old logs from log store -2025-01-28T14:47:26.400Z [INFO] server-018cc126-6d81-d83e-7e22-6c832322804d: compacting logs: from=1 to=2 -2025-01-28T14:47:26.401Z [INFO] server-018cc126-6d81-d83e-7e22-6c832322804d: Installed remote snapshot -2025-01-28T14:47:26.408Z [WARN] server-018cc126-6d81-d83e-7e22-6c832322804d: skipping application of old log: index=13 -2025-01-28T14:47:26.408Z [INFO] server-f86d04aa-4db5-551f-bd4e-29adf720afd7: pipelining replication: peer="{Voter server-018cc126-6d81-d83e-7e22-6c832322804d 018cc126-6d81-d83e-7e22-6c832322804d}" -2025-01-28T14:47:26.408Z [INFO] server-6bae03fd-423d-baff-c73a-4c44b86b68e5: snapshot restore progress: id=2-13-1738075646390 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:26.409Z [INFO] server-6bae03fd-423d-baff-c73a-4c44b86b68e5: removing all old logs from log store -2025-01-28T14:47:26.409Z [INFO] server-6bae03fd-423d-baff-c73a-4c44b86b68e5: compacting logs: from=1 to=2 -2025-01-28T14:47:26.409Z [INFO] server-6bae03fd-423d-baff-c73a-4c44b86b68e5: Installed remote snapshot -2025-01-28T14:47:26.410Z [WARN] server-6bae03fd-423d-baff-c73a-4c44b86b68e5: skipping application of old log: index=13 -2025-01-28T14:47:26.412Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:47:26.412Z [INFO] server-f86d04aa-4db5-551f-bd4e-29adf720afd7: pipelining replication: peer="{Voter server-6bae03fd-423d-baff-c73a-4c44b86b68e5 6bae03fd-423d-baff-c73a-4c44b86b68e5}" -2025-01-28T14:47:27.419Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:47:26.412929017 +0000 UTC m=+29.496334041 (2 nodes), 8.667µs from start of poll, 1.643143452s from cluster start. Timeout at 2025-01-28 14:47:27.418209359 +0000 UTC m=+30.501615716, 1.005281675s after stability -2025-01-28T14:47:27.440Z [INFO] server-f17939a7-fb26-6f31-6b75-88ca50ed48d1: aborting pipeline replication: peer="{Voter server-dc182dbf-004c-763d-20be-9957a307222e dc182dbf-004c-763d-20be-9957a307222e}" -2025-01-28T14:47:27.440Z [INFO] server-f17939a7-fb26-6f31-6b75-88ca50ed48d1: aborting pipeline replication: peer="{Voter server-0fe07390-0968-ea5e-60d1-4c2f880ef77b 0fe07390-0968-ea5e-60d1-4c2f880ef77b}" +2025-01-28T14:54:24.228Z [INFO] server-8dc342da-9b13-39ca-70b9-cb2d4de56941: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8dc342da-9b13-39ca-70b9-cb2d4de56941 Address:8dc342da-9b13-39ca-70b9-cb2d4de56941} {Suffrage:Voter ID:server-049fa6b8-6f27-43df-aa89-37941a186534 Address:049fa6b8-6f27-43df-aa89-37941a186534} {Suffrage:Voter ID:server-df36936f-e222-5616-bdb6-1bdc60c25a23 Address:df36936f-e222-5616-bdb6-1bdc60c25a23}]" +2025-01-28T14:54:24.229Z [INFO] server-049fa6b8-6f27-43df-aa89-37941a186534: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8dc342da-9b13-39ca-70b9-cb2d4de56941 Address:8dc342da-9b13-39ca-70b9-cb2d4de56941} {Suffrage:Voter ID:server-049fa6b8-6f27-43df-aa89-37941a186534 Address:049fa6b8-6f27-43df-aa89-37941a186534} {Suffrage:Voter ID:server-df36936f-e222-5616-bdb6-1bdc60c25a23 Address:df36936f-e222-5616-bdb6-1bdc60c25a23}]" +2025-01-28T14:54:24.229Z [INFO] server-8dc342da-9b13-39ca-70b9-cb2d4de56941: entering follower state: follower="Node at 8dc342da-9b13-39ca-70b9-cb2d4de56941 [Follower]" leader-address= leader-id= +2025-01-28T14:54:24.229Z [INFO] server-049fa6b8-6f27-43df-aa89-37941a186534: entering follower state: follower="Node at 049fa6b8-6f27-43df-aa89-37941a186534 [Follower]" leader-address= leader-id= +2025-01-28T14:54:24.229Z [INFO] server-df36936f-e222-5616-bdb6-1bdc60c25a23: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8dc342da-9b13-39ca-70b9-cb2d4de56941 Address:8dc342da-9b13-39ca-70b9-cb2d4de56941} {Suffrage:Voter ID:server-049fa6b8-6f27-43df-aa89-37941a186534 Address:049fa6b8-6f27-43df-aa89-37941a186534} {Suffrage:Voter ID:server-df36936f-e222-5616-bdb6-1bdc60c25a23 Address:df36936f-e222-5616-bdb6-1bdc60c25a23}]" +2025-01-28T14:54:24.230Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:24.230Z [INFO] server-df36936f-e222-5616-bdb6-1bdc60c25a23: entering follower state: follower="Node at df36936f-e222-5616-bdb6-1bdc60c25a23 [Follower]" leader-address= leader-id= +2025-01-28T14:54:24.741Z [WARN] server-df36936f-e222-5616-bdb6-1bdc60c25a23: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:24.741Z [INFO] server-df36936f-e222-5616-bdb6-1bdc60c25a23: entering candidate state: node="Node at df36936f-e222-5616-bdb6-1bdc60c25a23 [Candidate]" term=2 +2025-01-28T14:54:24.742Z [INFO] server-df36936f-e222-5616-bdb6-1bdc60c25a23: election won: term=2 tally=2 +2025-01-28T14:54:24.742Z [INFO] server-df36936f-e222-5616-bdb6-1bdc60c25a23: entering leader state: leader="Node at df36936f-e222-5616-bdb6-1bdc60c25a23 [Leader]" +2025-01-28T14:54:24.742Z [INFO] server-df36936f-e222-5616-bdb6-1bdc60c25a23: added peer, starting replication: peer=server-8dc342da-9b13-39ca-70b9-cb2d4de56941 +2025-01-28T14:54:24.742Z [INFO] server-df36936f-e222-5616-bdb6-1bdc60c25a23: added peer, starting replication: peer=server-049fa6b8-6f27-43df-aa89-37941a186534 +2025-01-28T14:54:24.742Z [INFO] server-df36936f-e222-5616-bdb6-1bdc60c25a23: pipelining replication: peer="{Voter server-8dc342da-9b13-39ca-70b9-cb2d4de56941 8dc342da-9b13-39ca-70b9-cb2d4de56941}" +2025-01-28T14:54:24.742Z [INFO] server-df36936f-e222-5616-bdb6-1bdc60c25a23: pipelining replication: peer="{Voter server-049fa6b8-6f27-43df-aa89-37941a186534 049fa6b8-6f27-43df-aa89-37941a186534}" +2025-01-28T14:54:25.747Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:24.742346165 +0000 UTC m=+50.416836185 (1 nodes), 512.104588ms from start of poll, 515.001475ms from cluster start. Timeout at 2025-01-28 14:54:25.747351984 +0000 UTC m=+51.421842024, 1.005005839s after stability +2025-01-28T14:54:25.748Z [INFO] server-df36936f-e222-5616-bdb6-1bdc60c25a23: starting snapshot up to: index=12 +2025-01-28T14:54:25.748Z [INFO] creating new snapshot: path=/tmp/raft1371664513/snapshots/2-12-1738076065748.tmp +2025-01-28T14:54:25.750Z [INFO] server-df36936f-e222-5616-bdb6-1bdc60c25a23: snapshot complete up to: index=12 +2025-01-28T14:54:25.753Z [INFO] server-3bcfe630-708f-09ed-eaef-81aa41e7668a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3bcfe630-708f-09ed-eaef-81aa41e7668a Address:3bcfe630-708f-09ed-eaef-81aa41e7668a} {Suffrage:Voter ID:server-90050a2c-afa6-9b84-fc4e-4109e8ad878e Address:90050a2c-afa6-9b84-fc4e-4109e8ad878e} {Suffrage:Voter ID:server-8eec329d-2933-8733-3329-234a5a421c3e Address:8eec329d-2933-8733-3329-234a5a421c3e}]" +2025-01-28T14:54:25.753Z [INFO] server-3bcfe630-708f-09ed-eaef-81aa41e7668a: entering follower state: follower="Node at 3bcfe630-708f-09ed-eaef-81aa41e7668a [Follower]" leader-address= leader-id= +2025-01-28T14:54:25.753Z [INFO] server-90050a2c-afa6-9b84-fc4e-4109e8ad878e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3bcfe630-708f-09ed-eaef-81aa41e7668a Address:3bcfe630-708f-09ed-eaef-81aa41e7668a} {Suffrage:Voter ID:server-90050a2c-afa6-9b84-fc4e-4109e8ad878e Address:90050a2c-afa6-9b84-fc4e-4109e8ad878e} {Suffrage:Voter ID:server-8eec329d-2933-8733-3329-234a5a421c3e Address:8eec329d-2933-8733-3329-234a5a421c3e}]" +2025-01-28T14:54:25.753Z [INFO] server-90050a2c-afa6-9b84-fc4e-4109e8ad878e: entering follower state: follower="Node at 90050a2c-afa6-9b84-fc4e-4109e8ad878e [Follower]" leader-address= leader-id= +2025-01-28T14:54:25.753Z [INFO] server-8eec329d-2933-8733-3329-234a5a421c3e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3bcfe630-708f-09ed-eaef-81aa41e7668a Address:3bcfe630-708f-09ed-eaef-81aa41e7668a} {Suffrage:Voter ID:server-90050a2c-afa6-9b84-fc4e-4109e8ad878e Address:90050a2c-afa6-9b84-fc4e-4109e8ad878e} {Suffrage:Voter ID:server-8eec329d-2933-8733-3329-234a5a421c3e Address:8eec329d-2933-8733-3329-234a5a421c3e}]" +2025-01-28T14:54:25.753Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:25.753Z [INFO] server-8eec329d-2933-8733-3329-234a5a421c3e: entering follower state: follower="Node at 8eec329d-2933-8733-3329-234a5a421c3e [Follower]" leader-address= leader-id= +2025-01-28T14:54:26.649Z [WARN] server-8eec329d-2933-8733-3329-234a5a421c3e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:26.650Z [INFO] server-8eec329d-2933-8733-3329-234a5a421c3e: entering candidate state: node="Node at 8eec329d-2933-8733-3329-234a5a421c3e [Candidate]" term=2 +2025-01-28T14:54:26.650Z [INFO] server-8eec329d-2933-8733-3329-234a5a421c3e: election won: term=2 tally=2 +2025-01-28T14:54:26.650Z [INFO] server-8eec329d-2933-8733-3329-234a5a421c3e: entering leader state: leader="Node at 8eec329d-2933-8733-3329-234a5a421c3e [Leader]" +2025-01-28T14:54:26.650Z [INFO] server-8eec329d-2933-8733-3329-234a5a421c3e: added peer, starting replication: peer=server-3bcfe630-708f-09ed-eaef-81aa41e7668a +2025-01-28T14:54:26.650Z [INFO] server-8eec329d-2933-8733-3329-234a5a421c3e: added peer, starting replication: peer=server-90050a2c-afa6-9b84-fc4e-4109e8ad878e +2025-01-28T14:54:26.650Z [INFO] server-8eec329d-2933-8733-3329-234a5a421c3e: pipelining replication: peer="{Voter server-90050a2c-afa6-9b84-fc4e-4109e8ad878e 90050a2c-afa6-9b84-fc4e-4109e8ad878e}" +2025-01-28T14:54:26.650Z [INFO] server-8eec329d-2933-8733-3329-234a5a421c3e: pipelining replication: peer="{Voter server-3bcfe630-708f-09ed-eaef-81aa41e7668a 3bcfe630-708f-09ed-eaef-81aa41e7668a}" +2025-01-28T14:54:27.655Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:26.650374224 +0000 UTC m=+52.324864204 (1 nodes), 896.382256ms from start of poll, 897.423278ms from cluster start. Timeout at 2025-01-28 14:54:27.655395322 +0000 UTC m=+53.329885302, 1.005021098s after stability +2025-01-28T14:54:27.658Z [INFO] creating new snapshot: path=/tmp/raft4049951965/snapshots/2-13-1738076067658.tmp +2025-01-28T14:54:27.661Z [INFO] server-8eec329d-2933-8733-3329-234a5a421c3e: copied to local snapshot: bytes=71 +2025-01-28T14:54:27.661Z [INFO] server-8eec329d-2933-8733-3329-234a5a421c3e: snapshot restore progress: id=2-13-1738076067658 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:27.661Z [INFO] server-8eec329d-2933-8733-3329-234a5a421c3e: removing all old logs from log store +2025-01-28T14:54:27.662Z [INFO] server-8eec329d-2933-8733-3329-234a5a421c3e: compacting logs: from=1 to=2 +2025-01-28T14:54:27.662Z [INFO] server-8eec329d-2933-8733-3329-234a5a421c3e: restored user snapshot: index=13 +2025-01-28T14:54:27.662Z [ERROR] server-8eec329d-2933-8733-3329-234a5a421c3e: failed to get log: index=2 error="log not found" +2025-01-28T14:54:27.662Z [ERROR] server-8eec329d-2933-8733-3329-234a5a421c3e: failed to get log: index=2 error="log not found" +2025-01-28T14:54:27.662Z [INFO] server-8eec329d-2933-8733-3329-234a5a421c3e: aborting pipeline replication: peer="{Voter server-90050a2c-afa6-9b84-fc4e-4109e8ad878e 90050a2c-afa6-9b84-fc4e-4109e8ad878e}" +2025-01-28T14:54:27.662Z [INFO] server-8eec329d-2933-8733-3329-234a5a421c3e: aborting pipeline replication: peer="{Voter server-3bcfe630-708f-09ed-eaef-81aa41e7668a 3bcfe630-708f-09ed-eaef-81aa41e7668a}" +2025-01-28T14:54:27.669Z [ERROR] server-8eec329d-2933-8733-3329-234a5a421c3e: failed to get log: index=2 error="log not found" +2025-01-28T14:54:27.669Z [ERROR] server-8eec329d-2933-8733-3329-234a5a421c3e: failed to get log: index=2 error="log not found" +2025-01-28T14:54:27.669Z [INFO] creating new snapshot: path=/tmp/raft1430819158/snapshots/2-13-1738076067669.tmp +2025-01-28T14:54:27.669Z [INFO] creating new snapshot: path=/tmp/raft1329274814/snapshots/2-13-1738076067669.tmp +2025-01-28T14:54:27.670Z [INFO] server-90050a2c-afa6-9b84-fc4e-4109e8ad878e: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:27.670Z [INFO] server-3bcfe630-708f-09ed-eaef-81aa41e7668a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:27.671Z [INFO] server-90050a2c-afa6-9b84-fc4e-4109e8ad878e: copied to local snapshot: bytes=71 +2025-01-28T14:54:27.671Z [INFO] server-3bcfe630-708f-09ed-eaef-81aa41e7668a: copied to local snapshot: bytes=71 +2025-01-28T14:54:27.672Z [INFO] server-90050a2c-afa6-9b84-fc4e-4109e8ad878e: snapshot restore progress: id=2-13-1738076067669 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:27.672Z [INFO] server-3bcfe630-708f-09ed-eaef-81aa41e7668a: snapshot restore progress: id=2-13-1738076067669 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:27.672Z [INFO] server-3bcfe630-708f-09ed-eaef-81aa41e7668a: removing all old logs from log store +2025-01-28T14:54:27.672Z [INFO] server-3bcfe630-708f-09ed-eaef-81aa41e7668a: compacting logs: from=1 to=2 +2025-01-28T14:54:27.672Z [INFO] server-3bcfe630-708f-09ed-eaef-81aa41e7668a: Installed remote snapshot +2025-01-28T14:54:27.672Z [INFO] server-90050a2c-afa6-9b84-fc4e-4109e8ad878e: removing all old logs from log store +2025-01-28T14:54:27.672Z [INFO] server-90050a2c-afa6-9b84-fc4e-4109e8ad878e: compacting logs: from=1 to=2 +2025-01-28T14:54:27.672Z [INFO] server-90050a2c-afa6-9b84-fc4e-4109e8ad878e: Installed remote snapshot +2025-01-28T14:54:27.672Z [INFO] server-8eec329d-2933-8733-3329-234a5a421c3e: pipelining replication: peer="{Voter server-3bcfe630-708f-09ed-eaef-81aa41e7668a 3bcfe630-708f-09ed-eaef-81aa41e7668a}" +2025-01-28T14:54:27.672Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:54:27.672Z [WARN] server-90050a2c-afa6-9b84-fc4e-4109e8ad878e: skipping application of old log: index=13 +2025-01-28T14:54:27.672Z [INFO] server-8eec329d-2933-8733-3329-234a5a421c3e: pipelining replication: peer="{Voter server-90050a2c-afa6-9b84-fc4e-4109e8ad878e 90050a2c-afa6-9b84-fc4e-4109e8ad878e}" +2025-01-28T14:54:28.678Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:54:27.672698719 +0000 UTC m=+53.347188679 (2 nodes), 2.339µs from start of poll, 1.919747753s from cluster start. Timeout at 2025-01-28 14:54:28.677751254 +0000 UTC m=+54.352241234, 1.005052555s after stability +2025-01-28T14:54:28.694Z [INFO] server-df36936f-e222-5616-bdb6-1bdc60c25a23: aborting pipeline replication: peer="{Voter server-049fa6b8-6f27-43df-aa89-37941a186534 049fa6b8-6f27-43df-aa89-37941a186534}" +2025-01-28T14:54:28.694Z [INFO] server-df36936f-e222-5616-bdb6-1bdc60c25a23: aborting pipeline replication: peer="{Voter server-8dc342da-9b13-39ca-70b9-cb2d4de56941 8dc342da-9b13-39ca-70b9-cb2d4de56941}" === RUN TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_false -2025-01-28T14:47:27.466Z [INFO] server-c423691e-e148-ddfa-255b-fd8beaeafc7b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c423691e-e148-ddfa-255b-fd8beaeafc7b Address:c423691e-e148-ddfa-255b-fd8beaeafc7b} {Suffrage:Voter ID:server-9bf51f08-6efa-d61b-a29b-84fbc7861362 Address:9bf51f08-6efa-d61b-a29b-84fbc7861362} {Suffrage:Voter ID:server-32d4a8ac-aa2e-4064-2d52-965243d61aac Address:32d4a8ac-aa2e-4064-2d52-965243d61aac}]" -2025-01-28T14:47:27.466Z [INFO] server-c423691e-e148-ddfa-255b-fd8beaeafc7b: entering follower state: follower="Node at c423691e-e148-ddfa-255b-fd8beaeafc7b [Follower]" leader-address= leader-id= -2025-01-28T14:47:27.467Z [INFO] server-9bf51f08-6efa-d61b-a29b-84fbc7861362: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c423691e-e148-ddfa-255b-fd8beaeafc7b Address:c423691e-e148-ddfa-255b-fd8beaeafc7b} {Suffrage:Voter ID:server-9bf51f08-6efa-d61b-a29b-84fbc7861362 Address:9bf51f08-6efa-d61b-a29b-84fbc7861362} {Suffrage:Voter ID:server-32d4a8ac-aa2e-4064-2d52-965243d61aac Address:32d4a8ac-aa2e-4064-2d52-965243d61aac}]" -2025-01-28T14:47:27.467Z [INFO] server-9bf51f08-6efa-d61b-a29b-84fbc7861362: entering follower state: follower="Node at 9bf51f08-6efa-d61b-a29b-84fbc7861362 [Follower]" leader-address= leader-id= -2025-01-28T14:47:27.467Z [INFO] server-32d4a8ac-aa2e-4064-2d52-965243d61aac: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c423691e-e148-ddfa-255b-fd8beaeafc7b Address:c423691e-e148-ddfa-255b-fd8beaeafc7b} {Suffrage:Voter ID:server-9bf51f08-6efa-d61b-a29b-84fbc7861362 Address:9bf51f08-6efa-d61b-a29b-84fbc7861362} {Suffrage:Voter ID:server-32d4a8ac-aa2e-4064-2d52-965243d61aac Address:32d4a8ac-aa2e-4064-2d52-965243d61aac}]" -2025-01-28T14:47:27.468Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:27.468Z [INFO] server-32d4a8ac-aa2e-4064-2d52-965243d61aac: entering follower state: follower="Node at 32d4a8ac-aa2e-4064-2d52-965243d61aac [Follower]" leader-address= leader-id= -2025-01-28T14:47:28.188Z [WARN] server-9bf51f08-6efa-d61b-a29b-84fbc7861362: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:28.188Z [INFO] server-9bf51f08-6efa-d61b-a29b-84fbc7861362: entering candidate state: node="Node at 9bf51f08-6efa-d61b-a29b-84fbc7861362 [Candidate]" term=2 -2025-01-28T14:47:28.189Z [INFO] server-9bf51f08-6efa-d61b-a29b-84fbc7861362: election won: term=2 tally=2 -2025-01-28T14:47:28.189Z [INFO] server-9bf51f08-6efa-d61b-a29b-84fbc7861362: entering leader state: leader="Node at 9bf51f08-6efa-d61b-a29b-84fbc7861362 [Leader]" -2025-01-28T14:47:28.190Z [INFO] server-9bf51f08-6efa-d61b-a29b-84fbc7861362: added peer, starting replication: peer=server-c423691e-e148-ddfa-255b-fd8beaeafc7b -2025-01-28T14:47:28.190Z [INFO] server-9bf51f08-6efa-d61b-a29b-84fbc7861362: added peer, starting replication: peer=server-32d4a8ac-aa2e-4064-2d52-965243d61aac -2025-01-28T14:47:28.191Z [INFO] server-9bf51f08-6efa-d61b-a29b-84fbc7861362: pipelining replication: peer="{Voter server-c423691e-e148-ddfa-255b-fd8beaeafc7b c423691e-e148-ddfa-255b-fd8beaeafc7b}" -2025-01-28T14:47:28.192Z [INFO] server-9bf51f08-6efa-d61b-a29b-84fbc7861362: pipelining replication: peer="{Voter server-32d4a8ac-aa2e-4064-2d52-965243d61aac 32d4a8ac-aa2e-4064-2d52-965243d61aac}" -2025-01-28T14:47:29.195Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:28.190134116 +0000 UTC m=+31.273540807 (1 nodes), 721.842765ms from start of poll, 726.028223ms from cluster start. Timeout at 2025-01-28 14:47:29.19518945 +0000 UTC m=+32.278595475, 1.005054668s after stability -2025-01-28T14:47:29.206Z [INFO] server-9bf51f08-6efa-d61b-a29b-84fbc7861362: starting snapshot up to: index=12 -2025-01-28T14:47:29.206Z [INFO] creating new snapshot: path=/tmp/raft1870196071/snapshots/2-12-1738075649206.tmp -2025-01-28T14:47:29.213Z [INFO] server-9bf51f08-6efa-d61b-a29b-84fbc7861362: snapshot complete up to: index=12 -2025-01-28T14:47:29.220Z [INFO] creating new snapshot: path=/tmp/raft1870196071/snapshots/2-23-1738075649220.tmp -2025-01-28T14:47:29.225Z [INFO] server-9bf51f08-6efa-d61b-a29b-84fbc7861362: copied to local snapshot: bytes=71 -2025-01-28T14:47:29.227Z [INFO] server-9bf51f08-6efa-d61b-a29b-84fbc7861362: snapshot restore progress: id=2-23-1738075649220 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:29.228Z [INFO] server-9bf51f08-6efa-d61b-a29b-84fbc7861362: restored user snapshot: index=23 -2025-01-28T14:47:29.228Z [ERROR] server-9bf51f08-6efa-d61b-a29b-84fbc7861362: failed to get log: index=23 error="log not found" -2025-01-28T14:47:29.228Z [ERROR] server-9bf51f08-6efa-d61b-a29b-84fbc7861362: failed to get log: index=23 error="log not found" -2025-01-28T14:47:29.228Z [INFO] server-9bf51f08-6efa-d61b-a29b-84fbc7861362: aborting pipeline replication: peer="{Voter server-c423691e-e148-ddfa-255b-fd8beaeafc7b c423691e-e148-ddfa-255b-fd8beaeafc7b}" -2025-01-28T14:47:29.229Z [INFO] server-9bf51f08-6efa-d61b-a29b-84fbc7861362: aborting pipeline replication: peer="{Voter server-32d4a8ac-aa2e-4064-2d52-965243d61aac 32d4a8ac-aa2e-4064-2d52-965243d61aac}" -2025-01-28T14:47:29.237Z [ERROR] server-9bf51f08-6efa-d61b-a29b-84fbc7861362: failed to get log: index=23 error="log not found" -2025-01-28T14:47:29.245Z [ERROR] server-9bf51f08-6efa-d61b-a29b-84fbc7861362: failed to get log: index=23 error="log not found" -2025-01-28T14:47:29.245Z [INFO] creating new snapshot: path=/tmp/raft3072491704/snapshots/2-23-1738075649245.tmp -2025-01-28T14:47:29.250Z [INFO] server-c423691e-e148-ddfa-255b-fd8beaeafc7b: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:29.253Z [INFO] creating new snapshot: path=/tmp/raft1651862045/snapshots/2-23-1738075649253.tmp -2025-01-28T14:47:29.255Z [INFO] server-c423691e-e148-ddfa-255b-fd8beaeafc7b: copied to local snapshot: bytes=71 -2025-01-28T14:47:29.256Z [INFO] server-32d4a8ac-aa2e-4064-2d52-965243d61aac: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:29.257Z [INFO] server-c423691e-e148-ddfa-255b-fd8beaeafc7b: snapshot restore progress: id=2-23-1738075649245 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:29.257Z [INFO] server-c423691e-e148-ddfa-255b-fd8beaeafc7b: Installed remote snapshot -2025-01-28T14:47:29.258Z [INFO] server-9bf51f08-6efa-d61b-a29b-84fbc7861362: pipelining replication: peer="{Voter server-c423691e-e148-ddfa-255b-fd8beaeafc7b c423691e-e148-ddfa-255b-fd8beaeafc7b}" -2025-01-28T14:47:29.259Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:47:29.260Z [INFO] server-32d4a8ac-aa2e-4064-2d52-965243d61aac: copied to local snapshot: bytes=71 -2025-01-28T14:47:29.261Z [INFO] server-32d4a8ac-aa2e-4064-2d52-965243d61aac: snapshot restore progress: id=2-23-1738075649253 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:29.262Z [INFO] server-32d4a8ac-aa2e-4064-2d52-965243d61aac: Installed remote snapshot -2025-01-28T14:47:29.263Z [INFO] server-9bf51f08-6efa-d61b-a29b-84fbc7861362: pipelining replication: peer="{Voter server-32d4a8ac-aa2e-4064-2d52-965243d61aac 32d4a8ac-aa2e-4064-2d52-965243d61aac}" -2025-01-28T14:47:30.265Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:47:29.259314193 +0000 UTC m=+32.342719218 (2 nodes), 9.667µs from start of poll, 1.795206634s from cluster start. Timeout at 2025-01-28 14:47:30.264519533 +0000 UTC m=+33.347925558, 1.00520634s after stability -2025-01-28T14:47:30.285Z [INFO] server-9bf51f08-6efa-d61b-a29b-84fbc7861362: aborting pipeline replication: peer="{Voter server-32d4a8ac-aa2e-4064-2d52-965243d61aac 32d4a8ac-aa2e-4064-2d52-965243d61aac}" -2025-01-28T14:47:30.285Z [INFO] server-9bf51f08-6efa-d61b-a29b-84fbc7861362: aborting pipeline replication: peer="{Voter server-c423691e-e148-ddfa-255b-fd8beaeafc7b c423691e-e148-ddfa-255b-fd8beaeafc7b}" +2025-01-28T14:54:28.699Z [INFO] server-f16a4165-49eb-daee-52d5-fc7314684fa6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f16a4165-49eb-daee-52d5-fc7314684fa6 Address:f16a4165-49eb-daee-52d5-fc7314684fa6} {Suffrage:Voter ID:server-180fd759-cadb-e5b5-76be-531e34f87a8b Address:180fd759-cadb-e5b5-76be-531e34f87a8b} {Suffrage:Voter ID:server-85b6ad5f-b4b0-2169-f8ba-e0b70b426553 Address:85b6ad5f-b4b0-2169-f8ba-e0b70b426553}]" +2025-01-28T14:54:28.699Z [INFO] server-f16a4165-49eb-daee-52d5-fc7314684fa6: entering follower state: follower="Node at f16a4165-49eb-daee-52d5-fc7314684fa6 [Follower]" leader-address= leader-id= +2025-01-28T14:54:28.700Z [INFO] server-180fd759-cadb-e5b5-76be-531e34f87a8b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f16a4165-49eb-daee-52d5-fc7314684fa6 Address:f16a4165-49eb-daee-52d5-fc7314684fa6} {Suffrage:Voter ID:server-180fd759-cadb-e5b5-76be-531e34f87a8b Address:180fd759-cadb-e5b5-76be-531e34f87a8b} {Suffrage:Voter ID:server-85b6ad5f-b4b0-2169-f8ba-e0b70b426553 Address:85b6ad5f-b4b0-2169-f8ba-e0b70b426553}]" +2025-01-28T14:54:28.700Z [INFO] server-180fd759-cadb-e5b5-76be-531e34f87a8b: entering follower state: follower="Node at 180fd759-cadb-e5b5-76be-531e34f87a8b [Follower]" leader-address= leader-id= +2025-01-28T14:54:28.700Z [INFO] server-85b6ad5f-b4b0-2169-f8ba-e0b70b426553: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f16a4165-49eb-daee-52d5-fc7314684fa6 Address:f16a4165-49eb-daee-52d5-fc7314684fa6} {Suffrage:Voter ID:server-180fd759-cadb-e5b5-76be-531e34f87a8b Address:180fd759-cadb-e5b5-76be-531e34f87a8b} {Suffrage:Voter ID:server-85b6ad5f-b4b0-2169-f8ba-e0b70b426553 Address:85b6ad5f-b4b0-2169-f8ba-e0b70b426553}]" +2025-01-28T14:54:28.700Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:28.700Z [INFO] server-85b6ad5f-b4b0-2169-f8ba-e0b70b426553: entering follower state: follower="Node at 85b6ad5f-b4b0-2169-f8ba-e0b70b426553 [Follower]" leader-address= leader-id= +2025-01-28T14:54:29.276Z [WARN] server-f16a4165-49eb-daee-52d5-fc7314684fa6: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:29.276Z [INFO] server-f16a4165-49eb-daee-52d5-fc7314684fa6: entering candidate state: node="Node at f16a4165-49eb-daee-52d5-fc7314684fa6 [Candidate]" term=2 +2025-01-28T14:54:29.277Z [INFO] server-f16a4165-49eb-daee-52d5-fc7314684fa6: election won: term=2 tally=2 +2025-01-28T14:54:29.277Z [INFO] server-f16a4165-49eb-daee-52d5-fc7314684fa6: entering leader state: leader="Node at f16a4165-49eb-daee-52d5-fc7314684fa6 [Leader]" +2025-01-28T14:54:29.277Z [INFO] server-f16a4165-49eb-daee-52d5-fc7314684fa6: added peer, starting replication: peer=server-180fd759-cadb-e5b5-76be-531e34f87a8b +2025-01-28T14:54:29.277Z [INFO] server-f16a4165-49eb-daee-52d5-fc7314684fa6: added peer, starting replication: peer=server-85b6ad5f-b4b0-2169-f8ba-e0b70b426553 +2025-01-28T14:54:29.277Z [INFO] server-f16a4165-49eb-daee-52d5-fc7314684fa6: pipelining replication: peer="{Voter server-180fd759-cadb-e5b5-76be-531e34f87a8b 180fd759-cadb-e5b5-76be-531e34f87a8b}" +2025-01-28T14:54:29.277Z [INFO] server-f16a4165-49eb-daee-52d5-fc7314684fa6: pipelining replication: peer="{Voter server-85b6ad5f-b4b0-2169-f8ba-e0b70b426553 85b6ad5f-b4b0-2169-f8ba-e0b70b426553}" +2025-01-28T14:54:30.283Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:29.277492085 +0000 UTC m=+54.951982085 (1 nodes), 576.973349ms from start of poll, 578.207092ms from cluster start. Timeout at 2025-01-28 14:54:30.282501664 +0000 UTC m=+55.956991664, 1.005009579s after stability +2025-01-28T14:54:30.284Z [INFO] server-f16a4165-49eb-daee-52d5-fc7314684fa6: starting snapshot up to: index=12 +2025-01-28T14:54:30.284Z [INFO] creating new snapshot: path=/tmp/raft2156609689/snapshots/2-12-1738076070284.tmp +2025-01-28T14:54:30.286Z [INFO] server-f16a4165-49eb-daee-52d5-fc7314684fa6: snapshot complete up to: index=12 +2025-01-28T14:54:30.288Z [INFO] creating new snapshot: path=/tmp/raft2156609689/snapshots/2-23-1738076070288.tmp +2025-01-28T14:54:30.290Z [INFO] server-f16a4165-49eb-daee-52d5-fc7314684fa6: copied to local snapshot: bytes=71 +2025-01-28T14:54:30.290Z [INFO] server-f16a4165-49eb-daee-52d5-fc7314684fa6: snapshot restore progress: id=2-23-1738076070288 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:30.290Z [INFO] server-f16a4165-49eb-daee-52d5-fc7314684fa6: restored user snapshot: index=23 +2025-01-28T14:54:30.290Z [ERROR] server-f16a4165-49eb-daee-52d5-fc7314684fa6: failed to get log: index=23 error="log not found" +2025-01-28T14:54:30.290Z [INFO] server-f16a4165-49eb-daee-52d5-fc7314684fa6: aborting pipeline replication: peer="{Voter server-85b6ad5f-b4b0-2169-f8ba-e0b70b426553 85b6ad5f-b4b0-2169-f8ba-e0b70b426553}" +2025-01-28T14:54:30.291Z [ERROR] server-f16a4165-49eb-daee-52d5-fc7314684fa6: failed to get log: index=23 error="log not found" +2025-01-28T14:54:30.291Z [INFO] server-f16a4165-49eb-daee-52d5-fc7314684fa6: aborting pipeline replication: peer="{Voter server-180fd759-cadb-e5b5-76be-531e34f87a8b 180fd759-cadb-e5b5-76be-531e34f87a8b}" +2025-01-28T14:54:30.299Z [ERROR] server-f16a4165-49eb-daee-52d5-fc7314684fa6: failed to get log: index=23 error="log not found" +2025-01-28T14:54:30.300Z [INFO] creating new snapshot: path=/tmp/raft465191807/snapshots/2-23-1738076070300.tmp +2025-01-28T14:54:30.301Z [ERROR] server-f16a4165-49eb-daee-52d5-fc7314684fa6: failed to get log: index=23 error="log not found" +2025-01-28T14:54:30.301Z [INFO] server-180fd759-cadb-e5b5-76be-531e34f87a8b: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:30.302Z [INFO] creating new snapshot: path=/tmp/raft762961124/snapshots/2-23-1738076070302.tmp +2025-01-28T14:54:30.302Z [INFO] server-180fd759-cadb-e5b5-76be-531e34f87a8b: copied to local snapshot: bytes=71 +2025-01-28T14:54:30.302Z [INFO] server-85b6ad5f-b4b0-2169-f8ba-e0b70b426553: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:30.302Z [INFO] server-180fd759-cadb-e5b5-76be-531e34f87a8b: snapshot restore progress: id=2-23-1738076070300 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:30.303Z [INFO] server-180fd759-cadb-e5b5-76be-531e34f87a8b: Installed remote snapshot +2025-01-28T14:54:30.303Z [WARN] server-180fd759-cadb-e5b5-76be-531e34f87a8b: skipping application of old log: index=22 +2025-01-28T14:54:30.303Z [INFO] server-85b6ad5f-b4b0-2169-f8ba-e0b70b426553: copied to local snapshot: bytes=71 +2025-01-28T14:54:30.303Z [INFO] server-f16a4165-49eb-daee-52d5-fc7314684fa6: pipelining replication: peer="{Voter server-180fd759-cadb-e5b5-76be-531e34f87a8b 180fd759-cadb-e5b5-76be-531e34f87a8b}" +2025-01-28T14:54:30.303Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:54:30.304Z [INFO] server-85b6ad5f-b4b0-2169-f8ba-e0b70b426553: snapshot restore progress: id=2-23-1738076070302 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:30.304Z [INFO] server-85b6ad5f-b4b0-2169-f8ba-e0b70b426553: Installed remote snapshot +2025-01-28T14:54:30.304Z [INFO] server-f16a4165-49eb-daee-52d5-fc7314684fa6: pipelining replication: peer="{Voter server-85b6ad5f-b4b0-2169-f8ba-e0b70b426553 85b6ad5f-b4b0-2169-f8ba-e0b70b426553}" +2025-01-28T14:54:31.309Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:54:30.303868058 +0000 UTC m=+55.978358038 (2 nodes), 2.479µs from start of poll, 1.604583045s from cluster start. Timeout at 2025-01-28 14:54:31.308904634 +0000 UTC m=+56.983394654, 1.005036616s after stability +2025-01-28T14:54:31.321Z [INFO] server-f16a4165-49eb-daee-52d5-fc7314684fa6: aborting pipeline replication: peer="{Voter server-85b6ad5f-b4b0-2169-f8ba-e0b70b426553 85b6ad5f-b4b0-2169-f8ba-e0b70b426553}" +2025-01-28T14:54:31.321Z [INFO] server-f16a4165-49eb-daee-52d5-fc7314684fa6: aborting pipeline replication: peer="{Voter server-180fd759-cadb-e5b5-76be-531e34f87a8b 180fd759-cadb-e5b5-76be-531e34f87a8b}" === RUN TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_false -2025-01-28T14:47:30.310Z [INFO] server-38c5302d-dc2c-f38a-f9a2-1c455ce74c38: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-38c5302d-dc2c-f38a-f9a2-1c455ce74c38 Address:38c5302d-dc2c-f38a-f9a2-1c455ce74c38} {Suffrage:Voter ID:server-c1e37476-0327-fcca-2379-37c54a428795 Address:c1e37476-0327-fcca-2379-37c54a428795} {Suffrage:Voter ID:server-d584fb87-a943-9514-dc9f-51329f22d5fc Address:d584fb87-a943-9514-dc9f-51329f22d5fc}]" -2025-01-28T14:47:30.311Z [INFO] server-38c5302d-dc2c-f38a-f9a2-1c455ce74c38: entering follower state: follower="Node at 38c5302d-dc2c-f38a-f9a2-1c455ce74c38 [Follower]" leader-address= leader-id= -2025-01-28T14:47:30.312Z [INFO] server-c1e37476-0327-fcca-2379-37c54a428795: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-38c5302d-dc2c-f38a-f9a2-1c455ce74c38 Address:38c5302d-dc2c-f38a-f9a2-1c455ce74c38} {Suffrage:Voter ID:server-c1e37476-0327-fcca-2379-37c54a428795 Address:c1e37476-0327-fcca-2379-37c54a428795} {Suffrage:Voter ID:server-d584fb87-a943-9514-dc9f-51329f22d5fc Address:d584fb87-a943-9514-dc9f-51329f22d5fc}]" -2025-01-28T14:47:30.312Z [INFO] server-c1e37476-0327-fcca-2379-37c54a428795: entering follower state: follower="Node at c1e37476-0327-fcca-2379-37c54a428795 [Follower]" leader-address= leader-id= -2025-01-28T14:47:30.313Z [INFO] server-d584fb87-a943-9514-dc9f-51329f22d5fc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-38c5302d-dc2c-f38a-f9a2-1c455ce74c38 Address:38c5302d-dc2c-f38a-f9a2-1c455ce74c38} {Suffrage:Voter ID:server-c1e37476-0327-fcca-2379-37c54a428795 Address:c1e37476-0327-fcca-2379-37c54a428795} {Suffrage:Voter ID:server-d584fb87-a943-9514-dc9f-51329f22d5fc Address:d584fb87-a943-9514-dc9f-51329f22d5fc}]" -2025-01-28T14:47:30.314Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:30.314Z [INFO] server-d584fb87-a943-9514-dc9f-51329f22d5fc: entering follower state: follower="Node at d584fb87-a943-9514-dc9f-51329f22d5fc [Follower]" leader-address= leader-id= -2025-01-28T14:47:30.898Z [WARN] server-d584fb87-a943-9514-dc9f-51329f22d5fc: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:30.898Z [INFO] server-d584fb87-a943-9514-dc9f-51329f22d5fc: entering candidate state: node="Node at d584fb87-a943-9514-dc9f-51329f22d5fc [Candidate]" term=2 -2025-01-28T14:47:30.899Z [INFO] server-d584fb87-a943-9514-dc9f-51329f22d5fc: election won: term=2 tally=2 -2025-01-28T14:47:30.900Z [INFO] server-d584fb87-a943-9514-dc9f-51329f22d5fc: entering leader state: leader="Node at d584fb87-a943-9514-dc9f-51329f22d5fc [Leader]" -2025-01-28T14:47:30.900Z [INFO] server-d584fb87-a943-9514-dc9f-51329f22d5fc: added peer, starting replication: peer=server-38c5302d-dc2c-f38a-f9a2-1c455ce74c38 -2025-01-28T14:47:30.900Z [INFO] server-d584fb87-a943-9514-dc9f-51329f22d5fc: added peer, starting replication: peer=server-c1e37476-0327-fcca-2379-37c54a428795 -2025-01-28T14:47:30.902Z [INFO] server-d584fb87-a943-9514-dc9f-51329f22d5fc: pipelining replication: peer="{Voter server-c1e37476-0327-fcca-2379-37c54a428795 c1e37476-0327-fcca-2379-37c54a428795}" -2025-01-28T14:47:30.902Z [INFO] server-d584fb87-a943-9514-dc9f-51329f22d5fc: pipelining replication: peer="{Voter server-38c5302d-dc2c-f38a-f9a2-1c455ce74c38 38c5302d-dc2c-f38a-f9a2-1c455ce74c38}" -2025-01-28T14:47:31.906Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:30.900581924 +0000 UTC m=+33.983988281 (1 nodes), 586.138402ms from start of poll, 590.877877ms from cluster start. Timeout at 2025-01-28 14:47:31.905618591 +0000 UTC m=+34.989024949, 1.005036668s after stability -2025-01-28T14:47:31.914Z [INFO] server-d584fb87-a943-9514-dc9f-51329f22d5fc: starting snapshot up to: index=12 -2025-01-28T14:47:31.914Z [INFO] creating new snapshot: path=/tmp/raft1285573842/snapshots/2-12-1738075651914.tmp -2025-01-28T14:47:31.922Z [INFO] server-d584fb87-a943-9514-dc9f-51329f22d5fc: snapshot complete up to: index=12 -2025-01-28T14:47:31.937Z [INFO] creating new snapshot: path=/tmp/raft1285573842/snapshots/2-23-1738075651936.tmp -2025-01-28T14:47:31.946Z [INFO] server-d584fb87-a943-9514-dc9f-51329f22d5fc: copied to local snapshot: bytes=71 -2025-01-28T14:47:31.948Z [INFO] server-d584fb87-a943-9514-dc9f-51329f22d5fc: snapshot restore progress: id=2-23-1738075651936 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:31.948Z [INFO] server-d584fb87-a943-9514-dc9f-51329f22d5fc: removing all old logs from log store -2025-01-28T14:47:31.948Z [INFO] server-d584fb87-a943-9514-dc9f-51329f22d5fc: compacting logs: from=1 to=22 -2025-01-28T14:47:31.948Z [INFO] server-d584fb87-a943-9514-dc9f-51329f22d5fc: restored user snapshot: index=23 -2025-01-28T14:47:31.949Z [ERROR] server-d584fb87-a943-9514-dc9f-51329f22d5fc: failed to get log: index=22 error="log not found" -2025-01-28T14:47:31.949Z [INFO] server-d584fb87-a943-9514-dc9f-51329f22d5fc: aborting pipeline replication: peer="{Voter server-38c5302d-dc2c-f38a-f9a2-1c455ce74c38 38c5302d-dc2c-f38a-f9a2-1c455ce74c38}" -2025-01-28T14:47:31.949Z [ERROR] server-d584fb87-a943-9514-dc9f-51329f22d5fc: failed to get log: index=22 error="log not found" -2025-01-28T14:47:31.949Z [INFO] server-d584fb87-a943-9514-dc9f-51329f22d5fc: aborting pipeline replication: peer="{Voter server-c1e37476-0327-fcca-2379-37c54a428795 c1e37476-0327-fcca-2379-37c54a428795}" -2025-01-28T14:47:31.957Z [ERROR] server-d584fb87-a943-9514-dc9f-51329f22d5fc: failed to get log: index=22 error="log not found" -2025-01-28T14:47:31.958Z [ERROR] server-d584fb87-a943-9514-dc9f-51329f22d5fc: failed to get log: index=22 error="log not found" -2025-01-28T14:47:31.963Z [INFO] creating new snapshot: path=/tmp/raft946878823/snapshots/2-23-1738075651963.tmp -2025-01-28T14:47:31.963Z [INFO] creating new snapshot: path=/tmp/raft3328387268/snapshots/2-23-1738075651963.tmp -2025-01-28T14:47:31.965Z [INFO] server-c1e37476-0327-fcca-2379-37c54a428795: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:31.965Z [INFO] server-38c5302d-dc2c-f38a-f9a2-1c455ce74c38: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:31.968Z [INFO] server-c1e37476-0327-fcca-2379-37c54a428795: copied to local snapshot: bytes=71 -2025-01-28T14:47:31.968Z [INFO] server-38c5302d-dc2c-f38a-f9a2-1c455ce74c38: copied to local snapshot: bytes=71 -2025-01-28T14:47:31.969Z [INFO] server-38c5302d-dc2c-f38a-f9a2-1c455ce74c38: snapshot restore progress: id=2-23-1738075651963 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:31.969Z [INFO] server-c1e37476-0327-fcca-2379-37c54a428795: snapshot restore progress: id=2-23-1738075651963 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:31.969Z [INFO] server-38c5302d-dc2c-f38a-f9a2-1c455ce74c38: removing all old logs from log store -2025-01-28T14:47:31.969Z [INFO] server-38c5302d-dc2c-f38a-f9a2-1c455ce74c38: compacting logs: from=1 to=22 -2025-01-28T14:47:31.969Z [INFO] server-c1e37476-0327-fcca-2379-37c54a428795: removing all old logs from log store -2025-01-28T14:47:31.969Z [INFO] server-c1e37476-0327-fcca-2379-37c54a428795: compacting logs: from=1 to=22 -2025-01-28T14:47:31.970Z [INFO] server-38c5302d-dc2c-f38a-f9a2-1c455ce74c38: Installed remote snapshot -2025-01-28T14:47:31.970Z [INFO] server-c1e37476-0327-fcca-2379-37c54a428795: Installed remote snapshot -2025-01-28T14:47:31.970Z [INFO] server-d584fb87-a943-9514-dc9f-51329f22d5fc: pipelining replication: peer="{Voter server-c1e37476-0327-fcca-2379-37c54a428795 c1e37476-0327-fcca-2379-37c54a428795}" -2025-01-28T14:47:31.971Z [INFO] server-d584fb87-a943-9514-dc9f-51329f22d5fc: pipelining replication: peer="{Voter server-38c5302d-dc2c-f38a-f9a2-1c455ce74c38 38c5302d-dc2c-f38a-f9a2-1c455ce74c38}" -2025-01-28T14:47:31.971Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:47:32.977Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:47:31.971306381 +0000 UTC m=+35.054711405 (2 nodes), 8µs from start of poll, 1.661601001s from cluster start. Timeout at 2025-01-28 14:47:32.976529388 +0000 UTC m=+36.059935412, 1.005224007s after stability -2025-01-28T14:47:32.990Z [INFO] server-d584fb87-a943-9514-dc9f-51329f22d5fc: aborting pipeline replication: peer="{Voter server-c1e37476-0327-fcca-2379-37c54a428795 c1e37476-0327-fcca-2379-37c54a428795}" -2025-01-28T14:47:32.990Z [INFO] server-d584fb87-a943-9514-dc9f-51329f22d5fc: aborting pipeline replication: peer="{Voter server-38c5302d-dc2c-f38a-f9a2-1c455ce74c38 38c5302d-dc2c-f38a-f9a2-1c455ce74c38}" +2025-01-28T14:54:31.326Z [INFO] server-18d90f22-4c7a-b5d6-08af-5dba33fe2ed7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-18d90f22-4c7a-b5d6-08af-5dba33fe2ed7 Address:18d90f22-4c7a-b5d6-08af-5dba33fe2ed7} {Suffrage:Voter ID:server-e3d2edfd-26c4-44da-e721-c910e7daeb59 Address:e3d2edfd-26c4-44da-e721-c910e7daeb59} {Suffrage:Voter ID:server-82c19a4e-9157-25aa-f15b-126f6d879685 Address:82c19a4e-9157-25aa-f15b-126f6d879685}]" +2025-01-28T14:54:31.326Z [INFO] server-18d90f22-4c7a-b5d6-08af-5dba33fe2ed7: entering follower state: follower="Node at 18d90f22-4c7a-b5d6-08af-5dba33fe2ed7 [Follower]" leader-address= leader-id= +2025-01-28T14:54:31.326Z [INFO] server-e3d2edfd-26c4-44da-e721-c910e7daeb59: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-18d90f22-4c7a-b5d6-08af-5dba33fe2ed7 Address:18d90f22-4c7a-b5d6-08af-5dba33fe2ed7} {Suffrage:Voter ID:server-e3d2edfd-26c4-44da-e721-c910e7daeb59 Address:e3d2edfd-26c4-44da-e721-c910e7daeb59} {Suffrage:Voter ID:server-82c19a4e-9157-25aa-f15b-126f6d879685 Address:82c19a4e-9157-25aa-f15b-126f6d879685}]" +2025-01-28T14:54:31.326Z [INFO] server-e3d2edfd-26c4-44da-e721-c910e7daeb59: entering follower state: follower="Node at e3d2edfd-26c4-44da-e721-c910e7daeb59 [Follower]" leader-address= leader-id= +2025-01-28T14:54:31.326Z [INFO] server-82c19a4e-9157-25aa-f15b-126f6d879685: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-18d90f22-4c7a-b5d6-08af-5dba33fe2ed7 Address:18d90f22-4c7a-b5d6-08af-5dba33fe2ed7} {Suffrage:Voter ID:server-e3d2edfd-26c4-44da-e721-c910e7daeb59 Address:e3d2edfd-26c4-44da-e721-c910e7daeb59} {Suffrage:Voter ID:server-82c19a4e-9157-25aa-f15b-126f6d879685 Address:82c19a4e-9157-25aa-f15b-126f6d879685}]" +2025-01-28T14:54:31.326Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:31.326Z [INFO] server-82c19a4e-9157-25aa-f15b-126f6d879685: entering follower state: follower="Node at 82c19a4e-9157-25aa-f15b-126f6d879685 [Follower]" leader-address= leader-id= +2025-01-28T14:54:31.834Z [WARN] server-e3d2edfd-26c4-44da-e721-c910e7daeb59: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:31.834Z [INFO] server-e3d2edfd-26c4-44da-e721-c910e7daeb59: entering candidate state: node="Node at e3d2edfd-26c4-44da-e721-c910e7daeb59 [Candidate]" term=2 +2025-01-28T14:54:31.834Z [INFO] server-e3d2edfd-26c4-44da-e721-c910e7daeb59: election won: term=2 tally=2 +2025-01-28T14:54:31.834Z [INFO] server-e3d2edfd-26c4-44da-e721-c910e7daeb59: entering leader state: leader="Node at e3d2edfd-26c4-44da-e721-c910e7daeb59 [Leader]" +2025-01-28T14:54:31.834Z [INFO] server-e3d2edfd-26c4-44da-e721-c910e7daeb59: added peer, starting replication: peer=server-18d90f22-4c7a-b5d6-08af-5dba33fe2ed7 +2025-01-28T14:54:31.834Z [INFO] server-e3d2edfd-26c4-44da-e721-c910e7daeb59: added peer, starting replication: peer=server-82c19a4e-9157-25aa-f15b-126f6d879685 +2025-01-28T14:54:31.835Z [INFO] server-e3d2edfd-26c4-44da-e721-c910e7daeb59: pipelining replication: peer="{Voter server-18d90f22-4c7a-b5d6-08af-5dba33fe2ed7 18d90f22-4c7a-b5d6-08af-5dba33fe2ed7}" +2025-01-28T14:54:31.835Z [INFO] server-e3d2edfd-26c4-44da-e721-c910e7daeb59: pipelining replication: peer="{Voter server-82c19a4e-9157-25aa-f15b-126f6d879685 82c19a4e-9157-25aa-f15b-126f6d879685}" +2025-01-28T14:54:32.840Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:31.834724939 +0000 UTC m=+57.509214899 (1 nodes), 508.145122ms from start of poll, 508.921209ms from cluster start. Timeout at 2025-01-28 14:54:32.839728838 +0000 UTC m=+58.514218858, 1.005003959s after stability +2025-01-28T14:54:32.841Z [INFO] server-e3d2edfd-26c4-44da-e721-c910e7daeb59: starting snapshot up to: index=12 +2025-01-28T14:54:32.841Z [INFO] creating new snapshot: path=/tmp/raft1746061664/snapshots/2-12-1738076072841.tmp +2025-01-28T14:54:33.140Z [INFO] server-e3d2edfd-26c4-44da-e721-c910e7daeb59: snapshot complete up to: index=12 +2025-01-28T14:54:33.141Z [INFO] creating new snapshot: path=/tmp/raft1746061664/snapshots/2-23-1738076073141.tmp +2025-01-28T14:54:33.143Z [INFO] server-e3d2edfd-26c4-44da-e721-c910e7daeb59: copied to local snapshot: bytes=71 +2025-01-28T14:54:33.143Z [INFO] server-e3d2edfd-26c4-44da-e721-c910e7daeb59: snapshot restore progress: id=2-23-1738076073141 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:33.143Z [INFO] server-e3d2edfd-26c4-44da-e721-c910e7daeb59: removing all old logs from log store +2025-01-28T14:54:33.143Z [INFO] server-e3d2edfd-26c4-44da-e721-c910e7daeb59: compacting logs: from=1 to=22 +2025-01-28T14:54:33.143Z [INFO] server-e3d2edfd-26c4-44da-e721-c910e7daeb59: restored user snapshot: index=23 +2025-01-28T14:54:33.144Z [ERROR] server-e3d2edfd-26c4-44da-e721-c910e7daeb59: failed to get log: index=22 error="log not found" +2025-01-28T14:54:33.144Z [INFO] server-e3d2edfd-26c4-44da-e721-c910e7daeb59: aborting pipeline replication: peer="{Voter server-82c19a4e-9157-25aa-f15b-126f6d879685 82c19a4e-9157-25aa-f15b-126f6d879685}" +2025-01-28T14:54:33.144Z [ERROR] server-e3d2edfd-26c4-44da-e721-c910e7daeb59: failed to get log: index=22 error="log not found" +2025-01-28T14:54:33.144Z [INFO] server-e3d2edfd-26c4-44da-e721-c910e7daeb59: aborting pipeline replication: peer="{Voter server-18d90f22-4c7a-b5d6-08af-5dba33fe2ed7 18d90f22-4c7a-b5d6-08af-5dba33fe2ed7}" +2025-01-28T14:54:33.152Z [ERROR] server-e3d2edfd-26c4-44da-e721-c910e7daeb59: failed to get log: index=22 error="log not found" +2025-01-28T14:54:33.153Z [INFO] creating new snapshot: path=/tmp/raft2668722255/snapshots/2-23-1738076073153.tmp +2025-01-28T14:54:33.153Z [ERROR] server-e3d2edfd-26c4-44da-e721-c910e7daeb59: failed to get log: index=22 error="log not found" +2025-01-28T14:54:33.154Z [INFO] server-82c19a4e-9157-25aa-f15b-126f6d879685: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:33.154Z [INFO] creating new snapshot: path=/tmp/raft3123973954/snapshots/2-23-1738076073154.tmp +2025-01-28T14:54:33.155Z [INFO] server-82c19a4e-9157-25aa-f15b-126f6d879685: copied to local snapshot: bytes=71 +2025-01-28T14:54:33.155Z [INFO] server-18d90f22-4c7a-b5d6-08af-5dba33fe2ed7: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:33.155Z [INFO] server-82c19a4e-9157-25aa-f15b-126f6d879685: snapshot restore progress: id=2-23-1738076073153 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:33.155Z [INFO] server-82c19a4e-9157-25aa-f15b-126f6d879685: removing all old logs from log store +2025-01-28T14:54:33.155Z [INFO] server-82c19a4e-9157-25aa-f15b-126f6d879685: compacting logs: from=1 to=22 +2025-01-28T14:54:33.155Z [INFO] server-82c19a4e-9157-25aa-f15b-126f6d879685: Installed remote snapshot +2025-01-28T14:54:33.155Z [INFO] server-18d90f22-4c7a-b5d6-08af-5dba33fe2ed7: copied to local snapshot: bytes=71 +2025-01-28T14:54:33.156Z [WARN] server-82c19a4e-9157-25aa-f15b-126f6d879685: skipping application of old log: index=22 +2025-01-28T14:54:33.156Z [INFO] server-e3d2edfd-26c4-44da-e721-c910e7daeb59: pipelining replication: peer="{Voter server-82c19a4e-9157-25aa-f15b-126f6d879685 82c19a4e-9157-25aa-f15b-126f6d879685}" +2025-01-28T14:54:33.156Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:54:33.156Z [INFO] server-18d90f22-4c7a-b5d6-08af-5dba33fe2ed7: snapshot restore progress: id=2-23-1738076073154 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:33.156Z [INFO] server-18d90f22-4c7a-b5d6-08af-5dba33fe2ed7: removing all old logs from log store +2025-01-28T14:54:33.156Z [INFO] server-18d90f22-4c7a-b5d6-08af-5dba33fe2ed7: compacting logs: from=1 to=22 +2025-01-28T14:54:33.156Z [INFO] server-18d90f22-4c7a-b5d6-08af-5dba33fe2ed7: Installed remote snapshot +2025-01-28T14:54:33.156Z [INFO] server-e3d2edfd-26c4-44da-e721-c910e7daeb59: pipelining replication: peer="{Voter server-18d90f22-4c7a-b5d6-08af-5dba33fe2ed7 18d90f22-4c7a-b5d6-08af-5dba33fe2ed7}" +2025-01-28T14:54:34.162Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:54:33.156367398 +0000 UTC m=+58.830857358 (2 nodes), 2.32µs from start of poll, 1.830563668s from cluster start. Timeout at 2025-01-28 14:54:34.161404195 +0000 UTC m=+59.835894155, 1.005036797s after stability +2025-01-28T14:54:34.175Z [INFO] server-e3d2edfd-26c4-44da-e721-c910e7daeb59: aborting pipeline replication: peer="{Voter server-82c19a4e-9157-25aa-f15b-126f6d879685 82c19a4e-9157-25aa-f15b-126f6d879685}" +2025-01-28T14:54:34.175Z [INFO] server-e3d2edfd-26c4-44da-e721-c910e7daeb59: aborting pipeline replication: peer="{Voter server-18d90f22-4c7a-b5d6-08af-5dba33fe2ed7 18d90f22-4c7a-b5d6-08af-5dba33fe2ed7}" === RUN TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_true -2025-01-28T14:47:33.016Z [INFO] server-cad7fdee-dd1b-b5b3-8983-5972aea7e140: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cad7fdee-dd1b-b5b3-8983-5972aea7e140 Address:cad7fdee-dd1b-b5b3-8983-5972aea7e140} {Suffrage:Voter ID:server-bb582f92-2308-3fa9-7ed0-4487ae2e1e2d Address:bb582f92-2308-3fa9-7ed0-4487ae2e1e2d} {Suffrage:Voter ID:server-e76b4dbc-59c9-5243-69d8-ec7d906c1c31 Address:e76b4dbc-59c9-5243-69d8-ec7d906c1c31}]" -2025-01-28T14:47:33.018Z [INFO] server-bb582f92-2308-3fa9-7ed0-4487ae2e1e2d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cad7fdee-dd1b-b5b3-8983-5972aea7e140 Address:cad7fdee-dd1b-b5b3-8983-5972aea7e140} {Suffrage:Voter ID:server-bb582f92-2308-3fa9-7ed0-4487ae2e1e2d Address:bb582f92-2308-3fa9-7ed0-4487ae2e1e2d} {Suffrage:Voter ID:server-e76b4dbc-59c9-5243-69d8-ec7d906c1c31 Address:e76b4dbc-59c9-5243-69d8-ec7d906c1c31}]" -2025-01-28T14:47:33.018Z [INFO] server-cad7fdee-dd1b-b5b3-8983-5972aea7e140: entering follower state: follower="Node at cad7fdee-dd1b-b5b3-8983-5972aea7e140 [Follower]" leader-address= leader-id= -2025-01-28T14:47:33.018Z [INFO] server-bb582f92-2308-3fa9-7ed0-4487ae2e1e2d: entering follower state: follower="Node at bb582f92-2308-3fa9-7ed0-4487ae2e1e2d [Follower]" leader-address= leader-id= -2025-01-28T14:47:33.019Z [INFO] server-e76b4dbc-59c9-5243-69d8-ec7d906c1c31: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cad7fdee-dd1b-b5b3-8983-5972aea7e140 Address:cad7fdee-dd1b-b5b3-8983-5972aea7e140} {Suffrage:Voter ID:server-bb582f92-2308-3fa9-7ed0-4487ae2e1e2d Address:bb582f92-2308-3fa9-7ed0-4487ae2e1e2d} {Suffrage:Voter ID:server-e76b4dbc-59c9-5243-69d8-ec7d906c1c31 Address:e76b4dbc-59c9-5243-69d8-ec7d906c1c31}]" -2025-01-28T14:47:33.020Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:33.020Z [INFO] server-e76b4dbc-59c9-5243-69d8-ec7d906c1c31: entering follower state: follower="Node at e76b4dbc-59c9-5243-69d8-ec7d906c1c31 [Follower]" leader-address= leader-id= -2025-01-28T14:47:33.560Z [WARN] server-e76b4dbc-59c9-5243-69d8-ec7d906c1c31: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:33.560Z [INFO] server-e76b4dbc-59c9-5243-69d8-ec7d906c1c31: entering candidate state: node="Node at e76b4dbc-59c9-5243-69d8-ec7d906c1c31 [Candidate]" term=2 -2025-01-28T14:47:33.562Z [INFO] server-e76b4dbc-59c9-5243-69d8-ec7d906c1c31: election won: term=2 tally=2 -2025-01-28T14:47:33.562Z [INFO] server-e76b4dbc-59c9-5243-69d8-ec7d906c1c31: entering leader state: leader="Node at e76b4dbc-59c9-5243-69d8-ec7d906c1c31 [Leader]" -2025-01-28T14:47:33.562Z [INFO] server-e76b4dbc-59c9-5243-69d8-ec7d906c1c31: added peer, starting replication: peer=server-cad7fdee-dd1b-b5b3-8983-5972aea7e140 -2025-01-28T14:47:33.562Z [INFO] server-e76b4dbc-59c9-5243-69d8-ec7d906c1c31: added peer, starting replication: peer=server-bb582f92-2308-3fa9-7ed0-4487ae2e1e2d -2025-01-28T14:47:33.563Z [INFO] server-e76b4dbc-59c9-5243-69d8-ec7d906c1c31: pipelining replication: peer="{Voter server-cad7fdee-dd1b-b5b3-8983-5972aea7e140 cad7fdee-dd1b-b5b3-8983-5972aea7e140}" -2025-01-28T14:47:33.564Z [INFO] server-e76b4dbc-59c9-5243-69d8-ec7d906c1c31: pipelining replication: peer="{Voter server-bb582f92-2308-3fa9-7ed0-4487ae2e1e2d bb582f92-2308-3fa9-7ed0-4487ae2e1e2d}" -2025-01-28T14:47:34.567Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:33.562538292 +0000 UTC m=+36.645944984 (1 nodes), 542.121097ms from start of poll, 546.985908ms from cluster start. Timeout at 2025-01-28 14:47:34.567560959 +0000 UTC m=+37.650966984, 1.005022s after stability -2025-01-28T14:47:34.576Z [INFO] server-e76b4dbc-59c9-5243-69d8-ec7d906c1c31: starting snapshot up to: index=12 -2025-01-28T14:47:34.576Z [INFO] creating new snapshot: path=/tmp/raft4240931850/snapshots/2-12-1738075654576.tmp -2025-01-28T14:47:34.583Z [INFO] server-e76b4dbc-59c9-5243-69d8-ec7d906c1c31: snapshot complete up to: index=12 -2025-01-28T14:47:34.594Z [INFO] server-0c3d5030-e31e-a65f-ab98-18c6e1f535bb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0c3d5030-e31e-a65f-ab98-18c6e1f535bb Address:0c3d5030-e31e-a65f-ab98-18c6e1f535bb} {Suffrage:Voter ID:server-d9bdee70-33be-8ce5-cc95-b4ae2f9be867 Address:d9bdee70-33be-8ce5-cc95-b4ae2f9be867} {Suffrage:Voter ID:server-dbda4c06-c499-520d-52bc-2f751a7c30b4 Address:dbda4c06-c499-520d-52bc-2f751a7c30b4}]" -2025-01-28T14:47:34.595Z [INFO] server-0c3d5030-e31e-a65f-ab98-18c6e1f535bb: entering follower state: follower="Node at 0c3d5030-e31e-a65f-ab98-18c6e1f535bb [Follower]" leader-address= leader-id= -2025-01-28T14:47:34.596Z [INFO] server-d9bdee70-33be-8ce5-cc95-b4ae2f9be867: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0c3d5030-e31e-a65f-ab98-18c6e1f535bb Address:0c3d5030-e31e-a65f-ab98-18c6e1f535bb} {Suffrage:Voter ID:server-d9bdee70-33be-8ce5-cc95-b4ae2f9be867 Address:d9bdee70-33be-8ce5-cc95-b4ae2f9be867} {Suffrage:Voter ID:server-dbda4c06-c499-520d-52bc-2f751a7c30b4 Address:dbda4c06-c499-520d-52bc-2f751a7c30b4}]" -2025-01-28T14:47:34.596Z [INFO] server-d9bdee70-33be-8ce5-cc95-b4ae2f9be867: entering follower state: follower="Node at d9bdee70-33be-8ce5-cc95-b4ae2f9be867 [Follower]" leader-address= leader-id= -2025-01-28T14:47:34.597Z [INFO] server-dbda4c06-c499-520d-52bc-2f751a7c30b4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0c3d5030-e31e-a65f-ab98-18c6e1f535bb Address:0c3d5030-e31e-a65f-ab98-18c6e1f535bb} {Suffrage:Voter ID:server-d9bdee70-33be-8ce5-cc95-b4ae2f9be867 Address:d9bdee70-33be-8ce5-cc95-b4ae2f9be867} {Suffrage:Voter ID:server-dbda4c06-c499-520d-52bc-2f751a7c30b4 Address:dbda4c06-c499-520d-52bc-2f751a7c30b4}]" -2025-01-28T14:47:34.597Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:34.598Z [INFO] server-dbda4c06-c499-520d-52bc-2f751a7c30b4: entering follower state: follower="Node at dbda4c06-c499-520d-52bc-2f751a7c30b4 [Follower]" leader-address= leader-id= -2025-01-28T14:47:35.165Z [WARN] server-d9bdee70-33be-8ce5-cc95-b4ae2f9be867: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:35.165Z [INFO] server-d9bdee70-33be-8ce5-cc95-b4ae2f9be867: entering candidate state: node="Node at d9bdee70-33be-8ce5-cc95-b4ae2f9be867 [Candidate]" term=2 -2025-01-28T14:47:35.166Z [INFO] server-d9bdee70-33be-8ce5-cc95-b4ae2f9be867: election won: term=2 tally=2 -2025-01-28T14:47:35.167Z [INFO] server-d9bdee70-33be-8ce5-cc95-b4ae2f9be867: entering leader state: leader="Node at d9bdee70-33be-8ce5-cc95-b4ae2f9be867 [Leader]" -2025-01-28T14:47:35.167Z [INFO] server-d9bdee70-33be-8ce5-cc95-b4ae2f9be867: added peer, starting replication: peer=server-0c3d5030-e31e-a65f-ab98-18c6e1f535bb -2025-01-28T14:47:35.167Z [INFO] server-d9bdee70-33be-8ce5-cc95-b4ae2f9be867: added peer, starting replication: peer=server-dbda4c06-c499-520d-52bc-2f751a7c30b4 -2025-01-28T14:47:35.169Z [INFO] server-d9bdee70-33be-8ce5-cc95-b4ae2f9be867: pipelining replication: peer="{Voter server-0c3d5030-e31e-a65f-ab98-18c6e1f535bb 0c3d5030-e31e-a65f-ab98-18c6e1f535bb}" -2025-01-28T14:47:35.169Z [INFO] server-d9bdee70-33be-8ce5-cc95-b4ae2f9be867: pipelining replication: peer="{Voter server-dbda4c06-c499-520d-52bc-2f751a7c30b4 dbda4c06-c499-520d-52bc-2f751a7c30b4}" -2025-01-28T14:47:36.173Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:35.167439277 +0000 UTC m=+38.250845635 (1 nodes), 569.245235ms from start of poll, 574.337053ms from cluster start. Timeout at 2025-01-28 14:47:36.17245361 +0000 UTC m=+39.255858635, 1.005013s after stability -2025-01-28T14:47:36.174Z [INFO] creating new snapshot: path=/tmp/raft3868837951/snapshots/2-14-1738075656174.tmp -2025-01-28T14:47:36.179Z [INFO] server-d9bdee70-33be-8ce5-cc95-b4ae2f9be867: copied to local snapshot: bytes=71 -2025-01-28T14:47:36.182Z [INFO] server-d9bdee70-33be-8ce5-cc95-b4ae2f9be867: snapshot restore progress: id=2-14-1738075656174 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:36.182Z [INFO] server-d9bdee70-33be-8ce5-cc95-b4ae2f9be867: restored user snapshot: index=14 -2025-01-28T14:47:36.183Z [ERROR] server-d9bdee70-33be-8ce5-cc95-b4ae2f9be867: failed to get log: index=3 error="log not found" -2025-01-28T14:47:36.183Z [INFO] server-d9bdee70-33be-8ce5-cc95-b4ae2f9be867: aborting pipeline replication: peer="{Voter server-dbda4c06-c499-520d-52bc-2f751a7c30b4 dbda4c06-c499-520d-52bc-2f751a7c30b4}" -2025-01-28T14:47:36.184Z [ERROR] server-d9bdee70-33be-8ce5-cc95-b4ae2f9be867: failed to get log: index=3 error="log not found" -2025-01-28T14:47:36.184Z [INFO] server-d9bdee70-33be-8ce5-cc95-b4ae2f9be867: aborting pipeline replication: peer="{Voter server-0c3d5030-e31e-a65f-ab98-18c6e1f535bb 0c3d5030-e31e-a65f-ab98-18c6e1f535bb}" -2025-01-28T14:47:36.191Z [ERROR] server-d9bdee70-33be-8ce5-cc95-b4ae2f9be867: failed to get log: index=3 error="log not found" -2025-01-28T14:47:36.192Z [ERROR] server-d9bdee70-33be-8ce5-cc95-b4ae2f9be867: failed to get log: index=3 error="log not found" -2025-01-28T14:47:36.194Z [INFO] creating new snapshot: path=/tmp/raft457258100/snapshots/2-14-1738075656194.tmp -2025-01-28T14:47:36.195Z [INFO] creating new snapshot: path=/tmp/raft1367413152/snapshots/2-14-1738075656195.tmp -2025-01-28T14:47:36.196Z [INFO] server-0c3d5030-e31e-a65f-ab98-18c6e1f535bb: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:36.198Z [INFO] server-dbda4c06-c499-520d-52bc-2f751a7c30b4: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:36.199Z [INFO] server-0c3d5030-e31e-a65f-ab98-18c6e1f535bb: copied to local snapshot: bytes=71 -2025-01-28T14:47:36.200Z [INFO] server-dbda4c06-c499-520d-52bc-2f751a7c30b4: copied to local snapshot: bytes=71 -2025-01-28T14:47:36.201Z [INFO] server-0c3d5030-e31e-a65f-ab98-18c6e1f535bb: snapshot restore progress: id=2-14-1738075656194 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:36.201Z [INFO] server-0c3d5030-e31e-a65f-ab98-18c6e1f535bb: Installed remote snapshot -2025-01-28T14:47:36.202Z [INFO] server-dbda4c06-c499-520d-52bc-2f751a7c30b4: snapshot restore progress: id=2-14-1738075656195 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:36.202Z [INFO] server-dbda4c06-c499-520d-52bc-2f751a7c30b4: Installed remote snapshot -2025-01-28T14:47:36.203Z [INFO] server-d9bdee70-33be-8ce5-cc95-b4ae2f9be867: pipelining replication: peer="{Voter server-0c3d5030-e31e-a65f-ab98-18c6e1f535bb 0c3d5030-e31e-a65f-ab98-18c6e1f535bb}" -2025-01-28T14:47:36.203Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:47:36.204Z [INFO] server-d9bdee70-33be-8ce5-cc95-b4ae2f9be867: pipelining replication: peer="{Voter server-dbda4c06-c499-520d-52bc-2f751a7c30b4 dbda4c06-c499-520d-52bc-2f751a7c30b4}" -2025-01-28T14:47:37.209Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:47:36.203855383 +0000 UTC m=+39.287261740 (2 nodes), 55.001µs from start of poll, 1.610753158s from cluster start. Timeout at 2025-01-28 14:47:37.209075055 +0000 UTC m=+40.292481413, 1.005219673s after stability -2025-01-28T14:47:37.229Z [INFO] server-e76b4dbc-59c9-5243-69d8-ec7d906c1c31: aborting pipeline replication: peer="{Voter server-cad7fdee-dd1b-b5b3-8983-5972aea7e140 cad7fdee-dd1b-b5b3-8983-5972aea7e140}" -2025-01-28T14:47:37.229Z [INFO] server-e76b4dbc-59c9-5243-69d8-ec7d906c1c31: aborting pipeline replication: peer="{Voter server-bb582f92-2308-3fa9-7ed0-4487ae2e1e2d bb582f92-2308-3fa9-7ed0-4487ae2e1e2d}" +2025-01-28T14:54:34.180Z [INFO] server-c9f5320d-c1a0-128f-0032-7168be7c2b60: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c9f5320d-c1a0-128f-0032-7168be7c2b60 Address:c9f5320d-c1a0-128f-0032-7168be7c2b60} {Suffrage:Voter ID:server-6784c6c9-f9ef-f762-7dc7-ff3d4810281f Address:6784c6c9-f9ef-f762-7dc7-ff3d4810281f} {Suffrage:Voter ID:server-5a912079-e29a-fd62-7b1c-5bd6bda950d4 Address:5a912079-e29a-fd62-7b1c-5bd6bda950d4}]" +2025-01-28T14:54:34.180Z [INFO] server-c9f5320d-c1a0-128f-0032-7168be7c2b60: entering follower state: follower="Node at c9f5320d-c1a0-128f-0032-7168be7c2b60 [Follower]" leader-address= leader-id= +2025-01-28T14:54:34.180Z [INFO] server-6784c6c9-f9ef-f762-7dc7-ff3d4810281f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c9f5320d-c1a0-128f-0032-7168be7c2b60 Address:c9f5320d-c1a0-128f-0032-7168be7c2b60} {Suffrage:Voter ID:server-6784c6c9-f9ef-f762-7dc7-ff3d4810281f Address:6784c6c9-f9ef-f762-7dc7-ff3d4810281f} {Suffrage:Voter ID:server-5a912079-e29a-fd62-7b1c-5bd6bda950d4 Address:5a912079-e29a-fd62-7b1c-5bd6bda950d4}]" +2025-01-28T14:54:34.180Z [INFO] server-6784c6c9-f9ef-f762-7dc7-ff3d4810281f: entering follower state: follower="Node at 6784c6c9-f9ef-f762-7dc7-ff3d4810281f [Follower]" leader-address= leader-id= +2025-01-28T14:54:34.180Z [INFO] server-5a912079-e29a-fd62-7b1c-5bd6bda950d4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c9f5320d-c1a0-128f-0032-7168be7c2b60 Address:c9f5320d-c1a0-128f-0032-7168be7c2b60} {Suffrage:Voter ID:server-6784c6c9-f9ef-f762-7dc7-ff3d4810281f Address:6784c6c9-f9ef-f762-7dc7-ff3d4810281f} {Suffrage:Voter ID:server-5a912079-e29a-fd62-7b1c-5bd6bda950d4 Address:5a912079-e29a-fd62-7b1c-5bd6bda950d4}]" +2025-01-28T14:54:34.180Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:34.180Z [INFO] server-5a912079-e29a-fd62-7b1c-5bd6bda950d4: entering follower state: follower="Node at 5a912079-e29a-fd62-7b1c-5bd6bda950d4 [Follower]" leader-address= leader-id= +2025-01-28T14:54:34.776Z [WARN] server-c9f5320d-c1a0-128f-0032-7168be7c2b60: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:34.776Z [INFO] server-c9f5320d-c1a0-128f-0032-7168be7c2b60: entering candidate state: node="Node at c9f5320d-c1a0-128f-0032-7168be7c2b60 [Candidate]" term=2 +2025-01-28T14:54:34.776Z [INFO] server-c9f5320d-c1a0-128f-0032-7168be7c2b60: election won: term=2 tally=2 +2025-01-28T14:54:34.776Z [INFO] server-c9f5320d-c1a0-128f-0032-7168be7c2b60: entering leader state: leader="Node at c9f5320d-c1a0-128f-0032-7168be7c2b60 [Leader]" +2025-01-28T14:54:34.776Z [INFO] server-c9f5320d-c1a0-128f-0032-7168be7c2b60: added peer, starting replication: peer=server-6784c6c9-f9ef-f762-7dc7-ff3d4810281f +2025-01-28T14:54:34.776Z [INFO] server-c9f5320d-c1a0-128f-0032-7168be7c2b60: added peer, starting replication: peer=server-5a912079-e29a-fd62-7b1c-5bd6bda950d4 +2025-01-28T14:54:34.776Z [INFO] server-c9f5320d-c1a0-128f-0032-7168be7c2b60: pipelining replication: peer="{Voter server-6784c6c9-f9ef-f762-7dc7-ff3d4810281f 6784c6c9-f9ef-f762-7dc7-ff3d4810281f}" +2025-01-28T14:54:34.776Z [INFO] server-c9f5320d-c1a0-128f-0032-7168be7c2b60: pipelining replication: peer="{Voter server-5a912079-e29a-fd62-7b1c-5bd6bda950d4 5a912079-e29a-fd62-7b1c-5bd6bda950d4}" +2025-01-28T14:54:35.781Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:34.776401166 +0000 UTC m=+60.450891126 (1 nodes), 595.583473ms from start of poll, 596.527104ms from cluster start. Timeout at 2025-01-28 14:54:35.781416724 +0000 UTC m=+61.455906704, 1.005015578s after stability +2025-01-28T14:54:35.785Z [INFO] server-c9f5320d-c1a0-128f-0032-7168be7c2b60: starting snapshot up to: index=12 +2025-01-28T14:54:35.785Z [INFO] creating new snapshot: path=/tmp/raft671186994/snapshots/2-12-1738076075785.tmp +2025-01-28T14:54:35.787Z [INFO] server-c9f5320d-c1a0-128f-0032-7168be7c2b60: snapshot complete up to: index=12 +2025-01-28T14:54:35.790Z [INFO] server-6c849148-cc31-4bf7-9366-7dca4454f9d7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6c849148-cc31-4bf7-9366-7dca4454f9d7 Address:6c849148-cc31-4bf7-9366-7dca4454f9d7} {Suffrage:Voter ID:server-7ec272ff-ab2e-ea24-4b02-d10bef5a7b09 Address:7ec272ff-ab2e-ea24-4b02-d10bef5a7b09} {Suffrage:Voter ID:server-e4dc8683-71e7-b8c9-1c1c-d49653e8d5a7 Address:e4dc8683-71e7-b8c9-1c1c-d49653e8d5a7}]" +2025-01-28T14:54:35.791Z [INFO] server-6c849148-cc31-4bf7-9366-7dca4454f9d7: entering follower state: follower="Node at 6c849148-cc31-4bf7-9366-7dca4454f9d7 [Follower]" leader-address= leader-id= +2025-01-28T14:54:35.791Z [INFO] server-7ec272ff-ab2e-ea24-4b02-d10bef5a7b09: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6c849148-cc31-4bf7-9366-7dca4454f9d7 Address:6c849148-cc31-4bf7-9366-7dca4454f9d7} {Suffrage:Voter ID:server-7ec272ff-ab2e-ea24-4b02-d10bef5a7b09 Address:7ec272ff-ab2e-ea24-4b02-d10bef5a7b09} {Suffrage:Voter ID:server-e4dc8683-71e7-b8c9-1c1c-d49653e8d5a7 Address:e4dc8683-71e7-b8c9-1c1c-d49653e8d5a7}]" +2025-01-28T14:54:35.791Z [INFO] server-7ec272ff-ab2e-ea24-4b02-d10bef5a7b09: entering follower state: follower="Node at 7ec272ff-ab2e-ea24-4b02-d10bef5a7b09 [Follower]" leader-address= leader-id= +2025-01-28T14:54:35.791Z [INFO] server-e4dc8683-71e7-b8c9-1c1c-d49653e8d5a7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6c849148-cc31-4bf7-9366-7dca4454f9d7 Address:6c849148-cc31-4bf7-9366-7dca4454f9d7} {Suffrage:Voter ID:server-7ec272ff-ab2e-ea24-4b02-d10bef5a7b09 Address:7ec272ff-ab2e-ea24-4b02-d10bef5a7b09} {Suffrage:Voter ID:server-e4dc8683-71e7-b8c9-1c1c-d49653e8d5a7 Address:e4dc8683-71e7-b8c9-1c1c-d49653e8d5a7}]" +2025-01-28T14:54:35.791Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:35.791Z [INFO] server-e4dc8683-71e7-b8c9-1c1c-d49653e8d5a7: entering follower state: follower="Node at e4dc8683-71e7-b8c9-1c1c-d49653e8d5a7 [Follower]" leader-address= leader-id= +2025-01-28T14:54:36.368Z [WARN] server-e4dc8683-71e7-b8c9-1c1c-d49653e8d5a7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:36.368Z [INFO] server-e4dc8683-71e7-b8c9-1c1c-d49653e8d5a7: entering candidate state: node="Node at e4dc8683-71e7-b8c9-1c1c-d49653e8d5a7 [Candidate]" term=2 +2025-01-28T14:54:36.369Z [INFO] server-e4dc8683-71e7-b8c9-1c1c-d49653e8d5a7: election won: term=2 tally=2 +2025-01-28T14:54:36.369Z [INFO] server-e4dc8683-71e7-b8c9-1c1c-d49653e8d5a7: entering leader state: leader="Node at e4dc8683-71e7-b8c9-1c1c-d49653e8d5a7 [Leader]" +2025-01-28T14:54:36.369Z [INFO] server-e4dc8683-71e7-b8c9-1c1c-d49653e8d5a7: added peer, starting replication: peer=server-6c849148-cc31-4bf7-9366-7dca4454f9d7 +2025-01-28T14:54:36.369Z [INFO] server-e4dc8683-71e7-b8c9-1c1c-d49653e8d5a7: added peer, starting replication: peer=server-7ec272ff-ab2e-ea24-4b02-d10bef5a7b09 +2025-01-28T14:54:36.369Z [INFO] server-e4dc8683-71e7-b8c9-1c1c-d49653e8d5a7: pipelining replication: peer="{Voter server-6c849148-cc31-4bf7-9366-7dca4454f9d7 6c849148-cc31-4bf7-9366-7dca4454f9d7}" +2025-01-28T14:54:36.369Z [INFO] server-e4dc8683-71e7-b8c9-1c1c-d49653e8d5a7: pipelining replication: peer="{Voter server-7ec272ff-ab2e-ea24-4b02-d10bef5a7b09 7ec272ff-ab2e-ea24-4b02-d10bef5a7b09}" +2025-01-28T14:54:37.374Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:36.369160728 +0000 UTC m=+62.043650708 (1 nodes), 577.462803ms from start of poll, 578.489227ms from cluster start. Timeout at 2025-01-28 14:54:37.374173207 +0000 UTC m=+63.048663227, 1.005012519s after stability +2025-01-28T14:54:37.375Z [INFO] creating new snapshot: path=/tmp/raft3047435954/snapshots/2-14-1738076077375.tmp +2025-01-28T14:54:37.376Z [INFO] server-e4dc8683-71e7-b8c9-1c1c-d49653e8d5a7: copied to local snapshot: bytes=71 +2025-01-28T14:54:37.377Z [INFO] server-e4dc8683-71e7-b8c9-1c1c-d49653e8d5a7: snapshot restore progress: id=2-14-1738076077375 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:37.377Z [INFO] server-e4dc8683-71e7-b8c9-1c1c-d49653e8d5a7: restored user snapshot: index=14 +2025-01-28T14:54:37.377Z [ERROR] server-e4dc8683-71e7-b8c9-1c1c-d49653e8d5a7: failed to get log: index=3 error="log not found" +2025-01-28T14:54:37.377Z [INFO] server-e4dc8683-71e7-b8c9-1c1c-d49653e8d5a7: aborting pipeline replication: peer="{Voter server-7ec272ff-ab2e-ea24-4b02-d10bef5a7b09 7ec272ff-ab2e-ea24-4b02-d10bef5a7b09}" +2025-01-28T14:54:37.377Z [ERROR] server-e4dc8683-71e7-b8c9-1c1c-d49653e8d5a7: failed to get log: index=3 error="log not found" +2025-01-28T14:54:37.378Z [INFO] server-e4dc8683-71e7-b8c9-1c1c-d49653e8d5a7: aborting pipeline replication: peer="{Voter server-6c849148-cc31-4bf7-9366-7dca4454f9d7 6c849148-cc31-4bf7-9366-7dca4454f9d7}" +2025-01-28T14:54:37.383Z [ERROR] server-e4dc8683-71e7-b8c9-1c1c-d49653e8d5a7: failed to get log: index=3 error="log not found" +2025-01-28T14:54:37.384Z [INFO] creating new snapshot: path=/tmp/raft2901170872/snapshots/2-14-1738076077384.tmp +2025-01-28T14:54:37.385Z [INFO] server-7ec272ff-ab2e-ea24-4b02-d10bef5a7b09: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:37.386Z [INFO] server-7ec272ff-ab2e-ea24-4b02-d10bef5a7b09: copied to local snapshot: bytes=71 +2025-01-28T14:54:37.387Z [ERROR] server-e4dc8683-71e7-b8c9-1c1c-d49653e8d5a7: failed to get log: index=3 error="log not found" +2025-01-28T14:54:37.387Z [INFO] server-7ec272ff-ab2e-ea24-4b02-d10bef5a7b09: snapshot restore progress: id=2-14-1738076077384 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:37.387Z [INFO] server-7ec272ff-ab2e-ea24-4b02-d10bef5a7b09: Installed remote snapshot +2025-01-28T14:54:37.388Z [INFO] creating new snapshot: path=/tmp/raft296555598/snapshots/2-14-1738076077388.tmp +2025-01-28T14:54:37.388Z [INFO] server-e4dc8683-71e7-b8c9-1c1c-d49653e8d5a7: pipelining replication: peer="{Voter server-7ec272ff-ab2e-ea24-4b02-d10bef5a7b09 7ec272ff-ab2e-ea24-4b02-d10bef5a7b09}" +2025-01-28T14:54:37.388Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:54:37.389Z [INFO] server-6c849148-cc31-4bf7-9366-7dca4454f9d7: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:37.389Z [INFO] server-6c849148-cc31-4bf7-9366-7dca4454f9d7: copied to local snapshot: bytes=71 +2025-01-28T14:54:37.390Z [INFO] server-6c849148-cc31-4bf7-9366-7dca4454f9d7: snapshot restore progress: id=2-14-1738076077388 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:37.390Z [INFO] server-6c849148-cc31-4bf7-9366-7dca4454f9d7: Installed remote snapshot +2025-01-28T14:54:37.391Z [INFO] server-e4dc8683-71e7-b8c9-1c1c-d49653e8d5a7: pipelining replication: peer="{Voter server-6c849148-cc31-4bf7-9366-7dca4454f9d7 6c849148-cc31-4bf7-9366-7dca4454f9d7}" +2025-01-28T14:54:38.393Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:54:37.388434731 +0000 UTC m=+63.062924711 (2 nodes), 2.58µs from start of poll, 1.59776323s from cluster start. Timeout at 2025-01-28 14:54:38.393482946 +0000 UTC m=+64.067972926, 1.005048215s after stability +2025-01-28T14:54:38.411Z [INFO] server-c9f5320d-c1a0-128f-0032-7168be7c2b60: aborting pipeline replication: peer="{Voter server-5a912079-e29a-fd62-7b1c-5bd6bda950d4 5a912079-e29a-fd62-7b1c-5bd6bda950d4}" +2025-01-28T14:54:38.411Z [INFO] server-c9f5320d-c1a0-128f-0032-7168be7c2b60: aborting pipeline replication: peer="{Voter server-6784c6c9-f9ef-f762-7dc7-ff3d4810281f 6784c6c9-f9ef-f762-7dc7-ff3d4810281f}" === RUN TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_true -2025-01-28T14:47:37.251Z [INFO] server-b62ac313-8e79-67ab-1df5-b2c266cd8ce8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b62ac313-8e79-67ab-1df5-b2c266cd8ce8 Address:b62ac313-8e79-67ab-1df5-b2c266cd8ce8} {Suffrage:Voter ID:server-806d8b01-e676-6255-2534-d40684ac061c Address:806d8b01-e676-6255-2534-d40684ac061c} {Suffrage:Voter ID:server-199bf27d-a513-3515-241b-2c3295ed158d Address:199bf27d-a513-3515-241b-2c3295ed158d}]" -2025-01-28T14:47:37.251Z [INFO] server-b62ac313-8e79-67ab-1df5-b2c266cd8ce8: entering follower state: follower="Node at b62ac313-8e79-67ab-1df5-b2c266cd8ce8 [Follower]" leader-address= leader-id= -2025-01-28T14:47:37.252Z [INFO] server-806d8b01-e676-6255-2534-d40684ac061c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b62ac313-8e79-67ab-1df5-b2c266cd8ce8 Address:b62ac313-8e79-67ab-1df5-b2c266cd8ce8} {Suffrage:Voter ID:server-806d8b01-e676-6255-2534-d40684ac061c Address:806d8b01-e676-6255-2534-d40684ac061c} {Suffrage:Voter ID:server-199bf27d-a513-3515-241b-2c3295ed158d Address:199bf27d-a513-3515-241b-2c3295ed158d}]" -2025-01-28T14:47:37.253Z [INFO] server-806d8b01-e676-6255-2534-d40684ac061c: entering follower state: follower="Node at 806d8b01-e676-6255-2534-d40684ac061c [Follower]" leader-address= leader-id= -2025-01-28T14:47:37.254Z [INFO] server-199bf27d-a513-3515-241b-2c3295ed158d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b62ac313-8e79-67ab-1df5-b2c266cd8ce8 Address:b62ac313-8e79-67ab-1df5-b2c266cd8ce8} {Suffrage:Voter ID:server-806d8b01-e676-6255-2534-d40684ac061c Address:806d8b01-e676-6255-2534-d40684ac061c} {Suffrage:Voter ID:server-199bf27d-a513-3515-241b-2c3295ed158d Address:199bf27d-a513-3515-241b-2c3295ed158d}]" -2025-01-28T14:47:37.254Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:37.254Z [INFO] server-199bf27d-a513-3515-241b-2c3295ed158d: entering follower state: follower="Node at 199bf27d-a513-3515-241b-2c3295ed158d [Follower]" leader-address= leader-id= -2025-01-28T14:47:37.757Z [WARN] server-199bf27d-a513-3515-241b-2c3295ed158d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:37.757Z [INFO] server-199bf27d-a513-3515-241b-2c3295ed158d: entering candidate state: node="Node at 199bf27d-a513-3515-241b-2c3295ed158d [Candidate]" term=2 -2025-01-28T14:47:37.758Z [INFO] server-199bf27d-a513-3515-241b-2c3295ed158d: election won: term=2 tally=2 -2025-01-28T14:47:37.758Z [INFO] server-199bf27d-a513-3515-241b-2c3295ed158d: entering leader state: leader="Node at 199bf27d-a513-3515-241b-2c3295ed158d [Leader]" -2025-01-28T14:47:37.759Z [INFO] server-199bf27d-a513-3515-241b-2c3295ed158d: added peer, starting replication: peer=server-b62ac313-8e79-67ab-1df5-b2c266cd8ce8 -2025-01-28T14:47:37.759Z [INFO] server-199bf27d-a513-3515-241b-2c3295ed158d: added peer, starting replication: peer=server-806d8b01-e676-6255-2534-d40684ac061c -2025-01-28T14:47:37.760Z [INFO] server-199bf27d-a513-3515-241b-2c3295ed158d: pipelining replication: peer="{Voter server-b62ac313-8e79-67ab-1df5-b2c266cd8ce8 b62ac313-8e79-67ab-1df5-b2c266cd8ce8}" -2025-01-28T14:47:37.761Z [INFO] server-199bf27d-a513-3515-241b-2c3295ed158d: pipelining replication: peer="{Voter server-806d8b01-e676-6255-2534-d40684ac061c 806d8b01-e676-6255-2534-d40684ac061c}" -2025-01-28T14:47:38.765Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:37.759387902 +0000 UTC m=+40.842794259 (1 nodes), 504.529313ms from start of poll, 509.795803ms from cluster start. Timeout at 2025-01-28 14:47:38.764411569 +0000 UTC m=+41.847817593, 1.005023334s after stability -2025-01-28T14:47:38.772Z [INFO] server-199bf27d-a513-3515-241b-2c3295ed158d: starting snapshot up to: index=12 -2025-01-28T14:47:38.772Z [INFO] creating new snapshot: path=/tmp/raft1921814711/snapshots/2-12-1738075658772.tmp -2025-01-28T14:47:38.776Z [INFO] server-199bf27d-a513-3515-241b-2c3295ed158d: snapshot complete up to: index=12 -2025-01-28T14:47:38.791Z [INFO] server-ebc7083a-6739-76b2-4583-bc6fc34432e8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ebc7083a-6739-76b2-4583-bc6fc34432e8 Address:ebc7083a-6739-76b2-4583-bc6fc34432e8} {Suffrage:Voter ID:server-7497df4b-5a7a-1c46-9b74-89cb4986583b Address:7497df4b-5a7a-1c46-9b74-89cb4986583b} {Suffrage:Voter ID:server-99561da2-acdf-619f-ae13-365931711180 Address:99561da2-acdf-619f-ae13-365931711180}]" -2025-01-28T14:47:38.791Z [INFO] server-ebc7083a-6739-76b2-4583-bc6fc34432e8: entering follower state: follower="Node at ebc7083a-6739-76b2-4583-bc6fc34432e8 [Follower]" leader-address= leader-id= -2025-01-28T14:47:38.792Z [INFO] server-7497df4b-5a7a-1c46-9b74-89cb4986583b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ebc7083a-6739-76b2-4583-bc6fc34432e8 Address:ebc7083a-6739-76b2-4583-bc6fc34432e8} {Suffrage:Voter ID:server-7497df4b-5a7a-1c46-9b74-89cb4986583b Address:7497df4b-5a7a-1c46-9b74-89cb4986583b} {Suffrage:Voter ID:server-99561da2-acdf-619f-ae13-365931711180 Address:99561da2-acdf-619f-ae13-365931711180}]" -2025-01-28T14:47:38.792Z [INFO] server-7497df4b-5a7a-1c46-9b74-89cb4986583b: entering follower state: follower="Node at 7497df4b-5a7a-1c46-9b74-89cb4986583b [Follower]" leader-address= leader-id= -2025-01-28T14:47:38.792Z [INFO] server-99561da2-acdf-619f-ae13-365931711180: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ebc7083a-6739-76b2-4583-bc6fc34432e8 Address:ebc7083a-6739-76b2-4583-bc6fc34432e8} {Suffrage:Voter ID:server-7497df4b-5a7a-1c46-9b74-89cb4986583b Address:7497df4b-5a7a-1c46-9b74-89cb4986583b} {Suffrage:Voter ID:server-99561da2-acdf-619f-ae13-365931711180 Address:99561da2-acdf-619f-ae13-365931711180}]" -2025-01-28T14:47:38.792Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:38.792Z [INFO] server-99561da2-acdf-619f-ae13-365931711180: entering follower state: follower="Node at 99561da2-acdf-619f-ae13-365931711180 [Follower]" leader-address= leader-id= -2025-01-28T14:47:39.310Z [WARN] server-7497df4b-5a7a-1c46-9b74-89cb4986583b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:39.310Z [INFO] server-7497df4b-5a7a-1c46-9b74-89cb4986583b: entering candidate state: node="Node at 7497df4b-5a7a-1c46-9b74-89cb4986583b [Candidate]" term=2 -2025-01-28T14:47:39.311Z [INFO] server-7497df4b-5a7a-1c46-9b74-89cb4986583b: election won: term=2 tally=2 -2025-01-28T14:47:39.311Z [INFO] server-7497df4b-5a7a-1c46-9b74-89cb4986583b: entering leader state: leader="Node at 7497df4b-5a7a-1c46-9b74-89cb4986583b [Leader]" -2025-01-28T14:47:39.312Z [INFO] server-7497df4b-5a7a-1c46-9b74-89cb4986583b: added peer, starting replication: peer=server-ebc7083a-6739-76b2-4583-bc6fc34432e8 -2025-01-28T14:47:39.312Z [INFO] server-7497df4b-5a7a-1c46-9b74-89cb4986583b: added peer, starting replication: peer=server-99561da2-acdf-619f-ae13-365931711180 -2025-01-28T14:47:39.313Z [INFO] server-7497df4b-5a7a-1c46-9b74-89cb4986583b: pipelining replication: peer="{Voter server-ebc7083a-6739-76b2-4583-bc6fc34432e8 ebc7083a-6739-76b2-4583-bc6fc34432e8}" -2025-01-28T14:47:39.314Z [INFO] server-7497df4b-5a7a-1c46-9b74-89cb4986583b: pipelining replication: peer="{Voter server-99561da2-acdf-619f-ae13-365931711180 99561da2-acdf-619f-ae13-365931711180}" -2025-01-28T14:47:40.318Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:39.312480015 +0000 UTC m=+42.395886373 (1 nodes), 519.397755ms from start of poll, 522.939194ms from cluster start. Timeout at 2025-01-28 14:47:40.317503349 +0000 UTC m=+43.400909373, 1.005023s after stability -2025-01-28T14:47:40.320Z [INFO] creating new snapshot: path=/tmp/raft3960274483/snapshots/2-14-1738075660320.tmp -2025-01-28T14:47:40.328Z [INFO] server-7497df4b-5a7a-1c46-9b74-89cb4986583b: copied to local snapshot: bytes=71 -2025-01-28T14:47:40.330Z [INFO] server-7497df4b-5a7a-1c46-9b74-89cb4986583b: snapshot restore progress: id=2-14-1738075660320 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:40.330Z [INFO] server-7497df4b-5a7a-1c46-9b74-89cb4986583b: removing all old logs from log store -2025-01-28T14:47:40.330Z [INFO] server-7497df4b-5a7a-1c46-9b74-89cb4986583b: compacting logs: from=1 to=2 -2025-01-28T14:47:40.331Z [INFO] server-7497df4b-5a7a-1c46-9b74-89cb4986583b: restored user snapshot: index=14 -2025-01-28T14:47:40.331Z [ERROR] server-7497df4b-5a7a-1c46-9b74-89cb4986583b: failed to get log: index=2 error="log not found" -2025-01-28T14:47:40.331Z [INFO] server-7497df4b-5a7a-1c46-9b74-89cb4986583b: aborting pipeline replication: peer="{Voter server-99561da2-acdf-619f-ae13-365931711180 99561da2-acdf-619f-ae13-365931711180}" -2025-01-28T14:47:40.331Z [ERROR] server-7497df4b-5a7a-1c46-9b74-89cb4986583b: failed to get log: index=2 error="log not found" -2025-01-28T14:47:40.331Z [INFO] server-7497df4b-5a7a-1c46-9b74-89cb4986583b: aborting pipeline replication: peer="{Voter server-ebc7083a-6739-76b2-4583-bc6fc34432e8 ebc7083a-6739-76b2-4583-bc6fc34432e8}" -2025-01-28T14:47:40.339Z [ERROR] server-7497df4b-5a7a-1c46-9b74-89cb4986583b: failed to get log: index=2 error="log not found" -2025-01-28T14:47:40.339Z [ERROR] server-7497df4b-5a7a-1c46-9b74-89cb4986583b: failed to get log: index=2 error="log not found" -2025-01-28T14:47:40.348Z [INFO] creating new snapshot: path=/tmp/raft3906985356/snapshots/2-14-1738075660348.tmp -2025-01-28T14:47:40.355Z [INFO] creating new snapshot: path=/tmp/raft3568417245/snapshots/2-14-1738075660354.tmp -2025-01-28T14:47:40.356Z [INFO] server-99561da2-acdf-619f-ae13-365931711180: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:40.359Z [INFO] server-99561da2-acdf-619f-ae13-365931711180: copied to local snapshot: bytes=71 -2025-01-28T14:47:40.360Z [INFO] server-ebc7083a-6739-76b2-4583-bc6fc34432e8: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:40.361Z [INFO] server-99561da2-acdf-619f-ae13-365931711180: snapshot restore progress: id=2-14-1738075660348 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:40.361Z [INFO] server-99561da2-acdf-619f-ae13-365931711180: removing all old logs from log store -2025-01-28T14:47:40.361Z [INFO] server-99561da2-acdf-619f-ae13-365931711180: compacting logs: from=1 to=2 -2025-01-28T14:47:40.361Z [INFO] server-99561da2-acdf-619f-ae13-365931711180: Installed remote snapshot -2025-01-28T14:47:40.363Z [INFO] server-ebc7083a-6739-76b2-4583-bc6fc34432e8: copied to local snapshot: bytes=71 -2025-01-28T14:47:40.363Z [INFO] server-7497df4b-5a7a-1c46-9b74-89cb4986583b: pipelining replication: peer="{Voter server-99561da2-acdf-619f-ae13-365931711180 99561da2-acdf-619f-ae13-365931711180}" -2025-01-28T14:47:40.363Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:47:40.364Z [INFO] server-ebc7083a-6739-76b2-4583-bc6fc34432e8: snapshot restore progress: id=2-14-1738075660354 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:40.364Z [INFO] server-ebc7083a-6739-76b2-4583-bc6fc34432e8: removing all old logs from log store -2025-01-28T14:47:40.364Z [INFO] server-ebc7083a-6739-76b2-4583-bc6fc34432e8: compacting logs: from=1 to=2 -2025-01-28T14:47:40.364Z [INFO] server-ebc7083a-6739-76b2-4583-bc6fc34432e8: Installed remote snapshot -2025-01-28T14:47:40.365Z [INFO] server-7497df4b-5a7a-1c46-9b74-89cb4986583b: pipelining replication: peer="{Voter server-ebc7083a-6739-76b2-4583-bc6fc34432e8 ebc7083a-6739-76b2-4583-bc6fc34432e8}" -2025-01-28T14:47:41.370Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:47:40.363964569 +0000 UTC m=+43.447369593 (2 nodes), 9.667µs from start of poll, 1.574422414s from cluster start. Timeout at 2025-01-28 14:47:41.369271577 +0000 UTC m=+44.452678268, 1.005308675s after stability -2025-01-28T14:47:41.391Z [INFO] server-199bf27d-a513-3515-241b-2c3295ed158d: aborting pipeline replication: peer="{Voter server-806d8b01-e676-6255-2534-d40684ac061c 806d8b01-e676-6255-2534-d40684ac061c}" -2025-01-28T14:47:41.391Z [INFO] server-199bf27d-a513-3515-241b-2c3295ed158d: aborting pipeline replication: peer="{Voter server-b62ac313-8e79-67ab-1df5-b2c266cd8ce8 b62ac313-8e79-67ab-1df5-b2c266cd8ce8}" +2025-01-28T14:54:38.415Z [INFO] server-d85694f6-9270-bc6d-dfa5-cdf46aa50b17: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d85694f6-9270-bc6d-dfa5-cdf46aa50b17 Address:d85694f6-9270-bc6d-dfa5-cdf46aa50b17} {Suffrage:Voter ID:server-2e7e0eb5-9008-2a88-2729-cf2a3139fb04 Address:2e7e0eb5-9008-2a88-2729-cf2a3139fb04} {Suffrage:Voter ID:server-e07cbf22-1478-ee4d-b495-9ede2565a1a4 Address:e07cbf22-1478-ee4d-b495-9ede2565a1a4}]" +2025-01-28T14:54:38.415Z [INFO] server-d85694f6-9270-bc6d-dfa5-cdf46aa50b17: entering follower state: follower="Node at d85694f6-9270-bc6d-dfa5-cdf46aa50b17 [Follower]" leader-address= leader-id= +2025-01-28T14:54:38.415Z [INFO] server-2e7e0eb5-9008-2a88-2729-cf2a3139fb04: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d85694f6-9270-bc6d-dfa5-cdf46aa50b17 Address:d85694f6-9270-bc6d-dfa5-cdf46aa50b17} {Suffrage:Voter ID:server-2e7e0eb5-9008-2a88-2729-cf2a3139fb04 Address:2e7e0eb5-9008-2a88-2729-cf2a3139fb04} {Suffrage:Voter ID:server-e07cbf22-1478-ee4d-b495-9ede2565a1a4 Address:e07cbf22-1478-ee4d-b495-9ede2565a1a4}]" +2025-01-28T14:54:38.415Z [INFO] server-2e7e0eb5-9008-2a88-2729-cf2a3139fb04: entering follower state: follower="Node at 2e7e0eb5-9008-2a88-2729-cf2a3139fb04 [Follower]" leader-address= leader-id= +2025-01-28T14:54:38.415Z [INFO] server-e07cbf22-1478-ee4d-b495-9ede2565a1a4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d85694f6-9270-bc6d-dfa5-cdf46aa50b17 Address:d85694f6-9270-bc6d-dfa5-cdf46aa50b17} {Suffrage:Voter ID:server-2e7e0eb5-9008-2a88-2729-cf2a3139fb04 Address:2e7e0eb5-9008-2a88-2729-cf2a3139fb04} {Suffrage:Voter ID:server-e07cbf22-1478-ee4d-b495-9ede2565a1a4 Address:e07cbf22-1478-ee4d-b495-9ede2565a1a4}]" +2025-01-28T14:54:38.415Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:38.415Z [INFO] server-e07cbf22-1478-ee4d-b495-9ede2565a1a4: entering follower state: follower="Node at e07cbf22-1478-ee4d-b495-9ede2565a1a4 [Follower]" leader-address= leader-id= +2025-01-28T14:54:38.932Z [WARN] server-d85694f6-9270-bc6d-dfa5-cdf46aa50b17: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:38.932Z [INFO] server-d85694f6-9270-bc6d-dfa5-cdf46aa50b17: entering candidate state: node="Node at d85694f6-9270-bc6d-dfa5-cdf46aa50b17 [Candidate]" term=2 +2025-01-28T14:54:38.932Z [INFO] server-d85694f6-9270-bc6d-dfa5-cdf46aa50b17: election won: term=2 tally=2 +2025-01-28T14:54:38.932Z [INFO] server-d85694f6-9270-bc6d-dfa5-cdf46aa50b17: entering leader state: leader="Node at d85694f6-9270-bc6d-dfa5-cdf46aa50b17 [Leader]" +2025-01-28T14:54:38.932Z [INFO] server-d85694f6-9270-bc6d-dfa5-cdf46aa50b17: added peer, starting replication: peer=server-2e7e0eb5-9008-2a88-2729-cf2a3139fb04 +2025-01-28T14:54:38.932Z [INFO] server-d85694f6-9270-bc6d-dfa5-cdf46aa50b17: added peer, starting replication: peer=server-e07cbf22-1478-ee4d-b495-9ede2565a1a4 +2025-01-28T14:54:38.932Z [INFO] server-d85694f6-9270-bc6d-dfa5-cdf46aa50b17: pipelining replication: peer="{Voter server-e07cbf22-1478-ee4d-b495-9ede2565a1a4 e07cbf22-1478-ee4d-b495-9ede2565a1a4}" +2025-01-28T14:54:38.932Z [INFO] server-d85694f6-9270-bc6d-dfa5-cdf46aa50b17: pipelining replication: peer="{Voter server-2e7e0eb5-9008-2a88-2729-cf2a3139fb04 2e7e0eb5-9008-2a88-2729-cf2a3139fb04}" +2025-01-28T14:54:39.938Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:38.932510886 +0000 UTC m=+64.607000866 (1 nodes), 516.517313ms from start of poll, 517.61211ms from cluster start. Timeout at 2025-01-28 14:54:39.937522444 +0000 UTC m=+65.612012424, 1.005011558s after stability +2025-01-28T14:54:39.939Z [INFO] server-d85694f6-9270-bc6d-dfa5-cdf46aa50b17: starting snapshot up to: index=12 +2025-01-28T14:54:39.939Z [INFO] creating new snapshot: path=/tmp/raft1876669511/snapshots/2-12-1738076079939.tmp +2025-01-28T14:54:39.941Z [INFO] server-d85694f6-9270-bc6d-dfa5-cdf46aa50b17: snapshot complete up to: index=12 +2025-01-28T14:54:39.944Z [INFO] server-6a958917-308c-5240-b5d2-0ea19e5f7840: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6a958917-308c-5240-b5d2-0ea19e5f7840 Address:6a958917-308c-5240-b5d2-0ea19e5f7840} {Suffrage:Voter ID:server-4a507f22-e57a-19c4-cdee-16ed98661d9c Address:4a507f22-e57a-19c4-cdee-16ed98661d9c} {Suffrage:Voter ID:server-720e4830-346e-712f-d45b-967af9c1b10d Address:720e4830-346e-712f-d45b-967af9c1b10d}]" +2025-01-28T14:54:39.944Z [INFO] server-6a958917-308c-5240-b5d2-0ea19e5f7840: entering follower state: follower="Node at 6a958917-308c-5240-b5d2-0ea19e5f7840 [Follower]" leader-address= leader-id= +2025-01-28T14:54:39.944Z [INFO] server-4a507f22-e57a-19c4-cdee-16ed98661d9c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6a958917-308c-5240-b5d2-0ea19e5f7840 Address:6a958917-308c-5240-b5d2-0ea19e5f7840} {Suffrage:Voter ID:server-4a507f22-e57a-19c4-cdee-16ed98661d9c Address:4a507f22-e57a-19c4-cdee-16ed98661d9c} {Suffrage:Voter ID:server-720e4830-346e-712f-d45b-967af9c1b10d Address:720e4830-346e-712f-d45b-967af9c1b10d}]" +2025-01-28T14:54:39.944Z [INFO] server-4a507f22-e57a-19c4-cdee-16ed98661d9c: entering follower state: follower="Node at 4a507f22-e57a-19c4-cdee-16ed98661d9c [Follower]" leader-address= leader-id= +2025-01-28T14:54:39.944Z [INFO] server-720e4830-346e-712f-d45b-967af9c1b10d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6a958917-308c-5240-b5d2-0ea19e5f7840 Address:6a958917-308c-5240-b5d2-0ea19e5f7840} {Suffrage:Voter ID:server-4a507f22-e57a-19c4-cdee-16ed98661d9c Address:4a507f22-e57a-19c4-cdee-16ed98661d9c} {Suffrage:Voter ID:server-720e4830-346e-712f-d45b-967af9c1b10d Address:720e4830-346e-712f-d45b-967af9c1b10d}]" +2025-01-28T14:54:39.944Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:39.944Z [INFO] server-720e4830-346e-712f-d45b-967af9c1b10d: entering follower state: follower="Node at 720e4830-346e-712f-d45b-967af9c1b10d [Follower]" leader-address= leader-id= +2025-01-28T14:54:40.622Z [WARN] server-6a958917-308c-5240-b5d2-0ea19e5f7840: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:40.622Z [INFO] server-6a958917-308c-5240-b5d2-0ea19e5f7840: entering candidate state: node="Node at 6a958917-308c-5240-b5d2-0ea19e5f7840 [Candidate]" term=2 +2025-01-28T14:54:40.622Z [INFO] server-6a958917-308c-5240-b5d2-0ea19e5f7840: election won: term=2 tally=2 +2025-01-28T14:54:40.622Z [INFO] server-6a958917-308c-5240-b5d2-0ea19e5f7840: entering leader state: leader="Node at 6a958917-308c-5240-b5d2-0ea19e5f7840 [Leader]" +2025-01-28T14:54:40.622Z [INFO] server-6a958917-308c-5240-b5d2-0ea19e5f7840: added peer, starting replication: peer=server-4a507f22-e57a-19c4-cdee-16ed98661d9c +2025-01-28T14:54:40.623Z [INFO] server-6a958917-308c-5240-b5d2-0ea19e5f7840: added peer, starting replication: peer=server-720e4830-346e-712f-d45b-967af9c1b10d +2025-01-28T14:54:40.623Z [INFO] server-6a958917-308c-5240-b5d2-0ea19e5f7840: pipelining replication: peer="{Voter server-720e4830-346e-712f-d45b-967af9c1b10d 720e4830-346e-712f-d45b-967af9c1b10d}" +2025-01-28T14:54:40.623Z [INFO] server-6a958917-308c-5240-b5d2-0ea19e5f7840: pipelining replication: peer="{Voter server-4a507f22-e57a-19c4-cdee-16ed98661d9c 4a507f22-e57a-19c4-cdee-16ed98661d9c}" +2025-01-28T14:54:41.628Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:40.623015186 +0000 UTC m=+66.297505146 (1 nodes), 678.067072ms from start of poll, 679.032241ms from cluster start. Timeout at 2025-01-28 14:54:41.628021425 +0000 UTC m=+67.302511485, 1.005006339s after stability +2025-01-28T14:54:41.628Z [INFO] creating new snapshot: path=/tmp/raft4177038571/snapshots/2-14-1738076081628.tmp +2025-01-28T14:54:41.630Z [INFO] server-6a958917-308c-5240-b5d2-0ea19e5f7840: copied to local snapshot: bytes=71 +2025-01-28T14:54:41.630Z [INFO] server-6a958917-308c-5240-b5d2-0ea19e5f7840: snapshot restore progress: id=2-14-1738076081628 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:41.631Z [INFO] server-6a958917-308c-5240-b5d2-0ea19e5f7840: removing all old logs from log store +2025-01-28T14:54:41.631Z [INFO] server-6a958917-308c-5240-b5d2-0ea19e5f7840: compacting logs: from=1 to=2 +2025-01-28T14:54:41.631Z [INFO] server-6a958917-308c-5240-b5d2-0ea19e5f7840: restored user snapshot: index=14 +2025-01-28T14:54:41.631Z [ERROR] server-6a958917-308c-5240-b5d2-0ea19e5f7840: failed to get log: index=2 error="log not found" +2025-01-28T14:54:41.631Z [ERROR] server-6a958917-308c-5240-b5d2-0ea19e5f7840: failed to get log: index=2 error="log not found" +2025-01-28T14:54:41.631Z [INFO] server-6a958917-308c-5240-b5d2-0ea19e5f7840: aborting pipeline replication: peer="{Voter server-4a507f22-e57a-19c4-cdee-16ed98661d9c 4a507f22-e57a-19c4-cdee-16ed98661d9c}" +2025-01-28T14:54:41.631Z [INFO] server-6a958917-308c-5240-b5d2-0ea19e5f7840: aborting pipeline replication: peer="{Voter server-720e4830-346e-712f-d45b-967af9c1b10d 720e4830-346e-712f-d45b-967af9c1b10d}" +2025-01-28T14:54:41.641Z [ERROR] server-6a958917-308c-5240-b5d2-0ea19e5f7840: failed to get log: index=2 error="log not found" +2025-01-28T14:54:41.641Z [ERROR] server-6a958917-308c-5240-b5d2-0ea19e5f7840: failed to get log: index=2 error="log not found" +2025-01-28T14:54:41.642Z [INFO] creating new snapshot: path=/tmp/raft726200438/snapshots/2-14-1738076081642.tmp +2025-01-28T14:54:41.642Z [INFO] creating new snapshot: path=/tmp/raft1142058795/snapshots/2-14-1738076081642.tmp +2025-01-28T14:54:41.643Z [INFO] server-720e4830-346e-712f-d45b-967af9c1b10d: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:41.643Z [INFO] server-4a507f22-e57a-19c4-cdee-16ed98661d9c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:41.643Z [INFO] server-4a507f22-e57a-19c4-cdee-16ed98661d9c: copied to local snapshot: bytes=71 +2025-01-28T14:54:41.643Z [INFO] server-720e4830-346e-712f-d45b-967af9c1b10d: copied to local snapshot: bytes=71 +2025-01-28T14:54:41.644Z [INFO] server-4a507f22-e57a-19c4-cdee-16ed98661d9c: snapshot restore progress: id=2-14-1738076081642 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:41.644Z [INFO] server-4a507f22-e57a-19c4-cdee-16ed98661d9c: removing all old logs from log store +2025-01-28T14:54:41.644Z [INFO] server-4a507f22-e57a-19c4-cdee-16ed98661d9c: compacting logs: from=1 to=2 +2025-01-28T14:54:41.644Z [INFO] server-4a507f22-e57a-19c4-cdee-16ed98661d9c: Installed remote snapshot +2025-01-28T14:54:41.644Z [INFO] server-720e4830-346e-712f-d45b-967af9c1b10d: snapshot restore progress: id=2-14-1738076081642 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:41.644Z [INFO] server-720e4830-346e-712f-d45b-967af9c1b10d: removing all old logs from log store +2025-01-28T14:54:41.644Z [INFO] server-720e4830-346e-712f-d45b-967af9c1b10d: compacting logs: from=1 to=2 +2025-01-28T14:54:41.644Z [INFO] server-720e4830-346e-712f-d45b-967af9c1b10d: Installed remote snapshot +2025-01-28T14:54:41.644Z [INFO] server-6a958917-308c-5240-b5d2-0ea19e5f7840: pipelining replication: peer="{Voter server-4a507f22-e57a-19c4-cdee-16ed98661d9c 4a507f22-e57a-19c4-cdee-16ed98661d9c}" +2025-01-28T14:54:41.644Z [INFO] server-6a958917-308c-5240-b5d2-0ea19e5f7840: pipelining replication: peer="{Voter server-720e4830-346e-712f-d45b-967af9c1b10d 720e4830-346e-712f-d45b-967af9c1b10d}" +2025-01-28T14:54:41.644Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:54:42.650Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:54:41.64483655 +0000 UTC m=+67.319326530 (2 nodes), 2.32µs from start of poll, 1.700853625s from cluster start. Timeout at 2025-01-28 14:54:42.649896385 +0000 UTC m=+68.324386405, 1.005059875s after stability +2025-01-28T14:54:42.662Z [INFO] server-d85694f6-9270-bc6d-dfa5-cdf46aa50b17: aborting pipeline replication: peer="{Voter server-e07cbf22-1478-ee4d-b495-9ede2565a1a4 e07cbf22-1478-ee4d-b495-9ede2565a1a4}" +2025-01-28T14:54:42.662Z [INFO] server-d85694f6-9270-bc6d-dfa5-cdf46aa50b17: aborting pipeline replication: peer="{Voter server-2e7e0eb5-9008-2a88-2729-cf2a3139fb04 2e7e0eb5-9008-2a88-2729-cf2a3139fb04}" === RUN TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_false -2025-01-28T14:47:41.407Z [INFO] server-541f64d1-861e-e225-e2bb-bc12467aad06: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-541f64d1-861e-e225-e2bb-bc12467aad06 Address:541f64d1-861e-e225-e2bb-bc12467aad06} {Suffrage:Voter ID:server-c1d90d27-5eb2-d124-263e-38a4c1230359 Address:c1d90d27-5eb2-d124-263e-38a4c1230359} {Suffrage:Voter ID:server-c7fc58ce-93ea-adc0-5c6a-39e639d65742 Address:c7fc58ce-93ea-adc0-5c6a-39e639d65742}]" -2025-01-28T14:47:41.407Z [INFO] server-541f64d1-861e-e225-e2bb-bc12467aad06: entering follower state: follower="Node at 541f64d1-861e-e225-e2bb-bc12467aad06 [Follower]" leader-address= leader-id= -2025-01-28T14:47:41.408Z [INFO] server-c1d90d27-5eb2-d124-263e-38a4c1230359: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-541f64d1-861e-e225-e2bb-bc12467aad06 Address:541f64d1-861e-e225-e2bb-bc12467aad06} {Suffrage:Voter ID:server-c1d90d27-5eb2-d124-263e-38a4c1230359 Address:c1d90d27-5eb2-d124-263e-38a4c1230359} {Suffrage:Voter ID:server-c7fc58ce-93ea-adc0-5c6a-39e639d65742 Address:c7fc58ce-93ea-adc0-5c6a-39e639d65742}]" -2025-01-28T14:47:41.409Z [INFO] server-c1d90d27-5eb2-d124-263e-38a4c1230359: entering follower state: follower="Node at c1d90d27-5eb2-d124-263e-38a4c1230359 [Follower]" leader-address= leader-id= -2025-01-28T14:47:41.409Z [INFO] server-c7fc58ce-93ea-adc0-5c6a-39e639d65742: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-541f64d1-861e-e225-e2bb-bc12467aad06 Address:541f64d1-861e-e225-e2bb-bc12467aad06} {Suffrage:Voter ID:server-c1d90d27-5eb2-d124-263e-38a4c1230359 Address:c1d90d27-5eb2-d124-263e-38a4c1230359} {Suffrage:Voter ID:server-c7fc58ce-93ea-adc0-5c6a-39e639d65742 Address:c7fc58ce-93ea-adc0-5c6a-39e639d65742}]" -2025-01-28T14:47:41.409Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:41.409Z [INFO] server-c7fc58ce-93ea-adc0-5c6a-39e639d65742: entering follower state: follower="Node at c7fc58ce-93ea-adc0-5c6a-39e639d65742 [Follower]" leader-address= leader-id= -2025-01-28T14:47:42.164Z [WARN] server-541f64d1-861e-e225-e2bb-bc12467aad06: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:42.164Z [INFO] server-541f64d1-861e-e225-e2bb-bc12467aad06: entering candidate state: node="Node at 541f64d1-861e-e225-e2bb-bc12467aad06 [Candidate]" term=2 -2025-01-28T14:47:42.166Z [INFO] server-541f64d1-861e-e225-e2bb-bc12467aad06: election won: term=2 tally=2 -2025-01-28T14:47:42.166Z [INFO] server-541f64d1-861e-e225-e2bb-bc12467aad06: entering leader state: leader="Node at 541f64d1-861e-e225-e2bb-bc12467aad06 [Leader]" -2025-01-28T14:47:42.166Z [INFO] server-541f64d1-861e-e225-e2bb-bc12467aad06: added peer, starting replication: peer=server-c1d90d27-5eb2-d124-263e-38a4c1230359 -2025-01-28T14:47:42.167Z [INFO] server-541f64d1-861e-e225-e2bb-bc12467aad06: added peer, starting replication: peer=server-c7fc58ce-93ea-adc0-5c6a-39e639d65742 -2025-01-28T14:47:42.169Z [INFO] server-541f64d1-861e-e225-e2bb-bc12467aad06: pipelining replication: peer="{Voter server-c1d90d27-5eb2-d124-263e-38a4c1230359 c1d90d27-5eb2-d124-263e-38a4c1230359}" -2025-01-28T14:47:42.170Z [INFO] server-541f64d1-861e-e225-e2bb-bc12467aad06: pipelining replication: peer="{Voter server-c7fc58ce-93ea-adc0-5c6a-39e639d65742 c7fc58ce-93ea-adc0-5c6a-39e639d65742}" -2025-01-28T14:47:43.172Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:42.166750763 +0000 UTC m=+45.250157121 (1 nodes), 757.118147ms from start of poll, 759.776559ms from cluster start. Timeout at 2025-01-28 14:47:43.171764764 +0000 UTC m=+46.255169455, 1.005012334s after stability -2025-01-28T14:47:43.191Z [INFO] server-541f64d1-861e-e225-e2bb-bc12467aad06: starting snapshot up to: index=12 -2025-01-28T14:47:43.191Z [INFO] creating new snapshot: path=/tmp/raft1314639082/snapshots/2-12-1738075663191.tmp -2025-01-28T14:47:43.195Z [INFO] server-541f64d1-861e-e225-e2bb-bc12467aad06: snapshot complete up to: index=12 -2025-01-28T14:47:43.201Z [INFO] creating new snapshot: path=/tmp/raft1314639082/snapshots/2-23-1738075663201.tmp -2025-01-28T14:47:43.205Z [INFO] server-541f64d1-861e-e225-e2bb-bc12467aad06: copied to local snapshot: bytes=71 -2025-01-28T14:47:43.207Z [INFO] server-541f64d1-861e-e225-e2bb-bc12467aad06: snapshot restore progress: id=2-23-1738075663201 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:43.207Z [INFO] server-541f64d1-861e-e225-e2bb-bc12467aad06: restored user snapshot: index=23 -2025-01-28T14:47:43.207Z [ERROR] server-541f64d1-861e-e225-e2bb-bc12467aad06: failed to get log: index=23 error="log not found" -2025-01-28T14:47:43.208Z [INFO] server-541f64d1-861e-e225-e2bb-bc12467aad06: aborting pipeline replication: peer="{Voter server-c1d90d27-5eb2-d124-263e-38a4c1230359 c1d90d27-5eb2-d124-263e-38a4c1230359}" -2025-01-28T14:47:43.207Z [ERROR] server-541f64d1-861e-e225-e2bb-bc12467aad06: failed to get log: index=23 error="log not found" -2025-01-28T14:47:43.208Z [INFO] server-541f64d1-861e-e225-e2bb-bc12467aad06: aborting pipeline replication: peer="{Voter server-c7fc58ce-93ea-adc0-5c6a-39e639d65742 c7fc58ce-93ea-adc0-5c6a-39e639d65742}" -2025-01-28T14:47:43.214Z [ERROR] server-541f64d1-861e-e225-e2bb-bc12467aad06: failed to get log: index=23 error="log not found" -2025-01-28T14:47:43.215Z [ERROR] server-541f64d1-861e-e225-e2bb-bc12467aad06: failed to get log: index=23 error="log not found" -2025-01-28T14:47:43.217Z [INFO] creating new snapshot: path=/tmp/raft1599551405/snapshots/2-23-1738075663217.tmp -2025-01-28T14:47:43.218Z [INFO] creating new snapshot: path=/tmp/raft2577505623/snapshots/2-23-1738075663218.tmp -2025-01-28T14:47:43.219Z [INFO] server-c1d90d27-5eb2-d124-263e-38a4c1230359: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:43.220Z [INFO] server-c7fc58ce-93ea-adc0-5c6a-39e639d65742: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:43.221Z [INFO] server-c1d90d27-5eb2-d124-263e-38a4c1230359: copied to local snapshot: bytes=71 -2025-01-28T14:47:43.223Z [INFO] server-c1d90d27-5eb2-d124-263e-38a4c1230359: snapshot restore progress: id=2-23-1738075663217 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:43.223Z [INFO] server-c1d90d27-5eb2-d124-263e-38a4c1230359: Installed remote snapshot -2025-01-28T14:47:43.224Z [INFO] server-c7fc58ce-93ea-adc0-5c6a-39e639d65742: copied to local snapshot: bytes=71 -2025-01-28T14:47:43.224Z [INFO] server-541f64d1-861e-e225-e2bb-bc12467aad06: pipelining replication: peer="{Voter server-c1d90d27-5eb2-d124-263e-38a4c1230359 c1d90d27-5eb2-d124-263e-38a4c1230359}" -2025-01-28T14:47:43.225Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:47:43.225Z [INFO] server-c7fc58ce-93ea-adc0-5c6a-39e639d65742: snapshot restore progress: id=2-23-1738075663218 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:43.226Z [INFO] server-c7fc58ce-93ea-adc0-5c6a-39e639d65742: Installed remote snapshot -2025-01-28T14:47:43.227Z [INFO] server-541f64d1-861e-e225-e2bb-bc12467aad06: pipelining replication: peer="{Voter server-c7fc58ce-93ea-adc0-5c6a-39e639d65742 c7fc58ce-93ea-adc0-5c6a-39e639d65742}" -2025-01-28T14:47:44.231Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:47:43.225627203 +0000 UTC m=+46.309031894 (2 nodes), 9.667µs from start of poll, 1.818651332s from cluster start. Timeout at 2025-01-28 14:47:44.23097188 +0000 UTC m=+47.314377571, 1.005345677s after stability -2025-01-28T14:47:44.250Z [INFO] server-541f64d1-861e-e225-e2bb-bc12467aad06: aborting pipeline replication: peer="{Voter server-c7fc58ce-93ea-adc0-5c6a-39e639d65742 c7fc58ce-93ea-adc0-5c6a-39e639d65742}" -2025-01-28T14:47:44.251Z [INFO] server-541f64d1-861e-e225-e2bb-bc12467aad06: aborting pipeline replication: peer="{Voter server-c1d90d27-5eb2-d124-263e-38a4c1230359 c1d90d27-5eb2-d124-263e-38a4c1230359}" +2025-01-28T14:54:42.668Z [INFO] server-8b04e788-ec63-8cae-cc75-0aac69070353: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8b04e788-ec63-8cae-cc75-0aac69070353 Address:8b04e788-ec63-8cae-cc75-0aac69070353} {Suffrage:Voter ID:server-2050504c-cc88-a166-4194-a85f5a64e661 Address:2050504c-cc88-a166-4194-a85f5a64e661} {Suffrage:Voter ID:server-cd12e50e-bd0a-d539-b229-a440b0aa9b44 Address:cd12e50e-bd0a-d539-b229-a440b0aa9b44}]" +2025-01-28T14:54:42.668Z [INFO] server-2050504c-cc88-a166-4194-a85f5a64e661: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8b04e788-ec63-8cae-cc75-0aac69070353 Address:8b04e788-ec63-8cae-cc75-0aac69070353} {Suffrage:Voter ID:server-2050504c-cc88-a166-4194-a85f5a64e661 Address:2050504c-cc88-a166-4194-a85f5a64e661} {Suffrage:Voter ID:server-cd12e50e-bd0a-d539-b229-a440b0aa9b44 Address:cd12e50e-bd0a-d539-b229-a440b0aa9b44}]" +2025-01-28T14:54:42.669Z [INFO] server-cd12e50e-bd0a-d539-b229-a440b0aa9b44: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8b04e788-ec63-8cae-cc75-0aac69070353 Address:8b04e788-ec63-8cae-cc75-0aac69070353} {Suffrage:Voter ID:server-2050504c-cc88-a166-4194-a85f5a64e661 Address:2050504c-cc88-a166-4194-a85f5a64e661} {Suffrage:Voter ID:server-cd12e50e-bd0a-d539-b229-a440b0aa9b44 Address:cd12e50e-bd0a-d539-b229-a440b0aa9b44}]" +2025-01-28T14:54:42.669Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:42.669Z [INFO] server-8b04e788-ec63-8cae-cc75-0aac69070353: entering follower state: follower="Node at 8b04e788-ec63-8cae-cc75-0aac69070353 [Follower]" leader-address= leader-id= +2025-01-28T14:54:42.669Z [INFO] server-2050504c-cc88-a166-4194-a85f5a64e661: entering follower state: follower="Node at 2050504c-cc88-a166-4194-a85f5a64e661 [Follower]" leader-address= leader-id= +2025-01-28T14:54:42.669Z [INFO] server-cd12e50e-bd0a-d539-b229-a440b0aa9b44: entering follower state: follower="Node at cd12e50e-bd0a-d539-b229-a440b0aa9b44 [Follower]" leader-address= leader-id= +2025-01-28T14:54:43.350Z [WARN] server-2050504c-cc88-a166-4194-a85f5a64e661: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:43.350Z [INFO] server-2050504c-cc88-a166-4194-a85f5a64e661: entering candidate state: node="Node at 2050504c-cc88-a166-4194-a85f5a64e661 [Candidate]" term=2 +2025-01-28T14:54:43.350Z [INFO] server-2050504c-cc88-a166-4194-a85f5a64e661: election won: term=2 tally=2 +2025-01-28T14:54:43.350Z [INFO] server-2050504c-cc88-a166-4194-a85f5a64e661: entering leader state: leader="Node at 2050504c-cc88-a166-4194-a85f5a64e661 [Leader]" +2025-01-28T14:54:43.350Z [INFO] server-2050504c-cc88-a166-4194-a85f5a64e661: added peer, starting replication: peer=server-8b04e788-ec63-8cae-cc75-0aac69070353 +2025-01-28T14:54:43.350Z [INFO] server-2050504c-cc88-a166-4194-a85f5a64e661: added peer, starting replication: peer=server-cd12e50e-bd0a-d539-b229-a440b0aa9b44 +2025-01-28T14:54:43.350Z [INFO] server-2050504c-cc88-a166-4194-a85f5a64e661: pipelining replication: peer="{Voter server-cd12e50e-bd0a-d539-b229-a440b0aa9b44 cd12e50e-bd0a-d539-b229-a440b0aa9b44}" +2025-01-28T14:54:43.351Z [INFO] server-2050504c-cc88-a166-4194-a85f5a64e661: pipelining replication: peer="{Voter server-8b04e788-ec63-8cae-cc75-0aac69070353 8b04e788-ec63-8cae-cc75-0aac69070353}" +2025-01-28T14:54:44.355Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:43.350661725 +0000 UTC m=+69.025151705 (1 nodes), 681.447373ms from start of poll, 682.641921ms from cluster start. Timeout at 2025-01-28 14:54:44.355667305 +0000 UTC m=+70.030157345, 1.00500564s after stability +2025-01-28T14:54:44.356Z [INFO] server-2050504c-cc88-a166-4194-a85f5a64e661: starting snapshot up to: index=12 +2025-01-28T14:54:44.356Z [INFO] creating new snapshot: path=/tmp/raft188398636/snapshots/2-12-1738076084356.tmp +2025-01-28T14:54:44.358Z [INFO] server-2050504c-cc88-a166-4194-a85f5a64e661: snapshot complete up to: index=12 +2025-01-28T14:54:44.359Z [INFO] creating new snapshot: path=/tmp/raft188398636/snapshots/2-23-1738076084359.tmp +2025-01-28T14:54:44.361Z [INFO] server-2050504c-cc88-a166-4194-a85f5a64e661: copied to local snapshot: bytes=71 +2025-01-28T14:54:44.361Z [INFO] server-2050504c-cc88-a166-4194-a85f5a64e661: snapshot restore progress: id=2-23-1738076084359 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:44.361Z [INFO] server-2050504c-cc88-a166-4194-a85f5a64e661: restored user snapshot: index=23 +2025-01-28T14:54:44.361Z [ERROR] server-2050504c-cc88-a166-4194-a85f5a64e661: failed to get log: index=23 error="log not found" +2025-01-28T14:54:44.362Z [INFO] server-2050504c-cc88-a166-4194-a85f5a64e661: aborting pipeline replication: peer="{Voter server-8b04e788-ec63-8cae-cc75-0aac69070353 8b04e788-ec63-8cae-cc75-0aac69070353}" +2025-01-28T14:54:44.361Z [ERROR] server-2050504c-cc88-a166-4194-a85f5a64e661: failed to get log: index=23 error="log not found" +2025-01-28T14:54:44.362Z [INFO] server-2050504c-cc88-a166-4194-a85f5a64e661: aborting pipeline replication: peer="{Voter server-cd12e50e-bd0a-d539-b229-a440b0aa9b44 cd12e50e-bd0a-d539-b229-a440b0aa9b44}" +2025-01-28T14:54:44.371Z [ERROR] server-2050504c-cc88-a166-4194-a85f5a64e661: failed to get log: index=23 error="log not found" +2025-01-28T14:54:44.372Z [INFO] creating new snapshot: path=/tmp/raft3424123677/snapshots/2-23-1738076084372.tmp +2025-01-28T14:54:44.372Z [ERROR] server-2050504c-cc88-a166-4194-a85f5a64e661: failed to get log: index=23 error="log not found" +2025-01-28T14:54:44.373Z [INFO] server-8b04e788-ec63-8cae-cc75-0aac69070353: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:44.373Z [INFO] creating new snapshot: path=/tmp/raft3075059098/snapshots/2-23-1738076084373.tmp +2025-01-28T14:54:44.373Z [INFO] server-8b04e788-ec63-8cae-cc75-0aac69070353: copied to local snapshot: bytes=71 +2025-01-28T14:54:44.373Z [INFO] server-cd12e50e-bd0a-d539-b229-a440b0aa9b44: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:44.374Z [INFO] server-8b04e788-ec63-8cae-cc75-0aac69070353: snapshot restore progress: id=2-23-1738076084372 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:44.374Z [INFO] server-8b04e788-ec63-8cae-cc75-0aac69070353: Installed remote snapshot +2025-01-28T14:54:44.374Z [WARN] server-8b04e788-ec63-8cae-cc75-0aac69070353: skipping application of old log: index=22 +2025-01-28T14:54:44.374Z [INFO] server-cd12e50e-bd0a-d539-b229-a440b0aa9b44: copied to local snapshot: bytes=71 +2025-01-28T14:54:44.374Z [INFO] server-2050504c-cc88-a166-4194-a85f5a64e661: pipelining replication: peer="{Voter server-8b04e788-ec63-8cae-cc75-0aac69070353 8b04e788-ec63-8cae-cc75-0aac69070353}" +2025-01-28T14:54:44.374Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:54:44.375Z [INFO] server-cd12e50e-bd0a-d539-b229-a440b0aa9b44: snapshot restore progress: id=2-23-1738076084373 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:44.375Z [INFO] server-cd12e50e-bd0a-d539-b229-a440b0aa9b44: Installed remote snapshot +2025-01-28T14:54:44.375Z [INFO] server-2050504c-cc88-a166-4194-a85f5a64e661: pipelining replication: peer="{Voter server-cd12e50e-bd0a-d539-b229-a440b0aa9b44 cd12e50e-bd0a-d539-b229-a440b0aa9b44}" +2025-01-28T14:54:45.380Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:54:44.374991072 +0000 UTC m=+70.049481052 (2 nodes), 2.48µs from start of poll, 1.706971268s from cluster start. Timeout at 2025-01-28 14:54:45.380044847 +0000 UTC m=+71.054534827, 1.005053775s after stability +2025-01-28T14:54:45.393Z [INFO] server-2050504c-cc88-a166-4194-a85f5a64e661: aborting pipeline replication: peer="{Voter server-cd12e50e-bd0a-d539-b229-a440b0aa9b44 cd12e50e-bd0a-d539-b229-a440b0aa9b44}" +2025-01-28T14:54:45.393Z [INFO] server-2050504c-cc88-a166-4194-a85f5a64e661: aborting pipeline replication: peer="{Voter server-8b04e788-ec63-8cae-cc75-0aac69070353 8b04e788-ec63-8cae-cc75-0aac69070353}" === RUN TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_false -2025-01-28T14:47:44.274Z [INFO] server-e7c6f8f8-cdef-8d82-e864-83ca31daaff5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e7c6f8f8-cdef-8d82-e864-83ca31daaff5 Address:e7c6f8f8-cdef-8d82-e864-83ca31daaff5} {Suffrage:Voter ID:server-20f127e6-58df-01b2-44af-b7412c8a286f Address:20f127e6-58df-01b2-44af-b7412c8a286f} {Suffrage:Voter ID:server-c9525c34-7e72-5157-463b-041c518d97fa Address:c9525c34-7e72-5157-463b-041c518d97fa}]" -2025-01-28T14:47:44.274Z [INFO] server-e7c6f8f8-cdef-8d82-e864-83ca31daaff5: entering follower state: follower="Node at e7c6f8f8-cdef-8d82-e864-83ca31daaff5 [Follower]" leader-address= leader-id= -2025-01-28T14:47:44.275Z [INFO] server-20f127e6-58df-01b2-44af-b7412c8a286f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e7c6f8f8-cdef-8d82-e864-83ca31daaff5 Address:e7c6f8f8-cdef-8d82-e864-83ca31daaff5} {Suffrage:Voter ID:server-20f127e6-58df-01b2-44af-b7412c8a286f Address:20f127e6-58df-01b2-44af-b7412c8a286f} {Suffrage:Voter ID:server-c9525c34-7e72-5157-463b-041c518d97fa Address:c9525c34-7e72-5157-463b-041c518d97fa}]" -2025-01-28T14:47:44.276Z [INFO] server-20f127e6-58df-01b2-44af-b7412c8a286f: entering follower state: follower="Node at 20f127e6-58df-01b2-44af-b7412c8a286f [Follower]" leader-address= leader-id= -2025-01-28T14:47:44.277Z [INFO] server-c9525c34-7e72-5157-463b-041c518d97fa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e7c6f8f8-cdef-8d82-e864-83ca31daaff5 Address:e7c6f8f8-cdef-8d82-e864-83ca31daaff5} {Suffrage:Voter ID:server-20f127e6-58df-01b2-44af-b7412c8a286f Address:20f127e6-58df-01b2-44af-b7412c8a286f} {Suffrage:Voter ID:server-c9525c34-7e72-5157-463b-041c518d97fa Address:c9525c34-7e72-5157-463b-041c518d97fa}]" -2025-01-28T14:47:44.277Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:44.277Z [INFO] server-c9525c34-7e72-5157-463b-041c518d97fa: entering follower state: follower="Node at c9525c34-7e72-5157-463b-041c518d97fa [Follower]" leader-address= leader-id= -2025-01-28T14:47:44.853Z [WARN] server-e7c6f8f8-cdef-8d82-e864-83ca31daaff5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:44.854Z [INFO] server-e7c6f8f8-cdef-8d82-e864-83ca31daaff5: entering candidate state: node="Node at e7c6f8f8-cdef-8d82-e864-83ca31daaff5 [Candidate]" term=2 -2025-01-28T14:47:44.855Z [INFO] server-e7c6f8f8-cdef-8d82-e864-83ca31daaff5: election won: term=2 tally=2 -2025-01-28T14:47:44.855Z [INFO] server-e7c6f8f8-cdef-8d82-e864-83ca31daaff5: entering leader state: leader="Node at e7c6f8f8-cdef-8d82-e864-83ca31daaff5 [Leader]" -2025-01-28T14:47:44.855Z [INFO] server-e7c6f8f8-cdef-8d82-e864-83ca31daaff5: added peer, starting replication: peer=server-20f127e6-58df-01b2-44af-b7412c8a286f -2025-01-28T14:47:44.855Z [INFO] server-e7c6f8f8-cdef-8d82-e864-83ca31daaff5: added peer, starting replication: peer=server-c9525c34-7e72-5157-463b-041c518d97fa -2025-01-28T14:47:44.857Z [INFO] server-e7c6f8f8-cdef-8d82-e864-83ca31daaff5: pipelining replication: peer="{Voter server-20f127e6-58df-01b2-44af-b7412c8a286f 20f127e6-58df-01b2-44af-b7412c8a286f}" -2025-01-28T14:47:44.857Z [INFO] server-e7c6f8f8-cdef-8d82-e864-83ca31daaff5: pipelining replication: peer="{Voter server-c9525c34-7e72-5157-463b-041c518d97fa c9525c34-7e72-5157-463b-041c518d97fa}" -2025-01-28T14:47:45.861Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:44.85594361 +0000 UTC m=+47.939350301 (1 nodes), 578.139832ms from start of poll, 583.187649ms from cluster start. Timeout at 2025-01-28 14:47:45.860989612 +0000 UTC m=+48.944395636, 1.005045335s after stability -2025-01-28T14:47:45.870Z [INFO] server-e7c6f8f8-cdef-8d82-e864-83ca31daaff5: starting snapshot up to: index=12 -2025-01-28T14:47:45.870Z [INFO] creating new snapshot: path=/tmp/raft3587898646/snapshots/2-12-1738075665870.tmp -2025-01-28T14:47:45.874Z [INFO] server-e7c6f8f8-cdef-8d82-e864-83ca31daaff5: snapshot complete up to: index=12 -2025-01-28T14:47:45.880Z [INFO] creating new snapshot: path=/tmp/raft3587898646/snapshots/2-23-1738075665880.tmp -2025-01-28T14:47:45.888Z [INFO] server-e7c6f8f8-cdef-8d82-e864-83ca31daaff5: copied to local snapshot: bytes=71 -2025-01-28T14:47:45.891Z [INFO] server-e7c6f8f8-cdef-8d82-e864-83ca31daaff5: snapshot restore progress: id=2-23-1738075665880 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:45.891Z [INFO] server-e7c6f8f8-cdef-8d82-e864-83ca31daaff5: removing all old logs from log store -2025-01-28T14:47:45.891Z [INFO] server-e7c6f8f8-cdef-8d82-e864-83ca31daaff5: compacting logs: from=1 to=22 -2025-01-28T14:47:45.891Z [INFO] server-e7c6f8f8-cdef-8d82-e864-83ca31daaff5: restored user snapshot: index=23 -2025-01-28T14:47:45.892Z [ERROR] server-e7c6f8f8-cdef-8d82-e864-83ca31daaff5: failed to get log: index=22 error="log not found" -2025-01-28T14:47:45.892Z [INFO] server-e7c6f8f8-cdef-8d82-e864-83ca31daaff5: aborting pipeline replication: peer="{Voter server-c9525c34-7e72-5157-463b-041c518d97fa c9525c34-7e72-5157-463b-041c518d97fa}" -2025-01-28T14:47:45.892Z [ERROR] server-e7c6f8f8-cdef-8d82-e864-83ca31daaff5: failed to get log: index=22 error="log not found" -2025-01-28T14:47:45.892Z [INFO] server-e7c6f8f8-cdef-8d82-e864-83ca31daaff5: aborting pipeline replication: peer="{Voter server-20f127e6-58df-01b2-44af-b7412c8a286f 20f127e6-58df-01b2-44af-b7412c8a286f}" -2025-01-28T14:47:45.898Z [ERROR] server-e7c6f8f8-cdef-8d82-e864-83ca31daaff5: failed to get log: index=22 error="log not found" -2025-01-28T14:47:45.900Z [ERROR] server-e7c6f8f8-cdef-8d82-e864-83ca31daaff5: failed to get log: index=22 error="log not found" -2025-01-28T14:47:45.901Z [INFO] creating new snapshot: path=/tmp/raft2000382351/snapshots/2-23-1738075665901.tmp -2025-01-28T14:47:45.905Z [INFO] server-c9525c34-7e72-5157-463b-041c518d97fa: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:45.908Z [INFO] server-c9525c34-7e72-5157-463b-041c518d97fa: copied to local snapshot: bytes=71 -2025-01-28T14:47:45.910Z [INFO] server-c9525c34-7e72-5157-463b-041c518d97fa: snapshot restore progress: id=2-23-1738075665901 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:45.910Z [INFO] server-c9525c34-7e72-5157-463b-041c518d97fa: removing all old logs from log store -2025-01-28T14:47:45.910Z [INFO] server-c9525c34-7e72-5157-463b-041c518d97fa: compacting logs: from=1 to=22 -2025-01-28T14:47:45.910Z [INFO] server-c9525c34-7e72-5157-463b-041c518d97fa: Installed remote snapshot -2025-01-28T14:47:45.913Z [INFO] server-e7c6f8f8-cdef-8d82-e864-83ca31daaff5: pipelining replication: peer="{Voter server-c9525c34-7e72-5157-463b-041c518d97fa c9525c34-7e72-5157-463b-041c518d97fa}" -2025-01-28T14:47:45.914Z [INFO] creating new snapshot: path=/tmp/raft3574893251/snapshots/2-23-1738075665914.tmp -2025-01-28T14:47:45.914Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:47:45.916Z [INFO] server-20f127e6-58df-01b2-44af-b7412c8a286f: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:45.919Z [INFO] server-20f127e6-58df-01b2-44af-b7412c8a286f: copied to local snapshot: bytes=71 -2025-01-28T14:47:45.920Z [INFO] server-20f127e6-58df-01b2-44af-b7412c8a286f: snapshot restore progress: id=2-23-1738075665914 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:45.920Z [INFO] server-20f127e6-58df-01b2-44af-b7412c8a286f: removing all old logs from log store -2025-01-28T14:47:45.920Z [INFO] server-20f127e6-58df-01b2-44af-b7412c8a286f: compacting logs: from=1 to=22 -2025-01-28T14:47:45.920Z [INFO] server-20f127e6-58df-01b2-44af-b7412c8a286f: Installed remote snapshot -2025-01-28T14:47:45.921Z [INFO] server-e7c6f8f8-cdef-8d82-e864-83ca31daaff5: pipelining replication: peer="{Voter server-20f127e6-58df-01b2-44af-b7412c8a286f 20f127e6-58df-01b2-44af-b7412c8a286f}" -2025-01-28T14:47:46.921Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:47:45.914374704 +0000 UTC m=+48.997779728 (2 nodes), 8.333µs from start of poll, 1.641617076s from cluster start. Timeout at 2025-01-28 14:47:46.919519375 +0000 UTC m=+50.002924399, 1.005144671s after stability -2025-01-28T14:47:46.941Z [INFO] server-e7c6f8f8-cdef-8d82-e864-83ca31daaff5: aborting pipeline replication: peer="{Voter server-c9525c34-7e72-5157-463b-041c518d97fa c9525c34-7e72-5157-463b-041c518d97fa}" -2025-01-28T14:47:46.941Z [INFO] server-e7c6f8f8-cdef-8d82-e864-83ca31daaff5: aborting pipeline replication: peer="{Voter server-20f127e6-58df-01b2-44af-b7412c8a286f 20f127e6-58df-01b2-44af-b7412c8a286f}" +2025-01-28T14:54:45.398Z [INFO] server-4790534a-3157-b26e-07f4-dee3e0c556e1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4790534a-3157-b26e-07f4-dee3e0c556e1 Address:4790534a-3157-b26e-07f4-dee3e0c556e1} {Suffrage:Voter ID:server-0a3c39e7-7a7a-1dd4-ceac-0bbe87744006 Address:0a3c39e7-7a7a-1dd4-ceac-0bbe87744006} {Suffrage:Voter ID:server-2887f76f-7354-7282-f73c-f12c386312f1 Address:2887f76f-7354-7282-f73c-f12c386312f1}]" +2025-01-28T14:54:45.399Z [INFO] server-0a3c39e7-7a7a-1dd4-ceac-0bbe87744006: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4790534a-3157-b26e-07f4-dee3e0c556e1 Address:4790534a-3157-b26e-07f4-dee3e0c556e1} {Suffrage:Voter ID:server-0a3c39e7-7a7a-1dd4-ceac-0bbe87744006 Address:0a3c39e7-7a7a-1dd4-ceac-0bbe87744006} {Suffrage:Voter ID:server-2887f76f-7354-7282-f73c-f12c386312f1 Address:2887f76f-7354-7282-f73c-f12c386312f1}]" +2025-01-28T14:54:45.400Z [INFO] server-4790534a-3157-b26e-07f4-dee3e0c556e1: entering follower state: follower="Node at 4790534a-3157-b26e-07f4-dee3e0c556e1 [Follower]" leader-address= leader-id= +2025-01-28T14:54:45.400Z [INFO] server-2887f76f-7354-7282-f73c-f12c386312f1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4790534a-3157-b26e-07f4-dee3e0c556e1 Address:4790534a-3157-b26e-07f4-dee3e0c556e1} {Suffrage:Voter ID:server-0a3c39e7-7a7a-1dd4-ceac-0bbe87744006 Address:0a3c39e7-7a7a-1dd4-ceac-0bbe87744006} {Suffrage:Voter ID:server-2887f76f-7354-7282-f73c-f12c386312f1 Address:2887f76f-7354-7282-f73c-f12c386312f1}]" +2025-01-28T14:54:45.400Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:45.400Z [INFO] server-2887f76f-7354-7282-f73c-f12c386312f1: entering follower state: follower="Node at 2887f76f-7354-7282-f73c-f12c386312f1 [Follower]" leader-address= leader-id= +2025-01-28T14:54:45.400Z [INFO] server-0a3c39e7-7a7a-1dd4-ceac-0bbe87744006: entering follower state: follower="Node at 0a3c39e7-7a7a-1dd4-ceac-0bbe87744006 [Follower]" leader-address= leader-id= +2025-01-28T14:54:45.946Z [WARN] server-4790534a-3157-b26e-07f4-dee3e0c556e1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:45.946Z [INFO] server-4790534a-3157-b26e-07f4-dee3e0c556e1: entering candidate state: node="Node at 4790534a-3157-b26e-07f4-dee3e0c556e1 [Candidate]" term=2 +2025-01-28T14:54:45.947Z [INFO] server-4790534a-3157-b26e-07f4-dee3e0c556e1: election won: term=2 tally=2 +2025-01-28T14:54:45.947Z [INFO] server-4790534a-3157-b26e-07f4-dee3e0c556e1: entering leader state: leader="Node at 4790534a-3157-b26e-07f4-dee3e0c556e1 [Leader]" +2025-01-28T14:54:45.947Z [INFO] server-4790534a-3157-b26e-07f4-dee3e0c556e1: added peer, starting replication: peer=server-0a3c39e7-7a7a-1dd4-ceac-0bbe87744006 +2025-01-28T14:54:45.947Z [INFO] server-4790534a-3157-b26e-07f4-dee3e0c556e1: added peer, starting replication: peer=server-2887f76f-7354-7282-f73c-f12c386312f1 +2025-01-28T14:54:45.947Z [INFO] server-4790534a-3157-b26e-07f4-dee3e0c556e1: pipelining replication: peer="{Voter server-2887f76f-7354-7282-f73c-f12c386312f1 2887f76f-7354-7282-f73c-f12c386312f1}" +2025-01-28T14:54:45.947Z [INFO] server-4790534a-3157-b26e-07f4-dee3e0c556e1: pipelining replication: peer="{Voter server-0a3c39e7-7a7a-1dd4-ceac-0bbe87744006 0a3c39e7-7a7a-1dd4-ceac-0bbe87744006}" +2025-01-28T14:54:46.953Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:45.947399454 +0000 UTC m=+71.621889434 (1 nodes), 546.695806ms from start of poll, 549.905243ms from cluster start. Timeout at 2025-01-28 14:54:46.952410953 +0000 UTC m=+72.626900933, 1.005011499s after stability +2025-01-28T14:54:46.954Z [INFO] server-4790534a-3157-b26e-07f4-dee3e0c556e1: starting snapshot up to: index=12 +2025-01-28T14:54:46.955Z [INFO] creating new snapshot: path=/tmp/raft3698249626/snapshots/2-12-1738076086955.tmp +2025-01-28T14:54:46.956Z [INFO] server-4790534a-3157-b26e-07f4-dee3e0c556e1: snapshot complete up to: index=12 +2025-01-28T14:54:46.958Z [INFO] creating new snapshot: path=/tmp/raft3698249626/snapshots/2-23-1738076086958.tmp +2025-01-28T14:54:46.959Z [INFO] server-4790534a-3157-b26e-07f4-dee3e0c556e1: copied to local snapshot: bytes=71 +2025-01-28T14:54:46.960Z [INFO] server-4790534a-3157-b26e-07f4-dee3e0c556e1: snapshot restore progress: id=2-23-1738076086958 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:46.960Z [INFO] server-4790534a-3157-b26e-07f4-dee3e0c556e1: removing all old logs from log store +2025-01-28T14:54:46.960Z [INFO] server-4790534a-3157-b26e-07f4-dee3e0c556e1: compacting logs: from=1 to=22 +2025-01-28T14:54:46.960Z [INFO] server-4790534a-3157-b26e-07f4-dee3e0c556e1: restored user snapshot: index=23 +2025-01-28T14:54:46.960Z [ERROR] server-4790534a-3157-b26e-07f4-dee3e0c556e1: failed to get log: index=22 error="log not found" +2025-01-28T14:54:46.960Z [ERROR] server-4790534a-3157-b26e-07f4-dee3e0c556e1: failed to get log: index=22 error="log not found" +2025-01-28T14:54:46.960Z [INFO] server-4790534a-3157-b26e-07f4-dee3e0c556e1: aborting pipeline replication: peer="{Voter server-0a3c39e7-7a7a-1dd4-ceac-0bbe87744006 0a3c39e7-7a7a-1dd4-ceac-0bbe87744006}" +2025-01-28T14:54:46.960Z [INFO] server-4790534a-3157-b26e-07f4-dee3e0c556e1: aborting pipeline replication: peer="{Voter server-2887f76f-7354-7282-f73c-f12c386312f1 2887f76f-7354-7282-f73c-f12c386312f1}" +2025-01-28T14:54:46.969Z [ERROR] server-4790534a-3157-b26e-07f4-dee3e0c556e1: failed to get log: index=22 error="log not found" +2025-01-28T14:54:46.970Z [ERROR] server-4790534a-3157-b26e-07f4-dee3e0c556e1: failed to get log: index=22 error="log not found" +2025-01-28T14:54:46.970Z [INFO] creating new snapshot: path=/tmp/raft1331227272/snapshots/2-23-1738076086970.tmp +2025-01-28T14:54:46.970Z [INFO] creating new snapshot: path=/tmp/raft486685482/snapshots/2-23-1738076086970.tmp +2025-01-28T14:54:46.971Z [INFO] server-0a3c39e7-7a7a-1dd4-ceac-0bbe87744006: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:46.972Z [INFO] server-2887f76f-7354-7282-f73c-f12c386312f1: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:46.972Z [INFO] server-2887f76f-7354-7282-f73c-f12c386312f1: copied to local snapshot: bytes=71 +2025-01-28T14:54:46.972Z [INFO] server-0a3c39e7-7a7a-1dd4-ceac-0bbe87744006: copied to local snapshot: bytes=71 +2025-01-28T14:54:46.974Z [INFO] server-2887f76f-7354-7282-f73c-f12c386312f1: snapshot restore progress: id=2-23-1738076086970 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:46.974Z [INFO] server-0a3c39e7-7a7a-1dd4-ceac-0bbe87744006: snapshot restore progress: id=2-23-1738076086970 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:46.974Z [INFO] server-0a3c39e7-7a7a-1dd4-ceac-0bbe87744006: removing all old logs from log store +2025-01-28T14:54:46.974Z [INFO] server-0a3c39e7-7a7a-1dd4-ceac-0bbe87744006: compacting logs: from=1 to=22 +2025-01-28T14:54:46.974Z [INFO] server-0a3c39e7-7a7a-1dd4-ceac-0bbe87744006: Installed remote snapshot +2025-01-28T14:54:46.974Z [INFO] server-2887f76f-7354-7282-f73c-f12c386312f1: removing all old logs from log store +2025-01-28T14:54:46.974Z [INFO] server-2887f76f-7354-7282-f73c-f12c386312f1: compacting logs: from=1 to=22 +2025-01-28T14:54:46.974Z [INFO] server-2887f76f-7354-7282-f73c-f12c386312f1: Installed remote snapshot +2025-01-28T14:54:46.974Z [WARN] server-0a3c39e7-7a7a-1dd4-ceac-0bbe87744006: skipping application of old log: index=22 +2025-01-28T14:54:46.975Z [INFO] server-4790534a-3157-b26e-07f4-dee3e0c556e1: pipelining replication: peer="{Voter server-0a3c39e7-7a7a-1dd4-ceac-0bbe87744006 0a3c39e7-7a7a-1dd4-ceac-0bbe87744006}" +2025-01-28T14:54:46.975Z [WARN] server-2887f76f-7354-7282-f73c-f12c386312f1: skipping application of old log: index=22 +2025-01-28T14:54:46.975Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:54:46.975Z [INFO] server-4790534a-3157-b26e-07f4-dee3e0c556e1: pipelining replication: peer="{Voter server-2887f76f-7354-7282-f73c-f12c386312f1 2887f76f-7354-7282-f73c-f12c386312f1}" +2025-01-28T14:54:47.981Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:54:46.975348779 +0000 UTC m=+72.649838759 (2 nodes), 2.659µs from start of poll, 1.577854568s from cluster start. Timeout at 2025-01-28 14:54:47.980422652 +0000 UTC m=+73.654912712, 1.005073953s after stability +2025-01-28T14:54:47.992Z [INFO] server-4790534a-3157-b26e-07f4-dee3e0c556e1: aborting pipeline replication: peer="{Voter server-2887f76f-7354-7282-f73c-f12c386312f1 2887f76f-7354-7282-f73c-f12c386312f1}" +2025-01-28T14:54:47.992Z [INFO] server-4790534a-3157-b26e-07f4-dee3e0c556e1: aborting pipeline replication: peer="{Voter server-0a3c39e7-7a7a-1dd4-ceac-0bbe87744006 0a3c39e7-7a7a-1dd4-ceac-0bbe87744006}" === RUN TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_true -2025-01-28T14:47:46.961Z [INFO] server-91ed2ad3-61f0-e889-7647-ffe91f847860: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-91ed2ad3-61f0-e889-7647-ffe91f847860 Address:91ed2ad3-61f0-e889-7647-ffe91f847860} {Suffrage:Voter ID:server-b38473ca-6276-4d1b-e145-f87a31535893 Address:b38473ca-6276-4d1b-e145-f87a31535893} {Suffrage:Voter ID:server-5f3ea835-045e-5e32-edb3-d81c30891701 Address:5f3ea835-045e-5e32-edb3-d81c30891701}]" -2025-01-28T14:47:46.961Z [INFO] server-91ed2ad3-61f0-e889-7647-ffe91f847860: entering follower state: follower="Node at 91ed2ad3-61f0-e889-7647-ffe91f847860 [Follower]" leader-address= leader-id= -2025-01-28T14:47:46.962Z [INFO] server-b38473ca-6276-4d1b-e145-f87a31535893: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-91ed2ad3-61f0-e889-7647-ffe91f847860 Address:91ed2ad3-61f0-e889-7647-ffe91f847860} {Suffrage:Voter ID:server-b38473ca-6276-4d1b-e145-f87a31535893 Address:b38473ca-6276-4d1b-e145-f87a31535893} {Suffrage:Voter ID:server-5f3ea835-045e-5e32-edb3-d81c30891701 Address:5f3ea835-045e-5e32-edb3-d81c30891701}]" -2025-01-28T14:47:46.962Z [INFO] server-b38473ca-6276-4d1b-e145-f87a31535893: entering follower state: follower="Node at b38473ca-6276-4d1b-e145-f87a31535893 [Follower]" leader-address= leader-id= -2025-01-28T14:47:46.963Z [INFO] server-5f3ea835-045e-5e32-edb3-d81c30891701: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-91ed2ad3-61f0-e889-7647-ffe91f847860 Address:91ed2ad3-61f0-e889-7647-ffe91f847860} {Suffrage:Voter ID:server-b38473ca-6276-4d1b-e145-f87a31535893 Address:b38473ca-6276-4d1b-e145-f87a31535893} {Suffrage:Voter ID:server-5f3ea835-045e-5e32-edb3-d81c30891701 Address:5f3ea835-045e-5e32-edb3-d81c30891701}]" -2025-01-28T14:47:46.963Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:46.963Z [INFO] server-5f3ea835-045e-5e32-edb3-d81c30891701: entering follower state: follower="Node at 5f3ea835-045e-5e32-edb3-d81c30891701 [Follower]" leader-address= leader-id= -2025-01-28T14:47:47.692Z [WARN] server-91ed2ad3-61f0-e889-7647-ffe91f847860: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:47.693Z [INFO] server-91ed2ad3-61f0-e889-7647-ffe91f847860: entering candidate state: node="Node at 91ed2ad3-61f0-e889-7647-ffe91f847860 [Candidate]" term=2 -2025-01-28T14:47:47.694Z [INFO] server-91ed2ad3-61f0-e889-7647-ffe91f847860: election won: term=2 tally=2 -2025-01-28T14:47:47.694Z [INFO] server-91ed2ad3-61f0-e889-7647-ffe91f847860: entering leader state: leader="Node at 91ed2ad3-61f0-e889-7647-ffe91f847860 [Leader]" -2025-01-28T14:47:47.694Z [INFO] server-91ed2ad3-61f0-e889-7647-ffe91f847860: added peer, starting replication: peer=server-b38473ca-6276-4d1b-e145-f87a31535893 -2025-01-28T14:47:47.695Z [INFO] server-91ed2ad3-61f0-e889-7647-ffe91f847860: added peer, starting replication: peer=server-5f3ea835-045e-5e32-edb3-d81c30891701 -2025-01-28T14:47:47.696Z [INFO] server-91ed2ad3-61f0-e889-7647-ffe91f847860: pipelining replication: peer="{Voter server-b38473ca-6276-4d1b-e145-f87a31535893 b38473ca-6276-4d1b-e145-f87a31535893}" -2025-01-28T14:47:47.697Z [INFO] server-91ed2ad3-61f0-e889-7647-ffe91f847860: pipelining replication: peer="{Voter server-5f3ea835-045e-5e32-edb3-d81c30891701 5f3ea835-045e-5e32-edb3-d81c30891701}" -2025-01-28T14:47:48.700Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:47.694743901 +0000 UTC m=+50.778150258 (1 nodes), 731.213711ms from start of poll, 734.131464ms from cluster start. Timeout at 2025-01-28 14:47:48.699763901 +0000 UTC m=+51.783169925, 1.005019667s after stability -2025-01-28T14:47:48.708Z [INFO] server-91ed2ad3-61f0-e889-7647-ffe91f847860: starting snapshot up to: index=12 -2025-01-28T14:47:48.708Z [INFO] creating new snapshot: path=/tmp/raft3470672819/snapshots/2-12-1738075668708.tmp -2025-01-28T14:47:48.712Z [INFO] server-91ed2ad3-61f0-e889-7647-ffe91f847860: snapshot complete up to: index=12 -2025-01-28T14:47:48.724Z [INFO] server-bb52452f-f1be-9b4d-6a01-ebc8ba97cabd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bb52452f-f1be-9b4d-6a01-ebc8ba97cabd Address:bb52452f-f1be-9b4d-6a01-ebc8ba97cabd} {Suffrage:Voter ID:server-b8fb1426-7437-8a41-6fb0-6533fd60b357 Address:b8fb1426-7437-8a41-6fb0-6533fd60b357} {Suffrage:Voter ID:server-5bd05750-583d-16dc-cbab-1bc2a5fc8d03 Address:5bd05750-583d-16dc-cbab-1bc2a5fc8d03}]" -2025-01-28T14:47:48.725Z [INFO] server-bb52452f-f1be-9b4d-6a01-ebc8ba97cabd: entering follower state: follower="Node at bb52452f-f1be-9b4d-6a01-ebc8ba97cabd [Follower]" leader-address= leader-id= -2025-01-28T14:47:48.726Z [INFO] server-b8fb1426-7437-8a41-6fb0-6533fd60b357: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bb52452f-f1be-9b4d-6a01-ebc8ba97cabd Address:bb52452f-f1be-9b4d-6a01-ebc8ba97cabd} {Suffrage:Voter ID:server-b8fb1426-7437-8a41-6fb0-6533fd60b357 Address:b8fb1426-7437-8a41-6fb0-6533fd60b357} {Suffrage:Voter ID:server-5bd05750-583d-16dc-cbab-1bc2a5fc8d03 Address:5bd05750-583d-16dc-cbab-1bc2a5fc8d03}]" -2025-01-28T14:47:48.727Z [INFO] server-b8fb1426-7437-8a41-6fb0-6533fd60b357: entering follower state: follower="Node at b8fb1426-7437-8a41-6fb0-6533fd60b357 [Follower]" leader-address= leader-id= -2025-01-28T14:47:48.728Z [INFO] server-5bd05750-583d-16dc-cbab-1bc2a5fc8d03: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bb52452f-f1be-9b4d-6a01-ebc8ba97cabd Address:bb52452f-f1be-9b4d-6a01-ebc8ba97cabd} {Suffrage:Voter ID:server-b8fb1426-7437-8a41-6fb0-6533fd60b357 Address:b8fb1426-7437-8a41-6fb0-6533fd60b357} {Suffrage:Voter ID:server-5bd05750-583d-16dc-cbab-1bc2a5fc8d03 Address:5bd05750-583d-16dc-cbab-1bc2a5fc8d03}]" -2025-01-28T14:47:48.728Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:48.728Z [INFO] server-5bd05750-583d-16dc-cbab-1bc2a5fc8d03: entering follower state: follower="Node at 5bd05750-583d-16dc-cbab-1bc2a5fc8d03 [Follower]" leader-address= leader-id= -2025-01-28T14:47:49.251Z [WARN] server-bb52452f-f1be-9b4d-6a01-ebc8ba97cabd: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:49.251Z [INFO] server-bb52452f-f1be-9b4d-6a01-ebc8ba97cabd: entering candidate state: node="Node at bb52452f-f1be-9b4d-6a01-ebc8ba97cabd [Candidate]" term=2 -2025-01-28T14:47:49.252Z [INFO] server-bb52452f-f1be-9b4d-6a01-ebc8ba97cabd: election won: term=2 tally=2 -2025-01-28T14:47:49.252Z [INFO] server-bb52452f-f1be-9b4d-6a01-ebc8ba97cabd: entering leader state: leader="Node at bb52452f-f1be-9b4d-6a01-ebc8ba97cabd [Leader]" -2025-01-28T14:47:49.252Z [INFO] server-bb52452f-f1be-9b4d-6a01-ebc8ba97cabd: added peer, starting replication: peer=server-b8fb1426-7437-8a41-6fb0-6533fd60b357 -2025-01-28T14:47:49.253Z [INFO] server-bb52452f-f1be-9b4d-6a01-ebc8ba97cabd: added peer, starting replication: peer=server-5bd05750-583d-16dc-cbab-1bc2a5fc8d03 -2025-01-28T14:47:49.254Z [INFO] server-bb52452f-f1be-9b4d-6a01-ebc8ba97cabd: pipelining replication: peer="{Voter server-b8fb1426-7437-8a41-6fb0-6533fd60b357 b8fb1426-7437-8a41-6fb0-6533fd60b357}" -2025-01-28T14:47:49.254Z [INFO] server-bb52452f-f1be-9b4d-6a01-ebc8ba97cabd: pipelining replication: peer="{Voter server-5bd05750-583d-16dc-cbab-1bc2a5fc8d03 5bd05750-583d-16dc-cbab-1bc2a5fc8d03}" -2025-01-28T14:47:50.258Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:49.252814164 +0000 UTC m=+52.336220521 (1 nodes), 523.995892ms from start of poll, 529.401719ms from cluster start. Timeout at 2025-01-28 14:47:50.257838831 +0000 UTC m=+53.341244188, 1.005023667s after stability -2025-01-28T14:47:50.259Z [INFO] creating new snapshot: path=/tmp/raft1063363986/snapshots/2-15-1738075670259.tmp -2025-01-28T14:47:50.263Z [INFO] server-bb52452f-f1be-9b4d-6a01-ebc8ba97cabd: copied to local snapshot: bytes=71 -2025-01-28T14:47:50.265Z [INFO] server-bb52452f-f1be-9b4d-6a01-ebc8ba97cabd: snapshot restore progress: id=2-15-1738075670259 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:50.265Z [INFO] server-bb52452f-f1be-9b4d-6a01-ebc8ba97cabd: restored user snapshot: index=15 -2025-01-28T14:47:50.266Z [ERROR] server-bb52452f-f1be-9b4d-6a01-ebc8ba97cabd: failed to get log: index=3 error="log not found" -2025-01-28T14:47:50.266Z [INFO] server-bb52452f-f1be-9b4d-6a01-ebc8ba97cabd: aborting pipeline replication: peer="{Voter server-b8fb1426-7437-8a41-6fb0-6533fd60b357 b8fb1426-7437-8a41-6fb0-6533fd60b357}" -2025-01-28T14:47:50.266Z [ERROR] server-bb52452f-f1be-9b4d-6a01-ebc8ba97cabd: failed to get log: index=3 error="log not found" -2025-01-28T14:47:50.266Z [INFO] server-bb52452f-f1be-9b4d-6a01-ebc8ba97cabd: aborting pipeline replication: peer="{Voter server-5bd05750-583d-16dc-cbab-1bc2a5fc8d03 5bd05750-583d-16dc-cbab-1bc2a5fc8d03}" -2025-01-28T14:47:50.273Z [ERROR] server-bb52452f-f1be-9b4d-6a01-ebc8ba97cabd: failed to get log: index=3 error="log not found" -2025-01-28T14:47:50.275Z [INFO] creating new snapshot: path=/tmp/raft1006998353/snapshots/2-15-1738075670275.tmp -2025-01-28T14:47:50.276Z [ERROR] server-bb52452f-f1be-9b4d-6a01-ebc8ba97cabd: failed to get log: index=3 error="log not found" -2025-01-28T14:47:50.277Z [INFO] server-5bd05750-583d-16dc-cbab-1bc2a5fc8d03: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:50.280Z [INFO] creating new snapshot: path=/tmp/raft3734439899/snapshots/2-15-1738075670280.tmp -2025-01-28T14:47:50.283Z [INFO] server-5bd05750-583d-16dc-cbab-1bc2a5fc8d03: copied to local snapshot: bytes=71 -2025-01-28T14:47:50.283Z [INFO] server-b8fb1426-7437-8a41-6fb0-6533fd60b357: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:50.287Z [INFO] server-5bd05750-583d-16dc-cbab-1bc2a5fc8d03: snapshot restore progress: id=2-15-1738075670275 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:50.287Z [INFO] server-5bd05750-583d-16dc-cbab-1bc2a5fc8d03: Installed remote snapshot -2025-01-28T14:47:50.288Z [INFO] server-b8fb1426-7437-8a41-6fb0-6533fd60b357: copied to local snapshot: bytes=71 -2025-01-28T14:47:50.289Z [INFO] server-bb52452f-f1be-9b4d-6a01-ebc8ba97cabd: pipelining replication: peer="{Voter server-5bd05750-583d-16dc-cbab-1bc2a5fc8d03 5bd05750-583d-16dc-cbab-1bc2a5fc8d03}" -2025-01-28T14:47:50.297Z [INFO] server-b8fb1426-7437-8a41-6fb0-6533fd60b357: snapshot restore progress: id=2-15-1738075670280 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:50.299Z [INFO] server-b8fb1426-7437-8a41-6fb0-6533fd60b357: Installed remote snapshot -2025-01-28T14:47:50.299Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:47:50.302Z [INFO] server-bb52452f-f1be-9b4d-6a01-ebc8ba97cabd: pipelining replication: peer="{Voter server-b8fb1426-7437-8a41-6fb0-6533fd60b357 b8fb1426-7437-8a41-6fb0-6533fd60b357}" -2025-01-28T14:47:51.306Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:47:50.299769917 +0000 UTC m=+53.383174941 (2 nodes), 10.667µs from start of poll, 1.576356139s from cluster start. Timeout at 2025-01-28 14:47:51.305112261 +0000 UTC m=+54.388518618, 1.005343677s after stability -2025-01-28T14:47:51.326Z [INFO] server-91ed2ad3-61f0-e889-7647-ffe91f847860: aborting pipeline replication: peer="{Voter server-5f3ea835-045e-5e32-edb3-d81c30891701 5f3ea835-045e-5e32-edb3-d81c30891701}" -2025-01-28T14:47:51.326Z [INFO] server-91ed2ad3-61f0-e889-7647-ffe91f847860: aborting pipeline replication: peer="{Voter server-b38473ca-6276-4d1b-e145-f87a31535893 b38473ca-6276-4d1b-e145-f87a31535893}" +2025-01-28T14:54:47.998Z [INFO] server-d13178ff-0d53-802d-08d0-32f53f675c97: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d13178ff-0d53-802d-08d0-32f53f675c97 Address:d13178ff-0d53-802d-08d0-32f53f675c97} {Suffrage:Voter ID:server-72d04ff1-e490-ae1e-1a02-ecf28d1e6be0 Address:72d04ff1-e490-ae1e-1a02-ecf28d1e6be0} {Suffrage:Voter ID:server-128d5b43-dd5c-8edc-0b80-6485ef0f2ad6 Address:128d5b43-dd5c-8edc-0b80-6485ef0f2ad6}]" +2025-01-28T14:54:47.998Z [INFO] server-d13178ff-0d53-802d-08d0-32f53f675c97: entering follower state: follower="Node at d13178ff-0d53-802d-08d0-32f53f675c97 [Follower]" leader-address= leader-id= +2025-01-28T14:54:47.998Z [INFO] server-72d04ff1-e490-ae1e-1a02-ecf28d1e6be0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d13178ff-0d53-802d-08d0-32f53f675c97 Address:d13178ff-0d53-802d-08d0-32f53f675c97} {Suffrage:Voter ID:server-72d04ff1-e490-ae1e-1a02-ecf28d1e6be0 Address:72d04ff1-e490-ae1e-1a02-ecf28d1e6be0} {Suffrage:Voter ID:server-128d5b43-dd5c-8edc-0b80-6485ef0f2ad6 Address:128d5b43-dd5c-8edc-0b80-6485ef0f2ad6}]" +2025-01-28T14:54:47.999Z [INFO] server-72d04ff1-e490-ae1e-1a02-ecf28d1e6be0: entering follower state: follower="Node at 72d04ff1-e490-ae1e-1a02-ecf28d1e6be0 [Follower]" leader-address= leader-id= +2025-01-28T14:54:47.999Z [INFO] server-128d5b43-dd5c-8edc-0b80-6485ef0f2ad6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d13178ff-0d53-802d-08d0-32f53f675c97 Address:d13178ff-0d53-802d-08d0-32f53f675c97} {Suffrage:Voter ID:server-72d04ff1-e490-ae1e-1a02-ecf28d1e6be0 Address:72d04ff1-e490-ae1e-1a02-ecf28d1e6be0} {Suffrage:Voter ID:server-128d5b43-dd5c-8edc-0b80-6485ef0f2ad6 Address:128d5b43-dd5c-8edc-0b80-6485ef0f2ad6}]" +2025-01-28T14:54:47.999Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:47.999Z [INFO] server-128d5b43-dd5c-8edc-0b80-6485ef0f2ad6: entering follower state: follower="Node at 128d5b43-dd5c-8edc-0b80-6485ef0f2ad6 [Follower]" leader-address= leader-id= +2025-01-28T14:54:48.698Z [WARN] server-128d5b43-dd5c-8edc-0b80-6485ef0f2ad6: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:48.698Z [INFO] server-128d5b43-dd5c-8edc-0b80-6485ef0f2ad6: entering candidate state: node="Node at 128d5b43-dd5c-8edc-0b80-6485ef0f2ad6 [Candidate]" term=2 +2025-01-28T14:54:48.698Z [INFO] server-128d5b43-dd5c-8edc-0b80-6485ef0f2ad6: election won: term=2 tally=2 +2025-01-28T14:54:48.698Z [INFO] server-128d5b43-dd5c-8edc-0b80-6485ef0f2ad6: entering leader state: leader="Node at 128d5b43-dd5c-8edc-0b80-6485ef0f2ad6 [Leader]" +2025-01-28T14:54:48.698Z [INFO] server-128d5b43-dd5c-8edc-0b80-6485ef0f2ad6: added peer, starting replication: peer=server-d13178ff-0d53-802d-08d0-32f53f675c97 +2025-01-28T14:54:48.698Z [INFO] server-128d5b43-dd5c-8edc-0b80-6485ef0f2ad6: added peer, starting replication: peer=server-72d04ff1-e490-ae1e-1a02-ecf28d1e6be0 +2025-01-28T14:54:48.698Z [INFO] server-128d5b43-dd5c-8edc-0b80-6485ef0f2ad6: pipelining replication: peer="{Voter server-d13178ff-0d53-802d-08d0-32f53f675c97 d13178ff-0d53-802d-08d0-32f53f675c97}" +2025-01-28T14:54:48.698Z [INFO] server-128d5b43-dd5c-8edc-0b80-6485ef0f2ad6: pipelining replication: peer="{Voter server-72d04ff1-e490-ae1e-1a02-ecf28d1e6be0 72d04ff1-e490-ae1e-1a02-ecf28d1e6be0}" +2025-01-28T14:54:49.704Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:48.698670585 +0000 UTC m=+74.373160585 (1 nodes), 698.747442ms from start of poll, 700.653862ms from cluster start. Timeout at 2025-01-28 14:54:49.703677605 +0000 UTC m=+75.378167665, 1.00500708s after stability +2025-01-28T14:54:49.705Z [INFO] server-128d5b43-dd5c-8edc-0b80-6485ef0f2ad6: starting snapshot up to: index=12 +2025-01-28T14:54:49.706Z [INFO] creating new snapshot: path=/tmp/raft3000744062/snapshots/2-12-1738076089706.tmp +2025-01-28T14:54:49.774Z [INFO] server-128d5b43-dd5c-8edc-0b80-6485ef0f2ad6: snapshot complete up to: index=12 +2025-01-28T14:54:49.778Z [INFO] server-bb3db447-5cdd-3761-3e19-7628726f0fae: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bb3db447-5cdd-3761-3e19-7628726f0fae Address:bb3db447-5cdd-3761-3e19-7628726f0fae} {Suffrage:Voter ID:server-972471b3-7b48-7d02-2c42-8a3273317c9a Address:972471b3-7b48-7d02-2c42-8a3273317c9a} {Suffrage:Voter ID:server-bcec3f66-5293-d022-29a2-4ac2cb5adc8b Address:bcec3f66-5293-d022-29a2-4ac2cb5adc8b}]" +2025-01-28T14:54:49.778Z [INFO] server-bb3db447-5cdd-3761-3e19-7628726f0fae: entering follower state: follower="Node at bb3db447-5cdd-3761-3e19-7628726f0fae [Follower]" leader-address= leader-id= +2025-01-28T14:54:49.778Z [INFO] server-972471b3-7b48-7d02-2c42-8a3273317c9a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bb3db447-5cdd-3761-3e19-7628726f0fae Address:bb3db447-5cdd-3761-3e19-7628726f0fae} {Suffrage:Voter ID:server-972471b3-7b48-7d02-2c42-8a3273317c9a Address:972471b3-7b48-7d02-2c42-8a3273317c9a} {Suffrage:Voter ID:server-bcec3f66-5293-d022-29a2-4ac2cb5adc8b Address:bcec3f66-5293-d022-29a2-4ac2cb5adc8b}]" +2025-01-28T14:54:49.779Z [INFO] server-972471b3-7b48-7d02-2c42-8a3273317c9a: entering follower state: follower="Node at 972471b3-7b48-7d02-2c42-8a3273317c9a [Follower]" leader-address= leader-id= +2025-01-28T14:54:49.779Z [INFO] server-bcec3f66-5293-d022-29a2-4ac2cb5adc8b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bb3db447-5cdd-3761-3e19-7628726f0fae Address:bb3db447-5cdd-3761-3e19-7628726f0fae} {Suffrage:Voter ID:server-972471b3-7b48-7d02-2c42-8a3273317c9a Address:972471b3-7b48-7d02-2c42-8a3273317c9a} {Suffrage:Voter ID:server-bcec3f66-5293-d022-29a2-4ac2cb5adc8b Address:bcec3f66-5293-d022-29a2-4ac2cb5adc8b}]" +2025-01-28T14:54:49.779Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:49.779Z [INFO] server-bcec3f66-5293-d022-29a2-4ac2cb5adc8b: entering follower state: follower="Node at bcec3f66-5293-d022-29a2-4ac2cb5adc8b [Follower]" leader-address= leader-id= +2025-01-28T14:54:50.376Z [WARN] server-972471b3-7b48-7d02-2c42-8a3273317c9a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:50.377Z [INFO] server-972471b3-7b48-7d02-2c42-8a3273317c9a: entering candidate state: node="Node at 972471b3-7b48-7d02-2c42-8a3273317c9a [Candidate]" term=2 +2025-01-28T14:54:50.377Z [INFO] server-972471b3-7b48-7d02-2c42-8a3273317c9a: election won: term=2 tally=2 +2025-01-28T14:54:50.377Z [INFO] server-972471b3-7b48-7d02-2c42-8a3273317c9a: entering leader state: leader="Node at 972471b3-7b48-7d02-2c42-8a3273317c9a [Leader]" +2025-01-28T14:54:50.377Z [INFO] server-972471b3-7b48-7d02-2c42-8a3273317c9a: added peer, starting replication: peer=server-bb3db447-5cdd-3761-3e19-7628726f0fae +2025-01-28T14:54:50.377Z [INFO] server-972471b3-7b48-7d02-2c42-8a3273317c9a: added peer, starting replication: peer=server-bcec3f66-5293-d022-29a2-4ac2cb5adc8b +2025-01-28T14:54:50.377Z [INFO] server-972471b3-7b48-7d02-2c42-8a3273317c9a: pipelining replication: peer="{Voter server-bb3db447-5cdd-3761-3e19-7628726f0fae bb3db447-5cdd-3761-3e19-7628726f0fae}" +2025-01-28T14:54:50.377Z [INFO] server-972471b3-7b48-7d02-2c42-8a3273317c9a: pipelining replication: peer="{Voter server-bcec3f66-5293-d022-29a2-4ac2cb5adc8b bcec3f66-5293-d022-29a2-4ac2cb5adc8b}" +2025-01-28T14:54:51.383Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:50.377420155 +0000 UTC m=+76.051910135 (1 nodes), 597.999466ms from start of poll, 599.293164ms from cluster start. Timeout at 2025-01-28 14:54:51.382424335 +0000 UTC m=+77.056914375, 1.00500424s after stability +2025-01-28T14:54:51.383Z [INFO] creating new snapshot: path=/tmp/raft1648756860/snapshots/2-15-1738076091383.tmp +2025-01-28T14:54:51.385Z [INFO] server-972471b3-7b48-7d02-2c42-8a3273317c9a: copied to local snapshot: bytes=71 +2025-01-28T14:54:51.386Z [INFO] server-972471b3-7b48-7d02-2c42-8a3273317c9a: snapshot restore progress: id=2-15-1738076091383 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:51.386Z [INFO] server-972471b3-7b48-7d02-2c42-8a3273317c9a: restored user snapshot: index=15 +2025-01-28T14:54:51.386Z [ERROR] server-972471b3-7b48-7d02-2c42-8a3273317c9a: failed to get log: index=3 error="log not found" +2025-01-28T14:54:51.386Z [INFO] server-972471b3-7b48-7d02-2c42-8a3273317c9a: aborting pipeline replication: peer="{Voter server-bcec3f66-5293-d022-29a2-4ac2cb5adc8b bcec3f66-5293-d022-29a2-4ac2cb5adc8b}" +2025-01-28T14:54:51.386Z [ERROR] server-972471b3-7b48-7d02-2c42-8a3273317c9a: failed to get log: index=3 error="log not found" +2025-01-28T14:54:51.386Z [INFO] server-972471b3-7b48-7d02-2c42-8a3273317c9a: aborting pipeline replication: peer="{Voter server-bb3db447-5cdd-3761-3e19-7628726f0fae bb3db447-5cdd-3761-3e19-7628726f0fae}" +2025-01-28T14:54:51.396Z [ERROR] server-972471b3-7b48-7d02-2c42-8a3273317c9a: failed to get log: index=3 error="log not found" +2025-01-28T14:54:51.397Z [INFO] creating new snapshot: path=/tmp/raft624318904/snapshots/2-15-1738076091397.tmp +2025-01-28T14:54:51.397Z [ERROR] server-972471b3-7b48-7d02-2c42-8a3273317c9a: failed to get log: index=3 error="log not found" +2025-01-28T14:54:51.398Z [INFO] creating new snapshot: path=/tmp/raft1980657660/snapshots/2-15-1738076091398.tmp +2025-01-28T14:54:51.398Z [INFO] server-bb3db447-5cdd-3761-3e19-7628726f0fae: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:51.398Z [INFO] server-bcec3f66-5293-d022-29a2-4ac2cb5adc8b: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:51.399Z [INFO] server-bb3db447-5cdd-3761-3e19-7628726f0fae: copied to local snapshot: bytes=71 +2025-01-28T14:54:51.399Z [INFO] server-bcec3f66-5293-d022-29a2-4ac2cb5adc8b: copied to local snapshot: bytes=71 +2025-01-28T14:54:51.400Z [INFO] server-bb3db447-5cdd-3761-3e19-7628726f0fae: snapshot restore progress: id=2-15-1738076091397 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:51.400Z [INFO] server-bb3db447-5cdd-3761-3e19-7628726f0fae: Installed remote snapshot +2025-01-28T14:54:51.400Z [INFO] server-bcec3f66-5293-d022-29a2-4ac2cb5adc8b: snapshot restore progress: id=2-15-1738076091398 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:51.400Z [INFO] server-bcec3f66-5293-d022-29a2-4ac2cb5adc8b: Installed remote snapshot +2025-01-28T14:54:51.400Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:54:51.400Z [INFO] server-972471b3-7b48-7d02-2c42-8a3273317c9a: pipelining replication: peer="{Voter server-bb3db447-5cdd-3761-3e19-7628726f0fae bb3db447-5cdd-3761-3e19-7628726f0fae}" +2025-01-28T14:54:51.401Z [INFO] server-972471b3-7b48-7d02-2c42-8a3273317c9a: pipelining replication: peer="{Voter server-bcec3f66-5293-d022-29a2-4ac2cb5adc8b bcec3f66-5293-d022-29a2-4ac2cb5adc8b}" +2025-01-28T14:54:52.406Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:54:51.400862206 +0000 UTC m=+77.075352166 (2 nodes), 2.92µs from start of poll, 1.622735195s from cluster start. Timeout at 2025-01-28 14:54:52.405907282 +0000 UTC m=+78.080397262, 1.005045096s after stability +2025-01-28T14:54:52.418Z [INFO] server-128d5b43-dd5c-8edc-0b80-6485ef0f2ad6: aborting pipeline replication: peer="{Voter server-d13178ff-0d53-802d-08d0-32f53f675c97 d13178ff-0d53-802d-08d0-32f53f675c97}" +2025-01-28T14:54:52.418Z [INFO] server-128d5b43-dd5c-8edc-0b80-6485ef0f2ad6: aborting pipeline replication: peer="{Voter server-72d04ff1-e490-ae1e-1a02-ecf28d1e6be0 72d04ff1-e490-ae1e-1a02-ecf28d1e6be0}" === RUN TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_true -2025-01-28T14:47:51.345Z [INFO] server-36cc9eee-2f39-55d1-243e-166dafc379e0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-36cc9eee-2f39-55d1-243e-166dafc379e0 Address:36cc9eee-2f39-55d1-243e-166dafc379e0} {Suffrage:Voter ID:server-c0fbd62f-fc1d-1366-1ad4-3edece012848 Address:c0fbd62f-fc1d-1366-1ad4-3edece012848} {Suffrage:Voter ID:server-1e0b3e56-ce15-a30a-33d5-5d1af16656e0 Address:1e0b3e56-ce15-a30a-33d5-5d1af16656e0}]" -2025-01-28T14:47:51.348Z [INFO] server-c0fbd62f-fc1d-1366-1ad4-3edece012848: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-36cc9eee-2f39-55d1-243e-166dafc379e0 Address:36cc9eee-2f39-55d1-243e-166dafc379e0} {Suffrage:Voter ID:server-c0fbd62f-fc1d-1366-1ad4-3edece012848 Address:c0fbd62f-fc1d-1366-1ad4-3edece012848} {Suffrage:Voter ID:server-1e0b3e56-ce15-a30a-33d5-5d1af16656e0 Address:1e0b3e56-ce15-a30a-33d5-5d1af16656e0}]" -2025-01-28T14:47:51.353Z [INFO] server-36cc9eee-2f39-55d1-243e-166dafc379e0: entering follower state: follower="Node at 36cc9eee-2f39-55d1-243e-166dafc379e0 [Follower]" leader-address= leader-id= -2025-01-28T14:47:51.355Z [INFO] server-c0fbd62f-fc1d-1366-1ad4-3edece012848: entering follower state: follower="Node at c0fbd62f-fc1d-1366-1ad4-3edece012848 [Follower]" leader-address= leader-id= -2025-01-28T14:47:51.349Z [INFO] server-1e0b3e56-ce15-a30a-33d5-5d1af16656e0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-36cc9eee-2f39-55d1-243e-166dafc379e0 Address:36cc9eee-2f39-55d1-243e-166dafc379e0} {Suffrage:Voter ID:server-c0fbd62f-fc1d-1366-1ad4-3edece012848 Address:c0fbd62f-fc1d-1366-1ad4-3edece012848} {Suffrage:Voter ID:server-1e0b3e56-ce15-a30a-33d5-5d1af16656e0 Address:1e0b3e56-ce15-a30a-33d5-5d1af16656e0}]" -2025-01-28T14:47:51.356Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:51.356Z [INFO] server-1e0b3e56-ce15-a30a-33d5-5d1af16656e0: entering follower state: follower="Node at 1e0b3e56-ce15-a30a-33d5-5d1af16656e0 [Follower]" leader-address= leader-id= -2025-01-28T14:47:51.884Z [WARN] server-36cc9eee-2f39-55d1-243e-166dafc379e0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:51.885Z [INFO] server-36cc9eee-2f39-55d1-243e-166dafc379e0: entering candidate state: node="Node at 36cc9eee-2f39-55d1-243e-166dafc379e0 [Candidate]" term=2 -2025-01-28T14:47:51.890Z [INFO] server-36cc9eee-2f39-55d1-243e-166dafc379e0: election won: term=2 tally=2 -2025-01-28T14:47:51.891Z [INFO] server-36cc9eee-2f39-55d1-243e-166dafc379e0: entering leader state: leader="Node at 36cc9eee-2f39-55d1-243e-166dafc379e0 [Leader]" -2025-01-28T14:47:51.891Z [INFO] server-36cc9eee-2f39-55d1-243e-166dafc379e0: added peer, starting replication: peer=server-c0fbd62f-fc1d-1366-1ad4-3edece012848 -2025-01-28T14:47:51.891Z [INFO] server-36cc9eee-2f39-55d1-243e-166dafc379e0: added peer, starting replication: peer=server-1e0b3e56-ce15-a30a-33d5-5d1af16656e0 -2025-01-28T14:47:51.893Z [INFO] server-36cc9eee-2f39-55d1-243e-166dafc379e0: pipelining replication: peer="{Voter server-1e0b3e56-ce15-a30a-33d5-5d1af16656e0 1e0b3e56-ce15-a30a-33d5-5d1af16656e0}" -2025-01-28T14:47:51.895Z [INFO] server-36cc9eee-2f39-55d1-243e-166dafc379e0: pipelining replication: peer="{Voter server-c0fbd62f-fc1d-1366-1ad4-3edece012848 c0fbd62f-fc1d-1366-1ad4-3edece012848}" -2025-01-28T14:47:52.898Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:51.893155896 +0000 UTC m=+54.976562253 (1 nodes), 536.505263ms from start of poll, 548.237945ms from cluster start. Timeout at 2025-01-28 14:47:52.898178563 +0000 UTC m=+55.981583588, 1.005021335s after stability -2025-01-28T14:47:52.902Z [INFO] server-36cc9eee-2f39-55d1-243e-166dafc379e0: starting snapshot up to: index=12 -2025-01-28T14:47:52.902Z [INFO] creating new snapshot: path=/tmp/raft3564985301/snapshots/2-12-1738075672902.tmp -2025-01-28T14:47:52.907Z [INFO] server-36cc9eee-2f39-55d1-243e-166dafc379e0: snapshot complete up to: index=12 -2025-01-28T14:47:52.918Z [INFO] server-60ac5e63-3a0b-93e9-9fef-7d5db297c0ff: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-60ac5e63-3a0b-93e9-9fef-7d5db297c0ff Address:60ac5e63-3a0b-93e9-9fef-7d5db297c0ff} {Suffrage:Voter ID:server-b635cf67-4438-7efa-f573-372b1284a9ac Address:b635cf67-4438-7efa-f573-372b1284a9ac} {Suffrage:Voter ID:server-ac05d0aa-5e14-5de7-1638-0049e764b418 Address:ac05d0aa-5e14-5de7-1638-0049e764b418}]" -2025-01-28T14:47:52.918Z [INFO] server-60ac5e63-3a0b-93e9-9fef-7d5db297c0ff: entering follower state: follower="Node at 60ac5e63-3a0b-93e9-9fef-7d5db297c0ff [Follower]" leader-address= leader-id= -2025-01-28T14:47:52.919Z [INFO] server-b635cf67-4438-7efa-f573-372b1284a9ac: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-60ac5e63-3a0b-93e9-9fef-7d5db297c0ff Address:60ac5e63-3a0b-93e9-9fef-7d5db297c0ff} {Suffrage:Voter ID:server-b635cf67-4438-7efa-f573-372b1284a9ac Address:b635cf67-4438-7efa-f573-372b1284a9ac} {Suffrage:Voter ID:server-ac05d0aa-5e14-5de7-1638-0049e764b418 Address:ac05d0aa-5e14-5de7-1638-0049e764b418}]" -2025-01-28T14:47:52.919Z [INFO] server-b635cf67-4438-7efa-f573-372b1284a9ac: entering follower state: follower="Node at b635cf67-4438-7efa-f573-372b1284a9ac [Follower]" leader-address= leader-id= -2025-01-28T14:47:52.920Z [INFO] server-ac05d0aa-5e14-5de7-1638-0049e764b418: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-60ac5e63-3a0b-93e9-9fef-7d5db297c0ff Address:60ac5e63-3a0b-93e9-9fef-7d5db297c0ff} {Suffrage:Voter ID:server-b635cf67-4438-7efa-f573-372b1284a9ac Address:b635cf67-4438-7efa-f573-372b1284a9ac} {Suffrage:Voter ID:server-ac05d0aa-5e14-5de7-1638-0049e764b418 Address:ac05d0aa-5e14-5de7-1638-0049e764b418}]" -2025-01-28T14:47:52.920Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:52.920Z [INFO] server-ac05d0aa-5e14-5de7-1638-0049e764b418: entering follower state: follower="Node at ac05d0aa-5e14-5de7-1638-0049e764b418 [Follower]" leader-address= leader-id= -2025-01-28T14:47:53.438Z [WARN] server-b635cf67-4438-7efa-f573-372b1284a9ac: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:53.438Z [INFO] server-b635cf67-4438-7efa-f573-372b1284a9ac: entering candidate state: node="Node at b635cf67-4438-7efa-f573-372b1284a9ac [Candidate]" term=2 -2025-01-28T14:47:53.439Z [INFO] server-b635cf67-4438-7efa-f573-372b1284a9ac: election won: term=2 tally=2 -2025-01-28T14:47:53.439Z [INFO] server-b635cf67-4438-7efa-f573-372b1284a9ac: entering leader state: leader="Node at b635cf67-4438-7efa-f573-372b1284a9ac [Leader]" -2025-01-28T14:47:53.439Z [INFO] server-b635cf67-4438-7efa-f573-372b1284a9ac: added peer, starting replication: peer=server-60ac5e63-3a0b-93e9-9fef-7d5db297c0ff -2025-01-28T14:47:53.442Z [INFO] server-b635cf67-4438-7efa-f573-372b1284a9ac: added peer, starting replication: peer=server-ac05d0aa-5e14-5de7-1638-0049e764b418 -2025-01-28T14:47:53.456Z [INFO] server-b635cf67-4438-7efa-f573-372b1284a9ac: pipelining replication: peer="{Voter server-60ac5e63-3a0b-93e9-9fef-7d5db297c0ff 60ac5e63-3a0b-93e9-9fef-7d5db297c0ff}" -2025-01-28T14:47:53.458Z [INFO] server-b635cf67-4438-7efa-f573-372b1284a9ac: pipelining replication: peer="{Voter server-ac05d0aa-5e14-5de7-1638-0049e764b418 ac05d0aa-5e14-5de7-1638-0049e764b418}" -2025-01-28T14:47:54.445Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:53.439864822 +0000 UTC m=+56.523269846 (1 nodes), 519.498759ms from start of poll, 522.230506ms from cluster start. Timeout at 2025-01-28 14:47:54.444877489 +0000 UTC m=+57.528282513, 1.005012667s after stability -2025-01-28T14:47:54.446Z [INFO] creating new snapshot: path=/tmp/raft1104535091/snapshots/2-15-1738075674446.tmp -2025-01-28T14:47:54.451Z [INFO] server-b635cf67-4438-7efa-f573-372b1284a9ac: copied to local snapshot: bytes=71 -2025-01-28T14:47:54.452Z [INFO] server-b635cf67-4438-7efa-f573-372b1284a9ac: snapshot restore progress: id=2-15-1738075674446 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:54.453Z [INFO] server-b635cf67-4438-7efa-f573-372b1284a9ac: removing all old logs from log store -2025-01-28T14:47:54.453Z [INFO] server-b635cf67-4438-7efa-f573-372b1284a9ac: compacting logs: from=1 to=2 -2025-01-28T14:47:54.453Z [INFO] server-b635cf67-4438-7efa-f573-372b1284a9ac: restored user snapshot: index=15 -2025-01-28T14:47:54.453Z [ERROR] server-b635cf67-4438-7efa-f573-372b1284a9ac: failed to get log: index=2 error="log not found" -2025-01-28T14:47:54.453Z [INFO] server-b635cf67-4438-7efa-f573-372b1284a9ac: aborting pipeline replication: peer="{Voter server-ac05d0aa-5e14-5de7-1638-0049e764b418 ac05d0aa-5e14-5de7-1638-0049e764b418}" -2025-01-28T14:47:54.453Z [ERROR] server-b635cf67-4438-7efa-f573-372b1284a9ac: failed to get log: index=2 error="log not found" -2025-01-28T14:47:54.454Z [INFO] server-b635cf67-4438-7efa-f573-372b1284a9ac: aborting pipeline replication: peer="{Voter server-60ac5e63-3a0b-93e9-9fef-7d5db297c0ff 60ac5e63-3a0b-93e9-9fef-7d5db297c0ff}" -2025-01-28T14:47:54.460Z [ERROR] server-b635cf67-4438-7efa-f573-372b1284a9ac: failed to get log: index=2 error="log not found" -2025-01-28T14:47:54.462Z [ERROR] server-b635cf67-4438-7efa-f573-372b1284a9ac: failed to get log: index=2 error="log not found" -2025-01-28T14:47:54.462Z [INFO] creating new snapshot: path=/tmp/raft2247567764/snapshots/2-15-1738075674462.tmp -2025-01-28T14:47:54.468Z [INFO] server-60ac5e63-3a0b-93e9-9fef-7d5db297c0ff: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:54.474Z [INFO] creating new snapshot: path=/tmp/raft1343579901/snapshots/2-15-1738075674474.tmp -2025-01-28T14:47:54.477Z [INFO] server-60ac5e63-3a0b-93e9-9fef-7d5db297c0ff: copied to local snapshot: bytes=71 -2025-01-28T14:47:54.477Z [INFO] server-ac05d0aa-5e14-5de7-1638-0049e764b418: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:54.479Z [INFO] server-60ac5e63-3a0b-93e9-9fef-7d5db297c0ff: snapshot restore progress: id=2-15-1738075674462 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:54.479Z [INFO] server-60ac5e63-3a0b-93e9-9fef-7d5db297c0ff: removing all old logs from log store -2025-01-28T14:47:54.479Z [INFO] server-60ac5e63-3a0b-93e9-9fef-7d5db297c0ff: compacting logs: from=1 to=2 -2025-01-28T14:47:54.480Z [INFO] server-60ac5e63-3a0b-93e9-9fef-7d5db297c0ff: Installed remote snapshot -2025-01-28T14:47:54.480Z [INFO] server-ac05d0aa-5e14-5de7-1638-0049e764b418: copied to local snapshot: bytes=71 -2025-01-28T14:47:54.481Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:47:54.480Z [INFO] server-b635cf67-4438-7efa-f573-372b1284a9ac: pipelining replication: peer="{Voter server-60ac5e63-3a0b-93e9-9fef-7d5db297c0ff 60ac5e63-3a0b-93e9-9fef-7d5db297c0ff}" -2025-01-28T14:47:54.482Z [INFO] server-ac05d0aa-5e14-5de7-1638-0049e764b418: snapshot restore progress: id=2-15-1738075674474 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:54.482Z [INFO] server-ac05d0aa-5e14-5de7-1638-0049e764b418: removing all old logs from log store -2025-01-28T14:47:54.482Z [INFO] server-ac05d0aa-5e14-5de7-1638-0049e764b418: compacting logs: from=1 to=2 -2025-01-28T14:47:54.482Z [INFO] server-ac05d0aa-5e14-5de7-1638-0049e764b418: Installed remote snapshot -2025-01-28T14:47:54.483Z [INFO] server-b635cf67-4438-7efa-f573-372b1284a9ac: pipelining replication: peer="{Voter server-ac05d0aa-5e14-5de7-1638-0049e764b418 ac05d0aa-5e14-5de7-1638-0049e764b418}" -2025-01-28T14:47:55.486Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:47:54.481243077 +0000 UTC m=+57.564647768 (2 nodes), 7.334µs from start of poll, 1.563608428s from cluster start. Timeout at 2025-01-28 14:47:55.486488417 +0000 UTC m=+58.569893442, 1.005245674s after stability -2025-01-28T14:47:55.503Z [INFO] server-36cc9eee-2f39-55d1-243e-166dafc379e0: aborting pipeline replication: peer="{Voter server-c0fbd62f-fc1d-1366-1ad4-3edece012848 c0fbd62f-fc1d-1366-1ad4-3edece012848}" -2025-01-28T14:47:55.503Z [INFO] server-36cc9eee-2f39-55d1-243e-166dafc379e0: aborting pipeline replication: peer="{Voter server-1e0b3e56-ce15-a30a-33d5-5d1af16656e0 1e0b3e56-ce15-a30a-33d5-5d1af16656e0}" +2025-01-28T14:54:52.422Z [INFO] server-ede50f2d-5fb1-af3a-0a15-465889c97889: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ede50f2d-5fb1-af3a-0a15-465889c97889 Address:ede50f2d-5fb1-af3a-0a15-465889c97889} {Suffrage:Voter ID:server-8812df09-ccb8-6aab-ebb7-734d1399fcdc Address:8812df09-ccb8-6aab-ebb7-734d1399fcdc} {Suffrage:Voter ID:server-20cc3556-5843-71c7-21cc-de081e2e48fa Address:20cc3556-5843-71c7-21cc-de081e2e48fa}]" +2025-01-28T14:54:52.422Z [INFO] server-ede50f2d-5fb1-af3a-0a15-465889c97889: entering follower state: follower="Node at ede50f2d-5fb1-af3a-0a15-465889c97889 [Follower]" leader-address= leader-id= +2025-01-28T14:54:52.422Z [INFO] server-8812df09-ccb8-6aab-ebb7-734d1399fcdc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ede50f2d-5fb1-af3a-0a15-465889c97889 Address:ede50f2d-5fb1-af3a-0a15-465889c97889} {Suffrage:Voter ID:server-8812df09-ccb8-6aab-ebb7-734d1399fcdc Address:8812df09-ccb8-6aab-ebb7-734d1399fcdc} {Suffrage:Voter ID:server-20cc3556-5843-71c7-21cc-de081e2e48fa Address:20cc3556-5843-71c7-21cc-de081e2e48fa}]" +2025-01-28T14:54:52.423Z [INFO] server-8812df09-ccb8-6aab-ebb7-734d1399fcdc: entering follower state: follower="Node at 8812df09-ccb8-6aab-ebb7-734d1399fcdc [Follower]" leader-address= leader-id= +2025-01-28T14:54:52.423Z [INFO] server-20cc3556-5843-71c7-21cc-de081e2e48fa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ede50f2d-5fb1-af3a-0a15-465889c97889 Address:ede50f2d-5fb1-af3a-0a15-465889c97889} {Suffrage:Voter ID:server-8812df09-ccb8-6aab-ebb7-734d1399fcdc Address:8812df09-ccb8-6aab-ebb7-734d1399fcdc} {Suffrage:Voter ID:server-20cc3556-5843-71c7-21cc-de081e2e48fa Address:20cc3556-5843-71c7-21cc-de081e2e48fa}]" +2025-01-28T14:54:52.423Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:52.423Z [INFO] server-20cc3556-5843-71c7-21cc-de081e2e48fa: entering follower state: follower="Node at 20cc3556-5843-71c7-21cc-de081e2e48fa [Follower]" leader-address= leader-id= +2025-01-28T14:54:52.954Z [WARN] server-ede50f2d-5fb1-af3a-0a15-465889c97889: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:52.954Z [INFO] server-ede50f2d-5fb1-af3a-0a15-465889c97889: entering candidate state: node="Node at ede50f2d-5fb1-af3a-0a15-465889c97889 [Candidate]" term=2 +2025-01-28T14:54:52.954Z [INFO] server-ede50f2d-5fb1-af3a-0a15-465889c97889: election won: term=2 tally=2 +2025-01-28T14:54:52.954Z [INFO] server-ede50f2d-5fb1-af3a-0a15-465889c97889: entering leader state: leader="Node at ede50f2d-5fb1-af3a-0a15-465889c97889 [Leader]" +2025-01-28T14:54:52.954Z [INFO] server-ede50f2d-5fb1-af3a-0a15-465889c97889: added peer, starting replication: peer=server-8812df09-ccb8-6aab-ebb7-734d1399fcdc +2025-01-28T14:54:52.954Z [INFO] server-ede50f2d-5fb1-af3a-0a15-465889c97889: added peer, starting replication: peer=server-20cc3556-5843-71c7-21cc-de081e2e48fa +2025-01-28T14:54:52.954Z [INFO] server-ede50f2d-5fb1-af3a-0a15-465889c97889: pipelining replication: peer="{Voter server-8812df09-ccb8-6aab-ebb7-734d1399fcdc 8812df09-ccb8-6aab-ebb7-734d1399fcdc}" +2025-01-28T14:54:52.954Z [INFO] server-ede50f2d-5fb1-af3a-0a15-465889c97889: pipelining replication: peer="{Voter server-20cc3556-5843-71c7-21cc-de081e2e48fa 20cc3556-5843-71c7-21cc-de081e2e48fa}" +2025-01-28T14:54:53.960Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:52.954685861 +0000 UTC m=+78.629175821 (1 nodes), 531.350913ms from start of poll, 532.34202ms from cluster start. Timeout at 2025-01-28 14:54:53.959716038 +0000 UTC m=+79.634206058, 1.005030237s after stability +2025-01-28T14:54:53.961Z [INFO] server-ede50f2d-5fb1-af3a-0a15-465889c97889: starting snapshot up to: index=12 +2025-01-28T14:54:53.961Z [INFO] creating new snapshot: path=/tmp/raft1107055528/snapshots/2-12-1738076093961.tmp +2025-01-28T14:54:53.962Z [INFO] server-ede50f2d-5fb1-af3a-0a15-465889c97889: snapshot complete up to: index=12 +2025-01-28T14:54:53.966Z [INFO] server-6bcf3569-ccc9-970b-3b69-5d4fa7c89fdf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6bcf3569-ccc9-970b-3b69-5d4fa7c89fdf Address:6bcf3569-ccc9-970b-3b69-5d4fa7c89fdf} {Suffrage:Voter ID:server-97b2c5a6-b641-0e49-e151-7932751d5f66 Address:97b2c5a6-b641-0e49-e151-7932751d5f66} {Suffrage:Voter ID:server-0d9871ee-1c3d-4090-9e42-bd029d482157 Address:0d9871ee-1c3d-4090-9e42-bd029d482157}]" +2025-01-28T14:54:53.966Z [INFO] server-6bcf3569-ccc9-970b-3b69-5d4fa7c89fdf: entering follower state: follower="Node at 6bcf3569-ccc9-970b-3b69-5d4fa7c89fdf [Follower]" leader-address= leader-id= +2025-01-28T14:54:53.966Z [INFO] server-97b2c5a6-b641-0e49-e151-7932751d5f66: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6bcf3569-ccc9-970b-3b69-5d4fa7c89fdf Address:6bcf3569-ccc9-970b-3b69-5d4fa7c89fdf} {Suffrage:Voter ID:server-97b2c5a6-b641-0e49-e151-7932751d5f66 Address:97b2c5a6-b641-0e49-e151-7932751d5f66} {Suffrage:Voter ID:server-0d9871ee-1c3d-4090-9e42-bd029d482157 Address:0d9871ee-1c3d-4090-9e42-bd029d482157}]" +2025-01-28T14:54:53.966Z [INFO] server-97b2c5a6-b641-0e49-e151-7932751d5f66: entering follower state: follower="Node at 97b2c5a6-b641-0e49-e151-7932751d5f66 [Follower]" leader-address= leader-id= +2025-01-28T14:54:53.966Z [INFO] server-0d9871ee-1c3d-4090-9e42-bd029d482157: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6bcf3569-ccc9-970b-3b69-5d4fa7c89fdf Address:6bcf3569-ccc9-970b-3b69-5d4fa7c89fdf} {Suffrage:Voter ID:server-97b2c5a6-b641-0e49-e151-7932751d5f66 Address:97b2c5a6-b641-0e49-e151-7932751d5f66} {Suffrage:Voter ID:server-0d9871ee-1c3d-4090-9e42-bd029d482157 Address:0d9871ee-1c3d-4090-9e42-bd029d482157}]" +2025-01-28T14:54:53.967Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:53.967Z [INFO] server-0d9871ee-1c3d-4090-9e42-bd029d482157: entering follower state: follower="Node at 0d9871ee-1c3d-4090-9e42-bd029d482157 [Follower]" leader-address= leader-id= +2025-01-28T14:54:54.624Z [WARN] server-0d9871ee-1c3d-4090-9e42-bd029d482157: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:54.624Z [INFO] server-0d9871ee-1c3d-4090-9e42-bd029d482157: entering candidate state: node="Node at 0d9871ee-1c3d-4090-9e42-bd029d482157 [Candidate]" term=2 +2025-01-28T14:54:54.624Z [INFO] server-0d9871ee-1c3d-4090-9e42-bd029d482157: election won: term=2 tally=2 +2025-01-28T14:54:54.624Z [INFO] server-0d9871ee-1c3d-4090-9e42-bd029d482157: entering leader state: leader="Node at 0d9871ee-1c3d-4090-9e42-bd029d482157 [Leader]" +2025-01-28T14:54:54.624Z [INFO] server-0d9871ee-1c3d-4090-9e42-bd029d482157: added peer, starting replication: peer=server-6bcf3569-ccc9-970b-3b69-5d4fa7c89fdf +2025-01-28T14:54:54.624Z [INFO] server-0d9871ee-1c3d-4090-9e42-bd029d482157: added peer, starting replication: peer=server-97b2c5a6-b641-0e49-e151-7932751d5f66 +2025-01-28T14:54:54.625Z [INFO] server-0d9871ee-1c3d-4090-9e42-bd029d482157: pipelining replication: peer="{Voter server-97b2c5a6-b641-0e49-e151-7932751d5f66 97b2c5a6-b641-0e49-e151-7932751d5f66}" +2025-01-28T14:54:54.625Z [INFO] server-0d9871ee-1c3d-4090-9e42-bd029d482157: pipelining replication: peer="{Voter server-6bcf3569-ccc9-970b-3b69-5d4fa7c89fdf 6bcf3569-ccc9-970b-3b69-5d4fa7c89fdf}" +2025-01-28T14:54:55.630Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:54.624848982 +0000 UTC m=+80.299338962 (1 nodes), 657.735891ms from start of poll, 658.815949ms from cluster start. Timeout at 2025-01-28 14:54:55.6298629 +0000 UTC m=+81.304352860, 1.005013898s after stability +2025-01-28T14:54:55.633Z [INFO] creating new snapshot: path=/tmp/raft3579785621/snapshots/2-15-1738076095632.tmp +2025-01-28T14:54:55.643Z [INFO] server-0d9871ee-1c3d-4090-9e42-bd029d482157: copied to local snapshot: bytes=71 +2025-01-28T14:54:55.644Z [INFO] server-0d9871ee-1c3d-4090-9e42-bd029d482157: snapshot restore progress: id=2-15-1738076095632 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:55.647Z [INFO] server-0d9871ee-1c3d-4090-9e42-bd029d482157: removing all old logs from log store +2025-01-28T14:54:55.647Z [INFO] server-0d9871ee-1c3d-4090-9e42-bd029d482157: compacting logs: from=1 to=2 +2025-01-28T14:54:55.647Z [INFO] server-0d9871ee-1c3d-4090-9e42-bd029d482157: restored user snapshot: index=15 +2025-01-28T14:54:55.647Z [ERROR] server-0d9871ee-1c3d-4090-9e42-bd029d482157: failed to get log: index=2 error="log not found" +2025-01-28T14:54:55.647Z [INFO] server-0d9871ee-1c3d-4090-9e42-bd029d482157: aborting pipeline replication: peer="{Voter server-97b2c5a6-b641-0e49-e151-7932751d5f66 97b2c5a6-b641-0e49-e151-7932751d5f66}" +2025-01-28T14:54:55.647Z [ERROR] server-0d9871ee-1c3d-4090-9e42-bd029d482157: failed to get log: index=2 error="log not found" +2025-01-28T14:54:55.648Z [INFO] server-0d9871ee-1c3d-4090-9e42-bd029d482157: aborting pipeline replication: peer="{Voter server-6bcf3569-ccc9-970b-3b69-5d4fa7c89fdf 6bcf3569-ccc9-970b-3b69-5d4fa7c89fdf}" +2025-01-28T14:54:55.656Z [ERROR] server-0d9871ee-1c3d-4090-9e42-bd029d482157: failed to get log: index=2 error="log not found" +2025-01-28T14:54:55.656Z [INFO] creating new snapshot: path=/tmp/raft4134343914/snapshots/2-15-1738076095656.tmp +2025-01-28T14:54:55.657Z [INFO] server-6bcf3569-ccc9-970b-3b69-5d4fa7c89fdf: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:55.658Z [INFO] server-6bcf3569-ccc9-970b-3b69-5d4fa7c89fdf: copied to local snapshot: bytes=71 +2025-01-28T14:54:55.658Z [ERROR] server-0d9871ee-1c3d-4090-9e42-bd029d482157: failed to get log: index=2 error="log not found" +2025-01-28T14:54:55.659Z [INFO] creating new snapshot: path=/tmp/raft4059423763/snapshots/2-15-1738076095659.tmp +2025-01-28T14:54:55.659Z [INFO] server-6bcf3569-ccc9-970b-3b69-5d4fa7c89fdf: snapshot restore progress: id=2-15-1738076095656 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:55.659Z [INFO] server-6bcf3569-ccc9-970b-3b69-5d4fa7c89fdf: removing all old logs from log store +2025-01-28T14:54:55.659Z [INFO] server-6bcf3569-ccc9-970b-3b69-5d4fa7c89fdf: compacting logs: from=1 to=2 +2025-01-28T14:54:55.659Z [INFO] server-6bcf3569-ccc9-970b-3b69-5d4fa7c89fdf: Installed remote snapshot +2025-01-28T14:54:55.659Z [INFO] server-97b2c5a6-b641-0e49-e151-7932751d5f66: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:55.660Z [INFO] server-0d9871ee-1c3d-4090-9e42-bd029d482157: pipelining replication: peer="{Voter server-6bcf3569-ccc9-970b-3b69-5d4fa7c89fdf 6bcf3569-ccc9-970b-3b69-5d4fa7c89fdf}" +2025-01-28T14:54:55.660Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:54:55.660Z [INFO] server-97b2c5a6-b641-0e49-e151-7932751d5f66: copied to local snapshot: bytes=71 +2025-01-28T14:54:55.661Z [INFO] server-97b2c5a6-b641-0e49-e151-7932751d5f66: snapshot restore progress: id=2-15-1738076095659 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:55.661Z [INFO] server-97b2c5a6-b641-0e49-e151-7932751d5f66: removing all old logs from log store +2025-01-28T14:54:55.661Z [INFO] server-97b2c5a6-b641-0e49-e151-7932751d5f66: compacting logs: from=1 to=2 +2025-01-28T14:54:55.661Z [INFO] server-97b2c5a6-b641-0e49-e151-7932751d5f66: Installed remote snapshot +2025-01-28T14:54:55.661Z [INFO] server-0d9871ee-1c3d-4090-9e42-bd029d482157: pipelining replication: peer="{Voter server-97b2c5a6-b641-0e49-e151-7932751d5f66 97b2c5a6-b641-0e49-e151-7932751d5f66}" +2025-01-28T14:54:56.666Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:54:55.660325258 +0000 UTC m=+81.334815218 (2 nodes), 2.38µs from start of poll, 1.694292205s from cluster start. Timeout at 2025-01-28 14:54:56.665391792 +0000 UTC m=+82.339881772, 1.005066554s after stability +2025-01-28T14:54:56.677Z [INFO] server-ede50f2d-5fb1-af3a-0a15-465889c97889: aborting pipeline replication: peer="{Voter server-20cc3556-5843-71c7-21cc-de081e2e48fa 20cc3556-5843-71c7-21cc-de081e2e48fa}" +2025-01-28T14:54:56.677Z [INFO] server-ede50f2d-5fb1-af3a-0a15-465889c97889: aborting pipeline replication: peer="{Voter server-8812df09-ccb8-6aab-ebb7-734d1399fcdc 8812df09-ccb8-6aab-ebb7-734d1399fcdc}" === RUN TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_false -2025-01-28T14:47:55.523Z [INFO] server-99072298-1ad9-33df-1284-b10c1061da78: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-99072298-1ad9-33df-1284-b10c1061da78 Address:99072298-1ad9-33df-1284-b10c1061da78} {Suffrage:Voter ID:server-527ea787-4f72-17a2-fc7e-7b8b74974f56 Address:527ea787-4f72-17a2-fc7e-7b8b74974f56} {Suffrage:Voter ID:server-ac7419ce-610e-ddda-e2b7-9e2f5abb5bd3 Address:ac7419ce-610e-ddda-e2b7-9e2f5abb5bd3}]" -2025-01-28T14:47:55.523Z [INFO] server-99072298-1ad9-33df-1284-b10c1061da78: entering follower state: follower="Node at 99072298-1ad9-33df-1284-b10c1061da78 [Follower]" leader-address= leader-id= -2025-01-28T14:47:55.523Z [INFO] server-527ea787-4f72-17a2-fc7e-7b8b74974f56: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-99072298-1ad9-33df-1284-b10c1061da78 Address:99072298-1ad9-33df-1284-b10c1061da78} {Suffrage:Voter ID:server-527ea787-4f72-17a2-fc7e-7b8b74974f56 Address:527ea787-4f72-17a2-fc7e-7b8b74974f56} {Suffrage:Voter ID:server-ac7419ce-610e-ddda-e2b7-9e2f5abb5bd3 Address:ac7419ce-610e-ddda-e2b7-9e2f5abb5bd3}]" -2025-01-28T14:47:55.524Z [INFO] server-527ea787-4f72-17a2-fc7e-7b8b74974f56: entering follower state: follower="Node at 527ea787-4f72-17a2-fc7e-7b8b74974f56 [Follower]" leader-address= leader-id= -2025-01-28T14:47:55.524Z [INFO] server-ac7419ce-610e-ddda-e2b7-9e2f5abb5bd3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-99072298-1ad9-33df-1284-b10c1061da78 Address:99072298-1ad9-33df-1284-b10c1061da78} {Suffrage:Voter ID:server-527ea787-4f72-17a2-fc7e-7b8b74974f56 Address:527ea787-4f72-17a2-fc7e-7b8b74974f56} {Suffrage:Voter ID:server-ac7419ce-610e-ddda-e2b7-9e2f5abb5bd3 Address:ac7419ce-610e-ddda-e2b7-9e2f5abb5bd3}]" -2025-01-28T14:47:55.524Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:55.524Z [INFO] server-ac7419ce-610e-ddda-e2b7-9e2f5abb5bd3: entering follower state: follower="Node at ac7419ce-610e-ddda-e2b7-9e2f5abb5bd3 [Follower]" leader-address= leader-id= -2025-01-28T14:47:56.075Z [WARN] server-527ea787-4f72-17a2-fc7e-7b8b74974f56: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:56.075Z [INFO] server-527ea787-4f72-17a2-fc7e-7b8b74974f56: entering candidate state: node="Node at 527ea787-4f72-17a2-fc7e-7b8b74974f56 [Candidate]" term=2 -2025-01-28T14:47:56.076Z [INFO] server-527ea787-4f72-17a2-fc7e-7b8b74974f56: election won: term=2 tally=2 -2025-01-28T14:47:56.076Z [INFO] server-527ea787-4f72-17a2-fc7e-7b8b74974f56: entering leader state: leader="Node at 527ea787-4f72-17a2-fc7e-7b8b74974f56 [Leader]" -2025-01-28T14:47:56.076Z [INFO] server-527ea787-4f72-17a2-fc7e-7b8b74974f56: added peer, starting replication: peer=server-99072298-1ad9-33df-1284-b10c1061da78 -2025-01-28T14:47:56.076Z [INFO] server-527ea787-4f72-17a2-fc7e-7b8b74974f56: added peer, starting replication: peer=server-ac7419ce-610e-ddda-e2b7-9e2f5abb5bd3 -2025-01-28T14:47:56.077Z [INFO] server-527ea787-4f72-17a2-fc7e-7b8b74974f56: pipelining replication: peer="{Voter server-99072298-1ad9-33df-1284-b10c1061da78 99072298-1ad9-33df-1284-b10c1061da78}" -2025-01-28T14:47:56.077Z [INFO] server-527ea787-4f72-17a2-fc7e-7b8b74974f56: pipelining replication: peer="{Voter server-ac7419ce-610e-ddda-e2b7-9e2f5abb5bd3 ac7419ce-610e-ddda-e2b7-9e2f5abb5bd3}" -2025-01-28T14:47:57.082Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:56.076457444 +0000 UTC m=+59.159862468 (1 nodes), 551.515376ms from start of poll, 554.153121ms from cluster start. Timeout at 2025-01-28 14:47:57.081470111 +0000 UTC m=+60.164874802, 1.005012334s after stability -2025-01-28T14:47:57.086Z [INFO] server-527ea787-4f72-17a2-fc7e-7b8b74974f56: starting snapshot up to: index=12 -2025-01-28T14:47:57.087Z [INFO] creating new snapshot: path=/tmp/raft3769061558/snapshots/2-12-1738075677087.tmp -2025-01-28T14:47:57.090Z [INFO] server-527ea787-4f72-17a2-fc7e-7b8b74974f56: snapshot complete up to: index=12 -2025-01-28T14:47:57.101Z [INFO] creating new snapshot: path=/tmp/raft3769061558/snapshots/2-113-1738075677101.tmp -2025-01-28T14:47:57.106Z [INFO] server-527ea787-4f72-17a2-fc7e-7b8b74974f56: copied to local snapshot: bytes=71 -2025-01-28T14:47:57.108Z [INFO] server-527ea787-4f72-17a2-fc7e-7b8b74974f56: snapshot restore progress: id=2-113-1738075677101 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:57.108Z [INFO] server-527ea787-4f72-17a2-fc7e-7b8b74974f56: restored user snapshot: index=113 -2025-01-28T14:47:57.108Z [ERROR] server-527ea787-4f72-17a2-fc7e-7b8b74974f56: failed to get log: index=23 error="log not found" -2025-01-28T14:47:57.108Z [INFO] server-527ea787-4f72-17a2-fc7e-7b8b74974f56: aborting pipeline replication: peer="{Voter server-99072298-1ad9-33df-1284-b10c1061da78 99072298-1ad9-33df-1284-b10c1061da78}" -2025-01-28T14:47:57.108Z [ERROR] server-527ea787-4f72-17a2-fc7e-7b8b74974f56: failed to get log: index=23 error="log not found" -2025-01-28T14:47:57.109Z [INFO] server-527ea787-4f72-17a2-fc7e-7b8b74974f56: aborting pipeline replication: peer="{Voter server-ac7419ce-610e-ddda-e2b7-9e2f5abb5bd3 ac7419ce-610e-ddda-e2b7-9e2f5abb5bd3}" -2025-01-28T14:47:57.118Z [ERROR] server-527ea787-4f72-17a2-fc7e-7b8b74974f56: failed to get log: index=23 error="log not found" -2025-01-28T14:47:57.119Z [ERROR] server-527ea787-4f72-17a2-fc7e-7b8b74974f56: failed to get log: index=23 error="log not found" -2025-01-28T14:47:57.136Z [INFO] creating new snapshot: path=/tmp/raft1917326730/snapshots/2-113-1738075677136.tmp -2025-01-28T14:47:57.139Z [INFO] server-99072298-1ad9-33df-1284-b10c1061da78: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:57.140Z [INFO] creating new snapshot: path=/tmp/raft4064220794/snapshots/2-113-1738075677140.tmp -2025-01-28T14:47:57.142Z [INFO] server-ac7419ce-610e-ddda-e2b7-9e2f5abb5bd3: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:57.142Z [INFO] server-99072298-1ad9-33df-1284-b10c1061da78: copied to local snapshot: bytes=71 -2025-01-28T14:47:57.144Z [INFO] server-99072298-1ad9-33df-1284-b10c1061da78: snapshot restore progress: id=2-113-1738075677136 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:57.144Z [INFO] server-99072298-1ad9-33df-1284-b10c1061da78: Installed remote snapshot -2025-01-28T14:47:57.144Z [INFO] server-ac7419ce-610e-ddda-e2b7-9e2f5abb5bd3: copied to local snapshot: bytes=71 -2025-01-28T14:47:57.145Z [INFO] server-527ea787-4f72-17a2-fc7e-7b8b74974f56: pipelining replication: peer="{Voter server-99072298-1ad9-33df-1284-b10c1061da78 99072298-1ad9-33df-1284-b10c1061da78}" -2025-01-28T14:47:57.146Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:47:57.146Z [INFO] server-ac7419ce-610e-ddda-e2b7-9e2f5abb5bd3: snapshot restore progress: id=2-113-1738075677140 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:57.146Z [INFO] server-ac7419ce-610e-ddda-e2b7-9e2f5abb5bd3: Installed remote snapshot -2025-01-28T14:47:57.147Z [INFO] server-527ea787-4f72-17a2-fc7e-7b8b74974f56: pipelining replication: peer="{Voter server-ac7419ce-610e-ddda-e2b7-9e2f5abb5bd3 ac7419ce-610e-ddda-e2b7-9e2f5abb5bd3}" -2025-01-28T14:47:58.152Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:47:57.146333545 +0000 UTC m=+60.229738236 (2 nodes), 8µs from start of poll, 1.624028889s from cluster start. Timeout at 2025-01-28 14:47:58.151634221 +0000 UTC m=+61.235040579, 1.005302343s after stability -2025-01-28T14:47:58.169Z [INFO] server-527ea787-4f72-17a2-fc7e-7b8b74974f56: aborting pipeline replication: peer="{Voter server-99072298-1ad9-33df-1284-b10c1061da78 99072298-1ad9-33df-1284-b10c1061da78}" -2025-01-28T14:47:58.169Z [INFO] server-527ea787-4f72-17a2-fc7e-7b8b74974f56: aborting pipeline replication: peer="{Voter server-ac7419ce-610e-ddda-e2b7-9e2f5abb5bd3 ac7419ce-610e-ddda-e2b7-9e2f5abb5bd3}" +2025-01-28T14:54:56.681Z [INFO] server-0e1f90fe-8b5c-6c34-fe80-c0dc24a42f64: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0e1f90fe-8b5c-6c34-fe80-c0dc24a42f64 Address:0e1f90fe-8b5c-6c34-fe80-c0dc24a42f64} {Suffrage:Voter ID:server-9a11845a-33b0-4ffa-107c-4ee13bafdae3 Address:9a11845a-33b0-4ffa-107c-4ee13bafdae3} {Suffrage:Voter ID:server-df0b2715-797e-cfb5-76ad-eaffe3e65769 Address:df0b2715-797e-cfb5-76ad-eaffe3e65769}]" +2025-01-28T14:54:56.682Z [INFO] server-0e1f90fe-8b5c-6c34-fe80-c0dc24a42f64: entering follower state: follower="Node at 0e1f90fe-8b5c-6c34-fe80-c0dc24a42f64 [Follower]" leader-address= leader-id= +2025-01-28T14:54:56.682Z [INFO] server-9a11845a-33b0-4ffa-107c-4ee13bafdae3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0e1f90fe-8b5c-6c34-fe80-c0dc24a42f64 Address:0e1f90fe-8b5c-6c34-fe80-c0dc24a42f64} {Suffrage:Voter ID:server-9a11845a-33b0-4ffa-107c-4ee13bafdae3 Address:9a11845a-33b0-4ffa-107c-4ee13bafdae3} {Suffrage:Voter ID:server-df0b2715-797e-cfb5-76ad-eaffe3e65769 Address:df0b2715-797e-cfb5-76ad-eaffe3e65769}]" +2025-01-28T14:54:56.682Z [INFO] server-9a11845a-33b0-4ffa-107c-4ee13bafdae3: entering follower state: follower="Node at 9a11845a-33b0-4ffa-107c-4ee13bafdae3 [Follower]" leader-address= leader-id= +2025-01-28T14:54:56.682Z [INFO] server-df0b2715-797e-cfb5-76ad-eaffe3e65769: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0e1f90fe-8b5c-6c34-fe80-c0dc24a42f64 Address:0e1f90fe-8b5c-6c34-fe80-c0dc24a42f64} {Suffrage:Voter ID:server-9a11845a-33b0-4ffa-107c-4ee13bafdae3 Address:9a11845a-33b0-4ffa-107c-4ee13bafdae3} {Suffrage:Voter ID:server-df0b2715-797e-cfb5-76ad-eaffe3e65769 Address:df0b2715-797e-cfb5-76ad-eaffe3e65769}]" +2025-01-28T14:54:56.682Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:56.682Z [INFO] server-df0b2715-797e-cfb5-76ad-eaffe3e65769: entering follower state: follower="Node at df0b2715-797e-cfb5-76ad-eaffe3e65769 [Follower]" leader-address= leader-id= +2025-01-28T14:54:57.548Z [WARN] server-9a11845a-33b0-4ffa-107c-4ee13bafdae3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:54:57.548Z [INFO] server-9a11845a-33b0-4ffa-107c-4ee13bafdae3: entering candidate state: node="Node at 9a11845a-33b0-4ffa-107c-4ee13bafdae3 [Candidate]" term=2 +2025-01-28T14:54:57.548Z [INFO] server-9a11845a-33b0-4ffa-107c-4ee13bafdae3: election won: term=2 tally=2 +2025-01-28T14:54:57.548Z [INFO] server-9a11845a-33b0-4ffa-107c-4ee13bafdae3: entering leader state: leader="Node at 9a11845a-33b0-4ffa-107c-4ee13bafdae3 [Leader]" +2025-01-28T14:54:57.548Z [INFO] server-9a11845a-33b0-4ffa-107c-4ee13bafdae3: added peer, starting replication: peer=server-0e1f90fe-8b5c-6c34-fe80-c0dc24a42f64 +2025-01-28T14:54:57.548Z [INFO] server-9a11845a-33b0-4ffa-107c-4ee13bafdae3: added peer, starting replication: peer=server-df0b2715-797e-cfb5-76ad-eaffe3e65769 +2025-01-28T14:54:57.549Z [INFO] server-9a11845a-33b0-4ffa-107c-4ee13bafdae3: pipelining replication: peer="{Voter server-0e1f90fe-8b5c-6c34-fe80-c0dc24a42f64 0e1f90fe-8b5c-6c34-fe80-c0dc24a42f64}" +2025-01-28T14:54:57.549Z [INFO] server-9a11845a-33b0-4ffa-107c-4ee13bafdae3: pipelining replication: peer="{Voter server-df0b2715-797e-cfb5-76ad-eaffe3e65769 df0b2715-797e-cfb5-76ad-eaffe3e65769}" +2025-01-28T14:54:58.554Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:54:57.548868957 +0000 UTC m=+83.223358917 (1 nodes), 866.14141ms from start of poll, 867.140896ms from cluster start. Timeout at 2025-01-28 14:54:58.553877417 +0000 UTC m=+84.228367397, 1.00500848s after stability +2025-01-28T14:54:58.556Z [INFO] server-9a11845a-33b0-4ffa-107c-4ee13bafdae3: starting snapshot up to: index=12 +2025-01-28T14:54:58.556Z [INFO] creating new snapshot: path=/tmp/raft2960184890/snapshots/2-12-1738076098556.tmp +2025-01-28T14:54:58.557Z [INFO] server-9a11845a-33b0-4ffa-107c-4ee13bafdae3: snapshot complete up to: index=12 +2025-01-28T14:54:58.559Z [INFO] creating new snapshot: path=/tmp/raft2960184890/snapshots/2-113-1738076098559.tmp +2025-01-28T14:54:58.560Z [INFO] server-9a11845a-33b0-4ffa-107c-4ee13bafdae3: copied to local snapshot: bytes=71 +2025-01-28T14:54:58.561Z [INFO] server-9a11845a-33b0-4ffa-107c-4ee13bafdae3: snapshot restore progress: id=2-113-1738076098559 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:58.561Z [INFO] server-9a11845a-33b0-4ffa-107c-4ee13bafdae3: restored user snapshot: index=113 +2025-01-28T14:54:58.561Z [ERROR] server-9a11845a-33b0-4ffa-107c-4ee13bafdae3: failed to get log: index=23 error="log not found" +2025-01-28T14:54:58.561Z [INFO] server-9a11845a-33b0-4ffa-107c-4ee13bafdae3: aborting pipeline replication: peer="{Voter server-df0b2715-797e-cfb5-76ad-eaffe3e65769 df0b2715-797e-cfb5-76ad-eaffe3e65769}" +2025-01-28T14:54:58.561Z [ERROR] server-9a11845a-33b0-4ffa-107c-4ee13bafdae3: failed to get log: index=23 error="log not found" +2025-01-28T14:54:58.561Z [INFO] server-9a11845a-33b0-4ffa-107c-4ee13bafdae3: aborting pipeline replication: peer="{Voter server-0e1f90fe-8b5c-6c34-fe80-c0dc24a42f64 0e1f90fe-8b5c-6c34-fe80-c0dc24a42f64}" +2025-01-28T14:54:58.569Z [ERROR] server-9a11845a-33b0-4ffa-107c-4ee13bafdae3: failed to get log: index=23 error="log not found" +2025-01-28T14:54:58.570Z [ERROR] server-9a11845a-33b0-4ffa-107c-4ee13bafdae3: failed to get log: index=23 error="log not found" +2025-01-28T14:54:58.571Z [INFO] creating new snapshot: path=/tmp/raft2530845631/snapshots/2-113-1738076098571.tmp +2025-01-28T14:54:58.573Z [INFO] creating new snapshot: path=/tmp/raft2314347603/snapshots/2-113-1738076098573.tmp +2025-01-28T14:54:58.573Z [INFO] server-0e1f90fe-8b5c-6c34-fe80-c0dc24a42f64: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:58.574Z [INFO] server-df0b2715-797e-cfb5-76ad-eaffe3e65769: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:58.575Z [INFO] server-df0b2715-797e-cfb5-76ad-eaffe3e65769: copied to local snapshot: bytes=71 +2025-01-28T14:54:58.575Z [INFO] server-0e1f90fe-8b5c-6c34-fe80-c0dc24a42f64: copied to local snapshot: bytes=71 +2025-01-28T14:54:58.575Z [INFO] server-df0b2715-797e-cfb5-76ad-eaffe3e65769: snapshot restore progress: id=2-113-1738076098573 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:58.575Z [INFO] server-0e1f90fe-8b5c-6c34-fe80-c0dc24a42f64: snapshot restore progress: id=2-113-1738076098571 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:54:58.575Z [INFO] server-df0b2715-797e-cfb5-76ad-eaffe3e65769: Installed remote snapshot +2025-01-28T14:54:58.575Z [INFO] server-0e1f90fe-8b5c-6c34-fe80-c0dc24a42f64: Installed remote snapshot +2025-01-28T14:54:58.576Z [WARN] server-0e1f90fe-8b5c-6c34-fe80-c0dc24a42f64: skipping application of old log: index=113 +2025-01-28T14:54:58.576Z [WARN] server-df0b2715-797e-cfb5-76ad-eaffe3e65769: skipping application of old log: index=113 +2025-01-28T14:54:58.576Z [INFO] server-9a11845a-33b0-4ffa-107c-4ee13bafdae3: pipelining replication: peer="{Voter server-0e1f90fe-8b5c-6c34-fe80-c0dc24a42f64 0e1f90fe-8b5c-6c34-fe80-c0dc24a42f64}" +2025-01-28T14:54:58.576Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:54:58.576Z [INFO] server-9a11845a-33b0-4ffa-107c-4ee13bafdae3: pipelining replication: peer="{Voter server-df0b2715-797e-cfb5-76ad-eaffe3e65769 df0b2715-797e-cfb5-76ad-eaffe3e65769}" +2025-01-28T14:54:59.583Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:54:58.576446038 +0000 UTC m=+84.250936018 (2 nodes), 2.66µs from start of poll, 1.894717997s from cluster start. Timeout at 2025-01-28 14:54:59.581554068 +0000 UTC m=+85.256044128, 1.00510811s after stability +2025-01-28T14:54:59.599Z [INFO] server-9a11845a-33b0-4ffa-107c-4ee13bafdae3: aborting pipeline replication: peer="{Voter server-df0b2715-797e-cfb5-76ad-eaffe3e65769 df0b2715-797e-cfb5-76ad-eaffe3e65769}" +2025-01-28T14:54:59.599Z [INFO] server-9a11845a-33b0-4ffa-107c-4ee13bafdae3: aborting pipeline replication: peer="{Voter server-0e1f90fe-8b5c-6c34-fe80-c0dc24a42f64 0e1f90fe-8b5c-6c34-fe80-c0dc24a42f64}" === RUN TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_false -2025-01-28T14:47:58.199Z [INFO] server-ee8041f9-8ff0-6c9a-7e70-1e3a6d037379: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ee8041f9-8ff0-6c9a-7e70-1e3a6d037379 Address:ee8041f9-8ff0-6c9a-7e70-1e3a6d037379} {Suffrage:Voter ID:server-f1f21a48-a3f9-eca1-2244-ac9fa37313f8 Address:f1f21a48-a3f9-eca1-2244-ac9fa37313f8} {Suffrage:Voter ID:server-35185b9a-5a01-5523-f0ef-6182e329818a Address:35185b9a-5a01-5523-f0ef-6182e329818a}]" -2025-01-28T14:47:58.199Z [INFO] server-ee8041f9-8ff0-6c9a-7e70-1e3a6d037379: entering follower state: follower="Node at ee8041f9-8ff0-6c9a-7e70-1e3a6d037379 [Follower]" leader-address= leader-id= -2025-01-28T14:47:58.200Z [INFO] server-f1f21a48-a3f9-eca1-2244-ac9fa37313f8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ee8041f9-8ff0-6c9a-7e70-1e3a6d037379 Address:ee8041f9-8ff0-6c9a-7e70-1e3a6d037379} {Suffrage:Voter ID:server-f1f21a48-a3f9-eca1-2244-ac9fa37313f8 Address:f1f21a48-a3f9-eca1-2244-ac9fa37313f8} {Suffrage:Voter ID:server-35185b9a-5a01-5523-f0ef-6182e329818a Address:35185b9a-5a01-5523-f0ef-6182e329818a}]" -2025-01-28T14:47:58.201Z [INFO] server-f1f21a48-a3f9-eca1-2244-ac9fa37313f8: entering follower state: follower="Node at f1f21a48-a3f9-eca1-2244-ac9fa37313f8 [Follower]" leader-address= leader-id= -2025-01-28T14:47:58.201Z [INFO] server-35185b9a-5a01-5523-f0ef-6182e329818a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ee8041f9-8ff0-6c9a-7e70-1e3a6d037379 Address:ee8041f9-8ff0-6c9a-7e70-1e3a6d037379} {Suffrage:Voter ID:server-f1f21a48-a3f9-eca1-2244-ac9fa37313f8 Address:f1f21a48-a3f9-eca1-2244-ac9fa37313f8} {Suffrage:Voter ID:server-35185b9a-5a01-5523-f0ef-6182e329818a Address:35185b9a-5a01-5523-f0ef-6182e329818a}]" -2025-01-28T14:47:58.201Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:47:58.201Z [INFO] server-35185b9a-5a01-5523-f0ef-6182e329818a: entering follower state: follower="Node at 35185b9a-5a01-5523-f0ef-6182e329818a [Follower]" leader-address= leader-id= -2025-01-28T14:47:58.779Z [WARN] server-f1f21a48-a3f9-eca1-2244-ac9fa37313f8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:47:58.779Z [INFO] server-f1f21a48-a3f9-eca1-2244-ac9fa37313f8: entering candidate state: node="Node at f1f21a48-a3f9-eca1-2244-ac9fa37313f8 [Candidate]" term=2 -2025-01-28T14:47:58.780Z [INFO] server-f1f21a48-a3f9-eca1-2244-ac9fa37313f8: election won: term=2 tally=2 -2025-01-28T14:47:58.780Z [INFO] server-f1f21a48-a3f9-eca1-2244-ac9fa37313f8: entering leader state: leader="Node at f1f21a48-a3f9-eca1-2244-ac9fa37313f8 [Leader]" -2025-01-28T14:47:58.780Z [INFO] server-f1f21a48-a3f9-eca1-2244-ac9fa37313f8: added peer, starting replication: peer=server-ee8041f9-8ff0-6c9a-7e70-1e3a6d037379 -2025-01-28T14:47:58.781Z [INFO] server-f1f21a48-a3f9-eca1-2244-ac9fa37313f8: added peer, starting replication: peer=server-35185b9a-5a01-5523-f0ef-6182e329818a -2025-01-28T14:47:58.785Z [INFO] server-f1f21a48-a3f9-eca1-2244-ac9fa37313f8: pipelining replication: peer="{Voter server-ee8041f9-8ff0-6c9a-7e70-1e3a6d037379 ee8041f9-8ff0-6c9a-7e70-1e3a6d037379}" -2025-01-28T14:47:58.787Z [INFO] server-f1f21a48-a3f9-eca1-2244-ac9fa37313f8: pipelining replication: peer="{Voter server-35185b9a-5a01-5523-f0ef-6182e329818a 35185b9a-5a01-5523-f0ef-6182e329818a}" -2025-01-28T14:47:59.787Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:47:58.78120209 +0000 UTC m=+61.864607114 (1 nodes), 579.392871ms from start of poll, 582.360625ms from cluster start. Timeout at 2025-01-28 14:47:59.786222091 +0000 UTC m=+62.869626782, 1.005019668s after stability -2025-01-28T14:47:59.793Z [INFO] server-f1f21a48-a3f9-eca1-2244-ac9fa37313f8: starting snapshot up to: index=12 -2025-01-28T14:47:59.793Z [INFO] creating new snapshot: path=/tmp/raft3157924745/snapshots/2-12-1738075679793.tmp -2025-01-28T14:47:59.797Z [INFO] server-f1f21a48-a3f9-eca1-2244-ac9fa37313f8: snapshot complete up to: index=12 -2025-01-28T14:47:59.804Z [INFO] creating new snapshot: path=/tmp/raft3157924745/snapshots/2-113-1738075679804.tmp -2025-01-28T14:47:59.808Z [INFO] server-f1f21a48-a3f9-eca1-2244-ac9fa37313f8: copied to local snapshot: bytes=71 -2025-01-28T14:47:59.810Z [INFO] server-f1f21a48-a3f9-eca1-2244-ac9fa37313f8: snapshot restore progress: id=2-113-1738075679804 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:59.810Z [INFO] server-f1f21a48-a3f9-eca1-2244-ac9fa37313f8: removing all old logs from log store -2025-01-28T14:47:59.810Z [INFO] server-f1f21a48-a3f9-eca1-2244-ac9fa37313f8: compacting logs: from=1 to=22 -2025-01-28T14:47:59.810Z [INFO] server-f1f21a48-a3f9-eca1-2244-ac9fa37313f8: restored user snapshot: index=113 -2025-01-28T14:47:59.810Z [ERROR] server-f1f21a48-a3f9-eca1-2244-ac9fa37313f8: failed to get log: index=22 error="log not found" -2025-01-28T14:47:59.810Z [ERROR] server-f1f21a48-a3f9-eca1-2244-ac9fa37313f8: failed to get log: index=22 error="log not found" -2025-01-28T14:47:59.810Z [INFO] server-f1f21a48-a3f9-eca1-2244-ac9fa37313f8: aborting pipeline replication: peer="{Voter server-ee8041f9-8ff0-6c9a-7e70-1e3a6d037379 ee8041f9-8ff0-6c9a-7e70-1e3a6d037379}" -2025-01-28T14:47:59.810Z [INFO] server-f1f21a48-a3f9-eca1-2244-ac9fa37313f8: aborting pipeline replication: peer="{Voter server-35185b9a-5a01-5523-f0ef-6182e329818a 35185b9a-5a01-5523-f0ef-6182e329818a}" -2025-01-28T14:47:59.818Z [ERROR] server-f1f21a48-a3f9-eca1-2244-ac9fa37313f8: failed to get log: index=22 error="log not found" -2025-01-28T14:47:59.819Z [ERROR] server-f1f21a48-a3f9-eca1-2244-ac9fa37313f8: failed to get log: index=22 error="log not found" -2025-01-28T14:47:59.823Z [INFO] creating new snapshot: path=/tmp/raft3670346112/snapshots/2-113-1738075679823.tmp -2025-01-28T14:47:59.829Z [INFO] creating new snapshot: path=/tmp/raft1653024333/snapshots/2-113-1738075679829.tmp -2025-01-28T14:47:59.836Z [INFO] server-35185b9a-5a01-5523-f0ef-6182e329818a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:59.839Z [INFO] server-35185b9a-5a01-5523-f0ef-6182e329818a: copied to local snapshot: bytes=71 -2025-01-28T14:47:59.841Z [INFO] server-35185b9a-5a01-5523-f0ef-6182e329818a: snapshot restore progress: id=2-113-1738075679823 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:59.841Z [INFO] server-35185b9a-5a01-5523-f0ef-6182e329818a: removing all old logs from log store -2025-01-28T14:47:59.841Z [INFO] server-35185b9a-5a01-5523-f0ef-6182e329818a: compacting logs: from=1 to=22 -2025-01-28T14:47:59.841Z [INFO] server-35185b9a-5a01-5523-f0ef-6182e329818a: Installed remote snapshot -2025-01-28T14:47:59.842Z [INFO] server-f1f21a48-a3f9-eca1-2244-ac9fa37313f8: pipelining replication: peer="{Voter server-35185b9a-5a01-5523-f0ef-6182e329818a 35185b9a-5a01-5523-f0ef-6182e329818a}" -2025-01-28T14:47:59.843Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:47:59.847Z [INFO] server-ee8041f9-8ff0-6c9a-7e70-1e3a6d037379: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:59.849Z [INFO] server-ee8041f9-8ff0-6c9a-7e70-1e3a6d037379: copied to local snapshot: bytes=71 -2025-01-28T14:47:59.852Z [INFO] server-ee8041f9-8ff0-6c9a-7e70-1e3a6d037379: snapshot restore progress: id=2-113-1738075679829 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:47:59.852Z [INFO] server-ee8041f9-8ff0-6c9a-7e70-1e3a6d037379: removing all old logs from log store -2025-01-28T14:47:59.852Z [INFO] server-ee8041f9-8ff0-6c9a-7e70-1e3a6d037379: compacting logs: from=1 to=22 -2025-01-28T14:47:59.852Z [INFO] server-ee8041f9-8ff0-6c9a-7e70-1e3a6d037379: Installed remote snapshot -2025-01-28T14:47:59.853Z [INFO] server-f1f21a48-a3f9-eca1-2244-ac9fa37313f8: pipelining replication: peer="{Voter server-ee8041f9-8ff0-6c9a-7e70-1e3a6d037379 ee8041f9-8ff0-6c9a-7e70-1e3a6d037379}" -2025-01-28T14:48:00.848Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:47:59.843199624 +0000 UTC m=+62.926604315 (2 nodes), 6.666µs from start of poll, 1.644357826s from cluster start. Timeout at 2025-01-28 14:48:00.848474632 +0000 UTC m=+63.931879657, 1.005275342s after stability -2025-01-28T14:48:00.867Z [INFO] server-f1f21a48-a3f9-eca1-2244-ac9fa37313f8: aborting pipeline replication: peer="{Voter server-35185b9a-5a01-5523-f0ef-6182e329818a 35185b9a-5a01-5523-f0ef-6182e329818a}" -2025-01-28T14:48:00.867Z [INFO] server-f1f21a48-a3f9-eca1-2244-ac9fa37313f8: aborting pipeline replication: peer="{Voter server-ee8041f9-8ff0-6c9a-7e70-1e3a6d037379 ee8041f9-8ff0-6c9a-7e70-1e3a6d037379}" +2025-01-28T14:54:59.604Z [INFO] server-45c840c3-9891-b0c0-25ca-e35fd62c4468: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-45c840c3-9891-b0c0-25ca-e35fd62c4468 Address:45c840c3-9891-b0c0-25ca-e35fd62c4468} {Suffrage:Voter ID:server-d74f6f89-3eee-9ea4-f555-95b6e56918dd Address:d74f6f89-3eee-9ea4-f555-95b6e56918dd} {Suffrage:Voter ID:server-ed822476-b77f-a013-afa7-5c4c70037fe4 Address:ed822476-b77f-a013-afa7-5c4c70037fe4}]" +2025-01-28T14:54:59.604Z [INFO] server-45c840c3-9891-b0c0-25ca-e35fd62c4468: entering follower state: follower="Node at 45c840c3-9891-b0c0-25ca-e35fd62c4468 [Follower]" leader-address= leader-id= +2025-01-28T14:54:59.604Z [INFO] server-d74f6f89-3eee-9ea4-f555-95b6e56918dd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-45c840c3-9891-b0c0-25ca-e35fd62c4468 Address:45c840c3-9891-b0c0-25ca-e35fd62c4468} {Suffrage:Voter ID:server-d74f6f89-3eee-9ea4-f555-95b6e56918dd Address:d74f6f89-3eee-9ea4-f555-95b6e56918dd} {Suffrage:Voter ID:server-ed822476-b77f-a013-afa7-5c4c70037fe4 Address:ed822476-b77f-a013-afa7-5c4c70037fe4}]" +2025-01-28T14:54:59.605Z [INFO] server-d74f6f89-3eee-9ea4-f555-95b6e56918dd: entering follower state: follower="Node at d74f6f89-3eee-9ea4-f555-95b6e56918dd [Follower]" leader-address= leader-id= +2025-01-28T14:54:59.605Z [INFO] server-ed822476-b77f-a013-afa7-5c4c70037fe4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-45c840c3-9891-b0c0-25ca-e35fd62c4468 Address:45c840c3-9891-b0c0-25ca-e35fd62c4468} {Suffrage:Voter ID:server-d74f6f89-3eee-9ea4-f555-95b6e56918dd Address:d74f6f89-3eee-9ea4-f555-95b6e56918dd} {Suffrage:Voter ID:server-ed822476-b77f-a013-afa7-5c4c70037fe4 Address:ed822476-b77f-a013-afa7-5c4c70037fe4}]" +2025-01-28T14:54:59.605Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:54:59.605Z [INFO] server-ed822476-b77f-a013-afa7-5c4c70037fe4: entering follower state: follower="Node at ed822476-b77f-a013-afa7-5c4c70037fe4 [Follower]" leader-address= leader-id= +2025-01-28T14:55:00.175Z [WARN] server-45c840c3-9891-b0c0-25ca-e35fd62c4468: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:00.175Z [INFO] server-45c840c3-9891-b0c0-25ca-e35fd62c4468: entering candidate state: node="Node at 45c840c3-9891-b0c0-25ca-e35fd62c4468 [Candidate]" term=2 +2025-01-28T14:55:00.175Z [INFO] server-45c840c3-9891-b0c0-25ca-e35fd62c4468: election won: term=2 tally=2 +2025-01-28T14:55:00.175Z [INFO] server-45c840c3-9891-b0c0-25ca-e35fd62c4468: entering leader state: leader="Node at 45c840c3-9891-b0c0-25ca-e35fd62c4468 [Leader]" +2025-01-28T14:55:00.175Z [INFO] server-45c840c3-9891-b0c0-25ca-e35fd62c4468: added peer, starting replication: peer=server-d74f6f89-3eee-9ea4-f555-95b6e56918dd +2025-01-28T14:55:00.175Z [INFO] server-45c840c3-9891-b0c0-25ca-e35fd62c4468: added peer, starting replication: peer=server-ed822476-b77f-a013-afa7-5c4c70037fe4 +2025-01-28T14:55:00.176Z [INFO] server-45c840c3-9891-b0c0-25ca-e35fd62c4468: pipelining replication: peer="{Voter server-ed822476-b77f-a013-afa7-5c4c70037fe4 ed822476-b77f-a013-afa7-5c4c70037fe4}" +2025-01-28T14:55:00.176Z [INFO] server-45c840c3-9891-b0c0-25ca-e35fd62c4468: pipelining replication: peer="{Voter server-d74f6f89-3eee-9ea4-f555-95b6e56918dd d74f6f89-3eee-9ea4-f555-95b6e56918dd}" +2025-01-28T14:55:01.181Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:00.175916008 +0000 UTC m=+85.850405988 (1 nodes), 570.580853ms from start of poll, 571.650732ms from cluster start. Timeout at 2025-01-28 14:55:01.180919808 +0000 UTC m=+86.855409808, 1.00500382s after stability +2025-01-28T14:55:01.182Z [INFO] server-45c840c3-9891-b0c0-25ca-e35fd62c4468: starting snapshot up to: index=12 +2025-01-28T14:55:01.182Z [INFO] creating new snapshot: path=/tmp/raft3954512927/snapshots/2-12-1738076101182.tmp +2025-01-28T14:55:01.184Z [INFO] server-45c840c3-9891-b0c0-25ca-e35fd62c4468: snapshot complete up to: index=12 +2025-01-28T14:55:01.185Z [INFO] creating new snapshot: path=/tmp/raft3954512927/snapshots/2-113-1738076101185.tmp +2025-01-28T14:55:01.187Z [INFO] server-45c840c3-9891-b0c0-25ca-e35fd62c4468: copied to local snapshot: bytes=71 +2025-01-28T14:55:01.187Z [INFO] server-45c840c3-9891-b0c0-25ca-e35fd62c4468: snapshot restore progress: id=2-113-1738076101185 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:01.188Z [INFO] server-45c840c3-9891-b0c0-25ca-e35fd62c4468: removing all old logs from log store +2025-01-28T14:55:01.188Z [INFO] server-45c840c3-9891-b0c0-25ca-e35fd62c4468: compacting logs: from=1 to=22 +2025-01-28T14:55:01.188Z [INFO] server-45c840c3-9891-b0c0-25ca-e35fd62c4468: restored user snapshot: index=113 +2025-01-28T14:55:01.188Z [ERROR] server-45c840c3-9891-b0c0-25ca-e35fd62c4468: failed to get log: index=22 error="log not found" +2025-01-28T14:55:01.188Z [ERROR] server-45c840c3-9891-b0c0-25ca-e35fd62c4468: failed to get log: index=22 error="log not found" +2025-01-28T14:55:01.188Z [INFO] server-45c840c3-9891-b0c0-25ca-e35fd62c4468: aborting pipeline replication: peer="{Voter server-d74f6f89-3eee-9ea4-f555-95b6e56918dd d74f6f89-3eee-9ea4-f555-95b6e56918dd}" +2025-01-28T14:55:01.188Z [INFO] server-45c840c3-9891-b0c0-25ca-e35fd62c4468: aborting pipeline replication: peer="{Voter server-ed822476-b77f-a013-afa7-5c4c70037fe4 ed822476-b77f-a013-afa7-5c4c70037fe4}" +2025-01-28T14:55:01.196Z [ERROR] server-45c840c3-9891-b0c0-25ca-e35fd62c4468: failed to get log: index=22 error="log not found" +2025-01-28T14:55:01.197Z [INFO] creating new snapshot: path=/tmp/raft222557369/snapshots/2-113-1738076101197.tmp +2025-01-28T14:55:01.197Z [INFO] server-ed822476-b77f-a013-afa7-5c4c70037fe4: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:01.198Z [ERROR] server-45c840c3-9891-b0c0-25ca-e35fd62c4468: failed to get log: index=22 error="log not found" +2025-01-28T14:55:01.198Z [INFO] server-ed822476-b77f-a013-afa7-5c4c70037fe4: copied to local snapshot: bytes=71 +2025-01-28T14:55:01.199Z [INFO] server-ed822476-b77f-a013-afa7-5c4c70037fe4: snapshot restore progress: id=2-113-1738076101197 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:01.199Z [INFO] creating new snapshot: path=/tmp/raft2543610462/snapshots/2-113-1738076101199.tmp +2025-01-28T14:55:01.199Z [INFO] server-ed822476-b77f-a013-afa7-5c4c70037fe4: removing all old logs from log store +2025-01-28T14:55:01.199Z [INFO] server-ed822476-b77f-a013-afa7-5c4c70037fe4: compacting logs: from=1 to=22 +2025-01-28T14:55:01.199Z [INFO] server-ed822476-b77f-a013-afa7-5c4c70037fe4: Installed remote snapshot +2025-01-28T14:55:01.199Z [WARN] server-ed822476-b77f-a013-afa7-5c4c70037fe4: skipping application of old log: index=22 +2025-01-28T14:55:01.199Z [INFO] server-45c840c3-9891-b0c0-25ca-e35fd62c4468: pipelining replication: peer="{Voter server-ed822476-b77f-a013-afa7-5c4c70037fe4 ed822476-b77f-a013-afa7-5c4c70037fe4}" +2025-01-28T14:55:01.199Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:55:01.199Z [INFO] server-d74f6f89-3eee-9ea4-f555-95b6e56918dd: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:01.200Z [INFO] server-d74f6f89-3eee-9ea4-f555-95b6e56918dd: copied to local snapshot: bytes=71 +2025-01-28T14:55:01.201Z [INFO] server-d74f6f89-3eee-9ea4-f555-95b6e56918dd: snapshot restore progress: id=2-113-1738076101199 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:01.201Z [INFO] server-d74f6f89-3eee-9ea4-f555-95b6e56918dd: removing all old logs from log store +2025-01-28T14:55:01.201Z [INFO] server-d74f6f89-3eee-9ea4-f555-95b6e56918dd: compacting logs: from=1 to=22 +2025-01-28T14:55:01.201Z [INFO] server-d74f6f89-3eee-9ea4-f555-95b6e56918dd: Installed remote snapshot +2025-01-28T14:55:01.201Z [INFO] server-45c840c3-9891-b0c0-25ca-e35fd62c4468: pipelining replication: peer="{Voter server-d74f6f89-3eee-9ea4-f555-95b6e56918dd d74f6f89-3eee-9ea4-f555-95b6e56918dd}" +2025-01-28T14:55:02.206Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:55:01.19997926 +0000 UTC m=+86.874469220 (2 nodes), 2.319µs from start of poll, 1.595713964s from cluster start. Timeout at 2025-01-28 14:55:02.205064532 +0000 UTC m=+87.879554592, 1.005085372s after stability +2025-01-28T14:55:02.218Z [INFO] server-45c840c3-9891-b0c0-25ca-e35fd62c4468: aborting pipeline replication: peer="{Voter server-ed822476-b77f-a013-afa7-5c4c70037fe4 ed822476-b77f-a013-afa7-5c4c70037fe4}" +2025-01-28T14:55:02.218Z [INFO] server-45c840c3-9891-b0c0-25ca-e35fd62c4468: aborting pipeline replication: peer="{Voter server-d74f6f89-3eee-9ea4-f555-95b6e56918dd d74f6f89-3eee-9ea4-f555-95b6e56918dd}" === RUN TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_true -2025-01-28T14:48:00.879Z [INFO] server-1cf17c00-771a-8084-cb31-ef3934ccad69: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1cf17c00-771a-8084-cb31-ef3934ccad69 Address:1cf17c00-771a-8084-cb31-ef3934ccad69} {Suffrage:Voter ID:server-3068c999-712b-9dd4-25d1-ad785e71f54a Address:3068c999-712b-9dd4-25d1-ad785e71f54a} {Suffrage:Voter ID:server-91815518-0e58-36e5-ac77-c6d606e0eec1 Address:91815518-0e58-36e5-ac77-c6d606e0eec1}]" -2025-01-28T14:48:00.880Z [INFO] server-1cf17c00-771a-8084-cb31-ef3934ccad69: entering follower state: follower="Node at 1cf17c00-771a-8084-cb31-ef3934ccad69 [Follower]" leader-address= leader-id= -2025-01-28T14:48:00.880Z [INFO] server-3068c999-712b-9dd4-25d1-ad785e71f54a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1cf17c00-771a-8084-cb31-ef3934ccad69 Address:1cf17c00-771a-8084-cb31-ef3934ccad69} {Suffrage:Voter ID:server-3068c999-712b-9dd4-25d1-ad785e71f54a Address:3068c999-712b-9dd4-25d1-ad785e71f54a} {Suffrage:Voter ID:server-91815518-0e58-36e5-ac77-c6d606e0eec1 Address:91815518-0e58-36e5-ac77-c6d606e0eec1}]" -2025-01-28T14:48:00.881Z [INFO] server-91815518-0e58-36e5-ac77-c6d606e0eec1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1cf17c00-771a-8084-cb31-ef3934ccad69 Address:1cf17c00-771a-8084-cb31-ef3934ccad69} {Suffrage:Voter ID:server-3068c999-712b-9dd4-25d1-ad785e71f54a Address:3068c999-712b-9dd4-25d1-ad785e71f54a} {Suffrage:Voter ID:server-91815518-0e58-36e5-ac77-c6d606e0eec1 Address:91815518-0e58-36e5-ac77-c6d606e0eec1}]" -2025-01-28T14:48:00.881Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:00.881Z [INFO] server-3068c999-712b-9dd4-25d1-ad785e71f54a: entering follower state: follower="Node at 3068c999-712b-9dd4-25d1-ad785e71f54a [Follower]" leader-address= leader-id= -2025-01-28T14:48:00.881Z [INFO] server-91815518-0e58-36e5-ac77-c6d606e0eec1: entering follower state: follower="Node at 91815518-0e58-36e5-ac77-c6d606e0eec1 [Follower]" leader-address= leader-id= -2025-01-28T14:48:01.414Z [WARN] server-91815518-0e58-36e5-ac77-c6d606e0eec1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:01.414Z [INFO] server-91815518-0e58-36e5-ac77-c6d606e0eec1: entering candidate state: node="Node at 91815518-0e58-36e5-ac77-c6d606e0eec1 [Candidate]" term=2 -2025-01-28T14:48:01.415Z [INFO] server-91815518-0e58-36e5-ac77-c6d606e0eec1: election won: term=2 tally=2 -2025-01-28T14:48:01.415Z [INFO] server-91815518-0e58-36e5-ac77-c6d606e0eec1: entering leader state: leader="Node at 91815518-0e58-36e5-ac77-c6d606e0eec1 [Leader]" -2025-01-28T14:48:01.415Z [INFO] server-91815518-0e58-36e5-ac77-c6d606e0eec1: added peer, starting replication: peer=server-1cf17c00-771a-8084-cb31-ef3934ccad69 -2025-01-28T14:48:01.415Z [INFO] server-91815518-0e58-36e5-ac77-c6d606e0eec1: added peer, starting replication: peer=server-3068c999-712b-9dd4-25d1-ad785e71f54a -2025-01-28T14:48:01.416Z [INFO] server-91815518-0e58-36e5-ac77-c6d606e0eec1: pipelining replication: peer="{Voter server-3068c999-712b-9dd4-25d1-ad785e71f54a 3068c999-712b-9dd4-25d1-ad785e71f54a}" -2025-01-28T14:48:01.417Z [INFO] server-91815518-0e58-36e5-ac77-c6d606e0eec1: pipelining replication: peer="{Voter server-1cf17c00-771a-8084-cb31-ef3934ccad69 1cf17c00-771a-8084-cb31-ef3934ccad69}" -2025-01-28T14:48:02.427Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:01.416402005 +0000 UTC m=+64.499806696 (1 nodes), 534.985885ms from start of poll, 537.103948ms from cluster start. Timeout at 2025-01-28 14:48:02.421487341 +0000 UTC m=+65.504893032, 1.005086336s after stability -2025-01-28T14:48:02.441Z [INFO] server-91815518-0e58-36e5-ac77-c6d606e0eec1: starting snapshot up to: index=12 -2025-01-28T14:48:02.441Z [INFO] creating new snapshot: path=/tmp/raft2173949338/snapshots/2-12-1738075682441.tmp -2025-01-28T14:48:02.448Z [INFO] server-91815518-0e58-36e5-ac77-c6d606e0eec1: snapshot complete up to: index=12 -2025-01-28T14:48:02.481Z [INFO] server-0ea1d87d-35be-1ad0-59f8-9bc6b48a4081: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0ea1d87d-35be-1ad0-59f8-9bc6b48a4081 Address:0ea1d87d-35be-1ad0-59f8-9bc6b48a4081} {Suffrage:Voter ID:server-db5d333b-7afc-fc9b-acb1-82caafa66ae9 Address:db5d333b-7afc-fc9b-acb1-82caafa66ae9} {Suffrage:Voter ID:server-16e15cc4-70b3-6d04-e109-efa7fedf2cfc Address:16e15cc4-70b3-6d04-e109-efa7fedf2cfc}]" -2025-01-28T14:48:02.482Z [INFO] server-0ea1d87d-35be-1ad0-59f8-9bc6b48a4081: entering follower state: follower="Node at 0ea1d87d-35be-1ad0-59f8-9bc6b48a4081 [Follower]" leader-address= leader-id= -2025-01-28T14:48:02.482Z [INFO] server-db5d333b-7afc-fc9b-acb1-82caafa66ae9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0ea1d87d-35be-1ad0-59f8-9bc6b48a4081 Address:0ea1d87d-35be-1ad0-59f8-9bc6b48a4081} {Suffrage:Voter ID:server-db5d333b-7afc-fc9b-acb1-82caafa66ae9 Address:db5d333b-7afc-fc9b-acb1-82caafa66ae9} {Suffrage:Voter ID:server-16e15cc4-70b3-6d04-e109-efa7fedf2cfc Address:16e15cc4-70b3-6d04-e109-efa7fedf2cfc}]" -2025-01-28T14:48:02.482Z [INFO] server-db5d333b-7afc-fc9b-acb1-82caafa66ae9: entering follower state: follower="Node at db5d333b-7afc-fc9b-acb1-82caafa66ae9 [Follower]" leader-address= leader-id= -2025-01-28T14:48:02.483Z [INFO] server-16e15cc4-70b3-6d04-e109-efa7fedf2cfc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0ea1d87d-35be-1ad0-59f8-9bc6b48a4081 Address:0ea1d87d-35be-1ad0-59f8-9bc6b48a4081} {Suffrage:Voter ID:server-db5d333b-7afc-fc9b-acb1-82caafa66ae9 Address:db5d333b-7afc-fc9b-acb1-82caafa66ae9} {Suffrage:Voter ID:server-16e15cc4-70b3-6d04-e109-efa7fedf2cfc Address:16e15cc4-70b3-6d04-e109-efa7fedf2cfc}]" -2025-01-28T14:48:02.483Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:02.483Z [INFO] server-16e15cc4-70b3-6d04-e109-efa7fedf2cfc: entering follower state: follower="Node at 16e15cc4-70b3-6d04-e109-efa7fedf2cfc [Follower]" leader-address= leader-id= -2025-01-28T14:48:03.047Z [WARN] server-db5d333b-7afc-fc9b-acb1-82caafa66ae9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:03.048Z [INFO] server-db5d333b-7afc-fc9b-acb1-82caafa66ae9: entering candidate state: node="Node at db5d333b-7afc-fc9b-acb1-82caafa66ae9 [Candidate]" term=2 -2025-01-28T14:48:03.048Z [INFO] server-db5d333b-7afc-fc9b-acb1-82caafa66ae9: election won: term=2 tally=2 -2025-01-28T14:48:03.048Z [INFO] server-db5d333b-7afc-fc9b-acb1-82caafa66ae9: entering leader state: leader="Node at db5d333b-7afc-fc9b-acb1-82caafa66ae9 [Leader]" -2025-01-28T14:48:03.049Z [INFO] server-db5d333b-7afc-fc9b-acb1-82caafa66ae9: added peer, starting replication: peer=server-0ea1d87d-35be-1ad0-59f8-9bc6b48a4081 -2025-01-28T14:48:03.049Z [INFO] server-db5d333b-7afc-fc9b-acb1-82caafa66ae9: added peer, starting replication: peer=server-16e15cc4-70b3-6d04-e109-efa7fedf2cfc -2025-01-28T14:48:03.050Z [INFO] server-db5d333b-7afc-fc9b-acb1-82caafa66ae9: pipelining replication: peer="{Voter server-0ea1d87d-35be-1ad0-59f8-9bc6b48a4081 0ea1d87d-35be-1ad0-59f8-9bc6b48a4081}" -2025-01-28T14:48:03.050Z [INFO] server-db5d333b-7afc-fc9b-acb1-82caafa66ae9: pipelining replication: peer="{Voter server-16e15cc4-70b3-6d04-e109-efa7fedf2cfc 16e15cc4-70b3-6d04-e109-efa7fedf2cfc}" -2025-01-28T14:48:04.054Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:03.049086819 +0000 UTC m=+66.132491844 (1 nodes), 565.30312ms from start of poll, 568.768556ms from cluster start. Timeout at 2025-01-28 14:48:04.054106486 +0000 UTC m=+67.137511510, 1.005019666s after stability -2025-01-28T14:48:04.055Z [INFO] creating new snapshot: path=/tmp/raft2155486308/snapshots/2-113-1738075684055.tmp -2025-01-28T14:48:04.060Z [INFO] server-db5d333b-7afc-fc9b-acb1-82caafa66ae9: copied to local snapshot: bytes=71 -2025-01-28T14:48:04.062Z [INFO] server-db5d333b-7afc-fc9b-acb1-82caafa66ae9: snapshot restore progress: id=2-113-1738075684055 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:04.063Z [INFO] server-db5d333b-7afc-fc9b-acb1-82caafa66ae9: restored user snapshot: index=113 -2025-01-28T14:48:04.063Z [ERROR] server-db5d333b-7afc-fc9b-acb1-82caafa66ae9: failed to get log: index=3 error="log not found" -2025-01-28T14:48:04.063Z [ERROR] server-db5d333b-7afc-fc9b-acb1-82caafa66ae9: failed to get log: index=3 error="log not found" -2025-01-28T14:48:04.063Z [INFO] server-db5d333b-7afc-fc9b-acb1-82caafa66ae9: aborting pipeline replication: peer="{Voter server-0ea1d87d-35be-1ad0-59f8-9bc6b48a4081 0ea1d87d-35be-1ad0-59f8-9bc6b48a4081}" -2025-01-28T14:48:04.063Z [INFO] server-db5d333b-7afc-fc9b-acb1-82caafa66ae9: aborting pipeline replication: peer="{Voter server-16e15cc4-70b3-6d04-e109-efa7fedf2cfc 16e15cc4-70b3-6d04-e109-efa7fedf2cfc}" -2025-01-28T14:48:04.063Z [ERROR] server-db5d333b-7afc-fc9b-acb1-82caafa66ae9: failed to get log: index=3 error="log not found" -2025-01-28T14:48:04.066Z [INFO] creating new snapshot: path=/tmp/raft2810845969/snapshots/2-113-1738075684066.tmp -2025-01-28T14:48:04.068Z [INFO] server-16e15cc4-70b3-6d04-e109-efa7fedf2cfc: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:04.070Z [INFO] server-16e15cc4-70b3-6d04-e109-efa7fedf2cfc: copied to local snapshot: bytes=71 -2025-01-28T14:48:04.071Z [ERROR] server-db5d333b-7afc-fc9b-acb1-82caafa66ae9: failed to get log: index=3 error="log not found" -2025-01-28T14:48:04.072Z [INFO] server-16e15cc4-70b3-6d04-e109-efa7fedf2cfc: snapshot restore progress: id=2-113-1738075684066 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:04.072Z [INFO] server-16e15cc4-70b3-6d04-e109-efa7fedf2cfc: Installed remote snapshot -2025-01-28T14:48:04.072Z [INFO] creating new snapshot: path=/tmp/raft1581725978/snapshots/2-113-1738075684072.tmp -2025-01-28T14:48:04.074Z [INFO] server-0ea1d87d-35be-1ad0-59f8-9bc6b48a4081: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:04.075Z [INFO] server-db5d333b-7afc-fc9b-acb1-82caafa66ae9: pipelining replication: peer="{Voter server-16e15cc4-70b3-6d04-e109-efa7fedf2cfc 16e15cc4-70b3-6d04-e109-efa7fedf2cfc}" -2025-01-28T14:48:04.075Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:48:04.079Z [INFO] server-0ea1d87d-35be-1ad0-59f8-9bc6b48a4081: copied to local snapshot: bytes=71 -2025-01-28T14:48:04.080Z [INFO] server-0ea1d87d-35be-1ad0-59f8-9bc6b48a4081: snapshot restore progress: id=2-113-1738075684072 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:04.081Z [INFO] server-0ea1d87d-35be-1ad0-59f8-9bc6b48a4081: Installed remote snapshot -2025-01-28T14:48:04.081Z [INFO] server-db5d333b-7afc-fc9b-acb1-82caafa66ae9: pipelining replication: peer="{Voter server-0ea1d87d-35be-1ad0-59f8-9bc6b48a4081 0ea1d87d-35be-1ad0-59f8-9bc6b48a4081}" -2025-01-28T14:48:05.081Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:48:04.075644968 +0000 UTC m=+67.159049660 (2 nodes), 9.001µs from start of poll, 1.595326372s from cluster start. Timeout at 2025-01-28 14:48:05.080822973 +0000 UTC m=+68.164227998, 1.005178338s after stability -2025-01-28T14:48:05.094Z [INFO] server-91815518-0e58-36e5-ac77-c6d606e0eec1: aborting pipeline replication: peer="{Voter server-3068c999-712b-9dd4-25d1-ad785e71f54a 3068c999-712b-9dd4-25d1-ad785e71f54a}" -2025-01-28T14:48:05.094Z [INFO] server-91815518-0e58-36e5-ac77-c6d606e0eec1: aborting pipeline replication: peer="{Voter server-1cf17c00-771a-8084-cb31-ef3934ccad69 1cf17c00-771a-8084-cb31-ef3934ccad69}" +2025-01-28T14:55:02.224Z [INFO] server-13bdf423-95f8-2f7e-ca4a-f9e458ddcf39: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-13bdf423-95f8-2f7e-ca4a-f9e458ddcf39 Address:13bdf423-95f8-2f7e-ca4a-f9e458ddcf39} {Suffrage:Voter ID:server-16702220-5698-7e54-81b6-f16843890f5a Address:16702220-5698-7e54-81b6-f16843890f5a} {Suffrage:Voter ID:server-c22c2d0f-2bc0-2bef-fd79-45834aef386a Address:c22c2d0f-2bc0-2bef-fd79-45834aef386a}]" +2025-01-28T14:55:02.224Z [INFO] server-16702220-5698-7e54-81b6-f16843890f5a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-13bdf423-95f8-2f7e-ca4a-f9e458ddcf39 Address:13bdf423-95f8-2f7e-ca4a-f9e458ddcf39} {Suffrage:Voter ID:server-16702220-5698-7e54-81b6-f16843890f5a Address:16702220-5698-7e54-81b6-f16843890f5a} {Suffrage:Voter ID:server-c22c2d0f-2bc0-2bef-fd79-45834aef386a Address:c22c2d0f-2bc0-2bef-fd79-45834aef386a}]" +2025-01-28T14:55:02.224Z [INFO] server-13bdf423-95f8-2f7e-ca4a-f9e458ddcf39: entering follower state: follower="Node at 13bdf423-95f8-2f7e-ca4a-f9e458ddcf39 [Follower]" leader-address= leader-id= +2025-01-28T14:55:02.224Z [INFO] server-16702220-5698-7e54-81b6-f16843890f5a: entering follower state: follower="Node at 16702220-5698-7e54-81b6-f16843890f5a [Follower]" leader-address= leader-id= +2025-01-28T14:55:02.225Z [INFO] server-c22c2d0f-2bc0-2bef-fd79-45834aef386a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-13bdf423-95f8-2f7e-ca4a-f9e458ddcf39 Address:13bdf423-95f8-2f7e-ca4a-f9e458ddcf39} {Suffrage:Voter ID:server-16702220-5698-7e54-81b6-f16843890f5a Address:16702220-5698-7e54-81b6-f16843890f5a} {Suffrage:Voter ID:server-c22c2d0f-2bc0-2bef-fd79-45834aef386a Address:c22c2d0f-2bc0-2bef-fd79-45834aef386a}]" +2025-01-28T14:55:02.225Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:02.225Z [INFO] server-c22c2d0f-2bc0-2bef-fd79-45834aef386a: entering follower state: follower="Node at c22c2d0f-2bc0-2bef-fd79-45834aef386a [Follower]" leader-address= leader-id= +2025-01-28T14:55:02.876Z [WARN] server-c22c2d0f-2bc0-2bef-fd79-45834aef386a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:02.876Z [INFO] server-c22c2d0f-2bc0-2bef-fd79-45834aef386a: entering candidate state: node="Node at c22c2d0f-2bc0-2bef-fd79-45834aef386a [Candidate]" term=2 +2025-01-28T14:55:02.876Z [INFO] server-c22c2d0f-2bc0-2bef-fd79-45834aef386a: election won: term=2 tally=2 +2025-01-28T14:55:02.876Z [INFO] server-c22c2d0f-2bc0-2bef-fd79-45834aef386a: entering leader state: leader="Node at c22c2d0f-2bc0-2bef-fd79-45834aef386a [Leader]" +2025-01-28T14:55:02.876Z [INFO] server-c22c2d0f-2bc0-2bef-fd79-45834aef386a: added peer, starting replication: peer=server-13bdf423-95f8-2f7e-ca4a-f9e458ddcf39 +2025-01-28T14:55:02.876Z [INFO] server-c22c2d0f-2bc0-2bef-fd79-45834aef386a: added peer, starting replication: peer=server-16702220-5698-7e54-81b6-f16843890f5a +2025-01-28T14:55:02.877Z [INFO] server-c22c2d0f-2bc0-2bef-fd79-45834aef386a: pipelining replication: peer="{Voter server-13bdf423-95f8-2f7e-ca4a-f9e458ddcf39 13bdf423-95f8-2f7e-ca4a-f9e458ddcf39}" +2025-01-28T14:55:02.877Z [INFO] server-c22c2d0f-2bc0-2bef-fd79-45834aef386a: pipelining replication: peer="{Voter server-16702220-5698-7e54-81b6-f16843890f5a 16702220-5698-7e54-81b6-f16843890f5a}" +2025-01-28T14:55:03.882Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:02.876901024 +0000 UTC m=+88.551391004 (1 nodes), 651.59769ms from start of poll, 652.835733ms from cluster start. Timeout at 2025-01-28 14:55:03.881914163 +0000 UTC m=+89.556404183, 1.005013179s after stability +2025-01-28T14:55:03.884Z [INFO] server-c22c2d0f-2bc0-2bef-fd79-45834aef386a: starting snapshot up to: index=12 +2025-01-28T14:55:03.884Z [INFO] creating new snapshot: path=/tmp/raft2055875527/snapshots/2-12-1738076103884.tmp +2025-01-28T14:55:03.885Z [INFO] server-c22c2d0f-2bc0-2bef-fd79-45834aef386a: snapshot complete up to: index=12 +2025-01-28T14:55:03.892Z [INFO] server-21362484-cbbf-6c03-76cc-c7a73bc664e4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-21362484-cbbf-6c03-76cc-c7a73bc664e4 Address:21362484-cbbf-6c03-76cc-c7a73bc664e4} {Suffrage:Voter ID:server-05c9db10-5aff-04f6-6d99-33be0731ef44 Address:05c9db10-5aff-04f6-6d99-33be0731ef44} {Suffrage:Voter ID:server-5751c78e-cf79-0cd8-43e9-94f0ea0e3a44 Address:5751c78e-cf79-0cd8-43e9-94f0ea0e3a44}]" +2025-01-28T14:55:03.892Z [INFO] server-05c9db10-5aff-04f6-6d99-33be0731ef44: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-21362484-cbbf-6c03-76cc-c7a73bc664e4 Address:21362484-cbbf-6c03-76cc-c7a73bc664e4} {Suffrage:Voter ID:server-05c9db10-5aff-04f6-6d99-33be0731ef44 Address:05c9db10-5aff-04f6-6d99-33be0731ef44} {Suffrage:Voter ID:server-5751c78e-cf79-0cd8-43e9-94f0ea0e3a44 Address:5751c78e-cf79-0cd8-43e9-94f0ea0e3a44}]" +2025-01-28T14:55:03.892Z [INFO] server-21362484-cbbf-6c03-76cc-c7a73bc664e4: entering follower state: follower="Node at 21362484-cbbf-6c03-76cc-c7a73bc664e4 [Follower]" leader-address= leader-id= +2025-01-28T14:55:03.892Z [INFO] server-05c9db10-5aff-04f6-6d99-33be0731ef44: entering follower state: follower="Node at 05c9db10-5aff-04f6-6d99-33be0731ef44 [Follower]" leader-address= leader-id= +2025-01-28T14:55:03.892Z [INFO] server-5751c78e-cf79-0cd8-43e9-94f0ea0e3a44: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-21362484-cbbf-6c03-76cc-c7a73bc664e4 Address:21362484-cbbf-6c03-76cc-c7a73bc664e4} {Suffrage:Voter ID:server-05c9db10-5aff-04f6-6d99-33be0731ef44 Address:05c9db10-5aff-04f6-6d99-33be0731ef44} {Suffrage:Voter ID:server-5751c78e-cf79-0cd8-43e9-94f0ea0e3a44 Address:5751c78e-cf79-0cd8-43e9-94f0ea0e3a44}]" +2025-01-28T14:55:03.892Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:03.892Z [INFO] server-5751c78e-cf79-0cd8-43e9-94f0ea0e3a44: entering follower state: follower="Node at 5751c78e-cf79-0cd8-43e9-94f0ea0e3a44 [Follower]" leader-address= leader-id= +2025-01-28T14:55:04.523Z [WARN] server-05c9db10-5aff-04f6-6d99-33be0731ef44: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:04.523Z [INFO] server-05c9db10-5aff-04f6-6d99-33be0731ef44: entering candidate state: node="Node at 05c9db10-5aff-04f6-6d99-33be0731ef44 [Candidate]" term=2 +2025-01-28T14:55:04.523Z [INFO] server-05c9db10-5aff-04f6-6d99-33be0731ef44: election won: term=2 tally=2 +2025-01-28T14:55:04.523Z [INFO] server-05c9db10-5aff-04f6-6d99-33be0731ef44: entering leader state: leader="Node at 05c9db10-5aff-04f6-6d99-33be0731ef44 [Leader]" +2025-01-28T14:55:04.523Z [INFO] server-05c9db10-5aff-04f6-6d99-33be0731ef44: added peer, starting replication: peer=server-21362484-cbbf-6c03-76cc-c7a73bc664e4 +2025-01-28T14:55:04.523Z [INFO] server-05c9db10-5aff-04f6-6d99-33be0731ef44: added peer, starting replication: peer=server-5751c78e-cf79-0cd8-43e9-94f0ea0e3a44 +2025-01-28T14:55:04.524Z [INFO] server-05c9db10-5aff-04f6-6d99-33be0731ef44: pipelining replication: peer="{Voter server-5751c78e-cf79-0cd8-43e9-94f0ea0e3a44 5751c78e-cf79-0cd8-43e9-94f0ea0e3a44}" +2025-01-28T14:55:04.524Z [INFO] server-05c9db10-5aff-04f6-6d99-33be0731ef44: pipelining replication: peer="{Voter server-21362484-cbbf-6c03-76cc-c7a73bc664e4 21362484-cbbf-6c03-76cc-c7a73bc664e4}" +2025-01-28T14:55:05.529Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:04.523767543 +0000 UTC m=+90.198257523 (1 nodes), 630.949438ms from start of poll, 631.77364ms from cluster start. Timeout at 2025-01-28 14:55:05.528774942 +0000 UTC m=+91.203264902, 1.005007379s after stability +2025-01-28T14:55:05.530Z [INFO] creating new snapshot: path=/tmp/raft725788943/snapshots/2-113-1738076105530.tmp +2025-01-28T14:55:05.531Z [INFO] server-05c9db10-5aff-04f6-6d99-33be0731ef44: copied to local snapshot: bytes=71 +2025-01-28T14:55:05.532Z [INFO] server-05c9db10-5aff-04f6-6d99-33be0731ef44: snapshot restore progress: id=2-113-1738076105530 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:05.532Z [INFO] server-05c9db10-5aff-04f6-6d99-33be0731ef44: restored user snapshot: index=113 +2025-01-28T14:55:05.532Z [ERROR] server-05c9db10-5aff-04f6-6d99-33be0731ef44: failed to get log: index=3 error="log not found" +2025-01-28T14:55:05.532Z [INFO] server-05c9db10-5aff-04f6-6d99-33be0731ef44: aborting pipeline replication: peer="{Voter server-5751c78e-cf79-0cd8-43e9-94f0ea0e3a44 5751c78e-cf79-0cd8-43e9-94f0ea0e3a44}" +2025-01-28T14:55:05.532Z [ERROR] server-05c9db10-5aff-04f6-6d99-33be0731ef44: failed to get log: index=3 error="log not found" +2025-01-28T14:55:05.532Z [INFO] server-05c9db10-5aff-04f6-6d99-33be0731ef44: aborting pipeline replication: peer="{Voter server-21362484-cbbf-6c03-76cc-c7a73bc664e4 21362484-cbbf-6c03-76cc-c7a73bc664e4}" +2025-01-28T14:55:05.540Z [ERROR] server-05c9db10-5aff-04f6-6d99-33be0731ef44: failed to get log: index=3 error="log not found" +2025-01-28T14:55:05.541Z [INFO] creating new snapshot: path=/tmp/raft693359209/snapshots/2-113-1738076105541.tmp +2025-01-28T14:55:05.542Z [INFO] server-21362484-cbbf-6c03-76cc-c7a73bc664e4: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:05.542Z [ERROR] server-05c9db10-5aff-04f6-6d99-33be0731ef44: failed to get log: index=3 error="log not found" +2025-01-28T14:55:05.543Z [INFO] server-21362484-cbbf-6c03-76cc-c7a73bc664e4: copied to local snapshot: bytes=71 +2025-01-28T14:55:05.544Z [INFO] creating new snapshot: path=/tmp/raft814231172/snapshots/2-113-1738076105544.tmp +2025-01-28T14:55:05.545Z [INFO] server-21362484-cbbf-6c03-76cc-c7a73bc664e4: snapshot restore progress: id=2-113-1738076105541 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:05.545Z [INFO] server-21362484-cbbf-6c03-76cc-c7a73bc664e4: Installed remote snapshot +2025-01-28T14:55:05.546Z [INFO] server-05c9db10-5aff-04f6-6d99-33be0731ef44: pipelining replication: peer="{Voter server-21362484-cbbf-6c03-76cc-c7a73bc664e4 21362484-cbbf-6c03-76cc-c7a73bc664e4}" +2025-01-28T14:55:05.547Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:55:05.547Z [INFO] server-5751c78e-cf79-0cd8-43e9-94f0ea0e3a44: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:05.548Z [INFO] server-5751c78e-cf79-0cd8-43e9-94f0ea0e3a44: copied to local snapshot: bytes=71 +2025-01-28T14:55:05.548Z [INFO] server-5751c78e-cf79-0cd8-43e9-94f0ea0e3a44: snapshot restore progress: id=2-113-1738076105544 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:05.548Z [INFO] server-5751c78e-cf79-0cd8-43e9-94f0ea0e3a44: Installed remote snapshot +2025-01-28T14:55:05.548Z [INFO] server-05c9db10-5aff-04f6-6d99-33be0731ef44: pipelining replication: peer="{Voter server-5751c78e-cf79-0cd8-43e9-94f0ea0e3a44 5751c78e-cf79-0cd8-43e9-94f0ea0e3a44}" +2025-01-28T14:55:06.552Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:55:05.547072427 +0000 UTC m=+91.221562407 (2 nodes), 2.2µs from start of poll, 1.655078524s from cluster start. Timeout at 2025-01-28 14:55:06.552158059 +0000 UTC m=+92.226648079, 1.005085672s after stability +2025-01-28T14:55:06.564Z [INFO] server-c22c2d0f-2bc0-2bef-fd79-45834aef386a: aborting pipeline replication: peer="{Voter server-13bdf423-95f8-2f7e-ca4a-f9e458ddcf39 13bdf423-95f8-2f7e-ca4a-f9e458ddcf39}" +2025-01-28T14:55:06.564Z [INFO] server-c22c2d0f-2bc0-2bef-fd79-45834aef386a: aborting pipeline replication: peer="{Voter server-16702220-5698-7e54-81b6-f16843890f5a 16702220-5698-7e54-81b6-f16843890f5a}" === RUN TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_true -2025-01-28T14:48:05.104Z [INFO] server-fe0351f2-2ea1-fcdf-4e10-8f422e7ef51c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fe0351f2-2ea1-fcdf-4e10-8f422e7ef51c Address:fe0351f2-2ea1-fcdf-4e10-8f422e7ef51c} {Suffrage:Voter ID:server-36d02b3b-2e5c-7d34-c23f-f15f45d4b6aa Address:36d02b3b-2e5c-7d34-c23f-f15f45d4b6aa} {Suffrage:Voter ID:server-4071d816-93ca-339a-f9c1-5f652d35b900 Address:4071d816-93ca-339a-f9c1-5f652d35b900}]" -2025-01-28T14:48:05.105Z [INFO] server-36d02b3b-2e5c-7d34-c23f-f15f45d4b6aa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fe0351f2-2ea1-fcdf-4e10-8f422e7ef51c Address:fe0351f2-2ea1-fcdf-4e10-8f422e7ef51c} {Suffrage:Voter ID:server-36d02b3b-2e5c-7d34-c23f-f15f45d4b6aa Address:36d02b3b-2e5c-7d34-c23f-f15f45d4b6aa} {Suffrage:Voter ID:server-4071d816-93ca-339a-f9c1-5f652d35b900 Address:4071d816-93ca-339a-f9c1-5f652d35b900}]" -2025-01-28T14:48:05.106Z [INFO] server-fe0351f2-2ea1-fcdf-4e10-8f422e7ef51c: entering follower state: follower="Node at fe0351f2-2ea1-fcdf-4e10-8f422e7ef51c [Follower]" leader-address= leader-id= -2025-01-28T14:48:05.106Z [INFO] server-36d02b3b-2e5c-7d34-c23f-f15f45d4b6aa: entering follower state: follower="Node at 36d02b3b-2e5c-7d34-c23f-f15f45d4b6aa [Follower]" leader-address= leader-id= -2025-01-28T14:48:05.107Z [INFO] server-4071d816-93ca-339a-f9c1-5f652d35b900: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fe0351f2-2ea1-fcdf-4e10-8f422e7ef51c Address:fe0351f2-2ea1-fcdf-4e10-8f422e7ef51c} {Suffrage:Voter ID:server-36d02b3b-2e5c-7d34-c23f-f15f45d4b6aa Address:36d02b3b-2e5c-7d34-c23f-f15f45d4b6aa} {Suffrage:Voter ID:server-4071d816-93ca-339a-f9c1-5f652d35b900 Address:4071d816-93ca-339a-f9c1-5f652d35b900}]" -2025-01-28T14:48:05.107Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:05.107Z [INFO] server-4071d816-93ca-339a-f9c1-5f652d35b900: entering follower state: follower="Node at 4071d816-93ca-339a-f9c1-5f652d35b900 [Follower]" leader-address= leader-id= -2025-01-28T14:48:05.707Z [WARN] server-fe0351f2-2ea1-fcdf-4e10-8f422e7ef51c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:05.707Z [INFO] server-fe0351f2-2ea1-fcdf-4e10-8f422e7ef51c: entering candidate state: node="Node at fe0351f2-2ea1-fcdf-4e10-8f422e7ef51c [Candidate]" term=2 -2025-01-28T14:48:05.708Z [INFO] server-fe0351f2-2ea1-fcdf-4e10-8f422e7ef51c: election won: term=2 tally=2 -2025-01-28T14:48:05.708Z [INFO] server-fe0351f2-2ea1-fcdf-4e10-8f422e7ef51c: entering leader state: leader="Node at fe0351f2-2ea1-fcdf-4e10-8f422e7ef51c [Leader]" -2025-01-28T14:48:05.708Z [INFO] server-fe0351f2-2ea1-fcdf-4e10-8f422e7ef51c: added peer, starting replication: peer=server-36d02b3b-2e5c-7d34-c23f-f15f45d4b6aa -2025-01-28T14:48:05.708Z [INFO] server-fe0351f2-2ea1-fcdf-4e10-8f422e7ef51c: added peer, starting replication: peer=server-4071d816-93ca-339a-f9c1-5f652d35b900 -2025-01-28T14:48:05.709Z [INFO] server-fe0351f2-2ea1-fcdf-4e10-8f422e7ef51c: pipelining replication: peer="{Voter server-36d02b3b-2e5c-7d34-c23f-f15f45d4b6aa 36d02b3b-2e5c-7d34-c23f-f15f45d4b6aa}" -2025-01-28T14:48:05.711Z [INFO] server-fe0351f2-2ea1-fcdf-4e10-8f422e7ef51c: pipelining replication: peer="{Voter server-4071d816-93ca-339a-f9c1-5f652d35b900 4071d816-93ca-339a-f9c1-5f652d35b900}" -2025-01-28T14:48:06.714Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:05.709077472 +0000 UTC m=+68.792482163 (1 nodes), 601.22852ms from start of poll, 604.642955ms from cluster start. Timeout at 2025-01-28 14:48:06.714102139 +0000 UTC m=+69.797507164, 1.005025001s after stability -2025-01-28T14:48:06.721Z [INFO] server-fe0351f2-2ea1-fcdf-4e10-8f422e7ef51c: starting snapshot up to: index=12 -2025-01-28T14:48:06.721Z [INFO] creating new snapshot: path=/tmp/raft715067995/snapshots/2-12-1738075686721.tmp -2025-01-28T14:48:06.725Z [INFO] server-fe0351f2-2ea1-fcdf-4e10-8f422e7ef51c: snapshot complete up to: index=12 -2025-01-28T14:48:06.739Z [INFO] server-3c37f80e-90f7-498e-cc2d-503f26d3f261: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3c37f80e-90f7-498e-cc2d-503f26d3f261 Address:3c37f80e-90f7-498e-cc2d-503f26d3f261} {Suffrage:Voter ID:server-9515b21a-4ea1-2479-266f-4ed1a16e4eec Address:9515b21a-4ea1-2479-266f-4ed1a16e4eec} {Suffrage:Voter ID:server-80451566-3d16-6fd0-2dca-11a25a538455 Address:80451566-3d16-6fd0-2dca-11a25a538455}]" -2025-01-28T14:48:06.739Z [INFO] server-3c37f80e-90f7-498e-cc2d-503f26d3f261: entering follower state: follower="Node at 3c37f80e-90f7-498e-cc2d-503f26d3f261 [Follower]" leader-address= leader-id= -2025-01-28T14:48:06.739Z [INFO] server-9515b21a-4ea1-2479-266f-4ed1a16e4eec: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3c37f80e-90f7-498e-cc2d-503f26d3f261 Address:3c37f80e-90f7-498e-cc2d-503f26d3f261} {Suffrage:Voter ID:server-9515b21a-4ea1-2479-266f-4ed1a16e4eec Address:9515b21a-4ea1-2479-266f-4ed1a16e4eec} {Suffrage:Voter ID:server-80451566-3d16-6fd0-2dca-11a25a538455 Address:80451566-3d16-6fd0-2dca-11a25a538455}]" -2025-01-28T14:48:06.740Z [INFO] server-80451566-3d16-6fd0-2dca-11a25a538455: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3c37f80e-90f7-498e-cc2d-503f26d3f261 Address:3c37f80e-90f7-498e-cc2d-503f26d3f261} {Suffrage:Voter ID:server-9515b21a-4ea1-2479-266f-4ed1a16e4eec Address:9515b21a-4ea1-2479-266f-4ed1a16e4eec} {Suffrage:Voter ID:server-80451566-3d16-6fd0-2dca-11a25a538455 Address:80451566-3d16-6fd0-2dca-11a25a538455}]" -2025-01-28T14:48:06.740Z [INFO] server-9515b21a-4ea1-2479-266f-4ed1a16e4eec: entering follower state: follower="Node at 9515b21a-4ea1-2479-266f-4ed1a16e4eec [Follower]" leader-address= leader-id= -2025-01-28T14:48:06.740Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:06.740Z [INFO] server-80451566-3d16-6fd0-2dca-11a25a538455: entering follower state: follower="Node at 80451566-3d16-6fd0-2dca-11a25a538455 [Follower]" leader-address= leader-id= -2025-01-28T14:48:07.351Z [WARN] server-80451566-3d16-6fd0-2dca-11a25a538455: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:07.351Z [INFO] server-80451566-3d16-6fd0-2dca-11a25a538455: entering candidate state: node="Node at 80451566-3d16-6fd0-2dca-11a25a538455 [Candidate]" term=2 -2025-01-28T14:48:07.352Z [INFO] server-80451566-3d16-6fd0-2dca-11a25a538455: election won: term=2 tally=2 -2025-01-28T14:48:07.352Z [INFO] server-80451566-3d16-6fd0-2dca-11a25a538455: entering leader state: leader="Node at 80451566-3d16-6fd0-2dca-11a25a538455 [Leader]" -2025-01-28T14:48:07.352Z [INFO] server-80451566-3d16-6fd0-2dca-11a25a538455: added peer, starting replication: peer=server-3c37f80e-90f7-498e-cc2d-503f26d3f261 -2025-01-28T14:48:07.353Z [INFO] server-80451566-3d16-6fd0-2dca-11a25a538455: added peer, starting replication: peer=server-9515b21a-4ea1-2479-266f-4ed1a16e4eec -2025-01-28T14:48:07.354Z [INFO] server-80451566-3d16-6fd0-2dca-11a25a538455: pipelining replication: peer="{Voter server-3c37f80e-90f7-498e-cc2d-503f26d3f261 3c37f80e-90f7-498e-cc2d-503f26d3f261}" -2025-01-28T14:48:07.355Z [INFO] server-80451566-3d16-6fd0-2dca-11a25a538455: pipelining replication: peer="{Voter server-9515b21a-4ea1-2479-266f-4ed1a16e4eec 9515b21a-4ea1-2479-266f-4ed1a16e4eec}" -2025-01-28T14:48:08.361Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:07.353155625 +0000 UTC m=+70.436560649 (1 nodes), 612.469187ms from start of poll, 614.70292ms from cluster start. Timeout at 2025-01-28 14:48:08.358273295 +0000 UTC m=+71.441677986, 1.005117337s after stability -2025-01-28T14:48:08.362Z [INFO] creating new snapshot: path=/tmp/raft2368570834/snapshots/2-113-1738075688362.tmp -2025-01-28T14:48:08.366Z [INFO] server-80451566-3d16-6fd0-2dca-11a25a538455: copied to local snapshot: bytes=71 -2025-01-28T14:48:08.368Z [INFO] server-80451566-3d16-6fd0-2dca-11a25a538455: snapshot restore progress: id=2-113-1738075688362 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:08.368Z [INFO] server-80451566-3d16-6fd0-2dca-11a25a538455: removing all old logs from log store -2025-01-28T14:48:08.368Z [INFO] server-80451566-3d16-6fd0-2dca-11a25a538455: compacting logs: from=1 to=2 -2025-01-28T14:48:08.368Z [INFO] server-80451566-3d16-6fd0-2dca-11a25a538455: restored user snapshot: index=113 -2025-01-28T14:48:08.368Z [ERROR] server-80451566-3d16-6fd0-2dca-11a25a538455: failed to get log: index=2 error="log not found" -2025-01-28T14:48:08.369Z [INFO] server-80451566-3d16-6fd0-2dca-11a25a538455: aborting pipeline replication: peer="{Voter server-9515b21a-4ea1-2479-266f-4ed1a16e4eec 9515b21a-4ea1-2479-266f-4ed1a16e4eec}" -2025-01-28T14:48:08.369Z [ERROR] server-80451566-3d16-6fd0-2dca-11a25a538455: failed to get log: index=2 error="log not found" -2025-01-28T14:48:08.369Z [INFO] server-80451566-3d16-6fd0-2dca-11a25a538455: aborting pipeline replication: peer="{Voter server-3c37f80e-90f7-498e-cc2d-503f26d3f261 3c37f80e-90f7-498e-cc2d-503f26d3f261}" -2025-01-28T14:48:08.376Z [ERROR] server-80451566-3d16-6fd0-2dca-11a25a538455: failed to get log: index=2 error="log not found" -2025-01-28T14:48:08.378Z [INFO] creating new snapshot: path=/tmp/raft1238104714/snapshots/2-113-1738075688378.tmp -2025-01-28T14:48:08.380Z [ERROR] server-80451566-3d16-6fd0-2dca-11a25a538455: failed to get log: index=2 error="log not found" -2025-01-28T14:48:08.380Z [INFO] server-9515b21a-4ea1-2479-266f-4ed1a16e4eec: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:08.390Z [INFO] server-9515b21a-4ea1-2479-266f-4ed1a16e4eec: copied to local snapshot: bytes=71 -2025-01-28T14:48:08.391Z [INFO] creating new snapshot: path=/tmp/raft1867329354/snapshots/2-113-1738075688390.tmp -2025-01-28T14:48:08.392Z [INFO] server-9515b21a-4ea1-2479-266f-4ed1a16e4eec: snapshot restore progress: id=2-113-1738075688378 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:08.392Z [INFO] server-9515b21a-4ea1-2479-266f-4ed1a16e4eec: removing all old logs from log store -2025-01-28T14:48:08.392Z [INFO] server-9515b21a-4ea1-2479-266f-4ed1a16e4eec: compacting logs: from=1 to=2 -2025-01-28T14:48:08.392Z [INFO] server-9515b21a-4ea1-2479-266f-4ed1a16e4eec: Installed remote snapshot -2025-01-28T14:48:08.392Z [INFO] server-3c37f80e-90f7-498e-cc2d-503f26d3f261: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:08.393Z [INFO] server-80451566-3d16-6fd0-2dca-11a25a538455: pipelining replication: peer="{Voter server-9515b21a-4ea1-2479-266f-4ed1a16e4eec 9515b21a-4ea1-2479-266f-4ed1a16e4eec}" -2025-01-28T14:48:08.395Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:48:08.395Z [INFO] server-3c37f80e-90f7-498e-cc2d-503f26d3f261: copied to local snapshot: bytes=71 -2025-01-28T14:48:08.396Z [INFO] server-3c37f80e-90f7-498e-cc2d-503f26d3f261: snapshot restore progress: id=2-113-1738075688390 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:08.396Z [INFO] server-3c37f80e-90f7-498e-cc2d-503f26d3f261: removing all old logs from log store -2025-01-28T14:48:08.397Z [INFO] server-3c37f80e-90f7-498e-cc2d-503f26d3f261: compacting logs: from=1 to=2 -2025-01-28T14:48:08.397Z [INFO] server-3c37f80e-90f7-498e-cc2d-503f26d3f261: Installed remote snapshot -2025-01-28T14:48:08.397Z [INFO] server-80451566-3d16-6fd0-2dca-11a25a538455: pipelining replication: peer="{Voter server-3c37f80e-90f7-498e-cc2d-503f26d3f261 3c37f80e-90f7-498e-cc2d-503f26d3f261}" -2025-01-28T14:48:09.400Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:48:08.395144899 +0000 UTC m=+71.478549590 (2 nodes), 7.666µs from start of poll, 1.656691861s from cluster start. Timeout at 2025-01-28 14:48:09.400388907 +0000 UTC m=+72.483793931, 1.005244341s after stability -2025-01-28T14:48:09.415Z [INFO] server-fe0351f2-2ea1-fcdf-4e10-8f422e7ef51c: aborting pipeline replication: peer="{Voter server-4071d816-93ca-339a-f9c1-5f652d35b900 4071d816-93ca-339a-f9c1-5f652d35b900}" -2025-01-28T14:48:09.416Z [INFO] server-fe0351f2-2ea1-fcdf-4e10-8f422e7ef51c: aborting pipeline replication: peer="{Voter server-36d02b3b-2e5c-7d34-c23f-f15f45d4b6aa 36d02b3b-2e5c-7d34-c23f-f15f45d4b6aa}" +2025-01-28T14:55:06.570Z [INFO] server-3ce8adbc-fd9d-ed3f-890d-927ccf732217: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3ce8adbc-fd9d-ed3f-890d-927ccf732217 Address:3ce8adbc-fd9d-ed3f-890d-927ccf732217} {Suffrage:Voter ID:server-ce8157db-7e2c-3baf-6af7-5d7ef93d5cad Address:ce8157db-7e2c-3baf-6af7-5d7ef93d5cad} {Suffrage:Voter ID:server-3859bb36-6c31-79bf-81ca-d6aed133cd80 Address:3859bb36-6c31-79bf-81ca-d6aed133cd80}]" +2025-01-28T14:55:06.570Z [INFO] server-3ce8adbc-fd9d-ed3f-890d-927ccf732217: entering follower state: follower="Node at 3ce8adbc-fd9d-ed3f-890d-927ccf732217 [Follower]" leader-address= leader-id= +2025-01-28T14:55:06.570Z [INFO] server-ce8157db-7e2c-3baf-6af7-5d7ef93d5cad: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3ce8adbc-fd9d-ed3f-890d-927ccf732217 Address:3ce8adbc-fd9d-ed3f-890d-927ccf732217} {Suffrage:Voter ID:server-ce8157db-7e2c-3baf-6af7-5d7ef93d5cad Address:ce8157db-7e2c-3baf-6af7-5d7ef93d5cad} {Suffrage:Voter ID:server-3859bb36-6c31-79bf-81ca-d6aed133cd80 Address:3859bb36-6c31-79bf-81ca-d6aed133cd80}]" +2025-01-28T14:55:06.571Z [INFO] server-ce8157db-7e2c-3baf-6af7-5d7ef93d5cad: entering follower state: follower="Node at ce8157db-7e2c-3baf-6af7-5d7ef93d5cad [Follower]" leader-address= leader-id= +2025-01-28T14:55:06.571Z [INFO] server-3859bb36-6c31-79bf-81ca-d6aed133cd80: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3ce8adbc-fd9d-ed3f-890d-927ccf732217 Address:3ce8adbc-fd9d-ed3f-890d-927ccf732217} {Suffrage:Voter ID:server-ce8157db-7e2c-3baf-6af7-5d7ef93d5cad Address:ce8157db-7e2c-3baf-6af7-5d7ef93d5cad} {Suffrage:Voter ID:server-3859bb36-6c31-79bf-81ca-d6aed133cd80 Address:3859bb36-6c31-79bf-81ca-d6aed133cd80}]" +2025-01-28T14:55:06.571Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:06.571Z [INFO] server-3859bb36-6c31-79bf-81ca-d6aed133cd80: entering follower state: follower="Node at 3859bb36-6c31-79bf-81ca-d6aed133cd80 [Follower]" leader-address= leader-id= +2025-01-28T14:55:07.488Z [WARN] server-3859bb36-6c31-79bf-81ca-d6aed133cd80: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:07.488Z [INFO] server-3859bb36-6c31-79bf-81ca-d6aed133cd80: entering candidate state: node="Node at 3859bb36-6c31-79bf-81ca-d6aed133cd80 [Candidate]" term=2 +2025-01-28T14:55:07.488Z [INFO] server-3859bb36-6c31-79bf-81ca-d6aed133cd80: election won: term=2 tally=2 +2025-01-28T14:55:07.488Z [INFO] server-3859bb36-6c31-79bf-81ca-d6aed133cd80: entering leader state: leader="Node at 3859bb36-6c31-79bf-81ca-d6aed133cd80 [Leader]" +2025-01-28T14:55:07.488Z [INFO] server-3859bb36-6c31-79bf-81ca-d6aed133cd80: added peer, starting replication: peer=server-3ce8adbc-fd9d-ed3f-890d-927ccf732217 +2025-01-28T14:55:07.488Z [INFO] server-3859bb36-6c31-79bf-81ca-d6aed133cd80: added peer, starting replication: peer=server-ce8157db-7e2c-3baf-6af7-5d7ef93d5cad +2025-01-28T14:55:07.488Z [INFO] server-3859bb36-6c31-79bf-81ca-d6aed133cd80: pipelining replication: peer="{Voter server-3ce8adbc-fd9d-ed3f-890d-927ccf732217 3ce8adbc-fd9d-ed3f-890d-927ccf732217}" +2025-01-28T14:55:07.488Z [INFO] server-3859bb36-6c31-79bf-81ca-d6aed133cd80: pipelining replication: peer="{Voter server-ce8157db-7e2c-3baf-6af7-5d7ef93d5cad ce8157db-7e2c-3baf-6af7-5d7ef93d5cad}" +2025-01-28T14:55:08.494Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:07.488687401 +0000 UTC m=+93.163177361 (1 nodes), 917.228632ms from start of poll, 918.323568ms from cluster start. Timeout at 2025-01-28 14:55:08.49369612 +0000 UTC m=+94.168186100, 1.005008739s after stability +2025-01-28T14:55:08.495Z [INFO] server-3859bb36-6c31-79bf-81ca-d6aed133cd80: starting snapshot up to: index=12 +2025-01-28T14:55:08.495Z [INFO] creating new snapshot: path=/tmp/raft2328750132/snapshots/2-12-1738076108495.tmp +2025-01-28T14:55:08.496Z [INFO] server-3859bb36-6c31-79bf-81ca-d6aed133cd80: snapshot complete up to: index=12 +2025-01-28T14:55:08.499Z [INFO] server-cf010aad-ad2f-4235-d9c1-e40bc80247bd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cf010aad-ad2f-4235-d9c1-e40bc80247bd Address:cf010aad-ad2f-4235-d9c1-e40bc80247bd} {Suffrage:Voter ID:server-7f361493-4921-4358-9436-b9aab5e113f8 Address:7f361493-4921-4358-9436-b9aab5e113f8} {Suffrage:Voter ID:server-b92cd6e7-6a01-6d82-cda1-5ac3b2a7864a Address:b92cd6e7-6a01-6d82-cda1-5ac3b2a7864a}]" +2025-01-28T14:55:08.500Z [INFO] server-cf010aad-ad2f-4235-d9c1-e40bc80247bd: entering follower state: follower="Node at cf010aad-ad2f-4235-d9c1-e40bc80247bd [Follower]" leader-address= leader-id= +2025-01-28T14:55:08.500Z [INFO] server-7f361493-4921-4358-9436-b9aab5e113f8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cf010aad-ad2f-4235-d9c1-e40bc80247bd Address:cf010aad-ad2f-4235-d9c1-e40bc80247bd} {Suffrage:Voter ID:server-7f361493-4921-4358-9436-b9aab5e113f8 Address:7f361493-4921-4358-9436-b9aab5e113f8} {Suffrage:Voter ID:server-b92cd6e7-6a01-6d82-cda1-5ac3b2a7864a Address:b92cd6e7-6a01-6d82-cda1-5ac3b2a7864a}]" +2025-01-28T14:55:08.500Z [INFO] server-7f361493-4921-4358-9436-b9aab5e113f8: entering follower state: follower="Node at 7f361493-4921-4358-9436-b9aab5e113f8 [Follower]" leader-address= leader-id= +2025-01-28T14:55:08.500Z [INFO] server-b92cd6e7-6a01-6d82-cda1-5ac3b2a7864a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cf010aad-ad2f-4235-d9c1-e40bc80247bd Address:cf010aad-ad2f-4235-d9c1-e40bc80247bd} {Suffrage:Voter ID:server-7f361493-4921-4358-9436-b9aab5e113f8 Address:7f361493-4921-4358-9436-b9aab5e113f8} {Suffrage:Voter ID:server-b92cd6e7-6a01-6d82-cda1-5ac3b2a7864a Address:b92cd6e7-6a01-6d82-cda1-5ac3b2a7864a}]" +2025-01-28T14:55:08.500Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:08.500Z [INFO] server-b92cd6e7-6a01-6d82-cda1-5ac3b2a7864a: entering follower state: follower="Node at b92cd6e7-6a01-6d82-cda1-5ac3b2a7864a [Follower]" leader-address= leader-id= +2025-01-28T14:55:09.031Z [WARN] server-7f361493-4921-4358-9436-b9aab5e113f8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:09.031Z [INFO] server-7f361493-4921-4358-9436-b9aab5e113f8: entering candidate state: node="Node at 7f361493-4921-4358-9436-b9aab5e113f8 [Candidate]" term=2 +2025-01-28T14:55:09.031Z [INFO] server-7f361493-4921-4358-9436-b9aab5e113f8: election won: term=2 tally=2 +2025-01-28T14:55:09.031Z [INFO] server-7f361493-4921-4358-9436-b9aab5e113f8: entering leader state: leader="Node at 7f361493-4921-4358-9436-b9aab5e113f8 [Leader]" +2025-01-28T14:55:09.031Z [INFO] server-7f361493-4921-4358-9436-b9aab5e113f8: added peer, starting replication: peer=server-cf010aad-ad2f-4235-d9c1-e40bc80247bd +2025-01-28T14:55:09.031Z [INFO] server-7f361493-4921-4358-9436-b9aab5e113f8: added peer, starting replication: peer=server-b92cd6e7-6a01-6d82-cda1-5ac3b2a7864a +2025-01-28T14:55:09.032Z [INFO] server-7f361493-4921-4358-9436-b9aab5e113f8: pipelining replication: peer="{Voter server-cf010aad-ad2f-4235-d9c1-e40bc80247bd cf010aad-ad2f-4235-d9c1-e40bc80247bd}" +2025-01-28T14:55:09.032Z [INFO] server-7f361493-4921-4358-9436-b9aab5e113f8: pipelining replication: peer="{Voter server-b92cd6e7-6a01-6d82-cda1-5ac3b2a7864a b92cd6e7-6a01-6d82-cda1-5ac3b2a7864a}" +2025-01-28T14:55:10.037Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:09.031895199 +0000 UTC m=+94.706385179 (1 nodes), 531.102218ms from start of poll, 532.150459ms from cluster start. Timeout at 2025-01-28 14:55:10.036902638 +0000 UTC m=+95.711392618, 1.005007439s after stability +2025-01-28T14:55:10.037Z [INFO] creating new snapshot: path=/tmp/raft2926251092/snapshots/2-113-1738076110037.tmp +2025-01-28T14:55:10.039Z [INFO] server-7f361493-4921-4358-9436-b9aab5e113f8: copied to local snapshot: bytes=71 +2025-01-28T14:55:10.039Z [INFO] server-7f361493-4921-4358-9436-b9aab5e113f8: snapshot restore progress: id=2-113-1738076110037 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:10.040Z [INFO] server-7f361493-4921-4358-9436-b9aab5e113f8: removing all old logs from log store +2025-01-28T14:55:10.040Z [INFO] server-7f361493-4921-4358-9436-b9aab5e113f8: compacting logs: from=1 to=2 +2025-01-28T14:55:10.040Z [INFO] server-7f361493-4921-4358-9436-b9aab5e113f8: restored user snapshot: index=113 +2025-01-28T14:55:10.040Z [ERROR] server-7f361493-4921-4358-9436-b9aab5e113f8: failed to get log: index=2 error="log not found" +2025-01-28T14:55:10.040Z [INFO] server-7f361493-4921-4358-9436-b9aab5e113f8: aborting pipeline replication: peer="{Voter server-b92cd6e7-6a01-6d82-cda1-5ac3b2a7864a b92cd6e7-6a01-6d82-cda1-5ac3b2a7864a}" +2025-01-28T14:55:10.040Z [ERROR] server-7f361493-4921-4358-9436-b9aab5e113f8: failed to get log: index=2 error="log not found" +2025-01-28T14:55:10.040Z [INFO] server-7f361493-4921-4358-9436-b9aab5e113f8: aborting pipeline replication: peer="{Voter server-cf010aad-ad2f-4235-d9c1-e40bc80247bd cf010aad-ad2f-4235-d9c1-e40bc80247bd}" +2025-01-28T14:55:10.047Z [ERROR] server-7f361493-4921-4358-9436-b9aab5e113f8: failed to get log: index=2 error="log not found" +2025-01-28T14:55:10.047Z [INFO] creating new snapshot: path=/tmp/raft4191162390/snapshots/2-113-1738076110047.tmp +2025-01-28T14:55:10.047Z [ERROR] server-7f361493-4921-4358-9436-b9aab5e113f8: failed to get log: index=2 error="log not found" +2025-01-28T14:55:10.048Z [INFO] server-b92cd6e7-6a01-6d82-cda1-5ac3b2a7864a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:10.048Z [INFO] creating new snapshot: path=/tmp/raft1426473459/snapshots/2-113-1738076110048.tmp +2025-01-28T14:55:10.049Z [INFO] server-b92cd6e7-6a01-6d82-cda1-5ac3b2a7864a: copied to local snapshot: bytes=71 +2025-01-28T14:55:10.049Z [INFO] server-cf010aad-ad2f-4235-d9c1-e40bc80247bd: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:10.049Z [INFO] server-b92cd6e7-6a01-6d82-cda1-5ac3b2a7864a: snapshot restore progress: id=2-113-1738076110047 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:10.049Z [INFO] server-b92cd6e7-6a01-6d82-cda1-5ac3b2a7864a: removing all old logs from log store +2025-01-28T14:55:10.049Z [INFO] server-b92cd6e7-6a01-6d82-cda1-5ac3b2a7864a: compacting logs: from=1 to=2 +2025-01-28T14:55:10.049Z [INFO] server-b92cd6e7-6a01-6d82-cda1-5ac3b2a7864a: Installed remote snapshot +2025-01-28T14:55:10.050Z [INFO] server-7f361493-4921-4358-9436-b9aab5e113f8: pipelining replication: peer="{Voter server-b92cd6e7-6a01-6d82-cda1-5ac3b2a7864a b92cd6e7-6a01-6d82-cda1-5ac3b2a7864a}" +2025-01-28T14:55:10.050Z [INFO] server-cf010aad-ad2f-4235-d9c1-e40bc80247bd: copied to local snapshot: bytes=71 +2025-01-28T14:55:10.050Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:55:10.050Z [INFO] server-cf010aad-ad2f-4235-d9c1-e40bc80247bd: snapshot restore progress: id=2-113-1738076110048 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:10.050Z [INFO] server-cf010aad-ad2f-4235-d9c1-e40bc80247bd: removing all old logs from log store +2025-01-28T14:55:10.050Z [INFO] server-cf010aad-ad2f-4235-d9c1-e40bc80247bd: compacting logs: from=1 to=2 +2025-01-28T14:55:10.050Z [INFO] server-cf010aad-ad2f-4235-d9c1-e40bc80247bd: Installed remote snapshot +2025-01-28T14:55:10.051Z [INFO] server-7f361493-4921-4358-9436-b9aab5e113f8: pipelining replication: peer="{Voter server-cf010aad-ad2f-4235-d9c1-e40bc80247bd cf010aad-ad2f-4235-d9c1-e40bc80247bd}" +2025-01-28T14:55:11.056Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:55:10.050429653 +0000 UTC m=+95.724919633 (2 nodes), 2.22µs from start of poll, 1.550684913s from cluster start. Timeout at 2025-01-28 14:55:11.055517645 +0000 UTC m=+96.730007625, 1.005087992s after stability +2025-01-28T14:55:11.067Z [INFO] server-3859bb36-6c31-79bf-81ca-d6aed133cd80: aborting pipeline replication: peer="{Voter server-ce8157db-7e2c-3baf-6af7-5d7ef93d5cad ce8157db-7e2c-3baf-6af7-5d7ef93d5cad}" +2025-01-28T14:55:11.067Z [INFO] server-3859bb36-6c31-79bf-81ca-d6aed133cd80: aborting pipeline replication: peer="{Voter server-3ce8adbc-fd9d-ed3f-890d-927ccf732217 3ce8adbc-fd9d-ed3f-890d-927ccf732217}" === RUN TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_false -2025-01-28T14:48:09.429Z [INFO] server-bfb4c83d-d7eb-2b5b-06d5-a0612343b7d9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bfb4c83d-d7eb-2b5b-06d5-a0612343b7d9 Address:bfb4c83d-d7eb-2b5b-06d5-a0612343b7d9} {Suffrage:Voter ID:server-cd8ee909-9a12-9e35-2c85-7164cba4429d Address:cd8ee909-9a12-9e35-2c85-7164cba4429d} {Suffrage:Voter ID:server-7c555bf9-8b41-2ade-331c-b3045e0b5bce Address:7c555bf9-8b41-2ade-331c-b3045e0b5bce}]" -2025-01-28T14:48:09.429Z [INFO] server-bfb4c83d-d7eb-2b5b-06d5-a0612343b7d9: entering follower state: follower="Node at bfb4c83d-d7eb-2b5b-06d5-a0612343b7d9 [Follower]" leader-address= leader-id= -2025-01-28T14:48:09.430Z [INFO] server-cd8ee909-9a12-9e35-2c85-7164cba4429d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bfb4c83d-d7eb-2b5b-06d5-a0612343b7d9 Address:bfb4c83d-d7eb-2b5b-06d5-a0612343b7d9} {Suffrage:Voter ID:server-cd8ee909-9a12-9e35-2c85-7164cba4429d Address:cd8ee909-9a12-9e35-2c85-7164cba4429d} {Suffrage:Voter ID:server-7c555bf9-8b41-2ade-331c-b3045e0b5bce Address:7c555bf9-8b41-2ade-331c-b3045e0b5bce}]" -2025-01-28T14:48:09.430Z [INFO] server-cd8ee909-9a12-9e35-2c85-7164cba4429d: entering follower state: follower="Node at cd8ee909-9a12-9e35-2c85-7164cba4429d [Follower]" leader-address= leader-id= -2025-01-28T14:48:09.430Z [INFO] server-7c555bf9-8b41-2ade-331c-b3045e0b5bce: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bfb4c83d-d7eb-2b5b-06d5-a0612343b7d9 Address:bfb4c83d-d7eb-2b5b-06d5-a0612343b7d9} {Suffrage:Voter ID:server-cd8ee909-9a12-9e35-2c85-7164cba4429d Address:cd8ee909-9a12-9e35-2c85-7164cba4429d} {Suffrage:Voter ID:server-7c555bf9-8b41-2ade-331c-b3045e0b5bce Address:7c555bf9-8b41-2ade-331c-b3045e0b5bce}]" -2025-01-28T14:48:09.431Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:09.431Z [INFO] server-7c555bf9-8b41-2ade-331c-b3045e0b5bce: entering follower state: follower="Node at 7c555bf9-8b41-2ade-331c-b3045e0b5bce [Follower]" leader-address= leader-id= -2025-01-28T14:48:10.211Z [WARN] server-cd8ee909-9a12-9e35-2c85-7164cba4429d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:10.211Z [INFO] server-cd8ee909-9a12-9e35-2c85-7164cba4429d: entering candidate state: node="Node at cd8ee909-9a12-9e35-2c85-7164cba4429d [Candidate]" term=2 -2025-01-28T14:48:10.212Z [INFO] server-cd8ee909-9a12-9e35-2c85-7164cba4429d: election won: term=2 tally=2 -2025-01-28T14:48:10.212Z [INFO] server-cd8ee909-9a12-9e35-2c85-7164cba4429d: entering leader state: leader="Node at cd8ee909-9a12-9e35-2c85-7164cba4429d [Leader]" -2025-01-28T14:48:10.212Z [INFO] server-cd8ee909-9a12-9e35-2c85-7164cba4429d: added peer, starting replication: peer=server-bfb4c83d-d7eb-2b5b-06d5-a0612343b7d9 -2025-01-28T14:48:10.212Z [INFO] server-cd8ee909-9a12-9e35-2c85-7164cba4429d: added peer, starting replication: peer=server-7c555bf9-8b41-2ade-331c-b3045e0b5bce -2025-01-28T14:48:10.213Z [INFO] server-cd8ee909-9a12-9e35-2c85-7164cba4429d: pipelining replication: peer="{Voter server-7c555bf9-8b41-2ade-331c-b3045e0b5bce 7c555bf9-8b41-2ade-331c-b3045e0b5bce}" -2025-01-28T14:48:10.215Z [INFO] server-cd8ee909-9a12-9e35-2c85-7164cba4429d: pipelining replication: peer="{Voter server-bfb4c83d-d7eb-2b5b-06d5-a0612343b7d9 bfb4c83d-d7eb-2b5b-06d5-a0612343b7d9}" -2025-01-28T14:48:11.218Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:10.213207553 +0000 UTC m=+73.296612577 (1 nodes), 782.010555ms from start of poll, 784.621966ms from cluster start. Timeout at 2025-01-28 14:48:11.218217219 +0000 UTC m=+74.301621910, 1.005009333s after stability -2025-01-28T14:48:11.222Z [INFO] server-cd8ee909-9a12-9e35-2c85-7164cba4429d: starting snapshot up to: index=12 -2025-01-28T14:48:11.222Z [INFO] creating new snapshot: path=/tmp/raft1853178945/snapshots/2-12-1738075691222.tmp -2025-01-28T14:48:11.228Z [INFO] server-cd8ee909-9a12-9e35-2c85-7164cba4429d: snapshot complete up to: index=12 -2025-01-28T14:48:11.249Z [INFO] creating new snapshot: path=/tmp/raft1853178945/snapshots/2-1013-1738075691249.tmp -2025-01-28T14:48:11.253Z [INFO] server-cd8ee909-9a12-9e35-2c85-7164cba4429d: copied to local snapshot: bytes=71 -2025-01-28T14:48:11.254Z [INFO] server-cd8ee909-9a12-9e35-2c85-7164cba4429d: snapshot restore progress: id=2-1013-1738075691249 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:11.255Z [INFO] server-cd8ee909-9a12-9e35-2c85-7164cba4429d: restored user snapshot: index=1013 -2025-01-28T14:48:11.255Z [ERROR] server-cd8ee909-9a12-9e35-2c85-7164cba4429d: failed to get log: index=23 error="log not found" -2025-01-28T14:48:11.255Z [INFO] server-cd8ee909-9a12-9e35-2c85-7164cba4429d: aborting pipeline replication: peer="{Voter server-bfb4c83d-d7eb-2b5b-06d5-a0612343b7d9 bfb4c83d-d7eb-2b5b-06d5-a0612343b7d9}" -2025-01-28T14:48:11.255Z [ERROR] server-cd8ee909-9a12-9e35-2c85-7164cba4429d: failed to get log: index=23 error="log not found" -2025-01-28T14:48:11.255Z [INFO] server-cd8ee909-9a12-9e35-2c85-7164cba4429d: aborting pipeline replication: peer="{Voter server-7c555bf9-8b41-2ade-331c-b3045e0b5bce 7c555bf9-8b41-2ade-331c-b3045e0b5bce}" -2025-01-28T14:48:11.263Z [ERROR] server-cd8ee909-9a12-9e35-2c85-7164cba4429d: failed to get log: index=23 error="log not found" -2025-01-28T14:48:11.263Z [ERROR] server-cd8ee909-9a12-9e35-2c85-7164cba4429d: failed to get log: index=23 error="log not found" -2025-01-28T14:48:11.265Z [INFO] creating new snapshot: path=/tmp/raft4057434078/snapshots/2-1013-1738075691265.tmp -2025-01-28T14:48:11.266Z [INFO] creating new snapshot: path=/tmp/raft2952460223/snapshots/2-1013-1738075691266.tmp -2025-01-28T14:48:11.268Z [INFO] server-7c555bf9-8b41-2ade-331c-b3045e0b5bce: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:11.268Z [INFO] server-bfb4c83d-d7eb-2b5b-06d5-a0612343b7d9: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:11.273Z [INFO] server-bfb4c83d-d7eb-2b5b-06d5-a0612343b7d9: copied to local snapshot: bytes=71 -2025-01-28T14:48:11.274Z [INFO] server-bfb4c83d-d7eb-2b5b-06d5-a0612343b7d9: snapshot restore progress: id=2-1013-1738075691265 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:11.274Z [INFO] server-bfb4c83d-d7eb-2b5b-06d5-a0612343b7d9: Installed remote snapshot -2025-01-28T14:48:11.275Z [INFO] server-cd8ee909-9a12-9e35-2c85-7164cba4429d: pipelining replication: peer="{Voter server-bfb4c83d-d7eb-2b5b-06d5-a0612343b7d9 bfb4c83d-d7eb-2b5b-06d5-a0612343b7d9}" -2025-01-28T14:48:11.276Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:48:11.278Z [INFO] server-7c555bf9-8b41-2ade-331c-b3045e0b5bce: copied to local snapshot: bytes=71 -2025-01-28T14:48:11.280Z [INFO] server-7c555bf9-8b41-2ade-331c-b3045e0b5bce: snapshot restore progress: id=2-1013-1738075691266 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:11.280Z [INFO] server-7c555bf9-8b41-2ade-331c-b3045e0b5bce: Installed remote snapshot -2025-01-28T14:48:11.281Z [INFO] server-cd8ee909-9a12-9e35-2c85-7164cba4429d: pipelining replication: peer="{Voter server-7c555bf9-8b41-2ade-331c-b3045e0b5bce 7c555bf9-8b41-2ade-331c-b3045e0b5bce}" -2025-01-28T14:48:12.282Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:48:11.276536795 +0000 UTC m=+74.359941819 (2 nodes), 6.667µs from start of poll, 1.847951208s from cluster start. Timeout at 2025-01-28 14:48:12.281784136 +0000 UTC m=+75.365189493, 1.005247674s after stability -2025-01-28T14:48:12.296Z [INFO] server-cd8ee909-9a12-9e35-2c85-7164cba4429d: aborting pipeline replication: peer="{Voter server-7c555bf9-8b41-2ade-331c-b3045e0b5bce 7c555bf9-8b41-2ade-331c-b3045e0b5bce}" -2025-01-28T14:48:12.296Z [INFO] server-cd8ee909-9a12-9e35-2c85-7164cba4429d: aborting pipeline replication: peer="{Voter server-bfb4c83d-d7eb-2b5b-06d5-a0612343b7d9 bfb4c83d-d7eb-2b5b-06d5-a0612343b7d9}" +2025-01-28T14:55:11.071Z [INFO] server-bb466cd6-b8f1-2de2-0eff-bca81df6afff: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bb466cd6-b8f1-2de2-0eff-bca81df6afff Address:bb466cd6-b8f1-2de2-0eff-bca81df6afff} {Suffrage:Voter ID:server-a89d7eb1-497f-06ca-e6ee-77d5fc0d91dc Address:a89d7eb1-497f-06ca-e6ee-77d5fc0d91dc} {Suffrage:Voter ID:server-b6f8f0b9-53f5-0bd4-18e9-f06acaaffffe Address:b6f8f0b9-53f5-0bd4-18e9-f06acaaffffe}]" +2025-01-28T14:55:11.071Z [INFO] server-bb466cd6-b8f1-2de2-0eff-bca81df6afff: entering follower state: follower="Node at bb466cd6-b8f1-2de2-0eff-bca81df6afff [Follower]" leader-address= leader-id= +2025-01-28T14:55:11.071Z [INFO] server-a89d7eb1-497f-06ca-e6ee-77d5fc0d91dc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bb466cd6-b8f1-2de2-0eff-bca81df6afff Address:bb466cd6-b8f1-2de2-0eff-bca81df6afff} {Suffrage:Voter ID:server-a89d7eb1-497f-06ca-e6ee-77d5fc0d91dc Address:a89d7eb1-497f-06ca-e6ee-77d5fc0d91dc} {Suffrage:Voter ID:server-b6f8f0b9-53f5-0bd4-18e9-f06acaaffffe Address:b6f8f0b9-53f5-0bd4-18e9-f06acaaffffe}]" +2025-01-28T14:55:11.071Z [INFO] server-a89d7eb1-497f-06ca-e6ee-77d5fc0d91dc: entering follower state: follower="Node at a89d7eb1-497f-06ca-e6ee-77d5fc0d91dc [Follower]" leader-address= leader-id= +2025-01-28T14:55:11.071Z [INFO] server-b6f8f0b9-53f5-0bd4-18e9-f06acaaffffe: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bb466cd6-b8f1-2de2-0eff-bca81df6afff Address:bb466cd6-b8f1-2de2-0eff-bca81df6afff} {Suffrage:Voter ID:server-a89d7eb1-497f-06ca-e6ee-77d5fc0d91dc Address:a89d7eb1-497f-06ca-e6ee-77d5fc0d91dc} {Suffrage:Voter ID:server-b6f8f0b9-53f5-0bd4-18e9-f06acaaffffe Address:b6f8f0b9-53f5-0bd4-18e9-f06acaaffffe}]" +2025-01-28T14:55:11.071Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:11.071Z [INFO] server-b6f8f0b9-53f5-0bd4-18e9-f06acaaffffe: entering follower state: follower="Node at b6f8f0b9-53f5-0bd4-18e9-f06acaaffffe [Follower]" leader-address= leader-id= +2025-01-28T14:55:11.653Z [WARN] server-b6f8f0b9-53f5-0bd4-18e9-f06acaaffffe: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:11.653Z [INFO] server-b6f8f0b9-53f5-0bd4-18e9-f06acaaffffe: entering candidate state: node="Node at b6f8f0b9-53f5-0bd4-18e9-f06acaaffffe [Candidate]" term=2 +2025-01-28T14:55:11.653Z [INFO] server-b6f8f0b9-53f5-0bd4-18e9-f06acaaffffe: election won: term=2 tally=2 +2025-01-28T14:55:11.653Z [INFO] server-b6f8f0b9-53f5-0bd4-18e9-f06acaaffffe: entering leader state: leader="Node at b6f8f0b9-53f5-0bd4-18e9-f06acaaffffe [Leader]" +2025-01-28T14:55:11.653Z [INFO] server-b6f8f0b9-53f5-0bd4-18e9-f06acaaffffe: added peer, starting replication: peer=server-bb466cd6-b8f1-2de2-0eff-bca81df6afff +2025-01-28T14:55:11.653Z [INFO] server-b6f8f0b9-53f5-0bd4-18e9-f06acaaffffe: added peer, starting replication: peer=server-a89d7eb1-497f-06ca-e6ee-77d5fc0d91dc +2025-01-28T14:55:11.654Z [INFO] server-b6f8f0b9-53f5-0bd4-18e9-f06acaaffffe: pipelining replication: peer="{Voter server-a89d7eb1-497f-06ca-e6ee-77d5fc0d91dc a89d7eb1-497f-06ca-e6ee-77d5fc0d91dc}" +2025-01-28T14:55:11.654Z [INFO] server-b6f8f0b9-53f5-0bd4-18e9-f06acaaffffe: pipelining replication: peer="{Voter server-bb466cd6-b8f1-2de2-0eff-bca81df6afff bb466cd6-b8f1-2de2-0eff-bca81df6afff}" +2025-01-28T14:55:12.659Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:11.653950622 +0000 UTC m=+97.328440602 (1 nodes), 581.997096ms from start of poll, 582.959466ms from cluster start. Timeout at 2025-01-28 14:55:12.658955241 +0000 UTC m=+98.333445241, 1.005004639s after stability +2025-01-28T14:55:12.660Z [INFO] server-b6f8f0b9-53f5-0bd4-18e9-f06acaaffffe: starting snapshot up to: index=12 +2025-01-28T14:55:12.660Z [INFO] creating new snapshot: path=/tmp/raft1038531717/snapshots/2-12-1738076112660.tmp +2025-01-28T14:55:12.665Z [INFO] server-b6f8f0b9-53f5-0bd4-18e9-f06acaaffffe: snapshot complete up to: index=12 +2025-01-28T14:55:12.666Z [INFO] creating new snapshot: path=/tmp/raft1038531717/snapshots/2-1013-1738076112666.tmp +2025-01-28T14:55:12.668Z [INFO] server-b6f8f0b9-53f5-0bd4-18e9-f06acaaffffe: copied to local snapshot: bytes=71 +2025-01-28T14:55:12.668Z [INFO] server-b6f8f0b9-53f5-0bd4-18e9-f06acaaffffe: snapshot restore progress: id=2-1013-1738076112666 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:12.669Z [INFO] server-b6f8f0b9-53f5-0bd4-18e9-f06acaaffffe: restored user snapshot: index=1013 +2025-01-28T14:55:12.669Z [ERROR] server-b6f8f0b9-53f5-0bd4-18e9-f06acaaffffe: failed to get log: index=23 error="log not found" +2025-01-28T14:55:12.669Z [INFO] server-b6f8f0b9-53f5-0bd4-18e9-f06acaaffffe: aborting pipeline replication: peer="{Voter server-a89d7eb1-497f-06ca-e6ee-77d5fc0d91dc a89d7eb1-497f-06ca-e6ee-77d5fc0d91dc}" +2025-01-28T14:55:12.669Z [ERROR] server-b6f8f0b9-53f5-0bd4-18e9-f06acaaffffe: failed to get log: index=23 error="log not found" +2025-01-28T14:55:12.669Z [INFO] server-b6f8f0b9-53f5-0bd4-18e9-f06acaaffffe: aborting pipeline replication: peer="{Voter server-bb466cd6-b8f1-2de2-0eff-bca81df6afff bb466cd6-b8f1-2de2-0eff-bca81df6afff}" +2025-01-28T14:55:12.676Z [ERROR] server-b6f8f0b9-53f5-0bd4-18e9-f06acaaffffe: failed to get log: index=23 error="log not found" +2025-01-28T14:55:12.677Z [INFO] creating new snapshot: path=/tmp/raft687380656/snapshots/2-1013-1738076112677.tmp +2025-01-28T14:55:12.677Z [INFO] server-bb466cd6-b8f1-2de2-0eff-bca81df6afff: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:12.678Z [INFO] server-bb466cd6-b8f1-2de2-0eff-bca81df6afff: copied to local snapshot: bytes=71 +2025-01-28T14:55:12.678Z [ERROR] server-b6f8f0b9-53f5-0bd4-18e9-f06acaaffffe: failed to get log: index=23 error="log not found" +2025-01-28T14:55:12.678Z [INFO] server-bb466cd6-b8f1-2de2-0eff-bca81df6afff: snapshot restore progress: id=2-1013-1738076112677 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:12.679Z [INFO] server-bb466cd6-b8f1-2de2-0eff-bca81df6afff: Installed remote snapshot +2025-01-28T14:55:12.679Z [WARN] server-bb466cd6-b8f1-2de2-0eff-bca81df6afff: skipping application of old log: index=22 +2025-01-28T14:55:12.679Z [INFO] server-b6f8f0b9-53f5-0bd4-18e9-f06acaaffffe: pipelining replication: peer="{Voter server-bb466cd6-b8f1-2de2-0eff-bca81df6afff bb466cd6-b8f1-2de2-0eff-bca81df6afff}" +2025-01-28T14:55:12.679Z [INFO] creating new snapshot: path=/tmp/raft3736291733/snapshots/2-1013-1738076112679.tmp +2025-01-28T14:55:12.679Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:55:12.680Z [INFO] server-a89d7eb1-497f-06ca-e6ee-77d5fc0d91dc: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:12.680Z [INFO] server-a89d7eb1-497f-06ca-e6ee-77d5fc0d91dc: copied to local snapshot: bytes=71 +2025-01-28T14:55:12.681Z [INFO] server-a89d7eb1-497f-06ca-e6ee-77d5fc0d91dc: snapshot restore progress: id=2-1013-1738076112679 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:12.681Z [INFO] server-a89d7eb1-497f-06ca-e6ee-77d5fc0d91dc: Installed remote snapshot +2025-01-28T14:55:12.681Z [INFO] server-b6f8f0b9-53f5-0bd4-18e9-f06acaaffffe: pipelining replication: peer="{Voter server-a89d7eb1-497f-06ca-e6ee-77d5fc0d91dc a89d7eb1-497f-06ca-e6ee-77d5fc0d91dc}" +2025-01-28T14:55:13.685Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:55:12.679688177 +0000 UTC m=+98.354178157 (2 nodes), 2.3µs from start of poll, 1.608697021s from cluster start. Timeout at 2025-01-28 14:55:13.684784008 +0000 UTC m=+99.359274008, 1.005095851s after stability +2025-01-28T14:55:13.697Z [INFO] server-b6f8f0b9-53f5-0bd4-18e9-f06acaaffffe: aborting pipeline replication: peer="{Voter server-a89d7eb1-497f-06ca-e6ee-77d5fc0d91dc a89d7eb1-497f-06ca-e6ee-77d5fc0d91dc}" +2025-01-28T14:55:13.697Z [INFO] server-b6f8f0b9-53f5-0bd4-18e9-f06acaaffffe: aborting pipeline replication: peer="{Voter server-bb466cd6-b8f1-2de2-0eff-bca81df6afff bb466cd6-b8f1-2de2-0eff-bca81df6afff}" === RUN TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_false -2025-01-28T14:48:12.309Z [INFO] server-342a93e5-d0a2-3cae-43f2-29f68eb32825: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-342a93e5-d0a2-3cae-43f2-29f68eb32825 Address:342a93e5-d0a2-3cae-43f2-29f68eb32825} {Suffrage:Voter ID:server-ec198e85-9faf-b877-8970-04c3b24214cb Address:ec198e85-9faf-b877-8970-04c3b24214cb} {Suffrage:Voter ID:server-fe702b79-a2fb-8bd2-23c1-1567725bd0e8 Address:fe702b79-a2fb-8bd2-23c1-1567725bd0e8}]" -2025-01-28T14:48:12.310Z [INFO] server-ec198e85-9faf-b877-8970-04c3b24214cb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-342a93e5-d0a2-3cae-43f2-29f68eb32825 Address:342a93e5-d0a2-3cae-43f2-29f68eb32825} {Suffrage:Voter ID:server-ec198e85-9faf-b877-8970-04c3b24214cb Address:ec198e85-9faf-b877-8970-04c3b24214cb} {Suffrage:Voter ID:server-fe702b79-a2fb-8bd2-23c1-1567725bd0e8 Address:fe702b79-a2fb-8bd2-23c1-1567725bd0e8}]" -2025-01-28T14:48:12.310Z [INFO] server-fe702b79-a2fb-8bd2-23c1-1567725bd0e8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-342a93e5-d0a2-3cae-43f2-29f68eb32825 Address:342a93e5-d0a2-3cae-43f2-29f68eb32825} {Suffrage:Voter ID:server-ec198e85-9faf-b877-8970-04c3b24214cb Address:ec198e85-9faf-b877-8970-04c3b24214cb} {Suffrage:Voter ID:server-fe702b79-a2fb-8bd2-23c1-1567725bd0e8 Address:fe702b79-a2fb-8bd2-23c1-1567725bd0e8}]" -2025-01-28T14:48:12.311Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:12.311Z [INFO] server-342a93e5-d0a2-3cae-43f2-29f68eb32825: entering follower state: follower="Node at 342a93e5-d0a2-3cae-43f2-29f68eb32825 [Follower]" leader-address= leader-id= -2025-01-28T14:48:12.311Z [INFO] server-ec198e85-9faf-b877-8970-04c3b24214cb: entering follower state: follower="Node at ec198e85-9faf-b877-8970-04c3b24214cb [Follower]" leader-address= leader-id= -2025-01-28T14:48:12.311Z [INFO] server-fe702b79-a2fb-8bd2-23c1-1567725bd0e8: entering follower state: follower="Node at fe702b79-a2fb-8bd2-23c1-1567725bd0e8 [Follower]" leader-address= leader-id= -2025-01-28T14:48:12.845Z [WARN] server-ec198e85-9faf-b877-8970-04c3b24214cb: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:12.846Z [INFO] server-ec198e85-9faf-b877-8970-04c3b24214cb: entering candidate state: node="Node at ec198e85-9faf-b877-8970-04c3b24214cb [Candidate]" term=2 -2025-01-28T14:48:12.846Z [INFO] server-ec198e85-9faf-b877-8970-04c3b24214cb: election won: term=2 tally=2 -2025-01-28T14:48:12.846Z [INFO] server-ec198e85-9faf-b877-8970-04c3b24214cb: entering leader state: leader="Node at ec198e85-9faf-b877-8970-04c3b24214cb [Leader]" -2025-01-28T14:48:12.846Z [INFO] server-ec198e85-9faf-b877-8970-04c3b24214cb: added peer, starting replication: peer=server-342a93e5-d0a2-3cae-43f2-29f68eb32825 -2025-01-28T14:48:12.846Z [INFO] server-ec198e85-9faf-b877-8970-04c3b24214cb: added peer, starting replication: peer=server-fe702b79-a2fb-8bd2-23c1-1567725bd0e8 -2025-01-28T14:48:12.847Z [INFO] server-ec198e85-9faf-b877-8970-04c3b24214cb: pipelining replication: peer="{Voter server-fe702b79-a2fb-8bd2-23c1-1567725bd0e8 fe702b79-a2fb-8bd2-23c1-1567725bd0e8}" -2025-01-28T14:48:12.847Z [INFO] server-ec198e85-9faf-b877-8970-04c3b24214cb: pipelining replication: peer="{Voter server-342a93e5-d0a2-3cae-43f2-29f68eb32825 342a93e5-d0a2-3cae-43f2-29f68eb32825}" -2025-01-28T14:48:13.857Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:12.847337106 +0000 UTC m=+75.930741797 (1 nodes), 536.097919ms from start of poll, 538.46599ms from cluster start. Timeout at 2025-01-28 14:48:13.852426109 +0000 UTC m=+76.935830800, 1.005089003s after stability -2025-01-28T14:48:13.872Z [INFO] server-ec198e85-9faf-b877-8970-04c3b24214cb: starting snapshot up to: index=12 -2025-01-28T14:48:13.872Z [INFO] creating new snapshot: path=/tmp/raft2419961856/snapshots/2-12-1738075693872.tmp -2025-01-28T14:48:13.878Z [INFO] server-ec198e85-9faf-b877-8970-04c3b24214cb: snapshot complete up to: index=12 -2025-01-28T14:48:13.885Z [INFO] creating new snapshot: path=/tmp/raft2419961856/snapshots/2-1013-1738075693885.tmp -2025-01-28T14:48:13.890Z [INFO] server-ec198e85-9faf-b877-8970-04c3b24214cb: copied to local snapshot: bytes=71 -2025-01-28T14:48:13.891Z [INFO] server-ec198e85-9faf-b877-8970-04c3b24214cb: snapshot restore progress: id=2-1013-1738075693885 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:13.892Z [INFO] server-ec198e85-9faf-b877-8970-04c3b24214cb: removing all old logs from log store -2025-01-28T14:48:13.892Z [INFO] server-ec198e85-9faf-b877-8970-04c3b24214cb: compacting logs: from=1 to=22 -2025-01-28T14:48:13.892Z [INFO] server-ec198e85-9faf-b877-8970-04c3b24214cb: restored user snapshot: index=1013 -2025-01-28T14:48:13.892Z [ERROR] server-ec198e85-9faf-b877-8970-04c3b24214cb: failed to get log: index=22 error="log not found" -2025-01-28T14:48:13.892Z [INFO] server-ec198e85-9faf-b877-8970-04c3b24214cb: aborting pipeline replication: peer="{Voter server-fe702b79-a2fb-8bd2-23c1-1567725bd0e8 fe702b79-a2fb-8bd2-23c1-1567725bd0e8}" -2025-01-28T14:48:13.892Z [ERROR] server-ec198e85-9faf-b877-8970-04c3b24214cb: failed to get log: index=22 error="log not found" -2025-01-28T14:48:13.893Z [INFO] server-ec198e85-9faf-b877-8970-04c3b24214cb: aborting pipeline replication: peer="{Voter server-342a93e5-d0a2-3cae-43f2-29f68eb32825 342a93e5-d0a2-3cae-43f2-29f68eb32825}" -2025-01-28T14:48:13.900Z [ERROR] server-ec198e85-9faf-b877-8970-04c3b24214cb: failed to get log: index=22 error="log not found" -2025-01-28T14:48:13.903Z [ERROR] server-ec198e85-9faf-b877-8970-04c3b24214cb: failed to get log: index=22 error="log not found" -2025-01-28T14:48:13.903Z [INFO] creating new snapshot: path=/tmp/raft2104220590/snapshots/2-1013-1738075693903.tmp -2025-01-28T14:48:13.905Z [INFO] server-fe702b79-a2fb-8bd2-23c1-1567725bd0e8: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:13.910Z [INFO] server-fe702b79-a2fb-8bd2-23c1-1567725bd0e8: copied to local snapshot: bytes=71 -2025-01-28T14:48:13.911Z [INFO] server-fe702b79-a2fb-8bd2-23c1-1567725bd0e8: snapshot restore progress: id=2-1013-1738075693903 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:13.911Z [INFO] server-fe702b79-a2fb-8bd2-23c1-1567725bd0e8: removing all old logs from log store -2025-01-28T14:48:13.911Z [INFO] server-fe702b79-a2fb-8bd2-23c1-1567725bd0e8: compacting logs: from=1 to=22 -2025-01-28T14:48:13.911Z [INFO] server-fe702b79-a2fb-8bd2-23c1-1567725bd0e8: Installed remote snapshot -2025-01-28T14:48:13.912Z [INFO] creating new snapshot: path=/tmp/raft1238313485/snapshots/2-1013-1738075693912.tmp -2025-01-28T14:48:13.913Z [INFO] server-ec198e85-9faf-b877-8970-04c3b24214cb: pipelining replication: peer="{Voter server-fe702b79-a2fb-8bd2-23c1-1567725bd0e8 fe702b79-a2fb-8bd2-23c1-1567725bd0e8}" -2025-01-28T14:48:13.913Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:48:13.914Z [INFO] server-342a93e5-d0a2-3cae-43f2-29f68eb32825: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:13.917Z [INFO] server-342a93e5-d0a2-3cae-43f2-29f68eb32825: copied to local snapshot: bytes=71 -2025-01-28T14:48:13.918Z [INFO] server-342a93e5-d0a2-3cae-43f2-29f68eb32825: snapshot restore progress: id=2-1013-1738075693912 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:13.919Z [INFO] server-342a93e5-d0a2-3cae-43f2-29f68eb32825: removing all old logs from log store -2025-01-28T14:48:13.919Z [INFO] server-342a93e5-d0a2-3cae-43f2-29f68eb32825: compacting logs: from=1 to=22 -2025-01-28T14:48:13.919Z [INFO] server-342a93e5-d0a2-3cae-43f2-29f68eb32825: Installed remote snapshot -2025-01-28T14:48:13.920Z [INFO] server-ec198e85-9faf-b877-8970-04c3b24214cb: pipelining replication: peer="{Voter server-342a93e5-d0a2-3cae-43f2-29f68eb32825 342a93e5-d0a2-3cae-43f2-29f68eb32825}" -2025-01-28T14:48:14.920Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:48:13.913615102 +0000 UTC m=+76.997019793 (2 nodes), 6.667µs from start of poll, 1.604743986s from cluster start. Timeout at 2025-01-28 14:48:14.918839443 +0000 UTC m=+78.002244134, 1.005224341s after stability -2025-01-28T14:48:14.935Z [INFO] server-ec198e85-9faf-b877-8970-04c3b24214cb: aborting pipeline replication: peer="{Voter server-fe702b79-a2fb-8bd2-23c1-1567725bd0e8 fe702b79-a2fb-8bd2-23c1-1567725bd0e8}" -2025-01-28T14:48:14.935Z [INFO] server-ec198e85-9faf-b877-8970-04c3b24214cb: aborting pipeline replication: peer="{Voter server-342a93e5-d0a2-3cae-43f2-29f68eb32825 342a93e5-d0a2-3cae-43f2-29f68eb32825}" +2025-01-28T14:55:13.702Z [INFO] server-9c7a1ee8-5a4f-d630-10e3-905196f37b7d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9c7a1ee8-5a4f-d630-10e3-905196f37b7d Address:9c7a1ee8-5a4f-d630-10e3-905196f37b7d} {Suffrage:Voter ID:server-978e1e58-ba7b-37c8-8e62-05f69c058a0d Address:978e1e58-ba7b-37c8-8e62-05f69c058a0d} {Suffrage:Voter ID:server-dad33df4-91fd-2588-99d0-ef28e498779d Address:dad33df4-91fd-2588-99d0-ef28e498779d}]" +2025-01-28T14:55:13.702Z [INFO] server-9c7a1ee8-5a4f-d630-10e3-905196f37b7d: entering follower state: follower="Node at 9c7a1ee8-5a4f-d630-10e3-905196f37b7d [Follower]" leader-address= leader-id= +2025-01-28T14:55:13.702Z [INFO] server-978e1e58-ba7b-37c8-8e62-05f69c058a0d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9c7a1ee8-5a4f-d630-10e3-905196f37b7d Address:9c7a1ee8-5a4f-d630-10e3-905196f37b7d} {Suffrage:Voter ID:server-978e1e58-ba7b-37c8-8e62-05f69c058a0d Address:978e1e58-ba7b-37c8-8e62-05f69c058a0d} {Suffrage:Voter ID:server-dad33df4-91fd-2588-99d0-ef28e498779d Address:dad33df4-91fd-2588-99d0-ef28e498779d}]" +2025-01-28T14:55:13.702Z [INFO] server-978e1e58-ba7b-37c8-8e62-05f69c058a0d: entering follower state: follower="Node at 978e1e58-ba7b-37c8-8e62-05f69c058a0d [Follower]" leader-address= leader-id= +2025-01-28T14:55:13.702Z [INFO] server-dad33df4-91fd-2588-99d0-ef28e498779d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9c7a1ee8-5a4f-d630-10e3-905196f37b7d Address:9c7a1ee8-5a4f-d630-10e3-905196f37b7d} {Suffrage:Voter ID:server-978e1e58-ba7b-37c8-8e62-05f69c058a0d Address:978e1e58-ba7b-37c8-8e62-05f69c058a0d} {Suffrage:Voter ID:server-dad33df4-91fd-2588-99d0-ef28e498779d Address:dad33df4-91fd-2588-99d0-ef28e498779d}]" +2025-01-28T14:55:13.703Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:13.703Z [INFO] server-dad33df4-91fd-2588-99d0-ef28e498779d: entering follower state: follower="Node at dad33df4-91fd-2588-99d0-ef28e498779d [Follower]" leader-address= leader-id= +2025-01-28T14:55:14.252Z [WARN] server-978e1e58-ba7b-37c8-8e62-05f69c058a0d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:14.252Z [INFO] server-978e1e58-ba7b-37c8-8e62-05f69c058a0d: entering candidate state: node="Node at 978e1e58-ba7b-37c8-8e62-05f69c058a0d [Candidate]" term=2 +2025-01-28T14:55:14.252Z [INFO] server-978e1e58-ba7b-37c8-8e62-05f69c058a0d: election won: term=2 tally=2 +2025-01-28T14:55:14.252Z [INFO] server-978e1e58-ba7b-37c8-8e62-05f69c058a0d: entering leader state: leader="Node at 978e1e58-ba7b-37c8-8e62-05f69c058a0d [Leader]" +2025-01-28T14:55:14.252Z [INFO] server-978e1e58-ba7b-37c8-8e62-05f69c058a0d: added peer, starting replication: peer=server-9c7a1ee8-5a4f-d630-10e3-905196f37b7d +2025-01-28T14:55:14.252Z [INFO] server-978e1e58-ba7b-37c8-8e62-05f69c058a0d: added peer, starting replication: peer=server-dad33df4-91fd-2588-99d0-ef28e498779d +2025-01-28T14:55:14.252Z [INFO] server-978e1e58-ba7b-37c8-8e62-05f69c058a0d: pipelining replication: peer="{Voter server-dad33df4-91fd-2588-99d0-ef28e498779d dad33df4-91fd-2588-99d0-ef28e498779d}" +2025-01-28T14:55:14.252Z [INFO] server-978e1e58-ba7b-37c8-8e62-05f69c058a0d: pipelining replication: peer="{Voter server-9c7a1ee8-5a4f-d630-10e3-905196f37b7d 9c7a1ee8-5a4f-d630-10e3-905196f37b7d}" +2025-01-28T14:55:15.258Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:14.25252776 +0000 UTC m=+99.927017740 (1 nodes), 549.41277ms from start of poll, 550.377179ms from cluster start. Timeout at 2025-01-28 14:55:15.2575307 +0000 UTC m=+100.932020740, 1.005003s after stability +2025-01-28T14:55:15.263Z [INFO] server-978e1e58-ba7b-37c8-8e62-05f69c058a0d: starting snapshot up to: index=12 +2025-01-28T14:55:15.263Z [INFO] creating new snapshot: path=/tmp/raft696981668/snapshots/2-12-1738076115263.tmp +2025-01-28T14:55:15.265Z [INFO] server-978e1e58-ba7b-37c8-8e62-05f69c058a0d: snapshot complete up to: index=12 +2025-01-28T14:55:15.266Z [INFO] creating new snapshot: path=/tmp/raft696981668/snapshots/2-1013-1738076115266.tmp +2025-01-28T14:55:15.268Z [INFO] server-978e1e58-ba7b-37c8-8e62-05f69c058a0d: copied to local snapshot: bytes=71 +2025-01-28T14:55:15.268Z [INFO] server-978e1e58-ba7b-37c8-8e62-05f69c058a0d: snapshot restore progress: id=2-1013-1738076115266 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:15.268Z [INFO] server-978e1e58-ba7b-37c8-8e62-05f69c058a0d: removing all old logs from log store +2025-01-28T14:55:15.268Z [INFO] server-978e1e58-ba7b-37c8-8e62-05f69c058a0d: compacting logs: from=1 to=22 +2025-01-28T14:55:15.268Z [INFO] server-978e1e58-ba7b-37c8-8e62-05f69c058a0d: restored user snapshot: index=1013 +2025-01-28T14:55:15.269Z [ERROR] server-978e1e58-ba7b-37c8-8e62-05f69c058a0d: failed to get log: index=22 error="log not found" +2025-01-28T14:55:15.269Z [ERROR] server-978e1e58-ba7b-37c8-8e62-05f69c058a0d: failed to get log: index=22 error="log not found" +2025-01-28T14:55:15.269Z [INFO] server-978e1e58-ba7b-37c8-8e62-05f69c058a0d: aborting pipeline replication: peer="{Voter server-dad33df4-91fd-2588-99d0-ef28e498779d dad33df4-91fd-2588-99d0-ef28e498779d}" +2025-01-28T14:55:15.269Z [INFO] server-978e1e58-ba7b-37c8-8e62-05f69c058a0d: aborting pipeline replication: peer="{Voter server-9c7a1ee8-5a4f-d630-10e3-905196f37b7d 9c7a1ee8-5a4f-d630-10e3-905196f37b7d}" +2025-01-28T14:55:15.276Z [ERROR] server-978e1e58-ba7b-37c8-8e62-05f69c058a0d: failed to get log: index=22 error="log not found" +2025-01-28T14:55:15.277Z [INFO] creating new snapshot: path=/tmp/raft4006968070/snapshots/2-1013-1738076115277.tmp +2025-01-28T14:55:15.277Z [INFO] server-9c7a1ee8-5a4f-d630-10e3-905196f37b7d: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:15.278Z [INFO] server-9c7a1ee8-5a4f-d630-10e3-905196f37b7d: copied to local snapshot: bytes=71 +2025-01-28T14:55:15.278Z [ERROR] server-978e1e58-ba7b-37c8-8e62-05f69c058a0d: failed to get log: index=22 error="log not found" +2025-01-28T14:55:15.279Z [INFO] server-9c7a1ee8-5a4f-d630-10e3-905196f37b7d: snapshot restore progress: id=2-1013-1738076115277 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:15.279Z [INFO] server-9c7a1ee8-5a4f-d630-10e3-905196f37b7d: removing all old logs from log store +2025-01-28T14:55:15.279Z [INFO] server-9c7a1ee8-5a4f-d630-10e3-905196f37b7d: compacting logs: from=1 to=22 +2025-01-28T14:55:15.279Z [INFO] server-9c7a1ee8-5a4f-d630-10e3-905196f37b7d: Installed remote snapshot +2025-01-28T14:55:15.279Z [WARN] server-9c7a1ee8-5a4f-d630-10e3-905196f37b7d: skipping application of old log: index=22 +2025-01-28T14:55:15.279Z [INFO] server-978e1e58-ba7b-37c8-8e62-05f69c058a0d: pipelining replication: peer="{Voter server-9c7a1ee8-5a4f-d630-10e3-905196f37b7d 9c7a1ee8-5a4f-d630-10e3-905196f37b7d}" +2025-01-28T14:55:15.279Z [INFO] creating new snapshot: path=/tmp/raft4185156934/snapshots/2-1013-1738076115279.tmp +2025-01-28T14:55:15.279Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:55:15.280Z [INFO] server-dad33df4-91fd-2588-99d0-ef28e498779d: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:15.281Z [INFO] server-dad33df4-91fd-2588-99d0-ef28e498779d: copied to local snapshot: bytes=71 +2025-01-28T14:55:15.281Z [INFO] server-dad33df4-91fd-2588-99d0-ef28e498779d: snapshot restore progress: id=2-1013-1738076115279 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:15.281Z [INFO] server-dad33df4-91fd-2588-99d0-ef28e498779d: removing all old logs from log store +2025-01-28T14:55:15.281Z [INFO] server-dad33df4-91fd-2588-99d0-ef28e498779d: compacting logs: from=1 to=22 +2025-01-28T14:55:15.281Z [INFO] server-dad33df4-91fd-2588-99d0-ef28e498779d: Installed remote snapshot +2025-01-28T14:55:15.282Z [INFO] server-978e1e58-ba7b-37c8-8e62-05f69c058a0d: pipelining replication: peer="{Voter server-dad33df4-91fd-2588-99d0-ef28e498779d dad33df4-91fd-2588-99d0-ef28e498779d}" +2025-01-28T14:55:16.285Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:55:15.279711259 +0000 UTC m=+100.954201239 (2 nodes), 2.3µs from start of poll, 1.577560678s from cluster start. Timeout at 2025-01-28 14:55:16.284815509 +0000 UTC m=+101.959305509, 1.00510427s after stability +2025-01-28T14:55:16.297Z [INFO] server-978e1e58-ba7b-37c8-8e62-05f69c058a0d: aborting pipeline replication: peer="{Voter server-dad33df4-91fd-2588-99d0-ef28e498779d dad33df4-91fd-2588-99d0-ef28e498779d}" +2025-01-28T14:55:16.297Z [INFO] server-978e1e58-ba7b-37c8-8e62-05f69c058a0d: aborting pipeline replication: peer="{Voter server-9c7a1ee8-5a4f-d630-10e3-905196f37b7d 9c7a1ee8-5a4f-d630-10e3-905196f37b7d}" === RUN TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_true -2025-01-28T14:48:14.970Z [INFO] server-7f6f48a9-7f76-8a0c-321b-fc74a0b1063e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7f6f48a9-7f76-8a0c-321b-fc74a0b1063e Address:7f6f48a9-7f76-8a0c-321b-fc74a0b1063e} {Suffrage:Voter ID:server-0a406d7e-1a30-97a6-97d9-a198341370d4 Address:0a406d7e-1a30-97a6-97d9-a198341370d4} {Suffrage:Voter ID:server-9c47da70-2268-570c-c05c-4a549f3e503a Address:9c47da70-2268-570c-c05c-4a549f3e503a}]" -2025-01-28T14:48:14.970Z [INFO] server-7f6f48a9-7f76-8a0c-321b-fc74a0b1063e: entering follower state: follower="Node at 7f6f48a9-7f76-8a0c-321b-fc74a0b1063e [Follower]" leader-address= leader-id= -2025-01-28T14:48:14.971Z [INFO] server-0a406d7e-1a30-97a6-97d9-a198341370d4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7f6f48a9-7f76-8a0c-321b-fc74a0b1063e Address:7f6f48a9-7f76-8a0c-321b-fc74a0b1063e} {Suffrage:Voter ID:server-0a406d7e-1a30-97a6-97d9-a198341370d4 Address:0a406d7e-1a30-97a6-97d9-a198341370d4} {Suffrage:Voter ID:server-9c47da70-2268-570c-c05c-4a549f3e503a Address:9c47da70-2268-570c-c05c-4a549f3e503a}]" -2025-01-28T14:48:14.971Z [INFO] server-0a406d7e-1a30-97a6-97d9-a198341370d4: entering follower state: follower="Node at 0a406d7e-1a30-97a6-97d9-a198341370d4 [Follower]" leader-address= leader-id= -2025-01-28T14:48:14.971Z [INFO] server-9c47da70-2268-570c-c05c-4a549f3e503a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7f6f48a9-7f76-8a0c-321b-fc74a0b1063e Address:7f6f48a9-7f76-8a0c-321b-fc74a0b1063e} {Suffrage:Voter ID:server-0a406d7e-1a30-97a6-97d9-a198341370d4 Address:0a406d7e-1a30-97a6-97d9-a198341370d4} {Suffrage:Voter ID:server-9c47da70-2268-570c-c05c-4a549f3e503a Address:9c47da70-2268-570c-c05c-4a549f3e503a}]" -2025-01-28T14:48:14.972Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:14.972Z [INFO] server-9c47da70-2268-570c-c05c-4a549f3e503a: entering follower state: follower="Node at 9c47da70-2268-570c-c05c-4a549f3e503a [Follower]" leader-address= leader-id= -2025-01-28T14:48:15.626Z [WARN] server-0a406d7e-1a30-97a6-97d9-a198341370d4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:15.626Z [INFO] server-0a406d7e-1a30-97a6-97d9-a198341370d4: entering candidate state: node="Node at 0a406d7e-1a30-97a6-97d9-a198341370d4 [Candidate]" term=2 -2025-01-28T14:48:15.627Z [INFO] server-0a406d7e-1a30-97a6-97d9-a198341370d4: election won: term=2 tally=2 -2025-01-28T14:48:15.627Z [INFO] server-0a406d7e-1a30-97a6-97d9-a198341370d4: entering leader state: leader="Node at 0a406d7e-1a30-97a6-97d9-a198341370d4 [Leader]" -2025-01-28T14:48:15.627Z [INFO] server-0a406d7e-1a30-97a6-97d9-a198341370d4: added peer, starting replication: peer=server-7f6f48a9-7f76-8a0c-321b-fc74a0b1063e -2025-01-28T14:48:15.627Z [INFO] server-0a406d7e-1a30-97a6-97d9-a198341370d4: added peer, starting replication: peer=server-9c47da70-2268-570c-c05c-4a549f3e503a -2025-01-28T14:48:15.628Z [INFO] server-0a406d7e-1a30-97a6-97d9-a198341370d4: pipelining replication: peer="{Voter server-7f6f48a9-7f76-8a0c-321b-fc74a0b1063e 7f6f48a9-7f76-8a0c-321b-fc74a0b1063e}" -2025-01-28T14:48:15.629Z [INFO] server-0a406d7e-1a30-97a6-97d9-a198341370d4: pipelining replication: peer="{Voter server-9c47da70-2268-570c-c05c-4a549f3e503a 9c47da70-2268-570c-c05c-4a549f3e503a}" -2025-01-28T14:48:16.633Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:15.627445661 +0000 UTC m=+78.710850352 (1 nodes), 655.249457ms from start of poll, 677.173775ms from cluster start. Timeout at 2025-01-28 14:48:16.632457329 +0000 UTC m=+79.715862020, 1.005011668s after stability -2025-01-28T14:48:16.638Z [INFO] server-0a406d7e-1a30-97a6-97d9-a198341370d4: starting snapshot up to: index=12 -2025-01-28T14:48:16.638Z [INFO] creating new snapshot: path=/tmp/raft1002841784/snapshots/2-12-1738075696638.tmp -2025-01-28T14:48:16.642Z [INFO] server-0a406d7e-1a30-97a6-97d9-a198341370d4: snapshot complete up to: index=12 -2025-01-28T14:48:16.666Z [INFO] server-6065237c-1a54-1c4c-0f7a-22e5cdb6d113: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6065237c-1a54-1c4c-0f7a-22e5cdb6d113 Address:6065237c-1a54-1c4c-0f7a-22e5cdb6d113} {Suffrage:Voter ID:server-59534215-baee-5112-685b-5f864bddd655 Address:59534215-baee-5112-685b-5f864bddd655} {Suffrage:Voter ID:server-59bb37bb-db13-50dd-ea44-3385e8588740 Address:59bb37bb-db13-50dd-ea44-3385e8588740}]" -2025-01-28T14:48:16.666Z [INFO] server-6065237c-1a54-1c4c-0f7a-22e5cdb6d113: entering follower state: follower="Node at 6065237c-1a54-1c4c-0f7a-22e5cdb6d113 [Follower]" leader-address= leader-id= -2025-01-28T14:48:16.667Z [INFO] server-59534215-baee-5112-685b-5f864bddd655: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6065237c-1a54-1c4c-0f7a-22e5cdb6d113 Address:6065237c-1a54-1c4c-0f7a-22e5cdb6d113} {Suffrage:Voter ID:server-59534215-baee-5112-685b-5f864bddd655 Address:59534215-baee-5112-685b-5f864bddd655} {Suffrage:Voter ID:server-59bb37bb-db13-50dd-ea44-3385e8588740 Address:59bb37bb-db13-50dd-ea44-3385e8588740}]" -2025-01-28T14:48:16.667Z [INFO] server-59534215-baee-5112-685b-5f864bddd655: entering follower state: follower="Node at 59534215-baee-5112-685b-5f864bddd655 [Follower]" leader-address= leader-id= -2025-01-28T14:48:16.668Z [INFO] server-59bb37bb-db13-50dd-ea44-3385e8588740: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6065237c-1a54-1c4c-0f7a-22e5cdb6d113 Address:6065237c-1a54-1c4c-0f7a-22e5cdb6d113} {Suffrage:Voter ID:server-59534215-baee-5112-685b-5f864bddd655 Address:59534215-baee-5112-685b-5f864bddd655} {Suffrage:Voter ID:server-59bb37bb-db13-50dd-ea44-3385e8588740 Address:59bb37bb-db13-50dd-ea44-3385e8588740}]" -2025-01-28T14:48:16.668Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:16.668Z [INFO] server-59bb37bb-db13-50dd-ea44-3385e8588740: entering follower state: follower="Node at 59bb37bb-db13-50dd-ea44-3385e8588740 [Follower]" leader-address= leader-id= -2025-01-28T14:48:17.244Z [WARN] server-59bb37bb-db13-50dd-ea44-3385e8588740: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:17.244Z [INFO] server-59bb37bb-db13-50dd-ea44-3385e8588740: entering candidate state: node="Node at 59bb37bb-db13-50dd-ea44-3385e8588740 [Candidate]" term=2 -2025-01-28T14:48:17.245Z [INFO] server-59bb37bb-db13-50dd-ea44-3385e8588740: election won: term=2 tally=2 -2025-01-28T14:48:17.245Z [INFO] server-59bb37bb-db13-50dd-ea44-3385e8588740: entering leader state: leader="Node at 59bb37bb-db13-50dd-ea44-3385e8588740 [Leader]" -2025-01-28T14:48:17.245Z [INFO] server-59bb37bb-db13-50dd-ea44-3385e8588740: added peer, starting replication: peer=server-6065237c-1a54-1c4c-0f7a-22e5cdb6d113 -2025-01-28T14:48:17.245Z [INFO] server-59bb37bb-db13-50dd-ea44-3385e8588740: added peer, starting replication: peer=server-59534215-baee-5112-685b-5f864bddd655 -2025-01-28T14:48:17.246Z [INFO] server-59bb37bb-db13-50dd-ea44-3385e8588740: pipelining replication: peer="{Voter server-59534215-baee-5112-685b-5f864bddd655 59534215-baee-5112-685b-5f864bddd655}" -2025-01-28T14:48:17.246Z [INFO] server-59bb37bb-db13-50dd-ea44-3385e8588740: pipelining replication: peer="{Voter server-6065237c-1a54-1c4c-0f7a-22e5cdb6d113 6065237c-1a54-1c4c-0f7a-22e5cdb6d113}" -2025-01-28T14:48:18.251Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:17.245737717 +0000 UTC m=+80.329142741 (1 nodes), 577.375812ms from start of poll, 579.717215ms from cluster start. Timeout at 2025-01-28 14:48:18.250758051 +0000 UTC m=+81.334163075, 1.005020334s after stability -2025-01-28T14:48:18.252Z [INFO] creating new snapshot: path=/tmp/raft2755412486/snapshots/2-1013-1738075698252.tmp -2025-01-28T14:48:18.258Z [INFO] server-59bb37bb-db13-50dd-ea44-3385e8588740: copied to local snapshot: bytes=71 -2025-01-28T14:48:18.260Z [INFO] server-59bb37bb-db13-50dd-ea44-3385e8588740: snapshot restore progress: id=2-1013-1738075698252 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:18.260Z [INFO] server-59bb37bb-db13-50dd-ea44-3385e8588740: restored user snapshot: index=1013 -2025-01-28T14:48:18.260Z [ERROR] server-59bb37bb-db13-50dd-ea44-3385e8588740: failed to get log: index=3 error="log not found" -2025-01-28T14:48:18.260Z [INFO] server-59bb37bb-db13-50dd-ea44-3385e8588740: aborting pipeline replication: peer="{Voter server-6065237c-1a54-1c4c-0f7a-22e5cdb6d113 6065237c-1a54-1c4c-0f7a-22e5cdb6d113}" -2025-01-28T14:48:18.260Z [ERROR] server-59bb37bb-db13-50dd-ea44-3385e8588740: failed to get log: index=3 error="log not found" -2025-01-28T14:48:18.261Z [INFO] server-59bb37bb-db13-50dd-ea44-3385e8588740: aborting pipeline replication: peer="{Voter server-59534215-baee-5112-685b-5f864bddd655 59534215-baee-5112-685b-5f864bddd655}" -2025-01-28T14:48:18.269Z [ERROR] server-59bb37bb-db13-50dd-ea44-3385e8588740: failed to get log: index=3 error="log not found" -2025-01-28T14:48:18.269Z [ERROR] server-59bb37bb-db13-50dd-ea44-3385e8588740: failed to get log: index=3 error="log not found" -2025-01-28T14:48:18.271Z [INFO] creating new snapshot: path=/tmp/raft1819036198/snapshots/2-1013-1738075698271.tmp -2025-01-28T14:48:18.271Z [INFO] creating new snapshot: path=/tmp/raft1622522723/snapshots/2-1013-1738075698271.tmp -2025-01-28T14:48:18.274Z [INFO] server-59534215-baee-5112-685b-5f864bddd655: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:18.274Z [INFO] server-6065237c-1a54-1c4c-0f7a-22e5cdb6d113: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:18.276Z [INFO] server-59534215-baee-5112-685b-5f864bddd655: copied to local snapshot: bytes=71 -2025-01-28T14:48:18.276Z [INFO] server-6065237c-1a54-1c4c-0f7a-22e5cdb6d113: copied to local snapshot: bytes=71 -2025-01-28T14:48:18.278Z [INFO] server-59534215-baee-5112-685b-5f864bddd655: snapshot restore progress: id=2-1013-1738075698271 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:18.278Z [INFO] server-6065237c-1a54-1c4c-0f7a-22e5cdb6d113: snapshot restore progress: id=2-1013-1738075698271 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:18.278Z [INFO] server-59534215-baee-5112-685b-5f864bddd655: Installed remote snapshot -2025-01-28T14:48:18.279Z [INFO] server-6065237c-1a54-1c4c-0f7a-22e5cdb6d113: Installed remote snapshot -2025-01-28T14:48:18.279Z [INFO] server-59bb37bb-db13-50dd-ea44-3385e8588740: pipelining replication: peer="{Voter server-6065237c-1a54-1c4c-0f7a-22e5cdb6d113 6065237c-1a54-1c4c-0f7a-22e5cdb6d113}" -2025-01-28T14:48:18.280Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:48:18.280Z [WARN] server-59534215-baee-5112-685b-5f864bddd655: skipping application of old log: index=1013 -2025-01-28T14:48:18.280Z [INFO] server-59bb37bb-db13-50dd-ea44-3385e8588740: pipelining replication: peer="{Voter server-59534215-baee-5112-685b-5f864bddd655 59534215-baee-5112-685b-5f864bddd655}" -2025-01-28T14:48:19.287Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:48:18.280420442 +0000 UTC m=+81.363825133 (2 nodes), 7.333µs from start of poll, 1.614399607s from cluster start. Timeout at 2025-01-28 14:48:19.285889124 +0000 UTC m=+82.369294481, 1.005469348s after stability -2025-01-28T14:48:19.304Z [INFO] server-0a406d7e-1a30-97a6-97d9-a198341370d4: aborting pipeline replication: peer="{Voter server-7f6f48a9-7f76-8a0c-321b-fc74a0b1063e 7f6f48a9-7f76-8a0c-321b-fc74a0b1063e}" -2025-01-28T14:48:19.304Z [INFO] server-0a406d7e-1a30-97a6-97d9-a198341370d4: aborting pipeline replication: peer="{Voter server-9c47da70-2268-570c-c05c-4a549f3e503a 9c47da70-2268-570c-c05c-4a549f3e503a}" +2025-01-28T14:55:16.302Z [INFO] server-38f2462b-6c70-a872-50af-c43450274792: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-38f2462b-6c70-a872-50af-c43450274792 Address:38f2462b-6c70-a872-50af-c43450274792} {Suffrage:Voter ID:server-06719bad-053e-2626-907e-0053f45f65ea Address:06719bad-053e-2626-907e-0053f45f65ea} {Suffrage:Voter ID:server-d008feb8-48ae-d268-8a96-33bbc5fce90b Address:d008feb8-48ae-d268-8a96-33bbc5fce90b}]" +2025-01-28T14:55:16.302Z [INFO] server-38f2462b-6c70-a872-50af-c43450274792: entering follower state: follower="Node at 38f2462b-6c70-a872-50af-c43450274792 [Follower]" leader-address= leader-id= +2025-01-28T14:55:16.302Z [INFO] server-06719bad-053e-2626-907e-0053f45f65ea: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-38f2462b-6c70-a872-50af-c43450274792 Address:38f2462b-6c70-a872-50af-c43450274792} {Suffrage:Voter ID:server-06719bad-053e-2626-907e-0053f45f65ea Address:06719bad-053e-2626-907e-0053f45f65ea} {Suffrage:Voter ID:server-d008feb8-48ae-d268-8a96-33bbc5fce90b Address:d008feb8-48ae-d268-8a96-33bbc5fce90b}]" +2025-01-28T14:55:16.302Z [INFO] server-06719bad-053e-2626-907e-0053f45f65ea: entering follower state: follower="Node at 06719bad-053e-2626-907e-0053f45f65ea [Follower]" leader-address= leader-id= +2025-01-28T14:55:16.302Z [INFO] server-d008feb8-48ae-d268-8a96-33bbc5fce90b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-38f2462b-6c70-a872-50af-c43450274792 Address:38f2462b-6c70-a872-50af-c43450274792} {Suffrage:Voter ID:server-06719bad-053e-2626-907e-0053f45f65ea Address:06719bad-053e-2626-907e-0053f45f65ea} {Suffrage:Voter ID:server-d008feb8-48ae-d268-8a96-33bbc5fce90b Address:d008feb8-48ae-d268-8a96-33bbc5fce90b}]" +2025-01-28T14:55:16.303Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:16.303Z [INFO] server-d008feb8-48ae-d268-8a96-33bbc5fce90b: entering follower state: follower="Node at d008feb8-48ae-d268-8a96-33bbc5fce90b [Follower]" leader-address= leader-id= +2025-01-28T14:55:16.825Z [WARN] server-38f2462b-6c70-a872-50af-c43450274792: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:16.825Z [INFO] server-38f2462b-6c70-a872-50af-c43450274792: entering candidate state: node="Node at 38f2462b-6c70-a872-50af-c43450274792 [Candidate]" term=2 +2025-01-28T14:55:16.825Z [INFO] server-38f2462b-6c70-a872-50af-c43450274792: election won: term=2 tally=2 +2025-01-28T14:55:16.825Z [INFO] server-38f2462b-6c70-a872-50af-c43450274792: entering leader state: leader="Node at 38f2462b-6c70-a872-50af-c43450274792 [Leader]" +2025-01-28T14:55:16.825Z [INFO] server-38f2462b-6c70-a872-50af-c43450274792: added peer, starting replication: peer=server-06719bad-053e-2626-907e-0053f45f65ea +2025-01-28T14:55:16.825Z [INFO] server-38f2462b-6c70-a872-50af-c43450274792: added peer, starting replication: peer=server-d008feb8-48ae-d268-8a96-33bbc5fce90b +2025-01-28T14:55:16.825Z [INFO] server-38f2462b-6c70-a872-50af-c43450274792: pipelining replication: peer="{Voter server-d008feb8-48ae-d268-8a96-33bbc5fce90b d008feb8-48ae-d268-8a96-33bbc5fce90b}" +2025-01-28T14:55:16.825Z [INFO] server-38f2462b-6c70-a872-50af-c43450274792: pipelining replication: peer="{Voter server-06719bad-053e-2626-907e-0053f45f65ea 06719bad-053e-2626-907e-0053f45f65ea}" +2025-01-28T14:55:17.831Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:16.825633421 +0000 UTC m=+102.500123401 (1 nodes), 522.571162ms from start of poll, 523.484636ms from cluster start. Timeout at 2025-01-28 14:55:17.830648579 +0000 UTC m=+103.505138559, 1.005015158s after stability +2025-01-28T14:55:17.837Z [INFO] server-38f2462b-6c70-a872-50af-c43450274792: starting snapshot up to: index=12 +2025-01-28T14:55:17.837Z [INFO] creating new snapshot: path=/tmp/raft4081606572/snapshots/2-12-1738076117837.tmp +2025-01-28T14:55:17.840Z [INFO] server-38f2462b-6c70-a872-50af-c43450274792: snapshot complete up to: index=12 +2025-01-28T14:55:17.843Z [INFO] server-cd9f0463-c26e-9ad4-50bf-fdd8aecac857: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cd9f0463-c26e-9ad4-50bf-fdd8aecac857 Address:cd9f0463-c26e-9ad4-50bf-fdd8aecac857} {Suffrage:Voter ID:server-1279708b-6d61-de12-355a-6937eaa3530b Address:1279708b-6d61-de12-355a-6937eaa3530b} {Suffrage:Voter ID:server-b6b8405e-9883-e6ab-ff3f-a54c520905b8 Address:b6b8405e-9883-e6ab-ff3f-a54c520905b8}]" +2025-01-28T14:55:17.843Z [INFO] server-cd9f0463-c26e-9ad4-50bf-fdd8aecac857: entering follower state: follower="Node at cd9f0463-c26e-9ad4-50bf-fdd8aecac857 [Follower]" leader-address= leader-id= +2025-01-28T14:55:17.843Z [INFO] server-1279708b-6d61-de12-355a-6937eaa3530b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cd9f0463-c26e-9ad4-50bf-fdd8aecac857 Address:cd9f0463-c26e-9ad4-50bf-fdd8aecac857} {Suffrage:Voter ID:server-1279708b-6d61-de12-355a-6937eaa3530b Address:1279708b-6d61-de12-355a-6937eaa3530b} {Suffrage:Voter ID:server-b6b8405e-9883-e6ab-ff3f-a54c520905b8 Address:b6b8405e-9883-e6ab-ff3f-a54c520905b8}]" +2025-01-28T14:55:17.843Z [INFO] server-1279708b-6d61-de12-355a-6937eaa3530b: entering follower state: follower="Node at 1279708b-6d61-de12-355a-6937eaa3530b [Follower]" leader-address= leader-id= +2025-01-28T14:55:17.844Z [INFO] server-b6b8405e-9883-e6ab-ff3f-a54c520905b8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cd9f0463-c26e-9ad4-50bf-fdd8aecac857 Address:cd9f0463-c26e-9ad4-50bf-fdd8aecac857} {Suffrage:Voter ID:server-1279708b-6d61-de12-355a-6937eaa3530b Address:1279708b-6d61-de12-355a-6937eaa3530b} {Suffrage:Voter ID:server-b6b8405e-9883-e6ab-ff3f-a54c520905b8 Address:b6b8405e-9883-e6ab-ff3f-a54c520905b8}]" +2025-01-28T14:55:17.844Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:17.844Z [INFO] server-b6b8405e-9883-e6ab-ff3f-a54c520905b8: entering follower state: follower="Node at b6b8405e-9883-e6ab-ff3f-a54c520905b8 [Follower]" leader-address= leader-id= +2025-01-28T14:55:18.434Z [WARN] server-1279708b-6d61-de12-355a-6937eaa3530b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:18.434Z [INFO] server-1279708b-6d61-de12-355a-6937eaa3530b: entering candidate state: node="Node at 1279708b-6d61-de12-355a-6937eaa3530b [Candidate]" term=2 +2025-01-28T14:55:18.435Z [INFO] server-1279708b-6d61-de12-355a-6937eaa3530b: election won: term=2 tally=2 +2025-01-28T14:55:18.435Z [INFO] server-1279708b-6d61-de12-355a-6937eaa3530b: entering leader state: leader="Node at 1279708b-6d61-de12-355a-6937eaa3530b [Leader]" +2025-01-28T14:55:18.435Z [INFO] server-1279708b-6d61-de12-355a-6937eaa3530b: added peer, starting replication: peer=server-cd9f0463-c26e-9ad4-50bf-fdd8aecac857 +2025-01-28T14:55:18.435Z [INFO] server-1279708b-6d61-de12-355a-6937eaa3530b: added peer, starting replication: peer=server-b6b8405e-9883-e6ab-ff3f-a54c520905b8 +2025-01-28T14:55:18.435Z [INFO] server-1279708b-6d61-de12-355a-6937eaa3530b: pipelining replication: peer="{Voter server-cd9f0463-c26e-9ad4-50bf-fdd8aecac857 cd9f0463-c26e-9ad4-50bf-fdd8aecac857}" +2025-01-28T14:55:18.435Z [INFO] server-1279708b-6d61-de12-355a-6937eaa3530b: pipelining replication: peer="{Voter server-b6b8405e-9883-e6ab-ff3f-a54c520905b8 b6b8405e-9883-e6ab-ff3f-a54c520905b8}" +2025-01-28T14:55:19.441Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:18.435155124 +0000 UTC m=+104.109645084 (1 nodes), 590.849161ms from start of poll, 591.996633ms from cluster start. Timeout at 2025-01-28 14:55:19.440164603 +0000 UTC m=+105.114654583, 1.005009499s after stability +2025-01-28T14:55:19.442Z [INFO] creating new snapshot: path=/tmp/raft3003612796/snapshots/2-1013-1738076119442.tmp +2025-01-28T14:55:19.443Z [INFO] server-1279708b-6d61-de12-355a-6937eaa3530b: copied to local snapshot: bytes=71 +2025-01-28T14:55:19.444Z [INFO] server-1279708b-6d61-de12-355a-6937eaa3530b: snapshot restore progress: id=2-1013-1738076119442 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:19.444Z [INFO] server-1279708b-6d61-de12-355a-6937eaa3530b: restored user snapshot: index=1013 +2025-01-28T14:55:19.444Z [ERROR] server-1279708b-6d61-de12-355a-6937eaa3530b: failed to get log: index=3 error="log not found" +2025-01-28T14:55:19.444Z [INFO] server-1279708b-6d61-de12-355a-6937eaa3530b: aborting pipeline replication: peer="{Voter server-b6b8405e-9883-e6ab-ff3f-a54c520905b8 b6b8405e-9883-e6ab-ff3f-a54c520905b8}" +2025-01-28T14:55:19.444Z [ERROR] server-1279708b-6d61-de12-355a-6937eaa3530b: failed to get log: index=3 error="log not found" +2025-01-28T14:55:19.444Z [INFO] server-1279708b-6d61-de12-355a-6937eaa3530b: aborting pipeline replication: peer="{Voter server-cd9f0463-c26e-9ad4-50bf-fdd8aecac857 cd9f0463-c26e-9ad4-50bf-fdd8aecac857}" +2025-01-28T14:55:19.450Z [ERROR] server-1279708b-6d61-de12-355a-6937eaa3530b: failed to get log: index=3 error="log not found" +2025-01-28T14:55:19.452Z [INFO] creating new snapshot: path=/tmp/raft538748978/snapshots/2-1013-1738076119452.tmp +2025-01-28T14:55:19.452Z [INFO] server-cd9f0463-c26e-9ad4-50bf-fdd8aecac857: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:19.453Z [INFO] server-cd9f0463-c26e-9ad4-50bf-fdd8aecac857: copied to local snapshot: bytes=71 +2025-01-28T14:55:19.454Z [INFO] server-cd9f0463-c26e-9ad4-50bf-fdd8aecac857: snapshot restore progress: id=2-1013-1738076119452 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:19.454Z [INFO] server-cd9f0463-c26e-9ad4-50bf-fdd8aecac857: Installed remote snapshot +2025-01-28T14:55:19.454Z [INFO] server-1279708b-6d61-de12-355a-6937eaa3530b: pipelining replication: peer="{Voter server-cd9f0463-c26e-9ad4-50bf-fdd8aecac857 cd9f0463-c26e-9ad4-50bf-fdd8aecac857}" +2025-01-28T14:55:19.455Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:55:19.457Z [ERROR] server-1279708b-6d61-de12-355a-6937eaa3530b: failed to get log: index=3 error="log not found" +2025-01-28T14:55:19.457Z [INFO] creating new snapshot: path=/tmp/raft3103627393/snapshots/2-1013-1738076119457.tmp +2025-01-28T14:55:19.458Z [INFO] server-b6b8405e-9883-e6ab-ff3f-a54c520905b8: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:19.459Z [INFO] server-b6b8405e-9883-e6ab-ff3f-a54c520905b8: copied to local snapshot: bytes=71 +2025-01-28T14:55:19.459Z [INFO] server-b6b8405e-9883-e6ab-ff3f-a54c520905b8: snapshot restore progress: id=2-1013-1738076119457 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:19.460Z [INFO] server-b6b8405e-9883-e6ab-ff3f-a54c520905b8: Installed remote snapshot +2025-01-28T14:55:19.460Z [INFO] server-1279708b-6d61-de12-355a-6937eaa3530b: pipelining replication: peer="{Voter server-b6b8405e-9883-e6ab-ff3f-a54c520905b8 b6b8405e-9883-e6ab-ff3f-a54c520905b8}" +2025-01-28T14:55:20.461Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:55:19.455082727 +0000 UTC m=+105.129572687 (2 nodes), 2.359µs from start of poll, 1.611924236s from cluster start. Timeout at 2025-01-28 14:55:20.460182778 +0000 UTC m=+106.134672838, 1.005100151s after stability +2025-01-28T14:55:20.472Z [INFO] server-38f2462b-6c70-a872-50af-c43450274792: aborting pipeline replication: peer="{Voter server-d008feb8-48ae-d268-8a96-33bbc5fce90b d008feb8-48ae-d268-8a96-33bbc5fce90b}" +2025-01-28T14:55:20.472Z [INFO] server-38f2462b-6c70-a872-50af-c43450274792: aborting pipeline replication: peer="{Voter server-06719bad-053e-2626-907e-0053f45f65ea 06719bad-053e-2626-907e-0053f45f65ea}" === RUN TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_true -2025-01-28T14:48:19.318Z [INFO] server-b3bd3d9e-323e-c040-6bce-3b3117c8385c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b3bd3d9e-323e-c040-6bce-3b3117c8385c Address:b3bd3d9e-323e-c040-6bce-3b3117c8385c} {Suffrage:Voter ID:server-9847a3c5-7878-ee88-fbe5-3e6f389019b1 Address:9847a3c5-7878-ee88-fbe5-3e6f389019b1} {Suffrage:Voter ID:server-2ccdaf21-38c6-83c8-04ac-671ed5f54eac Address:2ccdaf21-38c6-83c8-04ac-671ed5f54eac}]" -2025-01-28T14:48:19.318Z [INFO] server-b3bd3d9e-323e-c040-6bce-3b3117c8385c: entering follower state: follower="Node at b3bd3d9e-323e-c040-6bce-3b3117c8385c [Follower]" leader-address= leader-id= -2025-01-28T14:48:19.318Z [INFO] server-9847a3c5-7878-ee88-fbe5-3e6f389019b1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b3bd3d9e-323e-c040-6bce-3b3117c8385c Address:b3bd3d9e-323e-c040-6bce-3b3117c8385c} {Suffrage:Voter ID:server-9847a3c5-7878-ee88-fbe5-3e6f389019b1 Address:9847a3c5-7878-ee88-fbe5-3e6f389019b1} {Suffrage:Voter ID:server-2ccdaf21-38c6-83c8-04ac-671ed5f54eac Address:2ccdaf21-38c6-83c8-04ac-671ed5f54eac}]" -2025-01-28T14:48:19.319Z [INFO] server-9847a3c5-7878-ee88-fbe5-3e6f389019b1: entering follower state: follower="Node at 9847a3c5-7878-ee88-fbe5-3e6f389019b1 [Follower]" leader-address= leader-id= -2025-01-28T14:48:19.325Z [INFO] server-2ccdaf21-38c6-83c8-04ac-671ed5f54eac: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b3bd3d9e-323e-c040-6bce-3b3117c8385c Address:b3bd3d9e-323e-c040-6bce-3b3117c8385c} {Suffrage:Voter ID:server-9847a3c5-7878-ee88-fbe5-3e6f389019b1 Address:9847a3c5-7878-ee88-fbe5-3e6f389019b1} {Suffrage:Voter ID:server-2ccdaf21-38c6-83c8-04ac-671ed5f54eac Address:2ccdaf21-38c6-83c8-04ac-671ed5f54eac}]" -2025-01-28T14:48:19.326Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:19.328Z [INFO] server-2ccdaf21-38c6-83c8-04ac-671ed5f54eac: entering follower state: follower="Node at 2ccdaf21-38c6-83c8-04ac-671ed5f54eac [Follower]" leader-address= leader-id= -2025-01-28T14:48:19.839Z [WARN] server-2ccdaf21-38c6-83c8-04ac-671ed5f54eac: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:19.840Z [INFO] server-2ccdaf21-38c6-83c8-04ac-671ed5f54eac: entering candidate state: node="Node at 2ccdaf21-38c6-83c8-04ac-671ed5f54eac [Candidate]" term=2 -2025-01-28T14:48:19.841Z [INFO] server-2ccdaf21-38c6-83c8-04ac-671ed5f54eac: election won: term=2 tally=2 -2025-01-28T14:48:19.842Z [INFO] server-2ccdaf21-38c6-83c8-04ac-671ed5f54eac: entering leader state: leader="Node at 2ccdaf21-38c6-83c8-04ac-671ed5f54eac [Leader]" -2025-01-28T14:48:19.842Z [INFO] server-2ccdaf21-38c6-83c8-04ac-671ed5f54eac: added peer, starting replication: peer=server-b3bd3d9e-323e-c040-6bce-3b3117c8385c -2025-01-28T14:48:19.842Z [INFO] server-2ccdaf21-38c6-83c8-04ac-671ed5f54eac: added peer, starting replication: peer=server-9847a3c5-7878-ee88-fbe5-3e6f389019b1 -2025-01-28T14:48:19.844Z [INFO] server-2ccdaf21-38c6-83c8-04ac-671ed5f54eac: pipelining replication: peer="{Voter server-9847a3c5-7878-ee88-fbe5-3e6f389019b1 9847a3c5-7878-ee88-fbe5-3e6f389019b1}" -2025-01-28T14:48:19.845Z [INFO] server-2ccdaf21-38c6-83c8-04ac-671ed5f54eac: pipelining replication: peer="{Voter server-b3bd3d9e-323e-c040-6bce-3b3117c8385c b3bd3d9e-323e-c040-6bce-3b3117c8385c}" -2025-01-28T14:48:20.848Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:19.842850173 +0000 UTC m=+82.926256531 (1 nodes), 516.483338ms from start of poll, 525.402269ms from cluster start. Timeout at 2025-01-28 14:48:20.847894174 +0000 UTC m=+83.931300198, 1.005043667s after stability -2025-01-28T14:48:20.858Z [INFO] server-2ccdaf21-38c6-83c8-04ac-671ed5f54eac: starting snapshot up to: index=12 -2025-01-28T14:48:20.858Z [INFO] creating new snapshot: path=/tmp/raft299851659/snapshots/2-12-1738075700858.tmp -2025-01-28T14:48:20.876Z [INFO] server-2ccdaf21-38c6-83c8-04ac-671ed5f54eac: snapshot complete up to: index=12 -2025-01-28T14:48:20.886Z [INFO] server-c3b633cc-60ce-aeac-a5c9-2437f2bb711b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c3b633cc-60ce-aeac-a5c9-2437f2bb711b Address:c3b633cc-60ce-aeac-a5c9-2437f2bb711b} {Suffrage:Voter ID:server-622bf10a-b5d0-f174-c1c8-1b771b0914e9 Address:622bf10a-b5d0-f174-c1c8-1b771b0914e9} {Suffrage:Voter ID:server-577cee66-e0a3-b801-e8af-c79721ad7e75 Address:577cee66-e0a3-b801-e8af-c79721ad7e75}]" -2025-01-28T14:48:20.886Z [INFO] server-c3b633cc-60ce-aeac-a5c9-2437f2bb711b: entering follower state: follower="Node at c3b633cc-60ce-aeac-a5c9-2437f2bb711b [Follower]" leader-address= leader-id= -2025-01-28T14:48:20.886Z [INFO] server-622bf10a-b5d0-f174-c1c8-1b771b0914e9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c3b633cc-60ce-aeac-a5c9-2437f2bb711b Address:c3b633cc-60ce-aeac-a5c9-2437f2bb711b} {Suffrage:Voter ID:server-622bf10a-b5d0-f174-c1c8-1b771b0914e9 Address:622bf10a-b5d0-f174-c1c8-1b771b0914e9} {Suffrage:Voter ID:server-577cee66-e0a3-b801-e8af-c79721ad7e75 Address:577cee66-e0a3-b801-e8af-c79721ad7e75}]" -2025-01-28T14:48:20.887Z [INFO] server-622bf10a-b5d0-f174-c1c8-1b771b0914e9: entering follower state: follower="Node at 622bf10a-b5d0-f174-c1c8-1b771b0914e9 [Follower]" leader-address= leader-id= -2025-01-28T14:48:20.887Z [INFO] server-577cee66-e0a3-b801-e8af-c79721ad7e75: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c3b633cc-60ce-aeac-a5c9-2437f2bb711b Address:c3b633cc-60ce-aeac-a5c9-2437f2bb711b} {Suffrage:Voter ID:server-622bf10a-b5d0-f174-c1c8-1b771b0914e9 Address:622bf10a-b5d0-f174-c1c8-1b771b0914e9} {Suffrage:Voter ID:server-577cee66-e0a3-b801-e8af-c79721ad7e75 Address:577cee66-e0a3-b801-e8af-c79721ad7e75}]" -2025-01-28T14:48:20.887Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:20.887Z [INFO] server-577cee66-e0a3-b801-e8af-c79721ad7e75: entering follower state: follower="Node at 577cee66-e0a3-b801-e8af-c79721ad7e75 [Follower]" leader-address= leader-id= -2025-01-28T14:48:21.581Z [WARN] server-622bf10a-b5d0-f174-c1c8-1b771b0914e9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:21.581Z [INFO] server-622bf10a-b5d0-f174-c1c8-1b771b0914e9: entering candidate state: node="Node at 622bf10a-b5d0-f174-c1c8-1b771b0914e9 [Candidate]" term=2 -2025-01-28T14:48:21.582Z [INFO] server-622bf10a-b5d0-f174-c1c8-1b771b0914e9: election won: term=2 tally=2 -2025-01-28T14:48:21.582Z [INFO] server-622bf10a-b5d0-f174-c1c8-1b771b0914e9: entering leader state: leader="Node at 622bf10a-b5d0-f174-c1c8-1b771b0914e9 [Leader]" -2025-01-28T14:48:21.582Z [INFO] server-622bf10a-b5d0-f174-c1c8-1b771b0914e9: added peer, starting replication: peer=server-c3b633cc-60ce-aeac-a5c9-2437f2bb711b -2025-01-28T14:48:21.582Z [INFO] server-622bf10a-b5d0-f174-c1c8-1b771b0914e9: added peer, starting replication: peer=server-577cee66-e0a3-b801-e8af-c79721ad7e75 -2025-01-28T14:48:21.583Z [INFO] server-622bf10a-b5d0-f174-c1c8-1b771b0914e9: pipelining replication: peer="{Voter server-577cee66-e0a3-b801-e8af-c79721ad7e75 577cee66-e0a3-b801-e8af-c79721ad7e75}" -2025-01-28T14:48:21.583Z [INFO] server-622bf10a-b5d0-f174-c1c8-1b771b0914e9: pipelining replication: peer="{Voter server-c3b633cc-60ce-aeac-a5c9-2437f2bb711b c3b633cc-60ce-aeac-a5c9-2437f2bb711b}" -2025-01-28T14:48:22.588Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:21.582727506 +0000 UTC m=+84.666132197 (1 nodes), 694.847967ms from start of poll, 697.172703ms from cluster start. Timeout at 2025-01-28 14:48:22.587747174 +0000 UTC m=+85.671152198, 1.005020001s after stability -2025-01-28T14:48:22.589Z [INFO] creating new snapshot: path=/tmp/raft1310620579/snapshots/2-1013-1738075702589.tmp -2025-01-28T14:48:22.594Z [INFO] server-622bf10a-b5d0-f174-c1c8-1b771b0914e9: copied to local snapshot: bytes=71 -2025-01-28T14:48:22.596Z [INFO] server-622bf10a-b5d0-f174-c1c8-1b771b0914e9: snapshot restore progress: id=2-1013-1738075702589 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:22.596Z [INFO] server-622bf10a-b5d0-f174-c1c8-1b771b0914e9: removing all old logs from log store -2025-01-28T14:48:22.596Z [INFO] server-622bf10a-b5d0-f174-c1c8-1b771b0914e9: compacting logs: from=1 to=2 -2025-01-28T14:48:22.596Z [INFO] server-622bf10a-b5d0-f174-c1c8-1b771b0914e9: restored user snapshot: index=1013 -2025-01-28T14:48:22.597Z [ERROR] server-622bf10a-b5d0-f174-c1c8-1b771b0914e9: failed to get log: index=2 error="log not found" -2025-01-28T14:48:22.597Z [ERROR] server-622bf10a-b5d0-f174-c1c8-1b771b0914e9: failed to get log: index=2 error="log not found" -2025-01-28T14:48:22.597Z [INFO] server-622bf10a-b5d0-f174-c1c8-1b771b0914e9: aborting pipeline replication: peer="{Voter server-577cee66-e0a3-b801-e8af-c79721ad7e75 577cee66-e0a3-b801-e8af-c79721ad7e75}" -2025-01-28T14:48:22.597Z [INFO] server-622bf10a-b5d0-f174-c1c8-1b771b0914e9: aborting pipeline replication: peer="{Voter server-c3b633cc-60ce-aeac-a5c9-2437f2bb711b c3b633cc-60ce-aeac-a5c9-2437f2bb711b}" -2025-01-28T14:48:22.606Z [ERROR] server-622bf10a-b5d0-f174-c1c8-1b771b0914e9: failed to get log: index=2 error="log not found" -2025-01-28T14:48:22.617Z [ERROR] server-622bf10a-b5d0-f174-c1c8-1b771b0914e9: failed to get log: index=2 error="log not found" -2025-01-28T14:48:22.621Z [INFO] creating new snapshot: path=/tmp/raft1371859071/snapshots/2-1013-1738075702621.tmp -2025-01-28T14:48:22.624Z [INFO] server-577cee66-e0a3-b801-e8af-c79721ad7e75: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:22.626Z [INFO] creating new snapshot: path=/tmp/raft2633116013/snapshots/2-1013-1738075702626.tmp -2025-01-28T14:48:22.628Z [INFO] server-577cee66-e0a3-b801-e8af-c79721ad7e75: copied to local snapshot: bytes=71 -2025-01-28T14:48:22.628Z [INFO] server-c3b633cc-60ce-aeac-a5c9-2437f2bb711b: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:22.630Z [INFO] server-577cee66-e0a3-b801-e8af-c79721ad7e75: snapshot restore progress: id=2-1013-1738075702621 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:22.630Z [INFO] server-577cee66-e0a3-b801-e8af-c79721ad7e75: removing all old logs from log store -2025-01-28T14:48:22.630Z [INFO] server-577cee66-e0a3-b801-e8af-c79721ad7e75: compacting logs: from=1 to=2 -2025-01-28T14:48:22.630Z [INFO] server-577cee66-e0a3-b801-e8af-c79721ad7e75: Installed remote snapshot -2025-01-28T14:48:22.631Z [INFO] server-c3b633cc-60ce-aeac-a5c9-2437f2bb711b: copied to local snapshot: bytes=71 -2025-01-28T14:48:22.632Z [INFO] server-622bf10a-b5d0-f174-c1c8-1b771b0914e9: pipelining replication: peer="{Voter server-577cee66-e0a3-b801-e8af-c79721ad7e75 577cee66-e0a3-b801-e8af-c79721ad7e75}" -2025-01-28T14:48:22.632Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:48:22.632Z [INFO] server-c3b633cc-60ce-aeac-a5c9-2437f2bb711b: snapshot restore progress: id=2-1013-1738075702626 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:22.632Z [INFO] server-c3b633cc-60ce-aeac-a5c9-2437f2bb711b: removing all old logs from log store -2025-01-28T14:48:22.632Z [INFO] server-c3b633cc-60ce-aeac-a5c9-2437f2bb711b: compacting logs: from=1 to=2 -2025-01-28T14:48:22.633Z [INFO] server-c3b633cc-60ce-aeac-a5c9-2437f2bb711b: Installed remote snapshot -2025-01-28T14:48:22.633Z [INFO] server-622bf10a-b5d0-f174-c1c8-1b771b0914e9: pipelining replication: peer="{Voter server-c3b633cc-60ce-aeac-a5c9-2437f2bb711b c3b633cc-60ce-aeac-a5c9-2437f2bb711b}" -2025-01-28T14:48:23.638Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:48:22.632672352 +0000 UTC m=+85.716077043 (2 nodes), 7.667µs from start of poll, 1.747117549s from cluster start. Timeout at 2025-01-28 14:48:23.638124365 +0000 UTC m=+86.721529389, 1.005452346s after stability -2025-01-28T14:48:23.653Z [INFO] server-2ccdaf21-38c6-83c8-04ac-671ed5f54eac: aborting pipeline replication: peer="{Voter server-9847a3c5-7878-ee88-fbe5-3e6f389019b1 9847a3c5-7878-ee88-fbe5-3e6f389019b1}" -2025-01-28T14:48:23.653Z [INFO] server-2ccdaf21-38c6-83c8-04ac-671ed5f54eac: aborting pipeline replication: peer="{Voter server-b3bd3d9e-323e-c040-6bce-3b3117c8385c b3bd3d9e-323e-c040-6bce-3b3117c8385c}" +2025-01-28T14:55:20.476Z [INFO] server-42a1f769-6cd6-036d-3c0b-5a133b0f1eb9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-42a1f769-6cd6-036d-3c0b-5a133b0f1eb9 Address:42a1f769-6cd6-036d-3c0b-5a133b0f1eb9} {Suffrage:Voter ID:server-b9e3aad9-e9e6-85f9-361d-d215f4a207cd Address:b9e3aad9-e9e6-85f9-361d-d215f4a207cd} {Suffrage:Voter ID:server-60f4ed59-de49-f423-3161-c1381e85a1e0 Address:60f4ed59-de49-f423-3161-c1381e85a1e0}]" +2025-01-28T14:55:20.476Z [INFO] server-42a1f769-6cd6-036d-3c0b-5a133b0f1eb9: entering follower state: follower="Node at 42a1f769-6cd6-036d-3c0b-5a133b0f1eb9 [Follower]" leader-address= leader-id= +2025-01-28T14:55:20.476Z [INFO] server-b9e3aad9-e9e6-85f9-361d-d215f4a207cd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-42a1f769-6cd6-036d-3c0b-5a133b0f1eb9 Address:42a1f769-6cd6-036d-3c0b-5a133b0f1eb9} {Suffrage:Voter ID:server-b9e3aad9-e9e6-85f9-361d-d215f4a207cd Address:b9e3aad9-e9e6-85f9-361d-d215f4a207cd} {Suffrage:Voter ID:server-60f4ed59-de49-f423-3161-c1381e85a1e0 Address:60f4ed59-de49-f423-3161-c1381e85a1e0}]" +2025-01-28T14:55:20.476Z [INFO] server-b9e3aad9-e9e6-85f9-361d-d215f4a207cd: entering follower state: follower="Node at b9e3aad9-e9e6-85f9-361d-d215f4a207cd [Follower]" leader-address= leader-id= +2025-01-28T14:55:20.476Z [INFO] server-60f4ed59-de49-f423-3161-c1381e85a1e0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-42a1f769-6cd6-036d-3c0b-5a133b0f1eb9 Address:42a1f769-6cd6-036d-3c0b-5a133b0f1eb9} {Suffrage:Voter ID:server-b9e3aad9-e9e6-85f9-361d-d215f4a207cd Address:b9e3aad9-e9e6-85f9-361d-d215f4a207cd} {Suffrage:Voter ID:server-60f4ed59-de49-f423-3161-c1381e85a1e0 Address:60f4ed59-de49-f423-3161-c1381e85a1e0}]" +2025-01-28T14:55:20.476Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:20.477Z [INFO] server-60f4ed59-de49-f423-3161-c1381e85a1e0: entering follower state: follower="Node at 60f4ed59-de49-f423-3161-c1381e85a1e0 [Follower]" leader-address= leader-id= +2025-01-28T14:55:21.126Z [WARN] server-b9e3aad9-e9e6-85f9-361d-d215f4a207cd: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:21.126Z [INFO] server-b9e3aad9-e9e6-85f9-361d-d215f4a207cd: entering candidate state: node="Node at b9e3aad9-e9e6-85f9-361d-d215f4a207cd [Candidate]" term=2 +2025-01-28T14:55:21.126Z [INFO] server-b9e3aad9-e9e6-85f9-361d-d215f4a207cd: election won: term=2 tally=2 +2025-01-28T14:55:21.126Z [INFO] server-b9e3aad9-e9e6-85f9-361d-d215f4a207cd: entering leader state: leader="Node at b9e3aad9-e9e6-85f9-361d-d215f4a207cd [Leader]" +2025-01-28T14:55:21.126Z [INFO] server-b9e3aad9-e9e6-85f9-361d-d215f4a207cd: added peer, starting replication: peer=server-42a1f769-6cd6-036d-3c0b-5a133b0f1eb9 +2025-01-28T14:55:21.126Z [INFO] server-b9e3aad9-e9e6-85f9-361d-d215f4a207cd: added peer, starting replication: peer=server-60f4ed59-de49-f423-3161-c1381e85a1e0 +2025-01-28T14:55:21.126Z [INFO] server-b9e3aad9-e9e6-85f9-361d-d215f4a207cd: pipelining replication: peer="{Voter server-60f4ed59-de49-f423-3161-c1381e85a1e0 60f4ed59-de49-f423-3161-c1381e85a1e0}" +2025-01-28T14:55:21.127Z [INFO] server-b9e3aad9-e9e6-85f9-361d-d215f4a207cd: pipelining replication: peer="{Voter server-42a1f769-6cd6-036d-3c0b-5a133b0f1eb9 42a1f769-6cd6-036d-3c0b-5a133b0f1eb9}" +2025-01-28T14:55:22.132Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:21.126675635 +0000 UTC m=+106.801165615 (1 nodes), 649.649914ms from start of poll, 650.446619ms from cluster start. Timeout at 2025-01-28 14:55:22.131685434 +0000 UTC m=+107.806175414, 1.005009799s after stability +2025-01-28T14:55:22.133Z [INFO] server-b9e3aad9-e9e6-85f9-361d-d215f4a207cd: starting snapshot up to: index=12 +2025-01-28T14:55:22.133Z [INFO] creating new snapshot: path=/tmp/raft3002474244/snapshots/2-12-1738076122133.tmp +2025-01-28T14:55:22.136Z [INFO] server-b9e3aad9-e9e6-85f9-361d-d215f4a207cd: snapshot complete up to: index=12 +2025-01-28T14:55:22.141Z [INFO] server-bcae2595-402b-7bec-9a33-21ade35672fe: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bcae2595-402b-7bec-9a33-21ade35672fe Address:bcae2595-402b-7bec-9a33-21ade35672fe} {Suffrage:Voter ID:server-4b888034-289d-dca9-34f0-95dc436fecc5 Address:4b888034-289d-dca9-34f0-95dc436fecc5} {Suffrage:Voter ID:server-47773f58-222a-9b0f-314b-edf27ff170af Address:47773f58-222a-9b0f-314b-edf27ff170af}]" +2025-01-28T14:55:22.141Z [INFO] server-bcae2595-402b-7bec-9a33-21ade35672fe: entering follower state: follower="Node at bcae2595-402b-7bec-9a33-21ade35672fe [Follower]" leader-address= leader-id= +2025-01-28T14:55:22.141Z [INFO] server-4b888034-289d-dca9-34f0-95dc436fecc5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bcae2595-402b-7bec-9a33-21ade35672fe Address:bcae2595-402b-7bec-9a33-21ade35672fe} {Suffrage:Voter ID:server-4b888034-289d-dca9-34f0-95dc436fecc5 Address:4b888034-289d-dca9-34f0-95dc436fecc5} {Suffrage:Voter ID:server-47773f58-222a-9b0f-314b-edf27ff170af Address:47773f58-222a-9b0f-314b-edf27ff170af}]" +2025-01-28T14:55:22.141Z [INFO] server-4b888034-289d-dca9-34f0-95dc436fecc5: entering follower state: follower="Node at 4b888034-289d-dca9-34f0-95dc436fecc5 [Follower]" leader-address= leader-id= +2025-01-28T14:55:22.141Z [INFO] server-47773f58-222a-9b0f-314b-edf27ff170af: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bcae2595-402b-7bec-9a33-21ade35672fe Address:bcae2595-402b-7bec-9a33-21ade35672fe} {Suffrage:Voter ID:server-4b888034-289d-dca9-34f0-95dc436fecc5 Address:4b888034-289d-dca9-34f0-95dc436fecc5} {Suffrage:Voter ID:server-47773f58-222a-9b0f-314b-edf27ff170af Address:47773f58-222a-9b0f-314b-edf27ff170af}]" +2025-01-28T14:55:22.141Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:22.141Z [INFO] server-47773f58-222a-9b0f-314b-edf27ff170af: entering follower state: follower="Node at 47773f58-222a-9b0f-314b-edf27ff170af [Follower]" leader-address= leader-id= +2025-01-28T14:55:22.747Z [WARN] server-47773f58-222a-9b0f-314b-edf27ff170af: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:22.748Z [INFO] server-47773f58-222a-9b0f-314b-edf27ff170af: entering candidate state: node="Node at 47773f58-222a-9b0f-314b-edf27ff170af [Candidate]" term=2 +2025-01-28T14:55:22.748Z [INFO] server-47773f58-222a-9b0f-314b-edf27ff170af: election won: term=2 tally=2 +2025-01-28T14:55:22.748Z [INFO] server-47773f58-222a-9b0f-314b-edf27ff170af: entering leader state: leader="Node at 47773f58-222a-9b0f-314b-edf27ff170af [Leader]" +2025-01-28T14:55:22.749Z [INFO] server-47773f58-222a-9b0f-314b-edf27ff170af: added peer, starting replication: peer=server-bcae2595-402b-7bec-9a33-21ade35672fe +2025-01-28T14:55:22.749Z [INFO] server-47773f58-222a-9b0f-314b-edf27ff170af: added peer, starting replication: peer=server-4b888034-289d-dca9-34f0-95dc436fecc5 +2025-01-28T14:55:22.751Z [INFO] server-47773f58-222a-9b0f-314b-edf27ff170af: pipelining replication: peer="{Voter server-bcae2595-402b-7bec-9a33-21ade35672fe bcae2595-402b-7bec-9a33-21ade35672fe}" +2025-01-28T14:55:22.752Z [INFO] server-47773f58-222a-9b0f-314b-edf27ff170af: pipelining replication: peer="{Voter server-4b888034-289d-dca9-34f0-95dc436fecc5 4b888034-289d-dca9-34f0-95dc436fecc5}" +2025-01-28T14:55:23.754Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:22.749302562 +0000 UTC m=+108.423792542 (1 nodes), 607.353884ms from start of poll, 608.200145ms from cluster start. Timeout at 2025-01-28 14:55:23.754310241 +0000 UTC m=+109.428800221, 1.005007679s after stability +2025-01-28T14:55:23.755Z [INFO] creating new snapshot: path=/tmp/raft3203437808/snapshots/2-1013-1738076123755.tmp +2025-01-28T14:55:23.757Z [INFO] server-47773f58-222a-9b0f-314b-edf27ff170af: copied to local snapshot: bytes=71 +2025-01-28T14:55:23.758Z [INFO] server-47773f58-222a-9b0f-314b-edf27ff170af: snapshot restore progress: id=2-1013-1738076123755 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:23.758Z [INFO] server-47773f58-222a-9b0f-314b-edf27ff170af: removing all old logs from log store +2025-01-28T14:55:23.758Z [INFO] server-47773f58-222a-9b0f-314b-edf27ff170af: compacting logs: from=1 to=2 +2025-01-28T14:55:23.758Z [INFO] server-47773f58-222a-9b0f-314b-edf27ff170af: restored user snapshot: index=1013 +2025-01-28T14:55:23.758Z [ERROR] server-47773f58-222a-9b0f-314b-edf27ff170af: failed to get log: index=2 error="log not found" +2025-01-28T14:55:23.758Z [INFO] server-47773f58-222a-9b0f-314b-edf27ff170af: aborting pipeline replication: peer="{Voter server-4b888034-289d-dca9-34f0-95dc436fecc5 4b888034-289d-dca9-34f0-95dc436fecc5}" +2025-01-28T14:55:23.758Z [ERROR] server-47773f58-222a-9b0f-314b-edf27ff170af: failed to get log: index=2 error="log not found" +2025-01-28T14:55:23.758Z [INFO] server-47773f58-222a-9b0f-314b-edf27ff170af: aborting pipeline replication: peer="{Voter server-bcae2595-402b-7bec-9a33-21ade35672fe bcae2595-402b-7bec-9a33-21ade35672fe}" +2025-01-28T14:55:23.763Z [ERROR] server-47773f58-222a-9b0f-314b-edf27ff170af: failed to get log: index=2 error="log not found" +2025-01-28T14:55:23.764Z [INFO] creating new snapshot: path=/tmp/raft2496577681/snapshots/2-1013-1738076123764.tmp +2025-01-28T14:55:23.764Z [ERROR] server-47773f58-222a-9b0f-314b-edf27ff170af: failed to get log: index=2 error="log not found" +2025-01-28T14:55:23.765Z [INFO] server-4b888034-289d-dca9-34f0-95dc436fecc5: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:23.765Z [INFO] creating new snapshot: path=/tmp/raft2947565179/snapshots/2-1013-1738076123765.tmp +2025-01-28T14:55:23.766Z [INFO] server-4b888034-289d-dca9-34f0-95dc436fecc5: copied to local snapshot: bytes=71 +2025-01-28T14:55:23.766Z [INFO] server-bcae2595-402b-7bec-9a33-21ade35672fe: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:23.767Z [INFO] server-bcae2595-402b-7bec-9a33-21ade35672fe: copied to local snapshot: bytes=71 +2025-01-28T14:55:23.767Z [INFO] server-4b888034-289d-dca9-34f0-95dc436fecc5: snapshot restore progress: id=2-1013-1738076123764 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:23.769Z [INFO] server-4b888034-289d-dca9-34f0-95dc436fecc5: removing all old logs from log store +2025-01-28T14:55:23.769Z [INFO] server-4b888034-289d-dca9-34f0-95dc436fecc5: compacting logs: from=1 to=2 +2025-01-28T14:55:23.769Z [INFO] server-4b888034-289d-dca9-34f0-95dc436fecc5: Installed remote snapshot +2025-01-28T14:55:23.770Z [INFO] server-47773f58-222a-9b0f-314b-edf27ff170af: pipelining replication: peer="{Voter server-4b888034-289d-dca9-34f0-95dc436fecc5 4b888034-289d-dca9-34f0-95dc436fecc5}" +2025-01-28T14:55:23.771Z [INFO] server-bcae2595-402b-7bec-9a33-21ade35672fe: snapshot restore progress: id=2-1013-1738076123765 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:23.771Z [INFO] server-bcae2595-402b-7bec-9a33-21ade35672fe: removing all old logs from log store +2025-01-28T14:55:23.771Z [INFO] server-bcae2595-402b-7bec-9a33-21ade35672fe: compacting logs: from=1 to=2 +2025-01-28T14:55:23.771Z [INFO] server-bcae2595-402b-7bec-9a33-21ade35672fe: Installed remote snapshot +2025-01-28T14:55:23.771Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:55:23.771Z [INFO] server-47773f58-222a-9b0f-314b-edf27ff170af: pipelining replication: peer="{Voter server-bcae2595-402b-7bec-9a33-21ade35672fe bcae2595-402b-7bec-9a33-21ade35672fe}" +2025-01-28T14:55:24.777Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:55:23.771577484 +0000 UTC m=+109.446067464 (2 nodes), 2.2µs from start of poll, 1.630475067s from cluster start. Timeout at 2025-01-28 14:55:24.776734489 +0000 UTC m=+110.451224449, 1.005156985s after stability +2025-01-28T14:55:24.789Z [INFO] server-b9e3aad9-e9e6-85f9-361d-d215f4a207cd: aborting pipeline replication: peer="{Voter server-60f4ed59-de49-f423-3161-c1381e85a1e0 60f4ed59-de49-f423-3161-c1381e85a1e0}" +2025-01-28T14:55:24.789Z [INFO] server-b9e3aad9-e9e6-85f9-361d-d215f4a207cd: aborting pipeline replication: peer="{Voter server-42a1f769-6cd6-036d-3c0b-5a133b0f1eb9 42a1f769-6cd6-036d-3c0b-5a133b0f1eb9}" === RUN TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_false -2025-01-28T14:48:23.666Z [INFO] server-e7cf8dca-da8d-b7f3-98d1-7bb15b3994b1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e7cf8dca-da8d-b7f3-98d1-7bb15b3994b1 Address:e7cf8dca-da8d-b7f3-98d1-7bb15b3994b1} {Suffrage:Voter ID:server-d6b9257c-51c8-af4e-866d-554a5007d821 Address:d6b9257c-51c8-af4e-866d-554a5007d821} {Suffrage:Voter ID:server-8818d339-8684-0522-92a9-44a956aa148f Address:8818d339-8684-0522-92a9-44a956aa148f}]" -2025-01-28T14:48:23.666Z [INFO] server-e7cf8dca-da8d-b7f3-98d1-7bb15b3994b1: entering follower state: follower="Node at e7cf8dca-da8d-b7f3-98d1-7bb15b3994b1 [Follower]" leader-address= leader-id= -2025-01-28T14:48:23.667Z [INFO] server-d6b9257c-51c8-af4e-866d-554a5007d821: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e7cf8dca-da8d-b7f3-98d1-7bb15b3994b1 Address:e7cf8dca-da8d-b7f3-98d1-7bb15b3994b1} {Suffrage:Voter ID:server-d6b9257c-51c8-af4e-866d-554a5007d821 Address:d6b9257c-51c8-af4e-866d-554a5007d821} {Suffrage:Voter ID:server-8818d339-8684-0522-92a9-44a956aa148f Address:8818d339-8684-0522-92a9-44a956aa148f}]" -2025-01-28T14:48:23.667Z [INFO] server-d6b9257c-51c8-af4e-866d-554a5007d821: entering follower state: follower="Node at d6b9257c-51c8-af4e-866d-554a5007d821 [Follower]" leader-address= leader-id= -2025-01-28T14:48:23.667Z [INFO] server-8818d339-8684-0522-92a9-44a956aa148f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e7cf8dca-da8d-b7f3-98d1-7bb15b3994b1 Address:e7cf8dca-da8d-b7f3-98d1-7bb15b3994b1} {Suffrage:Voter ID:server-d6b9257c-51c8-af4e-866d-554a5007d821 Address:d6b9257c-51c8-af4e-866d-554a5007d821} {Suffrage:Voter ID:server-8818d339-8684-0522-92a9-44a956aa148f Address:8818d339-8684-0522-92a9-44a956aa148f}]" -2025-01-28T14:48:23.667Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:23.667Z [INFO] server-8818d339-8684-0522-92a9-44a956aa148f: entering follower state: follower="Node at 8818d339-8684-0522-92a9-44a956aa148f [Follower]" leader-address= leader-id= -2025-01-28T14:48:24.223Z [WARN] server-e7cf8dca-da8d-b7f3-98d1-7bb15b3994b1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:24.223Z [INFO] server-e7cf8dca-da8d-b7f3-98d1-7bb15b3994b1: entering candidate state: node="Node at e7cf8dca-da8d-b7f3-98d1-7bb15b3994b1 [Candidate]" term=2 -2025-01-28T14:48:24.224Z [INFO] server-e7cf8dca-da8d-b7f3-98d1-7bb15b3994b1: election won: term=2 tally=2 -2025-01-28T14:48:24.224Z [INFO] server-e7cf8dca-da8d-b7f3-98d1-7bb15b3994b1: entering leader state: leader="Node at e7cf8dca-da8d-b7f3-98d1-7bb15b3994b1 [Leader]" -2025-01-28T14:48:24.224Z [INFO] server-e7cf8dca-da8d-b7f3-98d1-7bb15b3994b1: added peer, starting replication: peer=server-d6b9257c-51c8-af4e-866d-554a5007d821 -2025-01-28T14:48:24.225Z [INFO] server-e7cf8dca-da8d-b7f3-98d1-7bb15b3994b1: added peer, starting replication: peer=server-8818d339-8684-0522-92a9-44a956aa148f -2025-01-28T14:48:24.225Z [INFO] server-e7cf8dca-da8d-b7f3-98d1-7bb15b3994b1: pipelining replication: peer="{Voter server-8818d339-8684-0522-92a9-44a956aa148f 8818d339-8684-0522-92a9-44a956aa148f}" -2025-01-28T14:48:24.227Z [INFO] server-e7cf8dca-da8d-b7f3-98d1-7bb15b3994b1: pipelining replication: peer="{Voter server-d6b9257c-51c8-af4e-866d-554a5007d821 d6b9257c-51c8-af4e-866d-554a5007d821}" -2025-01-28T14:48:25.231Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:24.226462681 +0000 UTC m=+87.309868039 (1 nodes), 558.515252ms from start of poll, 560.854989ms from cluster start. Timeout at 2025-01-28 14:48:25.231473015 +0000 UTC m=+88.314878039, 1.00501s after stability -2025-01-28T14:48:25.236Z [INFO] server-e7cf8dca-da8d-b7f3-98d1-7bb15b3994b1: starting snapshot up to: index=12 -2025-01-28T14:48:25.237Z [INFO] creating new snapshot: path=/tmp/raft2833461981/snapshots/2-12-1738075705237.tmp -2025-01-28T14:48:25.240Z [INFO] server-e7cf8dca-da8d-b7f3-98d1-7bb15b3994b1: snapshot complete up to: index=12 -2025-01-28T14:48:25.246Z [INFO] creating new snapshot: path=/tmp/raft2833461981/snapshots/2-10013-1738075705246.tmp -2025-01-28T14:48:25.251Z [INFO] server-e7cf8dca-da8d-b7f3-98d1-7bb15b3994b1: copied to local snapshot: bytes=71 -2025-01-28T14:48:25.252Z [INFO] server-e7cf8dca-da8d-b7f3-98d1-7bb15b3994b1: snapshot restore progress: id=2-10013-1738075705246 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:25.253Z [INFO] server-e7cf8dca-da8d-b7f3-98d1-7bb15b3994b1: restored user snapshot: index=10013 -2025-01-28T14:48:25.253Z [ERROR] server-e7cf8dca-da8d-b7f3-98d1-7bb15b3994b1: failed to get log: index=23 error="log not found" -2025-01-28T14:48:25.253Z [INFO] server-e7cf8dca-da8d-b7f3-98d1-7bb15b3994b1: aborting pipeline replication: peer="{Voter server-8818d339-8684-0522-92a9-44a956aa148f 8818d339-8684-0522-92a9-44a956aa148f}" -2025-01-28T14:48:25.254Z [ERROR] server-e7cf8dca-da8d-b7f3-98d1-7bb15b3994b1: failed to get log: index=23 error="log not found" -2025-01-28T14:48:25.254Z [INFO] server-e7cf8dca-da8d-b7f3-98d1-7bb15b3994b1: aborting pipeline replication: peer="{Voter server-d6b9257c-51c8-af4e-866d-554a5007d821 d6b9257c-51c8-af4e-866d-554a5007d821}" -2025-01-28T14:48:25.261Z [ERROR] server-e7cf8dca-da8d-b7f3-98d1-7bb15b3994b1: failed to get log: index=23 error="log not found" -2025-01-28T14:48:25.267Z [INFO] creating new snapshot: path=/tmp/raft4065308969/snapshots/2-10013-1738075705267.tmp -2025-01-28T14:48:25.268Z [ERROR] server-e7cf8dca-da8d-b7f3-98d1-7bb15b3994b1: failed to get log: index=23 error="log not found" -2025-01-28T14:48:25.270Z [INFO] server-8818d339-8684-0522-92a9-44a956aa148f: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:25.272Z [INFO] creating new snapshot: path=/tmp/raft316726399/snapshots/2-10013-1738075705272.tmp -2025-01-28T14:48:25.272Z [INFO] server-8818d339-8684-0522-92a9-44a956aa148f: copied to local snapshot: bytes=71 -2025-01-28T14:48:25.273Z [INFO] server-8818d339-8684-0522-92a9-44a956aa148f: snapshot restore progress: id=2-10013-1738075705267 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:25.274Z [INFO] server-8818d339-8684-0522-92a9-44a956aa148f: Installed remote snapshot -2025-01-28T14:48:25.274Z [INFO] server-d6b9257c-51c8-af4e-866d-554a5007d821: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:25.275Z [INFO] server-e7cf8dca-da8d-b7f3-98d1-7bb15b3994b1: pipelining replication: peer="{Voter server-8818d339-8684-0522-92a9-44a956aa148f 8818d339-8684-0522-92a9-44a956aa148f}" -2025-01-28T14:48:25.275Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:48:25.277Z [INFO] server-d6b9257c-51c8-af4e-866d-554a5007d821: copied to local snapshot: bytes=71 -2025-01-28T14:48:25.279Z [INFO] server-d6b9257c-51c8-af4e-866d-554a5007d821: snapshot restore progress: id=2-10013-1738075705272 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:25.279Z [INFO] server-d6b9257c-51c8-af4e-866d-554a5007d821: Installed remote snapshot -2025-01-28T14:48:25.280Z [INFO] server-e7cf8dca-da8d-b7f3-98d1-7bb15b3994b1: pipelining replication: peer="{Voter server-d6b9257c-51c8-af4e-866d-554a5007d821 d6b9257c-51c8-af4e-866d-554a5007d821}" -2025-01-28T14:48:26.281Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:48:25.275328162 +0000 UTC m=+88.358732853 (2 nodes), 8.334µs from start of poll, 1.609719803s from cluster start. Timeout at 2025-01-28 14:48:26.280763842 +0000 UTC m=+89.364168866, 1.005436013s after stability -2025-01-28T14:48:26.308Z [INFO] server-e7cf8dca-da8d-b7f3-98d1-7bb15b3994b1: aborting pipeline replication: peer="{Voter server-8818d339-8684-0522-92a9-44a956aa148f 8818d339-8684-0522-92a9-44a956aa148f}" -2025-01-28T14:48:26.308Z [INFO] server-e7cf8dca-da8d-b7f3-98d1-7bb15b3994b1: aborting pipeline replication: peer="{Voter server-d6b9257c-51c8-af4e-866d-554a5007d821 d6b9257c-51c8-af4e-866d-554a5007d821}" +2025-01-28T14:55:24.793Z [INFO] server-1b0a5d29-a527-f3af-3fd5-278ce33bad5c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1b0a5d29-a527-f3af-3fd5-278ce33bad5c Address:1b0a5d29-a527-f3af-3fd5-278ce33bad5c} {Suffrage:Voter ID:server-71a325e5-0e8a-6458-1fe9-ab9fbaf89058 Address:71a325e5-0e8a-6458-1fe9-ab9fbaf89058} {Suffrage:Voter ID:server-f4cb0155-f07c-36ec-da11-8420eadfa1b6 Address:f4cb0155-f07c-36ec-da11-8420eadfa1b6}]" +2025-01-28T14:55:24.793Z [INFO] server-1b0a5d29-a527-f3af-3fd5-278ce33bad5c: entering follower state: follower="Node at 1b0a5d29-a527-f3af-3fd5-278ce33bad5c [Follower]" leader-address= leader-id= +2025-01-28T14:55:24.793Z [INFO] server-71a325e5-0e8a-6458-1fe9-ab9fbaf89058: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1b0a5d29-a527-f3af-3fd5-278ce33bad5c Address:1b0a5d29-a527-f3af-3fd5-278ce33bad5c} {Suffrage:Voter ID:server-71a325e5-0e8a-6458-1fe9-ab9fbaf89058 Address:71a325e5-0e8a-6458-1fe9-ab9fbaf89058} {Suffrage:Voter ID:server-f4cb0155-f07c-36ec-da11-8420eadfa1b6 Address:f4cb0155-f07c-36ec-da11-8420eadfa1b6}]" +2025-01-28T14:55:24.793Z [INFO] server-71a325e5-0e8a-6458-1fe9-ab9fbaf89058: entering follower state: follower="Node at 71a325e5-0e8a-6458-1fe9-ab9fbaf89058 [Follower]" leader-address= leader-id= +2025-01-28T14:55:24.794Z [INFO] server-f4cb0155-f07c-36ec-da11-8420eadfa1b6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1b0a5d29-a527-f3af-3fd5-278ce33bad5c Address:1b0a5d29-a527-f3af-3fd5-278ce33bad5c} {Suffrage:Voter ID:server-71a325e5-0e8a-6458-1fe9-ab9fbaf89058 Address:71a325e5-0e8a-6458-1fe9-ab9fbaf89058} {Suffrage:Voter ID:server-f4cb0155-f07c-36ec-da11-8420eadfa1b6 Address:f4cb0155-f07c-36ec-da11-8420eadfa1b6}]" +2025-01-28T14:55:24.794Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:24.794Z [INFO] server-f4cb0155-f07c-36ec-da11-8420eadfa1b6: entering follower state: follower="Node at f4cb0155-f07c-36ec-da11-8420eadfa1b6 [Follower]" leader-address= leader-id= +2025-01-28T14:55:25.559Z [WARN] server-1b0a5d29-a527-f3af-3fd5-278ce33bad5c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:25.559Z [INFO] server-1b0a5d29-a527-f3af-3fd5-278ce33bad5c: entering candidate state: node="Node at 1b0a5d29-a527-f3af-3fd5-278ce33bad5c [Candidate]" term=2 +2025-01-28T14:55:25.560Z [INFO] server-1b0a5d29-a527-f3af-3fd5-278ce33bad5c: election won: term=2 tally=2 +2025-01-28T14:55:25.560Z [INFO] server-1b0a5d29-a527-f3af-3fd5-278ce33bad5c: entering leader state: leader="Node at 1b0a5d29-a527-f3af-3fd5-278ce33bad5c [Leader]" +2025-01-28T14:55:25.560Z [INFO] server-1b0a5d29-a527-f3af-3fd5-278ce33bad5c: added peer, starting replication: peer=server-71a325e5-0e8a-6458-1fe9-ab9fbaf89058 +2025-01-28T14:55:25.560Z [INFO] server-1b0a5d29-a527-f3af-3fd5-278ce33bad5c: added peer, starting replication: peer=server-f4cb0155-f07c-36ec-da11-8420eadfa1b6 +2025-01-28T14:55:25.560Z [INFO] server-1b0a5d29-a527-f3af-3fd5-278ce33bad5c: pipelining replication: peer="{Voter server-71a325e5-0e8a-6458-1fe9-ab9fbaf89058 71a325e5-0e8a-6458-1fe9-ab9fbaf89058}" +2025-01-28T14:55:25.560Z [INFO] server-1b0a5d29-a527-f3af-3fd5-278ce33bad5c: pipelining replication: peer="{Voter server-f4cb0155-f07c-36ec-da11-8420eadfa1b6 f4cb0155-f07c-36ec-da11-8420eadfa1b6}" +2025-01-28T14:55:26.565Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:25.560191532 +0000 UTC m=+111.234681512 (1 nodes), 765.854512ms from start of poll, 766.919751ms from cluster start. Timeout at 2025-01-28 14:55:26.565193211 +0000 UTC m=+112.239683211, 1.005001699s after stability +2025-01-28T14:55:26.566Z [INFO] server-1b0a5d29-a527-f3af-3fd5-278ce33bad5c: starting snapshot up to: index=12 +2025-01-28T14:55:26.566Z [INFO] creating new snapshot: path=/tmp/raft449509607/snapshots/2-12-1738076126566.tmp +2025-01-28T14:55:26.567Z [INFO] server-1b0a5d29-a527-f3af-3fd5-278ce33bad5c: snapshot complete up to: index=12 +2025-01-28T14:55:26.569Z [INFO] creating new snapshot: path=/tmp/raft449509607/snapshots/2-10013-1738076126569.tmp +2025-01-28T14:55:26.570Z [INFO] server-1b0a5d29-a527-f3af-3fd5-278ce33bad5c: copied to local snapshot: bytes=71 +2025-01-28T14:55:26.571Z [INFO] server-1b0a5d29-a527-f3af-3fd5-278ce33bad5c: snapshot restore progress: id=2-10013-1738076126569 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:26.571Z [INFO] server-1b0a5d29-a527-f3af-3fd5-278ce33bad5c: restored user snapshot: index=10013 +2025-01-28T14:55:26.571Z [ERROR] server-1b0a5d29-a527-f3af-3fd5-278ce33bad5c: failed to get log: index=23 error="log not found" +2025-01-28T14:55:26.571Z [INFO] server-1b0a5d29-a527-f3af-3fd5-278ce33bad5c: aborting pipeline replication: peer="{Voter server-71a325e5-0e8a-6458-1fe9-ab9fbaf89058 71a325e5-0e8a-6458-1fe9-ab9fbaf89058}" +2025-01-28T14:55:26.571Z [ERROR] server-1b0a5d29-a527-f3af-3fd5-278ce33bad5c: failed to get log: index=23 error="log not found" +2025-01-28T14:55:26.571Z [INFO] server-1b0a5d29-a527-f3af-3fd5-278ce33bad5c: aborting pipeline replication: peer="{Voter server-f4cb0155-f07c-36ec-da11-8420eadfa1b6 f4cb0155-f07c-36ec-da11-8420eadfa1b6}" +2025-01-28T14:55:26.579Z [ERROR] server-1b0a5d29-a527-f3af-3fd5-278ce33bad5c: failed to get log: index=23 error="log not found" +2025-01-28T14:55:26.580Z [INFO] creating new snapshot: path=/tmp/raft2780785844/snapshots/2-10013-1738076126580.tmp +2025-01-28T14:55:26.580Z [ERROR] server-1b0a5d29-a527-f3af-3fd5-278ce33bad5c: failed to get log: index=23 error="log not found" +2025-01-28T14:55:26.580Z [INFO] server-71a325e5-0e8a-6458-1fe9-ab9fbaf89058: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:26.581Z [INFO] creating new snapshot: path=/tmp/raft1465756817/snapshots/2-10013-1738076126581.tmp +2025-01-28T14:55:26.581Z [INFO] server-71a325e5-0e8a-6458-1fe9-ab9fbaf89058: copied to local snapshot: bytes=71 +2025-01-28T14:55:26.581Z [INFO] server-f4cb0155-f07c-36ec-da11-8420eadfa1b6: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:26.582Z [INFO] server-71a325e5-0e8a-6458-1fe9-ab9fbaf89058: snapshot restore progress: id=2-10013-1738076126580 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:26.582Z [INFO] server-71a325e5-0e8a-6458-1fe9-ab9fbaf89058: Installed remote snapshot +2025-01-28T14:55:26.582Z [WARN] server-71a325e5-0e8a-6458-1fe9-ab9fbaf89058: skipping application of old log: index=22 +2025-01-28T14:55:26.582Z [INFO] server-1b0a5d29-a527-f3af-3fd5-278ce33bad5c: pipelining replication: peer="{Voter server-71a325e5-0e8a-6458-1fe9-ab9fbaf89058 71a325e5-0e8a-6458-1fe9-ab9fbaf89058}" +2025-01-28T14:55:26.582Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:55:26.582Z [INFO] server-f4cb0155-f07c-36ec-da11-8420eadfa1b6: copied to local snapshot: bytes=71 +2025-01-28T14:55:26.582Z [INFO] server-f4cb0155-f07c-36ec-da11-8420eadfa1b6: snapshot restore progress: id=2-10013-1738076126581 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:26.583Z [INFO] server-f4cb0155-f07c-36ec-da11-8420eadfa1b6: Installed remote snapshot +2025-01-28T14:55:26.583Z [INFO] server-1b0a5d29-a527-f3af-3fd5-278ce33bad5c: pipelining replication: peer="{Voter server-f4cb0155-f07c-36ec-da11-8420eadfa1b6 f4cb0155-f07c-36ec-da11-8420eadfa1b6}" +2025-01-28T14:55:27.587Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:55:26.582559786 +0000 UTC m=+112.257049766 (2 nodes), 2.46µs from start of poll, 1.789288005s from cluster start. Timeout at 2025-01-28 14:55:27.587688494 +0000 UTC m=+113.262178474, 1.005128708s after stability +2025-01-28T14:55:27.602Z [INFO] server-1b0a5d29-a527-f3af-3fd5-278ce33bad5c: aborting pipeline replication: peer="{Voter server-71a325e5-0e8a-6458-1fe9-ab9fbaf89058 71a325e5-0e8a-6458-1fe9-ab9fbaf89058}" +2025-01-28T14:55:27.602Z [INFO] server-1b0a5d29-a527-f3af-3fd5-278ce33bad5c: aborting pipeline replication: peer="{Voter server-f4cb0155-f07c-36ec-da11-8420eadfa1b6 f4cb0155-f07c-36ec-da11-8420eadfa1b6}" === RUN TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_false -2025-01-28T14:48:26.322Z [INFO] server-2ff64675-a041-218a-bfb7-7309389e7ef0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2ff64675-a041-218a-bfb7-7309389e7ef0 Address:2ff64675-a041-218a-bfb7-7309389e7ef0} {Suffrage:Voter ID:server-385b77c0-7d33-6911-3190-c3ba095b5eda Address:385b77c0-7d33-6911-3190-c3ba095b5eda} {Suffrage:Voter ID:server-eb806551-8db7-5a37-e142-e2f33f3531cd Address:eb806551-8db7-5a37-e142-e2f33f3531cd}]" -2025-01-28T14:48:26.322Z [INFO] server-2ff64675-a041-218a-bfb7-7309389e7ef0: entering follower state: follower="Node at 2ff64675-a041-218a-bfb7-7309389e7ef0 [Follower]" leader-address= leader-id= -2025-01-28T14:48:26.322Z [INFO] server-385b77c0-7d33-6911-3190-c3ba095b5eda: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2ff64675-a041-218a-bfb7-7309389e7ef0 Address:2ff64675-a041-218a-bfb7-7309389e7ef0} {Suffrage:Voter ID:server-385b77c0-7d33-6911-3190-c3ba095b5eda Address:385b77c0-7d33-6911-3190-c3ba095b5eda} {Suffrage:Voter ID:server-eb806551-8db7-5a37-e142-e2f33f3531cd Address:eb806551-8db7-5a37-e142-e2f33f3531cd}]" -2025-01-28T14:48:26.323Z [INFO] server-385b77c0-7d33-6911-3190-c3ba095b5eda: entering follower state: follower="Node at 385b77c0-7d33-6911-3190-c3ba095b5eda [Follower]" leader-address= leader-id= -2025-01-28T14:48:26.323Z [INFO] server-eb806551-8db7-5a37-e142-e2f33f3531cd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2ff64675-a041-218a-bfb7-7309389e7ef0 Address:2ff64675-a041-218a-bfb7-7309389e7ef0} {Suffrage:Voter ID:server-385b77c0-7d33-6911-3190-c3ba095b5eda Address:385b77c0-7d33-6911-3190-c3ba095b5eda} {Suffrage:Voter ID:server-eb806551-8db7-5a37-e142-e2f33f3531cd Address:eb806551-8db7-5a37-e142-e2f33f3531cd}]" -2025-01-28T14:48:26.323Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:26.323Z [INFO] server-eb806551-8db7-5a37-e142-e2f33f3531cd: entering follower state: follower="Node at eb806551-8db7-5a37-e142-e2f33f3531cd [Follower]" leader-address= leader-id= -2025-01-28T14:48:26.874Z [WARN] server-2ff64675-a041-218a-bfb7-7309389e7ef0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:26.874Z [INFO] server-2ff64675-a041-218a-bfb7-7309389e7ef0: entering candidate state: node="Node at 2ff64675-a041-218a-bfb7-7309389e7ef0 [Candidate]" term=2 -2025-01-28T14:48:26.875Z [INFO] server-2ff64675-a041-218a-bfb7-7309389e7ef0: election won: term=2 tally=2 -2025-01-28T14:48:26.875Z [INFO] server-2ff64675-a041-218a-bfb7-7309389e7ef0: entering leader state: leader="Node at 2ff64675-a041-218a-bfb7-7309389e7ef0 [Leader]" -2025-01-28T14:48:26.875Z [INFO] server-2ff64675-a041-218a-bfb7-7309389e7ef0: added peer, starting replication: peer=server-385b77c0-7d33-6911-3190-c3ba095b5eda -2025-01-28T14:48:26.875Z [INFO] server-2ff64675-a041-218a-bfb7-7309389e7ef0: added peer, starting replication: peer=server-eb806551-8db7-5a37-e142-e2f33f3531cd -2025-01-28T14:48:26.876Z [INFO] server-2ff64675-a041-218a-bfb7-7309389e7ef0: pipelining replication: peer="{Voter server-385b77c0-7d33-6911-3190-c3ba095b5eda 385b77c0-7d33-6911-3190-c3ba095b5eda}" -2025-01-28T14:48:26.876Z [INFO] server-2ff64675-a041-218a-bfb7-7309389e7ef0: pipelining replication: peer="{Voter server-eb806551-8db7-5a37-e142-e2f33f3531cd eb806551-8db7-5a37-e142-e2f33f3531cd}" -2025-01-28T14:48:27.881Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:26.876171701 +0000 UTC m=+89.959576725 (1 nodes), 552.561075ms from start of poll, 554.760141ms from cluster start. Timeout at 2025-01-28 14:48:27.881195035 +0000 UTC m=+90.964600059, 1.005023334s after stability -2025-01-28T14:48:27.886Z [INFO] server-2ff64675-a041-218a-bfb7-7309389e7ef0: starting snapshot up to: index=12 -2025-01-28T14:48:27.886Z [INFO] creating new snapshot: path=/tmp/raft2936564990/snapshots/2-12-1738075707886.tmp -2025-01-28T14:48:27.890Z [INFO] server-2ff64675-a041-218a-bfb7-7309389e7ef0: snapshot complete up to: index=12 -2025-01-28T14:48:27.896Z [INFO] creating new snapshot: path=/tmp/raft2936564990/snapshots/2-10013-1738075707896.tmp -2025-01-28T14:48:27.899Z [INFO] server-2ff64675-a041-218a-bfb7-7309389e7ef0: copied to local snapshot: bytes=71 -2025-01-28T14:48:27.901Z [INFO] server-2ff64675-a041-218a-bfb7-7309389e7ef0: snapshot restore progress: id=2-10013-1738075707896 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:27.901Z [INFO] server-2ff64675-a041-218a-bfb7-7309389e7ef0: removing all old logs from log store -2025-01-28T14:48:27.901Z [INFO] server-2ff64675-a041-218a-bfb7-7309389e7ef0: compacting logs: from=1 to=22 -2025-01-28T14:48:27.901Z [INFO] server-2ff64675-a041-218a-bfb7-7309389e7ef0: restored user snapshot: index=10013 -2025-01-28T14:48:27.901Z [ERROR] server-2ff64675-a041-218a-bfb7-7309389e7ef0: failed to get log: index=22 error="log not found" -2025-01-28T14:48:27.901Z [ERROR] server-2ff64675-a041-218a-bfb7-7309389e7ef0: failed to get log: index=22 error="log not found" -2025-01-28T14:48:27.902Z [INFO] server-2ff64675-a041-218a-bfb7-7309389e7ef0: aborting pipeline replication: peer="{Voter server-385b77c0-7d33-6911-3190-c3ba095b5eda 385b77c0-7d33-6911-3190-c3ba095b5eda}" -2025-01-28T14:48:27.902Z [INFO] server-2ff64675-a041-218a-bfb7-7309389e7ef0: aborting pipeline replication: peer="{Voter server-eb806551-8db7-5a37-e142-e2f33f3531cd eb806551-8db7-5a37-e142-e2f33f3531cd}" -2025-01-28T14:48:27.909Z [ERROR] server-2ff64675-a041-218a-bfb7-7309389e7ef0: failed to get log: index=22 error="log not found" -2025-01-28T14:48:27.910Z [ERROR] server-2ff64675-a041-218a-bfb7-7309389e7ef0: failed to get log: index=22 error="log not found" -2025-01-28T14:48:27.911Z [INFO] creating new snapshot: path=/tmp/raft1699396691/snapshots/2-10013-1738075707911.tmp -2025-01-28T14:48:27.913Z [INFO] creating new snapshot: path=/tmp/raft3793172068/snapshots/2-10013-1738075707913.tmp -2025-01-28T14:48:27.914Z [INFO] server-385b77c0-7d33-6911-3190-c3ba095b5eda: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:27.915Z [INFO] server-eb806551-8db7-5a37-e142-e2f33f3531cd: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:27.917Z [INFO] server-385b77c0-7d33-6911-3190-c3ba095b5eda: copied to local snapshot: bytes=71 -2025-01-28T14:48:27.918Z [INFO] server-eb806551-8db7-5a37-e142-e2f33f3531cd: copied to local snapshot: bytes=71 -2025-01-28T14:48:27.918Z [INFO] server-385b77c0-7d33-6911-3190-c3ba095b5eda: snapshot restore progress: id=2-10013-1738075707911 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:27.918Z [INFO] server-385b77c0-7d33-6911-3190-c3ba095b5eda: removing all old logs from log store -2025-01-28T14:48:27.918Z [INFO] server-385b77c0-7d33-6911-3190-c3ba095b5eda: compacting logs: from=1 to=22 -2025-01-28T14:48:27.918Z [INFO] server-385b77c0-7d33-6911-3190-c3ba095b5eda: Installed remote snapshot -2025-01-28T14:48:27.919Z [WARN] server-385b77c0-7d33-6911-3190-c3ba095b5eda: skipping application of old log: index=22 -2025-01-28T14:48:27.919Z [INFO] server-2ff64675-a041-218a-bfb7-7309389e7ef0: pipelining replication: peer="{Voter server-385b77c0-7d33-6911-3190-c3ba095b5eda 385b77c0-7d33-6911-3190-c3ba095b5eda}" -2025-01-28T14:48:27.920Z [INFO] server-eb806551-8db7-5a37-e142-e2f33f3531cd: snapshot restore progress: id=2-10013-1738075707913 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:27.920Z [INFO] server-eb806551-8db7-5a37-e142-e2f33f3531cd: removing all old logs from log store -2025-01-28T14:48:27.920Z [INFO] server-eb806551-8db7-5a37-e142-e2f33f3531cd: compacting logs: from=1 to=22 -2025-01-28T14:48:27.920Z [INFO] server-eb806551-8db7-5a37-e142-e2f33f3531cd: Installed remote snapshot -2025-01-28T14:48:27.920Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:48:27.921Z [INFO] server-2ff64675-a041-218a-bfb7-7309389e7ef0: pipelining replication: peer="{Voter server-eb806551-8db7-5a37-e142-e2f33f3531cd eb806551-8db7-5a37-e142-e2f33f3531cd}" -2025-01-28T14:48:28.926Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:48:27.920472046 +0000 UTC m=+91.003877404 (2 nodes), 8.667µs from start of poll, 1.59906082s from cluster start. Timeout at 2025-01-28 14:48:28.925782056 +0000 UTC m=+92.009187080, 1.005309676s after stability -2025-01-28T14:48:28.951Z [INFO] server-2ff64675-a041-218a-bfb7-7309389e7ef0: aborting pipeline replication: peer="{Voter server-eb806551-8db7-5a37-e142-e2f33f3531cd eb806551-8db7-5a37-e142-e2f33f3531cd}" -2025-01-28T14:48:28.951Z [INFO] server-2ff64675-a041-218a-bfb7-7309389e7ef0: aborting pipeline replication: peer="{Voter server-385b77c0-7d33-6911-3190-c3ba095b5eda 385b77c0-7d33-6911-3190-c3ba095b5eda}" +2025-01-28T14:55:27.606Z [INFO] server-04f6db8c-16a6-107a-6dda-cbb76cd6025d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-04f6db8c-16a6-107a-6dda-cbb76cd6025d Address:04f6db8c-16a6-107a-6dda-cbb76cd6025d} {Suffrage:Voter ID:server-c0e4b846-4b00-2b4e-a1d3-4b8075da1062 Address:c0e4b846-4b00-2b4e-a1d3-4b8075da1062} {Suffrage:Voter ID:server-c5bdfb74-bba4-3b2e-8174-33b86652d143 Address:c5bdfb74-bba4-3b2e-8174-33b86652d143}]" +2025-01-28T14:55:27.607Z [INFO] server-04f6db8c-16a6-107a-6dda-cbb76cd6025d: entering follower state: follower="Node at 04f6db8c-16a6-107a-6dda-cbb76cd6025d [Follower]" leader-address= leader-id= +2025-01-28T14:55:27.607Z [INFO] server-c0e4b846-4b00-2b4e-a1d3-4b8075da1062: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-04f6db8c-16a6-107a-6dda-cbb76cd6025d Address:04f6db8c-16a6-107a-6dda-cbb76cd6025d} {Suffrage:Voter ID:server-c0e4b846-4b00-2b4e-a1d3-4b8075da1062 Address:c0e4b846-4b00-2b4e-a1d3-4b8075da1062} {Suffrage:Voter ID:server-c5bdfb74-bba4-3b2e-8174-33b86652d143 Address:c5bdfb74-bba4-3b2e-8174-33b86652d143}]" +2025-01-28T14:55:27.607Z [INFO] server-c0e4b846-4b00-2b4e-a1d3-4b8075da1062: entering follower state: follower="Node at c0e4b846-4b00-2b4e-a1d3-4b8075da1062 [Follower]" leader-address= leader-id= +2025-01-28T14:55:27.607Z [INFO] server-c5bdfb74-bba4-3b2e-8174-33b86652d143: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-04f6db8c-16a6-107a-6dda-cbb76cd6025d Address:04f6db8c-16a6-107a-6dda-cbb76cd6025d} {Suffrage:Voter ID:server-c0e4b846-4b00-2b4e-a1d3-4b8075da1062 Address:c0e4b846-4b00-2b4e-a1d3-4b8075da1062} {Suffrage:Voter ID:server-c5bdfb74-bba4-3b2e-8174-33b86652d143 Address:c5bdfb74-bba4-3b2e-8174-33b86652d143}]" +2025-01-28T14:55:27.607Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:27.607Z [INFO] server-c5bdfb74-bba4-3b2e-8174-33b86652d143: entering follower state: follower="Node at c5bdfb74-bba4-3b2e-8174-33b86652d143 [Follower]" leader-address= leader-id= +2025-01-28T14:55:28.295Z [WARN] server-c5bdfb74-bba4-3b2e-8174-33b86652d143: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:28.295Z [INFO] server-c5bdfb74-bba4-3b2e-8174-33b86652d143: entering candidate state: node="Node at c5bdfb74-bba4-3b2e-8174-33b86652d143 [Candidate]" term=2 +2025-01-28T14:55:28.296Z [INFO] server-c5bdfb74-bba4-3b2e-8174-33b86652d143: election won: term=2 tally=2 +2025-01-28T14:55:28.296Z [INFO] server-c5bdfb74-bba4-3b2e-8174-33b86652d143: entering leader state: leader="Node at c5bdfb74-bba4-3b2e-8174-33b86652d143 [Leader]" +2025-01-28T14:55:28.296Z [INFO] server-c5bdfb74-bba4-3b2e-8174-33b86652d143: added peer, starting replication: peer=server-04f6db8c-16a6-107a-6dda-cbb76cd6025d +2025-01-28T14:55:28.296Z [INFO] server-c5bdfb74-bba4-3b2e-8174-33b86652d143: added peer, starting replication: peer=server-c0e4b846-4b00-2b4e-a1d3-4b8075da1062 +2025-01-28T14:55:28.296Z [INFO] server-c5bdfb74-bba4-3b2e-8174-33b86652d143: pipelining replication: peer="{Voter server-04f6db8c-16a6-107a-6dda-cbb76cd6025d 04f6db8c-16a6-107a-6dda-cbb76cd6025d}" +2025-01-28T14:55:28.296Z [INFO] server-c5bdfb74-bba4-3b2e-8174-33b86652d143: pipelining replication: peer="{Voter server-c0e4b846-4b00-2b4e-a1d3-4b8075da1062 c0e4b846-4b00-2b4e-a1d3-4b8075da1062}" +2025-01-28T14:55:29.301Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:28.296199547 +0000 UTC m=+113.970689527 (1 nodes), 688.421117ms from start of poll, 689.459299ms from cluster start. Timeout at 2025-01-28 14:55:29.301203447 +0000 UTC m=+114.975693467, 1.00500394s after stability +2025-01-28T14:55:29.302Z [INFO] server-c5bdfb74-bba4-3b2e-8174-33b86652d143: starting snapshot up to: index=12 +2025-01-28T14:55:29.303Z [INFO] creating new snapshot: path=/tmp/raft3367337895/snapshots/2-12-1738076129303.tmp +2025-01-28T14:55:29.304Z [INFO] server-c5bdfb74-bba4-3b2e-8174-33b86652d143: snapshot complete up to: index=12 +2025-01-28T14:55:29.306Z [INFO] creating new snapshot: path=/tmp/raft3367337895/snapshots/2-10013-1738076129306.tmp +2025-01-28T14:55:29.308Z [INFO] server-c5bdfb74-bba4-3b2e-8174-33b86652d143: copied to local snapshot: bytes=71 +2025-01-28T14:55:29.308Z [INFO] server-c5bdfb74-bba4-3b2e-8174-33b86652d143: snapshot restore progress: id=2-10013-1738076129306 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:29.308Z [INFO] server-c5bdfb74-bba4-3b2e-8174-33b86652d143: removing all old logs from log store +2025-01-28T14:55:29.308Z [INFO] server-c5bdfb74-bba4-3b2e-8174-33b86652d143: compacting logs: from=1 to=22 +2025-01-28T14:55:29.308Z [INFO] server-c5bdfb74-bba4-3b2e-8174-33b86652d143: restored user snapshot: index=10013 +2025-01-28T14:55:29.308Z [ERROR] server-c5bdfb74-bba4-3b2e-8174-33b86652d143: failed to get log: index=22 error="log not found" +2025-01-28T14:55:29.308Z [INFO] server-c5bdfb74-bba4-3b2e-8174-33b86652d143: aborting pipeline replication: peer="{Voter server-c0e4b846-4b00-2b4e-a1d3-4b8075da1062 c0e4b846-4b00-2b4e-a1d3-4b8075da1062}" +2025-01-28T14:55:29.308Z [ERROR] server-c5bdfb74-bba4-3b2e-8174-33b86652d143: failed to get log: index=22 error="log not found" +2025-01-28T14:55:29.308Z [INFO] server-c5bdfb74-bba4-3b2e-8174-33b86652d143: aborting pipeline replication: peer="{Voter server-04f6db8c-16a6-107a-6dda-cbb76cd6025d 04f6db8c-16a6-107a-6dda-cbb76cd6025d}" +2025-01-28T14:55:29.316Z [ERROR] server-c5bdfb74-bba4-3b2e-8174-33b86652d143: failed to get log: index=22 error="log not found" +2025-01-28T14:55:29.317Z [INFO] creating new snapshot: path=/tmp/raft1246544600/snapshots/2-10013-1738076129317.tmp +2025-01-28T14:55:29.317Z [INFO] server-04f6db8c-16a6-107a-6dda-cbb76cd6025d: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:29.318Z [INFO] server-04f6db8c-16a6-107a-6dda-cbb76cd6025d: copied to local snapshot: bytes=71 +2025-01-28T14:55:29.318Z [ERROR] server-c5bdfb74-bba4-3b2e-8174-33b86652d143: failed to get log: index=22 error="log not found" +2025-01-28T14:55:29.319Z [INFO] server-04f6db8c-16a6-107a-6dda-cbb76cd6025d: snapshot restore progress: id=2-10013-1738076129317 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:29.319Z [INFO] server-04f6db8c-16a6-107a-6dda-cbb76cd6025d: removing all old logs from log store +2025-01-28T14:55:29.319Z [INFO] server-04f6db8c-16a6-107a-6dda-cbb76cd6025d: compacting logs: from=1 to=22 +2025-01-28T14:55:29.319Z [INFO] server-04f6db8c-16a6-107a-6dda-cbb76cd6025d: Installed remote snapshot +2025-01-28T14:55:29.319Z [WARN] server-04f6db8c-16a6-107a-6dda-cbb76cd6025d: skipping application of old log: index=22 +2025-01-28T14:55:29.319Z [INFO] creating new snapshot: path=/tmp/raft3390697256/snapshots/2-10013-1738076129319.tmp +2025-01-28T14:55:29.319Z [INFO] server-c5bdfb74-bba4-3b2e-8174-33b86652d143: pipelining replication: peer="{Voter server-04f6db8c-16a6-107a-6dda-cbb76cd6025d 04f6db8c-16a6-107a-6dda-cbb76cd6025d}" +2025-01-28T14:55:29.319Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:55:29.320Z [INFO] server-c0e4b846-4b00-2b4e-a1d3-4b8075da1062: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:29.321Z [INFO] server-c0e4b846-4b00-2b4e-a1d3-4b8075da1062: copied to local snapshot: bytes=71 +2025-01-28T14:55:29.321Z [INFO] server-c0e4b846-4b00-2b4e-a1d3-4b8075da1062: snapshot restore progress: id=2-10013-1738076129319 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:29.321Z [INFO] server-c0e4b846-4b00-2b4e-a1d3-4b8075da1062: removing all old logs from log store +2025-01-28T14:55:29.321Z [INFO] server-c0e4b846-4b00-2b4e-a1d3-4b8075da1062: compacting logs: from=1 to=22 +2025-01-28T14:55:29.321Z [INFO] server-c0e4b846-4b00-2b4e-a1d3-4b8075da1062: Installed remote snapshot +2025-01-28T14:55:29.322Z [INFO] server-c5bdfb74-bba4-3b2e-8174-33b86652d143: pipelining replication: peer="{Voter server-c0e4b846-4b00-2b4e-a1d3-4b8075da1062 c0e4b846-4b00-2b4e-a1d3-4b8075da1062}" +2025-01-28T14:55:30.325Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:55:29.319862279 +0000 UTC m=+114.994352259 (2 nodes), 2.16µs from start of poll, 1.713122031s from cluster start. Timeout at 2025-01-28 14:55:30.324999585 +0000 UTC m=+115.999489625, 1.005137366s after stability +2025-01-28T14:55:30.338Z [INFO] server-c5bdfb74-bba4-3b2e-8174-33b86652d143: aborting pipeline replication: peer="{Voter server-c0e4b846-4b00-2b4e-a1d3-4b8075da1062 c0e4b846-4b00-2b4e-a1d3-4b8075da1062}" +2025-01-28T14:55:30.338Z [INFO] server-c5bdfb74-bba4-3b2e-8174-33b86652d143: aborting pipeline replication: peer="{Voter server-04f6db8c-16a6-107a-6dda-cbb76cd6025d 04f6db8c-16a6-107a-6dda-cbb76cd6025d}" === RUN TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_true -2025-01-28T14:48:28.963Z [INFO] server-d367fa51-6540-caf5-9e58-22323dcb89ff: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d367fa51-6540-caf5-9e58-22323dcb89ff Address:d367fa51-6540-caf5-9e58-22323dcb89ff} {Suffrage:Voter ID:server-0682e837-5360-7fa3-5476-d534c6de53c2 Address:0682e837-5360-7fa3-5476-d534c6de53c2} {Suffrage:Voter ID:server-db5e5232-35e9-1d78-0360-2dc309460a53 Address:db5e5232-35e9-1d78-0360-2dc309460a53}]" -2025-01-28T14:48:28.963Z [INFO] server-0682e837-5360-7fa3-5476-d534c6de53c2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d367fa51-6540-caf5-9e58-22323dcb89ff Address:d367fa51-6540-caf5-9e58-22323dcb89ff} {Suffrage:Voter ID:server-0682e837-5360-7fa3-5476-d534c6de53c2 Address:0682e837-5360-7fa3-5476-d534c6de53c2} {Suffrage:Voter ID:server-db5e5232-35e9-1d78-0360-2dc309460a53 Address:db5e5232-35e9-1d78-0360-2dc309460a53}]" -2025-01-28T14:48:28.963Z [INFO] server-d367fa51-6540-caf5-9e58-22323dcb89ff: entering follower state: follower="Node at d367fa51-6540-caf5-9e58-22323dcb89ff [Follower]" leader-address= leader-id= -2025-01-28T14:48:28.964Z [INFO] server-0682e837-5360-7fa3-5476-d534c6de53c2: entering follower state: follower="Node at 0682e837-5360-7fa3-5476-d534c6de53c2 [Follower]" leader-address= leader-id= -2025-01-28T14:48:28.964Z [INFO] server-db5e5232-35e9-1d78-0360-2dc309460a53: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d367fa51-6540-caf5-9e58-22323dcb89ff Address:d367fa51-6540-caf5-9e58-22323dcb89ff} {Suffrage:Voter ID:server-0682e837-5360-7fa3-5476-d534c6de53c2 Address:0682e837-5360-7fa3-5476-d534c6de53c2} {Suffrage:Voter ID:server-db5e5232-35e9-1d78-0360-2dc309460a53 Address:db5e5232-35e9-1d78-0360-2dc309460a53}]" -2025-01-28T14:48:28.964Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:28.964Z [INFO] server-db5e5232-35e9-1d78-0360-2dc309460a53: entering follower state: follower="Node at db5e5232-35e9-1d78-0360-2dc309460a53 [Follower]" leader-address= leader-id= -2025-01-28T14:48:29.490Z [WARN] server-d367fa51-6540-caf5-9e58-22323dcb89ff: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:29.491Z [INFO] server-d367fa51-6540-caf5-9e58-22323dcb89ff: entering candidate state: node="Node at d367fa51-6540-caf5-9e58-22323dcb89ff [Candidate]" term=2 -2025-01-28T14:48:29.491Z [INFO] server-d367fa51-6540-caf5-9e58-22323dcb89ff: election won: term=2 tally=2 -2025-01-28T14:48:29.491Z [INFO] server-d367fa51-6540-caf5-9e58-22323dcb89ff: entering leader state: leader="Node at d367fa51-6540-caf5-9e58-22323dcb89ff [Leader]" -2025-01-28T14:48:29.491Z [INFO] server-d367fa51-6540-caf5-9e58-22323dcb89ff: added peer, starting replication: peer=server-0682e837-5360-7fa3-5476-d534c6de53c2 -2025-01-28T14:48:29.491Z [INFO] server-d367fa51-6540-caf5-9e58-22323dcb89ff: added peer, starting replication: peer=server-db5e5232-35e9-1d78-0360-2dc309460a53 -2025-01-28T14:48:29.492Z [INFO] server-d367fa51-6540-caf5-9e58-22323dcb89ff: pipelining replication: peer="{Voter server-0682e837-5360-7fa3-5476-d534c6de53c2 0682e837-5360-7fa3-5476-d534c6de53c2}" -2025-01-28T14:48:29.492Z [INFO] server-d367fa51-6540-caf5-9e58-22323dcb89ff: pipelining replication: peer="{Voter server-db5e5232-35e9-1d78-0360-2dc309460a53 db5e5232-35e9-1d78-0360-2dc309460a53}" -2025-01-28T14:48:30.497Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:29.492048717 +0000 UTC m=+92.575453741 (1 nodes), 527.387662ms from start of poll, 529.401055ms from cluster start. Timeout at 2025-01-28 14:48:30.497092718 +0000 UTC m=+93.580498742, 1.005045001s after stability -2025-01-28T14:48:30.501Z [INFO] server-d367fa51-6540-caf5-9e58-22323dcb89ff: starting snapshot up to: index=12 -2025-01-28T14:48:30.501Z [INFO] creating new snapshot: path=/tmp/raft3576803583/snapshots/2-12-1738075710501.tmp -2025-01-28T14:48:30.507Z [INFO] server-d367fa51-6540-caf5-9e58-22323dcb89ff: snapshot complete up to: index=12 -2025-01-28T14:48:30.532Z [INFO] server-a007de91-f4df-05bf-e6be-220d9180296c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a007de91-f4df-05bf-e6be-220d9180296c Address:a007de91-f4df-05bf-e6be-220d9180296c} {Suffrage:Voter ID:server-6d74e363-32a7-7140-f5d0-2908e65ca9ed Address:6d74e363-32a7-7140-f5d0-2908e65ca9ed} {Suffrage:Voter ID:server-be78217d-49d1-f754-b626-ffbbdcdb9017 Address:be78217d-49d1-f754-b626-ffbbdcdb9017}]" -2025-01-28T14:48:30.532Z [INFO] server-a007de91-f4df-05bf-e6be-220d9180296c: entering follower state: follower="Node at a007de91-f4df-05bf-e6be-220d9180296c [Follower]" leader-address= leader-id= -2025-01-28T14:48:30.532Z [INFO] server-6d74e363-32a7-7140-f5d0-2908e65ca9ed: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a007de91-f4df-05bf-e6be-220d9180296c Address:a007de91-f4df-05bf-e6be-220d9180296c} {Suffrage:Voter ID:server-6d74e363-32a7-7140-f5d0-2908e65ca9ed Address:6d74e363-32a7-7140-f5d0-2908e65ca9ed} {Suffrage:Voter ID:server-be78217d-49d1-f754-b626-ffbbdcdb9017 Address:be78217d-49d1-f754-b626-ffbbdcdb9017}]" -2025-01-28T14:48:30.533Z [INFO] server-6d74e363-32a7-7140-f5d0-2908e65ca9ed: entering follower state: follower="Node at 6d74e363-32a7-7140-f5d0-2908e65ca9ed [Follower]" leader-address= leader-id= -2025-01-28T14:48:30.533Z [INFO] server-be78217d-49d1-f754-b626-ffbbdcdb9017: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a007de91-f4df-05bf-e6be-220d9180296c Address:a007de91-f4df-05bf-e6be-220d9180296c} {Suffrage:Voter ID:server-6d74e363-32a7-7140-f5d0-2908e65ca9ed Address:6d74e363-32a7-7140-f5d0-2908e65ca9ed} {Suffrage:Voter ID:server-be78217d-49d1-f754-b626-ffbbdcdb9017 Address:be78217d-49d1-f754-b626-ffbbdcdb9017}]" -2025-01-28T14:48:30.533Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:30.533Z [INFO] server-be78217d-49d1-f754-b626-ffbbdcdb9017: entering follower state: follower="Node at be78217d-49d1-f754-b626-ffbbdcdb9017 [Follower]" leader-address= leader-id= -2025-01-28T14:48:31.082Z [WARN] server-6d74e363-32a7-7140-f5d0-2908e65ca9ed: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:31.082Z [INFO] server-6d74e363-32a7-7140-f5d0-2908e65ca9ed: entering candidate state: node="Node at 6d74e363-32a7-7140-f5d0-2908e65ca9ed [Candidate]" term=2 -2025-01-28T14:48:31.083Z [INFO] server-6d74e363-32a7-7140-f5d0-2908e65ca9ed: election won: term=2 tally=2 -2025-01-28T14:48:31.083Z [INFO] server-6d74e363-32a7-7140-f5d0-2908e65ca9ed: entering leader state: leader="Node at 6d74e363-32a7-7140-f5d0-2908e65ca9ed [Leader]" -2025-01-28T14:48:31.083Z [INFO] server-6d74e363-32a7-7140-f5d0-2908e65ca9ed: added peer, starting replication: peer=server-a007de91-f4df-05bf-e6be-220d9180296c -2025-01-28T14:48:31.083Z [INFO] server-6d74e363-32a7-7140-f5d0-2908e65ca9ed: added peer, starting replication: peer=server-be78217d-49d1-f754-b626-ffbbdcdb9017 -2025-01-28T14:48:31.084Z [INFO] server-6d74e363-32a7-7140-f5d0-2908e65ca9ed: pipelining replication: peer="{Voter server-a007de91-f4df-05bf-e6be-220d9180296c a007de91-f4df-05bf-e6be-220d9180296c}" -2025-01-28T14:48:31.084Z [INFO] server-6d74e363-32a7-7140-f5d0-2908e65ca9ed: pipelining replication: peer="{Voter server-be78217d-49d1-f754-b626-ffbbdcdb9017 be78217d-49d1-f754-b626-ffbbdcdb9017}" -2025-01-28T14:48:32.090Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:31.083532978 +0000 UTC m=+94.166938003 (1 nodes), 549.793661ms from start of poll, 551.984059ms from cluster start. Timeout at 2025-01-28 14:48:32.088948324 +0000 UTC m=+95.172355015, 1.005417012s after stability -2025-01-28T14:48:32.091Z [INFO] creating new snapshot: path=/tmp/raft3162444630/snapshots/2-10013-1738075712091.tmp -2025-01-28T14:48:32.095Z [INFO] server-6d74e363-32a7-7140-f5d0-2908e65ca9ed: copied to local snapshot: bytes=71 -2025-01-28T14:48:32.097Z [INFO] server-6d74e363-32a7-7140-f5d0-2908e65ca9ed: snapshot restore progress: id=2-10013-1738075712091 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:32.097Z [INFO] server-6d74e363-32a7-7140-f5d0-2908e65ca9ed: restored user snapshot: index=10013 -2025-01-28T14:48:32.097Z [ERROR] server-6d74e363-32a7-7140-f5d0-2908e65ca9ed: failed to get log: index=3 error="log not found" -2025-01-28T14:48:32.097Z [INFO] server-6d74e363-32a7-7140-f5d0-2908e65ca9ed: aborting pipeline replication: peer="{Voter server-a007de91-f4df-05bf-e6be-220d9180296c a007de91-f4df-05bf-e6be-220d9180296c}" -2025-01-28T14:48:32.097Z [ERROR] server-6d74e363-32a7-7140-f5d0-2908e65ca9ed: failed to get log: index=3 error="log not found" -2025-01-28T14:48:32.098Z [INFO] server-6d74e363-32a7-7140-f5d0-2908e65ca9ed: aborting pipeline replication: peer="{Voter server-be78217d-49d1-f754-b626-ffbbdcdb9017 be78217d-49d1-f754-b626-ffbbdcdb9017}" -2025-01-28T14:48:32.107Z [ERROR] server-6d74e363-32a7-7140-f5d0-2908e65ca9ed: failed to get log: index=3 error="log not found" -2025-01-28T14:48:32.109Z [INFO] creating new snapshot: path=/tmp/raft75711000/snapshots/2-10013-1738075712109.tmp -2025-01-28T14:48:32.110Z [ERROR] server-6d74e363-32a7-7140-f5d0-2908e65ca9ed: failed to get log: index=3 error="log not found" -2025-01-28T14:48:32.112Z [INFO] server-be78217d-49d1-f754-b626-ffbbdcdb9017: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:32.113Z [INFO] creating new snapshot: path=/tmp/raft74456927/snapshots/2-10013-1738075712113.tmp -2025-01-28T14:48:32.114Z [INFO] server-be78217d-49d1-f754-b626-ffbbdcdb9017: copied to local snapshot: bytes=71 -2025-01-28T14:48:32.115Z [INFO] server-a007de91-f4df-05bf-e6be-220d9180296c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:32.116Z [INFO] server-be78217d-49d1-f754-b626-ffbbdcdb9017: snapshot restore progress: id=2-10013-1738075712109 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:32.116Z [INFO] server-be78217d-49d1-f754-b626-ffbbdcdb9017: Installed remote snapshot -2025-01-28T14:48:32.117Z [INFO] server-6d74e363-32a7-7140-f5d0-2908e65ca9ed: pipelining replication: peer="{Voter server-be78217d-49d1-f754-b626-ffbbdcdb9017 be78217d-49d1-f754-b626-ffbbdcdb9017}" -2025-01-28T14:48:32.117Z [INFO] server-a007de91-f4df-05bf-e6be-220d9180296c: copied to local snapshot: bytes=71 -2025-01-28T14:48:32.118Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:48:32.118Z [INFO] server-a007de91-f4df-05bf-e6be-220d9180296c: snapshot restore progress: id=2-10013-1738075712113 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:32.119Z [INFO] server-a007de91-f4df-05bf-e6be-220d9180296c: Installed remote snapshot -2025-01-28T14:48:32.119Z [INFO] server-6d74e363-32a7-7140-f5d0-2908e65ca9ed: pipelining replication: peer="{Voter server-a007de91-f4df-05bf-e6be-220d9180296c a007de91-f4df-05bf-e6be-220d9180296c}" -2025-01-28T14:48:33.123Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:48:32.118172037 +0000 UTC m=+95.201577062 (2 nodes), 7.334µs from start of poll, 1.586623118s from cluster start. Timeout at 2025-01-28 14:48:33.123543382 +0000 UTC m=+96.206948406, 1.005371344s after stability -2025-01-28T14:48:33.139Z [INFO] server-d367fa51-6540-caf5-9e58-22323dcb89ff: aborting pipeline replication: peer="{Voter server-db5e5232-35e9-1d78-0360-2dc309460a53 db5e5232-35e9-1d78-0360-2dc309460a53}" -2025-01-28T14:48:33.140Z [INFO] server-d367fa51-6540-caf5-9e58-22323dcb89ff: aborting pipeline replication: peer="{Voter server-0682e837-5360-7fa3-5476-d534c6de53c2 0682e837-5360-7fa3-5476-d534c6de53c2}" +2025-01-28T14:55:30.344Z [INFO] server-dd0ab195-ec18-fd17-33af-5cbc8ef3d20e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dd0ab195-ec18-fd17-33af-5cbc8ef3d20e Address:dd0ab195-ec18-fd17-33af-5cbc8ef3d20e} {Suffrage:Voter ID:server-72c1e8bd-8f17-13e4-0964-413c5ff8d25f Address:72c1e8bd-8f17-13e4-0964-413c5ff8d25f} {Suffrage:Voter ID:server-d01ad084-15fb-8aed-0d0f-655a24fbf32c Address:d01ad084-15fb-8aed-0d0f-655a24fbf32c}]" +2025-01-28T14:55:30.344Z [INFO] server-dd0ab195-ec18-fd17-33af-5cbc8ef3d20e: entering follower state: follower="Node at dd0ab195-ec18-fd17-33af-5cbc8ef3d20e [Follower]" leader-address= leader-id= +2025-01-28T14:55:30.344Z [INFO] server-72c1e8bd-8f17-13e4-0964-413c5ff8d25f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dd0ab195-ec18-fd17-33af-5cbc8ef3d20e Address:dd0ab195-ec18-fd17-33af-5cbc8ef3d20e} {Suffrage:Voter ID:server-72c1e8bd-8f17-13e4-0964-413c5ff8d25f Address:72c1e8bd-8f17-13e4-0964-413c5ff8d25f} {Suffrage:Voter ID:server-d01ad084-15fb-8aed-0d0f-655a24fbf32c Address:d01ad084-15fb-8aed-0d0f-655a24fbf32c}]" +2025-01-28T14:55:30.345Z [INFO] server-72c1e8bd-8f17-13e4-0964-413c5ff8d25f: entering follower state: follower="Node at 72c1e8bd-8f17-13e4-0964-413c5ff8d25f [Follower]" leader-address= leader-id= +2025-01-28T14:55:30.345Z [INFO] server-d01ad084-15fb-8aed-0d0f-655a24fbf32c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dd0ab195-ec18-fd17-33af-5cbc8ef3d20e Address:dd0ab195-ec18-fd17-33af-5cbc8ef3d20e} {Suffrage:Voter ID:server-72c1e8bd-8f17-13e4-0964-413c5ff8d25f Address:72c1e8bd-8f17-13e4-0964-413c5ff8d25f} {Suffrage:Voter ID:server-d01ad084-15fb-8aed-0d0f-655a24fbf32c Address:d01ad084-15fb-8aed-0d0f-655a24fbf32c}]" +2025-01-28T14:55:30.345Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:30.345Z [INFO] server-d01ad084-15fb-8aed-0d0f-655a24fbf32c: entering follower state: follower="Node at d01ad084-15fb-8aed-0d0f-655a24fbf32c [Follower]" leader-address= leader-id= +2025-01-28T14:55:30.891Z [WARN] server-72c1e8bd-8f17-13e4-0964-413c5ff8d25f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:30.891Z [INFO] server-72c1e8bd-8f17-13e4-0964-413c5ff8d25f: entering candidate state: node="Node at 72c1e8bd-8f17-13e4-0964-413c5ff8d25f [Candidate]" term=2 +2025-01-28T14:55:30.891Z [INFO] server-72c1e8bd-8f17-13e4-0964-413c5ff8d25f: election won: term=2 tally=2 +2025-01-28T14:55:30.891Z [INFO] server-72c1e8bd-8f17-13e4-0964-413c5ff8d25f: entering leader state: leader="Node at 72c1e8bd-8f17-13e4-0964-413c5ff8d25f [Leader]" +2025-01-28T14:55:30.891Z [INFO] server-72c1e8bd-8f17-13e4-0964-413c5ff8d25f: added peer, starting replication: peer=server-dd0ab195-ec18-fd17-33af-5cbc8ef3d20e +2025-01-28T14:55:30.891Z [INFO] server-72c1e8bd-8f17-13e4-0964-413c5ff8d25f: added peer, starting replication: peer=server-d01ad084-15fb-8aed-0d0f-655a24fbf32c +2025-01-28T14:55:30.892Z [INFO] server-72c1e8bd-8f17-13e4-0964-413c5ff8d25f: pipelining replication: peer="{Voter server-d01ad084-15fb-8aed-0d0f-655a24fbf32c d01ad084-15fb-8aed-0d0f-655a24fbf32c}" +2025-01-28T14:55:30.892Z [INFO] server-72c1e8bd-8f17-13e4-0964-413c5ff8d25f: pipelining replication: peer="{Voter server-dd0ab195-ec18-fd17-33af-5cbc8ef3d20e dd0ab195-ec18-fd17-33af-5cbc8ef3d20e}" +2025-01-28T14:55:31.896Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:30.891697897 +0000 UTC m=+116.566187877 (1 nodes), 546.410093ms from start of poll, 547.405159ms from cluster start. Timeout at 2025-01-28 14:55:31.896701017 +0000 UTC m=+117.571191017, 1.00500314s after stability +2025-01-28T14:55:31.898Z [INFO] server-72c1e8bd-8f17-13e4-0964-413c5ff8d25f: starting snapshot up to: index=12 +2025-01-28T14:55:31.898Z [INFO] creating new snapshot: path=/tmp/raft146499436/snapshots/2-12-1738076131898.tmp +2025-01-28T14:55:31.900Z [INFO] server-72c1e8bd-8f17-13e4-0964-413c5ff8d25f: snapshot complete up to: index=12 +2025-01-28T14:55:31.903Z [INFO] server-d9eb3999-75b8-bfe3-8539-cfc480e4fb0f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d9eb3999-75b8-bfe3-8539-cfc480e4fb0f Address:d9eb3999-75b8-bfe3-8539-cfc480e4fb0f} {Suffrage:Voter ID:server-b451307e-bf96-5198-3f54-a03762055e2c Address:b451307e-bf96-5198-3f54-a03762055e2c} {Suffrage:Voter ID:server-808e21be-0bb8-9a48-5555-3c8e9f1ac813 Address:808e21be-0bb8-9a48-5555-3c8e9f1ac813}]" +2025-01-28T14:55:31.904Z [INFO] server-b451307e-bf96-5198-3f54-a03762055e2c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d9eb3999-75b8-bfe3-8539-cfc480e4fb0f Address:d9eb3999-75b8-bfe3-8539-cfc480e4fb0f} {Suffrage:Voter ID:server-b451307e-bf96-5198-3f54-a03762055e2c Address:b451307e-bf96-5198-3f54-a03762055e2c} {Suffrage:Voter ID:server-808e21be-0bb8-9a48-5555-3c8e9f1ac813 Address:808e21be-0bb8-9a48-5555-3c8e9f1ac813}]" +2025-01-28T14:55:31.904Z [INFO] server-d9eb3999-75b8-bfe3-8539-cfc480e4fb0f: entering follower state: follower="Node at d9eb3999-75b8-bfe3-8539-cfc480e4fb0f [Follower]" leader-address= leader-id= +2025-01-28T14:55:31.904Z [INFO] server-b451307e-bf96-5198-3f54-a03762055e2c: entering follower state: follower="Node at b451307e-bf96-5198-3f54-a03762055e2c [Follower]" leader-address= leader-id= +2025-01-28T14:55:31.904Z [INFO] server-808e21be-0bb8-9a48-5555-3c8e9f1ac813: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d9eb3999-75b8-bfe3-8539-cfc480e4fb0f Address:d9eb3999-75b8-bfe3-8539-cfc480e4fb0f} {Suffrage:Voter ID:server-b451307e-bf96-5198-3f54-a03762055e2c Address:b451307e-bf96-5198-3f54-a03762055e2c} {Suffrage:Voter ID:server-808e21be-0bb8-9a48-5555-3c8e9f1ac813 Address:808e21be-0bb8-9a48-5555-3c8e9f1ac813}]" +2025-01-28T14:55:31.904Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:31.904Z [INFO] server-808e21be-0bb8-9a48-5555-3c8e9f1ac813: entering follower state: follower="Node at 808e21be-0bb8-9a48-5555-3c8e9f1ac813 [Follower]" leader-address= leader-id= +2025-01-28T14:55:32.570Z [WARN] server-d9eb3999-75b8-bfe3-8539-cfc480e4fb0f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:32.570Z [INFO] server-d9eb3999-75b8-bfe3-8539-cfc480e4fb0f: entering candidate state: node="Node at d9eb3999-75b8-bfe3-8539-cfc480e4fb0f [Candidate]" term=2 +2025-01-28T14:55:32.570Z [INFO] server-d9eb3999-75b8-bfe3-8539-cfc480e4fb0f: election won: term=2 tally=2 +2025-01-28T14:55:32.570Z [INFO] server-d9eb3999-75b8-bfe3-8539-cfc480e4fb0f: entering leader state: leader="Node at d9eb3999-75b8-bfe3-8539-cfc480e4fb0f [Leader]" +2025-01-28T14:55:32.571Z [INFO] server-d9eb3999-75b8-bfe3-8539-cfc480e4fb0f: added peer, starting replication: peer=server-b451307e-bf96-5198-3f54-a03762055e2c +2025-01-28T14:55:32.571Z [INFO] server-d9eb3999-75b8-bfe3-8539-cfc480e4fb0f: added peer, starting replication: peer=server-808e21be-0bb8-9a48-5555-3c8e9f1ac813 +2025-01-28T14:55:32.571Z [INFO] server-d9eb3999-75b8-bfe3-8539-cfc480e4fb0f: pipelining replication: peer="{Voter server-b451307e-bf96-5198-3f54-a03762055e2c b451307e-bf96-5198-3f54-a03762055e2c}" +2025-01-28T14:55:32.571Z [INFO] server-d9eb3999-75b8-bfe3-8539-cfc480e4fb0f: pipelining replication: peer="{Voter server-808e21be-0bb8-9a48-5555-3c8e9f1ac813 808e21be-0bb8-9a48-5555-3c8e9f1ac813}" +2025-01-28T14:55:33.576Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:32.571033555 +0000 UTC m=+118.245523535 (1 nodes), 666.359438ms from start of poll, 667.39854ms from cluster start. Timeout at 2025-01-28 14:55:33.576040335 +0000 UTC m=+119.250530415, 1.00500688s after stability +2025-01-28T14:55:33.576Z [INFO] creating new snapshot: path=/tmp/raft4183933472/snapshots/2-10013-1738076133576.tmp +2025-01-28T14:55:33.578Z [INFO] server-d9eb3999-75b8-bfe3-8539-cfc480e4fb0f: copied to local snapshot: bytes=71 +2025-01-28T14:55:33.579Z [INFO] server-d9eb3999-75b8-bfe3-8539-cfc480e4fb0f: snapshot restore progress: id=2-10013-1738076133576 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:33.579Z [INFO] server-d9eb3999-75b8-bfe3-8539-cfc480e4fb0f: restored user snapshot: index=10013 +2025-01-28T14:55:33.579Z [ERROR] server-d9eb3999-75b8-bfe3-8539-cfc480e4fb0f: failed to get log: index=3 error="log not found" +2025-01-28T14:55:33.579Z [INFO] server-d9eb3999-75b8-bfe3-8539-cfc480e4fb0f: aborting pipeline replication: peer="{Voter server-808e21be-0bb8-9a48-5555-3c8e9f1ac813 808e21be-0bb8-9a48-5555-3c8e9f1ac813}" +2025-01-28T14:55:33.579Z [ERROR] server-d9eb3999-75b8-bfe3-8539-cfc480e4fb0f: failed to get log: index=3 error="log not found" +2025-01-28T14:55:33.580Z [INFO] server-d9eb3999-75b8-bfe3-8539-cfc480e4fb0f: aborting pipeline replication: peer="{Voter server-b451307e-bf96-5198-3f54-a03762055e2c b451307e-bf96-5198-3f54-a03762055e2c}" +2025-01-28T14:55:33.589Z [ERROR] server-d9eb3999-75b8-bfe3-8539-cfc480e4fb0f: failed to get log: index=3 error="log not found" +2025-01-28T14:55:33.590Z [ERROR] server-d9eb3999-75b8-bfe3-8539-cfc480e4fb0f: failed to get log: index=3 error="log not found" +2025-01-28T14:55:33.590Z [INFO] creating new snapshot: path=/tmp/raft2639486270/snapshots/2-10013-1738076133590.tmp +2025-01-28T14:55:33.591Z [INFO] creating new snapshot: path=/tmp/raft3354896893/snapshots/2-10013-1738076133591.tmp +2025-01-28T14:55:33.591Z [INFO] server-b451307e-bf96-5198-3f54-a03762055e2c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:33.592Z [INFO] server-808e21be-0bb8-9a48-5555-3c8e9f1ac813: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:33.592Z [INFO] server-b451307e-bf96-5198-3f54-a03762055e2c: copied to local snapshot: bytes=71 +2025-01-28T14:55:33.592Z [INFO] server-808e21be-0bb8-9a48-5555-3c8e9f1ac813: copied to local snapshot: bytes=71 +2025-01-28T14:55:33.593Z [INFO] server-b451307e-bf96-5198-3f54-a03762055e2c: snapshot restore progress: id=2-10013-1738076133590 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:33.593Z [INFO] server-b451307e-bf96-5198-3f54-a03762055e2c: Installed remote snapshot +2025-01-28T14:55:33.593Z [INFO] server-808e21be-0bb8-9a48-5555-3c8e9f1ac813: snapshot restore progress: id=2-10013-1738076133591 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:33.593Z [WARN] server-b451307e-bf96-5198-3f54-a03762055e2c: skipping application of old log: index=10013 +2025-01-28T14:55:33.593Z [INFO] server-808e21be-0bb8-9a48-5555-3c8e9f1ac813: Installed remote snapshot +2025-01-28T14:55:33.593Z [INFO] server-d9eb3999-75b8-bfe3-8539-cfc480e4fb0f: pipelining replication: peer="{Voter server-b451307e-bf96-5198-3f54-a03762055e2c b451307e-bf96-5198-3f54-a03762055e2c}" +2025-01-28T14:55:33.593Z [WARN] server-808e21be-0bb8-9a48-5555-3c8e9f1ac813: skipping application of old log: index=10013 +2025-01-28T14:55:33.593Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:55:33.593Z [INFO] server-d9eb3999-75b8-bfe3-8539-cfc480e4fb0f: pipelining replication: peer="{Voter server-808e21be-0bb8-9a48-5555-3c8e9f1ac813 808e21be-0bb8-9a48-5555-3c8e9f1ac813}" +2025-01-28T14:55:34.599Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:55:33.593945718 +0000 UTC m=+119.268435698 (2 nodes), 2.86µs from start of poll, 1.690310703s from cluster start. Timeout at 2025-01-28 14:55:34.599189875 +0000 UTC m=+120.273679855, 1.005244157s after stability +2025-01-28T14:55:34.612Z [INFO] server-72c1e8bd-8f17-13e4-0964-413c5ff8d25f: aborting pipeline replication: peer="{Voter server-dd0ab195-ec18-fd17-33af-5cbc8ef3d20e dd0ab195-ec18-fd17-33af-5cbc8ef3d20e}" +2025-01-28T14:55:34.612Z [INFO] server-72c1e8bd-8f17-13e4-0964-413c5ff8d25f: aborting pipeline replication: peer="{Voter server-d01ad084-15fb-8aed-0d0f-655a24fbf32c d01ad084-15fb-8aed-0d0f-655a24fbf32c}" === RUN TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_true -2025-01-28T14:48:33.151Z [INFO] server-bc82fe28-6aeb-0a59-7442-be916a5753a8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bc82fe28-6aeb-0a59-7442-be916a5753a8 Address:bc82fe28-6aeb-0a59-7442-be916a5753a8} {Suffrage:Voter ID:server-77fc1848-4a8d-31ff-3deb-7368d978c495 Address:77fc1848-4a8d-31ff-3deb-7368d978c495} {Suffrage:Voter ID:server-6687842b-c9af-9218-8782-c176f6d73242 Address:6687842b-c9af-9218-8782-c176f6d73242}]" -2025-01-28T14:48:33.152Z [INFO] server-bc82fe28-6aeb-0a59-7442-be916a5753a8: entering follower state: follower="Node at bc82fe28-6aeb-0a59-7442-be916a5753a8 [Follower]" leader-address= leader-id= -2025-01-28T14:48:33.152Z [INFO] server-77fc1848-4a8d-31ff-3deb-7368d978c495: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bc82fe28-6aeb-0a59-7442-be916a5753a8 Address:bc82fe28-6aeb-0a59-7442-be916a5753a8} {Suffrage:Voter ID:server-77fc1848-4a8d-31ff-3deb-7368d978c495 Address:77fc1848-4a8d-31ff-3deb-7368d978c495} {Suffrage:Voter ID:server-6687842b-c9af-9218-8782-c176f6d73242 Address:6687842b-c9af-9218-8782-c176f6d73242}]" -2025-01-28T14:48:33.153Z [INFO] server-77fc1848-4a8d-31ff-3deb-7368d978c495: entering follower state: follower="Node at 77fc1848-4a8d-31ff-3deb-7368d978c495 [Follower]" leader-address= leader-id= -2025-01-28T14:48:33.153Z [INFO] server-6687842b-c9af-9218-8782-c176f6d73242: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bc82fe28-6aeb-0a59-7442-be916a5753a8 Address:bc82fe28-6aeb-0a59-7442-be916a5753a8} {Suffrage:Voter ID:server-77fc1848-4a8d-31ff-3deb-7368d978c495 Address:77fc1848-4a8d-31ff-3deb-7368d978c495} {Suffrage:Voter ID:server-6687842b-c9af-9218-8782-c176f6d73242 Address:6687842b-c9af-9218-8782-c176f6d73242}]" -2025-01-28T14:48:33.153Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:33.153Z [INFO] server-6687842b-c9af-9218-8782-c176f6d73242: entering follower state: follower="Node at 6687842b-c9af-9218-8782-c176f6d73242 [Follower]" leader-address= leader-id= -2025-01-28T14:48:33.856Z [WARN] server-bc82fe28-6aeb-0a59-7442-be916a5753a8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:33.856Z [INFO] server-bc82fe28-6aeb-0a59-7442-be916a5753a8: entering candidate state: node="Node at bc82fe28-6aeb-0a59-7442-be916a5753a8 [Candidate]" term=2 -2025-01-28T14:48:33.857Z [INFO] server-bc82fe28-6aeb-0a59-7442-be916a5753a8: election won: term=2 tally=2 -2025-01-28T14:48:33.857Z [INFO] server-bc82fe28-6aeb-0a59-7442-be916a5753a8: entering leader state: leader="Node at bc82fe28-6aeb-0a59-7442-be916a5753a8 [Leader]" -2025-01-28T14:48:33.857Z [INFO] server-bc82fe28-6aeb-0a59-7442-be916a5753a8: added peer, starting replication: peer=server-77fc1848-4a8d-31ff-3deb-7368d978c495 -2025-01-28T14:48:33.857Z [INFO] server-bc82fe28-6aeb-0a59-7442-be916a5753a8: added peer, starting replication: peer=server-6687842b-c9af-9218-8782-c176f6d73242 -2025-01-28T14:48:33.858Z [INFO] server-bc82fe28-6aeb-0a59-7442-be916a5753a8: pipelining replication: peer="{Voter server-6687842b-c9af-9218-8782-c176f6d73242 6687842b-c9af-9218-8782-c176f6d73242}" -2025-01-28T14:48:33.858Z [INFO] server-bc82fe28-6aeb-0a59-7442-be916a5753a8: pipelining replication: peer="{Voter server-77fc1848-4a8d-31ff-3deb-7368d978c495 77fc1848-4a8d-31ff-3deb-7368d978c495}" -2025-01-28T14:48:34.863Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:33.857351685 +0000 UTC m=+96.940756710 (1 nodes), 703.526893ms from start of poll, 706.008967ms from cluster start. Timeout at 2025-01-28 14:48:34.862363352 +0000 UTC m=+97.945768043, 1.005011333s after stability -2025-01-28T14:48:34.886Z [INFO] server-bc82fe28-6aeb-0a59-7442-be916a5753a8: starting snapshot up to: index=12 -2025-01-28T14:48:34.886Z [INFO] creating new snapshot: path=/tmp/raft3590269609/snapshots/2-12-1738075714886.tmp -2025-01-28T14:48:34.890Z [INFO] server-bc82fe28-6aeb-0a59-7442-be916a5753a8: snapshot complete up to: index=12 -2025-01-28T14:48:34.899Z [INFO] server-7cb18d25-4f78-51d1-3966-08f87c6cf016: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7cb18d25-4f78-51d1-3966-08f87c6cf016 Address:7cb18d25-4f78-51d1-3966-08f87c6cf016} {Suffrage:Voter ID:server-fe60e4bd-553c-0f14-8d47-afd82330b409 Address:fe60e4bd-553c-0f14-8d47-afd82330b409} {Suffrage:Voter ID:server-9e6b65ef-a63b-e514-909a-c8207b4711d6 Address:9e6b65ef-a63b-e514-909a-c8207b4711d6}]" -2025-01-28T14:48:34.900Z [INFO] server-7cb18d25-4f78-51d1-3966-08f87c6cf016: entering follower state: follower="Node at 7cb18d25-4f78-51d1-3966-08f87c6cf016 [Follower]" leader-address= leader-id= -2025-01-28T14:48:34.900Z [INFO] server-fe60e4bd-553c-0f14-8d47-afd82330b409: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7cb18d25-4f78-51d1-3966-08f87c6cf016 Address:7cb18d25-4f78-51d1-3966-08f87c6cf016} {Suffrage:Voter ID:server-fe60e4bd-553c-0f14-8d47-afd82330b409 Address:fe60e4bd-553c-0f14-8d47-afd82330b409} {Suffrage:Voter ID:server-9e6b65ef-a63b-e514-909a-c8207b4711d6 Address:9e6b65ef-a63b-e514-909a-c8207b4711d6}]" -2025-01-28T14:48:34.901Z [INFO] server-fe60e4bd-553c-0f14-8d47-afd82330b409: entering follower state: follower="Node at fe60e4bd-553c-0f14-8d47-afd82330b409 [Follower]" leader-address= leader-id= -2025-01-28T14:48:34.901Z [INFO] server-9e6b65ef-a63b-e514-909a-c8207b4711d6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7cb18d25-4f78-51d1-3966-08f87c6cf016 Address:7cb18d25-4f78-51d1-3966-08f87c6cf016} {Suffrage:Voter ID:server-fe60e4bd-553c-0f14-8d47-afd82330b409 Address:fe60e4bd-553c-0f14-8d47-afd82330b409} {Suffrage:Voter ID:server-9e6b65ef-a63b-e514-909a-c8207b4711d6 Address:9e6b65ef-a63b-e514-909a-c8207b4711d6}]" -2025-01-28T14:48:34.901Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:34.901Z [INFO] server-9e6b65ef-a63b-e514-909a-c8207b4711d6: entering follower state: follower="Node at 9e6b65ef-a63b-e514-909a-c8207b4711d6 [Follower]" leader-address= leader-id= -2025-01-28T14:48:35.461Z [WARN] server-7cb18d25-4f78-51d1-3966-08f87c6cf016: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:35.462Z [INFO] server-7cb18d25-4f78-51d1-3966-08f87c6cf016: entering candidate state: node="Node at 7cb18d25-4f78-51d1-3966-08f87c6cf016 [Candidate]" term=2 -2025-01-28T14:48:35.462Z [INFO] server-7cb18d25-4f78-51d1-3966-08f87c6cf016: election won: term=2 tally=2 -2025-01-28T14:48:35.462Z [INFO] server-7cb18d25-4f78-51d1-3966-08f87c6cf016: entering leader state: leader="Node at 7cb18d25-4f78-51d1-3966-08f87c6cf016 [Leader]" -2025-01-28T14:48:35.463Z [INFO] server-7cb18d25-4f78-51d1-3966-08f87c6cf016: added peer, starting replication: peer=server-fe60e4bd-553c-0f14-8d47-afd82330b409 -2025-01-28T14:48:35.463Z [INFO] server-7cb18d25-4f78-51d1-3966-08f87c6cf016: added peer, starting replication: peer=server-9e6b65ef-a63b-e514-909a-c8207b4711d6 -2025-01-28T14:48:35.464Z [INFO] server-7cb18d25-4f78-51d1-3966-08f87c6cf016: pipelining replication: peer="{Voter server-9e6b65ef-a63b-e514-909a-c8207b4711d6 9e6b65ef-a63b-e514-909a-c8207b4711d6}" -2025-01-28T14:48:35.464Z [INFO] server-7cb18d25-4f78-51d1-3966-08f87c6cf016: pipelining replication: peer="{Voter server-fe60e4bd-553c-0f14-8d47-afd82330b409 fe60e4bd-553c-0f14-8d47-afd82330b409}" -2025-01-28T14:48:36.470Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:35.464354742 +0000 UTC m=+98.547759767 (1 nodes), 562.698044ms from start of poll, 565.01978ms from cluster start. Timeout at 2025-01-28 14:48:36.469400078 +0000 UTC m=+99.552804769, 1.005045002s after stability -2025-01-28T14:48:36.485Z [INFO] creating new snapshot: path=/tmp/raft2220644668/snapshots/2-10013-1738075716485.tmp -2025-01-28T14:48:36.489Z [INFO] server-7cb18d25-4f78-51d1-3966-08f87c6cf016: copied to local snapshot: bytes=71 -2025-01-28T14:48:36.491Z [INFO] server-7cb18d25-4f78-51d1-3966-08f87c6cf016: snapshot restore progress: id=2-10013-1738075716485 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:36.491Z [INFO] server-7cb18d25-4f78-51d1-3966-08f87c6cf016: removing all old logs from log store -2025-01-28T14:48:36.491Z [INFO] server-7cb18d25-4f78-51d1-3966-08f87c6cf016: compacting logs: from=1 to=2 -2025-01-28T14:48:36.491Z [INFO] server-7cb18d25-4f78-51d1-3966-08f87c6cf016: restored user snapshot: index=10013 -2025-01-28T14:48:36.491Z [ERROR] server-7cb18d25-4f78-51d1-3966-08f87c6cf016: failed to get log: index=3 error="log not found" -2025-01-28T14:48:36.491Z [INFO] server-7cb18d25-4f78-51d1-3966-08f87c6cf016: aborting pipeline replication: peer="{Voter server-fe60e4bd-553c-0f14-8d47-afd82330b409 fe60e4bd-553c-0f14-8d47-afd82330b409}" -2025-01-28T14:48:36.492Z [ERROR] server-7cb18d25-4f78-51d1-3966-08f87c6cf016: failed to get log: index=2 error="log not found" -2025-01-28T14:48:36.492Z [ERROR] server-7cb18d25-4f78-51d1-3966-08f87c6cf016: failed to get log: index=2 error="log not found" -2025-01-28T14:48:36.492Z [INFO] server-7cb18d25-4f78-51d1-3966-08f87c6cf016: aborting pipeline replication: peer="{Voter server-9e6b65ef-a63b-e514-909a-c8207b4711d6 9e6b65ef-a63b-e514-909a-c8207b4711d6}" -2025-01-28T14:48:36.494Z [INFO] creating new snapshot: path=/tmp/raft1505187362/snapshots/2-10013-1738075716494.tmp -2025-01-28T14:48:36.495Z [INFO] server-fe60e4bd-553c-0f14-8d47-afd82330b409: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:36.498Z [INFO] server-fe60e4bd-553c-0f14-8d47-afd82330b409: copied to local snapshot: bytes=71 -2025-01-28T14:48:36.500Z [INFO] server-fe60e4bd-553c-0f14-8d47-afd82330b409: snapshot restore progress: id=2-10013-1738075716494 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:36.500Z [INFO] server-fe60e4bd-553c-0f14-8d47-afd82330b409: removing all old logs from log store -2025-01-28T14:48:36.500Z [INFO] server-fe60e4bd-553c-0f14-8d47-afd82330b409: compacting logs: from=1 to=2 -2025-01-28T14:48:36.500Z [INFO] server-fe60e4bd-553c-0f14-8d47-afd82330b409: Installed remote snapshot -2025-01-28T14:48:36.500Z [ERROR] server-7cb18d25-4f78-51d1-3966-08f87c6cf016: failed to get log: index=2 error="log not found" -2025-01-28T14:48:36.501Z [INFO] server-7cb18d25-4f78-51d1-3966-08f87c6cf016: pipelining replication: peer="{Voter server-fe60e4bd-553c-0f14-8d47-afd82330b409 fe60e4bd-553c-0f14-8d47-afd82330b409}" -2025-01-28T14:48:36.502Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:48:36.503Z [INFO] creating new snapshot: path=/tmp/raft1641485788/snapshots/2-10013-1738075716503.tmp -2025-01-28T14:48:36.505Z [INFO] server-9e6b65ef-a63b-e514-909a-c8207b4711d6: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:36.508Z [INFO] server-9e6b65ef-a63b-e514-909a-c8207b4711d6: copied to local snapshot: bytes=71 -2025-01-28T14:48:36.509Z [INFO] server-9e6b65ef-a63b-e514-909a-c8207b4711d6: snapshot restore progress: id=2-10013-1738075716503 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-01-28T14:48:36.509Z [INFO] server-9e6b65ef-a63b-e514-909a-c8207b4711d6: removing all old logs from log store -2025-01-28T14:48:36.509Z [INFO] server-9e6b65ef-a63b-e514-909a-c8207b4711d6: compacting logs: from=1 to=2 -2025-01-28T14:48:36.509Z [INFO] server-9e6b65ef-a63b-e514-909a-c8207b4711d6: Installed remote snapshot -2025-01-28T14:48:36.510Z [INFO] server-7cb18d25-4f78-51d1-3966-08f87c6cf016: pipelining replication: peer="{Voter server-9e6b65ef-a63b-e514-909a-c8207b4711d6 9e6b65ef-a63b-e514-909a-c8207b4711d6}" -2025-01-28T14:48:37.508Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:48:36.502789581 +0000 UTC m=+99.586194272 (2 nodes), 9µs from start of poll, 1.603454285s from cluster start. Timeout at 2025-01-28 14:48:37.508242261 +0000 UTC m=+100.591647286, 1.005453014s after stability -2025-01-28T14:48:37.523Z [INFO] server-bc82fe28-6aeb-0a59-7442-be916a5753a8: aborting pipeline replication: peer="{Voter server-77fc1848-4a8d-31ff-3deb-7368d978c495 77fc1848-4a8d-31ff-3deb-7368d978c495}" -2025-01-28T14:48:37.523Z [INFO] server-bc82fe28-6aeb-0a59-7442-be916a5753a8: aborting pipeline replication: peer="{Voter server-6687842b-c9af-9218-8782-c176f6d73242 6687842b-c9af-9218-8782-c176f6d73242}" ---- PASS: TestRaft_UserRestore (84.10s) - --- PASS: TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_false (2.66s) - --- PASS: TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_false (2.71s) - --- PASS: TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_true (4.39s) - --- PASS: TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_true (4.25s) - --- PASS: TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_false (2.85s) - --- PASS: TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_false (2.70s) +2025-01-28T14:55:34.621Z [INFO] server-104efdcf-a081-9f52-7794-f9a4ae371d5d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-104efdcf-a081-9f52-7794-f9a4ae371d5d Address:104efdcf-a081-9f52-7794-f9a4ae371d5d} {Suffrage:Voter ID:server-373cd188-51fc-85d5-97b3-492821fe0623 Address:373cd188-51fc-85d5-97b3-492821fe0623} {Suffrage:Voter ID:server-d4c1da89-f157-768b-7e09-66e59e01dfa5 Address:d4c1da89-f157-768b-7e09-66e59e01dfa5}]" +2025-01-28T14:55:34.621Z [INFO] server-104efdcf-a081-9f52-7794-f9a4ae371d5d: entering follower state: follower="Node at 104efdcf-a081-9f52-7794-f9a4ae371d5d [Follower]" leader-address= leader-id= +2025-01-28T14:55:34.621Z [INFO] server-373cd188-51fc-85d5-97b3-492821fe0623: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-104efdcf-a081-9f52-7794-f9a4ae371d5d Address:104efdcf-a081-9f52-7794-f9a4ae371d5d} {Suffrage:Voter ID:server-373cd188-51fc-85d5-97b3-492821fe0623 Address:373cd188-51fc-85d5-97b3-492821fe0623} {Suffrage:Voter ID:server-d4c1da89-f157-768b-7e09-66e59e01dfa5 Address:d4c1da89-f157-768b-7e09-66e59e01dfa5}]" +2025-01-28T14:55:34.621Z [INFO] server-373cd188-51fc-85d5-97b3-492821fe0623: entering follower state: follower="Node at 373cd188-51fc-85d5-97b3-492821fe0623 [Follower]" leader-address= leader-id= +2025-01-28T14:55:34.621Z [INFO] server-d4c1da89-f157-768b-7e09-66e59e01dfa5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-104efdcf-a081-9f52-7794-f9a4ae371d5d Address:104efdcf-a081-9f52-7794-f9a4ae371d5d} {Suffrage:Voter ID:server-373cd188-51fc-85d5-97b3-492821fe0623 Address:373cd188-51fc-85d5-97b3-492821fe0623} {Suffrage:Voter ID:server-d4c1da89-f157-768b-7e09-66e59e01dfa5 Address:d4c1da89-f157-768b-7e09-66e59e01dfa5}]" +2025-01-28T14:55:34.621Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:34.621Z [INFO] server-d4c1da89-f157-768b-7e09-66e59e01dfa5: entering follower state: follower="Node at d4c1da89-f157-768b-7e09-66e59e01dfa5 [Follower]" leader-address= leader-id= +2025-01-28T14:55:35.135Z [WARN] server-d4c1da89-f157-768b-7e09-66e59e01dfa5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:35.135Z [INFO] server-d4c1da89-f157-768b-7e09-66e59e01dfa5: entering candidate state: node="Node at d4c1da89-f157-768b-7e09-66e59e01dfa5 [Candidate]" term=2 +2025-01-28T14:55:35.135Z [INFO] server-d4c1da89-f157-768b-7e09-66e59e01dfa5: election won: term=2 tally=2 +2025-01-28T14:55:35.135Z [INFO] server-d4c1da89-f157-768b-7e09-66e59e01dfa5: entering leader state: leader="Node at d4c1da89-f157-768b-7e09-66e59e01dfa5 [Leader]" +2025-01-28T14:55:35.135Z [INFO] server-d4c1da89-f157-768b-7e09-66e59e01dfa5: added peer, starting replication: peer=server-104efdcf-a081-9f52-7794-f9a4ae371d5d +2025-01-28T14:55:35.135Z [INFO] server-d4c1da89-f157-768b-7e09-66e59e01dfa5: added peer, starting replication: peer=server-373cd188-51fc-85d5-97b3-492821fe0623 +2025-01-28T14:55:35.136Z [INFO] server-d4c1da89-f157-768b-7e09-66e59e01dfa5: pipelining replication: peer="{Voter server-373cd188-51fc-85d5-97b3-492821fe0623 373cd188-51fc-85d5-97b3-492821fe0623}" +2025-01-28T14:55:35.136Z [INFO] server-d4c1da89-f157-768b-7e09-66e59e01dfa5: pipelining replication: peer="{Voter server-104efdcf-a081-9f52-7794-f9a4ae371d5d 104efdcf-a081-9f52-7794-f9a4ae371d5d}" +2025-01-28T14:55:36.141Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:35.135727952 +0000 UTC m=+120.810217932 (1 nodes), 514.057766ms from start of poll, 514.951362ms from cluster start. Timeout at 2025-01-28 14:55:36.140751889 +0000 UTC m=+121.815241869, 1.005023937s after stability +2025-01-28T14:55:36.142Z [INFO] server-d4c1da89-f157-768b-7e09-66e59e01dfa5: starting snapshot up to: index=12 +2025-01-28T14:55:36.142Z [INFO] creating new snapshot: path=/tmp/raft291674726/snapshots/2-12-1738076136142.tmp +2025-01-28T14:55:36.144Z [INFO] server-d4c1da89-f157-768b-7e09-66e59e01dfa5: snapshot complete up to: index=12 +2025-01-28T14:55:36.147Z [INFO] server-b1db11d7-518d-da10-aff3-ca6b3ab71779: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b1db11d7-518d-da10-aff3-ca6b3ab71779 Address:b1db11d7-518d-da10-aff3-ca6b3ab71779} {Suffrage:Voter ID:server-adfb35ab-4c3a-5d71-ff65-714e5999c9af Address:adfb35ab-4c3a-5d71-ff65-714e5999c9af} {Suffrage:Voter ID:server-41ef8dd7-3fc5-2de4-d9d4-1acbe98d81dc Address:41ef8dd7-3fc5-2de4-d9d4-1acbe98d81dc}]" +2025-01-28T14:55:36.148Z [INFO] server-b1db11d7-518d-da10-aff3-ca6b3ab71779: entering follower state: follower="Node at b1db11d7-518d-da10-aff3-ca6b3ab71779 [Follower]" leader-address= leader-id= +2025-01-28T14:55:36.148Z [INFO] server-adfb35ab-4c3a-5d71-ff65-714e5999c9af: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b1db11d7-518d-da10-aff3-ca6b3ab71779 Address:b1db11d7-518d-da10-aff3-ca6b3ab71779} {Suffrage:Voter ID:server-adfb35ab-4c3a-5d71-ff65-714e5999c9af Address:adfb35ab-4c3a-5d71-ff65-714e5999c9af} {Suffrage:Voter ID:server-41ef8dd7-3fc5-2de4-d9d4-1acbe98d81dc Address:41ef8dd7-3fc5-2de4-d9d4-1acbe98d81dc}]" +2025-01-28T14:55:36.148Z [INFO] server-adfb35ab-4c3a-5d71-ff65-714e5999c9af: entering follower state: follower="Node at adfb35ab-4c3a-5d71-ff65-714e5999c9af [Follower]" leader-address= leader-id= +2025-01-28T14:55:36.148Z [INFO] server-41ef8dd7-3fc5-2de4-d9d4-1acbe98d81dc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b1db11d7-518d-da10-aff3-ca6b3ab71779 Address:b1db11d7-518d-da10-aff3-ca6b3ab71779} {Suffrage:Voter ID:server-adfb35ab-4c3a-5d71-ff65-714e5999c9af Address:adfb35ab-4c3a-5d71-ff65-714e5999c9af} {Suffrage:Voter ID:server-41ef8dd7-3fc5-2de4-d9d4-1acbe98d81dc Address:41ef8dd7-3fc5-2de4-d9d4-1acbe98d81dc}]" +2025-01-28T14:55:36.148Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:36.148Z [INFO] server-41ef8dd7-3fc5-2de4-d9d4-1acbe98d81dc: entering follower state: follower="Node at 41ef8dd7-3fc5-2de4-d9d4-1acbe98d81dc [Follower]" leader-address= leader-id= +2025-01-28T14:55:36.723Z [WARN] server-b1db11d7-518d-da10-aff3-ca6b3ab71779: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:36.724Z [INFO] server-b1db11d7-518d-da10-aff3-ca6b3ab71779: entering candidate state: node="Node at b1db11d7-518d-da10-aff3-ca6b3ab71779 [Candidate]" term=2 +2025-01-28T14:55:36.724Z [INFO] server-b1db11d7-518d-da10-aff3-ca6b3ab71779: election won: term=2 tally=2 +2025-01-28T14:55:36.724Z [INFO] server-b1db11d7-518d-da10-aff3-ca6b3ab71779: entering leader state: leader="Node at b1db11d7-518d-da10-aff3-ca6b3ab71779 [Leader]" +2025-01-28T14:55:36.724Z [INFO] server-b1db11d7-518d-da10-aff3-ca6b3ab71779: added peer, starting replication: peer=server-adfb35ab-4c3a-5d71-ff65-714e5999c9af +2025-01-28T14:55:36.724Z [INFO] server-b1db11d7-518d-da10-aff3-ca6b3ab71779: added peer, starting replication: peer=server-41ef8dd7-3fc5-2de4-d9d4-1acbe98d81dc +2025-01-28T14:55:36.724Z [INFO] server-b1db11d7-518d-da10-aff3-ca6b3ab71779: pipelining replication: peer="{Voter server-adfb35ab-4c3a-5d71-ff65-714e5999c9af adfb35ab-4c3a-5d71-ff65-714e5999c9af}" +2025-01-28T14:55:36.724Z [INFO] server-b1db11d7-518d-da10-aff3-ca6b3ab71779: pipelining replication: peer="{Voter server-41ef8dd7-3fc5-2de4-d9d4-1acbe98d81dc 41ef8dd7-3fc5-2de4-d9d4-1acbe98d81dc}" +2025-01-28T14:55:37.729Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:36.724406042 +0000 UTC m=+122.398896022 (1 nodes), 575.747586ms from start of poll, 576.949993ms from cluster start. Timeout at 2025-01-28 14:55:37.729418681 +0000 UTC m=+123.403908641, 1.005012619s after stability +2025-01-28T14:55:37.730Z [INFO] creating new snapshot: path=/tmp/raft2894792163/snapshots/2-10013-1738076137730.tmp +2025-01-28T14:55:37.731Z [INFO] server-b1db11d7-518d-da10-aff3-ca6b3ab71779: copied to local snapshot: bytes=71 +2025-01-28T14:55:37.732Z [INFO] server-b1db11d7-518d-da10-aff3-ca6b3ab71779: snapshot restore progress: id=2-10013-1738076137730 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:37.732Z [INFO] server-b1db11d7-518d-da10-aff3-ca6b3ab71779: removing all old logs from log store +2025-01-28T14:55:37.732Z [INFO] server-b1db11d7-518d-da10-aff3-ca6b3ab71779: compacting logs: from=1 to=2 +2025-01-28T14:55:37.732Z [INFO] server-b1db11d7-518d-da10-aff3-ca6b3ab71779: restored user snapshot: index=10013 +2025-01-28T14:55:37.732Z [ERROR] server-b1db11d7-518d-da10-aff3-ca6b3ab71779: failed to get log: index=2 error="log not found" +2025-01-28T14:55:37.733Z [ERROR] server-b1db11d7-518d-da10-aff3-ca6b3ab71779: failed to get log: index=2 error="log not found" +2025-01-28T14:55:37.733Z [INFO] server-b1db11d7-518d-da10-aff3-ca6b3ab71779: aborting pipeline replication: peer="{Voter server-adfb35ab-4c3a-5d71-ff65-714e5999c9af adfb35ab-4c3a-5d71-ff65-714e5999c9af}" +2025-01-28T14:55:37.733Z [INFO] server-b1db11d7-518d-da10-aff3-ca6b3ab71779: aborting pipeline replication: peer="{Voter server-41ef8dd7-3fc5-2de4-d9d4-1acbe98d81dc 41ef8dd7-3fc5-2de4-d9d4-1acbe98d81dc}" +2025-01-28T14:55:37.739Z [ERROR] server-b1db11d7-518d-da10-aff3-ca6b3ab71779: failed to get log: index=2 error="log not found" +2025-01-28T14:55:37.740Z [INFO] creating new snapshot: path=/tmp/raft531455212/snapshots/2-10013-1738076137740.tmp +2025-01-28T14:55:37.741Z [INFO] server-adfb35ab-4c3a-5d71-ff65-714e5999c9af: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:37.742Z [ERROR] server-b1db11d7-518d-da10-aff3-ca6b3ab71779: failed to get log: index=2 error="log not found" +2025-01-28T14:55:37.742Z [INFO] server-adfb35ab-4c3a-5d71-ff65-714e5999c9af: copied to local snapshot: bytes=71 +2025-01-28T14:55:37.743Z [INFO] creating new snapshot: path=/tmp/raft1391662181/snapshots/2-10013-1738076137743.tmp +2025-01-28T14:55:37.743Z [INFO] server-adfb35ab-4c3a-5d71-ff65-714e5999c9af: snapshot restore progress: id=2-10013-1738076137740 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:37.743Z [INFO] server-adfb35ab-4c3a-5d71-ff65-714e5999c9af: removing all old logs from log store +2025-01-28T14:55:37.743Z [INFO] server-adfb35ab-4c3a-5d71-ff65-714e5999c9af: compacting logs: from=1 to=2 +2025-01-28T14:55:37.743Z [INFO] server-adfb35ab-4c3a-5d71-ff65-714e5999c9af: Installed remote snapshot +2025-01-28T14:55:37.743Z [WARN] server-adfb35ab-4c3a-5d71-ff65-714e5999c9af: skipping application of old log: index=10013 +2025-01-28T14:55:37.743Z [INFO] server-41ef8dd7-3fc5-2de4-d9d4-1acbe98d81dc: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:37.743Z [INFO] server-b1db11d7-518d-da10-aff3-ca6b3ab71779: pipelining replication: peer="{Voter server-adfb35ab-4c3a-5d71-ff65-714e5999c9af adfb35ab-4c3a-5d71-ff65-714e5999c9af}" +2025-01-28T14:55:37.743Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:55:37.744Z [INFO] server-41ef8dd7-3fc5-2de4-d9d4-1acbe98d81dc: copied to local snapshot: bytes=71 +2025-01-28T14:55:37.745Z [INFO] server-41ef8dd7-3fc5-2de4-d9d4-1acbe98d81dc: snapshot restore progress: id=2-10013-1738076137743 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2025-01-28T14:55:37.745Z [INFO] server-41ef8dd7-3fc5-2de4-d9d4-1acbe98d81dc: removing all old logs from log store +2025-01-28T14:55:37.745Z [INFO] server-41ef8dd7-3fc5-2de4-d9d4-1acbe98d81dc: compacting logs: from=1 to=2 +2025-01-28T14:55:37.745Z [INFO] server-41ef8dd7-3fc5-2de4-d9d4-1acbe98d81dc: Installed remote snapshot +2025-01-28T14:55:37.746Z [INFO] server-b1db11d7-518d-da10-aff3-ca6b3ab71779: pipelining replication: peer="{Voter server-41ef8dd7-3fc5-2de4-d9d4-1acbe98d81dc 41ef8dd7-3fc5-2de4-d9d4-1acbe98d81dc}" +2025-01-28T14:55:38.750Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:55:37.74397652 +0000 UTC m=+123.418466500 (2 nodes), 2.24µs from start of poll, 1.596520471s from cluster start. Timeout at 2025-01-28 14:55:38.749233836 +0000 UTC m=+124.423723876, 1.005257376s after stability +2025-01-28T14:55:38.762Z [INFO] server-d4c1da89-f157-768b-7e09-66e59e01dfa5: aborting pipeline replication: peer="{Voter server-104efdcf-a081-9f52-7794-f9a4ae371d5d 104efdcf-a081-9f52-7794-f9a4ae371d5d}" +2025-01-28T14:55:38.762Z [INFO] server-d4c1da89-f157-768b-7e09-66e59e01dfa5: aborting pipeline replication: peer="{Voter server-373cd188-51fc-85d5-97b3-492821fe0623 373cd188-51fc-85d5-97b3-492821fe0623}" +--- PASS: TestRaft_UserRestore (84.14s) + --- PASS: TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_false (2.58s) + --- PASS: TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_false (2.75s) + --- PASS: TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_true (4.27s) + --- PASS: TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_true (4.47s) + --- PASS: TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_false (2.63s) + --- PASS: TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_false (2.85s) --- PASS: TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_true (4.23s) - --- PASS: TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_true (4.16s) - --- PASS: TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_false (2.86s) - --- PASS: TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_false (2.69s) - --- PASS: TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_true (4.38s) - --- PASS: TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_true (4.17s) - --- PASS: TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_false (2.68s) - --- PASS: TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_false (2.68s) - --- PASS: TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_true (4.22s) - --- PASS: TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_true (4.32s) - --- PASS: TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_false (2.88s) - --- PASS: TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_false (2.64s) - --- PASS: TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_true (4.37s) - --- PASS: TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_true (4.35s) - --- PASS: TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_false (2.66s) - --- PASS: TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_false (2.64s) - --- PASS: TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_true (4.19s) - --- PASS: TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_true (4.38s) + --- PASS: TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_true (4.25s) + --- PASS: TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_false (2.73s) + --- PASS: TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_false (2.60s) + --- PASS: TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_true (4.42s) + --- PASS: TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_true (4.26s) + --- PASS: TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_false (2.92s) + --- PASS: TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_false (2.62s) + --- PASS: TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_true (4.35s) + --- PASS: TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_true (4.50s) + --- PASS: TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_false (2.63s) + --- PASS: TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_false (2.60s) + --- PASS: TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_true (4.17s) + --- PASS: TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_true (4.32s) + --- PASS: TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_false (2.81s) + --- PASS: TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_false (2.74s) + --- PASS: TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_true (4.28s) + --- PASS: TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_true (4.15s) === RUN TestRaft_SendSnapshotFollower -2025-01-28T14:48:37.535Z [INFO] server-36dfa77f-42ea-aecc-e97a-0fe8582e3fdb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-36dfa77f-42ea-aecc-e97a-0fe8582e3fdb Address:36dfa77f-42ea-aecc-e97a-0fe8582e3fdb} {Suffrage:Voter ID:server-7bacb526-80a8-a9cf-06f9-21febbffef8e Address:7bacb526-80a8-a9cf-06f9-21febbffef8e} {Suffrage:Voter ID:server-316ada7c-e3c1-50a4-7107-315421fc3f4f Address:316ada7c-e3c1-50a4-7107-315421fc3f4f}]" -2025-01-28T14:48:37.535Z [INFO] server-7bacb526-80a8-a9cf-06f9-21febbffef8e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-36dfa77f-42ea-aecc-e97a-0fe8582e3fdb Address:36dfa77f-42ea-aecc-e97a-0fe8582e3fdb} {Suffrage:Voter ID:server-7bacb526-80a8-a9cf-06f9-21febbffef8e Address:7bacb526-80a8-a9cf-06f9-21febbffef8e} {Suffrage:Voter ID:server-316ada7c-e3c1-50a4-7107-315421fc3f4f Address:316ada7c-e3c1-50a4-7107-315421fc3f4f}]" -2025-01-28T14:48:37.536Z [INFO] server-316ada7c-e3c1-50a4-7107-315421fc3f4f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-36dfa77f-42ea-aecc-e97a-0fe8582e3fdb Address:36dfa77f-42ea-aecc-e97a-0fe8582e3fdb} {Suffrage:Voter ID:server-7bacb526-80a8-a9cf-06f9-21febbffef8e Address:7bacb526-80a8-a9cf-06f9-21febbffef8e} {Suffrage:Voter ID:server-316ada7c-e3c1-50a4-7107-315421fc3f4f Address:316ada7c-e3c1-50a4-7107-315421fc3f4f}]" -2025-01-28T14:48:37.536Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:48:37.536Z [INFO] server-36dfa77f-42ea-aecc-e97a-0fe8582e3fdb: entering follower state: follower="Node at 36dfa77f-42ea-aecc-e97a-0fe8582e3fdb [Follower]" leader-address= leader-id= -2025-01-28T14:48:37.537Z [INFO] server-7bacb526-80a8-a9cf-06f9-21febbffef8e: entering follower state: follower="Node at 7bacb526-80a8-a9cf-06f9-21febbffef8e [Follower]" leader-address= leader-id= -2025-01-28T14:48:37.537Z [INFO] server-316ada7c-e3c1-50a4-7107-315421fc3f4f: entering follower state: follower="Node at 316ada7c-e3c1-50a4-7107-315421fc3f4f [Follower]" leader-address= leader-id= -2025-01-28T14:48:37.591Z [WARN] server-316ada7c-e3c1-50a4-7107-315421fc3f4f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:37.591Z [INFO] server-316ada7c-e3c1-50a4-7107-315421fc3f4f: entering candidate state: node="Node at 316ada7c-e3c1-50a4-7107-315421fc3f4f [Candidate]" term=2 -2025-01-28T14:48:37.591Z [INFO] server-316ada7c-e3c1-50a4-7107-315421fc3f4f: election won: term=2 tally=2 -2025-01-28T14:48:37.591Z [INFO] server-316ada7c-e3c1-50a4-7107-315421fc3f4f: entering leader state: leader="Node at 316ada7c-e3c1-50a4-7107-315421fc3f4f [Leader]" -2025-01-28T14:48:37.592Z [INFO] server-316ada7c-e3c1-50a4-7107-315421fc3f4f: added peer, starting replication: peer=server-36dfa77f-42ea-aecc-e97a-0fe8582e3fdb -2025-01-28T14:48:37.592Z [INFO] server-316ada7c-e3c1-50a4-7107-315421fc3f4f: added peer, starting replication: peer=server-7bacb526-80a8-a9cf-06f9-21febbffef8e -2025-01-28T14:48:37.593Z [INFO] server-316ada7c-e3c1-50a4-7107-315421fc3f4f: pipelining replication: peer="{Voter server-7bacb526-80a8-a9cf-06f9-21febbffef8e 7bacb526-80a8-a9cf-06f9-21febbffef8e}" -2025-01-28T14:48:37.593Z [INFO] server-316ada7c-e3c1-50a4-7107-315421fc3f4f: pipelining replication: peer="{Voter server-36dfa77f-42ea-aecc-e97a-0fe8582e3fdb 36dfa77f-42ea-aecc-e97a-0fe8582e3fdb}" -2025-01-28T14:48:37.698Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:48:37.592478942 +0000 UTC m=+100.675883966 (2 nodes), 55.722321ms from start of poll, 57.803383ms from cluster start. Timeout at 2025-01-28 14:48:37.697488609 +0000 UTC m=+100.780893634, 105.009668ms after stability -2025-01-28T14:48:37.698Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:37.805Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:37.698465423 +0000 UTC m=+100.781870447 (1 nodes), 6.667µs from start of poll, 163.789864ms from cluster start. Timeout at 2025-01-28 14:48:37.803892769 +0000 UTC m=+100.887298126, 105.427679ms after stability -2025-01-28T14:48:37.806Z [ERROR] server-316ada7c-e3c1-50a4-7107-315421fc3f4f: failed to pipeline appendEntries: peer="{Voter server-36dfa77f-42ea-aecc-e97a-0fe8582e3fdb 36dfa77f-42ea-aecc-e97a-0fe8582e3fdb}" error="append pipeline closed" -2025-01-28T14:48:37.806Z [INFO] server-316ada7c-e3c1-50a4-7107-315421fc3f4f: aborting pipeline replication: peer="{Voter server-36dfa77f-42ea-aecc-e97a-0fe8582e3fdb 36dfa77f-42ea-aecc-e97a-0fe8582e3fdb}" -2025-01-28T14:48:37.806Z [ERROR] server-316ada7c-e3c1-50a4-7107-315421fc3f4f: failed to appendEntries to: peer="{Voter server-36dfa77f-42ea-aecc-e97a-0fe8582e3fdb 36dfa77f-42ea-aecc-e97a-0fe8582e3fdb}" error="failed to connect to peer: 36dfa77f-42ea-aecc-e97a-0fe8582e3fdb" -2025-01-28T14:48:37.807Z [ERROR] server-316ada7c-e3c1-50a4-7107-315421fc3f4f: failed to heartbeat to: peer=36dfa77f-42ea-aecc-e97a-0fe8582e3fdb backoff time=10ms error="failed to connect to peer: 36dfa77f-42ea-aecc-e97a-0fe8582e3fdb" -2025-01-28T14:48:37.817Z [ERROR] server-316ada7c-e3c1-50a4-7107-315421fc3f4f: failed to appendEntries to: peer="{Voter server-36dfa77f-42ea-aecc-e97a-0fe8582e3fdb 36dfa77f-42ea-aecc-e97a-0fe8582e3fdb}" error="failed to connect to peer: 36dfa77f-42ea-aecc-e97a-0fe8582e3fdb" -2025-01-28T14:48:37.823Z [ERROR] server-316ada7c-e3c1-50a4-7107-315421fc3f4f: failed to heartbeat to: peer=36dfa77f-42ea-aecc-e97a-0fe8582e3fdb backoff time=10ms error="failed to connect to peer: 36dfa77f-42ea-aecc-e97a-0fe8582e3fdb" -2025-01-28T14:48:37.827Z [ERROR] server-316ada7c-e3c1-50a4-7107-315421fc3f4f: failed to appendEntries to: peer="{Voter server-36dfa77f-42ea-aecc-e97a-0fe8582e3fdb 36dfa77f-42ea-aecc-e97a-0fe8582e3fdb}" error="failed to connect to peer: 36dfa77f-42ea-aecc-e97a-0fe8582e3fdb" -2025-01-28T14:48:37.842Z [ERROR] server-316ada7c-e3c1-50a4-7107-315421fc3f4f: failed to heartbeat to: peer=36dfa77f-42ea-aecc-e97a-0fe8582e3fdb backoff time=10ms error="failed to connect to peer: 36dfa77f-42ea-aecc-e97a-0fe8582e3fdb" -2025-01-28T14:48:37.847Z [ERROR] server-316ada7c-e3c1-50a4-7107-315421fc3f4f: failed to appendEntries to: peer="{Voter server-36dfa77f-42ea-aecc-e97a-0fe8582e3fdb 36dfa77f-42ea-aecc-e97a-0fe8582e3fdb}" error="failed to connect to peer: 36dfa77f-42ea-aecc-e97a-0fe8582e3fdb" +2025-01-28T14:55:38.767Z [INFO] server-dcc7b185-d18b-cc0c-f895-c87db52ff555: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dcc7b185-d18b-cc0c-f895-c87db52ff555 Address:dcc7b185-d18b-cc0c-f895-c87db52ff555} {Suffrage:Voter ID:server-3f84320c-7a18-ccee-4e2c-bb37f3c08d1e Address:3f84320c-7a18-ccee-4e2c-bb37f3c08d1e} {Suffrage:Voter ID:server-69650491-b359-5331-f391-71ddfce09298 Address:69650491-b359-5331-f391-71ddfce09298}]" +2025-01-28T14:55:38.767Z [INFO] server-dcc7b185-d18b-cc0c-f895-c87db52ff555: entering follower state: follower="Node at dcc7b185-d18b-cc0c-f895-c87db52ff555 [Follower]" leader-address= leader-id= +2025-01-28T14:55:38.767Z [INFO] server-3f84320c-7a18-ccee-4e2c-bb37f3c08d1e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dcc7b185-d18b-cc0c-f895-c87db52ff555 Address:dcc7b185-d18b-cc0c-f895-c87db52ff555} {Suffrage:Voter ID:server-3f84320c-7a18-ccee-4e2c-bb37f3c08d1e Address:3f84320c-7a18-ccee-4e2c-bb37f3c08d1e} {Suffrage:Voter ID:server-69650491-b359-5331-f391-71ddfce09298 Address:69650491-b359-5331-f391-71ddfce09298}]" +2025-01-28T14:55:38.767Z [INFO] server-3f84320c-7a18-ccee-4e2c-bb37f3c08d1e: entering follower state: follower="Node at 3f84320c-7a18-ccee-4e2c-bb37f3c08d1e [Follower]" leader-address= leader-id= +2025-01-28T14:55:38.768Z [INFO] server-69650491-b359-5331-f391-71ddfce09298: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dcc7b185-d18b-cc0c-f895-c87db52ff555 Address:dcc7b185-d18b-cc0c-f895-c87db52ff555} {Suffrage:Voter ID:server-3f84320c-7a18-ccee-4e2c-bb37f3c08d1e Address:3f84320c-7a18-ccee-4e2c-bb37f3c08d1e} {Suffrage:Voter ID:server-69650491-b359-5331-f391-71ddfce09298 Address:69650491-b359-5331-f391-71ddfce09298}]" +2025-01-28T14:55:38.768Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:55:38.768Z [INFO] server-69650491-b359-5331-f391-71ddfce09298: entering follower state: follower="Node at 69650491-b359-5331-f391-71ddfce09298 [Follower]" leader-address= leader-id= +2025-01-28T14:55:38.845Z [WARN] server-69650491-b359-5331-f391-71ddfce09298: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:38.845Z [INFO] server-69650491-b359-5331-f391-71ddfce09298: entering candidate state: node="Node at 69650491-b359-5331-f391-71ddfce09298 [Candidate]" term=2 +2025-01-28T14:55:38.846Z [INFO] server-69650491-b359-5331-f391-71ddfce09298: election won: term=2 tally=2 +2025-01-28T14:55:38.846Z [INFO] server-69650491-b359-5331-f391-71ddfce09298: entering leader state: leader="Node at 69650491-b359-5331-f391-71ddfce09298 [Leader]" +2025-01-28T14:55:38.846Z [INFO] server-69650491-b359-5331-f391-71ddfce09298: added peer, starting replication: peer=server-dcc7b185-d18b-cc0c-f895-c87db52ff555 +2025-01-28T14:55:38.846Z [INFO] server-69650491-b359-5331-f391-71ddfce09298: added peer, starting replication: peer=server-3f84320c-7a18-ccee-4e2c-bb37f3c08d1e +2025-01-28T14:55:38.846Z [INFO] server-69650491-b359-5331-f391-71ddfce09298: pipelining replication: peer="{Voter server-dcc7b185-d18b-cc0c-f895-c87db52ff555 dcc7b185-d18b-cc0c-f895-c87db52ff555}" +2025-01-28T14:55:38.846Z [INFO] server-69650491-b359-5331-f391-71ddfce09298: pipelining replication: peer="{Voter server-3f84320c-7a18-ccee-4e2c-bb37f3c08d1e 3f84320c-7a18-ccee-4e2c-bb37f3c08d1e}" +2025-01-28T14:55:38.952Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:55:38.846319029 +0000 UTC m=+124.520808989 (2 nodes), 78.221181ms from start of poll, 79.241544ms from cluster start. Timeout at 2025-01-28 14:55:38.951328188 +0000 UTC m=+124.625818168, 105.009179ms after stability +2025-01-28T14:55:38.952Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:39.057Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:38.952176623 +0000 UTC m=+124.626666603 (1 nodes), 2.34µs from start of poll, 185.099158ms from cluster start. Timeout at 2025-01-28 14:55:39.057329529 +0000 UTC m=+124.731819649, 105.153046ms after stability +2025-01-28T14:55:39.058Z [ERROR] server-69650491-b359-5331-f391-71ddfce09298: failed to pipeline appendEntries: peer="{Voter server-dcc7b185-d18b-cc0c-f895-c87db52ff555 dcc7b185-d18b-cc0c-f895-c87db52ff555}" error="append pipeline closed" +2025-01-28T14:55:39.058Z [INFO] server-69650491-b359-5331-f391-71ddfce09298: aborting pipeline replication: peer="{Voter server-dcc7b185-d18b-cc0c-f895-c87db52ff555 dcc7b185-d18b-cc0c-f895-c87db52ff555}" +2025-01-28T14:55:39.058Z [ERROR] server-69650491-b359-5331-f391-71ddfce09298: failed to appendEntries to: peer="{Voter server-dcc7b185-d18b-cc0c-f895-c87db52ff555 dcc7b185-d18b-cc0c-f895-c87db52ff555}" error="failed to connect to peer: dcc7b185-d18b-cc0c-f895-c87db52ff555" +2025-01-28T14:55:39.061Z [ERROR] server-69650491-b359-5331-f391-71ddfce09298: failed to heartbeat to: peer=dcc7b185-d18b-cc0c-f895-c87db52ff555 backoff time=10ms error="failed to connect to peer: dcc7b185-d18b-cc0c-f895-c87db52ff555" raft_test.go:1627: [INFO] Finished apply without behind follower -2025-01-28T14:48:37.855Z [ERROR] server-36dfa77f-42ea-aecc-e97a-0fe8582e3fdb: failed to take snapshot: error="nothing new to snapshot" -2025-01-28T14:48:37.855Z [INFO] server-7bacb526-80a8-a9cf-06f9-21febbffef8e: starting snapshot up to: index=101 -2025-01-28T14:48:37.855Z [INFO] creating new snapshot: path=/tmp/raft854547184/snapshots/2-101-1738075717855.tmp -2025-01-28T14:48:37.859Z [INFO] server-7bacb526-80a8-a9cf-06f9-21febbffef8e: compacting logs: from=1 to=92 -2025-01-28T14:48:37.859Z [INFO] server-7bacb526-80a8-a9cf-06f9-21febbffef8e: snapshot complete up to: index=101 -2025-01-28T14:48:37.859Z [WARN] server-316ada7c-e3c1-50a4-7107-315421fc3f4f: failed to contact: server-id=server-36dfa77f-42ea-aecc-e97a-0fe8582e3fdb time=54.792627ms -2025-01-28T14:48:37.860Z [ERROR] server-316ada7c-e3c1-50a4-7107-315421fc3f4f: failed to heartbeat to: peer=36dfa77f-42ea-aecc-e97a-0fe8582e3fdb backoff time=20ms error="failed to connect to peer: 36dfa77f-42ea-aecc-e97a-0fe8582e3fdb" -2025-01-28T14:48:37.860Z [INFO] server-316ada7c-e3c1-50a4-7107-315421fc3f4f: starting snapshot up to: index=102 -2025-01-28T14:48:37.860Z [INFO] creating new snapshot: path=/tmp/raft1106525746/snapshots/2-102-1738075717860.tmp -2025-01-28T14:48:37.864Z [INFO] server-316ada7c-e3c1-50a4-7107-315421fc3f4f: compacting logs: from=1 to=92 -2025-01-28T14:48:37.864Z [INFO] server-316ada7c-e3c1-50a4-7107-315421fc3f4f: snapshot complete up to: index=102 -2025-01-28T14:48:37.888Z [ERROR] server-316ada7c-e3c1-50a4-7107-315421fc3f4f: failed to get log: index=2 error="log not found" -2025-01-28T14:48:37.890Z [INFO] creating new snapshot: path=/tmp/raft1923064124/snapshots/2-102-1738075717890.tmp -2025-01-28T14:48:37.892Z [INFO] server-36dfa77f-42ea-aecc-e97a-0fe8582e3fdb: snapshot network transfer progress: read-bytes=693 percent-complete=100.00% -2025-01-28T14:48:37.897Z [INFO] server-36dfa77f-42ea-aecc-e97a-0fe8582e3fdb: copied to local snapshot: bytes=693 -2025-01-28T14:48:37.899Z [INFO] server-36dfa77f-42ea-aecc-e97a-0fe8582e3fdb: snapshot restore progress: id=2-102-1738075717890 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% -2025-01-28T14:48:37.899Z [INFO] server-36dfa77f-42ea-aecc-e97a-0fe8582e3fdb: Installed remote snapshot -2025-01-28T14:48:37.900Z [WARN] server-36dfa77f-42ea-aecc-e97a-0fe8582e3fdb: skipping application of old log: index=102 -2025-01-28T14:48:37.900Z [INFO] server-316ada7c-e3c1-50a4-7107-315421fc3f4f: pipelining replication: peer="{Voter server-36dfa77f-42ea-aecc-e97a-0fe8582e3fdb 36dfa77f-42ea-aecc-e97a-0fe8582e3fdb}" -2025-01-28T14:48:37.903Z [INFO] server-316ada7c-e3c1-50a4-7107-315421fc3f4f: aborting pipeline replication: peer="{Voter server-7bacb526-80a8-a9cf-06f9-21febbffef8e 7bacb526-80a8-a9cf-06f9-21febbffef8e}" -2025-01-28T14:48:37.903Z [INFO] server-316ada7c-e3c1-50a4-7107-315421fc3f4f: aborting pipeline replication: peer="{Voter server-36dfa77f-42ea-aecc-e97a-0fe8582e3fdb 36dfa77f-42ea-aecc-e97a-0fe8582e3fdb}" ---- PASS: TestRaft_SendSnapshotFollower (0.38s) +2025-01-28T14:55:39.066Z [ERROR] server-dcc7b185-d18b-cc0c-f895-c87db52ff555: failed to take snapshot: error="nothing new to snapshot" +2025-01-28T14:55:39.066Z [INFO] server-3f84320c-7a18-ccee-4e2c-bb37f3c08d1e: starting snapshot up to: index=99 +2025-01-28T14:55:39.066Z [INFO] creating new snapshot: path=/tmp/raft2545984320/snapshots/2-99-1738076139066.tmp +2025-01-28T14:55:39.067Z [INFO] server-3f84320c-7a18-ccee-4e2c-bb37f3c08d1e: compacting logs: from=1 to=92 +2025-01-28T14:55:39.068Z [INFO] server-3f84320c-7a18-ccee-4e2c-bb37f3c08d1e: snapshot complete up to: index=99 +2025-01-28T14:55:39.068Z [INFO] server-69650491-b359-5331-f391-71ddfce09298: starting snapshot up to: index=102 +2025-01-28T14:55:39.068Z [INFO] creating new snapshot: path=/tmp/raft642241934/snapshots/2-102-1738076139068.tmp +2025-01-28T14:55:39.068Z [ERROR] server-69650491-b359-5331-f391-71ddfce09298: failed to appendEntries to: peer="{Voter server-dcc7b185-d18b-cc0c-f895-c87db52ff555 dcc7b185-d18b-cc0c-f895-c87db52ff555}" error="failed to connect to peer: dcc7b185-d18b-cc0c-f895-c87db52ff555" +2025-01-28T14:55:39.069Z [INFO] server-69650491-b359-5331-f391-71ddfce09298: compacting logs: from=1 to=92 +2025-01-28T14:55:39.069Z [INFO] server-69650491-b359-5331-f391-71ddfce09298: snapshot complete up to: index=102 +2025-01-28T14:55:39.079Z [ERROR] server-69650491-b359-5331-f391-71ddfce09298: failed to get log: index=2 error="log not found" +2025-01-28T14:55:39.080Z [INFO] creating new snapshot: path=/tmp/raft547618876/snapshots/2-102-1738076139080.tmp +2025-01-28T14:55:39.081Z [INFO] server-dcc7b185-d18b-cc0c-f895-c87db52ff555: snapshot network transfer progress: read-bytes=693 percent-complete=100.00% +2025-01-28T14:55:39.081Z [INFO] server-dcc7b185-d18b-cc0c-f895-c87db52ff555: copied to local snapshot: bytes=693 +2025-01-28T14:55:39.082Z [INFO] server-dcc7b185-d18b-cc0c-f895-c87db52ff555: snapshot restore progress: id=2-102-1738076139080 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% +2025-01-28T14:55:39.082Z [INFO] server-dcc7b185-d18b-cc0c-f895-c87db52ff555: Installed remote snapshot +2025-01-28T14:55:39.082Z [INFO] server-69650491-b359-5331-f391-71ddfce09298: aborting pipeline replication: peer="{Voter server-3f84320c-7a18-ccee-4e2c-bb37f3c08d1e 3f84320c-7a18-ccee-4e2c-bb37f3c08d1e}" +--- PASS: TestRaft_SendSnapshotFollower (0.32s) === RUN TestRaft_SendSnapshotAndLogsFollower -2025-01-28T14:48:37.916Z [INFO] server-4d1decea-a770-faf8-210c-86a6e5d06545: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4d1decea-a770-faf8-210c-86a6e5d06545 Address:4d1decea-a770-faf8-210c-86a6e5d06545} {Suffrage:Voter ID:server-031e1864-785b-cf2a-7814-174409746957 Address:031e1864-785b-cf2a-7814-174409746957} {Suffrage:Voter ID:server-fa678abe-f03d-7e8f-0bf3-453214a9dba1 Address:fa678abe-f03d-7e8f-0bf3-453214a9dba1}]" -2025-01-28T14:48:37.917Z [INFO] server-031e1864-785b-cf2a-7814-174409746957: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4d1decea-a770-faf8-210c-86a6e5d06545 Address:4d1decea-a770-faf8-210c-86a6e5d06545} {Suffrage:Voter ID:server-031e1864-785b-cf2a-7814-174409746957 Address:031e1864-785b-cf2a-7814-174409746957} {Suffrage:Voter ID:server-fa678abe-f03d-7e8f-0bf3-453214a9dba1 Address:fa678abe-f03d-7e8f-0bf3-453214a9dba1}]" -2025-01-28T14:48:37.917Z [INFO] server-4d1decea-a770-faf8-210c-86a6e5d06545: entering follower state: follower="Node at 4d1decea-a770-faf8-210c-86a6e5d06545 [Follower]" leader-address= leader-id= -2025-01-28T14:48:37.918Z [INFO] server-031e1864-785b-cf2a-7814-174409746957: entering follower state: follower="Node at 031e1864-785b-cf2a-7814-174409746957 [Follower]" leader-address= leader-id= -2025-01-28T14:48:37.918Z [INFO] server-fa678abe-f03d-7e8f-0bf3-453214a9dba1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4d1decea-a770-faf8-210c-86a6e5d06545 Address:4d1decea-a770-faf8-210c-86a6e5d06545} {Suffrage:Voter ID:server-031e1864-785b-cf2a-7814-174409746957 Address:031e1864-785b-cf2a-7814-174409746957} {Suffrage:Voter ID:server-fa678abe-f03d-7e8f-0bf3-453214a9dba1 Address:fa678abe-f03d-7e8f-0bf3-453214a9dba1}]" -2025-01-28T14:48:37.918Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:48:37.918Z [INFO] server-fa678abe-f03d-7e8f-0bf3-453214a9dba1: entering follower state: follower="Node at fa678abe-f03d-7e8f-0bf3-453214a9dba1 [Follower]" leader-address= leader-id= -2025-01-28T14:48:37.976Z [WARN] server-4d1decea-a770-faf8-210c-86a6e5d06545: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:37.977Z [INFO] server-4d1decea-a770-faf8-210c-86a6e5d06545: entering candidate state: node="Node at 4d1decea-a770-faf8-210c-86a6e5d06545 [Candidate]" term=2 -2025-01-28T14:48:37.977Z [INFO] server-4d1decea-a770-faf8-210c-86a6e5d06545: election won: term=2 tally=2 -2025-01-28T14:48:37.977Z [INFO] server-4d1decea-a770-faf8-210c-86a6e5d06545: entering leader state: leader="Node at 4d1decea-a770-faf8-210c-86a6e5d06545 [Leader]" -2025-01-28T14:48:37.977Z [INFO] server-4d1decea-a770-faf8-210c-86a6e5d06545: added peer, starting replication: peer=server-031e1864-785b-cf2a-7814-174409746957 -2025-01-28T14:48:37.978Z [INFO] server-4d1decea-a770-faf8-210c-86a6e5d06545: added peer, starting replication: peer=server-fa678abe-f03d-7e8f-0bf3-453214a9dba1 -2025-01-28T14:48:37.979Z [INFO] server-4d1decea-a770-faf8-210c-86a6e5d06545: pipelining replication: peer="{Voter server-fa678abe-f03d-7e8f-0bf3-453214a9dba1 fa678abe-f03d-7e8f-0bf3-453214a9dba1}" -2025-01-28T14:48:37.979Z [INFO] server-4d1decea-a770-faf8-210c-86a6e5d06545: pipelining replication: peer="{Voter server-031e1864-785b-cf2a-7814-174409746957 031e1864-785b-cf2a-7814-174409746957}" -2025-01-28T14:48:38.084Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:48:37.979269428 +0000 UTC m=+101.062674119 (2 nodes), 60.556131ms from start of poll, 63.337881ms from cluster start. Timeout at 2025-01-28 14:48:38.084279762 +0000 UTC m=+101.167684453, 105.010334ms after stability -2025-01-28T14:48:38.084Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:38.192Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:38.084873565 +0000 UTC m=+101.168278256 (1 nodes), 6.667µs from start of poll, 168.942018ms from cluster start. Timeout at 2025-01-28 14:48:38.190349912 +0000 UTC m=+101.273754936, 105.47668ms after stability -2025-01-28T14:48:38.193Z [ERROR] server-4d1decea-a770-faf8-210c-86a6e5d06545: failed to pipeline appendEntries: peer="{Voter server-031e1864-785b-cf2a-7814-174409746957 031e1864-785b-cf2a-7814-174409746957}" error="append pipeline closed" -2025-01-28T14:48:38.193Z [INFO] server-4d1decea-a770-faf8-210c-86a6e5d06545: aborting pipeline replication: peer="{Voter server-031e1864-785b-cf2a-7814-174409746957 031e1864-785b-cf2a-7814-174409746957}" -2025-01-28T14:48:38.193Z [ERROR] server-4d1decea-a770-faf8-210c-86a6e5d06545: failed to appendEntries to: peer="{Voter server-031e1864-785b-cf2a-7814-174409746957 031e1864-785b-cf2a-7814-174409746957}" error="failed to connect to peer: 031e1864-785b-cf2a-7814-174409746957" -2025-01-28T14:48:38.194Z [ERROR] server-4d1decea-a770-faf8-210c-86a6e5d06545: failed to heartbeat to: peer=031e1864-785b-cf2a-7814-174409746957 backoff time=10ms error="failed to connect to peer: 031e1864-785b-cf2a-7814-174409746957" -2025-01-28T14:48:38.204Z [ERROR] server-4d1decea-a770-faf8-210c-86a6e5d06545: failed to appendEntries to: peer="{Voter server-031e1864-785b-cf2a-7814-174409746957 031e1864-785b-cf2a-7814-174409746957}" error="failed to connect to peer: 031e1864-785b-cf2a-7814-174409746957" -2025-01-28T14:48:38.211Z [ERROR] server-4d1decea-a770-faf8-210c-86a6e5d06545: failed to heartbeat to: peer=031e1864-785b-cf2a-7814-174409746957 backoff time=10ms error="failed to connect to peer: 031e1864-785b-cf2a-7814-174409746957" -2025-01-28T14:48:38.214Z [ERROR] server-4d1decea-a770-faf8-210c-86a6e5d06545: failed to appendEntries to: peer="{Voter server-031e1864-785b-cf2a-7814-174409746957 031e1864-785b-cf2a-7814-174409746957}" error="failed to connect to peer: 031e1864-785b-cf2a-7814-174409746957" +2025-01-28T14:55:39.086Z [INFO] server-dd8e445e-8394-7ca9-3e23-a84b340ccc8b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dd8e445e-8394-7ca9-3e23-a84b340ccc8b Address:dd8e445e-8394-7ca9-3e23-a84b340ccc8b} {Suffrage:Voter ID:server-c604e684-364d-3dc5-bdcd-f79bc77965e1 Address:c604e684-364d-3dc5-bdcd-f79bc77965e1} {Suffrage:Voter ID:server-44ceefb6-23a5-1f80-f094-962dd4830e58 Address:44ceefb6-23a5-1f80-f094-962dd4830e58}]" +2025-01-28T14:55:39.086Z [INFO] server-dd8e445e-8394-7ca9-3e23-a84b340ccc8b: entering follower state: follower="Node at dd8e445e-8394-7ca9-3e23-a84b340ccc8b [Follower]" leader-address= leader-id= +2025-01-28T14:55:39.087Z [INFO] server-c604e684-364d-3dc5-bdcd-f79bc77965e1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dd8e445e-8394-7ca9-3e23-a84b340ccc8b Address:dd8e445e-8394-7ca9-3e23-a84b340ccc8b} {Suffrage:Voter ID:server-c604e684-364d-3dc5-bdcd-f79bc77965e1 Address:c604e684-364d-3dc5-bdcd-f79bc77965e1} {Suffrage:Voter ID:server-44ceefb6-23a5-1f80-f094-962dd4830e58 Address:44ceefb6-23a5-1f80-f094-962dd4830e58}]" +2025-01-28T14:55:39.087Z [INFO] server-c604e684-364d-3dc5-bdcd-f79bc77965e1: entering follower state: follower="Node at c604e684-364d-3dc5-bdcd-f79bc77965e1 [Follower]" leader-address= leader-id= +2025-01-28T14:55:39.087Z [INFO] server-44ceefb6-23a5-1f80-f094-962dd4830e58: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dd8e445e-8394-7ca9-3e23-a84b340ccc8b Address:dd8e445e-8394-7ca9-3e23-a84b340ccc8b} {Suffrage:Voter ID:server-c604e684-364d-3dc5-bdcd-f79bc77965e1 Address:c604e684-364d-3dc5-bdcd-f79bc77965e1} {Suffrage:Voter ID:server-44ceefb6-23a5-1f80-f094-962dd4830e58 Address:44ceefb6-23a5-1f80-f094-962dd4830e58}]" +2025-01-28T14:55:39.087Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:55:39.087Z [INFO] server-44ceefb6-23a5-1f80-f094-962dd4830e58: entering follower state: follower="Node at 44ceefb6-23a5-1f80-f094-962dd4830e58 [Follower]" leader-address= leader-id= +2025-01-28T14:55:39.164Z [WARN] server-dd8e445e-8394-7ca9-3e23-a84b340ccc8b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:39.164Z [INFO] server-dd8e445e-8394-7ca9-3e23-a84b340ccc8b: entering candidate state: node="Node at dd8e445e-8394-7ca9-3e23-a84b340ccc8b [Candidate]" term=2 +2025-01-28T14:55:39.164Z [INFO] server-dd8e445e-8394-7ca9-3e23-a84b340ccc8b: election won: term=2 tally=2 +2025-01-28T14:55:39.164Z [INFO] server-dd8e445e-8394-7ca9-3e23-a84b340ccc8b: entering leader state: leader="Node at dd8e445e-8394-7ca9-3e23-a84b340ccc8b [Leader]" +2025-01-28T14:55:39.164Z [INFO] server-dd8e445e-8394-7ca9-3e23-a84b340ccc8b: added peer, starting replication: peer=server-c604e684-364d-3dc5-bdcd-f79bc77965e1 +2025-01-28T14:55:39.164Z [INFO] server-dd8e445e-8394-7ca9-3e23-a84b340ccc8b: added peer, starting replication: peer=server-44ceefb6-23a5-1f80-f094-962dd4830e58 +2025-01-28T14:55:39.164Z [INFO] server-dd8e445e-8394-7ca9-3e23-a84b340ccc8b: pipelining replication: peer="{Voter server-c604e684-364d-3dc5-bdcd-f79bc77965e1 c604e684-364d-3dc5-bdcd-f79bc77965e1}" +2025-01-28T14:55:39.165Z [INFO] server-dd8e445e-8394-7ca9-3e23-a84b340ccc8b: pipelining replication: peer="{Voter server-44ceefb6-23a5-1f80-f094-962dd4830e58 44ceefb6-23a5-1f80-f094-962dd4830e58}" +2025-01-28T14:55:39.269Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:55:39.164795626 +0000 UTC m=+124.839285586 (2 nodes), 77.360783ms from start of poll, 78.23724ms from cluster start. Timeout at 2025-01-28 14:55:39.269800466 +0000 UTC m=+124.944290466, 105.00488ms after stability +2025-01-28T14:55:39.269Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:39.375Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:39.269999641 +0000 UTC m=+124.944489621 (1 nodes), 2.06µs from start of poll, 183.441275ms from cluster start. Timeout at 2025-01-28 14:55:39.375181344 +0000 UTC m=+125.049671304, 105.181683ms after stability +2025-01-28T14:55:39.375Z [ERROR] server-dd8e445e-8394-7ca9-3e23-a84b340ccc8b: failed to pipeline appendEntries: peer="{Voter server-c604e684-364d-3dc5-bdcd-f79bc77965e1 c604e684-364d-3dc5-bdcd-f79bc77965e1}" error="append pipeline closed" +2025-01-28T14:55:39.375Z [INFO] server-dd8e445e-8394-7ca9-3e23-a84b340ccc8b: aborting pipeline replication: peer="{Voter server-c604e684-364d-3dc5-bdcd-f79bc77965e1 c604e684-364d-3dc5-bdcd-f79bc77965e1}" +2025-01-28T14:55:39.375Z [ERROR] server-dd8e445e-8394-7ca9-3e23-a84b340ccc8b: failed to appendEntries to: peer="{Voter server-c604e684-364d-3dc5-bdcd-f79bc77965e1 c604e684-364d-3dc5-bdcd-f79bc77965e1}" error="failed to connect to peer: c604e684-364d-3dc5-bdcd-f79bc77965e1" +2025-01-28T14:55:39.385Z [ERROR] server-dd8e445e-8394-7ca9-3e23-a84b340ccc8b: failed to heartbeat to: peer=c604e684-364d-3dc5-bdcd-f79bc77965e1 backoff time=10ms error="failed to connect to peer: c604e684-364d-3dc5-bdcd-f79bc77965e1" +2025-01-28T14:55:39.385Z [ERROR] server-dd8e445e-8394-7ca9-3e23-a84b340ccc8b: failed to appendEntries to: peer="{Voter server-c604e684-364d-3dc5-bdcd-f79bc77965e1 c604e684-364d-3dc5-bdcd-f79bc77965e1}" error="failed to connect to peer: c604e684-364d-3dc5-bdcd-f79bc77965e1" raft_test.go:1669: [INFO] Finished apply without behind follower -2025-01-28T14:48:38.227Z [INFO] server-4d1decea-a770-faf8-210c-86a6e5d06545: starting snapshot up to: index=102 -2025-01-28T14:48:38.228Z [INFO] creating new snapshot: path=/tmp/raft1686706354/snapshots/2-102-1738075718228.tmp -2025-01-28T14:48:38.228Z [ERROR] server-4d1decea-a770-faf8-210c-86a6e5d06545: failed to heartbeat to: peer=031e1864-785b-cf2a-7814-174409746957 backoff time=10ms error="failed to connect to peer: 031e1864-785b-cf2a-7814-174409746957" -2025-01-28T14:48:38.234Z [INFO] server-4d1decea-a770-faf8-210c-86a6e5d06545: compacting logs: from=1 to=92 -2025-01-28T14:48:38.234Z [INFO] server-4d1decea-a770-faf8-210c-86a6e5d06545: snapshot complete up to: index=102 -2025-01-28T14:48:38.236Z [ERROR] server-4d1decea-a770-faf8-210c-86a6e5d06545: failed to get log: index=2 error="log not found" -2025-01-28T14:48:38.243Z [ERROR] server-031e1864-785b-cf2a-7814-174409746957: failed to take snapshot: error="nothing new to snapshot" -2025-01-28T14:48:38.244Z [ERROR] server-4d1decea-a770-faf8-210c-86a6e5d06545: failed to install snapshot: id=2-102-1738075718228 error="failed to connect to peer: 031e1864-785b-cf2a-7814-174409746957" -2025-01-28T14:48:38.245Z [ERROR] server-4d1decea-a770-faf8-210c-86a6e5d06545: failed to send snapshot to: peer="{Voter server-031e1864-785b-cf2a-7814-174409746957 031e1864-785b-cf2a-7814-174409746957}" error="failed to connect to peer: 031e1864-785b-cf2a-7814-174409746957" -2025-01-28T14:48:38.247Z [WARN] server-4d1decea-a770-faf8-210c-86a6e5d06545: failed to contact: server-id=server-031e1864-785b-cf2a-7814-174409746957 time=59.91178ms -2025-01-28T14:48:38.249Z [INFO] server-fa678abe-f03d-7e8f-0bf3-453214a9dba1: starting snapshot up to: index=102 -2025-01-28T14:48:38.249Z [INFO] creating new snapshot: path=/tmp/raft956992359/snapshots/2-102-1738075718249.tmp -2025-01-28T14:48:38.253Z [INFO] server-fa678abe-f03d-7e8f-0bf3-453214a9dba1: compacting logs: from=1 to=92 -2025-01-28T14:48:38.253Z [INFO] server-fa678abe-f03d-7e8f-0bf3-453214a9dba1: snapshot complete up to: index=102 -2025-01-28T14:48:38.257Z [ERROR] server-4d1decea-a770-faf8-210c-86a6e5d06545: failed to heartbeat to: peer=031e1864-785b-cf2a-7814-174409746957 backoff time=20ms error="failed to connect to peer: 031e1864-785b-cf2a-7814-174409746957" -2025-01-28T14:48:38.264Z [WARN] server-031e1864-785b-cf2a-7814-174409746957: heartbeat timeout reached, starting election: last-leader-addr=4d1decea-a770-faf8-210c-86a6e5d06545 last-leader-id=server-4d1decea-a770-faf8-210c-86a6e5d06545 -2025-01-28T14:48:38.265Z [INFO] server-031e1864-785b-cf2a-7814-174409746957: entering candidate state: node="Node at 031e1864-785b-cf2a-7814-174409746957 [Candidate]" term=3 -2025-01-28T14:48:38.265Z [ERROR] server-031e1864-785b-cf2a-7814-174409746957: failed to make requestVote RPC: target="{Voter server-fa678abe-f03d-7e8f-0bf3-453214a9dba1 fa678abe-f03d-7e8f-0bf3-453214a9dba1}" error="failed to connect to peer: fa678abe-f03d-7e8f-0bf3-453214a9dba1" term=3 -2025-01-28T14:48:38.265Z [ERROR] server-031e1864-785b-cf2a-7814-174409746957: failed to make requestVote RPC: target="{Voter server-4d1decea-a770-faf8-210c-86a6e5d06545 4d1decea-a770-faf8-210c-86a6e5d06545}" error="failed to connect to peer: 4d1decea-a770-faf8-210c-86a6e5d06545" term=3 +2025-01-28T14:55:39.387Z [INFO] server-dd8e445e-8394-7ca9-3e23-a84b340ccc8b: starting snapshot up to: index=102 +2025-01-28T14:55:39.387Z [INFO] creating new snapshot: path=/tmp/raft342791242/snapshots/2-102-1738076139387.tmp +2025-01-28T14:55:39.389Z [INFO] server-dd8e445e-8394-7ca9-3e23-a84b340ccc8b: compacting logs: from=1 to=92 +2025-01-28T14:55:39.389Z [INFO] server-dd8e445e-8394-7ca9-3e23-a84b340ccc8b: snapshot complete up to: index=102 +2025-01-28T14:55:39.389Z [ERROR] server-c604e684-364d-3dc5-bdcd-f79bc77965e1: failed to take snapshot: error="nothing new to snapshot" +2025-01-28T14:55:39.389Z [INFO] server-44ceefb6-23a5-1f80-f094-962dd4830e58: starting snapshot up to: index=98 +2025-01-28T14:55:39.390Z [INFO] creating new snapshot: path=/tmp/raft3205141601/snapshots/2-98-1738076139389.tmp +2025-01-28T14:55:39.391Z [INFO] server-44ceefb6-23a5-1f80-f094-962dd4830e58: compacting logs: from=1 to=92 +2025-01-28T14:55:39.391Z [INFO] server-44ceefb6-23a5-1f80-f094-962dd4830e58: snapshot complete up to: index=98 +2025-01-28T14:55:39.396Z [ERROR] server-dd8e445e-8394-7ca9-3e23-a84b340ccc8b: failed to get log: index=2 error="log not found" +2025-01-28T14:55:39.396Z [ERROR] server-dd8e445e-8394-7ca9-3e23-a84b340ccc8b: failed to install snapshot: id=2-102-1738076139387 error="failed to connect to peer: c604e684-364d-3dc5-bdcd-f79bc77965e1" +2025-01-28T14:55:39.396Z [ERROR] server-dd8e445e-8394-7ca9-3e23-a84b340ccc8b: failed to send snapshot to: peer="{Voter server-c604e684-364d-3dc5-bdcd-f79bc77965e1 c604e684-364d-3dc5-bdcd-f79bc77965e1}" error="failed to connect to peer: c604e684-364d-3dc5-bdcd-f79bc77965e1" raft_test.go:1690: [INFO] Finished apply without behind follower -2025-01-28T14:48:38.285Z [ERROR] server-4d1decea-a770-faf8-210c-86a6e5d06545: failed to get log: index=2 error="log not found" -2025-01-28T14:48:38.286Z [INFO] server-031e1864-785b-cf2a-7814-174409746957: ignoring installSnapshot request with older term than current term: request-term=2 current-term=3 -2025-01-28T14:48:38.287Z [ERROR] server-4d1decea-a770-faf8-210c-86a6e5d06545: peer has newer term, stopping replication: peer="{Voter server-031e1864-785b-cf2a-7814-174409746957 031e1864-785b-cf2a-7814-174409746957}" -2025-01-28T14:48:38.287Z [INFO] server-4d1decea-a770-faf8-210c-86a6e5d06545: entering follower state: follower="Node at 4d1decea-a770-faf8-210c-86a6e5d06545 [Follower]" leader-address= leader-id= -2025-01-28T14:48:38.287Z [INFO] server-4d1decea-a770-faf8-210c-86a6e5d06545: aborting pipeline replication: peer="{Voter server-fa678abe-f03d-7e8f-0bf3-453214a9dba1 fa678abe-f03d-7e8f-0bf3-453214a9dba1}" -2025-01-28T14:48:38.359Z [WARN] server-fa678abe-f03d-7e8f-0bf3-453214a9dba1: heartbeat timeout reached, starting election: last-leader-addr=4d1decea-a770-faf8-210c-86a6e5d06545 last-leader-id=server-4d1decea-a770-faf8-210c-86a6e5d06545 -2025-01-28T14:48:38.359Z [INFO] server-fa678abe-f03d-7e8f-0bf3-453214a9dba1: entering candidate state: node="Node at fa678abe-f03d-7e8f-0bf3-453214a9dba1 [Candidate]" term=3 -2025-01-28T14:48:38.361Z [WARN] server-4d1decea-a770-faf8-210c-86a6e5d06545: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:38.362Z [INFO] server-4d1decea-a770-faf8-210c-86a6e5d06545: entering candidate state: node="Node at 4d1decea-a770-faf8-210c-86a6e5d06545 [Candidate]" term=3 -2025-01-28T14:48:38.362Z [INFO] server-fa678abe-f03d-7e8f-0bf3-453214a9dba1: duplicate requestVote for same term: term=3 -2025-01-28T14:48:38.362Z [INFO] server-031e1864-785b-cf2a-7814-174409746957: duplicate requestVote for same term: term=3 -2025-01-28T14:48:38.363Z [INFO] server-031e1864-785b-cf2a-7814-174409746957: duplicate requestVote for same term: term=3 -2025-01-28T14:48:38.363Z [INFO] server-4d1decea-a770-faf8-210c-86a6e5d06545: duplicate requestVote for same term: term=3 -2025-01-28T14:48:38.364Z [WARN] server-031e1864-785b-cf2a-7814-174409746957: Election timeout reached, restarting election -2025-01-28T14:48:38.364Z [INFO] server-031e1864-785b-cf2a-7814-174409746957: entering candidate state: node="Node at 031e1864-785b-cf2a-7814-174409746957 [Candidate]" term=4 -2025-01-28T14:48:38.365Z [WARN] server-fa678abe-f03d-7e8f-0bf3-453214a9dba1: rejecting vote request since our last index is greater: candidate=031e1864-785b-cf2a-7814-174409746957 last-index=202 last-candidate-index=2 -2025-01-28T14:48:38.365Z [INFO] server-fa678abe-f03d-7e8f-0bf3-453214a9dba1: entering follower state: follower="Node at fa678abe-f03d-7e8f-0bf3-453214a9dba1 [Follower]" leader-address= leader-id= -2025-01-28T14:48:38.366Z [WARN] server-4d1decea-a770-faf8-210c-86a6e5d06545: rejecting vote request since our last index is greater: candidate=031e1864-785b-cf2a-7814-174409746957 last-index=202 last-candidate-index=2 -2025-01-28T14:48:38.366Z [INFO] server-4d1decea-a770-faf8-210c-86a6e5d06545: entering follower state: follower="Node at 4d1decea-a770-faf8-210c-86a6e5d06545 [Follower]" leader-address= leader-id= -2025-01-28T14:48:38.439Z [WARN] server-031e1864-785b-cf2a-7814-174409746957: Election timeout reached, restarting election -2025-01-28T14:48:38.439Z [INFO] server-031e1864-785b-cf2a-7814-174409746957: entering candidate state: node="Node at 031e1864-785b-cf2a-7814-174409746957 [Candidate]" term=5 -2025-01-28T14:48:38.439Z [WARN] server-4d1decea-a770-faf8-210c-86a6e5d06545: rejecting vote request since our last index is greater: candidate=031e1864-785b-cf2a-7814-174409746957 last-index=202 last-candidate-index=2 -2025-01-28T14:48:38.439Z [WARN] server-fa678abe-f03d-7e8f-0bf3-453214a9dba1: rejecting vote request since our last index is greater: candidate=031e1864-785b-cf2a-7814-174409746957 last-index=202 last-candidate-index=2 -2025-01-28T14:48:38.457Z [WARN] server-4d1decea-a770-faf8-210c-86a6e5d06545: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:38.457Z [INFO] server-4d1decea-a770-faf8-210c-86a6e5d06545: entering candidate state: node="Node at 4d1decea-a770-faf8-210c-86a6e5d06545 [Candidate]" term=6 -2025-01-28T14:48:38.458Z [INFO] server-031e1864-785b-cf2a-7814-174409746957: entering follower state: follower="Node at 031e1864-785b-cf2a-7814-174409746957 [Follower]" leader-address= leader-id= -2025-01-28T14:48:38.459Z [INFO] server-4d1decea-a770-faf8-210c-86a6e5d06545: election won: term=6 tally=2 -2025-01-28T14:48:38.459Z [INFO] server-4d1decea-a770-faf8-210c-86a6e5d06545: entering leader state: leader="Node at 4d1decea-a770-faf8-210c-86a6e5d06545 [Leader]" -2025-01-28T14:48:38.459Z [INFO] server-4d1decea-a770-faf8-210c-86a6e5d06545: added peer, starting replication: peer=server-031e1864-785b-cf2a-7814-174409746957 -2025-01-28T14:48:38.459Z [INFO] server-4d1decea-a770-faf8-210c-86a6e5d06545: added peer, starting replication: peer=server-fa678abe-f03d-7e8f-0bf3-453214a9dba1 -2025-01-28T14:48:38.460Z [WARN] server-031e1864-785b-cf2a-7814-174409746957: failed to get previous log: previous-index=202 last-index=2 error="log not found" -2025-01-28T14:48:38.460Z [WARN] server-4d1decea-a770-faf8-210c-86a6e5d06545: appendEntries rejected, sending older logs: peer="{Voter server-031e1864-785b-cf2a-7814-174409746957 031e1864-785b-cf2a-7814-174409746957}" next=3 -2025-01-28T14:48:38.460Z [ERROR] server-4d1decea-a770-faf8-210c-86a6e5d06545: failed to get log: index=2 error="log not found" -2025-01-28T14:48:38.461Z [INFO] server-4d1decea-a770-faf8-210c-86a6e5d06545: pipelining replication: peer="{Voter server-fa678abe-f03d-7e8f-0bf3-453214a9dba1 fa678abe-f03d-7e8f-0bf3-453214a9dba1}" -2025-01-28T14:48:38.463Z [INFO] creating new snapshot: path=/tmp/raft931015257/snapshots/2-102-1738075718463.tmp -2025-01-28T14:48:38.465Z [INFO] server-031e1864-785b-cf2a-7814-174409746957: snapshot network transfer progress: read-bytes=693 percent-complete=100.00% -2025-01-28T14:48:38.467Z [INFO] server-031e1864-785b-cf2a-7814-174409746957: copied to local snapshot: bytes=693 -2025-01-28T14:48:38.470Z [INFO] server-031e1864-785b-cf2a-7814-174409746957: snapshot restore progress: id=2-102-1738075718463 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% -2025-01-28T14:48:38.470Z [INFO] server-031e1864-785b-cf2a-7814-174409746957: Installed remote snapshot -2025-01-28T14:48:38.472Z [INFO] server-4d1decea-a770-faf8-210c-86a6e5d06545: pipelining replication: peer="{Voter server-031e1864-785b-cf2a-7814-174409746957 031e1864-785b-cf2a-7814-174409746957}" -2025-01-28T14:48:38.476Z [INFO] server-4d1decea-a770-faf8-210c-86a6e5d06545: aborting pipeline replication: peer="{Voter server-031e1864-785b-cf2a-7814-174409746957 031e1864-785b-cf2a-7814-174409746957}" -2025-01-28T14:48:38.477Z [INFO] server-4d1decea-a770-faf8-210c-86a6e5d06545: aborting pipeline replication: peer="{Voter server-fa678abe-f03d-7e8f-0bf3-453214a9dba1 fa678abe-f03d-7e8f-0bf3-453214a9dba1}" ---- PASS: TestRaft_SendSnapshotAndLogsFollower (0.59s) +2025-01-28T14:55:39.417Z [ERROR] server-dd8e445e-8394-7ca9-3e23-a84b340ccc8b: failed to get log: index=2 error="log not found" +2025-01-28T14:55:39.418Z [INFO] creating new snapshot: path=/tmp/raft2693038633/snapshots/2-102-1738076139418.tmp +2025-01-28T14:55:39.419Z [INFO] server-c604e684-364d-3dc5-bdcd-f79bc77965e1: snapshot network transfer progress: read-bytes=693 percent-complete=100.00% +2025-01-28T14:55:39.419Z [INFO] server-c604e684-364d-3dc5-bdcd-f79bc77965e1: copied to local snapshot: bytes=693 +2025-01-28T14:55:39.420Z [INFO] server-c604e684-364d-3dc5-bdcd-f79bc77965e1: snapshot restore progress: id=2-102-1738076139418 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% +2025-01-28T14:55:39.420Z [INFO] server-c604e684-364d-3dc5-bdcd-f79bc77965e1: Installed remote snapshot +2025-01-28T14:55:39.421Z [INFO] server-dd8e445e-8394-7ca9-3e23-a84b340ccc8b: pipelining replication: peer="{Voter server-c604e684-364d-3dc5-bdcd-f79bc77965e1 c604e684-364d-3dc5-bdcd-f79bc77965e1}" +2025-01-28T14:55:39.426Z [INFO] server-dd8e445e-8394-7ca9-3e23-a84b340ccc8b: aborting pipeline replication: peer="{Voter server-44ceefb6-23a5-1f80-f094-962dd4830e58 44ceefb6-23a5-1f80-f094-962dd4830e58}" +2025-01-28T14:55:39.426Z [INFO] server-dd8e445e-8394-7ca9-3e23-a84b340ccc8b: aborting pipeline replication: peer="{Voter server-c604e684-364d-3dc5-bdcd-f79bc77965e1 c604e684-364d-3dc5-bdcd-f79bc77965e1}" +--- PASS: TestRaft_SendSnapshotAndLogsFollower (0.34s) === RUN TestRaft_ReJoinFollower -2025-01-28T14:48:38.503Z [INFO] server-becdd36e-e96c-187b-3eb7-8a992d4eedfb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-becdd36e-e96c-187b-3eb7-8a992d4eedfb Address:becdd36e-e96c-187b-3eb7-8a992d4eedfb} {Suffrage:Voter ID:server-04a0601a-25ab-1658-3c9e-4426f4ec72e4 Address:04a0601a-25ab-1658-3c9e-4426f4ec72e4} {Suffrage:Voter ID:server-ac425c94-2ab9-dbef-5131-ef711499ccef Address:ac425c94-2ab9-dbef-5131-ef711499ccef}]" -2025-01-28T14:48:38.504Z [INFO] server-becdd36e-e96c-187b-3eb7-8a992d4eedfb: entering follower state: follower="Node at becdd36e-e96c-187b-3eb7-8a992d4eedfb [Follower]" leader-address= leader-id= -2025-01-28T14:48:38.504Z [INFO] server-04a0601a-25ab-1658-3c9e-4426f4ec72e4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-becdd36e-e96c-187b-3eb7-8a992d4eedfb Address:becdd36e-e96c-187b-3eb7-8a992d4eedfb} {Suffrage:Voter ID:server-04a0601a-25ab-1658-3c9e-4426f4ec72e4 Address:04a0601a-25ab-1658-3c9e-4426f4ec72e4} {Suffrage:Voter ID:server-ac425c94-2ab9-dbef-5131-ef711499ccef Address:ac425c94-2ab9-dbef-5131-ef711499ccef}]" -2025-01-28T14:48:38.504Z [INFO] server-04a0601a-25ab-1658-3c9e-4426f4ec72e4: entering follower state: follower="Node at 04a0601a-25ab-1658-3c9e-4426f4ec72e4 [Follower]" leader-address= leader-id= -2025-01-28T14:48:38.505Z [INFO] server-ac425c94-2ab9-dbef-5131-ef711499ccef: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-becdd36e-e96c-187b-3eb7-8a992d4eedfb Address:becdd36e-e96c-187b-3eb7-8a992d4eedfb} {Suffrage:Voter ID:server-04a0601a-25ab-1658-3c9e-4426f4ec72e4 Address:04a0601a-25ab-1658-3c9e-4426f4ec72e4} {Suffrage:Voter ID:server-ac425c94-2ab9-dbef-5131-ef711499ccef Address:ac425c94-2ab9-dbef-5131-ef711499ccef}]" -2025-01-28T14:48:38.505Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:38.505Z [INFO] server-ac425c94-2ab9-dbef-5131-ef711499ccef: entering follower state: follower="Node at ac425c94-2ab9-dbef-5131-ef711499ccef [Follower]" leader-address= leader-id= -2025-01-28T14:48:38.563Z [WARN] server-becdd36e-e96c-187b-3eb7-8a992d4eedfb: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:38.563Z [INFO] server-becdd36e-e96c-187b-3eb7-8a992d4eedfb: entering candidate state: node="Node at becdd36e-e96c-187b-3eb7-8a992d4eedfb [Candidate]" term=2 -2025-01-28T14:48:38.563Z [WARN] server-04a0601a-25ab-1658-3c9e-4426f4ec72e4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:38.563Z [INFO] server-04a0601a-25ab-1658-3c9e-4426f4ec72e4: entering candidate state: node="Node at 04a0601a-25ab-1658-3c9e-4426f4ec72e4 [Candidate]" term=2 -2025-01-28T14:48:38.564Z [INFO] server-04a0601a-25ab-1658-3c9e-4426f4ec72e4: duplicate requestVote for same term: term=2 -2025-01-28T14:48:38.564Z [INFO] server-becdd36e-e96c-187b-3eb7-8a992d4eedfb: duplicate requestVote for same term: term=2 -2025-01-28T14:48:38.564Z [INFO] server-becdd36e-e96c-187b-3eb7-8a992d4eedfb: election won: term=2 tally=2 -2025-01-28T14:48:38.564Z [INFO] server-becdd36e-e96c-187b-3eb7-8a992d4eedfb: entering leader state: leader="Node at becdd36e-e96c-187b-3eb7-8a992d4eedfb [Leader]" -2025-01-28T14:48:38.565Z [INFO] server-becdd36e-e96c-187b-3eb7-8a992d4eedfb: added peer, starting replication: peer=server-04a0601a-25ab-1658-3c9e-4426f4ec72e4 -2025-01-28T14:48:38.564Z [INFO] server-ac425c94-2ab9-dbef-5131-ef711499ccef: duplicate requestVote for same term: term=2 -2025-01-28T14:48:38.565Z [INFO] server-becdd36e-e96c-187b-3eb7-8a992d4eedfb: added peer, starting replication: peer=server-ac425c94-2ab9-dbef-5131-ef711499ccef -2025-01-28T14:48:38.565Z [INFO] server-04a0601a-25ab-1658-3c9e-4426f4ec72e4: entering follower state: follower="Node at 04a0601a-25ab-1658-3c9e-4426f4ec72e4 [Follower]" leader-address=becdd36e-e96c-187b-3eb7-8a992d4eedfb leader-id=server-becdd36e-e96c-187b-3eb7-8a992d4eedfb -2025-01-28T14:48:38.566Z [INFO] server-becdd36e-e96c-187b-3eb7-8a992d4eedfb: pipelining replication: peer="{Voter server-ac425c94-2ab9-dbef-5131-ef711499ccef ac425c94-2ab9-dbef-5131-ef711499ccef}" -2025-01-28T14:48:38.566Z [INFO] server-becdd36e-e96c-187b-3eb7-8a992d4eedfb: pipelining replication: peer="{Voter server-04a0601a-25ab-1658-3c9e-4426f4ec72e4 04a0601a-25ab-1658-3c9e-4426f4ec72e4}" -2025-01-28T14:48:38.671Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:38.566404198 +0000 UTC m=+101.649809556 (1 nodes), 61.011145ms from start of poll, 63.362882ms from cluster start. Timeout at 2025-01-28 14:48:38.671416198 +0000 UTC m=+101.754821223, 105.011667ms after stability -2025-01-28T14:48:38.677Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:48:38.782Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:48:38.677168821 +0000 UTC m=+101.760573512 (2 nodes), 7µs from start of poll, 174.126838ms from cluster start. Timeout at 2025-01-28 14:48:38.782660835 +0000 UTC m=+101.866065859, 105.492347ms after stability -2025-01-28T14:48:38.783Z [INFO] server-becdd36e-e96c-187b-3eb7-8a992d4eedfb: updating configuration: command=RemoveServer server-id=server-04a0601a-25ab-1658-3c9e-4426f4ec72e4 server-addr= servers="[{Suffrage:Voter ID:server-becdd36e-e96c-187b-3eb7-8a992d4eedfb Address:becdd36e-e96c-187b-3eb7-8a992d4eedfb} {Suffrage:Voter ID:server-ac425c94-2ab9-dbef-5131-ef711499ccef Address:ac425c94-2ab9-dbef-5131-ef711499ccef}]" -2025-01-28T14:48:38.783Z [INFO] server-becdd36e-e96c-187b-3eb7-8a992d4eedfb: removed peer, stopping replication: peer=server-04a0601a-25ab-1658-3c9e-4426f4ec72e4 last-index=3 -2025-01-28T14:48:38.783Z [INFO] server-becdd36e-e96c-187b-3eb7-8a992d4eedfb: aborting pipeline replication: peer="{Voter server-04a0601a-25ab-1658-3c9e-4426f4ec72e4 04a0601a-25ab-1658-3c9e-4426f4ec72e4}" -2025-01-28T14:48:38.871Z [WARN] server-04a0601a-25ab-1658-3c9e-4426f4ec72e4: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election -2025-01-28T14:48:38.891Z [INFO] server-becdd36e-e96c-187b-3eb7-8a992d4eedfb: updating configuration: command=AddVoter server-id=server-04a0601a-25ab-1658-3c9e-4426f4ec72e4 server-addr=04a0601a-25ab-1658-3c9e-4426f4ec72e4 servers="[{Suffrage:Voter ID:server-becdd36e-e96c-187b-3eb7-8a992d4eedfb Address:becdd36e-e96c-187b-3eb7-8a992d4eedfb} {Suffrage:Voter ID:server-ac425c94-2ab9-dbef-5131-ef711499ccef Address:ac425c94-2ab9-dbef-5131-ef711499ccef} {Suffrage:Voter ID:server-04a0601a-25ab-1658-3c9e-4426f4ec72e4 Address:04a0601a-25ab-1658-3c9e-4426f4ec72e4}]" -2025-01-28T14:48:38.891Z [INFO] server-becdd36e-e96c-187b-3eb7-8a992d4eedfb: added peer, starting replication: peer=server-04a0601a-25ab-1658-3c9e-4426f4ec72e4 -2025-01-28T14:48:38.892Z [WARN] server-04a0601a-25ab-1658-3c9e-4426f4ec72e4: failed to get previous log: previous-index=4 last-index=3 error="log not found" -2025-01-28T14:48:38.892Z [WARN] server-becdd36e-e96c-187b-3eb7-8a992d4eedfb: appendEntries rejected, sending older logs: peer="{Voter server-04a0601a-25ab-1658-3c9e-4426f4ec72e4 04a0601a-25ab-1658-3c9e-4426f4ec72e4}" next=4 -2025-01-28T14:48:38.892Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:38.892Z [INFO] server-becdd36e-e96c-187b-3eb7-8a992d4eedfb: pipelining replication: peer="{Voter server-04a0601a-25ab-1658-3c9e-4426f4ec72e4 04a0601a-25ab-1658-3c9e-4426f4ec72e4}" -2025-01-28T14:48:38.998Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:38.892517883 +0000 UTC m=+101.975922574 (1 nodes), 5.667µs from start of poll, 389.4759ms from cluster start. Timeout at 2025-01-28 14:48:38.998063232 +0000 UTC m=+102.081468590, 105.546016ms after stability -2025-01-28T14:48:38.998Z [INFO] server-becdd36e-e96c-187b-3eb7-8a992d4eedfb: aborting pipeline replication: peer="{Voter server-ac425c94-2ab9-dbef-5131-ef711499ccef ac425c94-2ab9-dbef-5131-ef711499ccef}" -2025-01-28T14:48:38.998Z [INFO] server-becdd36e-e96c-187b-3eb7-8a992d4eedfb: aborting pipeline replication: peer="{Voter server-04a0601a-25ab-1658-3c9e-4426f4ec72e4 04a0601a-25ab-1658-3c9e-4426f4ec72e4}" ---- PASS: TestRaft_ReJoinFollower (0.50s) +2025-01-28T14:55:39.430Z [INFO] server-f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf Address:f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf} {Suffrage:Voter ID:server-33fc2ea1-4723-c033-f027-777b733dea56 Address:33fc2ea1-4723-c033-f027-777b733dea56} {Suffrage:Voter ID:server-013a953e-cba9-58ed-a479-fe8165deb59f Address:013a953e-cba9-58ed-a479-fe8165deb59f}]" +2025-01-28T14:55:39.430Z [INFO] server-f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf: entering follower state: follower="Node at f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf [Follower]" leader-address= leader-id= +2025-01-28T14:55:39.430Z [INFO] server-33fc2ea1-4723-c033-f027-777b733dea56: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf Address:f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf} {Suffrage:Voter ID:server-33fc2ea1-4723-c033-f027-777b733dea56 Address:33fc2ea1-4723-c033-f027-777b733dea56} {Suffrage:Voter ID:server-013a953e-cba9-58ed-a479-fe8165deb59f Address:013a953e-cba9-58ed-a479-fe8165deb59f}]" +2025-01-28T14:55:39.430Z [INFO] server-33fc2ea1-4723-c033-f027-777b733dea56: entering follower state: follower="Node at 33fc2ea1-4723-c033-f027-777b733dea56 [Follower]" leader-address= leader-id= +2025-01-28T14:55:39.431Z [INFO] server-013a953e-cba9-58ed-a479-fe8165deb59f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf Address:f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf} {Suffrage:Voter ID:server-33fc2ea1-4723-c033-f027-777b733dea56 Address:33fc2ea1-4723-c033-f027-777b733dea56} {Suffrage:Voter ID:server-013a953e-cba9-58ed-a479-fe8165deb59f Address:013a953e-cba9-58ed-a479-fe8165deb59f}]" +2025-01-28T14:55:39.431Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:39.431Z [INFO] server-013a953e-cba9-58ed-a479-fe8165deb59f: entering follower state: follower="Node at 013a953e-cba9-58ed-a479-fe8165deb59f [Follower]" leader-address= leader-id= +2025-01-28T14:55:39.481Z [WARN] server-013a953e-cba9-58ed-a479-fe8165deb59f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:39.481Z [WARN] server-f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:39.481Z [INFO] server-f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf: entering candidate state: node="Node at f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf [Candidate]" term=2 +2025-01-28T14:55:39.481Z [INFO] server-013a953e-cba9-58ed-a479-fe8165deb59f: entering candidate state: node="Node at 013a953e-cba9-58ed-a479-fe8165deb59f [Candidate]" term=2 +2025-01-28T14:55:39.481Z [INFO] server-013a953e-cba9-58ed-a479-fe8165deb59f: duplicate requestVote for same term: term=2 +2025-01-28T14:55:39.481Z [INFO] server-f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf: election won: term=2 tally=2 +2025-01-28T14:55:39.482Z [INFO] server-f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf: entering leader state: leader="Node at f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf [Leader]" +2025-01-28T14:55:39.482Z [INFO] server-33fc2ea1-4723-c033-f027-777b733dea56: duplicate requestVote for same term: term=2 +2025-01-28T14:55:39.482Z [INFO] server-f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf: added peer, starting replication: peer=server-33fc2ea1-4723-c033-f027-777b733dea56 +2025-01-28T14:55:39.482Z [INFO] server-f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf: added peer, starting replication: peer=server-013a953e-cba9-58ed-a479-fe8165deb59f +2025-01-28T14:55:39.482Z [WARN] server-f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf: rejecting vote request since we have a leader: from=013a953e-cba9-58ed-a479-fe8165deb59f leader=f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf leader-id=server-f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf +2025-01-28T14:55:39.482Z [INFO] server-013a953e-cba9-58ed-a479-fe8165deb59f: entering follower state: follower="Node at 013a953e-cba9-58ed-a479-fe8165deb59f [Follower]" leader-address=f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf leader-id=server-f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf +2025-01-28T14:55:39.482Z [INFO] server-f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf: pipelining replication: peer="{Voter server-33fc2ea1-4723-c033-f027-777b733dea56 33fc2ea1-4723-c033-f027-777b733dea56}" +2025-01-28T14:55:39.482Z [INFO] server-f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf: pipelining replication: peer="{Voter server-013a953e-cba9-58ed-a479-fe8165deb59f 013a953e-cba9-58ed-a479-fe8165deb59f}" +2025-01-28T14:55:39.588Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:39.482318912 +0000 UTC m=+125.156808872 (1 nodes), 50.983191ms from start of poll, 51.998395ms from cluster start. Timeout at 2025-01-28 14:55:39.587330271 +0000 UTC m=+125.261820291, 105.011419ms after stability +2025-01-28T14:55:39.594Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:55:39.700Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:55:39.594188918 +0000 UTC m=+125.268678898 (2 nodes), 2.399µs from start of poll, 163.868421ms from cluster start. Timeout at 2025-01-28 14:55:39.699358282 +0000 UTC m=+125.373848262, 105.169364ms after stability +2025-01-28T14:55:39.700Z [INFO] server-f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf: updating configuration: command=RemoveServer server-id=server-33fc2ea1-4723-c033-f027-777b733dea56 server-addr= servers="[{Suffrage:Voter ID:server-f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf Address:f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf} {Suffrage:Voter ID:server-013a953e-cba9-58ed-a479-fe8165deb59f Address:013a953e-cba9-58ed-a479-fe8165deb59f}]" +2025-01-28T14:55:39.700Z [INFO] server-f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf: removed peer, stopping replication: peer=server-33fc2ea1-4723-c033-f027-777b733dea56 last-index=3 +2025-01-28T14:55:39.701Z [INFO] server-f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf: aborting pipeline replication: peer="{Voter server-33fc2ea1-4723-c033-f027-777b733dea56 33fc2ea1-4723-c033-f027-777b733dea56}" +2025-01-28T14:55:39.804Z [WARN] server-33fc2ea1-4723-c033-f027-777b733dea56: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +2025-01-28T14:55:39.806Z [INFO] server-f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf: updating configuration: command=AddVoter server-id=server-33fc2ea1-4723-c033-f027-777b733dea56 server-addr=33fc2ea1-4723-c033-f027-777b733dea56 servers="[{Suffrage:Voter ID:server-f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf Address:f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf} {Suffrage:Voter ID:server-013a953e-cba9-58ed-a479-fe8165deb59f Address:013a953e-cba9-58ed-a479-fe8165deb59f} {Suffrage:Voter ID:server-33fc2ea1-4723-c033-f027-777b733dea56 Address:33fc2ea1-4723-c033-f027-777b733dea56}]" +2025-01-28T14:55:39.806Z [INFO] server-f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf: added peer, starting replication: peer=server-33fc2ea1-4723-c033-f027-777b733dea56 +2025-01-28T14:55:39.806Z [WARN] server-33fc2ea1-4723-c033-f027-777b733dea56: failed to get previous log: previous-index=4 last-index=3 error="log not found" +2025-01-28T14:55:39.806Z [WARN] server-f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf: appendEntries rejected, sending older logs: peer="{Voter server-33fc2ea1-4723-c033-f027-777b733dea56 33fc2ea1-4723-c033-f027-777b733dea56}" next=4 +2025-01-28T14:55:39.807Z [INFO] server-f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf: pipelining replication: peer="{Voter server-33fc2ea1-4723-c033-f027-777b733dea56 33fc2ea1-4723-c033-f027-777b733dea56}" +2025-01-28T14:55:39.807Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:39.913Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:39.807374507 +0000 UTC m=+125.481864487 (1 nodes), 1.62µs from start of poll, 377.05401ms from cluster start. Timeout at 2025-01-28 14:55:39.912561389 +0000 UTC m=+125.587051469, 105.186982ms after stability +2025-01-28T14:55:39.913Z [INFO] server-f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf: aborting pipeline replication: peer="{Voter server-013a953e-cba9-58ed-a479-fe8165deb59f 013a953e-cba9-58ed-a479-fe8165deb59f}" +2025-01-28T14:55:39.913Z [INFO] server-f0ff7cb0-b6bb-25bf-50da-cb3fec9201bf: aborting pipeline replication: peer="{Voter server-33fc2ea1-4723-c033-f027-777b733dea56 33fc2ea1-4723-c033-f027-777b733dea56}" +--- PASS: TestRaft_ReJoinFollower (0.49s) === RUN TestRaft_LeaderLeaseExpire -2025-01-28T14:48:39.006Z [INFO] server-83d2b01a-b337-f10e-82b1-c0df4243cedb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-83d2b01a-b337-f10e-82b1-c0df4243cedb Address:83d2b01a-b337-f10e-82b1-c0df4243cedb} {Suffrage:Voter ID:server-6c4c558b-5e1d-a75b-a0c7-d2006e611008 Address:6c4c558b-5e1d-a75b-a0c7-d2006e611008}]" -2025-01-28T14:48:39.007Z [INFO] server-83d2b01a-b337-f10e-82b1-c0df4243cedb: entering follower state: follower="Node at 83d2b01a-b337-f10e-82b1-c0df4243cedb [Follower]" leader-address= leader-id= -2025-01-28T14:48:39.007Z [INFO] server-6c4c558b-5e1d-a75b-a0c7-d2006e611008: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-83d2b01a-b337-f10e-82b1-c0df4243cedb Address:83d2b01a-b337-f10e-82b1-c0df4243cedb} {Suffrage:Voter ID:server-6c4c558b-5e1d-a75b-a0c7-d2006e611008 Address:6c4c558b-5e1d-a75b-a0c7-d2006e611008}]" -2025-01-28T14:48:39.007Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:39.007Z [INFO] server-6c4c558b-5e1d-a75b-a0c7-d2006e611008: entering follower state: follower="Node at 6c4c558b-5e1d-a75b-a0c7-d2006e611008 [Follower]" leader-address= leader-id= -2025-01-28T14:48:39.077Z [WARN] server-83d2b01a-b337-f10e-82b1-c0df4243cedb: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:39.077Z [INFO] server-83d2b01a-b337-f10e-82b1-c0df4243cedb: entering candidate state: node="Node at 83d2b01a-b337-f10e-82b1-c0df4243cedb [Candidate]" term=2 -2025-01-28T14:48:39.078Z [INFO] server-83d2b01a-b337-f10e-82b1-c0df4243cedb: election won: term=2 tally=2 -2025-01-28T14:48:39.078Z [INFO] server-83d2b01a-b337-f10e-82b1-c0df4243cedb: entering leader state: leader="Node at 83d2b01a-b337-f10e-82b1-c0df4243cedb [Leader]" -2025-01-28T14:48:39.078Z [INFO] server-83d2b01a-b337-f10e-82b1-c0df4243cedb: added peer, starting replication: peer=server-6c4c558b-5e1d-a75b-a0c7-d2006e611008 -2025-01-28T14:48:39.079Z [INFO] server-83d2b01a-b337-f10e-82b1-c0df4243cedb: pipelining replication: peer="{Voter server-6c4c558b-5e1d-a75b-a0c7-d2006e611008 6c4c558b-5e1d-a75b-a0c7-d2006e611008}" -2025-01-28T14:48:39.184Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:39.079322764 +0000 UTC m=+102.162727788 (1 nodes), 71.752797ms from start of poll, 73.226841ms from cluster start. Timeout at 2025-01-28 14:48:39.184343432 +0000 UTC m=+102.267748123, 105.020335ms after stability -2025-01-28T14:48:39.191Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:48:39.297Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:48:39.191318423 +0000 UTC m=+102.274723114 (1 nodes), 5.667µs from start of poll, 185.222167ms from cluster start. Timeout at 2025-01-28 14:48:39.297095446 +0000 UTC m=+102.380500470, 105.777356ms after stability - raft_test.go:1799: [INFO] Disconnecting Node at 6c4c558b-5e1d-a75b-a0c7-d2006e611008 [Follower] -2025-01-28T14:48:39.298Z [ERROR] server-83d2b01a-b337-f10e-82b1-c0df4243cedb: failed to pipeline appendEntries: peer="{Voter server-6c4c558b-5e1d-a75b-a0c7-d2006e611008 6c4c558b-5e1d-a75b-a0c7-d2006e611008}" error="append pipeline closed" -2025-01-28T14:48:39.298Z [INFO] server-83d2b01a-b337-f10e-82b1-c0df4243cedb: aborting pipeline replication: peer="{Voter server-6c4c558b-5e1d-a75b-a0c7-d2006e611008 6c4c558b-5e1d-a75b-a0c7-d2006e611008}" -2025-01-28T14:48:39.303Z [ERROR] server-83d2b01a-b337-f10e-82b1-c0df4243cedb: failed to heartbeat to: peer=6c4c558b-5e1d-a75b-a0c7-d2006e611008 backoff time=10ms error="failed to connect to peer: 6c4c558b-5e1d-a75b-a0c7-d2006e611008" -2025-01-28T14:48:39.307Z [ERROR] server-83d2b01a-b337-f10e-82b1-c0df4243cedb: failed to appendEntries to: peer="{Voter server-6c4c558b-5e1d-a75b-a0c7-d2006e611008 6c4c558b-5e1d-a75b-a0c7-d2006e611008}" error="failed to connect to peer: 6c4c558b-5e1d-a75b-a0c7-d2006e611008" -2025-01-28T14:48:39.323Z [ERROR] server-83d2b01a-b337-f10e-82b1-c0df4243cedb: failed to appendEntries to: peer="{Voter server-6c4c558b-5e1d-a75b-a0c7-d2006e611008 6c4c558b-5e1d-a75b-a0c7-d2006e611008}" error="failed to connect to peer: 6c4c558b-5e1d-a75b-a0c7-d2006e611008" -2025-01-28T14:48:39.325Z [ERROR] server-83d2b01a-b337-f10e-82b1-c0df4243cedb: failed to heartbeat to: peer=6c4c558b-5e1d-a75b-a0c7-d2006e611008 backoff time=10ms error="failed to connect to peer: 6c4c558b-5e1d-a75b-a0c7-d2006e611008" -2025-01-28T14:48:39.342Z [ERROR] server-83d2b01a-b337-f10e-82b1-c0df4243cedb: failed to appendEntries to: peer="{Voter server-6c4c558b-5e1d-a75b-a0c7-d2006e611008 6c4c558b-5e1d-a75b-a0c7-d2006e611008}" error="failed to connect to peer: 6c4c558b-5e1d-a75b-a0c7-d2006e611008" -2025-01-28T14:48:39.344Z [WARN] server-83d2b01a-b337-f10e-82b1-c0df4243cedb: failed to contact: server-id=server-6c4c558b-5e1d-a75b-a0c7-d2006e611008 time=50.158823ms -2025-01-28T14:48:39.344Z [WARN] server-83d2b01a-b337-f10e-82b1-c0df4243cedb: failed to contact quorum of nodes, stepping down -2025-01-28T14:48:39.344Z [INFO] server-83d2b01a-b337-f10e-82b1-c0df4243cedb: entering follower state: follower="Node at 83d2b01a-b337-f10e-82b1-c0df4243cedb [Follower]" leader-address= leader-id= -2025-01-28T14:48:39.345Z [ERROR] server-83d2b01a-b337-f10e-82b1-c0df4243cedb: failed to heartbeat to: peer=6c4c558b-5e1d-a75b-a0c7-d2006e611008 backoff time=10ms error="failed to connect to peer: 6c4c558b-5e1d-a75b-a0c7-d2006e611008" -2025-01-28T14:48:39.345Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:39.351Z [WARN] server-6c4c558b-5e1d-a75b-a0c7-d2006e611008: heartbeat timeout reached, starting election: last-leader-addr=83d2b01a-b337-f10e-82b1-c0df4243cedb last-leader-id=server-83d2b01a-b337-f10e-82b1-c0df4243cedb -2025-01-28T14:48:39.351Z [INFO] server-6c4c558b-5e1d-a75b-a0c7-d2006e611008: entering candidate state: node="Node at 6c4c558b-5e1d-a75b-a0c7-d2006e611008 [Candidate]" term=3 -2025-01-28T14:48:39.352Z [ERROR] server-6c4c558b-5e1d-a75b-a0c7-d2006e611008: failed to make requestVote RPC: target="{Voter server-83d2b01a-b337-f10e-82b1-c0df4243cedb 83d2b01a-b337-f10e-82b1-c0df4243cedb}" error="failed to connect to peer: 83d2b01a-b337-f10e-82b1-c0df4243cedb" term=3 -2025-01-28T14:48:39.400Z [WARN] server-83d2b01a-b337-f10e-82b1-c0df4243cedb: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:39.400Z [INFO] server-83d2b01a-b337-f10e-82b1-c0df4243cedb: entering candidate state: node="Node at 83d2b01a-b337-f10e-82b1-c0df4243cedb [Candidate]" term=3 -2025-01-28T14:48:39.400Z [ERROR] server-83d2b01a-b337-f10e-82b1-c0df4243cedb: failed to make requestVote RPC: target="{Voter server-6c4c558b-5e1d-a75b-a0c7-d2006e611008 6c4c558b-5e1d-a75b-a0c7-d2006e611008}" error="failed to connect to peer: 6c4c558b-5e1d-a75b-a0c7-d2006e611008" term=3 -2025-01-28T14:48:39.443Z [WARN] server-6c4c558b-5e1d-a75b-a0c7-d2006e611008: Election timeout reached, restarting election -2025-01-28T14:48:39.443Z [INFO] server-6c4c558b-5e1d-a75b-a0c7-d2006e611008: entering candidate state: node="Node at 6c4c558b-5e1d-a75b-a0c7-d2006e611008 [Candidate]" term=4 -2025-01-28T14:48:39.443Z [ERROR] server-6c4c558b-5e1d-a75b-a0c7-d2006e611008: failed to make requestVote RPC: target="{Voter server-83d2b01a-b337-f10e-82b1-c0df4243cedb 83d2b01a-b337-f10e-82b1-c0df4243cedb}" error="failed to connect to peer: 83d2b01a-b337-f10e-82b1-c0df4243cedb" term=4 -2025-01-28T14:48:39.489Z [WARN] server-83d2b01a-b337-f10e-82b1-c0df4243cedb: Election timeout reached, restarting election -2025-01-28T14:48:39.490Z [INFO] server-83d2b01a-b337-f10e-82b1-c0df4243cedb: entering candidate state: node="Node at 83d2b01a-b337-f10e-82b1-c0df4243cedb [Candidate]" term=4 -2025-01-28T14:48:39.490Z [ERROR] server-83d2b01a-b337-f10e-82b1-c0df4243cedb: failed to make requestVote RPC: target="{Voter server-6c4c558b-5e1d-a75b-a0c7-d2006e611008 6c4c558b-5e1d-a75b-a0c7-d2006e611008}" error="failed to connect to peer: 6c4c558b-5e1d-a75b-a0c7-d2006e611008" term=4 -2025-01-28T14:48:39.506Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:39.400769977 +0000 UTC m=+102.484175335 (0 nodes), 55.350644ms from start of poll, 394.674388ms from cluster start. Timeout at 2025-01-28 14:48:39.505800978 +0000 UTC m=+102.589236670, 105.061335ms after stability -2025-01-28T14:48:39.536Z [WARN] server-6c4c558b-5e1d-a75b-a0c7-d2006e611008: Election timeout reached, restarting election -2025-01-28T14:48:39.536Z [INFO] server-6c4c558b-5e1d-a75b-a0c7-d2006e611008: entering candidate state: node="Node at 6c4c558b-5e1d-a75b-a0c7-d2006e611008 [Candidate]" term=5 -2025-01-28T14:48:39.537Z [ERROR] server-6c4c558b-5e1d-a75b-a0c7-d2006e611008: failed to make requestVote RPC: target="{Voter server-83d2b01a-b337-f10e-82b1-c0df4243cedb 83d2b01a-b337-f10e-82b1-c0df4243cedb}" error="failed to connect to peer: 83d2b01a-b337-f10e-82b1-c0df4243cedb" term=5 -2025-01-28T14:48:39.552Z [WARN] server-83d2b01a-b337-f10e-82b1-c0df4243cedb: Election timeout reached, restarting election -2025-01-28T14:48:39.553Z [INFO] server-83d2b01a-b337-f10e-82b1-c0df4243cedb: entering candidate state: node="Node at 83d2b01a-b337-f10e-82b1-c0df4243cedb [Candidate]" term=5 -2025-01-28T14:48:39.561Z [ERROR] server-83d2b01a-b337-f10e-82b1-c0df4243cedb: failed to make requestVote RPC: target="{Voter server-6c4c558b-5e1d-a75b-a0c7-d2006e611008 6c4c558b-5e1d-a75b-a0c7-d2006e611008}" error="failed to connect to peer: 6c4c558b-5e1d-a75b-a0c7-d2006e611008" term=5 ---- PASS: TestRaft_LeaderLeaseExpire (0.61s) +2025-01-28T14:55:39.919Z [INFO] server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd Address:f6d2dff6-b296-cafa-6dc0-7072a484f0fd} {Suffrage:Voter ID:server-5a3c7731-9dba-e855-7774-7b5c03278f2b Address:5a3c7731-9dba-e855-7774-7b5c03278f2b}]" +2025-01-28T14:55:39.921Z [INFO] server-5a3c7731-9dba-e855-7774-7b5c03278f2b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd Address:f6d2dff6-b296-cafa-6dc0-7072a484f0fd} {Suffrage:Voter ID:server-5a3c7731-9dba-e855-7774-7b5c03278f2b Address:5a3c7731-9dba-e855-7774-7b5c03278f2b}]" +2025-01-28T14:55:39.921Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:39.922Z [INFO] server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd: entering follower state: follower="Node at f6d2dff6-b296-cafa-6dc0-7072a484f0fd [Follower]" leader-address= leader-id= +2025-01-28T14:55:39.923Z [INFO] server-5a3c7731-9dba-e855-7774-7b5c03278f2b: entering follower state: follower="Node at 5a3c7731-9dba-e855-7774-7b5c03278f2b [Follower]" leader-address= leader-id= +2025-01-28T14:55:39.976Z [WARN] server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:39.976Z [INFO] server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd: entering candidate state: node="Node at f6d2dff6-b296-cafa-6dc0-7072a484f0fd [Candidate]" term=2 +2025-01-28T14:55:39.977Z [INFO] server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd: election won: term=2 tally=2 +2025-01-28T14:55:39.977Z [INFO] server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd: entering leader state: leader="Node at f6d2dff6-b296-cafa-6dc0-7072a484f0fd [Leader]" +2025-01-28T14:55:39.977Z [INFO] server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd: added peer, starting replication: peer=server-5a3c7731-9dba-e855-7774-7b5c03278f2b +2025-01-28T14:55:39.977Z [INFO] server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd: pipelining replication: peer="{Voter server-5a3c7731-9dba-e855-7774-7b5c03278f2b 5a3c7731-9dba-e855-7774-7b5c03278f2b}" +2025-01-28T14:55:40.083Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:39.977129993 +0000 UTC m=+125.651619973 (1 nodes), 55.900306ms from start of poll, 57.731514ms from cluster start. Timeout at 2025-01-28 14:55:40.082132833 +0000 UTC m=+125.756622833, 105.00286ms after stability +2025-01-28T14:55:40.088Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:55:40.194Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:55:40.089042856 +0000 UTC m=+125.763532836 (1 nodes), 1.74µs from start of poll, 169.644377ms from cluster start. Timeout at 2025-01-28 14:55:40.194183063 +0000 UTC m=+125.868673123, 105.140287ms after stability + raft_test.go:1799: [INFO] Disconnecting Node at 5a3c7731-9dba-e855-7774-7b5c03278f2b [Follower] +2025-01-28T14:55:40.194Z [ERROR] server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd: failed to pipeline appendEntries: peer="{Voter server-5a3c7731-9dba-e855-7774-7b5c03278f2b 5a3c7731-9dba-e855-7774-7b5c03278f2b}" error="append pipeline closed" +2025-01-28T14:55:40.194Z [INFO] server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd: aborting pipeline replication: peer="{Voter server-5a3c7731-9dba-e855-7774-7b5c03278f2b 5a3c7731-9dba-e855-7774-7b5c03278f2b}" +2025-01-28T14:55:40.196Z [ERROR] server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd: failed to heartbeat to: peer=5a3c7731-9dba-e855-7774-7b5c03278f2b backoff time=10ms error="failed to connect to peer: 5a3c7731-9dba-e855-7774-7b5c03278f2b" +2025-01-28T14:55:40.204Z [ERROR] server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd: failed to appendEntries to: peer="{Voter server-5a3c7731-9dba-e855-7774-7b5c03278f2b 5a3c7731-9dba-e855-7774-7b5c03278f2b}" error="failed to connect to peer: 5a3c7731-9dba-e855-7774-7b5c03278f2b" +2025-01-28T14:55:40.216Z [ERROR] server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd: failed to heartbeat to: peer=5a3c7731-9dba-e855-7774-7b5c03278f2b backoff time=10ms error="failed to connect to peer: 5a3c7731-9dba-e855-7774-7b5c03278f2b" +2025-01-28T14:55:40.223Z [ERROR] server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd: failed to appendEntries to: peer="{Voter server-5a3c7731-9dba-e855-7774-7b5c03278f2b 5a3c7731-9dba-e855-7774-7b5c03278f2b}" error="failed to connect to peer: 5a3c7731-9dba-e855-7774-7b5c03278f2b" +2025-01-28T14:55:40.236Z [ERROR] server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd: failed to heartbeat to: peer=5a3c7731-9dba-e855-7774-7b5c03278f2b backoff time=10ms error="failed to connect to peer: 5a3c7731-9dba-e855-7774-7b5c03278f2b" +2025-01-28T14:55:40.239Z [WARN] server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd: failed to contact: server-id=server-5a3c7731-9dba-e855-7774-7b5c03278f2b time=50.212463ms +2025-01-28T14:55:40.239Z [WARN] server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd: failed to contact quorum of nodes, stepping down +2025-01-28T14:55:40.239Z [INFO] server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd: entering follower state: follower="Node at f6d2dff6-b296-cafa-6dc0-7072a484f0fd [Follower]" leader-address= leader-id= +2025-01-28T14:55:40.239Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:40.243Z [ERROR] server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd: failed to appendEntries to: peer="{Voter server-5a3c7731-9dba-e855-7774-7b5c03278f2b 5a3c7731-9dba-e855-7774-7b5c03278f2b}" error="failed to connect to peer: 5a3c7731-9dba-e855-7774-7b5c03278f2b" +2025-01-28T14:55:40.288Z [WARN] server-5a3c7731-9dba-e855-7774-7b5c03278f2b: heartbeat timeout reached, starting election: last-leader-addr=f6d2dff6-b296-cafa-6dc0-7072a484f0fd last-leader-id=server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd +2025-01-28T14:55:40.288Z [INFO] server-5a3c7731-9dba-e855-7774-7b5c03278f2b: entering candidate state: node="Node at 5a3c7731-9dba-e855-7774-7b5c03278f2b [Candidate]" term=3 +2025-01-28T14:55:40.288Z [ERROR] server-5a3c7731-9dba-e855-7774-7b5c03278f2b: failed to make requestVote RPC: target="{Voter server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd f6d2dff6-b296-cafa-6dc0-7072a484f0fd}" error="failed to connect to peer: f6d2dff6-b296-cafa-6dc0-7072a484f0fd" term=3 +2025-01-28T14:55:40.333Z [WARN] server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:40.333Z [INFO] server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd: entering candidate state: node="Node at f6d2dff6-b296-cafa-6dc0-7072a484f0fd [Candidate]" term=3 +2025-01-28T14:55:40.333Z [ERROR] server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd: failed to make requestVote RPC: target="{Voter server-5a3c7731-9dba-e855-7774-7b5c03278f2b 5a3c7731-9dba-e855-7774-7b5c03278f2b}" error="failed to connect to peer: 5a3c7731-9dba-e855-7774-7b5c03278f2b" term=3 +2025-01-28T14:55:40.339Z [WARN] server-5a3c7731-9dba-e855-7774-7b5c03278f2b: Election timeout reached, restarting election +2025-01-28T14:55:40.339Z [INFO] server-5a3c7731-9dba-e855-7774-7b5c03278f2b: entering candidate state: node="Node at 5a3c7731-9dba-e855-7774-7b5c03278f2b [Candidate]" term=4 +2025-01-28T14:55:40.339Z [ERROR] server-5a3c7731-9dba-e855-7774-7b5c03278f2b: failed to make requestVote RPC: target="{Voter server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd f6d2dff6-b296-cafa-6dc0-7072a484f0fd}" error="failed to connect to peer: f6d2dff6-b296-cafa-6dc0-7072a484f0fd" term=4 +2025-01-28T14:55:40.414Z [WARN] server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd: Election timeout reached, restarting election +2025-01-28T14:55:40.415Z [INFO] server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd: entering candidate state: node="Node at f6d2dff6-b296-cafa-6dc0-7072a484f0fd [Candidate]" term=4 +2025-01-28T14:55:40.415Z [ERROR] server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd: failed to make requestVote RPC: target="{Voter server-5a3c7731-9dba-e855-7774-7b5c03278f2b 5a3c7731-9dba-e855-7774-7b5c03278f2b}" error="failed to connect to peer: 5a3c7731-9dba-e855-7774-7b5c03278f2b" term=4 +2025-01-28T14:55:40.436Z [WARN] server-5a3c7731-9dba-e855-7774-7b5c03278f2b: Election timeout reached, restarting election +2025-01-28T14:55:40.436Z [INFO] server-5a3c7731-9dba-e855-7774-7b5c03278f2b: entering candidate state: node="Node at 5a3c7731-9dba-e855-7774-7b5c03278f2b [Candidate]" term=5 +2025-01-28T14:55:40.436Z [ERROR] server-5a3c7731-9dba-e855-7774-7b5c03278f2b: failed to make requestVote RPC: target="{Voter server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd f6d2dff6-b296-cafa-6dc0-7072a484f0fd}" error="failed to connect to peer: f6d2dff6-b296-cafa-6dc0-7072a484f0fd" term=5 +2025-01-28T14:55:40.439Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:40.333846782 +0000 UTC m=+126.008336762 (0 nodes), 94.089704ms from start of poll, 414.448303ms from cluster start. Timeout at 2025-01-28 14:55:40.43886818 +0000 UTC m=+126.113358200, 105.021438ms after stability +2025-01-28T14:55:40.504Z [WARN] server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd: Election timeout reached, restarting election +2025-01-28T14:55:40.504Z [INFO] server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd: entering candidate state: node="Node at f6d2dff6-b296-cafa-6dc0-7072a484f0fd [Candidate]" term=5 +2025-01-28T14:55:40.504Z [ERROR] server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd: failed to make requestVote RPC: target="{Voter server-5a3c7731-9dba-e855-7774-7b5c03278f2b 5a3c7731-9dba-e855-7774-7b5c03278f2b}" error="failed to connect to peer: 5a3c7731-9dba-e855-7774-7b5c03278f2b" term=5 +2025-01-28T14:55:40.527Z [WARN] server-5a3c7731-9dba-e855-7774-7b5c03278f2b: Election timeout reached, restarting election +2025-01-28T14:55:40.528Z [INFO] server-5a3c7731-9dba-e855-7774-7b5c03278f2b: entering candidate state: node="Node at 5a3c7731-9dba-e855-7774-7b5c03278f2b [Candidate]" term=6 +2025-01-28T14:55:40.528Z [ERROR] server-5a3c7731-9dba-e855-7774-7b5c03278f2b: failed to make requestVote RPC: target="{Voter server-f6d2dff6-b296-cafa-6dc0-7072a484f0fd f6d2dff6-b296-cafa-6dc0-7072a484f0fd}" error="failed to connect to peer: f6d2dff6-b296-cafa-6dc0-7072a484f0fd" term=6 +--- PASS: TestRaft_LeaderLeaseExpire (0.63s) === RUN TestRaft_Barrier -2025-01-28T14:48:39.621Z [INFO] server-0e1a995e-8e97-c882-ff10-9cd227dcef60: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0e1a995e-8e97-c882-ff10-9cd227dcef60 Address:0e1a995e-8e97-c882-ff10-9cd227dcef60} {Suffrage:Voter ID:server-d15228fb-5ccc-b586-c2fc-2dd38233c8b3 Address:d15228fb-5ccc-b586-c2fc-2dd38233c8b3} {Suffrage:Voter ID:server-06136c5b-2af1-b22a-6c95-671429b8c757 Address:06136c5b-2af1-b22a-6c95-671429b8c757}]" -2025-01-28T14:48:39.621Z [INFO] server-0e1a995e-8e97-c882-ff10-9cd227dcef60: entering follower state: follower="Node at 0e1a995e-8e97-c882-ff10-9cd227dcef60 [Follower]" leader-address= leader-id= -2025-01-28T14:48:39.622Z [INFO] server-d15228fb-5ccc-b586-c2fc-2dd38233c8b3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0e1a995e-8e97-c882-ff10-9cd227dcef60 Address:0e1a995e-8e97-c882-ff10-9cd227dcef60} {Suffrage:Voter ID:server-d15228fb-5ccc-b586-c2fc-2dd38233c8b3 Address:d15228fb-5ccc-b586-c2fc-2dd38233c8b3} {Suffrage:Voter ID:server-06136c5b-2af1-b22a-6c95-671429b8c757 Address:06136c5b-2af1-b22a-6c95-671429b8c757}]" -2025-01-28T14:48:39.622Z [INFO] server-d15228fb-5ccc-b586-c2fc-2dd38233c8b3: entering follower state: follower="Node at d15228fb-5ccc-b586-c2fc-2dd38233c8b3 [Follower]" leader-address= leader-id= -2025-01-28T14:48:39.622Z [INFO] server-06136c5b-2af1-b22a-6c95-671429b8c757: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0e1a995e-8e97-c882-ff10-9cd227dcef60 Address:0e1a995e-8e97-c882-ff10-9cd227dcef60} {Suffrage:Voter ID:server-d15228fb-5ccc-b586-c2fc-2dd38233c8b3 Address:d15228fb-5ccc-b586-c2fc-2dd38233c8b3} {Suffrage:Voter ID:server-06136c5b-2af1-b22a-6c95-671429b8c757 Address:06136c5b-2af1-b22a-6c95-671429b8c757}]" -2025-01-28T14:48:39.622Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:39.622Z [INFO] server-06136c5b-2af1-b22a-6c95-671429b8c757: entering follower state: follower="Node at 06136c5b-2af1-b22a-6c95-671429b8c757 [Follower]" leader-address= leader-id= -2025-01-28T14:48:39.677Z [WARN] server-d15228fb-5ccc-b586-c2fc-2dd38233c8b3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:39.677Z [INFO] server-d15228fb-5ccc-b586-c2fc-2dd38233c8b3: entering candidate state: node="Node at d15228fb-5ccc-b586-c2fc-2dd38233c8b3 [Candidate]" term=2 -2025-01-28T14:48:39.678Z [INFO] server-d15228fb-5ccc-b586-c2fc-2dd38233c8b3: election won: term=2 tally=2 -2025-01-28T14:48:39.678Z [INFO] server-d15228fb-5ccc-b586-c2fc-2dd38233c8b3: entering leader state: leader="Node at d15228fb-5ccc-b586-c2fc-2dd38233c8b3 [Leader]" -2025-01-28T14:48:39.678Z [INFO] server-d15228fb-5ccc-b586-c2fc-2dd38233c8b3: added peer, starting replication: peer=server-0e1a995e-8e97-c882-ff10-9cd227dcef60 -2025-01-28T14:48:39.678Z [INFO] server-d15228fb-5ccc-b586-c2fc-2dd38233c8b3: added peer, starting replication: peer=server-06136c5b-2af1-b22a-6c95-671429b8c757 -2025-01-28T14:48:39.679Z [INFO] server-d15228fb-5ccc-b586-c2fc-2dd38233c8b3: pipelining replication: peer="{Voter server-06136c5b-2af1-b22a-6c95-671429b8c757 06136c5b-2af1-b22a-6c95-671429b8c757}" -2025-01-28T14:48:39.679Z [INFO] server-d15228fb-5ccc-b586-c2fc-2dd38233c8b3: pipelining replication: peer="{Voter server-0e1a995e-8e97-c882-ff10-9cd227dcef60 0e1a995e-8e97-c882-ff10-9cd227dcef60}" -2025-01-28T14:48:39.784Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:39.679110576 +0000 UTC m=+102.762515934 (1 nodes), 56.106666ms from start of poll, 58.390734ms from cluster start. Timeout at 2025-01-28 14:48:39.784119577 +0000 UTC m=+102.867524268, 105.008334ms after stability -2025-01-28T14:48:39.851Z [INFO] server-d15228fb-5ccc-b586-c2fc-2dd38233c8b3: aborting pipeline replication: peer="{Voter server-06136c5b-2af1-b22a-6c95-671429b8c757 06136c5b-2af1-b22a-6c95-671429b8c757}" -2025-01-28T14:48:39.851Z [INFO] server-d15228fb-5ccc-b586-c2fc-2dd38233c8b3: aborting pipeline replication: peer="{Voter server-0e1a995e-8e97-c882-ff10-9cd227dcef60 0e1a995e-8e97-c882-ff10-9cd227dcef60}" ---- PASS: TestRaft_Barrier (0.24s) +2025-01-28T14:55:40.549Z [INFO] server-d768e263-4d16-2bb6-c0b7-ac3a6f49bd24: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d768e263-4d16-2bb6-c0b7-ac3a6f49bd24 Address:d768e263-4d16-2bb6-c0b7-ac3a6f49bd24} {Suffrage:Voter ID:server-9c1dadd7-1a54-09f6-2eb0-c6149f3c1c28 Address:9c1dadd7-1a54-09f6-2eb0-c6149f3c1c28} {Suffrage:Voter ID:server-40d862e6-1667-eb8c-e8e4-dede8fbdc63f Address:40d862e6-1667-eb8c-e8e4-dede8fbdc63f}]" +2025-01-28T14:55:40.549Z [INFO] server-9c1dadd7-1a54-09f6-2eb0-c6149f3c1c28: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d768e263-4d16-2bb6-c0b7-ac3a6f49bd24 Address:d768e263-4d16-2bb6-c0b7-ac3a6f49bd24} {Suffrage:Voter ID:server-9c1dadd7-1a54-09f6-2eb0-c6149f3c1c28 Address:9c1dadd7-1a54-09f6-2eb0-c6149f3c1c28} {Suffrage:Voter ID:server-40d862e6-1667-eb8c-e8e4-dede8fbdc63f Address:40d862e6-1667-eb8c-e8e4-dede8fbdc63f}]" +2025-01-28T14:55:40.549Z [INFO] server-d768e263-4d16-2bb6-c0b7-ac3a6f49bd24: entering follower state: follower="Node at d768e263-4d16-2bb6-c0b7-ac3a6f49bd24 [Follower]" leader-address= leader-id= +2025-01-28T14:55:40.549Z [INFO] server-9c1dadd7-1a54-09f6-2eb0-c6149f3c1c28: entering follower state: follower="Node at 9c1dadd7-1a54-09f6-2eb0-c6149f3c1c28 [Follower]" leader-address= leader-id= +2025-01-28T14:55:40.549Z [INFO] server-40d862e6-1667-eb8c-e8e4-dede8fbdc63f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d768e263-4d16-2bb6-c0b7-ac3a6f49bd24 Address:d768e263-4d16-2bb6-c0b7-ac3a6f49bd24} {Suffrage:Voter ID:server-9c1dadd7-1a54-09f6-2eb0-c6149f3c1c28 Address:9c1dadd7-1a54-09f6-2eb0-c6149f3c1c28} {Suffrage:Voter ID:server-40d862e6-1667-eb8c-e8e4-dede8fbdc63f Address:40d862e6-1667-eb8c-e8e4-dede8fbdc63f}]" +2025-01-28T14:55:40.549Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:40.549Z [INFO] server-40d862e6-1667-eb8c-e8e4-dede8fbdc63f: entering follower state: follower="Node at 40d862e6-1667-eb8c-e8e4-dede8fbdc63f [Follower]" leader-address= leader-id= +2025-01-28T14:55:40.608Z [WARN] server-9c1dadd7-1a54-09f6-2eb0-c6149f3c1c28: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:40.608Z [INFO] server-9c1dadd7-1a54-09f6-2eb0-c6149f3c1c28: entering candidate state: node="Node at 9c1dadd7-1a54-09f6-2eb0-c6149f3c1c28 [Candidate]" term=2 +2025-01-28T14:55:40.608Z [INFO] server-9c1dadd7-1a54-09f6-2eb0-c6149f3c1c28: election won: term=2 tally=2 +2025-01-28T14:55:40.608Z [INFO] server-9c1dadd7-1a54-09f6-2eb0-c6149f3c1c28: entering leader state: leader="Node at 9c1dadd7-1a54-09f6-2eb0-c6149f3c1c28 [Leader]" +2025-01-28T14:55:40.608Z [INFO] server-9c1dadd7-1a54-09f6-2eb0-c6149f3c1c28: added peer, starting replication: peer=server-d768e263-4d16-2bb6-c0b7-ac3a6f49bd24 +2025-01-28T14:55:40.608Z [INFO] server-9c1dadd7-1a54-09f6-2eb0-c6149f3c1c28: added peer, starting replication: peer=server-40d862e6-1667-eb8c-e8e4-dede8fbdc63f +2025-01-28T14:55:40.608Z [INFO] server-9c1dadd7-1a54-09f6-2eb0-c6149f3c1c28: pipelining replication: peer="{Voter server-d768e263-4d16-2bb6-c0b7-ac3a6f49bd24 d768e263-4d16-2bb6-c0b7-ac3a6f49bd24}" +2025-01-28T14:55:40.608Z [INFO] server-9c1dadd7-1a54-09f6-2eb0-c6149f3c1c28: pipelining replication: peer="{Voter server-40d862e6-1667-eb8c-e8e4-dede8fbdc63f 40d862e6-1667-eb8c-e8e4-dede8fbdc63f}" +2025-01-28T14:55:40.714Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:40.608504372 +0000 UTC m=+126.282994352 (1 nodes), 58.657826ms from start of poll, 59.67533ms from cluster start. Timeout at 2025-01-28 14:55:40.713508212 +0000 UTC m=+126.387998252, 105.0039ms after stability +2025-01-28T14:55:40.733Z [INFO] server-9c1dadd7-1a54-09f6-2eb0-c6149f3c1c28: aborting pipeline replication: peer="{Voter server-40d862e6-1667-eb8c-e8e4-dede8fbdc63f 40d862e6-1667-eb8c-e8e4-dede8fbdc63f}" +2025-01-28T14:55:40.733Z [INFO] server-9c1dadd7-1a54-09f6-2eb0-c6149f3c1c28: aborting pipeline replication: peer="{Voter server-d768e263-4d16-2bb6-c0b7-ac3a6f49bd24 d768e263-4d16-2bb6-c0b7-ac3a6f49bd24}" +2025-01-28T14:55:41.234Z [ERROR] server-9c1dadd7-1a54-09f6-2eb0-c6149f3c1c28: failed to heartbeat to: peer=d768e263-4d16-2bb6-c0b7-ac3a6f49bd24 backoff time=10ms error="command timed out" +--- PASS: TestRaft_Barrier (0.69s) === RUN TestRaft_VerifyLeader -2025-01-28T14:48:39.861Z [INFO] server-8d2aaa6e-a27f-f099-6d7b-5e48d001b477: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8d2aaa6e-a27f-f099-6d7b-5e48d001b477 Address:8d2aaa6e-a27f-f099-6d7b-5e48d001b477} {Suffrage:Voter ID:server-d52a784d-5778-b9dc-aa2d-ecb714aaf07d Address:d52a784d-5778-b9dc-aa2d-ecb714aaf07d} {Suffrage:Voter ID:server-1d6b40aa-55f0-ff68-adb0-84e34ae7c65b Address:1d6b40aa-55f0-ff68-adb0-84e34ae7c65b}]" -2025-01-28T14:48:39.863Z [INFO] server-d52a784d-5778-b9dc-aa2d-ecb714aaf07d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8d2aaa6e-a27f-f099-6d7b-5e48d001b477 Address:8d2aaa6e-a27f-f099-6d7b-5e48d001b477} {Suffrage:Voter ID:server-d52a784d-5778-b9dc-aa2d-ecb714aaf07d Address:d52a784d-5778-b9dc-aa2d-ecb714aaf07d} {Suffrage:Voter ID:server-1d6b40aa-55f0-ff68-adb0-84e34ae7c65b Address:1d6b40aa-55f0-ff68-adb0-84e34ae7c65b}]" -2025-01-28T14:48:39.864Z [INFO] server-8d2aaa6e-a27f-f099-6d7b-5e48d001b477: entering follower state: follower="Node at 8d2aaa6e-a27f-f099-6d7b-5e48d001b477 [Follower]" leader-address= leader-id= -2025-01-28T14:48:39.864Z [INFO] server-1d6b40aa-55f0-ff68-adb0-84e34ae7c65b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8d2aaa6e-a27f-f099-6d7b-5e48d001b477 Address:8d2aaa6e-a27f-f099-6d7b-5e48d001b477} {Suffrage:Voter ID:server-d52a784d-5778-b9dc-aa2d-ecb714aaf07d Address:d52a784d-5778-b9dc-aa2d-ecb714aaf07d} {Suffrage:Voter ID:server-1d6b40aa-55f0-ff68-adb0-84e34ae7c65b Address:1d6b40aa-55f0-ff68-adb0-84e34ae7c65b}]" -2025-01-28T14:48:39.864Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:39.864Z [INFO] server-d52a784d-5778-b9dc-aa2d-ecb714aaf07d: entering follower state: follower="Node at d52a784d-5778-b9dc-aa2d-ecb714aaf07d [Follower]" leader-address= leader-id= -2025-01-28T14:48:39.865Z [INFO] server-1d6b40aa-55f0-ff68-adb0-84e34ae7c65b: entering follower state: follower="Node at 1d6b40aa-55f0-ff68-adb0-84e34ae7c65b [Follower]" leader-address= leader-id= -2025-01-28T14:48:39.921Z [WARN] server-1d6b40aa-55f0-ff68-adb0-84e34ae7c65b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:39.921Z [INFO] server-1d6b40aa-55f0-ff68-adb0-84e34ae7c65b: entering candidate state: node="Node at 1d6b40aa-55f0-ff68-adb0-84e34ae7c65b [Candidate]" term=2 -2025-01-28T14:48:39.922Z [INFO] server-1d6b40aa-55f0-ff68-adb0-84e34ae7c65b: election won: term=2 tally=2 -2025-01-28T14:48:39.922Z [INFO] server-1d6b40aa-55f0-ff68-adb0-84e34ae7c65b: entering leader state: leader="Node at 1d6b40aa-55f0-ff68-adb0-84e34ae7c65b [Leader]" -2025-01-28T14:48:39.922Z [INFO] server-1d6b40aa-55f0-ff68-adb0-84e34ae7c65b: added peer, starting replication: peer=server-8d2aaa6e-a27f-f099-6d7b-5e48d001b477 -2025-01-28T14:48:39.922Z [INFO] server-1d6b40aa-55f0-ff68-adb0-84e34ae7c65b: added peer, starting replication: peer=server-d52a784d-5778-b9dc-aa2d-ecb714aaf07d -2025-01-28T14:48:39.923Z [INFO] server-1d6b40aa-55f0-ff68-adb0-84e34ae7c65b: pipelining replication: peer="{Voter server-8d2aaa6e-a27f-f099-6d7b-5e48d001b477 8d2aaa6e-a27f-f099-6d7b-5e48d001b477}" -2025-01-28T14:48:39.923Z [INFO] server-1d6b40aa-55f0-ff68-adb0-84e34ae7c65b: pipelining replication: peer="{Voter server-d52a784d-5778-b9dc-aa2d-ecb714aaf07d d52a784d-5778-b9dc-aa2d-ecb714aaf07d}" -2025-01-28T14:48:40.028Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:39.923062488 +0000 UTC m=+103.006467179 (1 nodes), 58.035724ms from start of poll, 61.809169ms from cluster start. Timeout at 2025-01-28 14:48:40.028071155 +0000 UTC m=+103.111475846, 105.008667ms after stability -2025-01-28T14:48:40.029Z [INFO] server-1d6b40aa-55f0-ff68-adb0-84e34ae7c65b: aborting pipeline replication: peer="{Voter server-d52a784d-5778-b9dc-aa2d-ecb714aaf07d d52a784d-5778-b9dc-aa2d-ecb714aaf07d}" -2025-01-28T14:48:40.029Z [INFO] server-1d6b40aa-55f0-ff68-adb0-84e34ae7c65b: aborting pipeline replication: peer="{Voter server-8d2aaa6e-a27f-f099-6d7b-5e48d001b477 8d2aaa6e-a27f-f099-6d7b-5e48d001b477}" +2025-01-28T14:55:41.238Z [INFO] server-dfb5e5c1-6f86-2ccc-c6cc-431cbe8f29a8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dfb5e5c1-6f86-2ccc-c6cc-431cbe8f29a8 Address:dfb5e5c1-6f86-2ccc-c6cc-431cbe8f29a8} {Suffrage:Voter ID:server-64c34394-dc1f-2562-d666-122ceb03db6c Address:64c34394-dc1f-2562-d666-122ceb03db6c} {Suffrage:Voter ID:server-c07244a4-770d-3caf-ddab-c27998ead838 Address:c07244a4-770d-3caf-ddab-c27998ead838}]" +2025-01-28T14:55:41.238Z [INFO] server-dfb5e5c1-6f86-2ccc-c6cc-431cbe8f29a8: entering follower state: follower="Node at dfb5e5c1-6f86-2ccc-c6cc-431cbe8f29a8 [Follower]" leader-address= leader-id= +2025-01-28T14:55:41.239Z [INFO] server-64c34394-dc1f-2562-d666-122ceb03db6c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dfb5e5c1-6f86-2ccc-c6cc-431cbe8f29a8 Address:dfb5e5c1-6f86-2ccc-c6cc-431cbe8f29a8} {Suffrage:Voter ID:server-64c34394-dc1f-2562-d666-122ceb03db6c Address:64c34394-dc1f-2562-d666-122ceb03db6c} {Suffrage:Voter ID:server-c07244a4-770d-3caf-ddab-c27998ead838 Address:c07244a4-770d-3caf-ddab-c27998ead838}]" +2025-01-28T14:55:41.239Z [INFO] server-64c34394-dc1f-2562-d666-122ceb03db6c: entering follower state: follower="Node at 64c34394-dc1f-2562-d666-122ceb03db6c [Follower]" leader-address= leader-id= +2025-01-28T14:55:41.239Z [INFO] server-c07244a4-770d-3caf-ddab-c27998ead838: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dfb5e5c1-6f86-2ccc-c6cc-431cbe8f29a8 Address:dfb5e5c1-6f86-2ccc-c6cc-431cbe8f29a8} {Suffrage:Voter ID:server-64c34394-dc1f-2562-d666-122ceb03db6c Address:64c34394-dc1f-2562-d666-122ceb03db6c} {Suffrage:Voter ID:server-c07244a4-770d-3caf-ddab-c27998ead838 Address:c07244a4-770d-3caf-ddab-c27998ead838}]" +2025-01-28T14:55:41.239Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:41.239Z [INFO] server-c07244a4-770d-3caf-ddab-c27998ead838: entering follower state: follower="Node at c07244a4-770d-3caf-ddab-c27998ead838 [Follower]" leader-address= leader-id= +2025-01-28T14:55:41.304Z [WARN] server-c07244a4-770d-3caf-ddab-c27998ead838: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:41.304Z [INFO] server-c07244a4-770d-3caf-ddab-c27998ead838: entering candidate state: node="Node at c07244a4-770d-3caf-ddab-c27998ead838 [Candidate]" term=2 +2025-01-28T14:55:41.304Z [INFO] server-c07244a4-770d-3caf-ddab-c27998ead838: election won: term=2 tally=2 +2025-01-28T14:55:41.304Z [INFO] server-c07244a4-770d-3caf-ddab-c27998ead838: entering leader state: leader="Node at c07244a4-770d-3caf-ddab-c27998ead838 [Leader]" +2025-01-28T14:55:41.304Z [INFO] server-c07244a4-770d-3caf-ddab-c27998ead838: added peer, starting replication: peer=server-dfb5e5c1-6f86-2ccc-c6cc-431cbe8f29a8 +2025-01-28T14:55:41.304Z [INFO] server-c07244a4-770d-3caf-ddab-c27998ead838: added peer, starting replication: peer=server-64c34394-dc1f-2562-d666-122ceb03db6c +2025-01-28T14:55:41.304Z [INFO] server-c07244a4-770d-3caf-ddab-c27998ead838: pipelining replication: peer="{Voter server-dfb5e5c1-6f86-2ccc-c6cc-431cbe8f29a8 dfb5e5c1-6f86-2ccc-c6cc-431cbe8f29a8}" +2025-01-28T14:55:41.304Z [INFO] server-c07244a4-770d-3caf-ddab-c27998ead838: pipelining replication: peer="{Voter server-64c34394-dc1f-2562-d666-122ceb03db6c 64c34394-dc1f-2562-d666-122ceb03db6c}" +2025-01-28T14:55:41.410Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:41.304442 +0000 UTC m=+126.978931980 (1 nodes), 64.849062ms from start of poll, 66.098304ms from cluster start. Timeout at 2025-01-28 14:55:41.409458819 +0000 UTC m=+127.083948779, 105.016799ms after stability +2025-01-28T14:55:41.410Z [INFO] server-c07244a4-770d-3caf-ddab-c27998ead838: aborting pipeline replication: peer="{Voter server-64c34394-dc1f-2562-d666-122ceb03db6c 64c34394-dc1f-2562-d666-122ceb03db6c}" +2025-01-28T14:55:41.410Z [INFO] server-c07244a4-770d-3caf-ddab-c27998ead838: aborting pipeline replication: peer="{Voter server-dfb5e5c1-6f86-2ccc-c6cc-431cbe8f29a8 dfb5e5c1-6f86-2ccc-c6cc-431cbe8f29a8}" --- PASS: TestRaft_VerifyLeader (0.18s) === RUN TestRaft_VerifyLeader_Single -2025-01-28T14:48:40.036Z [INFO] server-efb8567d-ccf9-42a4-7ff0-47c3785ed9ba: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-efb8567d-ccf9-42a4-7ff0-47c3785ed9ba Address:efb8567d-ccf9-42a4-7ff0-47c3785ed9ba}]" -2025-01-28T14:48:40.037Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:40.037Z [INFO] server-efb8567d-ccf9-42a4-7ff0-47c3785ed9ba: entering follower state: follower="Node at efb8567d-ccf9-42a4-7ff0-47c3785ed9ba [Follower]" leader-address= leader-id= -2025-01-28T14:48:40.128Z [WARN] server-efb8567d-ccf9-42a4-7ff0-47c3785ed9ba: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:40.129Z [INFO] server-efb8567d-ccf9-42a4-7ff0-47c3785ed9ba: entering candidate state: node="Node at efb8567d-ccf9-42a4-7ff0-47c3785ed9ba [Candidate]" term=2 -2025-01-28T14:48:40.129Z [INFO] server-efb8567d-ccf9-42a4-7ff0-47c3785ed9ba: election won: term=2 tally=1 -2025-01-28T14:48:40.129Z [INFO] server-efb8567d-ccf9-42a4-7ff0-47c3785ed9ba: entering leader state: leader="Node at efb8567d-ccf9-42a4-7ff0-47c3785ed9ba [Leader]" -2025-01-28T14:48:40.235Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:40.129933298 +0000 UTC m=+103.213338322 (1 nodes), 92.637751ms from start of poll, 93.917122ms from cluster start. Timeout at 2025-01-28 14:48:40.234967633 +0000 UTC m=+103.318372324, 105.034002ms after stability ---- PASS: TestRaft_VerifyLeader_Single (0.20s) +2025-01-28T14:55:41.413Z [INFO] server-08ac384d-0d60-95fc-0e43-6d42ad6aad15: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-08ac384d-0d60-95fc-0e43-6d42ad6aad15 Address:08ac384d-0d60-95fc-0e43-6d42ad6aad15}]" +2025-01-28T14:55:41.413Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:41.414Z [INFO] server-08ac384d-0d60-95fc-0e43-6d42ad6aad15: entering follower state: follower="Node at 08ac384d-0d60-95fc-0e43-6d42ad6aad15 [Follower]" leader-address= leader-id= +2025-01-28T14:55:41.478Z [WARN] server-08ac384d-0d60-95fc-0e43-6d42ad6aad15: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:41.478Z [INFO] server-08ac384d-0d60-95fc-0e43-6d42ad6aad15: entering candidate state: node="Node at 08ac384d-0d60-95fc-0e43-6d42ad6aad15 [Candidate]" term=2 +2025-01-28T14:55:41.478Z [INFO] server-08ac384d-0d60-95fc-0e43-6d42ad6aad15: election won: term=2 tally=1 +2025-01-28T14:55:41.478Z [INFO] server-08ac384d-0d60-95fc-0e43-6d42ad6aad15: entering leader state: leader="Node at 08ac384d-0d60-95fc-0e43-6d42ad6aad15 [Leader]" +2025-01-28T14:55:41.584Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:41.47881831 +0000 UTC m=+127.153308270 (1 nodes), 64.806946ms from start of poll, 65.237326ms from cluster start. Timeout at 2025-01-28 14:55:41.583822189 +0000 UTC m=+127.258312209, 105.003939ms after stability +--- PASS: TestRaft_VerifyLeader_Single (0.17s) === RUN TestRaft_VerifyLeader_Fail -2025-01-28T14:48:40.243Z [INFO] server-15b46014-ada1-d7f7-fd8c-74f125d62b49: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-15b46014-ada1-d7f7-fd8c-74f125d62b49 Address:15b46014-ada1-d7f7-fd8c-74f125d62b49} {Suffrage:Voter ID:server-914e426a-804a-4c61-7a9b-fe9ce496db5b Address:914e426a-804a-4c61-7a9b-fe9ce496db5b}]" -2025-01-28T14:48:40.244Z [INFO] server-914e426a-804a-4c61-7a9b-fe9ce496db5b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-15b46014-ada1-d7f7-fd8c-74f125d62b49 Address:15b46014-ada1-d7f7-fd8c-74f125d62b49} {Suffrage:Voter ID:server-914e426a-804a-4c61-7a9b-fe9ce496db5b Address:914e426a-804a-4c61-7a9b-fe9ce496db5b}]" -2025-01-28T14:48:40.244Z [INFO] server-15b46014-ada1-d7f7-fd8c-74f125d62b49: entering follower state: follower="Node at 15b46014-ada1-d7f7-fd8c-74f125d62b49 [Follower]" leader-address= leader-id= -2025-01-28T14:48:40.244Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:40.244Z [INFO] server-914e426a-804a-4c61-7a9b-fe9ce496db5b: entering follower state: follower="Node at 914e426a-804a-4c61-7a9b-fe9ce496db5b [Follower]" leader-address= leader-id= -2025-01-28T14:48:40.312Z [WARN] server-15b46014-ada1-d7f7-fd8c-74f125d62b49: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:40.312Z [INFO] server-15b46014-ada1-d7f7-fd8c-74f125d62b49: entering candidate state: node="Node at 15b46014-ada1-d7f7-fd8c-74f125d62b49 [Candidate]" term=2 -2025-01-28T14:48:40.312Z [INFO] server-15b46014-ada1-d7f7-fd8c-74f125d62b49: election won: term=2 tally=2 -2025-01-28T14:48:40.312Z [INFO] server-15b46014-ada1-d7f7-fd8c-74f125d62b49: entering leader state: leader="Node at 15b46014-ada1-d7f7-fd8c-74f125d62b49 [Leader]" -2025-01-28T14:48:40.313Z [INFO] server-15b46014-ada1-d7f7-fd8c-74f125d62b49: added peer, starting replication: peer=server-914e426a-804a-4c61-7a9b-fe9ce496db5b -2025-01-28T14:48:40.313Z [INFO] server-15b46014-ada1-d7f7-fd8c-74f125d62b49: pipelining replication: peer="{Voter server-914e426a-804a-4c61-7a9b-fe9ce496db5b 914e426a-804a-4c61-7a9b-fe9ce496db5b}" -2025-01-28T14:48:40.418Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:40.313200741 +0000 UTC m=+103.396605765 (1 nodes), 68.604037ms from start of poll, 70.327089ms from cluster start. Timeout at 2025-01-28 14:48:40.418210407 +0000 UTC m=+103.501615098, 105.009333ms after stability -2025-01-28T14:48:40.418Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:48:40.524Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:48:40.418696874 +0000 UTC m=+103.502101565 (1 nodes), 5.667µs from start of poll, 175.822889ms from cluster start. Timeout at 2025-01-28 14:48:40.524109552 +0000 UTC m=+103.607514577, 105.413012ms after stability -2025-01-28T14:48:40.531Z [ERROR] server-15b46014-ada1-d7f7-fd8c-74f125d62b49: peer has newer term, stopping replication: peer="{Voter server-914e426a-804a-4c61-7a9b-fe9ce496db5b 914e426a-804a-4c61-7a9b-fe9ce496db5b}" -2025-01-28T14:48:40.532Z [INFO] server-15b46014-ada1-d7f7-fd8c-74f125d62b49: aborting pipeline replication: peer="{Voter server-914e426a-804a-4c61-7a9b-fe9ce496db5b 914e426a-804a-4c61-7a9b-fe9ce496db5b}" -2025-01-28T14:48:40.532Z [INFO] server-15b46014-ada1-d7f7-fd8c-74f125d62b49: entering follower state: follower="Node at 15b46014-ada1-d7f7-fd8c-74f125d62b49 [Follower]" leader-address= leader-id= +2025-01-28T14:55:41.587Z [INFO] server-62c70e57-c45c-4eee-17fd-27d5920c3174: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-62c70e57-c45c-4eee-17fd-27d5920c3174 Address:62c70e57-c45c-4eee-17fd-27d5920c3174} {Suffrage:Voter ID:server-3b9817b0-8e76-ff5f-db22-edc543bd75fa Address:3b9817b0-8e76-ff5f-db22-edc543bd75fa}]" +2025-01-28T14:55:41.587Z [INFO] server-3b9817b0-8e76-ff5f-db22-edc543bd75fa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-62c70e57-c45c-4eee-17fd-27d5920c3174 Address:62c70e57-c45c-4eee-17fd-27d5920c3174} {Suffrage:Voter ID:server-3b9817b0-8e76-ff5f-db22-edc543bd75fa Address:3b9817b0-8e76-ff5f-db22-edc543bd75fa}]" +2025-01-28T14:55:41.587Z [INFO] server-62c70e57-c45c-4eee-17fd-27d5920c3174: entering follower state: follower="Node at 62c70e57-c45c-4eee-17fd-27d5920c3174 [Follower]" leader-address= leader-id= +2025-01-28T14:55:41.587Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:41.587Z [INFO] server-3b9817b0-8e76-ff5f-db22-edc543bd75fa: entering follower state: follower="Node at 3b9817b0-8e76-ff5f-db22-edc543bd75fa [Follower]" leader-address= leader-id= +2025-01-28T14:55:41.670Z [WARN] server-62c70e57-c45c-4eee-17fd-27d5920c3174: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:41.670Z [INFO] server-62c70e57-c45c-4eee-17fd-27d5920c3174: entering candidate state: node="Node at 62c70e57-c45c-4eee-17fd-27d5920c3174 [Candidate]" term=2 +2025-01-28T14:55:41.670Z [INFO] server-62c70e57-c45c-4eee-17fd-27d5920c3174: election won: term=2 tally=2 +2025-01-28T14:55:41.670Z [INFO] server-62c70e57-c45c-4eee-17fd-27d5920c3174: entering leader state: leader="Node at 62c70e57-c45c-4eee-17fd-27d5920c3174 [Leader]" +2025-01-28T14:55:41.670Z [INFO] server-62c70e57-c45c-4eee-17fd-27d5920c3174: added peer, starting replication: peer=server-3b9817b0-8e76-ff5f-db22-edc543bd75fa +2025-01-28T14:55:41.670Z [INFO] server-62c70e57-c45c-4eee-17fd-27d5920c3174: pipelining replication: peer="{Voter server-3b9817b0-8e76-ff5f-db22-edc543bd75fa 3b9817b0-8e76-ff5f-db22-edc543bd75fa}" +2025-01-28T14:55:41.776Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:41.67068655 +0000 UTC m=+127.345176530 (1 nodes), 82.981992ms from start of poll, 83.699304ms from cluster start. Timeout at 2025-01-28 14:55:41.775690869 +0000 UTC m=+127.450180909, 105.004379ms after stability +2025-01-28T14:55:41.776Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:55:41.881Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:55:41.776399558 +0000 UTC m=+127.450889538 (1 nodes), 1.48µs from start of poll, 189.412312ms from cluster start. Timeout at 2025-01-28 14:55:41.881566782 +0000 UTC m=+127.556056742, 105.167204ms after stability +2025-01-28T14:55:41.883Z [ERROR] server-62c70e57-c45c-4eee-17fd-27d5920c3174: peer has newer term, stopping replication: peer="{Voter server-3b9817b0-8e76-ff5f-db22-edc543bd75fa 3b9817b0-8e76-ff5f-db22-edc543bd75fa}" +2025-01-28T14:55:41.883Z [INFO] server-62c70e57-c45c-4eee-17fd-27d5920c3174: aborting pipeline replication: peer="{Voter server-3b9817b0-8e76-ff5f-db22-edc543bd75fa 3b9817b0-8e76-ff5f-db22-edc543bd75fa}" +2025-01-28T14:55:41.883Z [INFO] server-62c70e57-c45c-4eee-17fd-27d5920c3174: entering follower state: follower="Node at 62c70e57-c45c-4eee-17fd-27d5920c3174 [Follower]" leader-address= leader-id= --- PASS: TestRaft_VerifyLeader_Fail (0.30s) === RUN TestRaft_VerifyLeader_PartialConnect -2025-01-28T14:48:40.542Z [INFO] server-0a048dae-4327-c16e-7a37-ed9e2e73814f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0a048dae-4327-c16e-7a37-ed9e2e73814f Address:0a048dae-4327-c16e-7a37-ed9e2e73814f} {Suffrage:Voter ID:server-fafe24b2-141e-b55c-c60b-afa8ac735a8f Address:fafe24b2-141e-b55c-c60b-afa8ac735a8f} {Suffrage:Voter ID:server-a215b2ae-1b73-b39c-cb9b-1774439b2582 Address:a215b2ae-1b73-b39c-cb9b-1774439b2582}]" -2025-01-28T14:48:40.542Z [INFO] server-0a048dae-4327-c16e-7a37-ed9e2e73814f: entering follower state: follower="Node at 0a048dae-4327-c16e-7a37-ed9e2e73814f [Follower]" leader-address= leader-id= -2025-01-28T14:48:40.543Z [INFO] server-fafe24b2-141e-b55c-c60b-afa8ac735a8f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0a048dae-4327-c16e-7a37-ed9e2e73814f Address:0a048dae-4327-c16e-7a37-ed9e2e73814f} {Suffrage:Voter ID:server-fafe24b2-141e-b55c-c60b-afa8ac735a8f Address:fafe24b2-141e-b55c-c60b-afa8ac735a8f} {Suffrage:Voter ID:server-a215b2ae-1b73-b39c-cb9b-1774439b2582 Address:a215b2ae-1b73-b39c-cb9b-1774439b2582}]" -2025-01-28T14:48:40.543Z [INFO] server-fafe24b2-141e-b55c-c60b-afa8ac735a8f: entering follower state: follower="Node at fafe24b2-141e-b55c-c60b-afa8ac735a8f [Follower]" leader-address= leader-id= -2025-01-28T14:48:40.544Z [INFO] server-a215b2ae-1b73-b39c-cb9b-1774439b2582: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0a048dae-4327-c16e-7a37-ed9e2e73814f Address:0a048dae-4327-c16e-7a37-ed9e2e73814f} {Suffrage:Voter ID:server-fafe24b2-141e-b55c-c60b-afa8ac735a8f Address:fafe24b2-141e-b55c-c60b-afa8ac735a8f} {Suffrage:Voter ID:server-a215b2ae-1b73-b39c-cb9b-1774439b2582 Address:a215b2ae-1b73-b39c-cb9b-1774439b2582}]" -2025-01-28T14:48:40.544Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:40.544Z [INFO] server-a215b2ae-1b73-b39c-cb9b-1774439b2582: entering follower state: follower="Node at a215b2ae-1b73-b39c-cb9b-1774439b2582 [Follower]" leader-address= leader-id= -2025-01-28T14:48:40.600Z [WARN] server-0a048dae-4327-c16e-7a37-ed9e2e73814f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:40.600Z [INFO] server-0a048dae-4327-c16e-7a37-ed9e2e73814f: entering candidate state: node="Node at 0a048dae-4327-c16e-7a37-ed9e2e73814f [Candidate]" term=2 -2025-01-28T14:48:40.601Z [INFO] server-0a048dae-4327-c16e-7a37-ed9e2e73814f: election won: term=2 tally=2 -2025-01-28T14:48:40.601Z [INFO] server-0a048dae-4327-c16e-7a37-ed9e2e73814f: entering leader state: leader="Node at 0a048dae-4327-c16e-7a37-ed9e2e73814f [Leader]" -2025-01-28T14:48:40.601Z [INFO] server-0a048dae-4327-c16e-7a37-ed9e2e73814f: added peer, starting replication: peer=server-fafe24b2-141e-b55c-c60b-afa8ac735a8f -2025-01-28T14:48:40.601Z [INFO] server-0a048dae-4327-c16e-7a37-ed9e2e73814f: added peer, starting replication: peer=server-a215b2ae-1b73-b39c-cb9b-1774439b2582 -2025-01-28T14:48:40.602Z [INFO] server-0a048dae-4327-c16e-7a37-ed9e2e73814f: pipelining replication: peer="{Voter server-fafe24b2-141e-b55c-c60b-afa8ac735a8f fafe24b2-141e-b55c-c60b-afa8ac735a8f}" -2025-01-28T14:48:40.602Z [INFO] server-0a048dae-4327-c16e-7a37-ed9e2e73814f: pipelining replication: peer="{Voter server-a215b2ae-1b73-b39c-cb9b-1774439b2582 a215b2ae-1b73-b39c-cb9b-1774439b2582}" -2025-01-28T14:48:40.707Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:40.601459634 +0000 UTC m=+103.684864659 (1 nodes), 56.759019ms from start of poll, 59.547435ms from cluster start. Timeout at 2025-01-28 14:48:40.706471635 +0000 UTC m=+103.789876659, 105.012ms after stability -2025-01-28T14:48:40.722Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:48:40.828Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:48:40.722617899 +0000 UTC m=+103.806022590 (2 nodes), 8.667µs from start of poll, 180.705366ms from cluster start. Timeout at 2025-01-28 14:48:40.828119248 +0000 UTC m=+103.911523939, 105.501349ms after stability - raft_test.go:1968: [INFO] Disconnecting Node at fafe24b2-141e-b55c-c60b-afa8ac735a8f [Follower] -2025-01-28T14:48:40.829Z [ERROR] server-0a048dae-4327-c16e-7a37-ed9e2e73814f: failed to heartbeat to: peer=fafe24b2-141e-b55c-c60b-afa8ac735a8f backoff time=10ms error="failed to connect to peer: fafe24b2-141e-b55c-c60b-afa8ac735a8f" -2025-01-28T14:48:40.829Z [INFO] server-0a048dae-4327-c16e-7a37-ed9e2e73814f: aborting pipeline replication: peer="{Voter server-a215b2ae-1b73-b39c-cb9b-1774439b2582 a215b2ae-1b73-b39c-cb9b-1774439b2582}" -2025-01-28T14:48:40.829Z [INFO] server-0a048dae-4327-c16e-7a37-ed9e2e73814f: aborting pipeline replication: peer="{Voter server-fafe24b2-141e-b55c-c60b-afa8ac735a8f fafe24b2-141e-b55c-c60b-afa8ac735a8f}" ---- PASS: TestRaft_VerifyLeader_PartialConnect (0.30s) +2025-01-28T14:55:41.886Z [INFO] server-55b86cf6-6a23-3b03-15df-f5aefa95ce66: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-55b86cf6-6a23-3b03-15df-f5aefa95ce66 Address:55b86cf6-6a23-3b03-15df-f5aefa95ce66} {Suffrage:Voter ID:server-27655c63-2055-a7b1-0052-2fbdd5ead241 Address:27655c63-2055-a7b1-0052-2fbdd5ead241} {Suffrage:Voter ID:server-b981a2b9-bc07-43c9-cb5e-0ce94217a371 Address:b981a2b9-bc07-43c9-cb5e-0ce94217a371}]" +2025-01-28T14:55:41.886Z [INFO] server-27655c63-2055-a7b1-0052-2fbdd5ead241: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-55b86cf6-6a23-3b03-15df-f5aefa95ce66 Address:55b86cf6-6a23-3b03-15df-f5aefa95ce66} {Suffrage:Voter ID:server-27655c63-2055-a7b1-0052-2fbdd5ead241 Address:27655c63-2055-a7b1-0052-2fbdd5ead241} {Suffrage:Voter ID:server-b981a2b9-bc07-43c9-cb5e-0ce94217a371 Address:b981a2b9-bc07-43c9-cb5e-0ce94217a371}]" +2025-01-28T14:55:41.886Z [INFO] server-55b86cf6-6a23-3b03-15df-f5aefa95ce66: entering follower state: follower="Node at 55b86cf6-6a23-3b03-15df-f5aefa95ce66 [Follower]" leader-address= leader-id= +2025-01-28T14:55:41.886Z [INFO] server-27655c63-2055-a7b1-0052-2fbdd5ead241: entering follower state: follower="Node at 27655c63-2055-a7b1-0052-2fbdd5ead241 [Follower]" leader-address= leader-id= +2025-01-28T14:55:41.887Z [INFO] server-b981a2b9-bc07-43c9-cb5e-0ce94217a371: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-55b86cf6-6a23-3b03-15df-f5aefa95ce66 Address:55b86cf6-6a23-3b03-15df-f5aefa95ce66} {Suffrage:Voter ID:server-27655c63-2055-a7b1-0052-2fbdd5ead241 Address:27655c63-2055-a7b1-0052-2fbdd5ead241} {Suffrage:Voter ID:server-b981a2b9-bc07-43c9-cb5e-0ce94217a371 Address:b981a2b9-bc07-43c9-cb5e-0ce94217a371}]" +2025-01-28T14:55:41.887Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:41.887Z [INFO] server-b981a2b9-bc07-43c9-cb5e-0ce94217a371: entering follower state: follower="Node at b981a2b9-bc07-43c9-cb5e-0ce94217a371 [Follower]" leader-address= leader-id= +2025-01-28T14:55:41.952Z [WARN] server-b981a2b9-bc07-43c9-cb5e-0ce94217a371: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:41.953Z [INFO] server-b981a2b9-bc07-43c9-cb5e-0ce94217a371: entering candidate state: node="Node at b981a2b9-bc07-43c9-cb5e-0ce94217a371 [Candidate]" term=2 +2025-01-28T14:55:41.953Z [INFO] server-b981a2b9-bc07-43c9-cb5e-0ce94217a371: election won: term=2 tally=2 +2025-01-28T14:55:41.953Z [INFO] server-b981a2b9-bc07-43c9-cb5e-0ce94217a371: entering leader state: leader="Node at b981a2b9-bc07-43c9-cb5e-0ce94217a371 [Leader]" +2025-01-28T14:55:41.953Z [INFO] server-b981a2b9-bc07-43c9-cb5e-0ce94217a371: added peer, starting replication: peer=server-55b86cf6-6a23-3b03-15df-f5aefa95ce66 +2025-01-28T14:55:41.953Z [INFO] server-b981a2b9-bc07-43c9-cb5e-0ce94217a371: added peer, starting replication: peer=server-27655c63-2055-a7b1-0052-2fbdd5ead241 +2025-01-28T14:55:41.953Z [INFO] server-b981a2b9-bc07-43c9-cb5e-0ce94217a371: pipelining replication: peer="{Voter server-55b86cf6-6a23-3b03-15df-f5aefa95ce66 55b86cf6-6a23-3b03-15df-f5aefa95ce66}" +2025-01-28T14:55:41.953Z [INFO] server-b981a2b9-bc07-43c9-cb5e-0ce94217a371: pipelining replication: peer="{Voter server-27655c63-2055-a7b1-0052-2fbdd5ead241 27655c63-2055-a7b1-0052-2fbdd5ead241}" +2025-01-28T14:55:42.059Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:41.953311948 +0000 UTC m=+127.627801928 (1 nodes), 66.134201ms from start of poll, 67.060413ms from cluster start. Timeout at 2025-01-28 14:55:42.058319647 +0000 UTC m=+127.732809627, 105.007699ms after stability +2025-01-28T14:55:42.065Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:55:42.170Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:55:42.065298884 +0000 UTC m=+127.739788864 (2 nodes), 2.08µs from start of poll, 179.047349ms from cluster start. Timeout at 2025-01-28 14:55:42.170494045 +0000 UTC m=+127.844984025, 105.195161ms after stability + raft_test.go:1968: [INFO] Disconnecting Node at 55b86cf6-6a23-3b03-15df-f5aefa95ce66 [Follower] +2025-01-28T14:55:42.171Z [ERROR] server-b981a2b9-bc07-43c9-cb5e-0ce94217a371: failed to heartbeat to: peer=55b86cf6-6a23-3b03-15df-f5aefa95ce66 backoff time=10ms error="failed to connect to peer: 55b86cf6-6a23-3b03-15df-f5aefa95ce66" +2025-01-28T14:55:42.171Z [INFO] server-b981a2b9-bc07-43c9-cb5e-0ce94217a371: aborting pipeline replication: peer="{Voter server-27655c63-2055-a7b1-0052-2fbdd5ead241 27655c63-2055-a7b1-0052-2fbdd5ead241}" +2025-01-28T14:55:42.171Z [INFO] server-b981a2b9-bc07-43c9-cb5e-0ce94217a371: aborting pipeline replication: peer="{Voter server-55b86cf6-6a23-3b03-15df-f5aefa95ce66 55b86cf6-6a23-3b03-15df-f5aefa95ce66}" +--- PASS: TestRaft_VerifyLeader_PartialConnect (0.29s) === RUN TestRaft_NotifyCh -2025-01-28T14:48:40.836Z [INFO] server-eba94215-cb5f-1dfa-936d-423dc8ad1c7f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eba94215-cb5f-1dfa-936d-423dc8ad1c7f Address:eba94215-cb5f-1dfa-936d-423dc8ad1c7f}]" -2025-01-28T14:48:40.836Z [INFO] server-eba94215-cb5f-1dfa-936d-423dc8ad1c7f: entering follower state: follower="Node at eba94215-cb5f-1dfa-936d-423dc8ad1c7f [Follower]" leader-address= leader-id= -2025-01-28T14:48:40.922Z [WARN] server-eba94215-cb5f-1dfa-936d-423dc8ad1c7f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:40.922Z [INFO] server-eba94215-cb5f-1dfa-936d-423dc8ad1c7f: entering candidate state: node="Node at eba94215-cb5f-1dfa-936d-423dc8ad1c7f [Candidate]" term=2 -2025-01-28T14:48:40.922Z [INFO] server-eba94215-cb5f-1dfa-936d-423dc8ad1c7f: election won: term=2 tally=1 -2025-01-28T14:48:40.922Z [INFO] server-eba94215-cb5f-1dfa-936d-423dc8ad1c7f: entering leader state: leader="Node at eba94215-cb5f-1dfa-936d-423dc8ad1c7f [Leader]" +2025-01-28T14:55:42.173Z [INFO] server-80b55dfb-55f3-519c-4e51-67d2a95bc98b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-80b55dfb-55f3-519c-4e51-67d2a95bc98b Address:80b55dfb-55f3-519c-4e51-67d2a95bc98b}]" +2025-01-28T14:55:42.173Z [INFO] server-80b55dfb-55f3-519c-4e51-67d2a95bc98b: entering follower state: follower="Node at 80b55dfb-55f3-519c-4e51-67d2a95bc98b [Follower]" leader-address= leader-id= +2025-01-28T14:55:42.258Z [WARN] server-80b55dfb-55f3-519c-4e51-67d2a95bc98b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:42.258Z [INFO] server-80b55dfb-55f3-519c-4e51-67d2a95bc98b: entering candidate state: node="Node at 80b55dfb-55f3-519c-4e51-67d2a95bc98b [Candidate]" term=2 +2025-01-28T14:55:42.258Z [INFO] server-80b55dfb-55f3-519c-4e51-67d2a95bc98b: election won: term=2 tally=1 +2025-01-28T14:55:42.258Z [INFO] server-80b55dfb-55f3-519c-4e51-67d2a95bc98b: entering leader state: leader="Node at 80b55dfb-55f3-519c-4e51-67d2a95bc98b [Leader]" --- PASS: TestRaft_NotifyCh (0.09s) === RUN TestRaft_AppendEntry -2025-01-28T14:48:40.929Z [INFO] server-1cb8c99c-787c-dfc2-f412-1a0e37c0ac84: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1cb8c99c-787c-dfc2-f412-1a0e37c0ac84 Address:1cb8c99c-787c-dfc2-f412-1a0e37c0ac84} {Suffrage:Voter ID:server-5a506305-9127-afcd-5100-8c0c2326fd47 Address:5a506305-9127-afcd-5100-8c0c2326fd47} {Suffrage:Voter ID:server-71610a26-48b6-e667-e077-ee4b48341756 Address:71610a26-48b6-e667-e077-ee4b48341756}]" -2025-01-28T14:48:40.930Z [INFO] server-5a506305-9127-afcd-5100-8c0c2326fd47: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1cb8c99c-787c-dfc2-f412-1a0e37c0ac84 Address:1cb8c99c-787c-dfc2-f412-1a0e37c0ac84} {Suffrage:Voter ID:server-5a506305-9127-afcd-5100-8c0c2326fd47 Address:5a506305-9127-afcd-5100-8c0c2326fd47} {Suffrage:Voter ID:server-71610a26-48b6-e667-e077-ee4b48341756 Address:71610a26-48b6-e667-e077-ee4b48341756}]" -2025-01-28T14:48:40.931Z [INFO] server-71610a26-48b6-e667-e077-ee4b48341756: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1cb8c99c-787c-dfc2-f412-1a0e37c0ac84 Address:1cb8c99c-787c-dfc2-f412-1a0e37c0ac84} {Suffrage:Voter ID:server-5a506305-9127-afcd-5100-8c0c2326fd47 Address:5a506305-9127-afcd-5100-8c0c2326fd47} {Suffrage:Voter ID:server-71610a26-48b6-e667-e077-ee4b48341756 Address:71610a26-48b6-e667-e077-ee4b48341756}]" -2025-01-28T14:48:40.931Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:48:40.931Z [INFO] server-5a506305-9127-afcd-5100-8c0c2326fd47: entering follower state: follower="Node at 5a506305-9127-afcd-5100-8c0c2326fd47 [Follower]" leader-address= leader-id= -2025-01-28T14:48:40.932Z [INFO] server-71610a26-48b6-e667-e077-ee4b48341756: entering follower state: follower="Node at 71610a26-48b6-e667-e077-ee4b48341756 [Follower]" leader-address= leader-id= -2025-01-28T14:48:40.932Z [INFO] server-1cb8c99c-787c-dfc2-f412-1a0e37c0ac84: entering follower state: follower="Node at 1cb8c99c-787c-dfc2-f412-1a0e37c0ac84 [Follower]" leader-address= leader-id= -2025-01-28T14:48:40.984Z [WARN] server-1cb8c99c-787c-dfc2-f412-1a0e37c0ac84: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:40.984Z [INFO] server-1cb8c99c-787c-dfc2-f412-1a0e37c0ac84: entering candidate state: node="Node at 1cb8c99c-787c-dfc2-f412-1a0e37c0ac84 [Candidate]" term=2 -2025-01-28T14:48:40.984Z [INFO] server-1cb8c99c-787c-dfc2-f412-1a0e37c0ac84: election won: term=2 tally=2 -2025-01-28T14:48:40.984Z [INFO] server-1cb8c99c-787c-dfc2-f412-1a0e37c0ac84: entering leader state: leader="Node at 1cb8c99c-787c-dfc2-f412-1a0e37c0ac84 [Leader]" -2025-01-28T14:48:40.985Z [INFO] server-1cb8c99c-787c-dfc2-f412-1a0e37c0ac84: added peer, starting replication: peer=server-5a506305-9127-afcd-5100-8c0c2326fd47 -2025-01-28T14:48:40.985Z [INFO] server-1cb8c99c-787c-dfc2-f412-1a0e37c0ac84: added peer, starting replication: peer=server-71610a26-48b6-e667-e077-ee4b48341756 -2025-01-28T14:48:40.985Z [WARN] server-71610a26-48b6-e667-e077-ee4b48341756: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:40.985Z [INFO] server-71610a26-48b6-e667-e077-ee4b48341756: entering candidate state: node="Node at 71610a26-48b6-e667-e077-ee4b48341756 [Candidate]" term=2 -2025-01-28T14:48:40.985Z [INFO] server-71610a26-48b6-e667-e077-ee4b48341756: duplicate requestVote for same term: term=2 -2025-01-28T14:48:40.985Z [WARN] server-1cb8c99c-787c-dfc2-f412-1a0e37c0ac84: rejecting vote request since we have a leader: from=71610a26-48b6-e667-e077-ee4b48341756 leader=1cb8c99c-787c-dfc2-f412-1a0e37c0ac84 leader-id=server-1cb8c99c-787c-dfc2-f412-1a0e37c0ac84 -2025-01-28T14:48:40.985Z [WARN] server-5a506305-9127-afcd-5100-8c0c2326fd47: rejecting vote request since we have a leader: from=71610a26-48b6-e667-e077-ee4b48341756 leader=1cb8c99c-787c-dfc2-f412-1a0e37c0ac84 leader-id=server-1cb8c99c-787c-dfc2-f412-1a0e37c0ac84 -2025-01-28T14:48:40.986Z [INFO] server-71610a26-48b6-e667-e077-ee4b48341756: entering follower state: follower="Node at 71610a26-48b6-e667-e077-ee4b48341756 [Follower]" leader-address=1cb8c99c-787c-dfc2-f412-1a0e37c0ac84 leader-id=server-1cb8c99c-787c-dfc2-f412-1a0e37c0ac84 -2025-01-28T14:48:40.986Z [INFO] server-1cb8c99c-787c-dfc2-f412-1a0e37c0ac84: pipelining replication: peer="{Voter server-5a506305-9127-afcd-5100-8c0c2326fd47 5a506305-9127-afcd-5100-8c0c2326fd47}" -2025-01-28T14:48:40.986Z [INFO] server-1cb8c99c-787c-dfc2-f412-1a0e37c0ac84: pipelining replication: peer="{Voter server-71610a26-48b6-e667-e077-ee4b48341756 71610a26-48b6-e667-e077-ee4b48341756}" -2025-01-28T14:48:41.091Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:48:40.986386399 +0000 UTC m=+104.069791424 (2 nodes), 54.65329ms from start of poll, 57.425039ms from cluster start. Timeout at 2025-01-28 14:48:41.0914434 +0000 UTC m=+104.174848091, 105.056667ms after stability -2025-01-28T14:48:41.091Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:41.204Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:41.092046537 +0000 UTC m=+104.175451228 (1 nodes), 6.667µs from start of poll, 163.084843ms from cluster start. Timeout at 2025-01-28 14:48:41.197716223 +0000 UTC m=+104.281120914, 105.669686ms after stability -2025-01-28T14:48:41.204Z [WARN] server-5a506305-9127-afcd-5100-8c0c2326fd47: clearing log suffix: from=1 to=2 -2025-01-28T14:48:41.204Z [WARN] server-5a506305-9127-afcd-5100-8c0c2326fd47: skipping application of old log: index=1 -2025-01-28T14:48:41.204Z [WARN] server-5a506305-9127-afcd-5100-8c0c2326fd47: skipping application of old log: index=1 -2025-01-28T14:48:41.207Z [INFO] server-1cb8c99c-787c-dfc2-f412-1a0e37c0ac84: aborting pipeline replication: peer="{Voter server-71610a26-48b6-e667-e077-ee4b48341756 71610a26-48b6-e667-e077-ee4b48341756}" -2025-01-28T14:48:41.207Z [INFO] server-1cb8c99c-787c-dfc2-f412-1a0e37c0ac84: aborting pipeline replication: peer="{Voter server-5a506305-9127-afcd-5100-8c0c2326fd47 5a506305-9127-afcd-5100-8c0c2326fd47}" ---- PASS: TestRaft_AppendEntry (0.29s) +2025-01-28T14:55:42.263Z [INFO] server-676b8923-473c-ba66-a195-c005c4ff1866: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-676b8923-473c-ba66-a195-c005c4ff1866 Address:676b8923-473c-ba66-a195-c005c4ff1866} {Suffrage:Voter ID:server-191c856e-43d0-57e6-ac74-7bd8b76f4125 Address:191c856e-43d0-57e6-ac74-7bd8b76f4125} {Suffrage:Voter ID:server-0a1710f5-c74f-de9f-75d6-2fed7b785acb Address:0a1710f5-c74f-de9f-75d6-2fed7b785acb}]" +2025-01-28T14:55:42.263Z [INFO] server-191c856e-43d0-57e6-ac74-7bd8b76f4125: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-676b8923-473c-ba66-a195-c005c4ff1866 Address:676b8923-473c-ba66-a195-c005c4ff1866} {Suffrage:Voter ID:server-191c856e-43d0-57e6-ac74-7bd8b76f4125 Address:191c856e-43d0-57e6-ac74-7bd8b76f4125} {Suffrage:Voter ID:server-0a1710f5-c74f-de9f-75d6-2fed7b785acb Address:0a1710f5-c74f-de9f-75d6-2fed7b785acb}]" +2025-01-28T14:55:42.264Z [INFO] server-0a1710f5-c74f-de9f-75d6-2fed7b785acb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-676b8923-473c-ba66-a195-c005c4ff1866 Address:676b8923-473c-ba66-a195-c005c4ff1866} {Suffrage:Voter ID:server-191c856e-43d0-57e6-ac74-7bd8b76f4125 Address:191c856e-43d0-57e6-ac74-7bd8b76f4125} {Suffrage:Voter ID:server-0a1710f5-c74f-de9f-75d6-2fed7b785acb Address:0a1710f5-c74f-de9f-75d6-2fed7b785acb}]" +2025-01-28T14:55:42.264Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:55:42.265Z [INFO] server-676b8923-473c-ba66-a195-c005c4ff1866: entering follower state: follower="Node at 676b8923-473c-ba66-a195-c005c4ff1866 [Follower]" leader-address= leader-id= +2025-01-28T14:55:42.266Z [INFO] server-191c856e-43d0-57e6-ac74-7bd8b76f4125: entering follower state: follower="Node at 191c856e-43d0-57e6-ac74-7bd8b76f4125 [Follower]" leader-address= leader-id= +2025-01-28T14:55:42.266Z [INFO] server-0a1710f5-c74f-de9f-75d6-2fed7b785acb: entering follower state: follower="Node at 0a1710f5-c74f-de9f-75d6-2fed7b785acb [Follower]" leader-address= leader-id= +2025-01-28T14:55:42.317Z [WARN] server-0a1710f5-c74f-de9f-75d6-2fed7b785acb: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:42.317Z [INFO] server-0a1710f5-c74f-de9f-75d6-2fed7b785acb: entering candidate state: node="Node at 0a1710f5-c74f-de9f-75d6-2fed7b785acb [Candidate]" term=2 +2025-01-28T14:55:42.317Z [INFO] server-0a1710f5-c74f-de9f-75d6-2fed7b785acb: election won: term=2 tally=2 +2025-01-28T14:55:42.318Z [INFO] server-0a1710f5-c74f-de9f-75d6-2fed7b785acb: entering leader state: leader="Node at 0a1710f5-c74f-de9f-75d6-2fed7b785acb [Leader]" +2025-01-28T14:55:42.318Z [INFO] server-0a1710f5-c74f-de9f-75d6-2fed7b785acb: added peer, starting replication: peer=server-676b8923-473c-ba66-a195-c005c4ff1866 +2025-01-28T14:55:42.318Z [INFO] server-0a1710f5-c74f-de9f-75d6-2fed7b785acb: added peer, starting replication: peer=server-191c856e-43d0-57e6-ac74-7bd8b76f4125 +2025-01-28T14:55:42.318Z [INFO] server-0a1710f5-c74f-de9f-75d6-2fed7b785acb: pipelining replication: peer="{Voter server-191c856e-43d0-57e6-ac74-7bd8b76f4125 191c856e-43d0-57e6-ac74-7bd8b76f4125}" +2025-01-28T14:55:42.318Z [INFO] server-0a1710f5-c74f-de9f-75d6-2fed7b785acb: pipelining replication: peer="{Voter server-676b8923-473c-ba66-a195-c005c4ff1866 676b8923-473c-ba66-a195-c005c4ff1866}" +2025-01-28T14:55:42.424Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:55:42.318172449 +0000 UTC m=+127.992662429 (2 nodes), 53.475535ms from start of poll, 55.390355ms from cluster start. Timeout at 2025-01-28 14:55:42.423178008 +0000 UTC m=+128.097668088, 105.005659ms after stability +2025-01-28T14:55:42.424Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:42.530Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:42.424440564 +0000 UTC m=+128.098930544 (1 nodes), 2.5µs from start of poll, 161.65847ms from cluster start. Timeout at 2025-01-28 14:55:42.529610948 +0000 UTC m=+128.204100908, 105.170364ms after stability +2025-01-28T14:55:42.530Z [WARN] server-676b8923-473c-ba66-a195-c005c4ff1866: clearing log suffix: from=1 to=2 +2025-01-28T14:55:42.530Z [WARN] server-676b8923-473c-ba66-a195-c005c4ff1866: skipping application of old log: index=1 +2025-01-28T14:55:42.531Z [WARN] server-676b8923-473c-ba66-a195-c005c4ff1866: skipping application of old log: index=1 +2025-01-28T14:55:42.531Z [INFO] server-0a1710f5-c74f-de9f-75d6-2fed7b785acb: aborting pipeline replication: peer="{Voter server-191c856e-43d0-57e6-ac74-7bd8b76f4125 191c856e-43d0-57e6-ac74-7bd8b76f4125}" +2025-01-28T14:55:42.531Z [INFO] server-0a1710f5-c74f-de9f-75d6-2fed7b785acb: aborting pipeline replication: peer="{Voter server-676b8923-473c-ba66-a195-c005c4ff1866 676b8923-473c-ba66-a195-c005c4ff1866}" +--- PASS: TestRaft_AppendEntry (0.27s) === RUN TestRaft_VotingGrant_WhenLeaderAvailable -2025-01-28T14:48:41.219Z [INFO] server-a6cb8458-a62f-6562-de79-96841dc92ce3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a6cb8458-a62f-6562-de79-96841dc92ce3 Address:a6cb8458-a62f-6562-de79-96841dc92ce3} {Suffrage:Voter ID:server-4bc584ce-f1f2-a354-7549-e4a7411870e4 Address:4bc584ce-f1f2-a354-7549-e4a7411870e4} {Suffrage:Voter ID:server-b67050a9-5a59-e238-2524-15516f7a1d21 Address:b67050a9-5a59-e238-2524-15516f7a1d21}]" -2025-01-28T14:48:41.219Z [INFO] server-a6cb8458-a62f-6562-de79-96841dc92ce3: entering follower state: follower="Node at a6cb8458-a62f-6562-de79-96841dc92ce3 [Follower]" leader-address= leader-id= -2025-01-28T14:48:41.220Z [INFO] server-4bc584ce-f1f2-a354-7549-e4a7411870e4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a6cb8458-a62f-6562-de79-96841dc92ce3 Address:a6cb8458-a62f-6562-de79-96841dc92ce3} {Suffrage:Voter ID:server-4bc584ce-f1f2-a354-7549-e4a7411870e4 Address:4bc584ce-f1f2-a354-7549-e4a7411870e4} {Suffrage:Voter ID:server-b67050a9-5a59-e238-2524-15516f7a1d21 Address:b67050a9-5a59-e238-2524-15516f7a1d21}]" -2025-01-28T14:48:41.220Z [INFO] server-4bc584ce-f1f2-a354-7549-e4a7411870e4: entering follower state: follower="Node at 4bc584ce-f1f2-a354-7549-e4a7411870e4 [Follower]" leader-address= leader-id= -2025-01-28T14:48:41.220Z [INFO] server-b67050a9-5a59-e238-2524-15516f7a1d21: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a6cb8458-a62f-6562-de79-96841dc92ce3 Address:a6cb8458-a62f-6562-de79-96841dc92ce3} {Suffrage:Voter ID:server-4bc584ce-f1f2-a354-7549-e4a7411870e4 Address:4bc584ce-f1f2-a354-7549-e4a7411870e4} {Suffrage:Voter ID:server-b67050a9-5a59-e238-2524-15516f7a1d21 Address:b67050a9-5a59-e238-2524-15516f7a1d21}]" -2025-01-28T14:48:41.221Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:48:41.221Z [INFO] server-b67050a9-5a59-e238-2524-15516f7a1d21: entering follower state: follower="Node at b67050a9-5a59-e238-2524-15516f7a1d21 [Follower]" leader-address= leader-id= -2025-01-28T14:48:41.271Z [WARN] server-a6cb8458-a62f-6562-de79-96841dc92ce3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:41.271Z [INFO] server-a6cb8458-a62f-6562-de79-96841dc92ce3: entering candidate state: node="Node at a6cb8458-a62f-6562-de79-96841dc92ce3 [Candidate]" term=2 -2025-01-28T14:48:41.272Z [INFO] server-a6cb8458-a62f-6562-de79-96841dc92ce3: election won: term=2 tally=2 -2025-01-28T14:48:41.272Z [INFO] server-a6cb8458-a62f-6562-de79-96841dc92ce3: entering leader state: leader="Node at a6cb8458-a62f-6562-de79-96841dc92ce3 [Leader]" -2025-01-28T14:48:41.272Z [INFO] server-a6cb8458-a62f-6562-de79-96841dc92ce3: added peer, starting replication: peer=server-4bc584ce-f1f2-a354-7549-e4a7411870e4 -2025-01-28T14:48:41.272Z [INFO] server-a6cb8458-a62f-6562-de79-96841dc92ce3: added peer, starting replication: peer=server-b67050a9-5a59-e238-2524-15516f7a1d21 -2025-01-28T14:48:41.273Z [INFO] server-a6cb8458-a62f-6562-de79-96841dc92ce3: pipelining replication: peer="{Voter server-4bc584ce-f1f2-a354-7549-e4a7411870e4 4bc584ce-f1f2-a354-7549-e4a7411870e4}" -2025-01-28T14:48:41.273Z [INFO] server-a6cb8458-a62f-6562-de79-96841dc92ce3: pipelining replication: peer="{Voter server-b67050a9-5a59-e238-2524-15516f7a1d21 b67050a9-5a59-e238-2524-15516f7a1d21}" -2025-01-28T14:48:41.377Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:48:41.272552564 +0000 UTC m=+104.355957255 (2 nodes), 51.447194ms from start of poll, 54.156941ms from cluster start. Timeout at 2025-01-28 14:48:41.377563231 +0000 UTC m=+104.460968256, 105.011001ms after stability -2025-01-28T14:48:41.378Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:41.483Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:41.378098699 +0000 UTC m=+104.461503390 (1 nodes), 5.667µs from start of poll, 159.703076ms from cluster start. Timeout at 2025-01-28 14:48:41.483534378 +0000 UTC m=+104.566939736, 105.436346ms after stability -2025-01-28T14:48:41.484Z [WARN] server-b67050a9-5a59-e238-2524-15516f7a1d21: rejecting vote request since we have a leader: from=4bc584ce-f1f2-a354-7549-e4a7411870e4 leader=a6cb8458-a62f-6562-de79-96841dc92ce3 leader-id=server-a6cb8458-a62f-6562-de79-96841dc92ce3 -2025-01-28T14:48:41.485Z [INFO] server-a6cb8458-a62f-6562-de79-96841dc92ce3: aborting pipeline replication: peer="{Voter server-4bc584ce-f1f2-a354-7549-e4a7411870e4 4bc584ce-f1f2-a354-7549-e4a7411870e4}" -2025-01-28T14:48:41.485Z [INFO] server-a6cb8458-a62f-6562-de79-96841dc92ce3: aborting pipeline replication: peer="{Voter server-b67050a9-5a59-e238-2524-15516f7a1d21 b67050a9-5a59-e238-2524-15516f7a1d21}" ---- PASS: TestRaft_VotingGrant_WhenLeaderAvailable (0.28s) +2025-01-28T14:55:42.538Z [INFO] server-374863fa-c980-13b6-5302-d34b4dafb566: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-374863fa-c980-13b6-5302-d34b4dafb566 Address:374863fa-c980-13b6-5302-d34b4dafb566} {Suffrage:Voter ID:server-c1cbb197-190d-1668-0f50-a7519d9cd324 Address:c1cbb197-190d-1668-0f50-a7519d9cd324} {Suffrage:Voter ID:server-018c737c-0d71-5c52-28b2-890d72878f19 Address:018c737c-0d71-5c52-28b2-890d72878f19}]" +2025-01-28T14:55:42.538Z [INFO] server-c1cbb197-190d-1668-0f50-a7519d9cd324: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-374863fa-c980-13b6-5302-d34b4dafb566 Address:374863fa-c980-13b6-5302-d34b4dafb566} {Suffrage:Voter ID:server-c1cbb197-190d-1668-0f50-a7519d9cd324 Address:c1cbb197-190d-1668-0f50-a7519d9cd324} {Suffrage:Voter ID:server-018c737c-0d71-5c52-28b2-890d72878f19 Address:018c737c-0d71-5c52-28b2-890d72878f19}]" +2025-01-28T14:55:42.539Z [INFO] server-018c737c-0d71-5c52-28b2-890d72878f19: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-374863fa-c980-13b6-5302-d34b4dafb566 Address:374863fa-c980-13b6-5302-d34b4dafb566} {Suffrage:Voter ID:server-c1cbb197-190d-1668-0f50-a7519d9cd324 Address:c1cbb197-190d-1668-0f50-a7519d9cd324} {Suffrage:Voter ID:server-018c737c-0d71-5c52-28b2-890d72878f19 Address:018c737c-0d71-5c52-28b2-890d72878f19}]" +2025-01-28T14:55:42.539Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:55:42.540Z [INFO] server-374863fa-c980-13b6-5302-d34b4dafb566: entering follower state: follower="Node at 374863fa-c980-13b6-5302-d34b4dafb566 [Follower]" leader-address= leader-id= +2025-01-28T14:55:42.540Z [INFO] server-c1cbb197-190d-1668-0f50-a7519d9cd324: entering follower state: follower="Node at c1cbb197-190d-1668-0f50-a7519d9cd324 [Follower]" leader-address= leader-id= +2025-01-28T14:55:42.541Z [INFO] server-018c737c-0d71-5c52-28b2-890d72878f19: entering follower state: follower="Node at 018c737c-0d71-5c52-28b2-890d72878f19 [Follower]" leader-address= leader-id= +2025-01-28T14:55:42.599Z [WARN] server-c1cbb197-190d-1668-0f50-a7519d9cd324: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:42.599Z [INFO] server-c1cbb197-190d-1668-0f50-a7519d9cd324: entering candidate state: node="Node at c1cbb197-190d-1668-0f50-a7519d9cd324 [Candidate]" term=2 +2025-01-28T14:55:42.599Z [INFO] server-c1cbb197-190d-1668-0f50-a7519d9cd324: election won: term=2 tally=2 +2025-01-28T14:55:42.599Z [INFO] server-c1cbb197-190d-1668-0f50-a7519d9cd324: entering leader state: leader="Node at c1cbb197-190d-1668-0f50-a7519d9cd324 [Leader]" +2025-01-28T14:55:42.599Z [INFO] server-c1cbb197-190d-1668-0f50-a7519d9cd324: added peer, starting replication: peer=server-374863fa-c980-13b6-5302-d34b4dafb566 +2025-01-28T14:55:42.599Z [INFO] server-c1cbb197-190d-1668-0f50-a7519d9cd324: added peer, starting replication: peer=server-018c737c-0d71-5c52-28b2-890d72878f19 +2025-01-28T14:55:42.599Z [INFO] server-c1cbb197-190d-1668-0f50-a7519d9cd324: pipelining replication: peer="{Voter server-374863fa-c980-13b6-5302-d34b4dafb566 374863fa-c980-13b6-5302-d34b4dafb566}" +2025-01-28T14:55:42.599Z [INFO] server-c1cbb197-190d-1668-0f50-a7519d9cd324: pipelining replication: peer="{Voter server-018c737c-0d71-5c52-28b2-890d72878f19 018c737c-0d71-5c52-28b2-890d72878f19}" +2025-01-28T14:55:42.705Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:55:42.599611319 +0000 UTC m=+128.274101299 (2 nodes), 59.863792ms from start of poll, 65.044684ms from cluster start. Timeout at 2025-01-28 14:55:42.704628977 +0000 UTC m=+128.379118957, 105.017658ms after stability +2025-01-28T14:55:42.705Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:42.810Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:42.705323427 +0000 UTC m=+128.379813507 (1 nodes), 2.02µs from start of poll, 170.756892ms from cluster start. Timeout at 2025-01-28 14:55:42.810497951 +0000 UTC m=+128.484987931, 105.174424ms after stability +2025-01-28T14:55:42.810Z [WARN] server-018c737c-0d71-5c52-28b2-890d72878f19: rejecting vote request since we have a leader: from=374863fa-c980-13b6-5302-d34b4dafb566 leader=c1cbb197-190d-1668-0f50-a7519d9cd324 leader-id=server-c1cbb197-190d-1668-0f50-a7519d9cd324 +2025-01-28T14:55:42.816Z [INFO] server-c1cbb197-190d-1668-0f50-a7519d9cd324: aborting pipeline replication: peer="{Voter server-018c737c-0d71-5c52-28b2-890d72878f19 018c737c-0d71-5c52-28b2-890d72878f19}" +2025-01-28T14:55:42.816Z [ERROR] server-c1cbb197-190d-1668-0f50-a7519d9cd324: peer has newer term, stopping replication: peer="{Voter server-374863fa-c980-13b6-5302-d34b4dafb566 374863fa-c980-13b6-5302-d34b4dafb566}" +2025-01-28T14:55:42.816Z [INFO] server-c1cbb197-190d-1668-0f50-a7519d9cd324: aborting pipeline replication: peer="{Voter server-374863fa-c980-13b6-5302-d34b4dafb566 374863fa-c980-13b6-5302-d34b4dafb566}" +--- PASS: TestRaft_VotingGrant_WhenLeaderAvailable (0.29s) === RUN TestRaft_ProtocolVersion_RejectRPC -2025-01-28T14:48:41.500Z [INFO] server-81272f9d-52cd-323c-d943-3d51c0be810e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-81272f9d-52cd-323c-d943-3d51c0be810e Address:81272f9d-52cd-323c-d943-3d51c0be810e} {Suffrage:Voter ID:server-d3d508de-49e3-b3e4-33e1-5930ee54cf5b Address:d3d508de-49e3-b3e4-33e1-5930ee54cf5b} {Suffrage:Voter ID:server-11c2317d-1b03-ec9f-bce6-04efd5e1ef99 Address:11c2317d-1b03-ec9f-bce6-04efd5e1ef99}]" -2025-01-28T14:48:41.501Z [INFO] server-d3d508de-49e3-b3e4-33e1-5930ee54cf5b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-81272f9d-52cd-323c-d943-3d51c0be810e Address:81272f9d-52cd-323c-d943-3d51c0be810e} {Suffrage:Voter ID:server-d3d508de-49e3-b3e4-33e1-5930ee54cf5b Address:d3d508de-49e3-b3e4-33e1-5930ee54cf5b} {Suffrage:Voter ID:server-11c2317d-1b03-ec9f-bce6-04efd5e1ef99 Address:11c2317d-1b03-ec9f-bce6-04efd5e1ef99}]" -2025-01-28T14:48:41.503Z [INFO] server-11c2317d-1b03-ec9f-bce6-04efd5e1ef99: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-81272f9d-52cd-323c-d943-3d51c0be810e Address:81272f9d-52cd-323c-d943-3d51c0be810e} {Suffrage:Voter ID:server-d3d508de-49e3-b3e4-33e1-5930ee54cf5b Address:d3d508de-49e3-b3e4-33e1-5930ee54cf5b} {Suffrage:Voter ID:server-11c2317d-1b03-ec9f-bce6-04efd5e1ef99 Address:11c2317d-1b03-ec9f-bce6-04efd5e1ef99}]" -2025-01-28T14:48:41.503Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:48:41.509Z [INFO] server-81272f9d-52cd-323c-d943-3d51c0be810e: entering follower state: follower="Node at 81272f9d-52cd-323c-d943-3d51c0be810e [Follower]" leader-address= leader-id= -2025-01-28T14:48:41.513Z [INFO] server-d3d508de-49e3-b3e4-33e1-5930ee54cf5b: entering follower state: follower="Node at d3d508de-49e3-b3e4-33e1-5930ee54cf5b [Follower]" leader-address= leader-id= -2025-01-28T14:48:41.515Z [INFO] server-11c2317d-1b03-ec9f-bce6-04efd5e1ef99: entering follower state: follower="Node at 11c2317d-1b03-ec9f-bce6-04efd5e1ef99 [Follower]" leader-address= leader-id= -2025-01-28T14:48:41.581Z [WARN] server-d3d508de-49e3-b3e4-33e1-5930ee54cf5b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:41.582Z [INFO] server-d3d508de-49e3-b3e4-33e1-5930ee54cf5b: entering candidate state: node="Node at d3d508de-49e3-b3e4-33e1-5930ee54cf5b [Candidate]" term=2 -2025-01-28T14:48:41.582Z [INFO] server-d3d508de-49e3-b3e4-33e1-5930ee54cf5b: election won: term=2 tally=2 -2025-01-28T14:48:41.582Z [INFO] server-d3d508de-49e3-b3e4-33e1-5930ee54cf5b: entering leader state: leader="Node at d3d508de-49e3-b3e4-33e1-5930ee54cf5b [Leader]" -2025-01-28T14:48:41.583Z [INFO] server-d3d508de-49e3-b3e4-33e1-5930ee54cf5b: added peer, starting replication: peer=server-81272f9d-52cd-323c-d943-3d51c0be810e -2025-01-28T14:48:41.583Z [INFO] server-d3d508de-49e3-b3e4-33e1-5930ee54cf5b: added peer, starting replication: peer=server-11c2317d-1b03-ec9f-bce6-04efd5e1ef99 -2025-01-28T14:48:41.583Z [WARN] server-11c2317d-1b03-ec9f-bce6-04efd5e1ef99: rejecting vote request since our last term is greater: candidate=d3d508de-49e3-b3e4-33e1-5930ee54cf5b last-term=2 last-candidate-term=1 -2025-01-28T14:48:41.583Z [INFO] server-d3d508de-49e3-b3e4-33e1-5930ee54cf5b: pipelining replication: peer="{Voter server-81272f9d-52cd-323c-d943-3d51c0be810e 81272f9d-52cd-323c-d943-3d51c0be810e}" -2025-01-28T14:48:41.584Z [INFO] server-d3d508de-49e3-b3e4-33e1-5930ee54cf5b: pipelining replication: peer="{Voter server-11c2317d-1b03-ec9f-bce6-04efd5e1ef99 11c2317d-1b03-ec9f-bce6-04efd5e1ef99}" -2025-01-28T14:48:41.691Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:48:41.584127484 +0000 UTC m=+104.667532509 (2 nodes), 80.335053ms from start of poll, 86.308897ms from cluster start. Timeout at 2025-01-28 14:48:41.689146818 +0000 UTC m=+104.772551510, 105.019001ms after stability -2025-01-28T14:48:41.691Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:41.796Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:41.691312001 +0000 UTC m=+104.774716692 (1 nodes), 5.667µs from start of poll, 193.49308ms from cluster start. Timeout at 2025-01-28 14:48:41.796761347 +0000 UTC m=+104.880166372, 105.44968ms after stability -2025-01-28T14:48:41.797Z [INFO] server-d3d508de-49e3-b3e4-33e1-5930ee54cf5b: aborting pipeline replication: peer="{Voter server-11c2317d-1b03-ec9f-bce6-04efd5e1ef99 11c2317d-1b03-ec9f-bce6-04efd5e1ef99}" -2025-01-28T14:48:41.797Z [INFO] server-d3d508de-49e3-b3e4-33e1-5930ee54cf5b: aborting pipeline replication: peer="{Voter server-81272f9d-52cd-323c-d943-3d51c0be810e 81272f9d-52cd-323c-d943-3d51c0be810e}" ---- PASS: TestRaft_ProtocolVersion_RejectRPC (0.31s) +2025-01-28T14:55:42.820Z [INFO] server-8f956ec1-4b23-9326-071b-cd8cfab8f882: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8f956ec1-4b23-9326-071b-cd8cfab8f882 Address:8f956ec1-4b23-9326-071b-cd8cfab8f882} {Suffrage:Voter ID:server-207670f0-993b-d09f-9dc8-7eb3ac0d9070 Address:207670f0-993b-d09f-9dc8-7eb3ac0d9070} {Suffrage:Voter ID:server-b551991d-4bcd-596e-328c-4dcdaf19a40f Address:b551991d-4bcd-596e-328c-4dcdaf19a40f}]" +2025-01-28T14:55:42.820Z [INFO] server-8f956ec1-4b23-9326-071b-cd8cfab8f882: entering follower state: follower="Node at 8f956ec1-4b23-9326-071b-cd8cfab8f882 [Follower]" leader-address= leader-id= +2025-01-28T14:55:42.820Z [INFO] server-207670f0-993b-d09f-9dc8-7eb3ac0d9070: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8f956ec1-4b23-9326-071b-cd8cfab8f882 Address:8f956ec1-4b23-9326-071b-cd8cfab8f882} {Suffrage:Voter ID:server-207670f0-993b-d09f-9dc8-7eb3ac0d9070 Address:207670f0-993b-d09f-9dc8-7eb3ac0d9070} {Suffrage:Voter ID:server-b551991d-4bcd-596e-328c-4dcdaf19a40f Address:b551991d-4bcd-596e-328c-4dcdaf19a40f}]" +2025-01-28T14:55:42.820Z [INFO] server-207670f0-993b-d09f-9dc8-7eb3ac0d9070: entering follower state: follower="Node at 207670f0-993b-d09f-9dc8-7eb3ac0d9070 [Follower]" leader-address= leader-id= +2025-01-28T14:55:42.821Z [INFO] server-b551991d-4bcd-596e-328c-4dcdaf19a40f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8f956ec1-4b23-9326-071b-cd8cfab8f882 Address:8f956ec1-4b23-9326-071b-cd8cfab8f882} {Suffrage:Voter ID:server-207670f0-993b-d09f-9dc8-7eb3ac0d9070 Address:207670f0-993b-d09f-9dc8-7eb3ac0d9070} {Suffrage:Voter ID:server-b551991d-4bcd-596e-328c-4dcdaf19a40f Address:b551991d-4bcd-596e-328c-4dcdaf19a40f}]" +2025-01-28T14:55:42.821Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:55:42.821Z [INFO] server-b551991d-4bcd-596e-328c-4dcdaf19a40f: entering follower state: follower="Node at b551991d-4bcd-596e-328c-4dcdaf19a40f [Follower]" leader-address= leader-id= +2025-01-28T14:55:42.871Z [WARN] server-b551991d-4bcd-596e-328c-4dcdaf19a40f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:42.871Z [INFO] server-b551991d-4bcd-596e-328c-4dcdaf19a40f: entering candidate state: node="Node at b551991d-4bcd-596e-328c-4dcdaf19a40f [Candidate]" term=2 +2025-01-28T14:55:42.871Z [INFO] server-b551991d-4bcd-596e-328c-4dcdaf19a40f: election won: term=2 tally=2 +2025-01-28T14:55:42.871Z [INFO] server-b551991d-4bcd-596e-328c-4dcdaf19a40f: entering leader state: leader="Node at b551991d-4bcd-596e-328c-4dcdaf19a40f [Leader]" +2025-01-28T14:55:42.871Z [INFO] server-b551991d-4bcd-596e-328c-4dcdaf19a40f: added peer, starting replication: peer=server-8f956ec1-4b23-9326-071b-cd8cfab8f882 +2025-01-28T14:55:42.871Z [INFO] server-b551991d-4bcd-596e-328c-4dcdaf19a40f: added peer, starting replication: peer=server-207670f0-993b-d09f-9dc8-7eb3ac0d9070 +2025-01-28T14:55:42.872Z [INFO] server-b551991d-4bcd-596e-328c-4dcdaf19a40f: pipelining replication: peer="{Voter server-8f956ec1-4b23-9326-071b-cd8cfab8f882 8f956ec1-4b23-9326-071b-cd8cfab8f882}" +2025-01-28T14:55:42.872Z [INFO] server-b551991d-4bcd-596e-328c-4dcdaf19a40f: pipelining replication: peer="{Voter server-207670f0-993b-d09f-9dc8-7eb3ac0d9070 207670f0-993b-d09f-9dc8-7eb3ac0d9070}" +2025-01-28T14:55:42.977Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:55:42.871859736 +0000 UTC m=+128.546349696 (2 nodes), 50.66248ms from start of poll, 51.587773ms from cluster start. Timeout at 2025-01-28 14:55:42.976865895 +0000 UTC m=+128.651355915, 105.006219ms after stability +2025-01-28T14:55:42.977Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:43.082Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:42.977198519 +0000 UTC m=+128.651688499 (1 nodes), 2.06µs from start of poll, 156.926576ms from cluster start. Timeout at 2025-01-28 14:55:43.082372783 +0000 UTC m=+128.756862783, 105.174284ms after stability +2025-01-28T14:55:43.082Z [INFO] server-b551991d-4bcd-596e-328c-4dcdaf19a40f: aborting pipeline replication: peer="{Voter server-8f956ec1-4b23-9326-071b-cd8cfab8f882 8f956ec1-4b23-9326-071b-cd8cfab8f882}" +2025-01-28T14:55:43.082Z [INFO] server-b551991d-4bcd-596e-328c-4dcdaf19a40f: aborting pipeline replication: peer="{Voter server-207670f0-993b-d09f-9dc8-7eb3ac0d9070 207670f0-993b-d09f-9dc8-7eb3ac0d9070}" +--- PASS: TestRaft_ProtocolVersion_RejectRPC (0.27s) === RUN TestRaft_ProtocolVersion_Upgrade_1_2 raft_test.go:2151: Skipping unreliable test --- SKIP: TestRaft_ProtocolVersion_Upgrade_1_2 (0.00s) @@ -3624,275 +3621,268 @@ raft_test.go:2196: Skipping unreliable test --- SKIP: TestRaft_ProtocolVersion_Upgrade_2_3 (0.00s) === RUN TestRaft_LeaderID_Propagated -2025-01-28T14:48:41.808Z [INFO] server-5fc5789d-ad75-ea7a-ce82-2306bbd06e97: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5fc5789d-ad75-ea7a-ce82-2306bbd06e97 Address:5fc5789d-ad75-ea7a-ce82-2306bbd06e97} {Suffrage:Voter ID:server-dbc2434c-2f9c-d5be-52f3-acdba4809358 Address:dbc2434c-2f9c-d5be-52f3-acdba4809358} {Suffrage:Voter ID:server-549a53a8-7c78-b20d-2257-020fdd6fde79 Address:549a53a8-7c78-b20d-2257-020fdd6fde79}]" -2025-01-28T14:48:41.812Z [INFO] server-dbc2434c-2f9c-d5be-52f3-acdba4809358: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5fc5789d-ad75-ea7a-ce82-2306bbd06e97 Address:5fc5789d-ad75-ea7a-ce82-2306bbd06e97} {Suffrage:Voter ID:server-dbc2434c-2f9c-d5be-52f3-acdba4809358 Address:dbc2434c-2f9c-d5be-52f3-acdba4809358} {Suffrage:Voter ID:server-549a53a8-7c78-b20d-2257-020fdd6fde79 Address:549a53a8-7c78-b20d-2257-020fdd6fde79}]" -2025-01-28T14:48:41.813Z [INFO] server-549a53a8-7c78-b20d-2257-020fdd6fde79: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5fc5789d-ad75-ea7a-ce82-2306bbd06e97 Address:5fc5789d-ad75-ea7a-ce82-2306bbd06e97} {Suffrage:Voter ID:server-dbc2434c-2f9c-d5be-52f3-acdba4809358 Address:dbc2434c-2f9c-d5be-52f3-acdba4809358} {Suffrage:Voter ID:server-549a53a8-7c78-b20d-2257-020fdd6fde79 Address:549a53a8-7c78-b20d-2257-020fdd6fde79}]" -2025-01-28T14:48:41.813Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:41.817Z [INFO] server-5fc5789d-ad75-ea7a-ce82-2306bbd06e97: entering follower state: follower="Node at 5fc5789d-ad75-ea7a-ce82-2306bbd06e97 [Follower]" leader-address= leader-id= -2025-01-28T14:48:41.821Z [INFO] server-dbc2434c-2f9c-d5be-52f3-acdba4809358: entering follower state: follower="Node at dbc2434c-2f9c-d5be-52f3-acdba4809358 [Follower]" leader-address= leader-id= -2025-01-28T14:48:41.824Z [INFO] server-549a53a8-7c78-b20d-2257-020fdd6fde79: entering follower state: follower="Node at 549a53a8-7c78-b20d-2257-020fdd6fde79 [Follower]" leader-address= leader-id= -2025-01-28T14:48:41.876Z [WARN] server-5fc5789d-ad75-ea7a-ce82-2306bbd06e97: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:41.876Z [INFO] server-5fc5789d-ad75-ea7a-ce82-2306bbd06e97: entering candidate state: node="Node at 5fc5789d-ad75-ea7a-ce82-2306bbd06e97 [Candidate]" term=2 -2025-01-28T14:48:41.877Z [INFO] server-5fc5789d-ad75-ea7a-ce82-2306bbd06e97: election won: term=2 tally=2 -2025-01-28T14:48:41.877Z [INFO] server-5fc5789d-ad75-ea7a-ce82-2306bbd06e97: entering leader state: leader="Node at 5fc5789d-ad75-ea7a-ce82-2306bbd06e97 [Leader]" -2025-01-28T14:48:41.877Z [INFO] server-5fc5789d-ad75-ea7a-ce82-2306bbd06e97: added peer, starting replication: peer=server-dbc2434c-2f9c-d5be-52f3-acdba4809358 -2025-01-28T14:48:41.878Z [INFO] server-5fc5789d-ad75-ea7a-ce82-2306bbd06e97: added peer, starting replication: peer=server-549a53a8-7c78-b20d-2257-020fdd6fde79 -2025-01-28T14:48:41.879Z [INFO] server-5fc5789d-ad75-ea7a-ce82-2306bbd06e97: pipelining replication: peer="{Voter server-549a53a8-7c78-b20d-2257-020fdd6fde79 549a53a8-7c78-b20d-2257-020fdd6fde79}" -2025-01-28T14:48:41.879Z [INFO] server-5fc5789d-ad75-ea7a-ce82-2306bbd06e97: pipelining replication: peer="{Voter server-dbc2434c-2f9c-d5be-52f3-acdba4809358 dbc2434c-2f9c-d5be-52f3-acdba4809358}" -2025-01-28T14:48:41.983Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:41.878498893 +0000 UTC m=+104.961903917 (1 nodes), 64.936261ms from start of poll, 70.422424ms from cluster start. Timeout at 2025-01-28 14:48:41.983531561 +0000 UTC m=+105.066936585, 105.032668ms after stability -2025-01-28T14:48:41.984Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:42.090Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:41.984311369 +0000 UTC m=+105.067716060 (1 nodes), 6.334µs from start of poll, 176.234567ms from cluster start. Timeout at 2025-01-28 14:48:42.089950388 +0000 UTC m=+105.173355413, 105.639353ms after stability -2025-01-28T14:48:42.092Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:42.199Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:42.092464247 +0000 UTC m=+105.175868938 (1 nodes), 7.333µs from start of poll, 284.387445ms from cluster start. Timeout at 2025-01-28 14:48:42.197932262 +0000 UTC m=+105.281336953, 105.468015ms after stability -2025-01-28T14:48:42.201Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:42.307Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:42.202055835 +0000 UTC m=+105.285460526 (1 nodes), 5.666µs from start of poll, 393.979033ms from cluster start. Timeout at 2025-01-28 14:48:42.307527516 +0000 UTC m=+105.390932207, 105.471681ms after stability -2025-01-28T14:48:42.309Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:42.415Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:42.309350022 +0000 UTC m=+105.392754713 (1 nodes), 5.667µs from start of poll, 501.27322ms from cluster start. Timeout at 2025-01-28 14:48:42.414795368 +0000 UTC m=+105.498200059, 105.445346ms after stability -2025-01-28T14:48:42.416Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:42.523Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:42.41626653 +0000 UTC m=+105.499671888 (1 nodes), 6.334µs from start of poll, 608.190395ms from cluster start. Timeout at 2025-01-28 14:48:42.521819546 +0000 UTC m=+105.605224237, 105.552349ms after stability -2025-01-28T14:48:42.630Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:42.746Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:42.631104911 +0000 UTC m=+105.714509935 (1 nodes), 7.667µs from start of poll, 823.028442ms from cluster start. Timeout at 2025-01-28 14:48:42.736439254 +0000 UTC m=+105.819844278, 105.334343ms after stability -2025-01-28T14:48:42.747Z [INFO] server-5fc5789d-ad75-ea7a-ce82-2306bbd06e97: aborting pipeline replication: peer="{Voter server-549a53a8-7c78-b20d-2257-020fdd6fde79 549a53a8-7c78-b20d-2257-020fdd6fde79}" -2025-01-28T14:48:42.748Z [INFO] server-5fc5789d-ad75-ea7a-ce82-2306bbd06e97: aborting pipeline replication: peer="{Voter server-dbc2434c-2f9c-d5be-52f3-acdba4809358 dbc2434c-2f9c-d5be-52f3-acdba4809358}" ---- PASS: TestRaft_LeaderID_Propagated (0.95s) +2025-01-28T14:55:43.087Z [INFO] server-68c9cb15-984b-6ca2-c9bd-2def67a97228: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-68c9cb15-984b-6ca2-c9bd-2def67a97228 Address:68c9cb15-984b-6ca2-c9bd-2def67a97228} {Suffrage:Voter ID:server-09f7553a-5545-8684-f197-f0e35932a4d1 Address:09f7553a-5545-8684-f197-f0e35932a4d1} {Suffrage:Voter ID:server-d23f2521-39d4-26ef-028d-f07ea2c33c6f Address:d23f2521-39d4-26ef-028d-f07ea2c33c6f}]" +2025-01-28T14:55:43.088Z [INFO] server-09f7553a-5545-8684-f197-f0e35932a4d1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-68c9cb15-984b-6ca2-c9bd-2def67a97228 Address:68c9cb15-984b-6ca2-c9bd-2def67a97228} {Suffrage:Voter ID:server-09f7553a-5545-8684-f197-f0e35932a4d1 Address:09f7553a-5545-8684-f197-f0e35932a4d1} {Suffrage:Voter ID:server-d23f2521-39d4-26ef-028d-f07ea2c33c6f Address:d23f2521-39d4-26ef-028d-f07ea2c33c6f}]" +2025-01-28T14:55:43.088Z [INFO] server-d23f2521-39d4-26ef-028d-f07ea2c33c6f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-68c9cb15-984b-6ca2-c9bd-2def67a97228 Address:68c9cb15-984b-6ca2-c9bd-2def67a97228} {Suffrage:Voter ID:server-09f7553a-5545-8684-f197-f0e35932a4d1 Address:09f7553a-5545-8684-f197-f0e35932a4d1} {Suffrage:Voter ID:server-d23f2521-39d4-26ef-028d-f07ea2c33c6f Address:d23f2521-39d4-26ef-028d-f07ea2c33c6f}]" +2025-01-28T14:55:43.088Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:43.090Z [INFO] server-68c9cb15-984b-6ca2-c9bd-2def67a97228: entering follower state: follower="Node at 68c9cb15-984b-6ca2-c9bd-2def67a97228 [Follower]" leader-address= leader-id= +2025-01-28T14:55:43.091Z [INFO] server-d23f2521-39d4-26ef-028d-f07ea2c33c6f: entering follower state: follower="Node at d23f2521-39d4-26ef-028d-f07ea2c33c6f [Follower]" leader-address= leader-id= +2025-01-28T14:55:43.091Z [INFO] server-09f7553a-5545-8684-f197-f0e35932a4d1: entering follower state: follower="Node at 09f7553a-5545-8684-f197-f0e35932a4d1 [Follower]" leader-address= leader-id= +2025-01-28T14:55:43.142Z [WARN] server-d23f2521-39d4-26ef-028d-f07ea2c33c6f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:43.142Z [INFO] server-d23f2521-39d4-26ef-028d-f07ea2c33c6f: entering candidate state: node="Node at d23f2521-39d4-26ef-028d-f07ea2c33c6f [Candidate]" term=2 +2025-01-28T14:55:43.142Z [INFO] server-d23f2521-39d4-26ef-028d-f07ea2c33c6f: election won: term=2 tally=2 +2025-01-28T14:55:43.142Z [INFO] server-d23f2521-39d4-26ef-028d-f07ea2c33c6f: entering leader state: leader="Node at d23f2521-39d4-26ef-028d-f07ea2c33c6f [Leader]" +2025-01-28T14:55:43.142Z [INFO] server-d23f2521-39d4-26ef-028d-f07ea2c33c6f: added peer, starting replication: peer=server-68c9cb15-984b-6ca2-c9bd-2def67a97228 +2025-01-28T14:55:43.142Z [INFO] server-d23f2521-39d4-26ef-028d-f07ea2c33c6f: added peer, starting replication: peer=server-09f7553a-5545-8684-f197-f0e35932a4d1 +2025-01-28T14:55:43.142Z [INFO] server-d23f2521-39d4-26ef-028d-f07ea2c33c6f: pipelining replication: peer="{Voter server-68c9cb15-984b-6ca2-c9bd-2def67a97228 68c9cb15-984b-6ca2-c9bd-2def67a97228}" +2025-01-28T14:55:43.142Z [INFO] server-d23f2521-39d4-26ef-028d-f07ea2c33c6f: pipelining replication: peer="{Voter server-09f7553a-5545-8684-f197-f0e35932a4d1 09f7553a-5545-8684-f197-f0e35932a4d1}" +2025-01-28T14:55:43.248Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:43.142463329 +0000 UTC m=+128.816953289 (1 nodes), 54.000866ms from start of poll, 55.076324ms from cluster start. Timeout at 2025-01-28 14:55:43.247477588 +0000 UTC m=+128.921967568, 105.014279ms after stability +2025-01-28T14:55:43.248Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:43.353Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:43.248278587 +0000 UTC m=+128.922768567 (1 nodes), 1.64µs from start of poll, 160.891602ms from cluster start. Timeout at 2025-01-28 14:55:43.35345401 +0000 UTC m=+129.027944110, 105.175543ms after stability +2025-01-28T14:55:43.353Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:43.459Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:43.354007613 +0000 UTC m=+129.028497593 (1 nodes), 1.74µs from start of poll, 266.620628ms from cluster start. Timeout at 2025-01-28 14:55:43.4591465 +0000 UTC m=+129.133636540, 105.138947ms after stability +2025-01-28T14:55:43.460Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:43.566Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:43.460415355 +0000 UTC m=+129.134905315 (1 nodes), 1.88µs from start of poll, 373.02835ms from cluster start. Timeout at 2025-01-28 14:55:43.565550162 +0000 UTC m=+129.240040242, 105.134927ms after stability +2025-01-28T14:55:43.567Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:43.672Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:43.56720868 +0000 UTC m=+129.241698640 (1 nodes), 2.18µs from start of poll, 479.821675ms from cluster start. Timeout at 2025-01-28 14:55:43.672390203 +0000 UTC m=+129.346880183, 105.181543ms after stability +2025-01-28T14:55:43.673Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:43.779Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:43.673347547 +0000 UTC m=+129.347837507 (1 nodes), 1.839µs from start of poll, 585.960542ms from cluster start. Timeout at 2025-01-28 14:55:43.778509891 +0000 UTC m=+129.452999871, 105.162364ms after stability +2025-01-28T14:55:43.885Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:43.991Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:43.885174385 +0000 UTC m=+129.559664365 (1 nodes), 1.84µs from start of poll, 797.7874ms from cluster start. Timeout at 2025-01-28 14:55:43.990271676 +0000 UTC m=+129.664761656, 105.097291ms after stability +2025-01-28T14:55:43.994Z [INFO] server-d23f2521-39d4-26ef-028d-f07ea2c33c6f: aborting pipeline replication: peer="{Voter server-09f7553a-5545-8684-f197-f0e35932a4d1 09f7553a-5545-8684-f197-f0e35932a4d1}" +2025-01-28T14:55:43.994Z [INFO] server-d23f2521-39d4-26ef-028d-f07ea2c33c6f: aborting pipeline replication: peer="{Voter server-68c9cb15-984b-6ca2-c9bd-2def67a97228 68c9cb15-984b-6ca2-c9bd-2def67a97228}" +2025-01-28T14:55:44.495Z [ERROR] server-d23f2521-39d4-26ef-028d-f07ea2c33c6f: failed to heartbeat to: peer=09f7553a-5545-8684-f197-f0e35932a4d1 backoff time=10ms error="command timed out" +--- PASS: TestRaft_LeaderID_Propagated (1.41s) === RUN TestRaft_LeadershipTransferInProgress --- PASS: TestRaft_LeadershipTransferInProgress (0.00s) === RUN TestRaft_LeadershipTransferPickServer --- PASS: TestRaft_LeadershipTransferPickServer (0.00s) === RUN TestRaft_LeadershipTransfer -2025-01-28T14:48:42.759Z [INFO] server-b607987d-b2f3-4c7d-28ea-d8df8113a996: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b607987d-b2f3-4c7d-28ea-d8df8113a996 Address:b607987d-b2f3-4c7d-28ea-d8df8113a996} {Suffrage:Voter ID:server-9c7d37f2-00e3-a392-65bb-e671d5b8b763 Address:9c7d37f2-00e3-a392-65bb-e671d5b8b763} {Suffrage:Voter ID:server-7a8ff00a-a994-a0cc-dc84-bbb8febb5d8f Address:7a8ff00a-a994-a0cc-dc84-bbb8febb5d8f}]" -2025-01-28T14:48:42.760Z [INFO] server-b607987d-b2f3-4c7d-28ea-d8df8113a996: entering follower state: follower="Node at b607987d-b2f3-4c7d-28ea-d8df8113a996 [Follower]" leader-address= leader-id= -2025-01-28T14:48:42.760Z [INFO] server-9c7d37f2-00e3-a392-65bb-e671d5b8b763: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b607987d-b2f3-4c7d-28ea-d8df8113a996 Address:b607987d-b2f3-4c7d-28ea-d8df8113a996} {Suffrage:Voter ID:server-9c7d37f2-00e3-a392-65bb-e671d5b8b763 Address:9c7d37f2-00e3-a392-65bb-e671d5b8b763} {Suffrage:Voter ID:server-7a8ff00a-a994-a0cc-dc84-bbb8febb5d8f Address:7a8ff00a-a994-a0cc-dc84-bbb8febb5d8f}]" -2025-01-28T14:48:42.760Z [INFO] server-9c7d37f2-00e3-a392-65bb-e671d5b8b763: entering follower state: follower="Node at 9c7d37f2-00e3-a392-65bb-e671d5b8b763 [Follower]" leader-address= leader-id= -2025-01-28T14:48:42.761Z [INFO] server-7a8ff00a-a994-a0cc-dc84-bbb8febb5d8f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b607987d-b2f3-4c7d-28ea-d8df8113a996 Address:b607987d-b2f3-4c7d-28ea-d8df8113a996} {Suffrage:Voter ID:server-9c7d37f2-00e3-a392-65bb-e671d5b8b763 Address:9c7d37f2-00e3-a392-65bb-e671d5b8b763} {Suffrage:Voter ID:server-7a8ff00a-a994-a0cc-dc84-bbb8febb5d8f Address:7a8ff00a-a994-a0cc-dc84-bbb8febb5d8f}]" -2025-01-28T14:48:42.761Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:42.761Z [INFO] server-7a8ff00a-a994-a0cc-dc84-bbb8febb5d8f: entering follower state: follower="Node at 7a8ff00a-a994-a0cc-dc84-bbb8febb5d8f [Follower]" leader-address= leader-id= -2025-01-28T14:48:42.829Z [WARN] server-9c7d37f2-00e3-a392-65bb-e671d5b8b763: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:42.829Z [INFO] server-9c7d37f2-00e3-a392-65bb-e671d5b8b763: entering candidate state: node="Node at 9c7d37f2-00e3-a392-65bb-e671d5b8b763 [Candidate]" term=2 -2025-01-28T14:48:42.830Z [INFO] server-9c7d37f2-00e3-a392-65bb-e671d5b8b763: election won: term=2 tally=2 -2025-01-28T14:48:42.830Z [INFO] server-9c7d37f2-00e3-a392-65bb-e671d5b8b763: entering leader state: leader="Node at 9c7d37f2-00e3-a392-65bb-e671d5b8b763 [Leader]" -2025-01-28T14:48:42.830Z [INFO] server-9c7d37f2-00e3-a392-65bb-e671d5b8b763: added peer, starting replication: peer=server-b607987d-b2f3-4c7d-28ea-d8df8113a996 -2025-01-28T14:48:42.830Z [INFO] server-9c7d37f2-00e3-a392-65bb-e671d5b8b763: added peer, starting replication: peer=server-7a8ff00a-a994-a0cc-dc84-bbb8febb5d8f -2025-01-28T14:48:42.831Z [INFO] server-9c7d37f2-00e3-a392-65bb-e671d5b8b763: pipelining replication: peer="{Voter server-7a8ff00a-a994-a0cc-dc84-bbb8febb5d8f 7a8ff00a-a994-a0cc-dc84-bbb8febb5d8f}" -2025-01-28T14:48:42.831Z [INFO] server-9c7d37f2-00e3-a392-65bb-e671d5b8b763: pipelining replication: peer="{Voter server-b607987d-b2f3-4c7d-28ea-d8df8113a996 b607987d-b2f3-4c7d-28ea-d8df8113a996}" -2025-01-28T14:48:42.937Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:42.830568501 +0000 UTC m=+105.913973525 (1 nodes), 68.79971ms from start of poll, 71.48979ms from cluster start. Timeout at 2025-01-28 14:48:42.935579169 +0000 UTC m=+106.018983860, 105.010335ms after stability -2025-01-28T14:48:42.938Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:43.045Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:42.938142362 +0000 UTC m=+106.021547053 (1 nodes), 6.333µs from start of poll, 179.063318ms from cluster start. Timeout at 2025-01-28 14:48:43.043570709 +0000 UTC m=+106.126975733, 105.42868ms after stability -2025-01-28T14:48:43.051Z [INFO] server-b607987d-b2f3-4c7d-28ea-d8df8113a996: entering candidate state: node="Node at b607987d-b2f3-4c7d-28ea-d8df8113a996 [Candidate]" term=3 -2025-01-28T14:48:43.053Z [INFO] server-9c7d37f2-00e3-a392-65bb-e671d5b8b763: entering follower state: follower="Node at 9c7d37f2-00e3-a392-65bb-e671d5b8b763 [Follower]" leader-address= leader-id= -2025-01-28T14:48:43.053Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:43.053Z [INFO] server-9c7d37f2-00e3-a392-65bb-e671d5b8b763: aborting pipeline replication: peer="{Voter server-7a8ff00a-a994-a0cc-dc84-bbb8febb5d8f 7a8ff00a-a994-a0cc-dc84-bbb8febb5d8f}" -2025-01-28T14:48:43.054Z [INFO] server-9c7d37f2-00e3-a392-65bb-e671d5b8b763: aborting pipeline replication: peer="{Voter server-b607987d-b2f3-4c7d-28ea-d8df8113a996 b607987d-b2f3-4c7d-28ea-d8df8113a996}" -2025-01-28T14:48:43.054Z [INFO] server-b607987d-b2f3-4c7d-28ea-d8df8113a996: election won: term=3 tally=2 -2025-01-28T14:48:43.054Z [INFO] server-b607987d-b2f3-4c7d-28ea-d8df8113a996: entering leader state: leader="Node at b607987d-b2f3-4c7d-28ea-d8df8113a996 [Leader]" -2025-01-28T14:48:43.054Z [INFO] server-b607987d-b2f3-4c7d-28ea-d8df8113a996: added peer, starting replication: peer=server-9c7d37f2-00e3-a392-65bb-e671d5b8b763 -2025-01-28T14:48:43.054Z [INFO] server-b607987d-b2f3-4c7d-28ea-d8df8113a996: added peer, starting replication: peer=server-7a8ff00a-a994-a0cc-dc84-bbb8febb5d8f -2025-01-28T14:48:43.056Z [INFO] server-b607987d-b2f3-4c7d-28ea-d8df8113a996: pipelining replication: peer="{Voter server-9c7d37f2-00e3-a392-65bb-e671d5b8b763 9c7d37f2-00e3-a392-65bb-e671d5b8b763}" -2025-01-28T14:48:43.056Z [INFO] server-b607987d-b2f3-4c7d-28ea-d8df8113a996: pipelining replication: peer="{Voter server-7a8ff00a-a994-a0cc-dc84-bbb8febb5d8f 7a8ff00a-a994-a0cc-dc84-bbb8febb5d8f}" -2025-01-28T14:48:43.160Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:43.05546318 +0000 UTC m=+106.138868204 (1 nodes), 1.959058ms from start of poll, 296.384469ms from cluster start. Timeout at 2025-01-28 14:48:43.160505181 +0000 UTC m=+106.243909872, 105.041668ms after stability -2025-01-28T14:48:43.161Z [INFO] server-b607987d-b2f3-4c7d-28ea-d8df8113a996: aborting pipeline replication: peer="{Voter server-9c7d37f2-00e3-a392-65bb-e671d5b8b763 9c7d37f2-00e3-a392-65bb-e671d5b8b763}" -2025-01-28T14:48:43.161Z [INFO] server-b607987d-b2f3-4c7d-28ea-d8df8113a996: aborting pipeline replication: peer="{Voter server-7a8ff00a-a994-a0cc-dc84-bbb8febb5d8f 7a8ff00a-a994-a0cc-dc84-bbb8febb5d8f}" ---- PASS: TestRaft_LeadershipTransfer (0.41s) +2025-01-28T14:55:44.499Z [INFO] server-955c351e-e622-2746-f130-abbd0c0ab2ca: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-955c351e-e622-2746-f130-abbd0c0ab2ca Address:955c351e-e622-2746-f130-abbd0c0ab2ca} {Suffrage:Voter ID:server-b060373a-87ff-c769-2110-10dcff83778e Address:b060373a-87ff-c769-2110-10dcff83778e} {Suffrage:Voter ID:server-7535ea26-0093-6cdc-5060-927a8bee1d14 Address:7535ea26-0093-6cdc-5060-927a8bee1d14}]" +2025-01-28T14:55:44.499Z [INFO] server-955c351e-e622-2746-f130-abbd0c0ab2ca: entering follower state: follower="Node at 955c351e-e622-2746-f130-abbd0c0ab2ca [Follower]" leader-address= leader-id= +2025-01-28T14:55:44.499Z [INFO] server-b060373a-87ff-c769-2110-10dcff83778e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-955c351e-e622-2746-f130-abbd0c0ab2ca Address:955c351e-e622-2746-f130-abbd0c0ab2ca} {Suffrage:Voter ID:server-b060373a-87ff-c769-2110-10dcff83778e Address:b060373a-87ff-c769-2110-10dcff83778e} {Suffrage:Voter ID:server-7535ea26-0093-6cdc-5060-927a8bee1d14 Address:7535ea26-0093-6cdc-5060-927a8bee1d14}]" +2025-01-28T14:55:44.499Z [INFO] server-b060373a-87ff-c769-2110-10dcff83778e: entering follower state: follower="Node at b060373a-87ff-c769-2110-10dcff83778e [Follower]" leader-address= leader-id= +2025-01-28T14:55:44.500Z [INFO] server-7535ea26-0093-6cdc-5060-927a8bee1d14: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-955c351e-e622-2746-f130-abbd0c0ab2ca Address:955c351e-e622-2746-f130-abbd0c0ab2ca} {Suffrage:Voter ID:server-b060373a-87ff-c769-2110-10dcff83778e Address:b060373a-87ff-c769-2110-10dcff83778e} {Suffrage:Voter ID:server-7535ea26-0093-6cdc-5060-927a8bee1d14 Address:7535ea26-0093-6cdc-5060-927a8bee1d14}]" +2025-01-28T14:55:44.500Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:44.500Z [INFO] server-7535ea26-0093-6cdc-5060-927a8bee1d14: entering follower state: follower="Node at 7535ea26-0093-6cdc-5060-927a8bee1d14 [Follower]" leader-address= leader-id= +2025-01-28T14:55:44.553Z [WARN] server-b060373a-87ff-c769-2110-10dcff83778e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:44.553Z [INFO] server-b060373a-87ff-c769-2110-10dcff83778e: entering candidate state: node="Node at b060373a-87ff-c769-2110-10dcff83778e [Candidate]" term=2 +2025-01-28T14:55:44.553Z [INFO] server-b060373a-87ff-c769-2110-10dcff83778e: election won: term=2 tally=2 +2025-01-28T14:55:44.553Z [INFO] server-b060373a-87ff-c769-2110-10dcff83778e: entering leader state: leader="Node at b060373a-87ff-c769-2110-10dcff83778e [Leader]" +2025-01-28T14:55:44.553Z [INFO] server-b060373a-87ff-c769-2110-10dcff83778e: added peer, starting replication: peer=server-955c351e-e622-2746-f130-abbd0c0ab2ca +2025-01-28T14:55:44.553Z [INFO] server-b060373a-87ff-c769-2110-10dcff83778e: added peer, starting replication: peer=server-7535ea26-0093-6cdc-5060-927a8bee1d14 +2025-01-28T14:55:44.553Z [INFO] server-b060373a-87ff-c769-2110-10dcff83778e: pipelining replication: peer="{Voter server-955c351e-e622-2746-f130-abbd0c0ab2ca 955c351e-e622-2746-f130-abbd0c0ab2ca}" +2025-01-28T14:55:44.553Z [INFO] server-b060373a-87ff-c769-2110-10dcff83778e: pipelining replication: peer="{Voter server-7535ea26-0093-6cdc-5060-927a8bee1d14 7535ea26-0093-6cdc-5060-927a8bee1d14}" +2025-01-28T14:55:44.658Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:44.553400047 +0000 UTC m=+130.227890027 (1 nodes), 53.243677ms from start of poll, 54.27406ms from cluster start. Timeout at 2025-01-28 14:55:44.658404827 +0000 UTC m=+130.332894807, 105.00478ms after stability +2025-01-28T14:55:44.658Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:44.763Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:44.658619301 +0000 UTC m=+130.333109281 (1 nodes), 1.88µs from start of poll, 159.493314ms from cluster start. Timeout at 2025-01-28 14:55:44.763784965 +0000 UTC m=+130.438274925, 105.165644ms after stability +2025-01-28T14:55:44.764Z [INFO] server-955c351e-e622-2746-f130-abbd0c0ab2ca: entering candidate state: node="Node at 955c351e-e622-2746-f130-abbd0c0ab2ca [Candidate]" term=3 +2025-01-28T14:55:44.764Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:44.764Z [INFO] server-b060373a-87ff-c769-2110-10dcff83778e: entering follower state: follower="Node at b060373a-87ff-c769-2110-10dcff83778e [Follower]" leader-address= leader-id= +2025-01-28T14:55:44.764Z [INFO] server-955c351e-e622-2746-f130-abbd0c0ab2ca: election won: term=3 tally=2 +2025-01-28T14:55:44.764Z [INFO] server-b060373a-87ff-c769-2110-10dcff83778e: aborting pipeline replication: peer="{Voter server-7535ea26-0093-6cdc-5060-927a8bee1d14 7535ea26-0093-6cdc-5060-927a8bee1d14}" +2025-01-28T14:55:44.764Z [INFO] server-955c351e-e622-2746-f130-abbd0c0ab2ca: entering leader state: leader="Node at 955c351e-e622-2746-f130-abbd0c0ab2ca [Leader]" +2025-01-28T14:55:44.764Z [INFO] server-955c351e-e622-2746-f130-abbd0c0ab2ca: added peer, starting replication: peer=server-b060373a-87ff-c769-2110-10dcff83778e +2025-01-28T14:55:44.764Z [INFO] server-955c351e-e622-2746-f130-abbd0c0ab2ca: added peer, starting replication: peer=server-7535ea26-0093-6cdc-5060-927a8bee1d14 +2025-01-28T14:55:44.764Z [INFO] server-b060373a-87ff-c769-2110-10dcff83778e: aborting pipeline replication: peer="{Voter server-955c351e-e622-2746-f130-abbd0c0ab2ca 955c351e-e622-2746-f130-abbd0c0ab2ca}" +2025-01-28T14:55:44.764Z [INFO] server-955c351e-e622-2746-f130-abbd0c0ab2ca: pipelining replication: peer="{Voter server-7535ea26-0093-6cdc-5060-927a8bee1d14 7535ea26-0093-6cdc-5060-927a8bee1d14}" +2025-01-28T14:55:44.764Z [INFO] server-955c351e-e622-2746-f130-abbd0c0ab2ca: pipelining replication: peer="{Voter server-b060373a-87ff-c769-2110-10dcff83778e b060373a-87ff-c769-2110-10dcff83778e}" +2025-01-28T14:55:44.869Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:44.76463604 +0000 UTC m=+130.439126020 (1 nodes), 242.897µs from start of poll, 265.510053ms from cluster start. Timeout at 2025-01-28 14:55:44.869761448 +0000 UTC m=+130.544251448, 105.125428ms after stability +2025-01-28T14:55:44.870Z [INFO] server-955c351e-e622-2746-f130-abbd0c0ab2ca: aborting pipeline replication: peer="{Voter server-7535ea26-0093-6cdc-5060-927a8bee1d14 7535ea26-0093-6cdc-5060-927a8bee1d14}" +2025-01-28T14:55:44.870Z [INFO] server-955c351e-e622-2746-f130-abbd0c0ab2ca: aborting pipeline replication: peer="{Voter server-b060373a-87ff-c769-2110-10dcff83778e b060373a-87ff-c769-2110-10dcff83778e}" +--- PASS: TestRaft_LeadershipTransfer (0.37s) === RUN TestRaft_LeadershipTransferWithOneNode -2025-01-28T14:48:43.167Z [INFO] server-bcc368e0-c68a-7746-681c-a9ae5f00023d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bcc368e0-c68a-7746-681c-a9ae5f00023d Address:bcc368e0-c68a-7746-681c-a9ae5f00023d}]" -2025-01-28T14:48:43.167Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:43.167Z [INFO] server-bcc368e0-c68a-7746-681c-a9ae5f00023d: entering follower state: follower="Node at bcc368e0-c68a-7746-681c-a9ae5f00023d [Follower]" leader-address= leader-id= -2025-01-28T14:48:43.226Z [WARN] server-bcc368e0-c68a-7746-681c-a9ae5f00023d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:43.226Z [INFO] server-bcc368e0-c68a-7746-681c-a9ae5f00023d: entering candidate state: node="Node at bcc368e0-c68a-7746-681c-a9ae5f00023d [Candidate]" term=2 -2025-01-28T14:48:43.226Z [INFO] server-bcc368e0-c68a-7746-681c-a9ae5f00023d: election won: term=2 tally=1 -2025-01-28T14:48:43.226Z [INFO] server-bcc368e0-c68a-7746-681c-a9ae5f00023d: entering leader state: leader="Node at bcc368e0-c68a-7746-681c-a9ae5f00023d [Leader]" -2025-01-28T14:48:43.333Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:43.227484622 +0000 UTC m=+106.310889313 (1 nodes), 59.736108ms from start of poll, 60.474796ms from cluster start. Timeout at 2025-01-28 14:48:43.332495955 +0000 UTC m=+106.415900980, 105.011667ms after stability ---- PASS: TestRaft_LeadershipTransferWithOneNode (0.17s) +2025-01-28T14:55:44.872Z [INFO] server-aa009046-31a1-409c-2e42-174b9a155edb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-aa009046-31a1-409c-2e42-174b9a155edb Address:aa009046-31a1-409c-2e42-174b9a155edb}]" +2025-01-28T14:55:44.872Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:44.873Z [INFO] server-aa009046-31a1-409c-2e42-174b9a155edb: entering follower state: follower="Node at aa009046-31a1-409c-2e42-174b9a155edb [Follower]" leader-address= leader-id= +2025-01-28T14:55:44.949Z [WARN] server-aa009046-31a1-409c-2e42-174b9a155edb: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:44.949Z [INFO] server-aa009046-31a1-409c-2e42-174b9a155edb: entering candidate state: node="Node at aa009046-31a1-409c-2e42-174b9a155edb [Candidate]" term=2 +2025-01-28T14:55:44.949Z [INFO] server-aa009046-31a1-409c-2e42-174b9a155edb: election won: term=2 tally=1 +2025-01-28T14:55:44.949Z [INFO] server-aa009046-31a1-409c-2e42-174b9a155edb: entering leader state: leader="Node at aa009046-31a1-409c-2e42-174b9a155edb [Leader]" +2025-01-28T14:55:45.055Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:44.949814771 +0000 UTC m=+130.624304731 (1 nodes), 76.826313ms from start of poll, 77.159261ms from cluster start. Timeout at 2025-01-28 14:55:45.054839628 +0000 UTC m=+130.729329688, 105.024957ms after stability +--- PASS: TestRaft_LeadershipTransferWithOneNode (0.19s) === RUN TestRaft_LeadershipTransferWithSevenNodes -2025-01-28T14:48:43.347Z [INFO] server-af54d44b-015a-18e6-da85-46538a6c4186: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-af54d44b-015a-18e6-da85-46538a6c4186 Address:af54d44b-015a-18e6-da85-46538a6c4186} {Suffrage:Voter ID:server-995f613f-bb00-44bd-a796-36ce725eb03a Address:995f613f-bb00-44bd-a796-36ce725eb03a} {Suffrage:Voter ID:server-9b7543b8-bf54-9d8a-5f42-a571c32df417 Address:9b7543b8-bf54-9d8a-5f42-a571c32df417} {Suffrage:Voter ID:server-45375828-775a-7428-e34a-83c3dfd99ef9 Address:45375828-775a-7428-e34a-83c3dfd99ef9} {Suffrage:Voter ID:server-2577410f-84cf-07c4-74d3-e5d05021e9d7 Address:2577410f-84cf-07c4-74d3-e5d05021e9d7} {Suffrage:Voter ID:server-9fdd3a18-c17b-507e-24f3-d63df555b804 Address:9fdd3a18-c17b-507e-24f3-d63df555b804} {Suffrage:Voter ID:server-faea0b1f-5b46-7485-0d7f-f26a767ab98a Address:faea0b1f-5b46-7485-0d7f-f26a767ab98a}]" -2025-01-28T14:48:43.348Z [INFO] server-af54d44b-015a-18e6-da85-46538a6c4186: entering follower state: follower="Node at af54d44b-015a-18e6-da85-46538a6c4186 [Follower]" leader-address= leader-id= -2025-01-28T14:48:43.348Z [INFO] server-995f613f-bb00-44bd-a796-36ce725eb03a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-af54d44b-015a-18e6-da85-46538a6c4186 Address:af54d44b-015a-18e6-da85-46538a6c4186} {Suffrage:Voter ID:server-995f613f-bb00-44bd-a796-36ce725eb03a Address:995f613f-bb00-44bd-a796-36ce725eb03a} {Suffrage:Voter ID:server-9b7543b8-bf54-9d8a-5f42-a571c32df417 Address:9b7543b8-bf54-9d8a-5f42-a571c32df417} {Suffrage:Voter ID:server-45375828-775a-7428-e34a-83c3dfd99ef9 Address:45375828-775a-7428-e34a-83c3dfd99ef9} {Suffrage:Voter ID:server-2577410f-84cf-07c4-74d3-e5d05021e9d7 Address:2577410f-84cf-07c4-74d3-e5d05021e9d7} {Suffrage:Voter ID:server-9fdd3a18-c17b-507e-24f3-d63df555b804 Address:9fdd3a18-c17b-507e-24f3-d63df555b804} {Suffrage:Voter ID:server-faea0b1f-5b46-7485-0d7f-f26a767ab98a Address:faea0b1f-5b46-7485-0d7f-f26a767ab98a}]" -2025-01-28T14:48:43.348Z [INFO] server-995f613f-bb00-44bd-a796-36ce725eb03a: entering follower state: follower="Node at 995f613f-bb00-44bd-a796-36ce725eb03a [Follower]" leader-address= leader-id= -2025-01-28T14:48:43.349Z [INFO] server-9b7543b8-bf54-9d8a-5f42-a571c32df417: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-af54d44b-015a-18e6-da85-46538a6c4186 Address:af54d44b-015a-18e6-da85-46538a6c4186} {Suffrage:Voter ID:server-995f613f-bb00-44bd-a796-36ce725eb03a Address:995f613f-bb00-44bd-a796-36ce725eb03a} {Suffrage:Voter ID:server-9b7543b8-bf54-9d8a-5f42-a571c32df417 Address:9b7543b8-bf54-9d8a-5f42-a571c32df417} {Suffrage:Voter ID:server-45375828-775a-7428-e34a-83c3dfd99ef9 Address:45375828-775a-7428-e34a-83c3dfd99ef9} {Suffrage:Voter ID:server-2577410f-84cf-07c4-74d3-e5d05021e9d7 Address:2577410f-84cf-07c4-74d3-e5d05021e9d7} {Suffrage:Voter ID:server-9fdd3a18-c17b-507e-24f3-d63df555b804 Address:9fdd3a18-c17b-507e-24f3-d63df555b804} {Suffrage:Voter ID:server-faea0b1f-5b46-7485-0d7f-f26a767ab98a Address:faea0b1f-5b46-7485-0d7f-f26a767ab98a}]" -2025-01-28T14:48:43.350Z [INFO] server-9b7543b8-bf54-9d8a-5f42-a571c32df417: entering follower state: follower="Node at 9b7543b8-bf54-9d8a-5f42-a571c32df417 [Follower]" leader-address= leader-id= -2025-01-28T14:48:43.350Z [INFO] server-45375828-775a-7428-e34a-83c3dfd99ef9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-af54d44b-015a-18e6-da85-46538a6c4186 Address:af54d44b-015a-18e6-da85-46538a6c4186} {Suffrage:Voter ID:server-995f613f-bb00-44bd-a796-36ce725eb03a Address:995f613f-bb00-44bd-a796-36ce725eb03a} {Suffrage:Voter ID:server-9b7543b8-bf54-9d8a-5f42-a571c32df417 Address:9b7543b8-bf54-9d8a-5f42-a571c32df417} {Suffrage:Voter ID:server-45375828-775a-7428-e34a-83c3dfd99ef9 Address:45375828-775a-7428-e34a-83c3dfd99ef9} {Suffrage:Voter ID:server-2577410f-84cf-07c4-74d3-e5d05021e9d7 Address:2577410f-84cf-07c4-74d3-e5d05021e9d7} {Suffrage:Voter ID:server-9fdd3a18-c17b-507e-24f3-d63df555b804 Address:9fdd3a18-c17b-507e-24f3-d63df555b804} {Suffrage:Voter ID:server-faea0b1f-5b46-7485-0d7f-f26a767ab98a Address:faea0b1f-5b46-7485-0d7f-f26a767ab98a}]" -2025-01-28T14:48:43.351Z [INFO] server-45375828-775a-7428-e34a-83c3dfd99ef9: entering follower state: follower="Node at 45375828-775a-7428-e34a-83c3dfd99ef9 [Follower]" leader-address= leader-id= -2025-01-28T14:48:43.351Z [INFO] server-2577410f-84cf-07c4-74d3-e5d05021e9d7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-af54d44b-015a-18e6-da85-46538a6c4186 Address:af54d44b-015a-18e6-da85-46538a6c4186} {Suffrage:Voter ID:server-995f613f-bb00-44bd-a796-36ce725eb03a Address:995f613f-bb00-44bd-a796-36ce725eb03a} {Suffrage:Voter ID:server-9b7543b8-bf54-9d8a-5f42-a571c32df417 Address:9b7543b8-bf54-9d8a-5f42-a571c32df417} {Suffrage:Voter ID:server-45375828-775a-7428-e34a-83c3dfd99ef9 Address:45375828-775a-7428-e34a-83c3dfd99ef9} {Suffrage:Voter ID:server-2577410f-84cf-07c4-74d3-e5d05021e9d7 Address:2577410f-84cf-07c4-74d3-e5d05021e9d7} {Suffrage:Voter ID:server-9fdd3a18-c17b-507e-24f3-d63df555b804 Address:9fdd3a18-c17b-507e-24f3-d63df555b804} {Suffrage:Voter ID:server-faea0b1f-5b46-7485-0d7f-f26a767ab98a Address:faea0b1f-5b46-7485-0d7f-f26a767ab98a}]" -2025-01-28T14:48:43.352Z [INFO] server-2577410f-84cf-07c4-74d3-e5d05021e9d7: entering follower state: follower="Node at 2577410f-84cf-07c4-74d3-e5d05021e9d7 [Follower]" leader-address= leader-id= -2025-01-28T14:48:43.355Z [INFO] server-9fdd3a18-c17b-507e-24f3-d63df555b804: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-af54d44b-015a-18e6-da85-46538a6c4186 Address:af54d44b-015a-18e6-da85-46538a6c4186} {Suffrage:Voter ID:server-995f613f-bb00-44bd-a796-36ce725eb03a Address:995f613f-bb00-44bd-a796-36ce725eb03a} {Suffrage:Voter ID:server-9b7543b8-bf54-9d8a-5f42-a571c32df417 Address:9b7543b8-bf54-9d8a-5f42-a571c32df417} {Suffrage:Voter ID:server-45375828-775a-7428-e34a-83c3dfd99ef9 Address:45375828-775a-7428-e34a-83c3dfd99ef9} {Suffrage:Voter ID:server-2577410f-84cf-07c4-74d3-e5d05021e9d7 Address:2577410f-84cf-07c4-74d3-e5d05021e9d7} {Suffrage:Voter ID:server-9fdd3a18-c17b-507e-24f3-d63df555b804 Address:9fdd3a18-c17b-507e-24f3-d63df555b804} {Suffrage:Voter ID:server-faea0b1f-5b46-7485-0d7f-f26a767ab98a Address:faea0b1f-5b46-7485-0d7f-f26a767ab98a}]" -2025-01-28T14:48:43.356Z [INFO] server-faea0b1f-5b46-7485-0d7f-f26a767ab98a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-af54d44b-015a-18e6-da85-46538a6c4186 Address:af54d44b-015a-18e6-da85-46538a6c4186} {Suffrage:Voter ID:server-995f613f-bb00-44bd-a796-36ce725eb03a Address:995f613f-bb00-44bd-a796-36ce725eb03a} {Suffrage:Voter ID:server-9b7543b8-bf54-9d8a-5f42-a571c32df417 Address:9b7543b8-bf54-9d8a-5f42-a571c32df417} {Suffrage:Voter ID:server-45375828-775a-7428-e34a-83c3dfd99ef9 Address:45375828-775a-7428-e34a-83c3dfd99ef9} {Suffrage:Voter ID:server-2577410f-84cf-07c4-74d3-e5d05021e9d7 Address:2577410f-84cf-07c4-74d3-e5d05021e9d7} {Suffrage:Voter ID:server-9fdd3a18-c17b-507e-24f3-d63df555b804 Address:9fdd3a18-c17b-507e-24f3-d63df555b804} {Suffrage:Voter ID:server-faea0b1f-5b46-7485-0d7f-f26a767ab98a Address:faea0b1f-5b46-7485-0d7f-f26a767ab98a}]" -2025-01-28T14:48:43.356Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:43.362Z [INFO] server-9fdd3a18-c17b-507e-24f3-d63df555b804: entering follower state: follower="Node at 9fdd3a18-c17b-507e-24f3-d63df555b804 [Follower]" leader-address= leader-id= -2025-01-28T14:48:43.365Z [INFO] server-faea0b1f-5b46-7485-0d7f-f26a767ab98a: entering follower state: follower="Node at faea0b1f-5b46-7485-0d7f-f26a767ab98a [Follower]" leader-address= leader-id= -2025-01-28T14:48:43.409Z [WARN] server-2577410f-84cf-07c4-74d3-e5d05021e9d7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:43.409Z [INFO] server-2577410f-84cf-07c4-74d3-e5d05021e9d7: entering candidate state: node="Node at 2577410f-84cf-07c4-74d3-e5d05021e9d7 [Candidate]" term=2 -2025-01-28T14:48:43.410Z [INFO] server-2577410f-84cf-07c4-74d3-e5d05021e9d7: election won: term=2 tally=4 -2025-01-28T14:48:43.410Z [INFO] server-2577410f-84cf-07c4-74d3-e5d05021e9d7: entering leader state: leader="Node at 2577410f-84cf-07c4-74d3-e5d05021e9d7 [Leader]" -2025-01-28T14:48:43.410Z [INFO] server-2577410f-84cf-07c4-74d3-e5d05021e9d7: added peer, starting replication: peer=server-af54d44b-015a-18e6-da85-46538a6c4186 -2025-01-28T14:48:43.411Z [INFO] server-2577410f-84cf-07c4-74d3-e5d05021e9d7: added peer, starting replication: peer=server-995f613f-bb00-44bd-a796-36ce725eb03a -2025-01-28T14:48:43.411Z [INFO] server-2577410f-84cf-07c4-74d3-e5d05021e9d7: added peer, starting replication: peer=server-9b7543b8-bf54-9d8a-5f42-a571c32df417 -2025-01-28T14:48:43.411Z [INFO] server-2577410f-84cf-07c4-74d3-e5d05021e9d7: added peer, starting replication: peer=server-45375828-775a-7428-e34a-83c3dfd99ef9 -2025-01-28T14:48:43.411Z [INFO] server-2577410f-84cf-07c4-74d3-e5d05021e9d7: added peer, starting replication: peer=server-9fdd3a18-c17b-507e-24f3-d63df555b804 -2025-01-28T14:48:43.411Z [INFO] server-2577410f-84cf-07c4-74d3-e5d05021e9d7: added peer, starting replication: peer=server-faea0b1f-5b46-7485-0d7f-f26a767ab98a -2025-01-28T14:48:43.411Z [INFO] server-2577410f-84cf-07c4-74d3-e5d05021e9d7: pipelining replication: peer="{Voter server-af54d44b-015a-18e6-da85-46538a6c4186 af54d44b-015a-18e6-da85-46538a6c4186}" -2025-01-28T14:48:43.412Z [INFO] server-2577410f-84cf-07c4-74d3-e5d05021e9d7: pipelining replication: peer="{Voter server-995f613f-bb00-44bd-a796-36ce725eb03a 995f613f-bb00-44bd-a796-36ce725eb03a}" -2025-01-28T14:48:43.412Z [INFO] server-2577410f-84cf-07c4-74d3-e5d05021e9d7: pipelining replication: peer="{Voter server-9fdd3a18-c17b-507e-24f3-d63df555b804 9fdd3a18-c17b-507e-24f3-d63df555b804}" -2025-01-28T14:48:43.412Z [INFO] server-2577410f-84cf-07c4-74d3-e5d05021e9d7: pipelining replication: peer="{Voter server-faea0b1f-5b46-7485-0d7f-f26a767ab98a faea0b1f-5b46-7485-0d7f-f26a767ab98a}" -2025-01-28T14:48:43.413Z [INFO] server-2577410f-84cf-07c4-74d3-e5d05021e9d7: pipelining replication: peer="{Voter server-45375828-775a-7428-e34a-83c3dfd99ef9 45375828-775a-7428-e34a-83c3dfd99ef9}" -2025-01-28T14:48:43.414Z [INFO] server-2577410f-84cf-07c4-74d3-e5d05021e9d7: pipelining replication: peer="{Voter server-9b7543b8-bf54-9d8a-5f42-a571c32df417 9b7543b8-bf54-9d8a-5f42-a571c32df417}" -2025-01-28T14:48:43.516Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:43.410593726 +0000 UTC m=+106.493999084 (1 nodes), 53.572591ms from start of poll, 63.792228ms from cluster start. Timeout at 2025-01-28 14:48:43.51560306 +0000 UTC m=+106.599008085, 105.009001ms after stability -2025-01-28T14:48:43.516Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:48:43.623Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:48:43.516410869 +0000 UTC m=+106.599815560 (6 nodes), 11.334µs from start of poll, 169.608704ms from cluster start. Timeout at 2025-01-28 14:48:43.621950885 +0000 UTC m=+106.705355909, 105.540349ms after stability -2025-01-28T14:48:43.623Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:43.729Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:43.623658387 +0000 UTC m=+106.707063078 (1 nodes), 9µs from start of poll, 276.856222ms from cluster start. Timeout at 2025-01-28 14:48:43.729067066 +0000 UTC m=+106.812472090, 105.409012ms after stability -2025-01-28T14:48:43.729Z [INFO] server-af54d44b-015a-18e6-da85-46538a6c4186: entering candidate state: node="Node at af54d44b-015a-18e6-da85-46538a6c4186 [Candidate]" term=3 -2025-01-28T14:48:43.731Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:43.731Z [INFO] server-af54d44b-015a-18e6-da85-46538a6c4186: election won: term=3 tally=4 -2025-01-28T14:48:43.731Z [INFO] server-2577410f-84cf-07c4-74d3-e5d05021e9d7: entering follower state: follower="Node at 2577410f-84cf-07c4-74d3-e5d05021e9d7 [Follower]" leader-address= leader-id= -2025-01-28T14:48:43.731Z [INFO] server-af54d44b-015a-18e6-da85-46538a6c4186: entering leader state: leader="Node at af54d44b-015a-18e6-da85-46538a6c4186 [Leader]" -2025-01-28T14:48:43.732Z [INFO] server-af54d44b-015a-18e6-da85-46538a6c4186: added peer, starting replication: peer=server-995f613f-bb00-44bd-a796-36ce725eb03a -2025-01-28T14:48:43.732Z [INFO] server-af54d44b-015a-18e6-da85-46538a6c4186: added peer, starting replication: peer=server-9b7543b8-bf54-9d8a-5f42-a571c32df417 -2025-01-28T14:48:43.732Z [INFO] server-af54d44b-015a-18e6-da85-46538a6c4186: added peer, starting replication: peer=server-45375828-775a-7428-e34a-83c3dfd99ef9 -2025-01-28T14:48:43.732Z [INFO] server-af54d44b-015a-18e6-da85-46538a6c4186: added peer, starting replication: peer=server-2577410f-84cf-07c4-74d3-e5d05021e9d7 -2025-01-28T14:48:43.732Z [INFO] server-2577410f-84cf-07c4-74d3-e5d05021e9d7: aborting pipeline replication: peer="{Voter server-9fdd3a18-c17b-507e-24f3-d63df555b804 9fdd3a18-c17b-507e-24f3-d63df555b804}" -2025-01-28T14:48:43.732Z [INFO] server-af54d44b-015a-18e6-da85-46538a6c4186: added peer, starting replication: peer=server-9fdd3a18-c17b-507e-24f3-d63df555b804 -2025-01-28T14:48:43.732Z [INFO] server-af54d44b-015a-18e6-da85-46538a6c4186: added peer, starting replication: peer=server-faea0b1f-5b46-7485-0d7f-f26a767ab98a -2025-01-28T14:48:43.732Z [INFO] server-2577410f-84cf-07c4-74d3-e5d05021e9d7: aborting pipeline replication: peer="{Voter server-9b7543b8-bf54-9d8a-5f42-a571c32df417 9b7543b8-bf54-9d8a-5f42-a571c32df417}" -2025-01-28T14:48:43.732Z [INFO] server-2577410f-84cf-07c4-74d3-e5d05021e9d7: aborting pipeline replication: peer="{Voter server-45375828-775a-7428-e34a-83c3dfd99ef9 45375828-775a-7428-e34a-83c3dfd99ef9}" -2025-01-28T14:48:43.733Z [INFO] server-2577410f-84cf-07c4-74d3-e5d05021e9d7: aborting pipeline replication: peer="{Voter server-faea0b1f-5b46-7485-0d7f-f26a767ab98a faea0b1f-5b46-7485-0d7f-f26a767ab98a}" -2025-01-28T14:48:43.733Z [INFO] server-2577410f-84cf-07c4-74d3-e5d05021e9d7: aborting pipeline replication: peer="{Voter server-af54d44b-015a-18e6-da85-46538a6c4186 af54d44b-015a-18e6-da85-46538a6c4186}" -2025-01-28T14:48:43.733Z [INFO] server-2577410f-84cf-07c4-74d3-e5d05021e9d7: aborting pipeline replication: peer="{Voter server-995f613f-bb00-44bd-a796-36ce725eb03a 995f613f-bb00-44bd-a796-36ce725eb03a}" -2025-01-28T14:48:43.733Z [INFO] server-af54d44b-015a-18e6-da85-46538a6c4186: pipelining replication: peer="{Voter server-2577410f-84cf-07c4-74d3-e5d05021e9d7 2577410f-84cf-07c4-74d3-e5d05021e9d7}" -2025-01-28T14:48:43.733Z [INFO] server-af54d44b-015a-18e6-da85-46538a6c4186: pipelining replication: peer="{Voter server-995f613f-bb00-44bd-a796-36ce725eb03a 995f613f-bb00-44bd-a796-36ce725eb03a}" -2025-01-28T14:48:43.734Z [INFO] server-af54d44b-015a-18e6-da85-46538a6c4186: pipelining replication: peer="{Voter server-faea0b1f-5b46-7485-0d7f-f26a767ab98a faea0b1f-5b46-7485-0d7f-f26a767ab98a}" -2025-01-28T14:48:43.734Z [INFO] server-af54d44b-015a-18e6-da85-46538a6c4186: pipelining replication: peer="{Voter server-9fdd3a18-c17b-507e-24f3-d63df555b804 9fdd3a18-c17b-507e-24f3-d63df555b804}" -2025-01-28T14:48:43.734Z [INFO] server-af54d44b-015a-18e6-da85-46538a6c4186: pipelining replication: peer="{Voter server-9b7543b8-bf54-9d8a-5f42-a571c32df417 9b7543b8-bf54-9d8a-5f42-a571c32df417}" -2025-01-28T14:48:43.733Z [INFO] server-af54d44b-015a-18e6-da85-46538a6c4186: pipelining replication: peer="{Voter server-45375828-775a-7428-e34a-83c3dfd99ef9 45375828-775a-7428-e34a-83c3dfd99ef9}" -2025-01-28T14:48:43.840Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:43.734250005 +0000 UTC m=+106.817654696 (1 nodes), 2.908086ms from start of poll, 387.44784ms from cluster start. Timeout at 2025-01-28 14:48:43.839323674 +0000 UTC m=+106.922728365, 105.073669ms after stability -2025-01-28T14:48:43.841Z [INFO] server-af54d44b-015a-18e6-da85-46538a6c4186: aborting pipeline replication: peer="{Voter server-faea0b1f-5b46-7485-0d7f-f26a767ab98a faea0b1f-5b46-7485-0d7f-f26a767ab98a}" -2025-01-28T14:48:43.841Z [INFO] server-af54d44b-015a-18e6-da85-46538a6c4186: aborting pipeline replication: peer="{Voter server-45375828-775a-7428-e34a-83c3dfd99ef9 45375828-775a-7428-e34a-83c3dfd99ef9}" -2025-01-28T14:48:43.841Z [INFO] server-af54d44b-015a-18e6-da85-46538a6c4186: aborting pipeline replication: peer="{Voter server-995f613f-bb00-44bd-a796-36ce725eb03a 995f613f-bb00-44bd-a796-36ce725eb03a}" -2025-01-28T14:48:43.841Z [INFO] server-af54d44b-015a-18e6-da85-46538a6c4186: aborting pipeline replication: peer="{Voter server-9b7543b8-bf54-9d8a-5f42-a571c32df417 9b7543b8-bf54-9d8a-5f42-a571c32df417}" -2025-01-28T14:48:43.841Z [INFO] server-af54d44b-015a-18e6-da85-46538a6c4186: aborting pipeline replication: peer="{Voter server-9fdd3a18-c17b-507e-24f3-d63df555b804 9fdd3a18-c17b-507e-24f3-d63df555b804}" -2025-01-28T14:48:43.841Z [INFO] server-af54d44b-015a-18e6-da85-46538a6c4186: aborting pipeline replication: peer="{Voter server-2577410f-84cf-07c4-74d3-e5d05021e9d7 2577410f-84cf-07c4-74d3-e5d05021e9d7}" -2025-01-28T14:48:44.340Z [ERROR] server-af54d44b-015a-18e6-da85-46538a6c4186: failed to heartbeat to: peer=9b7543b8-bf54-9d8a-5f42-a571c32df417 backoff time=10ms error="command timed out" ---- PASS: TestRaft_LeadershipTransferWithSevenNodes (1.01s) +2025-01-28T14:55:45.069Z [INFO] server-67f33648-437a-847e-a9cf-6bbd6b981188: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-67f33648-437a-847e-a9cf-6bbd6b981188 Address:67f33648-437a-847e-a9cf-6bbd6b981188} {Suffrage:Voter ID:server-48509be3-7c81-5153-ded7-68466927abae Address:48509be3-7c81-5153-ded7-68466927abae} {Suffrage:Voter ID:server-f73da7ab-19fa-ec89-0c4f-7831a90fffdc Address:f73da7ab-19fa-ec89-0c4f-7831a90fffdc} {Suffrage:Voter ID:server-7b554427-bd99-fe04-c520-fd1722bcebdf Address:7b554427-bd99-fe04-c520-fd1722bcebdf} {Suffrage:Voter ID:server-56b0cb4f-a302-06ed-3363-683976f5efa9 Address:56b0cb4f-a302-06ed-3363-683976f5efa9} {Suffrage:Voter ID:server-5b1e3e51-dcf4-3845-5297-0841920f5ae9 Address:5b1e3e51-dcf4-3845-5297-0841920f5ae9} {Suffrage:Voter ID:server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e Address:e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e}]" +2025-01-28T14:55:45.069Z [INFO] server-67f33648-437a-847e-a9cf-6bbd6b981188: entering follower state: follower="Node at 67f33648-437a-847e-a9cf-6bbd6b981188 [Follower]" leader-address= leader-id= +2025-01-28T14:55:45.069Z [INFO] server-48509be3-7c81-5153-ded7-68466927abae: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-67f33648-437a-847e-a9cf-6bbd6b981188 Address:67f33648-437a-847e-a9cf-6bbd6b981188} {Suffrage:Voter ID:server-48509be3-7c81-5153-ded7-68466927abae Address:48509be3-7c81-5153-ded7-68466927abae} {Suffrage:Voter ID:server-f73da7ab-19fa-ec89-0c4f-7831a90fffdc Address:f73da7ab-19fa-ec89-0c4f-7831a90fffdc} {Suffrage:Voter ID:server-7b554427-bd99-fe04-c520-fd1722bcebdf Address:7b554427-bd99-fe04-c520-fd1722bcebdf} {Suffrage:Voter ID:server-56b0cb4f-a302-06ed-3363-683976f5efa9 Address:56b0cb4f-a302-06ed-3363-683976f5efa9} {Suffrage:Voter ID:server-5b1e3e51-dcf4-3845-5297-0841920f5ae9 Address:5b1e3e51-dcf4-3845-5297-0841920f5ae9} {Suffrage:Voter ID:server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e Address:e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e}]" +2025-01-28T14:55:45.069Z [INFO] server-48509be3-7c81-5153-ded7-68466927abae: entering follower state: follower="Node at 48509be3-7c81-5153-ded7-68466927abae [Follower]" leader-address= leader-id= +2025-01-28T14:55:45.069Z [INFO] server-f73da7ab-19fa-ec89-0c4f-7831a90fffdc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-67f33648-437a-847e-a9cf-6bbd6b981188 Address:67f33648-437a-847e-a9cf-6bbd6b981188} {Suffrage:Voter ID:server-48509be3-7c81-5153-ded7-68466927abae Address:48509be3-7c81-5153-ded7-68466927abae} {Suffrage:Voter ID:server-f73da7ab-19fa-ec89-0c4f-7831a90fffdc Address:f73da7ab-19fa-ec89-0c4f-7831a90fffdc} {Suffrage:Voter ID:server-7b554427-bd99-fe04-c520-fd1722bcebdf Address:7b554427-bd99-fe04-c520-fd1722bcebdf} {Suffrage:Voter ID:server-56b0cb4f-a302-06ed-3363-683976f5efa9 Address:56b0cb4f-a302-06ed-3363-683976f5efa9} {Suffrage:Voter ID:server-5b1e3e51-dcf4-3845-5297-0841920f5ae9 Address:5b1e3e51-dcf4-3845-5297-0841920f5ae9} {Suffrage:Voter ID:server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e Address:e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e}]" +2025-01-28T14:55:45.070Z [INFO] server-f73da7ab-19fa-ec89-0c4f-7831a90fffdc: entering follower state: follower="Node at f73da7ab-19fa-ec89-0c4f-7831a90fffdc [Follower]" leader-address= leader-id= +2025-01-28T14:55:45.070Z [INFO] server-7b554427-bd99-fe04-c520-fd1722bcebdf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-67f33648-437a-847e-a9cf-6bbd6b981188 Address:67f33648-437a-847e-a9cf-6bbd6b981188} {Suffrage:Voter ID:server-48509be3-7c81-5153-ded7-68466927abae Address:48509be3-7c81-5153-ded7-68466927abae} {Suffrage:Voter ID:server-f73da7ab-19fa-ec89-0c4f-7831a90fffdc Address:f73da7ab-19fa-ec89-0c4f-7831a90fffdc} {Suffrage:Voter ID:server-7b554427-bd99-fe04-c520-fd1722bcebdf Address:7b554427-bd99-fe04-c520-fd1722bcebdf} {Suffrage:Voter ID:server-56b0cb4f-a302-06ed-3363-683976f5efa9 Address:56b0cb4f-a302-06ed-3363-683976f5efa9} {Suffrage:Voter ID:server-5b1e3e51-dcf4-3845-5297-0841920f5ae9 Address:5b1e3e51-dcf4-3845-5297-0841920f5ae9} {Suffrage:Voter ID:server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e Address:e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e}]" +2025-01-28T14:55:45.070Z [INFO] server-7b554427-bd99-fe04-c520-fd1722bcebdf: entering follower state: follower="Node at 7b554427-bd99-fe04-c520-fd1722bcebdf [Follower]" leader-address= leader-id= +2025-01-28T14:55:45.070Z [INFO] server-56b0cb4f-a302-06ed-3363-683976f5efa9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-67f33648-437a-847e-a9cf-6bbd6b981188 Address:67f33648-437a-847e-a9cf-6bbd6b981188} {Suffrage:Voter ID:server-48509be3-7c81-5153-ded7-68466927abae Address:48509be3-7c81-5153-ded7-68466927abae} {Suffrage:Voter ID:server-f73da7ab-19fa-ec89-0c4f-7831a90fffdc Address:f73da7ab-19fa-ec89-0c4f-7831a90fffdc} {Suffrage:Voter ID:server-7b554427-bd99-fe04-c520-fd1722bcebdf Address:7b554427-bd99-fe04-c520-fd1722bcebdf} {Suffrage:Voter ID:server-56b0cb4f-a302-06ed-3363-683976f5efa9 Address:56b0cb4f-a302-06ed-3363-683976f5efa9} {Suffrage:Voter ID:server-5b1e3e51-dcf4-3845-5297-0841920f5ae9 Address:5b1e3e51-dcf4-3845-5297-0841920f5ae9} {Suffrage:Voter ID:server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e Address:e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e}]" +2025-01-28T14:55:45.070Z [INFO] server-56b0cb4f-a302-06ed-3363-683976f5efa9: entering follower state: follower="Node at 56b0cb4f-a302-06ed-3363-683976f5efa9 [Follower]" leader-address= leader-id= +2025-01-28T14:55:45.070Z [INFO] server-5b1e3e51-dcf4-3845-5297-0841920f5ae9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-67f33648-437a-847e-a9cf-6bbd6b981188 Address:67f33648-437a-847e-a9cf-6bbd6b981188} {Suffrage:Voter ID:server-48509be3-7c81-5153-ded7-68466927abae Address:48509be3-7c81-5153-ded7-68466927abae} {Suffrage:Voter ID:server-f73da7ab-19fa-ec89-0c4f-7831a90fffdc Address:f73da7ab-19fa-ec89-0c4f-7831a90fffdc} {Suffrage:Voter ID:server-7b554427-bd99-fe04-c520-fd1722bcebdf Address:7b554427-bd99-fe04-c520-fd1722bcebdf} {Suffrage:Voter ID:server-56b0cb4f-a302-06ed-3363-683976f5efa9 Address:56b0cb4f-a302-06ed-3363-683976f5efa9} {Suffrage:Voter ID:server-5b1e3e51-dcf4-3845-5297-0841920f5ae9 Address:5b1e3e51-dcf4-3845-5297-0841920f5ae9} {Suffrage:Voter ID:server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e Address:e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e}]" +2025-01-28T14:55:45.071Z [INFO] server-5b1e3e51-dcf4-3845-5297-0841920f5ae9: entering follower state: follower="Node at 5b1e3e51-dcf4-3845-5297-0841920f5ae9 [Follower]" leader-address= leader-id= +2025-01-28T14:55:45.071Z [INFO] server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-67f33648-437a-847e-a9cf-6bbd6b981188 Address:67f33648-437a-847e-a9cf-6bbd6b981188} {Suffrage:Voter ID:server-48509be3-7c81-5153-ded7-68466927abae Address:48509be3-7c81-5153-ded7-68466927abae} {Suffrage:Voter ID:server-f73da7ab-19fa-ec89-0c4f-7831a90fffdc Address:f73da7ab-19fa-ec89-0c4f-7831a90fffdc} {Suffrage:Voter ID:server-7b554427-bd99-fe04-c520-fd1722bcebdf Address:7b554427-bd99-fe04-c520-fd1722bcebdf} {Suffrage:Voter ID:server-56b0cb4f-a302-06ed-3363-683976f5efa9 Address:56b0cb4f-a302-06ed-3363-683976f5efa9} {Suffrage:Voter ID:server-5b1e3e51-dcf4-3845-5297-0841920f5ae9 Address:5b1e3e51-dcf4-3845-5297-0841920f5ae9} {Suffrage:Voter ID:server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e Address:e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e}]" +2025-01-28T14:55:45.071Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:45.071Z [INFO] server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e: entering follower state: follower="Node at e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e [Follower]" leader-address= leader-id= +2025-01-28T14:55:45.130Z [WARN] server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:45.130Z [INFO] server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e: entering candidate state: node="Node at e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e [Candidate]" term=2 +2025-01-28T14:55:45.130Z [INFO] server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e: election won: term=2 tally=4 +2025-01-28T14:55:45.130Z [INFO] server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e: entering leader state: leader="Node at e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e [Leader]" +2025-01-28T14:55:45.130Z [INFO] server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e: added peer, starting replication: peer=server-67f33648-437a-847e-a9cf-6bbd6b981188 +2025-01-28T14:55:45.130Z [INFO] server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e: added peer, starting replication: peer=server-48509be3-7c81-5153-ded7-68466927abae +2025-01-28T14:55:45.130Z [INFO] server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e: added peer, starting replication: peer=server-f73da7ab-19fa-ec89-0c4f-7831a90fffdc +2025-01-28T14:55:45.130Z [INFO] server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e: added peer, starting replication: peer=server-7b554427-bd99-fe04-c520-fd1722bcebdf +2025-01-28T14:55:45.130Z [INFO] server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e: added peer, starting replication: peer=server-56b0cb4f-a302-06ed-3363-683976f5efa9 +2025-01-28T14:55:45.130Z [INFO] server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e: added peer, starting replication: peer=server-5b1e3e51-dcf4-3845-5297-0841920f5ae9 +2025-01-28T14:55:45.131Z [INFO] server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e: pipelining replication: peer="{Voter server-67f33648-437a-847e-a9cf-6bbd6b981188 67f33648-437a-847e-a9cf-6bbd6b981188}" +2025-01-28T14:55:45.131Z [INFO] server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e: pipelining replication: peer="{Voter server-f73da7ab-19fa-ec89-0c4f-7831a90fffdc f73da7ab-19fa-ec89-0c4f-7831a90fffdc}" +2025-01-28T14:55:45.131Z [INFO] server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e: pipelining replication: peer="{Voter server-5b1e3e51-dcf4-3845-5297-0841920f5ae9 5b1e3e51-dcf4-3845-5297-0841920f5ae9}" +2025-01-28T14:55:45.131Z [INFO] server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e: pipelining replication: peer="{Voter server-7b554427-bd99-fe04-c520-fd1722bcebdf 7b554427-bd99-fe04-c520-fd1722bcebdf}" +2025-01-28T14:55:45.131Z [INFO] server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e: pipelining replication: peer="{Voter server-48509be3-7c81-5153-ded7-68466927abae 48509be3-7c81-5153-ded7-68466927abae}" +2025-01-28T14:55:45.131Z [INFO] server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e: pipelining replication: peer="{Voter server-56b0cb4f-a302-06ed-3363-683976f5efa9 56b0cb4f-a302-06ed-3363-683976f5efa9}" +2025-01-28T14:55:45.236Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:45.130735044 +0000 UTC m=+130.805225064 (1 nodes), 59.298646ms from start of poll, 61.891102ms from cluster start. Timeout at 2025-01-28 14:55:45.235754322 +0000 UTC m=+130.910244302, 105.019238ms after stability +2025-01-28T14:55:45.236Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:55:45.342Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:55:45.236650592 +0000 UTC m=+130.911140592 (6 nodes), 3.26µs from start of poll, 167.80663ms from cluster start. Timeout at 2025-01-28 14:55:45.341802738 +0000 UTC m=+131.016292738, 105.152146ms after stability +2025-01-28T14:55:45.342Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:45.447Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:45.342272508 +0000 UTC m=+131.016762488 (1 nodes), 2.22µs from start of poll, 273.428526ms from cluster start. Timeout at 2025-01-28 14:55:45.44746263 +0000 UTC m=+131.121952630, 105.190142ms after stability +2025-01-28T14:55:45.447Z [INFO] server-67f33648-437a-847e-a9cf-6bbd6b981188: entering candidate state: node="Node at 67f33648-437a-847e-a9cf-6bbd6b981188 [Candidate]" term=3 +2025-01-28T14:55:45.447Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:45.448Z [INFO] server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e: entering follower state: follower="Node at e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e [Follower]" leader-address= leader-id= +2025-01-28T14:55:45.448Z [INFO] server-67f33648-437a-847e-a9cf-6bbd6b981188: election won: term=3 tally=4 +2025-01-28T14:55:45.448Z [INFO] server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e: aborting pipeline replication: peer="{Voter server-7b554427-bd99-fe04-c520-fd1722bcebdf 7b554427-bd99-fe04-c520-fd1722bcebdf}" +2025-01-28T14:55:45.448Z [INFO] server-67f33648-437a-847e-a9cf-6bbd6b981188: entering leader state: leader="Node at 67f33648-437a-847e-a9cf-6bbd6b981188 [Leader]" +2025-01-28T14:55:45.448Z [INFO] server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e: aborting pipeline replication: peer="{Voter server-67f33648-437a-847e-a9cf-6bbd6b981188 67f33648-437a-847e-a9cf-6bbd6b981188}" +2025-01-28T14:55:45.448Z [INFO] server-67f33648-437a-847e-a9cf-6bbd6b981188: added peer, starting replication: peer=server-48509be3-7c81-5153-ded7-68466927abae +2025-01-28T14:55:45.448Z [INFO] server-67f33648-437a-847e-a9cf-6bbd6b981188: added peer, starting replication: peer=server-f73da7ab-19fa-ec89-0c4f-7831a90fffdc +2025-01-28T14:55:45.448Z [INFO] server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e: aborting pipeline replication: peer="{Voter server-56b0cb4f-a302-06ed-3363-683976f5efa9 56b0cb4f-a302-06ed-3363-683976f5efa9}" +2025-01-28T14:55:45.448Z [INFO] server-67f33648-437a-847e-a9cf-6bbd6b981188: added peer, starting replication: peer=server-7b554427-bd99-fe04-c520-fd1722bcebdf +2025-01-28T14:55:45.448Z [INFO] server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e: aborting pipeline replication: peer="{Voter server-48509be3-7c81-5153-ded7-68466927abae 48509be3-7c81-5153-ded7-68466927abae}" +2025-01-28T14:55:45.448Z [INFO] server-67f33648-437a-847e-a9cf-6bbd6b981188: added peer, starting replication: peer=server-56b0cb4f-a302-06ed-3363-683976f5efa9 +2025-01-28T14:55:45.448Z [INFO] server-67f33648-437a-847e-a9cf-6bbd6b981188: added peer, starting replication: peer=server-5b1e3e51-dcf4-3845-5297-0841920f5ae9 +2025-01-28T14:55:45.448Z [INFO] server-67f33648-437a-847e-a9cf-6bbd6b981188: added peer, starting replication: peer=server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e +2025-01-28T14:55:45.448Z [INFO] server-67f33648-437a-847e-a9cf-6bbd6b981188: pipelining replication: peer="{Voter server-56b0cb4f-a302-06ed-3363-683976f5efa9 56b0cb4f-a302-06ed-3363-683976f5efa9}" +2025-01-28T14:55:45.448Z [INFO] server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e: aborting pipeline replication: peer="{Voter server-5b1e3e51-dcf4-3845-5297-0841920f5ae9 5b1e3e51-dcf4-3845-5297-0841920f5ae9}" +2025-01-28T14:55:45.448Z [INFO] server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e: aborting pipeline replication: peer="{Voter server-f73da7ab-19fa-ec89-0c4f-7831a90fffdc f73da7ab-19fa-ec89-0c4f-7831a90fffdc}" +2025-01-28T14:55:45.448Z [INFO] server-67f33648-437a-847e-a9cf-6bbd6b981188: pipelining replication: peer="{Voter server-f73da7ab-19fa-ec89-0c4f-7831a90fffdc f73da7ab-19fa-ec89-0c4f-7831a90fffdc}" +2025-01-28T14:55:45.448Z [INFO] server-67f33648-437a-847e-a9cf-6bbd6b981188: pipelining replication: peer="{Voter server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e}" +2025-01-28T14:55:45.448Z [INFO] server-67f33648-437a-847e-a9cf-6bbd6b981188: pipelining replication: peer="{Voter server-48509be3-7c81-5153-ded7-68466927abae 48509be3-7c81-5153-ded7-68466927abae}" +2025-01-28T14:55:45.448Z [INFO] server-67f33648-437a-847e-a9cf-6bbd6b981188: pipelining replication: peer="{Voter server-7b554427-bd99-fe04-c520-fd1722bcebdf 7b554427-bd99-fe04-c520-fd1722bcebdf}" +2025-01-28T14:55:45.448Z [INFO] server-67f33648-437a-847e-a9cf-6bbd6b981188: pipelining replication: peer="{Voter server-5b1e3e51-dcf4-3845-5297-0841920f5ae9 5b1e3e51-dcf4-3845-5297-0841920f5ae9}" +2025-01-28T14:55:45.553Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:45.448345142 +0000 UTC m=+131.122835122 (1 nodes), 278.134µs from start of poll, 379.50116ms from cluster start. Timeout at 2025-01-28 14:55:45.553349042 +0000 UTC m=+131.227839042, 105.00392ms after stability +2025-01-28T14:55:45.553Z [INFO] server-67f33648-437a-847e-a9cf-6bbd6b981188: aborting pipeline replication: peer="{Voter server-56b0cb4f-a302-06ed-3363-683976f5efa9 56b0cb4f-a302-06ed-3363-683976f5efa9}" +2025-01-28T14:55:45.553Z [INFO] server-67f33648-437a-847e-a9cf-6bbd6b981188: aborting pipeline replication: peer="{Voter server-5b1e3e51-dcf4-3845-5297-0841920f5ae9 5b1e3e51-dcf4-3845-5297-0841920f5ae9}" +2025-01-28T14:55:45.553Z [INFO] server-67f33648-437a-847e-a9cf-6bbd6b981188: aborting pipeline replication: peer="{Voter server-f73da7ab-19fa-ec89-0c4f-7831a90fffdc f73da7ab-19fa-ec89-0c4f-7831a90fffdc}" +2025-01-28T14:55:45.553Z [INFO] server-67f33648-437a-847e-a9cf-6bbd6b981188: aborting pipeline replication: peer="{Voter server-e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e e9bd0c8f-47f0-0d6f-d302-92b1caf70f8e}" +2025-01-28T14:55:45.553Z [INFO] server-67f33648-437a-847e-a9cf-6bbd6b981188: aborting pipeline replication: peer="{Voter server-7b554427-bd99-fe04-c520-fd1722bcebdf 7b554427-bd99-fe04-c520-fd1722bcebdf}" +2025-01-28T14:55:45.553Z [INFO] server-67f33648-437a-847e-a9cf-6bbd6b981188: aborting pipeline replication: peer="{Voter server-48509be3-7c81-5153-ded7-68466927abae 48509be3-7c81-5153-ded7-68466927abae}" +--- PASS: TestRaft_LeadershipTransferWithSevenNodes (0.49s) === RUN TestRaft_LeadershipTransferToInvalidID -2025-01-28T14:48:44.354Z [INFO] server-bcce9e67-cdac-8446-03eb-ac75fde74318: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bcce9e67-cdac-8446-03eb-ac75fde74318 Address:bcce9e67-cdac-8446-03eb-ac75fde74318} {Suffrage:Voter ID:server-71c1c501-e310-74cd-06da-c96f93de7aa7 Address:71c1c501-e310-74cd-06da-c96f93de7aa7} {Suffrage:Voter ID:server-edac89a2-dc6f-a176-fa2b-5420b288a2a9 Address:edac89a2-dc6f-a176-fa2b-5420b288a2a9}]" -2025-01-28T14:48:44.355Z [INFO] server-71c1c501-e310-74cd-06da-c96f93de7aa7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bcce9e67-cdac-8446-03eb-ac75fde74318 Address:bcce9e67-cdac-8446-03eb-ac75fde74318} {Suffrage:Voter ID:server-71c1c501-e310-74cd-06da-c96f93de7aa7 Address:71c1c501-e310-74cd-06da-c96f93de7aa7} {Suffrage:Voter ID:server-edac89a2-dc6f-a176-fa2b-5420b288a2a9 Address:edac89a2-dc6f-a176-fa2b-5420b288a2a9}]" -2025-01-28T14:48:44.356Z [INFO] server-bcce9e67-cdac-8446-03eb-ac75fde74318: entering follower state: follower="Node at bcce9e67-cdac-8446-03eb-ac75fde74318 [Follower]" leader-address= leader-id= -2025-01-28T14:48:44.356Z [INFO] server-71c1c501-e310-74cd-06da-c96f93de7aa7: entering follower state: follower="Node at 71c1c501-e310-74cd-06da-c96f93de7aa7 [Follower]" leader-address= leader-id= -2025-01-28T14:48:44.356Z [INFO] server-edac89a2-dc6f-a176-fa2b-5420b288a2a9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bcce9e67-cdac-8446-03eb-ac75fde74318 Address:bcce9e67-cdac-8446-03eb-ac75fde74318} {Suffrage:Voter ID:server-71c1c501-e310-74cd-06da-c96f93de7aa7 Address:71c1c501-e310-74cd-06da-c96f93de7aa7} {Suffrage:Voter ID:server-edac89a2-dc6f-a176-fa2b-5420b288a2a9 Address:edac89a2-dc6f-a176-fa2b-5420b288a2a9}]" -2025-01-28T14:48:44.356Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:44.356Z [INFO] server-edac89a2-dc6f-a176-fa2b-5420b288a2a9: entering follower state: follower="Node at edac89a2-dc6f-a176-fa2b-5420b288a2a9 [Follower]" leader-address= leader-id= -2025-01-28T14:48:44.418Z [WARN] server-71c1c501-e310-74cd-06da-c96f93de7aa7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:44.418Z [INFO] server-71c1c501-e310-74cd-06da-c96f93de7aa7: entering candidate state: node="Node at 71c1c501-e310-74cd-06da-c96f93de7aa7 [Candidate]" term=2 -2025-01-28T14:48:44.419Z [INFO] server-71c1c501-e310-74cd-06da-c96f93de7aa7: election won: term=2 tally=2 -2025-01-28T14:48:44.419Z [INFO] server-71c1c501-e310-74cd-06da-c96f93de7aa7: entering leader state: leader="Node at 71c1c501-e310-74cd-06da-c96f93de7aa7 [Leader]" -2025-01-28T14:48:44.419Z [INFO] server-71c1c501-e310-74cd-06da-c96f93de7aa7: added peer, starting replication: peer=server-bcce9e67-cdac-8446-03eb-ac75fde74318 -2025-01-28T14:48:44.419Z [INFO] server-71c1c501-e310-74cd-06da-c96f93de7aa7: added peer, starting replication: peer=server-edac89a2-dc6f-a176-fa2b-5420b288a2a9 -2025-01-28T14:48:44.440Z [INFO] server-71c1c501-e310-74cd-06da-c96f93de7aa7: pipelining replication: peer="{Voter server-edac89a2-dc6f-a176-fa2b-5420b288a2a9 edac89a2-dc6f-a176-fa2b-5420b288a2a9}" -2025-01-28T14:48:44.441Z [INFO] server-71c1c501-e310-74cd-06da-c96f93de7aa7: pipelining replication: peer="{Voter server-bcce9e67-cdac-8446-03eb-ac75fde74318 bcce9e67-cdac-8446-03eb-ac75fde74318}" -2025-01-28T14:48:44.524Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:44.419552691 +0000 UTC m=+107.502957715 (1 nodes), 62.62186ms from start of poll, 65.332607ms from cluster start. Timeout at 2025-01-28 14:48:44.524562024 +0000 UTC m=+107.607966715, 105.009ms after stability -2025-01-28T14:48:44.525Z [INFO] server-71c1c501-e310-74cd-06da-c96f93de7aa7: aborting pipeline replication: peer="{Voter server-edac89a2-dc6f-a176-fa2b-5420b288a2a9 edac89a2-dc6f-a176-fa2b-5420b288a2a9}" -2025-01-28T14:48:44.525Z [INFO] server-71c1c501-e310-74cd-06da-c96f93de7aa7: aborting pipeline replication: peer="{Voter server-bcce9e67-cdac-8446-03eb-ac75fde74318 bcce9e67-cdac-8446-03eb-ac75fde74318}" -2025-01-28T14:48:45.025Z [ERROR] server-71c1c501-e310-74cd-06da-c96f93de7aa7: failed to heartbeat to: peer=bcce9e67-cdac-8446-03eb-ac75fde74318 backoff time=10ms error="command timed out" ---- PASS: TestRaft_LeadershipTransferToInvalidID (0.68s) +2025-01-28T14:55:45.559Z [INFO] server-6d2a0a7b-da66-3929-6249-abc675f7b1f0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6d2a0a7b-da66-3929-6249-abc675f7b1f0 Address:6d2a0a7b-da66-3929-6249-abc675f7b1f0} {Suffrage:Voter ID:server-4800695e-99c8-aed5-49e7-1a183d4648f9 Address:4800695e-99c8-aed5-49e7-1a183d4648f9} {Suffrage:Voter ID:server-a901fd98-466c-c269-31f1-d8806e950d4c Address:a901fd98-466c-c269-31f1-d8806e950d4c}]" +2025-01-28T14:55:45.560Z [INFO] server-6d2a0a7b-da66-3929-6249-abc675f7b1f0: entering follower state: follower="Node at 6d2a0a7b-da66-3929-6249-abc675f7b1f0 [Follower]" leader-address= leader-id= +2025-01-28T14:55:45.560Z [INFO] server-4800695e-99c8-aed5-49e7-1a183d4648f9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6d2a0a7b-da66-3929-6249-abc675f7b1f0 Address:6d2a0a7b-da66-3929-6249-abc675f7b1f0} {Suffrage:Voter ID:server-4800695e-99c8-aed5-49e7-1a183d4648f9 Address:4800695e-99c8-aed5-49e7-1a183d4648f9} {Suffrage:Voter ID:server-a901fd98-466c-c269-31f1-d8806e950d4c Address:a901fd98-466c-c269-31f1-d8806e950d4c}]" +2025-01-28T14:55:45.560Z [INFO] server-4800695e-99c8-aed5-49e7-1a183d4648f9: entering follower state: follower="Node at 4800695e-99c8-aed5-49e7-1a183d4648f9 [Follower]" leader-address= leader-id= +2025-01-28T14:55:45.560Z [INFO] server-a901fd98-466c-c269-31f1-d8806e950d4c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6d2a0a7b-da66-3929-6249-abc675f7b1f0 Address:6d2a0a7b-da66-3929-6249-abc675f7b1f0} {Suffrage:Voter ID:server-4800695e-99c8-aed5-49e7-1a183d4648f9 Address:4800695e-99c8-aed5-49e7-1a183d4648f9} {Suffrage:Voter ID:server-a901fd98-466c-c269-31f1-d8806e950d4c Address:a901fd98-466c-c269-31f1-d8806e950d4c}]" +2025-01-28T14:55:45.560Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:45.560Z [INFO] server-a901fd98-466c-c269-31f1-d8806e950d4c: entering follower state: follower="Node at a901fd98-466c-c269-31f1-d8806e950d4c [Follower]" leader-address= leader-id= +2025-01-28T14:55:45.612Z [WARN] server-4800695e-99c8-aed5-49e7-1a183d4648f9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:45.612Z [INFO] server-4800695e-99c8-aed5-49e7-1a183d4648f9: entering candidate state: node="Node at 4800695e-99c8-aed5-49e7-1a183d4648f9 [Candidate]" term=2 +2025-01-28T14:55:45.612Z [INFO] server-4800695e-99c8-aed5-49e7-1a183d4648f9: election won: term=2 tally=2 +2025-01-28T14:55:45.612Z [INFO] server-4800695e-99c8-aed5-49e7-1a183d4648f9: entering leader state: leader="Node at 4800695e-99c8-aed5-49e7-1a183d4648f9 [Leader]" +2025-01-28T14:55:45.612Z [INFO] server-4800695e-99c8-aed5-49e7-1a183d4648f9: added peer, starting replication: peer=server-6d2a0a7b-da66-3929-6249-abc675f7b1f0 +2025-01-28T14:55:45.612Z [INFO] server-4800695e-99c8-aed5-49e7-1a183d4648f9: added peer, starting replication: peer=server-a901fd98-466c-c269-31f1-d8806e950d4c +2025-01-28T14:55:45.612Z [INFO] server-4800695e-99c8-aed5-49e7-1a183d4648f9: pipelining replication: peer="{Voter server-a901fd98-466c-c269-31f1-d8806e950d4c a901fd98-466c-c269-31f1-d8806e950d4c}" +2025-01-28T14:55:45.612Z [INFO] server-4800695e-99c8-aed5-49e7-1a183d4648f9: pipelining replication: peer="{Voter server-6d2a0a7b-da66-3929-6249-abc675f7b1f0 6d2a0a7b-da66-3929-6249-abc675f7b1f0}" +2025-01-28T14:55:45.718Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:45.612416104 +0000 UTC m=+131.286906064 (1 nodes), 51.680725ms from start of poll, 52.68519ms from cluster start. Timeout at 2025-01-28 14:55:45.717420284 +0000 UTC m=+131.391910304, 105.00424ms after stability +2025-01-28T14:55:45.718Z [INFO] server-4800695e-99c8-aed5-49e7-1a183d4648f9: aborting pipeline replication: peer="{Voter server-a901fd98-466c-c269-31f1-d8806e950d4c a901fd98-466c-c269-31f1-d8806e950d4c}" +2025-01-28T14:55:45.718Z [INFO] server-4800695e-99c8-aed5-49e7-1a183d4648f9: aborting pipeline replication: peer="{Voter server-6d2a0a7b-da66-3929-6249-abc675f7b1f0 6d2a0a7b-da66-3929-6249-abc675f7b1f0}" +--- PASS: TestRaft_LeadershipTransferToInvalidID (0.16s) === RUN TestRaft_LeadershipTransferToInvalidAddress -2025-01-28T14:48:45.036Z [INFO] server-b3939ce3-51d9-ea5f-5fd4-92ddc181e72d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b3939ce3-51d9-ea5f-5fd4-92ddc181e72d Address:b3939ce3-51d9-ea5f-5fd4-92ddc181e72d} {Suffrage:Voter ID:server-4a904f83-b0b9-6414-5409-934f37263ec1 Address:4a904f83-b0b9-6414-5409-934f37263ec1} {Suffrage:Voter ID:server-925a0735-4560-c63f-8b19-0a1fc7efaae2 Address:925a0735-4560-c63f-8b19-0a1fc7efaae2}]" -2025-01-28T14:48:45.036Z [INFO] server-b3939ce3-51d9-ea5f-5fd4-92ddc181e72d: entering follower state: follower="Node at b3939ce3-51d9-ea5f-5fd4-92ddc181e72d [Follower]" leader-address= leader-id= -2025-01-28T14:48:45.037Z [INFO] server-4a904f83-b0b9-6414-5409-934f37263ec1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b3939ce3-51d9-ea5f-5fd4-92ddc181e72d Address:b3939ce3-51d9-ea5f-5fd4-92ddc181e72d} {Suffrage:Voter ID:server-4a904f83-b0b9-6414-5409-934f37263ec1 Address:4a904f83-b0b9-6414-5409-934f37263ec1} {Suffrage:Voter ID:server-925a0735-4560-c63f-8b19-0a1fc7efaae2 Address:925a0735-4560-c63f-8b19-0a1fc7efaae2}]" -2025-01-28T14:48:45.037Z [INFO] server-4a904f83-b0b9-6414-5409-934f37263ec1: entering follower state: follower="Node at 4a904f83-b0b9-6414-5409-934f37263ec1 [Follower]" leader-address= leader-id= -2025-01-28T14:48:45.038Z [INFO] server-925a0735-4560-c63f-8b19-0a1fc7efaae2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b3939ce3-51d9-ea5f-5fd4-92ddc181e72d Address:b3939ce3-51d9-ea5f-5fd4-92ddc181e72d} {Suffrage:Voter ID:server-4a904f83-b0b9-6414-5409-934f37263ec1 Address:4a904f83-b0b9-6414-5409-934f37263ec1} {Suffrage:Voter ID:server-925a0735-4560-c63f-8b19-0a1fc7efaae2 Address:925a0735-4560-c63f-8b19-0a1fc7efaae2}]" -2025-01-28T14:48:45.038Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:48:45.038Z [INFO] server-925a0735-4560-c63f-8b19-0a1fc7efaae2: entering follower state: follower="Node at 925a0735-4560-c63f-8b19-0a1fc7efaae2 [Follower]" leader-address= leader-id= -2025-01-28T14:48:45.100Z [WARN] server-b3939ce3-51d9-ea5f-5fd4-92ddc181e72d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:45.100Z [INFO] server-b3939ce3-51d9-ea5f-5fd4-92ddc181e72d: entering candidate state: node="Node at b3939ce3-51d9-ea5f-5fd4-92ddc181e72d [Candidate]" term=2 -2025-01-28T14:48:45.101Z [INFO] server-b3939ce3-51d9-ea5f-5fd4-92ddc181e72d: election won: term=2 tally=2 -2025-01-28T14:48:45.101Z [INFO] server-b3939ce3-51d9-ea5f-5fd4-92ddc181e72d: entering leader state: leader="Node at b3939ce3-51d9-ea5f-5fd4-92ddc181e72d [Leader]" -2025-01-28T14:48:45.101Z [INFO] server-b3939ce3-51d9-ea5f-5fd4-92ddc181e72d: added peer, starting replication: peer=server-4a904f83-b0b9-6414-5409-934f37263ec1 -2025-01-28T14:48:45.101Z [INFO] server-b3939ce3-51d9-ea5f-5fd4-92ddc181e72d: added peer, starting replication: peer=server-925a0735-4560-c63f-8b19-0a1fc7efaae2 -2025-01-28T14:48:45.103Z [INFO] server-b3939ce3-51d9-ea5f-5fd4-92ddc181e72d: pipelining replication: peer="{Voter server-925a0735-4560-c63f-8b19-0a1fc7efaae2 925a0735-4560-c63f-8b19-0a1fc7efaae2}" -2025-01-28T14:48:45.103Z [INFO] server-b3939ce3-51d9-ea5f-5fd4-92ddc181e72d: pipelining replication: peer="{Voter server-4a904f83-b0b9-6414-5409-934f37263ec1 4a904f83-b0b9-6414-5409-934f37263ec1}" -2025-01-28T14:48:45.206Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:48:45.101130265 +0000 UTC m=+108.184535623 (2 nodes), 62.469188ms from start of poll, 65.271605ms from cluster start. Timeout at 2025-01-28 14:48:45.206221268 +0000 UTC m=+108.289626293, 105.09067ms after stability -2025-01-28T14:48:45.206Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:45.312Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:45.206887073 +0000 UTC m=+108.290292097 (1 nodes), 7µs from start of poll, 171.028079ms from cluster start. Timeout at 2025-01-28 14:48:45.31236042 +0000 UTC m=+108.395765111, 105.473014ms after stability -2025-01-28T14:48:45.313Z [INFO] server-b3939ce3-51d9-ea5f-5fd4-92ddc181e72d: aborting pipeline replication: peer="{Voter server-925a0735-4560-c63f-8b19-0a1fc7efaae2 925a0735-4560-c63f-8b19-0a1fc7efaae2}" -2025-01-28T14:48:45.313Z [INFO] server-b3939ce3-51d9-ea5f-5fd4-92ddc181e72d: aborting pipeline replication: peer="{Voter server-4a904f83-b0b9-6414-5409-934f37263ec1 4a904f83-b0b9-6414-5409-934f37263ec1}" +2025-01-28T14:55:45.722Z [INFO] server-ee87fc46-708f-2642-6a7b-fc1ad5437880: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ee87fc46-708f-2642-6a7b-fc1ad5437880 Address:ee87fc46-708f-2642-6a7b-fc1ad5437880} {Suffrage:Voter ID:server-f79ba009-bdf9-08dc-ba05-2dcd422dc759 Address:f79ba009-bdf9-08dc-ba05-2dcd422dc759} {Suffrage:Voter ID:server-df93ffd3-70da-146c-c89c-579662282942 Address:df93ffd3-70da-146c-c89c-579662282942}]" +2025-01-28T14:55:45.723Z [INFO] server-ee87fc46-708f-2642-6a7b-fc1ad5437880: entering follower state: follower="Node at ee87fc46-708f-2642-6a7b-fc1ad5437880 [Follower]" leader-address= leader-id= +2025-01-28T14:55:45.723Z [INFO] server-f79ba009-bdf9-08dc-ba05-2dcd422dc759: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ee87fc46-708f-2642-6a7b-fc1ad5437880 Address:ee87fc46-708f-2642-6a7b-fc1ad5437880} {Suffrage:Voter ID:server-f79ba009-bdf9-08dc-ba05-2dcd422dc759 Address:f79ba009-bdf9-08dc-ba05-2dcd422dc759} {Suffrage:Voter ID:server-df93ffd3-70da-146c-c89c-579662282942 Address:df93ffd3-70da-146c-c89c-579662282942}]" +2025-01-28T14:55:45.723Z [INFO] server-f79ba009-bdf9-08dc-ba05-2dcd422dc759: entering follower state: follower="Node at f79ba009-bdf9-08dc-ba05-2dcd422dc759 [Follower]" leader-address= leader-id= +2025-01-28T14:55:45.723Z [INFO] server-df93ffd3-70da-146c-c89c-579662282942: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ee87fc46-708f-2642-6a7b-fc1ad5437880 Address:ee87fc46-708f-2642-6a7b-fc1ad5437880} {Suffrage:Voter ID:server-f79ba009-bdf9-08dc-ba05-2dcd422dc759 Address:f79ba009-bdf9-08dc-ba05-2dcd422dc759} {Suffrage:Voter ID:server-df93ffd3-70da-146c-c89c-579662282942 Address:df93ffd3-70da-146c-c89c-579662282942}]" +2025-01-28T14:55:45.723Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:55:45.723Z [INFO] server-df93ffd3-70da-146c-c89c-579662282942: entering follower state: follower="Node at df93ffd3-70da-146c-c89c-579662282942 [Follower]" leader-address= leader-id= +2025-01-28T14:55:45.796Z [WARN] server-df93ffd3-70da-146c-c89c-579662282942: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:45.796Z [INFO] server-df93ffd3-70da-146c-c89c-579662282942: entering candidate state: node="Node at df93ffd3-70da-146c-c89c-579662282942 [Candidate]" term=2 +2025-01-28T14:55:45.796Z [INFO] server-df93ffd3-70da-146c-c89c-579662282942: election won: term=2 tally=2 +2025-01-28T14:55:45.796Z [INFO] server-df93ffd3-70da-146c-c89c-579662282942: entering leader state: leader="Node at df93ffd3-70da-146c-c89c-579662282942 [Leader]" +2025-01-28T14:55:45.796Z [INFO] server-df93ffd3-70da-146c-c89c-579662282942: added peer, starting replication: peer=server-ee87fc46-708f-2642-6a7b-fc1ad5437880 +2025-01-28T14:55:45.796Z [INFO] server-df93ffd3-70da-146c-c89c-579662282942: added peer, starting replication: peer=server-f79ba009-bdf9-08dc-ba05-2dcd422dc759 +2025-01-28T14:55:45.796Z [INFO] server-df93ffd3-70da-146c-c89c-579662282942: pipelining replication: peer="{Voter server-ee87fc46-708f-2642-6a7b-fc1ad5437880 ee87fc46-708f-2642-6a7b-fc1ad5437880}" +2025-01-28T14:55:45.797Z [INFO] server-df93ffd3-70da-146c-c89c-579662282942: pipelining replication: peer="{Voter server-f79ba009-bdf9-08dc-ba05-2dcd422dc759 f79ba009-bdf9-08dc-ba05-2dcd422dc759}" +2025-01-28T14:55:45.902Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:55:45.79648412 +0000 UTC m=+131.470974100 (2 nodes), 72.786894ms from start of poll, 73.898569ms from cluster start. Timeout at 2025-01-28 14:55:45.90148868 +0000 UTC m=+131.575978720, 105.00462ms after stability +2025-01-28T14:55:45.902Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:46.009Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:45.902975294 +0000 UTC m=+131.577465274 (1 nodes), 2.7µs from start of poll, 180.389743ms from cluster start. Timeout at 2025-01-28 14:55:46.008094662 +0000 UTC m=+131.682584682, 105.119408ms after stability +2025-01-28T14:55:46.009Z [INFO] server-df93ffd3-70da-146c-c89c-579662282942: aborting pipeline replication: peer="{Voter server-f79ba009-bdf9-08dc-ba05-2dcd422dc759 f79ba009-bdf9-08dc-ba05-2dcd422dc759}" +2025-01-28T14:55:46.009Z [INFO] server-df93ffd3-70da-146c-c89c-579662282942: aborting pipeline replication: peer="{Voter server-ee87fc46-708f-2642-6a7b-fc1ad5437880 ee87fc46-708f-2642-6a7b-fc1ad5437880}" --- PASS: TestRaft_LeadershipTransferToInvalidAddress (0.29s) === RUN TestRaft_LeadershipTransferToBehindServer -2025-01-28T14:48:45.322Z [INFO] server-7588cb91-eaf2-3bec-10c7-b9c66d449fb5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7588cb91-eaf2-3bec-10c7-b9c66d449fb5 Address:7588cb91-eaf2-3bec-10c7-b9c66d449fb5} {Suffrage:Voter ID:server-27644e4f-7bc4-5771-7a43-b70690d4c570 Address:27644e4f-7bc4-5771-7a43-b70690d4c570} {Suffrage:Voter ID:server-f2e555fc-6442-fcb9-6d01-ef7562218184 Address:f2e555fc-6442-fcb9-6d01-ef7562218184}]" -2025-01-28T14:48:45.323Z [INFO] server-27644e4f-7bc4-5771-7a43-b70690d4c570: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7588cb91-eaf2-3bec-10c7-b9c66d449fb5 Address:7588cb91-eaf2-3bec-10c7-b9c66d449fb5} {Suffrage:Voter ID:server-27644e4f-7bc4-5771-7a43-b70690d4c570 Address:27644e4f-7bc4-5771-7a43-b70690d4c570} {Suffrage:Voter ID:server-f2e555fc-6442-fcb9-6d01-ef7562218184 Address:f2e555fc-6442-fcb9-6d01-ef7562218184}]" -2025-01-28T14:48:45.323Z [INFO] server-7588cb91-eaf2-3bec-10c7-b9c66d449fb5: entering follower state: follower="Node at 7588cb91-eaf2-3bec-10c7-b9c66d449fb5 [Follower]" leader-address= leader-id= -2025-01-28T14:48:45.323Z [INFO] server-27644e4f-7bc4-5771-7a43-b70690d4c570: entering follower state: follower="Node at 27644e4f-7bc4-5771-7a43-b70690d4c570 [Follower]" leader-address= leader-id= -2025-01-28T14:48:45.323Z [INFO] server-f2e555fc-6442-fcb9-6d01-ef7562218184: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7588cb91-eaf2-3bec-10c7-b9c66d449fb5 Address:7588cb91-eaf2-3bec-10c7-b9c66d449fb5} {Suffrage:Voter ID:server-27644e4f-7bc4-5771-7a43-b70690d4c570 Address:27644e4f-7bc4-5771-7a43-b70690d4c570} {Suffrage:Voter ID:server-f2e555fc-6442-fcb9-6d01-ef7562218184 Address:f2e555fc-6442-fcb9-6d01-ef7562218184}]" -2025-01-28T14:48:45.323Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:45.324Z [INFO] server-f2e555fc-6442-fcb9-6d01-ef7562218184: entering follower state: follower="Node at f2e555fc-6442-fcb9-6d01-ef7562218184 [Follower]" leader-address= leader-id= -2025-01-28T14:48:45.386Z [WARN] server-27644e4f-7bc4-5771-7a43-b70690d4c570: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:45.386Z [INFO] server-27644e4f-7bc4-5771-7a43-b70690d4c570: entering candidate state: node="Node at 27644e4f-7bc4-5771-7a43-b70690d4c570 [Candidate]" term=2 -2025-01-28T14:48:45.387Z [INFO] server-27644e4f-7bc4-5771-7a43-b70690d4c570: election won: term=2 tally=2 -2025-01-28T14:48:45.387Z [INFO] server-27644e4f-7bc4-5771-7a43-b70690d4c570: entering leader state: leader="Node at 27644e4f-7bc4-5771-7a43-b70690d4c570 [Leader]" -2025-01-28T14:48:45.387Z [INFO] server-27644e4f-7bc4-5771-7a43-b70690d4c570: added peer, starting replication: peer=server-7588cb91-eaf2-3bec-10c7-b9c66d449fb5 -2025-01-28T14:48:45.387Z [INFO] server-27644e4f-7bc4-5771-7a43-b70690d4c570: added peer, starting replication: peer=server-f2e555fc-6442-fcb9-6d01-ef7562218184 -2025-01-28T14:48:45.388Z [INFO] server-27644e4f-7bc4-5771-7a43-b70690d4c570: pipelining replication: peer="{Voter server-7588cb91-eaf2-3bec-10c7-b9c66d449fb5 7588cb91-eaf2-3bec-10c7-b9c66d449fb5}" -2025-01-28T14:48:45.389Z [INFO] server-27644e4f-7bc4-5771-7a43-b70690d4c570: pipelining replication: peer="{Voter server-f2e555fc-6442-fcb9-6d01-ef7562218184 f2e555fc-6442-fcb9-6d01-ef7562218184}" -2025-01-28T14:48:45.493Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:45.388313128 +0000 UTC m=+108.471718152 (1 nodes), 64.271575ms from start of poll, 66.392305ms from cluster start. Timeout at 2025-01-28 14:48:45.493334795 +0000 UTC m=+108.576739820, 105.021668ms after stability -2025-01-28T14:48:45.493Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:48:45.599Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:48:45.493831928 +0000 UTC m=+108.577236952 (2 nodes), 9.333µs from start of poll, 171.911105ms from cluster start. Timeout at 2025-01-28 14:48:45.599393945 +0000 UTC m=+108.682798969, 105.562017ms after stability -2025-01-28T14:48:46.078Z [INFO] server-7588cb91-eaf2-3bec-10c7-b9c66d449fb5: entering candidate state: node="Node at 7588cb91-eaf2-3bec-10c7-b9c66d449fb5 [Candidate]" term=3 -2025-01-28T14:48:46.078Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:46.079Z [INFO] server-27644e4f-7bc4-5771-7a43-b70690d4c570: entering follower state: follower="Node at 27644e4f-7bc4-5771-7a43-b70690d4c570 [Follower]" leader-address= leader-id= -2025-01-28T14:48:46.079Z [INFO] server-7588cb91-eaf2-3bec-10c7-b9c66d449fb5: election won: term=3 tally=2 -2025-01-28T14:48:46.079Z [INFO] server-7588cb91-eaf2-3bec-10c7-b9c66d449fb5: entering leader state: leader="Node at 7588cb91-eaf2-3bec-10c7-b9c66d449fb5 [Leader]" -2025-01-28T14:48:46.079Z [INFO] server-27644e4f-7bc4-5771-7a43-b70690d4c570: aborting pipeline replication: peer="{Voter server-f2e555fc-6442-fcb9-6d01-ef7562218184 f2e555fc-6442-fcb9-6d01-ef7562218184}" -2025-01-28T14:48:46.079Z [INFO] server-7588cb91-eaf2-3bec-10c7-b9c66d449fb5: added peer, starting replication: peer=server-27644e4f-7bc4-5771-7a43-b70690d4c570 -2025-01-28T14:48:46.079Z [INFO] server-27644e4f-7bc4-5771-7a43-b70690d4c570: aborting pipeline replication: peer="{Voter server-7588cb91-eaf2-3bec-10c7-b9c66d449fb5 7588cb91-eaf2-3bec-10c7-b9c66d449fb5}" -2025-01-28T14:48:46.079Z [INFO] server-7588cb91-eaf2-3bec-10c7-b9c66d449fb5: added peer, starting replication: peer=server-f2e555fc-6442-fcb9-6d01-ef7562218184 -2025-01-28T14:48:46.080Z [INFO] server-7588cb91-eaf2-3bec-10c7-b9c66d449fb5: pipelining replication: peer="{Voter server-f2e555fc-6442-fcb9-6d01-ef7562218184 f2e555fc-6442-fcb9-6d01-ef7562218184}" -2025-01-28T14:48:46.080Z [INFO] server-7588cb91-eaf2-3bec-10c7-b9c66d449fb5: pipelining replication: peer="{Voter server-27644e4f-7bc4-5771-7a43-b70690d4c570 27644e4f-7bc4-5771-7a43-b70690d4c570}" -2025-01-28T14:48:46.185Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:46.07957499 +0000 UTC m=+109.162979681 (1 nodes), 667.02µs from start of poll, 757.653834ms from cluster start. Timeout at 2025-01-28 14:48:46.184701661 +0000 UTC m=+109.268106686, 105.127005ms after stability -2025-01-28T14:48:46.185Z [INFO] server-7588cb91-eaf2-3bec-10c7-b9c66d449fb5: aborting pipeline replication: peer="{Voter server-f2e555fc-6442-fcb9-6d01-ef7562218184 f2e555fc-6442-fcb9-6d01-ef7562218184}" -2025-01-28T14:48:46.185Z [INFO] server-7588cb91-eaf2-3bec-10c7-b9c66d449fb5: aborting pipeline replication: peer="{Voter server-27644e4f-7bc4-5771-7a43-b70690d4c570 27644e4f-7bc4-5771-7a43-b70690d4c570}" ---- PASS: TestRaft_LeadershipTransferToBehindServer (0.87s) +2025-01-28T14:55:46.013Z [INFO] server-351f5591-9548-908e-f887-eed3eb778e21: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-351f5591-9548-908e-f887-eed3eb778e21 Address:351f5591-9548-908e-f887-eed3eb778e21} {Suffrage:Voter ID:server-5bdb543d-ac5a-db90-a73e-36e36a7d7c6d Address:5bdb543d-ac5a-db90-a73e-36e36a7d7c6d} {Suffrage:Voter ID:server-a37e61c3-cf06-f465-d155-cad27c2b1fd9 Address:a37e61c3-cf06-f465-d155-cad27c2b1fd9}]" +2025-01-28T14:55:46.013Z [INFO] server-351f5591-9548-908e-f887-eed3eb778e21: entering follower state: follower="Node at 351f5591-9548-908e-f887-eed3eb778e21 [Follower]" leader-address= leader-id= +2025-01-28T14:55:46.013Z [INFO] server-5bdb543d-ac5a-db90-a73e-36e36a7d7c6d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-351f5591-9548-908e-f887-eed3eb778e21 Address:351f5591-9548-908e-f887-eed3eb778e21} {Suffrage:Voter ID:server-5bdb543d-ac5a-db90-a73e-36e36a7d7c6d Address:5bdb543d-ac5a-db90-a73e-36e36a7d7c6d} {Suffrage:Voter ID:server-a37e61c3-cf06-f465-d155-cad27c2b1fd9 Address:a37e61c3-cf06-f465-d155-cad27c2b1fd9}]" +2025-01-28T14:55:46.013Z [INFO] server-5bdb543d-ac5a-db90-a73e-36e36a7d7c6d: entering follower state: follower="Node at 5bdb543d-ac5a-db90-a73e-36e36a7d7c6d [Follower]" leader-address= leader-id= +2025-01-28T14:55:46.014Z [INFO] server-a37e61c3-cf06-f465-d155-cad27c2b1fd9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-351f5591-9548-908e-f887-eed3eb778e21 Address:351f5591-9548-908e-f887-eed3eb778e21} {Suffrage:Voter ID:server-5bdb543d-ac5a-db90-a73e-36e36a7d7c6d Address:5bdb543d-ac5a-db90-a73e-36e36a7d7c6d} {Suffrage:Voter ID:server-a37e61c3-cf06-f465-d155-cad27c2b1fd9 Address:a37e61c3-cf06-f465-d155-cad27c2b1fd9}]" +2025-01-28T14:55:46.014Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:46.014Z [INFO] server-a37e61c3-cf06-f465-d155-cad27c2b1fd9: entering follower state: follower="Node at a37e61c3-cf06-f465-d155-cad27c2b1fd9 [Follower]" leader-address= leader-id= +2025-01-28T14:55:46.070Z [WARN] server-351f5591-9548-908e-f887-eed3eb778e21: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:46.070Z [INFO] server-351f5591-9548-908e-f887-eed3eb778e21: entering candidate state: node="Node at 351f5591-9548-908e-f887-eed3eb778e21 [Candidate]" term=2 +2025-01-28T14:55:46.071Z [INFO] server-351f5591-9548-908e-f887-eed3eb778e21: election won: term=2 tally=2 +2025-01-28T14:55:46.071Z [INFO] server-351f5591-9548-908e-f887-eed3eb778e21: entering leader state: leader="Node at 351f5591-9548-908e-f887-eed3eb778e21 [Leader]" +2025-01-28T14:55:46.071Z [INFO] server-351f5591-9548-908e-f887-eed3eb778e21: added peer, starting replication: peer=server-5bdb543d-ac5a-db90-a73e-36e36a7d7c6d +2025-01-28T14:55:46.071Z [INFO] server-351f5591-9548-908e-f887-eed3eb778e21: added peer, starting replication: peer=server-a37e61c3-cf06-f465-d155-cad27c2b1fd9 +2025-01-28T14:55:46.071Z [INFO] server-351f5591-9548-908e-f887-eed3eb778e21: pipelining replication: peer="{Voter server-5bdb543d-ac5a-db90-a73e-36e36a7d7c6d 5bdb543d-ac5a-db90-a73e-36e36a7d7c6d}" +2025-01-28T14:55:46.071Z [INFO] server-351f5591-9548-908e-f887-eed3eb778e21: pipelining replication: peer="{Voter server-a37e61c3-cf06-f465-d155-cad27c2b1fd9 a37e61c3-cf06-f465-d155-cad27c2b1fd9}" +2025-01-28T14:55:46.176Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:46.071263778 +0000 UTC m=+131.745753758 (1 nodes), 56.976305ms from start of poll, 58.113197ms from cluster start. Timeout at 2025-01-28 14:55:46.176310354 +0000 UTC m=+131.850800334, 105.046576ms after stability +2025-01-28T14:55:46.176Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:55:46.282Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:55:46.176536627 +0000 UTC m=+131.851026627 (2 nodes), 2.779µs from start of poll, 163.386066ms from cluster start. Timeout at 2025-01-28 14:55:46.28172025 +0000 UTC m=+131.956210230, 105.183603ms after stability +2025-01-28T14:55:46.385Z [INFO] server-5bdb543d-ac5a-db90-a73e-36e36a7d7c6d: entering candidate state: node="Node at 5bdb543d-ac5a-db90-a73e-36e36a7d7c6d [Candidate]" term=3 +2025-01-28T14:55:46.385Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:46.385Z [INFO] server-5bdb543d-ac5a-db90-a73e-36e36a7d7c6d: election won: term=3 tally=2 +2025-01-28T14:55:46.385Z [INFO] server-5bdb543d-ac5a-db90-a73e-36e36a7d7c6d: entering leader state: leader="Node at 5bdb543d-ac5a-db90-a73e-36e36a7d7c6d [Leader]" +2025-01-28T14:55:46.385Z [INFO] server-5bdb543d-ac5a-db90-a73e-36e36a7d7c6d: added peer, starting replication: peer=server-351f5591-9548-908e-f887-eed3eb778e21 +2025-01-28T14:55:46.385Z [INFO] server-5bdb543d-ac5a-db90-a73e-36e36a7d7c6d: added peer, starting replication: peer=server-a37e61c3-cf06-f465-d155-cad27c2b1fd9 +2025-01-28T14:55:46.385Z [INFO] server-351f5591-9548-908e-f887-eed3eb778e21: entering follower state: follower="Node at 351f5591-9548-908e-f887-eed3eb778e21 [Follower]" leader-address= leader-id= +2025-01-28T14:55:46.385Z [INFO] server-351f5591-9548-908e-f887-eed3eb778e21: aborting pipeline replication: peer="{Voter server-5bdb543d-ac5a-db90-a73e-36e36a7d7c6d 5bdb543d-ac5a-db90-a73e-36e36a7d7c6d}" +2025-01-28T14:55:46.385Z [INFO] server-351f5591-9548-908e-f887-eed3eb778e21: aborting pipeline replication: peer="{Voter server-a37e61c3-cf06-f465-d155-cad27c2b1fd9 a37e61c3-cf06-f465-d155-cad27c2b1fd9}" +2025-01-28T14:55:46.385Z [INFO] server-5bdb543d-ac5a-db90-a73e-36e36a7d7c6d: pipelining replication: peer="{Voter server-a37e61c3-cf06-f465-d155-cad27c2b1fd9 a37e61c3-cf06-f465-d155-cad27c2b1fd9}" +2025-01-28T14:55:46.386Z [INFO] server-5bdb543d-ac5a-db90-a73e-36e36a7d7c6d: pipelining replication: peer="{Voter server-351f5591-9548-908e-f887-eed3eb778e21 351f5591-9548-908e-f887-eed3eb778e21}" +2025-01-28T14:55:46.491Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:46.38586748 +0000 UTC m=+132.060357460 (1 nodes), 334.028µs from start of poll, 372.716899ms from cluster start. Timeout at 2025-01-28 14:55:46.491141914 +0000 UTC m=+132.165631894, 105.274434ms after stability +2025-01-28T14:55:46.491Z [INFO] server-5bdb543d-ac5a-db90-a73e-36e36a7d7c6d: aborting pipeline replication: peer="{Voter server-a37e61c3-cf06-f465-d155-cad27c2b1fd9 a37e61c3-cf06-f465-d155-cad27c2b1fd9}" +2025-01-28T14:55:46.491Z [INFO] server-5bdb543d-ac5a-db90-a73e-36e36a7d7c6d: aborting pipeline replication: peer="{Voter server-351f5591-9548-908e-f887-eed3eb778e21 351f5591-9548-908e-f887-eed3eb778e21}" +--- PASS: TestRaft_LeadershipTransferToBehindServer (0.48s) === RUN TestRaft_LeadershipTransferToItself -2025-01-28T14:48:46.196Z [INFO] server-6bd40765-8b95-ac6c-7661-da9c1959975e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6bd40765-8b95-ac6c-7661-da9c1959975e Address:6bd40765-8b95-ac6c-7661-da9c1959975e} {Suffrage:Voter ID:server-98e2625f-f9f3-bb2f-af74-74bb426c2829 Address:98e2625f-f9f3-bb2f-af74-74bb426c2829} {Suffrage:Voter ID:server-fa269533-67e1-c719-201a-f91ad8d8b0b5 Address:fa269533-67e1-c719-201a-f91ad8d8b0b5}]" -2025-01-28T14:48:46.196Z [INFO] server-98e2625f-f9f3-bb2f-af74-74bb426c2829: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6bd40765-8b95-ac6c-7661-da9c1959975e Address:6bd40765-8b95-ac6c-7661-da9c1959975e} {Suffrage:Voter ID:server-98e2625f-f9f3-bb2f-af74-74bb426c2829 Address:98e2625f-f9f3-bb2f-af74-74bb426c2829} {Suffrage:Voter ID:server-fa269533-67e1-c719-201a-f91ad8d8b0b5 Address:fa269533-67e1-c719-201a-f91ad8d8b0b5}]" -2025-01-28T14:48:46.196Z [INFO] server-6bd40765-8b95-ac6c-7661-da9c1959975e: entering follower state: follower="Node at 6bd40765-8b95-ac6c-7661-da9c1959975e [Follower]" leader-address= leader-id= -2025-01-28T14:48:46.197Z [INFO] server-fa269533-67e1-c719-201a-f91ad8d8b0b5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6bd40765-8b95-ac6c-7661-da9c1959975e Address:6bd40765-8b95-ac6c-7661-da9c1959975e} {Suffrage:Voter ID:server-98e2625f-f9f3-bb2f-af74-74bb426c2829 Address:98e2625f-f9f3-bb2f-af74-74bb426c2829} {Suffrage:Voter ID:server-fa269533-67e1-c719-201a-f91ad8d8b0b5 Address:fa269533-67e1-c719-201a-f91ad8d8b0b5}]" -2025-01-28T14:48:46.197Z [INFO] server-98e2625f-f9f3-bb2f-af74-74bb426c2829: entering follower state: follower="Node at 98e2625f-f9f3-bb2f-af74-74bb426c2829 [Follower]" leader-address= leader-id= -2025-01-28T14:48:46.197Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:46.197Z [INFO] server-fa269533-67e1-c719-201a-f91ad8d8b0b5: entering follower state: follower="Node at fa269533-67e1-c719-201a-f91ad8d8b0b5 [Follower]" leader-address= leader-id= -2025-01-28T14:48:46.258Z [WARN] server-fa269533-67e1-c719-201a-f91ad8d8b0b5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:46.258Z [INFO] server-fa269533-67e1-c719-201a-f91ad8d8b0b5: entering candidate state: node="Node at fa269533-67e1-c719-201a-f91ad8d8b0b5 [Candidate]" term=2 -2025-01-28T14:48:46.258Z [INFO] server-fa269533-67e1-c719-201a-f91ad8d8b0b5: election won: term=2 tally=2 -2025-01-28T14:48:46.258Z [INFO] server-fa269533-67e1-c719-201a-f91ad8d8b0b5: entering leader state: leader="Node at fa269533-67e1-c719-201a-f91ad8d8b0b5 [Leader]" -2025-01-28T14:48:46.259Z [INFO] server-fa269533-67e1-c719-201a-f91ad8d8b0b5: added peer, starting replication: peer=server-6bd40765-8b95-ac6c-7661-da9c1959975e -2025-01-28T14:48:46.259Z [INFO] server-fa269533-67e1-c719-201a-f91ad8d8b0b5: added peer, starting replication: peer=server-98e2625f-f9f3-bb2f-af74-74bb426c2829 -2025-01-28T14:48:46.259Z [WARN] server-6bd40765-8b95-ac6c-7661-da9c1959975e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:46.259Z [INFO] server-6bd40765-8b95-ac6c-7661-da9c1959975e: entering candidate state: node="Node at 6bd40765-8b95-ac6c-7661-da9c1959975e [Candidate]" term=2 -2025-01-28T14:48:46.260Z [INFO] server-6bd40765-8b95-ac6c-7661-da9c1959975e: entering follower state: follower="Node at 6bd40765-8b95-ac6c-7661-da9c1959975e [Follower]" leader-address=fa269533-67e1-c719-201a-f91ad8d8b0b5 leader-id=server-fa269533-67e1-c719-201a-f91ad8d8b0b5 -2025-01-28T14:48:46.260Z [INFO] server-fa269533-67e1-c719-201a-f91ad8d8b0b5: pipelining replication: peer="{Voter server-98e2625f-f9f3-bb2f-af74-74bb426c2829 98e2625f-f9f3-bb2f-af74-74bb426c2829}" -2025-01-28T14:48:46.260Z [INFO] server-6bd40765-8b95-ac6c-7661-da9c1959975e: duplicate requestVote for same term: term=2 -2025-01-28T14:48:46.260Z [WARN] server-98e2625f-f9f3-bb2f-af74-74bb426c2829: rejecting vote request since we have a leader: from=6bd40765-8b95-ac6c-7661-da9c1959975e leader=fa269533-67e1-c719-201a-f91ad8d8b0b5 leader-id=server-fa269533-67e1-c719-201a-f91ad8d8b0b5 -2025-01-28T14:48:46.260Z [WARN] server-fa269533-67e1-c719-201a-f91ad8d8b0b5: rejecting vote request since we have a leader: from=6bd40765-8b95-ac6c-7661-da9c1959975e leader=fa269533-67e1-c719-201a-f91ad8d8b0b5 leader-id=server-fa269533-67e1-c719-201a-f91ad8d8b0b5 -2025-01-28T14:48:46.260Z [INFO] server-fa269533-67e1-c719-201a-f91ad8d8b0b5: pipelining replication: peer="{Voter server-6bd40765-8b95-ac6c-7661-da9c1959975e 6bd40765-8b95-ac6c-7661-da9c1959975e}" -2025-01-28T14:48:46.366Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:46.260980711 +0000 UTC m=+109.344385735 (1 nodes), 63.189876ms from start of poll, 65.426609ms from cluster start. Timeout at 2025-01-28 14:48:46.36605138 +0000 UTC m=+109.449455738, 105.070003ms after stability -2025-01-28T14:48:46.366Z [INFO] server-fa269533-67e1-c719-201a-f91ad8d8b0b5: cannot transfer leadership to itself -2025-01-28T14:48:46.366Z [INFO] server-fa269533-67e1-c719-201a-f91ad8d8b0b5: aborting pipeline replication: peer="{Voter server-6bd40765-8b95-ac6c-7661-da9c1959975e 6bd40765-8b95-ac6c-7661-da9c1959975e}" -2025-01-28T14:48:46.366Z [INFO] server-fa269533-67e1-c719-201a-f91ad8d8b0b5: aborting pipeline replication: peer="{Voter server-98e2625f-f9f3-bb2f-af74-74bb426c2829 98e2625f-f9f3-bb2f-af74-74bb426c2829}" ---- PASS: TestRaft_LeadershipTransferToItself (0.18s) +2025-01-28T14:55:46.495Z [INFO] server-4ec4f3e4-f559-0bcb-677d-e0afc991249e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4ec4f3e4-f559-0bcb-677d-e0afc991249e Address:4ec4f3e4-f559-0bcb-677d-e0afc991249e} {Suffrage:Voter ID:server-7612cd24-58bd-6876-f82b-22544ea61fc3 Address:7612cd24-58bd-6876-f82b-22544ea61fc3} {Suffrage:Voter ID:server-cc064f2c-0843-57fd-a345-8ef186b2cba1 Address:cc064f2c-0843-57fd-a345-8ef186b2cba1}]" +2025-01-28T14:55:46.495Z [INFO] server-4ec4f3e4-f559-0bcb-677d-e0afc991249e: entering follower state: follower="Node at 4ec4f3e4-f559-0bcb-677d-e0afc991249e [Follower]" leader-address= leader-id= +2025-01-28T14:55:46.495Z [INFO] server-7612cd24-58bd-6876-f82b-22544ea61fc3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4ec4f3e4-f559-0bcb-677d-e0afc991249e Address:4ec4f3e4-f559-0bcb-677d-e0afc991249e} {Suffrage:Voter ID:server-7612cd24-58bd-6876-f82b-22544ea61fc3 Address:7612cd24-58bd-6876-f82b-22544ea61fc3} {Suffrage:Voter ID:server-cc064f2c-0843-57fd-a345-8ef186b2cba1 Address:cc064f2c-0843-57fd-a345-8ef186b2cba1}]" +2025-01-28T14:55:46.495Z [INFO] server-7612cd24-58bd-6876-f82b-22544ea61fc3: entering follower state: follower="Node at 7612cd24-58bd-6876-f82b-22544ea61fc3 [Follower]" leader-address= leader-id= +2025-01-28T14:55:46.496Z [INFO] server-cc064f2c-0843-57fd-a345-8ef186b2cba1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4ec4f3e4-f559-0bcb-677d-e0afc991249e Address:4ec4f3e4-f559-0bcb-677d-e0afc991249e} {Suffrage:Voter ID:server-7612cd24-58bd-6876-f82b-22544ea61fc3 Address:7612cd24-58bd-6876-f82b-22544ea61fc3} {Suffrage:Voter ID:server-cc064f2c-0843-57fd-a345-8ef186b2cba1 Address:cc064f2c-0843-57fd-a345-8ef186b2cba1}]" +2025-01-28T14:55:46.496Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:46.496Z [INFO] server-cc064f2c-0843-57fd-a345-8ef186b2cba1: entering follower state: follower="Node at cc064f2c-0843-57fd-a345-8ef186b2cba1 [Follower]" leader-address= leader-id= +2025-01-28T14:55:46.548Z [WARN] server-4ec4f3e4-f559-0bcb-677d-e0afc991249e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:46.548Z [INFO] server-4ec4f3e4-f559-0bcb-677d-e0afc991249e: entering candidate state: node="Node at 4ec4f3e4-f559-0bcb-677d-e0afc991249e [Candidate]" term=2 +2025-01-28T14:55:46.549Z [INFO] server-4ec4f3e4-f559-0bcb-677d-e0afc991249e: election won: term=2 tally=2 +2025-01-28T14:55:46.549Z [INFO] server-4ec4f3e4-f559-0bcb-677d-e0afc991249e: entering leader state: leader="Node at 4ec4f3e4-f559-0bcb-677d-e0afc991249e [Leader]" +2025-01-28T14:55:46.549Z [INFO] server-4ec4f3e4-f559-0bcb-677d-e0afc991249e: added peer, starting replication: peer=server-7612cd24-58bd-6876-f82b-22544ea61fc3 +2025-01-28T14:55:46.549Z [INFO] server-4ec4f3e4-f559-0bcb-677d-e0afc991249e: added peer, starting replication: peer=server-cc064f2c-0843-57fd-a345-8ef186b2cba1 +2025-01-28T14:55:46.549Z [INFO] server-4ec4f3e4-f559-0bcb-677d-e0afc991249e: pipelining replication: peer="{Voter server-7612cd24-58bd-6876-f82b-22544ea61fc3 7612cd24-58bd-6876-f82b-22544ea61fc3}" +2025-01-28T14:55:46.549Z [INFO] server-4ec4f3e4-f559-0bcb-677d-e0afc991249e: pipelining replication: peer="{Voter server-cc064f2c-0843-57fd-a345-8ef186b2cba1 cc064f2c-0843-57fd-a345-8ef186b2cba1}" +2025-01-28T14:55:46.655Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:46.549259266 +0000 UTC m=+132.223749246 (1 nodes), 53.076893ms from start of poll, 54.154031ms from cluster start. Timeout at 2025-01-28 14:55:46.654266846 +0000 UTC m=+132.328756846, 105.0076ms after stability +2025-01-28T14:55:46.655Z [INFO] server-4ec4f3e4-f559-0bcb-677d-e0afc991249e: cannot transfer leadership to itself +2025-01-28T14:55:46.655Z [INFO] server-4ec4f3e4-f559-0bcb-677d-e0afc991249e: aborting pipeline replication: peer="{Voter server-cc064f2c-0843-57fd-a345-8ef186b2cba1 cc064f2c-0843-57fd-a345-8ef186b2cba1}" +2025-01-28T14:55:46.655Z [INFO] server-4ec4f3e4-f559-0bcb-677d-e0afc991249e: aborting pipeline replication: peer="{Voter server-7612cd24-58bd-6876-f82b-22544ea61fc3 7612cd24-58bd-6876-f82b-22544ea61fc3}" +--- PASS: TestRaft_LeadershipTransferToItself (0.16s) === RUN TestRaft_LeadershipTransferLeaderRejectsClientRequests -2025-01-28T14:48:46.376Z [INFO] server-70e7cabe-d4e1-5dbd-abb8-7d0e330afb19: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-70e7cabe-d4e1-5dbd-abb8-7d0e330afb19 Address:70e7cabe-d4e1-5dbd-abb8-7d0e330afb19} {Suffrage:Voter ID:server-83dd11f1-9edb-1cdd-a3c4-6ce7566542a0 Address:83dd11f1-9edb-1cdd-a3c4-6ce7566542a0} {Suffrage:Voter ID:server-f06a603f-d552-747c-f85d-f199b4b83304 Address:f06a603f-d552-747c-f85d-f199b4b83304}]" -2025-01-28T14:48:46.376Z [INFO] server-70e7cabe-d4e1-5dbd-abb8-7d0e330afb19: entering follower state: follower="Node at 70e7cabe-d4e1-5dbd-abb8-7d0e330afb19 [Follower]" leader-address= leader-id= -2025-01-28T14:48:46.376Z [INFO] server-83dd11f1-9edb-1cdd-a3c4-6ce7566542a0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-70e7cabe-d4e1-5dbd-abb8-7d0e330afb19 Address:70e7cabe-d4e1-5dbd-abb8-7d0e330afb19} {Suffrage:Voter ID:server-83dd11f1-9edb-1cdd-a3c4-6ce7566542a0 Address:83dd11f1-9edb-1cdd-a3c4-6ce7566542a0} {Suffrage:Voter ID:server-f06a603f-d552-747c-f85d-f199b4b83304 Address:f06a603f-d552-747c-f85d-f199b4b83304}]" -2025-01-28T14:48:46.377Z [INFO] server-83dd11f1-9edb-1cdd-a3c4-6ce7566542a0: entering follower state: follower="Node at 83dd11f1-9edb-1cdd-a3c4-6ce7566542a0 [Follower]" leader-address= leader-id= -2025-01-28T14:48:46.377Z [INFO] server-f06a603f-d552-747c-f85d-f199b4b83304: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-70e7cabe-d4e1-5dbd-abb8-7d0e330afb19 Address:70e7cabe-d4e1-5dbd-abb8-7d0e330afb19} {Suffrage:Voter ID:server-83dd11f1-9edb-1cdd-a3c4-6ce7566542a0 Address:83dd11f1-9edb-1cdd-a3c4-6ce7566542a0} {Suffrage:Voter ID:server-f06a603f-d552-747c-f85d-f199b4b83304 Address:f06a603f-d552-747c-f85d-f199b4b83304}]" -2025-01-28T14:48:46.377Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:46.378Z [INFO] server-f06a603f-d552-747c-f85d-f199b4b83304: entering follower state: follower="Node at f06a603f-d552-747c-f85d-f199b4b83304 [Follower]" leader-address= leader-id= -2025-01-28T14:48:46.431Z [WARN] server-83dd11f1-9edb-1cdd-a3c4-6ce7566542a0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:46.431Z [INFO] server-83dd11f1-9edb-1cdd-a3c4-6ce7566542a0: entering candidate state: node="Node at 83dd11f1-9edb-1cdd-a3c4-6ce7566542a0 [Candidate]" term=2 -2025-01-28T14:48:46.431Z [INFO] server-83dd11f1-9edb-1cdd-a3c4-6ce7566542a0: election won: term=2 tally=2 -2025-01-28T14:48:46.431Z [INFO] server-83dd11f1-9edb-1cdd-a3c4-6ce7566542a0: entering leader state: leader="Node at 83dd11f1-9edb-1cdd-a3c4-6ce7566542a0 [Leader]" -2025-01-28T14:48:46.432Z [INFO] server-83dd11f1-9edb-1cdd-a3c4-6ce7566542a0: added peer, starting replication: peer=server-70e7cabe-d4e1-5dbd-abb8-7d0e330afb19 -2025-01-28T14:48:46.432Z [INFO] server-83dd11f1-9edb-1cdd-a3c4-6ce7566542a0: added peer, starting replication: peer=server-f06a603f-d552-747c-f85d-f199b4b83304 -2025-01-28T14:48:46.432Z [INFO] server-83dd11f1-9edb-1cdd-a3c4-6ce7566542a0: pipelining replication: peer="{Voter server-f06a603f-d552-747c-f85d-f199b4b83304 f06a603f-d552-747c-f85d-f199b4b83304}" -2025-01-28T14:48:46.432Z [INFO] server-83dd11f1-9edb-1cdd-a3c4-6ce7566542a0: pipelining replication: peer="{Voter server-70e7cabe-d4e1-5dbd-abb8-7d0e330afb19 70e7cabe-d4e1-5dbd-abb8-7d0e330afb19}" -2025-01-28T14:48:46.537Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:46.432207129 +0000 UTC m=+109.515611820 (1 nodes), 54.374281ms from start of poll, 56.712684ms from cluster start. Timeout at 2025-01-28 14:48:46.537226463 +0000 UTC m=+109.620631154, 105.019334ms after stability +2025-01-28T14:55:46.658Z [INFO] server-8d995b7c-3e4f-48bc-db12-9401e9370a43: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8d995b7c-3e4f-48bc-db12-9401e9370a43 Address:8d995b7c-3e4f-48bc-db12-9401e9370a43} {Suffrage:Voter ID:server-3ca88f25-d55c-ca06-ab1d-bf92f86cf9c8 Address:3ca88f25-d55c-ca06-ab1d-bf92f86cf9c8} {Suffrage:Voter ID:server-99861c09-c48b-b03d-f3f8-5bfee700e81e Address:99861c09-c48b-b03d-f3f8-5bfee700e81e}]" +2025-01-28T14:55:46.659Z [INFO] server-3ca88f25-d55c-ca06-ab1d-bf92f86cf9c8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8d995b7c-3e4f-48bc-db12-9401e9370a43 Address:8d995b7c-3e4f-48bc-db12-9401e9370a43} {Suffrage:Voter ID:server-3ca88f25-d55c-ca06-ab1d-bf92f86cf9c8 Address:3ca88f25-d55c-ca06-ab1d-bf92f86cf9c8} {Suffrage:Voter ID:server-99861c09-c48b-b03d-f3f8-5bfee700e81e Address:99861c09-c48b-b03d-f3f8-5bfee700e81e}]" +2025-01-28T14:55:46.659Z [INFO] server-8d995b7c-3e4f-48bc-db12-9401e9370a43: entering follower state: follower="Node at 8d995b7c-3e4f-48bc-db12-9401e9370a43 [Follower]" leader-address= leader-id= +2025-01-28T14:55:46.659Z [INFO] server-3ca88f25-d55c-ca06-ab1d-bf92f86cf9c8: entering follower state: follower="Node at 3ca88f25-d55c-ca06-ab1d-bf92f86cf9c8 [Follower]" leader-address= leader-id= +2025-01-28T14:55:46.659Z [INFO] server-99861c09-c48b-b03d-f3f8-5bfee700e81e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8d995b7c-3e4f-48bc-db12-9401e9370a43 Address:8d995b7c-3e4f-48bc-db12-9401e9370a43} {Suffrage:Voter ID:server-3ca88f25-d55c-ca06-ab1d-bf92f86cf9c8 Address:3ca88f25-d55c-ca06-ab1d-bf92f86cf9c8} {Suffrage:Voter ID:server-99861c09-c48b-b03d-f3f8-5bfee700e81e Address:99861c09-c48b-b03d-f3f8-5bfee700e81e}]" +2025-01-28T14:55:46.659Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:46.659Z [INFO] server-99861c09-c48b-b03d-f3f8-5bfee700e81e: entering follower state: follower="Node at 99861c09-c48b-b03d-f3f8-5bfee700e81e [Follower]" leader-address= leader-id= +2025-01-28T14:55:46.722Z [WARN] server-3ca88f25-d55c-ca06-ab1d-bf92f86cf9c8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:46.722Z [INFO] server-3ca88f25-d55c-ca06-ab1d-bf92f86cf9c8: entering candidate state: node="Node at 3ca88f25-d55c-ca06-ab1d-bf92f86cf9c8 [Candidate]" term=2 +2025-01-28T14:55:46.723Z [INFO] server-3ca88f25-d55c-ca06-ab1d-bf92f86cf9c8: election won: term=2 tally=2 +2025-01-28T14:55:46.723Z [INFO] server-3ca88f25-d55c-ca06-ab1d-bf92f86cf9c8: entering leader state: leader="Node at 3ca88f25-d55c-ca06-ab1d-bf92f86cf9c8 [Leader]" +2025-01-28T14:55:46.723Z [INFO] server-3ca88f25-d55c-ca06-ab1d-bf92f86cf9c8: added peer, starting replication: peer=server-8d995b7c-3e4f-48bc-db12-9401e9370a43 +2025-01-28T14:55:46.723Z [INFO] server-3ca88f25-d55c-ca06-ab1d-bf92f86cf9c8: added peer, starting replication: peer=server-99861c09-c48b-b03d-f3f8-5bfee700e81e +2025-01-28T14:55:46.723Z [INFO] server-3ca88f25-d55c-ca06-ab1d-bf92f86cf9c8: pipelining replication: peer="{Voter server-8d995b7c-3e4f-48bc-db12-9401e9370a43 8d995b7c-3e4f-48bc-db12-9401e9370a43}" +2025-01-28T14:55:46.723Z [INFO] server-3ca88f25-d55c-ca06-ab1d-bf92f86cf9c8: pipelining replication: peer="{Voter server-99861c09-c48b-b03d-f3f8-5bfee700e81e 99861c09-c48b-b03d-f3f8-5bfee700e81e}" +2025-01-28T14:55:46.828Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:46.723192638 +0000 UTC m=+132.397682618 (1 nodes), 63.592161ms from start of poll, 64.510574ms from cluster start. Timeout at 2025-01-28 14:55:46.828239793 +0000 UTC m=+132.502729773, 105.047155ms after stability raft_test.go:2457: waiting on future 0 raft_test.go:2457: waiting on future 1 raft_test.go:2457: waiting on future 2 @@ -3900,342 +3890,336 @@ raft_test.go:2457: waiting on future 4 raft_test.go:2457: waiting on future 5 raft_test.go:2457: waiting on future 6 -2025-01-28T14:48:46.539Z [INFO] server-83dd11f1-9edb-1cdd-a3c4-6ce7566542a0: aborting pipeline replication: peer="{Voter server-f06a603f-d552-747c-f85d-f199b4b83304 f06a603f-d552-747c-f85d-f199b4b83304}" -2025-01-28T14:48:46.539Z [INFO] server-83dd11f1-9edb-1cdd-a3c4-6ce7566542a0: aborting pipeline replication: peer="{Voter server-70e7cabe-d4e1-5dbd-abb8-7d0e330afb19 70e7cabe-d4e1-5dbd-abb8-7d0e330afb19}" ---- PASS: TestRaft_LeadershipTransferLeaderRejectsClientRequests (0.18s) +2025-01-28T14:55:46.829Z [INFO] server-3ca88f25-d55c-ca06-ab1d-bf92f86cf9c8: aborting pipeline replication: peer="{Voter server-8d995b7c-3e4f-48bc-db12-9401e9370a43 8d995b7c-3e4f-48bc-db12-9401e9370a43}" +2025-01-28T14:55:46.829Z [INFO] server-3ca88f25-d55c-ca06-ab1d-bf92f86cf9c8: aborting pipeline replication: peer="{Voter server-99861c09-c48b-b03d-f3f8-5bfee700e81e 99861c09-c48b-b03d-f3f8-5bfee700e81e}" +--- PASS: TestRaft_LeadershipTransferLeaderRejectsClientRequests (0.17s) === RUN TestRaft_LeadershipTransferLeaderReplicationTimeout -2025-01-28T14:48:46.554Z [INFO] server-9137fac4-d9ad-ab45-4d94-29a930dc9895: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9137fac4-d9ad-ab45-4d94-29a930dc9895 Address:9137fac4-d9ad-ab45-4d94-29a930dc9895} {Suffrage:Voter ID:server-47698f02-310c-8f90-8f68-f90971275a51 Address:47698f02-310c-8f90-8f68-f90971275a51} {Suffrage:Voter ID:server-1d3883c9-a248-4cb1-cab0-1b28ab999b37 Address:1d3883c9-a248-4cb1-cab0-1b28ab999b37}]" -2025-01-28T14:48:46.554Z [INFO] server-9137fac4-d9ad-ab45-4d94-29a930dc9895: entering follower state: follower="Node at 9137fac4-d9ad-ab45-4d94-29a930dc9895 [Follower]" leader-address= leader-id= -2025-01-28T14:48:46.554Z [INFO] server-47698f02-310c-8f90-8f68-f90971275a51: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9137fac4-d9ad-ab45-4d94-29a930dc9895 Address:9137fac4-d9ad-ab45-4d94-29a930dc9895} {Suffrage:Voter ID:server-47698f02-310c-8f90-8f68-f90971275a51 Address:47698f02-310c-8f90-8f68-f90971275a51} {Suffrage:Voter ID:server-1d3883c9-a248-4cb1-cab0-1b28ab999b37 Address:1d3883c9-a248-4cb1-cab0-1b28ab999b37}]" -2025-01-28T14:48:46.555Z [INFO] server-47698f02-310c-8f90-8f68-f90971275a51: entering follower state: follower="Node at 47698f02-310c-8f90-8f68-f90971275a51 [Follower]" leader-address= leader-id= -2025-01-28T14:48:46.555Z [INFO] server-1d3883c9-a248-4cb1-cab0-1b28ab999b37: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9137fac4-d9ad-ab45-4d94-29a930dc9895 Address:9137fac4-d9ad-ab45-4d94-29a930dc9895} {Suffrage:Voter ID:server-47698f02-310c-8f90-8f68-f90971275a51 Address:47698f02-310c-8f90-8f68-f90971275a51} {Suffrage:Voter ID:server-1d3883c9-a248-4cb1-cab0-1b28ab999b37 Address:1d3883c9-a248-4cb1-cab0-1b28ab999b37}]" -2025-01-28T14:48:46.555Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:46.555Z [INFO] server-1d3883c9-a248-4cb1-cab0-1b28ab999b37: entering follower state: follower="Node at 1d3883c9-a248-4cb1-cab0-1b28ab999b37 [Follower]" leader-address= leader-id= -2025-01-28T14:48:46.611Z [WARN] server-1d3883c9-a248-4cb1-cab0-1b28ab999b37: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:46.612Z [INFO] server-1d3883c9-a248-4cb1-cab0-1b28ab999b37: entering candidate state: node="Node at 1d3883c9-a248-4cb1-cab0-1b28ab999b37 [Candidate]" term=2 -2025-01-28T14:48:46.613Z [INFO] server-1d3883c9-a248-4cb1-cab0-1b28ab999b37: election won: term=2 tally=2 -2025-01-28T14:48:46.613Z [INFO] server-1d3883c9-a248-4cb1-cab0-1b28ab999b37: entering leader state: leader="Node at 1d3883c9-a248-4cb1-cab0-1b28ab999b37 [Leader]" -2025-01-28T14:48:46.613Z [INFO] server-1d3883c9-a248-4cb1-cab0-1b28ab999b37: added peer, starting replication: peer=server-9137fac4-d9ad-ab45-4d94-29a930dc9895 -2025-01-28T14:48:46.613Z [INFO] server-1d3883c9-a248-4cb1-cab0-1b28ab999b37: added peer, starting replication: peer=server-47698f02-310c-8f90-8f68-f90971275a51 -2025-01-28T14:48:46.615Z [INFO] server-1d3883c9-a248-4cb1-cab0-1b28ab999b37: pipelining replication: peer="{Voter server-9137fac4-d9ad-ab45-4d94-29a930dc9895 9137fac4-d9ad-ab45-4d94-29a930dc9895}" -2025-01-28T14:48:46.615Z [INFO] server-1d3883c9-a248-4cb1-cab0-1b28ab999b37: pipelining replication: peer="{Voter server-47698f02-310c-8f90-8f68-f90971275a51 47698f02-310c-8f90-8f68-f90971275a51}" -2025-01-28T14:48:46.719Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:46.613781855 +0000 UTC m=+109.697187213 (1 nodes), 58.103726ms from start of poll, 60.379793ms from cluster start. Timeout at 2025-01-28 14:48:46.718807189 +0000 UTC m=+109.802212547, 105.025334ms after stability -2025-01-28T14:48:46.720Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:48:46.826Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:48:46.720225683 +0000 UTC m=+109.803630374 (2 nodes), 8.333µs from start of poll, 166.822954ms from cluster start. Timeout at 2025-01-28 14:48:46.825651029 +0000 UTC m=+109.909057720, 105.427346ms after stability -2025-01-28T14:48:51.812Z [INFO] server-1d3883c9-a248-4cb1-cab0-1b28ab999b37: aborting pipeline replication: peer="{Voter server-47698f02-310c-8f90-8f68-f90971275a51 47698f02-310c-8f90-8f68-f90971275a51}" -2025-01-28T14:48:51.812Z [INFO] server-1d3883c9-a248-4cb1-cab0-1b28ab999b37: aborting pipeline replication: peer="{Voter server-9137fac4-d9ad-ab45-4d94-29a930dc9895 9137fac4-d9ad-ab45-4d94-29a930dc9895}" ---- PASS: TestRaft_LeadershipTransferLeaderReplicationTimeout (5.27s) +2025-01-28T14:55:46.832Z [INFO] server-0419cf48-f9a9-9056-bee6-d2aa9e986112: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0419cf48-f9a9-9056-bee6-d2aa9e986112 Address:0419cf48-f9a9-9056-bee6-d2aa9e986112} {Suffrage:Voter ID:server-200a3c74-e46a-a974-6330-d711217f9fb1 Address:200a3c74-e46a-a974-6330-d711217f9fb1} {Suffrage:Voter ID:server-9a546608-1506-737c-6c0a-d0a273c6a27c Address:9a546608-1506-737c-6c0a-d0a273c6a27c}]" +2025-01-28T14:55:46.832Z [INFO] server-0419cf48-f9a9-9056-bee6-d2aa9e986112: entering follower state: follower="Node at 0419cf48-f9a9-9056-bee6-d2aa9e986112 [Follower]" leader-address= leader-id= +2025-01-28T14:55:46.833Z [INFO] server-200a3c74-e46a-a974-6330-d711217f9fb1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0419cf48-f9a9-9056-bee6-d2aa9e986112 Address:0419cf48-f9a9-9056-bee6-d2aa9e986112} {Suffrage:Voter ID:server-200a3c74-e46a-a974-6330-d711217f9fb1 Address:200a3c74-e46a-a974-6330-d711217f9fb1} {Suffrage:Voter ID:server-9a546608-1506-737c-6c0a-d0a273c6a27c Address:9a546608-1506-737c-6c0a-d0a273c6a27c}]" +2025-01-28T14:55:46.833Z [INFO] server-200a3c74-e46a-a974-6330-d711217f9fb1: entering follower state: follower="Node at 200a3c74-e46a-a974-6330-d711217f9fb1 [Follower]" leader-address= leader-id= +2025-01-28T14:55:46.833Z [INFO] server-9a546608-1506-737c-6c0a-d0a273c6a27c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0419cf48-f9a9-9056-bee6-d2aa9e986112 Address:0419cf48-f9a9-9056-bee6-d2aa9e986112} {Suffrage:Voter ID:server-200a3c74-e46a-a974-6330-d711217f9fb1 Address:200a3c74-e46a-a974-6330-d711217f9fb1} {Suffrage:Voter ID:server-9a546608-1506-737c-6c0a-d0a273c6a27c Address:9a546608-1506-737c-6c0a-d0a273c6a27c}]" +2025-01-28T14:55:46.833Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:46.833Z [INFO] server-9a546608-1506-737c-6c0a-d0a273c6a27c: entering follower state: follower="Node at 9a546608-1506-737c-6c0a-d0a273c6a27c [Follower]" leader-address= leader-id= +2025-01-28T14:55:46.894Z [WARN] server-200a3c74-e46a-a974-6330-d711217f9fb1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:46.894Z [INFO] server-200a3c74-e46a-a974-6330-d711217f9fb1: entering candidate state: node="Node at 200a3c74-e46a-a974-6330-d711217f9fb1 [Candidate]" term=2 +2025-01-28T14:55:46.894Z [INFO] server-200a3c74-e46a-a974-6330-d711217f9fb1: election won: term=2 tally=2 +2025-01-28T14:55:46.894Z [INFO] server-200a3c74-e46a-a974-6330-d711217f9fb1: entering leader state: leader="Node at 200a3c74-e46a-a974-6330-d711217f9fb1 [Leader]" +2025-01-28T14:55:46.895Z [INFO] server-200a3c74-e46a-a974-6330-d711217f9fb1: added peer, starting replication: peer=server-0419cf48-f9a9-9056-bee6-d2aa9e986112 +2025-01-28T14:55:46.895Z [INFO] server-200a3c74-e46a-a974-6330-d711217f9fb1: added peer, starting replication: peer=server-9a546608-1506-737c-6c0a-d0a273c6a27c +2025-01-28T14:55:46.895Z [INFO] server-200a3c74-e46a-a974-6330-d711217f9fb1: pipelining replication: peer="{Voter server-0419cf48-f9a9-9056-bee6-d2aa9e986112 0419cf48-f9a9-9056-bee6-d2aa9e986112}" +2025-01-28T14:55:46.895Z [INFO] server-200a3c74-e46a-a974-6330-d711217f9fb1: pipelining replication: peer="{Voter server-9a546608-1506-737c-6c0a-d0a273c6a27c 9a546608-1506-737c-6c0a-d0a273c6a27c}" +2025-01-28T14:55:47.000Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:46.89500769 +0000 UTC m=+132.569497670 (1 nodes), 61.675202ms from start of poll, 62.588456ms from cluster start. Timeout at 2025-01-28 14:55:47.00001003 +0000 UTC m=+132.674500050, 105.00238ms after stability +2025-01-28T14:55:47.000Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:55:47.105Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:55:47.00026722 +0000 UTC m=+132.674757200 (2 nodes), 2.24µs from start of poll, 167.847986ms from cluster start. Timeout at 2025-01-28 14:55:47.105430585 +0000 UTC m=+132.779920565, 105.163365ms after stability +2025-01-28T14:55:48.253Z [INFO] server-200a3c74-e46a-a974-6330-d711217f9fb1: aborting pipeline replication: peer="{Voter server-0419cf48-f9a9-9056-bee6-d2aa9e986112 0419cf48-f9a9-9056-bee6-d2aa9e986112}" +2025-01-28T14:55:48.253Z [INFO] server-200a3c74-e46a-a974-6330-d711217f9fb1: aborting pipeline replication: peer="{Voter server-9a546608-1506-737c-6c0a-d0a273c6a27c 9a546608-1506-737c-6c0a-d0a273c6a27c}" +--- PASS: TestRaft_LeadershipTransferLeaderReplicationTimeout (1.42s) === RUN TestRaft_LeadershipTransferIgnoresNonvoters -2025-01-28T14:48:51.822Z [INFO] server-78e2f2ac-9cce-1fa2-3b7d-cdd481acfcec: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-78e2f2ac-9cce-1fa2-3b7d-cdd481acfcec Address:78e2f2ac-9cce-1fa2-3b7d-cdd481acfcec} {Suffrage:Voter ID:server-3fa0a35f-4abe-899c-b34a-739c0a55fb0c Address:3fa0a35f-4abe-899c-b34a-739c0a55fb0c}]" -2025-01-28T14:48:51.822Z [INFO] server-3fa0a35f-4abe-899c-b34a-739c0a55fb0c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-78e2f2ac-9cce-1fa2-3b7d-cdd481acfcec Address:78e2f2ac-9cce-1fa2-3b7d-cdd481acfcec} {Suffrage:Voter ID:server-3fa0a35f-4abe-899c-b34a-739c0a55fb0c Address:3fa0a35f-4abe-899c-b34a-739c0a55fb0c}]" -2025-01-28T14:48:51.822Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:48:51.822Z [INFO] server-78e2f2ac-9cce-1fa2-3b7d-cdd481acfcec: entering follower state: follower="Node at 78e2f2ac-9cce-1fa2-3b7d-cdd481acfcec [Follower]" leader-address= leader-id= -2025-01-28T14:48:51.823Z [INFO] server-3fa0a35f-4abe-899c-b34a-739c0a55fb0c: entering follower state: follower="Node at 3fa0a35f-4abe-899c-b34a-739c0a55fb0c [Follower]" leader-address= leader-id= -2025-01-28T14:48:51.877Z [WARN] server-78e2f2ac-9cce-1fa2-3b7d-cdd481acfcec: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:51.878Z [INFO] server-78e2f2ac-9cce-1fa2-3b7d-cdd481acfcec: entering candidate state: node="Node at 78e2f2ac-9cce-1fa2-3b7d-cdd481acfcec [Candidate]" term=2 -2025-01-28T14:48:51.878Z [INFO] server-78e2f2ac-9cce-1fa2-3b7d-cdd481acfcec: election won: term=2 tally=2 -2025-01-28T14:48:51.878Z [INFO] server-78e2f2ac-9cce-1fa2-3b7d-cdd481acfcec: entering leader state: leader="Node at 78e2f2ac-9cce-1fa2-3b7d-cdd481acfcec [Leader]" -2025-01-28T14:48:51.878Z [INFO] server-78e2f2ac-9cce-1fa2-3b7d-cdd481acfcec: added peer, starting replication: peer=server-3fa0a35f-4abe-899c-b34a-739c0a55fb0c -2025-01-28T14:48:51.879Z [INFO] server-78e2f2ac-9cce-1fa2-3b7d-cdd481acfcec: pipelining replication: peer="{Voter server-3fa0a35f-4abe-899c-b34a-739c0a55fb0c 3fa0a35f-4abe-899c-b34a-739c0a55fb0c}" -2025-01-28T14:48:51.984Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:48:51.878934556 +0000 UTC m=+114.962339581 (1 nodes), 55.964663ms from start of poll, 57.525709ms from cluster start. Timeout at 2025-01-28 14:48:51.98395589 +0000 UTC m=+115.067360581, 105.021ms after stability -2025-01-28T14:48:51.984Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:52.090Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:51.984421689 +0000 UTC m=+115.067826380 (1 nodes), 5.333µs from start of poll, 163.012508ms from cluster start. Timeout at 2025-01-28 14:48:52.090075042 +0000 UTC m=+115.173479733, 105.653353ms after stability -2025-01-28T14:48:52.091Z [INFO] server-78e2f2ac-9cce-1fa2-3b7d-cdd481acfcec: updating configuration: command=DemoteVoter server-id=server-3fa0a35f-4abe-899c-b34a-739c0a55fb0c server-addr= servers="[{Suffrage:Voter ID:server-78e2f2ac-9cce-1fa2-3b7d-cdd481acfcec Address:78e2f2ac-9cce-1fa2-3b7d-cdd481acfcec} {Suffrage:Nonvoter ID:server-3fa0a35f-4abe-899c-b34a-739c0a55fb0c Address:3fa0a35f-4abe-899c-b34a-739c0a55fb0c}]" -2025-01-28T14:48:52.091Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:52.198Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:52.091933882 +0000 UTC m=+115.175338573 (1 nodes), 5µs from start of poll, 270.524701ms from cluster start. Timeout at 2025-01-28 14:48:52.197401896 +0000 UTC m=+115.280807254, 105.468681ms after stability -2025-01-28T14:48:52.198Z [INFO] server-78e2f2ac-9cce-1fa2-3b7d-cdd481acfcec: aborting pipeline replication: peer="{Voter server-3fa0a35f-4abe-899c-b34a-739c0a55fb0c 3fa0a35f-4abe-899c-b34a-739c0a55fb0c}" ---- PASS: TestRaft_LeadershipTransferIgnoresNonvoters (0.38s) +2025-01-28T14:55:48.257Z [INFO] server-e22cb081-05ea-6bc7-bc5c-a1830f1dc522: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e22cb081-05ea-6bc7-bc5c-a1830f1dc522 Address:e22cb081-05ea-6bc7-bc5c-a1830f1dc522} {Suffrage:Voter ID:server-a50880e4-77ae-ffc1-a203-e50d4c0d5e52 Address:a50880e4-77ae-ffc1-a203-e50d4c0d5e52}]" +2025-01-28T14:55:48.257Z [INFO] server-e22cb081-05ea-6bc7-bc5c-a1830f1dc522: entering follower state: follower="Node at e22cb081-05ea-6bc7-bc5c-a1830f1dc522 [Follower]" leader-address= leader-id= +2025-01-28T14:55:48.257Z [INFO] server-a50880e4-77ae-ffc1-a203-e50d4c0d5e52: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e22cb081-05ea-6bc7-bc5c-a1830f1dc522 Address:e22cb081-05ea-6bc7-bc5c-a1830f1dc522} {Suffrage:Voter ID:server-a50880e4-77ae-ffc1-a203-e50d4c0d5e52 Address:a50880e4-77ae-ffc1-a203-e50d4c0d5e52}]" +2025-01-28T14:55:48.257Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:55:48.257Z [INFO] server-a50880e4-77ae-ffc1-a203-e50d4c0d5e52: entering follower state: follower="Node at a50880e4-77ae-ffc1-a203-e50d4c0d5e52 [Follower]" leader-address= leader-id= +2025-01-28T14:55:48.322Z [WARN] server-e22cb081-05ea-6bc7-bc5c-a1830f1dc522: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:48.322Z [INFO] server-e22cb081-05ea-6bc7-bc5c-a1830f1dc522: entering candidate state: node="Node at e22cb081-05ea-6bc7-bc5c-a1830f1dc522 [Candidate]" term=2 +2025-01-28T14:55:48.322Z [INFO] server-e22cb081-05ea-6bc7-bc5c-a1830f1dc522: election won: term=2 tally=2 +2025-01-28T14:55:48.322Z [INFO] server-e22cb081-05ea-6bc7-bc5c-a1830f1dc522: entering leader state: leader="Node at e22cb081-05ea-6bc7-bc5c-a1830f1dc522 [Leader]" +2025-01-28T14:55:48.322Z [INFO] server-e22cb081-05ea-6bc7-bc5c-a1830f1dc522: added peer, starting replication: peer=server-a50880e4-77ae-ffc1-a203-e50d4c0d5e52 +2025-01-28T14:55:48.323Z [INFO] server-e22cb081-05ea-6bc7-bc5c-a1830f1dc522: pipelining replication: peer="{Voter server-a50880e4-77ae-ffc1-a203-e50d4c0d5e52 a50880e4-77ae-ffc1-a203-e50d4c0d5e52}" +2025-01-28T14:55:48.428Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:55:48.322940585 +0000 UTC m=+133.997430585 (1 nodes), 65.340036ms from start of poll, 65.947099ms from cluster start. Timeout at 2025-01-28 14:55:48.427946825 +0000 UTC m=+134.102436805, 105.00622ms after stability +2025-01-28T14:55:48.428Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:48.534Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:48.428954324 +0000 UTC m=+134.103444304 (1 nodes), 1.68µs from start of poll, 171.960818ms from cluster start. Timeout at 2025-01-28 14:55:48.534346807 +0000 UTC m=+134.208836767, 105.392463ms after stability +2025-01-28T14:55:48.534Z [INFO] server-e22cb081-05ea-6bc7-bc5c-a1830f1dc522: updating configuration: command=DemoteVoter server-id=server-a50880e4-77ae-ffc1-a203-e50d4c0d5e52 server-addr= servers="[{Suffrage:Voter ID:server-e22cb081-05ea-6bc7-bc5c-a1830f1dc522 Address:e22cb081-05ea-6bc7-bc5c-a1830f1dc522} {Suffrage:Nonvoter ID:server-a50880e4-77ae-ffc1-a203-e50d4c0d5e52 Address:a50880e4-77ae-ffc1-a203-e50d4c0d5e52}]" +2025-01-28T14:55:48.535Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:48.641Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:48.535153486 +0000 UTC m=+134.209643466 (1 nodes), 2.12µs from start of poll, 278.15998ms from cluster start. Timeout at 2025-01-28 14:55:48.640421421 +0000 UTC m=+134.314911401, 105.267935ms after stability +2025-01-28T14:55:48.641Z [INFO] server-e22cb081-05ea-6bc7-bc5c-a1830f1dc522: aborting pipeline replication: peer="{Voter server-a50880e4-77ae-ffc1-a203-e50d4c0d5e52 a50880e4-77ae-ffc1-a203-e50d4c0d5e52}" +--- PASS: TestRaft_LeadershipTransferIgnoresNonvoters (0.39s) === RUN TestRaft_LeadershipTransferStopRightAway --- PASS: TestRaft_LeadershipTransferStopRightAway (0.00s) === RUN TestRaft_GetConfigurationNoBootstrap -2025-01-28T14:48:52.206Z [INFO] server-d699ea89-e89c-78b2-290e-53716f271be2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d699ea89-e89c-78b2-290e-53716f271be2 Address:d699ea89-e89c-78b2-290e-53716f271be2} {Suffrage:Voter ID:server-932cea43-eb53-8ead-42e5-80de86d72d1c Address:932cea43-eb53-8ead-42e5-80de86d72d1c}]" -2025-01-28T14:48:52.207Z [INFO] server-d699ea89-e89c-78b2-290e-53716f271be2: entering follower state: follower="Node at d699ea89-e89c-78b2-290e-53716f271be2 [Follower]" leader-address= leader-id= -2025-01-28T14:48:52.207Z [INFO] server-932cea43-eb53-8ead-42e5-80de86d72d1c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d699ea89-e89c-78b2-290e-53716f271be2 Address:d699ea89-e89c-78b2-290e-53716f271be2} {Suffrage:Voter ID:server-932cea43-eb53-8ead-42e5-80de86d72d1c Address:932cea43-eb53-8ead-42e5-80de86d72d1c}]" -2025-01-28T14:48:52.207Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:48:52.207Z [INFO] server-932cea43-eb53-8ead-42e5-80de86d72d1c: entering follower state: follower="Node at 932cea43-eb53-8ead-42e5-80de86d72d1c [Follower]" leader-address= leader-id= -2025-01-28T14:48:52.268Z [WARN] server-932cea43-eb53-8ead-42e5-80de86d72d1c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:52.269Z [INFO] server-932cea43-eb53-8ead-42e5-80de86d72d1c: entering candidate state: node="Node at 932cea43-eb53-8ead-42e5-80de86d72d1c [Candidate]" term=2 -2025-01-28T14:48:52.269Z [INFO] server-932cea43-eb53-8ead-42e5-80de86d72d1c: election won: term=2 tally=2 -2025-01-28T14:48:52.269Z [INFO] server-932cea43-eb53-8ead-42e5-80de86d72d1c: entering leader state: leader="Node at 932cea43-eb53-8ead-42e5-80de86d72d1c [Leader]" -2025-01-28T14:48:52.270Z [INFO] server-932cea43-eb53-8ead-42e5-80de86d72d1c: added peer, starting replication: peer=server-d699ea89-e89c-78b2-290e-53716f271be2 -2025-01-28T14:48:52.271Z [INFO] server-932cea43-eb53-8ead-42e5-80de86d72d1c: pipelining replication: peer="{Voter server-d699ea89-e89c-78b2-290e-53716f271be2 d699ea89-e89c-78b2-290e-53716f271be2}" -2025-01-28T14:48:52.376Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:48:52.270692524 +0000 UTC m=+115.354097549 (1 nodes), 62.847533ms from start of poll, 64.516916ms from cluster start. Timeout at 2025-01-28 14:48:52.375727525 +0000 UTC m=+115.459132882, 105.035333ms after stability -2025-01-28T14:48:52.376Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:52.482Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:52.376289994 +0000 UTC m=+115.459694685 (1 nodes), 6.667µs from start of poll, 170.114052ms from cluster start. Timeout at 2025-01-28 14:48:52.481645004 +0000 UTC m=+115.565049695, 105.35501ms after stability -2025-01-28T14:48:52.495Z [INFO] server-932cea43-eb53-8ead-42e5-80de86d72d1c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d699ea89-e89c-78b2-290e-53716f271be2 Address:d699ea89-e89c-78b2-290e-53716f271be2} {Suffrage:Voter ID:server-932cea43-eb53-8ead-42e5-80de86d72d1c Address:932cea43-eb53-8ead-42e5-80de86d72d1c}]" -2025-01-28T14:48:52.496Z [INFO] server-932cea43-eb53-8ead-42e5-80de86d72d1c: aborting pipeline replication: peer="{Voter server-d699ea89-e89c-78b2-290e-53716f271be2 d699ea89-e89c-78b2-290e-53716f271be2}" +2025-01-28T14:55:48.644Z [INFO] server-9504af7e-f5a7-c933-2c5d-630cb7d275a9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9504af7e-f5a7-c933-2c5d-630cb7d275a9 Address:9504af7e-f5a7-c933-2c5d-630cb7d275a9} {Suffrage:Voter ID:server-1a9d936b-b713-e5f2-0aab-c55d0efada1b Address:1a9d936b-b713-e5f2-0aab-c55d0efada1b}]" +2025-01-28T14:55:48.645Z [INFO] server-1a9d936b-b713-e5f2-0aab-c55d0efada1b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9504af7e-f5a7-c933-2c5d-630cb7d275a9 Address:9504af7e-f5a7-c933-2c5d-630cb7d275a9} {Suffrage:Voter ID:server-1a9d936b-b713-e5f2-0aab-c55d0efada1b Address:1a9d936b-b713-e5f2-0aab-c55d0efada1b}]" +2025-01-28T14:55:48.645Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:55:48.645Z [INFO] server-1a9d936b-b713-e5f2-0aab-c55d0efada1b: entering follower state: follower="Node at 1a9d936b-b713-e5f2-0aab-c55d0efada1b [Follower]" leader-address= leader-id= +2025-01-28T14:55:48.645Z [INFO] server-9504af7e-f5a7-c933-2c5d-630cb7d275a9: entering follower state: follower="Node at 9504af7e-f5a7-c933-2c5d-630cb7d275a9 [Follower]" leader-address= leader-id= +2025-01-28T14:55:48.718Z [WARN] server-9504af7e-f5a7-c933-2c5d-630cb7d275a9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:48.718Z [INFO] server-9504af7e-f5a7-c933-2c5d-630cb7d275a9: entering candidate state: node="Node at 9504af7e-f5a7-c933-2c5d-630cb7d275a9 [Candidate]" term=2 +2025-01-28T14:55:48.719Z [INFO] server-9504af7e-f5a7-c933-2c5d-630cb7d275a9: election won: term=2 tally=2 +2025-01-28T14:55:48.719Z [INFO] server-9504af7e-f5a7-c933-2c5d-630cb7d275a9: entering leader state: leader="Node at 9504af7e-f5a7-c933-2c5d-630cb7d275a9 [Leader]" +2025-01-28T14:55:48.719Z [INFO] server-9504af7e-f5a7-c933-2c5d-630cb7d275a9: added peer, starting replication: peer=server-1a9d936b-b713-e5f2-0aab-c55d0efada1b +2025-01-28T14:55:48.719Z [INFO] server-9504af7e-f5a7-c933-2c5d-630cb7d275a9: pipelining replication: peer="{Voter server-1a9d936b-b713-e5f2-0aab-c55d0efada1b 1a9d936b-b713-e5f2-0aab-c55d0efada1b}" +2025-01-28T14:55:48.824Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:55:48.719202464 +0000 UTC m=+134.393692443 (1 nodes), 73.888129ms from start of poll, 74.457216ms from cluster start. Timeout at 2025-01-28 14:55:48.824212582 +0000 UTC m=+134.498702562, 105.010119ms after stability +2025-01-28T14:55:48.825Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:48.931Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:48.82556383 +0000 UTC m=+134.500053790 (1 nodes), 1.98µs from start of poll, 180.818563ms from cluster start. Timeout at 2025-01-28 14:55:48.930759011 +0000 UTC m=+134.605248971, 105.195181ms after stability +2025-01-28T14:55:48.943Z [INFO] server-1a9d936b-b713-e5f2-0aab-c55d0efada1b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9504af7e-f5a7-c933-2c5d-630cb7d275a9 Address:9504af7e-f5a7-c933-2c5d-630cb7d275a9} {Suffrage:Voter ID:server-1a9d936b-b713-e5f2-0aab-c55d0efada1b Address:1a9d936b-b713-e5f2-0aab-c55d0efada1b}]" +2025-01-28T14:55:48.944Z [INFO] server-9504af7e-f5a7-c933-2c5d-630cb7d275a9: aborting pipeline replication: peer="{Voter server-1a9d936b-b713-e5f2-0aab-c55d0efada1b 1a9d936b-b713-e5f2-0aab-c55d0efada1b}" --- PASS: TestRaft_GetConfigurationNoBootstrap (0.30s) === RUN TestRaft_LogStoreIsMonotonic -2025-01-28T14:48:52.502Z [INFO] server-2e4d42ae-cb6e-c1ac-6b30-c58d4fa0f40f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2e4d42ae-cb6e-c1ac-6b30-c58d4fa0f40f Address:2e4d42ae-cb6e-c1ac-6b30-c58d4fa0f40f}]" -2025-01-28T14:48:52.503Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:52.503Z [INFO] server-2e4d42ae-cb6e-c1ac-6b30-c58d4fa0f40f: entering follower state: follower="Node at 2e4d42ae-cb6e-c1ac-6b30-c58d4fa0f40f [Follower]" leader-address= leader-id= -2025-01-28T14:48:52.582Z [WARN] server-2e4d42ae-cb6e-c1ac-6b30-c58d4fa0f40f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:52.582Z [INFO] server-2e4d42ae-cb6e-c1ac-6b30-c58d4fa0f40f: entering candidate state: node="Node at 2e4d42ae-cb6e-c1ac-6b30-c58d4fa0f40f [Candidate]" term=2 -2025-01-28T14:48:52.582Z [INFO] server-2e4d42ae-cb6e-c1ac-6b30-c58d4fa0f40f: election won: term=2 tally=1 -2025-01-28T14:48:52.582Z [INFO] server-2e4d42ae-cb6e-c1ac-6b30-c58d4fa0f40f: entering leader state: leader="Node at 2e4d42ae-cb6e-c1ac-6b30-c58d4fa0f40f [Leader]" -2025-01-28T14:48:52.688Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:52.582989466 +0000 UTC m=+115.666394490 (1 nodes), 79.737701ms from start of poll, 80.723064ms from cluster start. Timeout at 2025-01-28 14:48:52.6880098 +0000 UTC m=+115.771414825, 105.020335ms after stability ---- PASS: TestRaft_LogStoreIsMonotonic (0.19s) +2025-01-28T14:55:48.946Z [INFO] server-6075dfdb-c5c3-c542-65d2-1b0d11eba32f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6075dfdb-c5c3-c542-65d2-1b0d11eba32f Address:6075dfdb-c5c3-c542-65d2-1b0d11eba32f}]" +2025-01-28T14:55:48.946Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:48.946Z [INFO] server-6075dfdb-c5c3-c542-65d2-1b0d11eba32f: entering follower state: follower="Node at 6075dfdb-c5c3-c542-65d2-1b0d11eba32f [Follower]" leader-address= leader-id= +2025-01-28T14:55:49.021Z [WARN] server-6075dfdb-c5c3-c542-65d2-1b0d11eba32f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:49.021Z [INFO] server-6075dfdb-c5c3-c542-65d2-1b0d11eba32f: entering candidate state: node="Node at 6075dfdb-c5c3-c542-65d2-1b0d11eba32f [Candidate]" term=2 +2025-01-28T14:55:49.021Z [INFO] server-6075dfdb-c5c3-c542-65d2-1b0d11eba32f: election won: term=2 tally=1 +2025-01-28T14:55:49.021Z [INFO] server-6075dfdb-c5c3-c542-65d2-1b0d11eba32f: entering leader state: leader="Node at 6075dfdb-c5c3-c542-65d2-1b0d11eba32f [Leader]" +2025-01-28T14:55:49.127Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:49.021611716 +0000 UTC m=+134.696101696 (1 nodes), 75.05206ms from start of poll, 75.436344ms from cluster start. Timeout at 2025-01-28 14:55:49.126616196 +0000 UTC m=+134.801106316, 105.00462ms after stability +--- PASS: TestRaft_LogStoreIsMonotonic (0.18s) === RUN TestRaft_CacheLogWithStoreError -2025-01-28T14:48:52.695Z [INFO] server-c1801c2c-c2cc-9592-1f1b-987e27cf26ea: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c1801c2c-c2cc-9592-1f1b-987e27cf26ea Address:c1801c2c-c2cc-9592-1f1b-987e27cf26ea} {Suffrage:Voter ID:server-e5696284-fc7d-53cb-f50e-429cb2693fee Address:e5696284-fc7d-53cb-f50e-429cb2693fee}]" -2025-01-28T14:48:52.695Z [INFO] server-c1801c2c-c2cc-9592-1f1b-987e27cf26ea: entering follower state: follower="Node at c1801c2c-c2cc-9592-1f1b-987e27cf26ea [Follower]" leader-address= leader-id= -2025-01-28T14:48:52.696Z [INFO] server-e5696284-fc7d-53cb-f50e-429cb2693fee: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c1801c2c-c2cc-9592-1f1b-987e27cf26ea Address:c1801c2c-c2cc-9592-1f1b-987e27cf26ea} {Suffrage:Voter ID:server-e5696284-fc7d-53cb-f50e-429cb2693fee Address:e5696284-fc7d-53cb-f50e-429cb2693fee}]" -2025-01-28T14:48:52.696Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:48:52.696Z [INFO] server-e5696284-fc7d-53cb-f50e-429cb2693fee: entering follower state: follower="Node at e5696284-fc7d-53cb-f50e-429cb2693fee [Follower]" leader-address= leader-id= -2025-01-28T14:48:52.750Z [WARN] server-c1801c2c-c2cc-9592-1f1b-987e27cf26ea: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:52.750Z [INFO] server-c1801c2c-c2cc-9592-1f1b-987e27cf26ea: entering candidate state: node="Node at c1801c2c-c2cc-9592-1f1b-987e27cf26ea [Candidate]" term=2 -2025-01-28T14:48:52.750Z [INFO] server-c1801c2c-c2cc-9592-1f1b-987e27cf26ea: election won: term=2 tally=2 -2025-01-28T14:48:52.751Z [INFO] server-c1801c2c-c2cc-9592-1f1b-987e27cf26ea: entering leader state: leader="Node at c1801c2c-c2cc-9592-1f1b-987e27cf26ea [Leader]" -2025-01-28T14:48:52.751Z [INFO] server-c1801c2c-c2cc-9592-1f1b-987e27cf26ea: added peer, starting replication: peer=server-e5696284-fc7d-53cb-f50e-429cb2693fee -2025-01-28T14:48:52.752Z [INFO] server-c1801c2c-c2cc-9592-1f1b-987e27cf26ea: pipelining replication: peer="{Voter server-e5696284-fc7d-53cb-f50e-429cb2693fee e5696284-fc7d-53cb-f50e-429cb2693fee}" -2025-01-28T14:48:52.859Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:48:52.751090125 +0000 UTC m=+115.834495149 (1 nodes), 54.779293ms from start of poll, 56.184002ms from cluster start. Timeout at 2025-01-28 14:48:52.856102125 +0000 UTC m=+115.939506816, 105.011667ms after stability -2025-01-28T14:48:52.859Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:52.965Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:52.859643682 +0000 UTC m=+115.943048707 (1 nodes), 6.334µs from start of poll, 164.73756ms from cluster start. Timeout at 2025-01-28 14:48:52.96502036 +0000 UTC m=+116.048425051, 105.376344ms after stability -2025-01-28T14:48:52.970Z [ERROR] server-e5696284-fc7d-53cb-f50e-429cb2693fee: failed to append to logs: error="unable to store logs within log store, err: "some error"" -2025-01-28T14:48:52.970Z [WARN] server-e5696284-fc7d-53cb-f50e-429cb2693fee: failed to get previous log: previous-index=9 last-index=7 error="log not found" -2025-01-28T14:48:52.971Z [INFO] server-c1801c2c-c2cc-9592-1f1b-987e27cf26ea: aborting pipeline replication: peer="{Voter server-e5696284-fc7d-53cb-f50e-429cb2693fee e5696284-fc7d-53cb-f50e-429cb2693fee}" -2025-01-28T14:48:52.979Z [INFO] server-c1801c2c-c2cc-9592-1f1b-987e27cf26ea: pipelining replication: peer="{Voter server-e5696284-fc7d-53cb-f50e-429cb2693fee e5696284-fc7d-53cb-f50e-429cb2693fee}" -2025-01-28T14:48:52.980Z [INFO] server-e5696284-fc7d-53cb-f50e-429cb2693fee: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c1801c2c-c2cc-9592-1f1b-987e27cf26ea Address:c1801c2c-c2cc-9592-1f1b-987e27cf26ea} {Suffrage:Voter ID:server-e5696284-fc7d-53cb-f50e-429cb2693fee Address:e5696284-fc7d-53cb-f50e-429cb2693fee}]" -2025-01-28T14:48:52.980Z [INFO] server-c1801c2c-c2cc-9592-1f1b-987e27cf26ea: aborting pipeline replication: peer="{Voter server-e5696284-fc7d-53cb-f50e-429cb2693fee e5696284-fc7d-53cb-f50e-429cb2693fee}" +2025-01-28T14:55:49.129Z [INFO] server-b0dbc48b-3325-97a5-3b59-17f6ac95743f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b0dbc48b-3325-97a5-3b59-17f6ac95743f Address:b0dbc48b-3325-97a5-3b59-17f6ac95743f} {Suffrage:Voter ID:server-87b305a3-5544-05f9-6f42-0e5bdae05c67 Address:87b305a3-5544-05f9-6f42-0e5bdae05c67}]" +2025-01-28T14:55:49.130Z [INFO] server-87b305a3-5544-05f9-6f42-0e5bdae05c67: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b0dbc48b-3325-97a5-3b59-17f6ac95743f Address:b0dbc48b-3325-97a5-3b59-17f6ac95743f} {Suffrage:Voter ID:server-87b305a3-5544-05f9-6f42-0e5bdae05c67 Address:87b305a3-5544-05f9-6f42-0e5bdae05c67}]" +2025-01-28T14:55:49.130Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:55:49.134Z [INFO] server-b0dbc48b-3325-97a5-3b59-17f6ac95743f: entering follower state: follower="Node at b0dbc48b-3325-97a5-3b59-17f6ac95743f [Follower]" leader-address= leader-id= +2025-01-28T14:55:49.134Z [INFO] server-87b305a3-5544-05f9-6f42-0e5bdae05c67: entering follower state: follower="Node at 87b305a3-5544-05f9-6f42-0e5bdae05c67 [Follower]" leader-address= leader-id= +2025-01-28T14:55:49.194Z [WARN] server-87b305a3-5544-05f9-6f42-0e5bdae05c67: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:49.195Z [INFO] server-87b305a3-5544-05f9-6f42-0e5bdae05c67: entering candidate state: node="Node at 87b305a3-5544-05f9-6f42-0e5bdae05c67 [Candidate]" term=2 +2025-01-28T14:55:49.195Z [INFO] server-87b305a3-5544-05f9-6f42-0e5bdae05c67: election won: term=2 tally=2 +2025-01-28T14:55:49.195Z [INFO] server-87b305a3-5544-05f9-6f42-0e5bdae05c67: entering leader state: leader="Node at 87b305a3-5544-05f9-6f42-0e5bdae05c67 [Leader]" +2025-01-28T14:55:49.195Z [INFO] server-87b305a3-5544-05f9-6f42-0e5bdae05c67: added peer, starting replication: peer=server-b0dbc48b-3325-97a5-3b59-17f6ac95743f +2025-01-28T14:55:49.195Z [INFO] server-87b305a3-5544-05f9-6f42-0e5bdae05c67: pipelining replication: peer="{Voter server-b0dbc48b-3325-97a5-3b59-17f6ac95743f b0dbc48b-3325-97a5-3b59-17f6ac95743f}" +2025-01-28T14:55:49.301Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:55:49.19540772 +0000 UTC m=+134.869897700 (1 nodes), 64.368587ms from start of poll, 65.701762ms from cluster start. Timeout at 2025-01-28 14:55:49.300431378 +0000 UTC m=+134.974921358, 105.023658ms after stability +2025-01-28T14:55:49.301Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:49.406Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:49.301459196 +0000 UTC m=+134.975949176 (1 nodes), 1.48µs from start of poll, 171.753238ms from cluster start. Timeout at 2025-01-28 14:55:49.406649538 +0000 UTC m=+135.081139518, 105.190342ms after stability +2025-01-28T14:55:49.408Z [ERROR] server-b0dbc48b-3325-97a5-3b59-17f6ac95743f: failed to append to logs: error="unable to store logs within log store, err: "some error"" +2025-01-28T14:55:49.408Z [WARN] server-b0dbc48b-3325-97a5-3b59-17f6ac95743f: failed to get previous log: previous-index=9 last-index=7 error="log not found" +2025-01-28T14:55:49.408Z [INFO] server-87b305a3-5544-05f9-6f42-0e5bdae05c67: aborting pipeline replication: peer="{Voter server-b0dbc48b-3325-97a5-3b59-17f6ac95743f b0dbc48b-3325-97a5-3b59-17f6ac95743f}" +2025-01-28T14:55:49.418Z [INFO] server-87b305a3-5544-05f9-6f42-0e5bdae05c67: pipelining replication: peer="{Voter server-b0dbc48b-3325-97a5-3b59-17f6ac95743f b0dbc48b-3325-97a5-3b59-17f6ac95743f}" +2025-01-28T14:55:49.418Z [INFO] server-b0dbc48b-3325-97a5-3b59-17f6ac95743f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b0dbc48b-3325-97a5-3b59-17f6ac95743f Address:b0dbc48b-3325-97a5-3b59-17f6ac95743f} {Suffrage:Voter ID:server-87b305a3-5544-05f9-6f42-0e5bdae05c67 Address:87b305a3-5544-05f9-6f42-0e5bdae05c67}]" +2025-01-28T14:55:49.418Z [INFO] server-87b305a3-5544-05f9-6f42-0e5bdae05c67: aborting pipeline replication: peer="{Voter server-b0dbc48b-3325-97a5-3b59-17f6ac95743f b0dbc48b-3325-97a5-3b59-17f6ac95743f}" --- PASS: TestRaft_CacheLogWithStoreError (0.29s) === RUN TestRaft_ReloadConfig -2025-01-28T14:48:52.986Z [INFO] server-928c4222-f6ba-02e3-7020-b64180a6433b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-928c4222-f6ba-02e3-7020-b64180a6433b Address:928c4222-f6ba-02e3-7020-b64180a6433b}]" +2025-01-28T14:55:49.420Z [INFO] server-afbd0443-ec9d-5a6f-1b46-d9896ecd3b3a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-afbd0443-ec9d-5a6f-1b46-d9896ecd3b3a Address:afbd0443-ec9d-5a6f-1b46-d9896ecd3b3a}]" --- PASS: TestRaft_ReloadConfig (0.00s) === RUN TestRaft_ReloadConfigValidates -2025-01-28T14:48:52.990Z [INFO] server-114adba3-5315-b185-90e8-8ebf73c56e6a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-114adba3-5315-b185-90e8-8ebf73c56e6a Address:114adba3-5315-b185-90e8-8ebf73c56e6a}]" -2025-01-28T14:48:52.991Z [INFO] server-114adba3-5315-b185-90e8-8ebf73c56e6a: entering follower state: follower="Node at 114adba3-5315-b185-90e8-8ebf73c56e6a [Follower]" leader-address= leader-id= +2025-01-28T14:55:49.422Z [INFO] server-cb2322c6-30f1-1ee3-4961-f36c5d1768e7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cb2322c6-30f1-1ee3-4961-f36c5d1768e7 Address:cb2322c6-30f1-1ee3-4961-f36c5d1768e7}]" --- PASS: TestRaft_ReloadConfigValidates (0.00s) === RUN TestRaft_InstallSnapshot_InvalidPeers -2025-01-28T14:48:52.993Z [ERROR] failed to install snapshot: error="failed to decode peers: codec.decoder: Only encoded map or array can be decoded into a slice. (valueType: 2)" +2025-01-28T14:55:49.423Z [ERROR] failed to install snapshot: error="failed to decode peers: codec.decoder: Only encoded map or array can be decoded into a slice. (valueType: 2)" --- PASS: TestRaft_InstallSnapshot_InvalidPeers (0.00s) === RUN TestRaft_VoteNotGranted_WhenNodeNotInCluster -2025-01-28T14:48:52.998Z [INFO] server-88b88133-5891-83ce-40d5-3d3fe98b0de4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-88b88133-5891-83ce-40d5-3d3fe98b0de4 Address:88b88133-5891-83ce-40d5-3d3fe98b0de4} {Suffrage:Voter ID:server-5c6b8d04-d082-09e9-06b2-3b08436b6107 Address:5c6b8d04-d082-09e9-06b2-3b08436b6107} {Suffrage:Voter ID:server-c192c0cc-d216-cd75-6587-0b16d94694e7 Address:c192c0cc-d216-cd75-6587-0b16d94694e7}]" -2025-01-28T14:48:52.999Z [INFO] server-88b88133-5891-83ce-40d5-3d3fe98b0de4: entering follower state: follower="Node at 88b88133-5891-83ce-40d5-3d3fe98b0de4 [Follower]" leader-address= leader-id= -2025-01-28T14:48:52.999Z [INFO] server-5c6b8d04-d082-09e9-06b2-3b08436b6107: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-88b88133-5891-83ce-40d5-3d3fe98b0de4 Address:88b88133-5891-83ce-40d5-3d3fe98b0de4} {Suffrage:Voter ID:server-5c6b8d04-d082-09e9-06b2-3b08436b6107 Address:5c6b8d04-d082-09e9-06b2-3b08436b6107} {Suffrage:Voter ID:server-c192c0cc-d216-cd75-6587-0b16d94694e7 Address:c192c0cc-d216-cd75-6587-0b16d94694e7}]" -2025-01-28T14:48:52.999Z [INFO] server-5c6b8d04-d082-09e9-06b2-3b08436b6107: entering follower state: follower="Node at 5c6b8d04-d082-09e9-06b2-3b08436b6107 [Follower]" leader-address= leader-id= -2025-01-28T14:48:53.000Z [INFO] server-c192c0cc-d216-cd75-6587-0b16d94694e7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-88b88133-5891-83ce-40d5-3d3fe98b0de4 Address:88b88133-5891-83ce-40d5-3d3fe98b0de4} {Suffrage:Voter ID:server-5c6b8d04-d082-09e9-06b2-3b08436b6107 Address:5c6b8d04-d082-09e9-06b2-3b08436b6107} {Suffrage:Voter ID:server-c192c0cc-d216-cd75-6587-0b16d94694e7 Address:c192c0cc-d216-cd75-6587-0b16d94694e7}]" -2025-01-28T14:48:53.000Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:53.000Z [INFO] server-c192c0cc-d216-cd75-6587-0b16d94694e7: entering follower state: follower="Node at c192c0cc-d216-cd75-6587-0b16d94694e7 [Follower]" leader-address= leader-id= -2025-01-28T14:48:53.061Z [WARN] server-5c6b8d04-d082-09e9-06b2-3b08436b6107: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:53.061Z [INFO] server-5c6b8d04-d082-09e9-06b2-3b08436b6107: entering candidate state: node="Node at 5c6b8d04-d082-09e9-06b2-3b08436b6107 [Candidate]" term=2 -2025-01-28T14:48:53.062Z [INFO] server-5c6b8d04-d082-09e9-06b2-3b08436b6107: election won: term=2 tally=2 -2025-01-28T14:48:53.062Z [INFO] server-5c6b8d04-d082-09e9-06b2-3b08436b6107: entering leader state: leader="Node at 5c6b8d04-d082-09e9-06b2-3b08436b6107 [Leader]" -2025-01-28T14:48:53.062Z [INFO] server-5c6b8d04-d082-09e9-06b2-3b08436b6107: added peer, starting replication: peer=server-88b88133-5891-83ce-40d5-3d3fe98b0de4 -2025-01-28T14:48:53.062Z [INFO] server-5c6b8d04-d082-09e9-06b2-3b08436b6107: added peer, starting replication: peer=server-c192c0cc-d216-cd75-6587-0b16d94694e7 -2025-01-28T14:48:53.063Z [INFO] server-5c6b8d04-d082-09e9-06b2-3b08436b6107: pipelining replication: peer="{Voter server-88b88133-5891-83ce-40d5-3d3fe98b0de4 88b88133-5891-83ce-40d5-3d3fe98b0de4}" -2025-01-28T14:48:53.063Z [INFO] server-5c6b8d04-d082-09e9-06b2-3b08436b6107: pipelining replication: peer="{Voter server-c192c0cc-d216-cd75-6587-0b16d94694e7 c192c0cc-d216-cd75-6587-0b16d94694e7}" -2025-01-28T14:48:53.167Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:53.06261171 +0000 UTC m=+116.146016734 (1 nodes), 62.268182ms from start of poll, 64.514249ms from cluster start. Timeout at 2025-01-28 14:48:53.167675045 +0000 UTC m=+116.251080070, 105.063336ms after stability -2025-01-28T14:48:53.173Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:48:53.279Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:48:53.173319665 +0000 UTC m=+116.256724689 (2 nodes), 9µs from start of poll, 175.222204ms from cluster start. Timeout at 2025-01-28 14:48:53.27873601 +0000 UTC m=+116.362141035, 105.416346ms after stability -2025-01-28T14:48:53.279Z [INFO] server-5c6b8d04-d082-09e9-06b2-3b08436b6107: updating configuration: command=RemoveServer server-id=server-88b88133-5891-83ce-40d5-3d3fe98b0de4 server-addr= servers="[{Suffrage:Voter ID:server-5c6b8d04-d082-09e9-06b2-3b08436b6107 Address:5c6b8d04-d082-09e9-06b2-3b08436b6107} {Suffrage:Voter ID:server-c192c0cc-d216-cd75-6587-0b16d94694e7 Address:c192c0cc-d216-cd75-6587-0b16d94694e7}]" -2025-01-28T14:48:53.279Z [INFO] server-5c6b8d04-d082-09e9-06b2-3b08436b6107: removed peer, stopping replication: peer=server-88b88133-5891-83ce-40d5-3d3fe98b0de4 last-index=3 -2025-01-28T14:48:53.280Z [INFO] server-5c6b8d04-d082-09e9-06b2-3b08436b6107: aborting pipeline replication: peer="{Voter server-88b88133-5891-83ce-40d5-3d3fe98b0de4 88b88133-5891-83ce-40d5-3d3fe98b0de4}" -2025-01-28T14:48:53.365Z [WARN] server-88b88133-5891-83ce-40d5-3d3fe98b0de4: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election -2025-01-28T14:48:53.387Z [ERROR] server-5c6b8d04-d082-09e9-06b2-3b08436b6107: failed to pipeline appendEntries: peer="{Voter server-c192c0cc-d216-cd75-6587-0b16d94694e7 c192c0cc-d216-cd75-6587-0b16d94694e7}" error="append pipeline closed" -2025-01-28T14:48:53.387Z [INFO] server-5c6b8d04-d082-09e9-06b2-3b08436b6107: aborting pipeline replication: peer="{Voter server-c192c0cc-d216-cd75-6587-0b16d94694e7 c192c0cc-d216-cd75-6587-0b16d94694e7}" -2025-01-28T14:48:53.389Z [ERROR] server-5c6b8d04-d082-09e9-06b2-3b08436b6107: failed to heartbeat to: peer=c192c0cc-d216-cd75-6587-0b16d94694e7 backoff time=10ms error="failed to connect to peer: c192c0cc-d216-cd75-6587-0b16d94694e7" -2025-01-28T14:48:53.393Z [ERROR] server-5c6b8d04-d082-09e9-06b2-3b08436b6107: failed to appendEntries to: peer="{Voter server-c192c0cc-d216-cd75-6587-0b16d94694e7 c192c0cc-d216-cd75-6587-0b16d94694e7}" error="failed to connect to peer: c192c0cc-d216-cd75-6587-0b16d94694e7" -2025-01-28T14:48:53.406Z [ERROR] server-5c6b8d04-d082-09e9-06b2-3b08436b6107: failed to heartbeat to: peer=c192c0cc-d216-cd75-6587-0b16d94694e7 backoff time=10ms error="failed to connect to peer: c192c0cc-d216-cd75-6587-0b16d94694e7" -2025-01-28T14:48:53.413Z [ERROR] server-5c6b8d04-d082-09e9-06b2-3b08436b6107: failed to appendEntries to: peer="{Voter server-c192c0cc-d216-cd75-6587-0b16d94694e7 c192c0cc-d216-cd75-6587-0b16d94694e7}" error="failed to connect to peer: c192c0cc-d216-cd75-6587-0b16d94694e7" -2025-01-28T14:48:53.426Z [ERROR] server-5c6b8d04-d082-09e9-06b2-3b08436b6107: failed to heartbeat to: peer=c192c0cc-d216-cd75-6587-0b16d94694e7 backoff time=10ms error="failed to connect to peer: c192c0cc-d216-cd75-6587-0b16d94694e7" -2025-01-28T14:48:53.430Z [WARN] server-5c6b8d04-d082-09e9-06b2-3b08436b6107: failed to contact: server-id=server-c192c0cc-d216-cd75-6587-0b16d94694e7 time=50.196157ms -2025-01-28T14:48:53.430Z [WARN] server-5c6b8d04-d082-09e9-06b2-3b08436b6107: failed to contact quorum of nodes, stepping down -2025-01-28T14:48:53.430Z [INFO] server-5c6b8d04-d082-09e9-06b2-3b08436b6107: entering follower state: follower="Node at 5c6b8d04-d082-09e9-06b2-3b08436b6107 [Follower]" leader-address= leader-id= -2025-01-28T14:48:53.433Z [ERROR] server-5c6b8d04-d082-09e9-06b2-3b08436b6107: failed to appendEntries to: peer="{Voter server-c192c0cc-d216-cd75-6587-0b16d94694e7 c192c0cc-d216-cd75-6587-0b16d94694e7}" error="failed to connect to peer: c192c0cc-d216-cd75-6587-0b16d94694e7" -2025-01-28T14:48:53.482Z [WARN] server-c192c0cc-d216-cd75-6587-0b16d94694e7: heartbeat timeout reached, starting election: last-leader-addr=5c6b8d04-d082-09e9-06b2-3b08436b6107 last-leader-id=server-5c6b8d04-d082-09e9-06b2-3b08436b6107 -2025-01-28T14:48:53.483Z [INFO] server-c192c0cc-d216-cd75-6587-0b16d94694e7: entering candidate state: node="Node at c192c0cc-d216-cd75-6587-0b16d94694e7 [Candidate]" term=3 -2025-01-28T14:48:53.483Z [ERROR] server-c192c0cc-d216-cd75-6587-0b16d94694e7: failed to make requestVote RPC: target="{Voter server-5c6b8d04-d082-09e9-06b2-3b08436b6107 5c6b8d04-d082-09e9-06b2-3b08436b6107}" error="failed to connect to peer: 5c6b8d04-d082-09e9-06b2-3b08436b6107" term=3 -2025-01-28T14:48:53.491Z [WARN] server-c192c0cc-d216-cd75-6587-0b16d94694e7: rejecting vote request since node is not in configuration: from=88b88133-5891-83ce-40d5-3d3fe98b0de4 ---- PASS: TestRaft_VoteNotGranted_WhenNodeNotInCluster (0.50s) +2025-01-28T14:55:49.425Z [INFO] server-fc9db554-9723-5936-cfa0-bb7cb96e09a3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fc9db554-9723-5936-cfa0-bb7cb96e09a3 Address:fc9db554-9723-5936-cfa0-bb7cb96e09a3} {Suffrage:Voter ID:server-51497266-e076-d678-05fa-8e87d0870285 Address:51497266-e076-d678-05fa-8e87d0870285} {Suffrage:Voter ID:server-e42861cd-4a11-860f-9cf8-f69f06cd95c0 Address:e42861cd-4a11-860f-9cf8-f69f06cd95c0}]" +2025-01-28T14:55:49.425Z [INFO] server-fc9db554-9723-5936-cfa0-bb7cb96e09a3: entering follower state: follower="Node at fc9db554-9723-5936-cfa0-bb7cb96e09a3 [Follower]" leader-address= leader-id= +2025-01-28T14:55:49.426Z [INFO] server-51497266-e076-d678-05fa-8e87d0870285: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fc9db554-9723-5936-cfa0-bb7cb96e09a3 Address:fc9db554-9723-5936-cfa0-bb7cb96e09a3} {Suffrage:Voter ID:server-51497266-e076-d678-05fa-8e87d0870285 Address:51497266-e076-d678-05fa-8e87d0870285} {Suffrage:Voter ID:server-e42861cd-4a11-860f-9cf8-f69f06cd95c0 Address:e42861cd-4a11-860f-9cf8-f69f06cd95c0}]" +2025-01-28T14:55:49.426Z [INFO] server-51497266-e076-d678-05fa-8e87d0870285: entering follower state: follower="Node at 51497266-e076-d678-05fa-8e87d0870285 [Follower]" leader-address= leader-id= +2025-01-28T14:55:49.426Z [INFO] server-e42861cd-4a11-860f-9cf8-f69f06cd95c0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fc9db554-9723-5936-cfa0-bb7cb96e09a3 Address:fc9db554-9723-5936-cfa0-bb7cb96e09a3} {Suffrage:Voter ID:server-51497266-e076-d678-05fa-8e87d0870285 Address:51497266-e076-d678-05fa-8e87d0870285} {Suffrage:Voter ID:server-e42861cd-4a11-860f-9cf8-f69f06cd95c0 Address:e42861cd-4a11-860f-9cf8-f69f06cd95c0}]" +2025-01-28T14:55:49.426Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:49.426Z [INFO] server-e42861cd-4a11-860f-9cf8-f69f06cd95c0: entering follower state: follower="Node at e42861cd-4a11-860f-9cf8-f69f06cd95c0 [Follower]" leader-address= leader-id= +2025-01-28T14:55:49.478Z [WARN] server-e42861cd-4a11-860f-9cf8-f69f06cd95c0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:49.478Z [INFO] server-e42861cd-4a11-860f-9cf8-f69f06cd95c0: entering candidate state: node="Node at e42861cd-4a11-860f-9cf8-f69f06cd95c0 [Candidate]" term=2 +2025-01-28T14:55:49.478Z [INFO] server-e42861cd-4a11-860f-9cf8-f69f06cd95c0: election won: term=2 tally=2 +2025-01-28T14:55:49.478Z [INFO] server-e42861cd-4a11-860f-9cf8-f69f06cd95c0: entering leader state: leader="Node at e42861cd-4a11-860f-9cf8-f69f06cd95c0 [Leader]" +2025-01-28T14:55:49.478Z [INFO] server-e42861cd-4a11-860f-9cf8-f69f06cd95c0: added peer, starting replication: peer=server-fc9db554-9723-5936-cfa0-bb7cb96e09a3 +2025-01-28T14:55:49.478Z [INFO] server-e42861cd-4a11-860f-9cf8-f69f06cd95c0: added peer, starting replication: peer=server-51497266-e076-d678-05fa-8e87d0870285 +2025-01-28T14:55:49.478Z [INFO] server-e42861cd-4a11-860f-9cf8-f69f06cd95c0: pipelining replication: peer="{Voter server-fc9db554-9723-5936-cfa0-bb7cb96e09a3 fc9db554-9723-5936-cfa0-bb7cb96e09a3}" +2025-01-28T14:55:49.479Z [INFO] server-e42861cd-4a11-860f-9cf8-f69f06cd95c0: pipelining replication: peer="{Voter server-51497266-e076-d678-05fa-8e87d0870285 51497266-e076-d678-05fa-8e87d0870285}" +2025-01-28T14:55:49.583Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:49.478777609 +0000 UTC m=+135.153267589 (1 nodes), 52.382119ms from start of poll, 53.300952ms from cluster start. Timeout at 2025-01-28 14:55:49.583787828 +0000 UTC m=+135.258277808, 105.010219ms after stability +2025-01-28T14:55:49.589Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:55:49.695Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:55:49.589603094 +0000 UTC m=+135.264093074 (2 nodes), 2.22µs from start of poll, 164.126437ms from cluster start. Timeout at 2025-01-28 14:55:49.694772458 +0000 UTC m=+135.369262518, 105.169444ms after stability +2025-01-28T14:55:49.695Z [INFO] server-e42861cd-4a11-860f-9cf8-f69f06cd95c0: updating configuration: command=RemoveServer server-id=server-fc9db554-9723-5936-cfa0-bb7cb96e09a3 server-addr= servers="[{Suffrage:Voter ID:server-51497266-e076-d678-05fa-8e87d0870285 Address:51497266-e076-d678-05fa-8e87d0870285} {Suffrage:Voter ID:server-e42861cd-4a11-860f-9cf8-f69f06cd95c0 Address:e42861cd-4a11-860f-9cf8-f69f06cd95c0}]" +2025-01-28T14:55:49.696Z [INFO] server-e42861cd-4a11-860f-9cf8-f69f06cd95c0: removed peer, stopping replication: peer=server-fc9db554-9723-5936-cfa0-bb7cb96e09a3 last-index=3 +2025-01-28T14:55:49.696Z [INFO] server-e42861cd-4a11-860f-9cf8-f69f06cd95c0: aborting pipeline replication: peer="{Voter server-fc9db554-9723-5936-cfa0-bb7cb96e09a3 fc9db554-9723-5936-cfa0-bb7cb96e09a3}" +2025-01-28T14:55:49.797Z [WARN] server-fc9db554-9723-5936-cfa0-bb7cb96e09a3: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +2025-01-28T14:55:49.805Z [ERROR] server-e42861cd-4a11-860f-9cf8-f69f06cd95c0: failed to pipeline appendEntries: peer="{Voter server-51497266-e076-d678-05fa-8e87d0870285 51497266-e076-d678-05fa-8e87d0870285}" error="append pipeline closed" +2025-01-28T14:55:49.805Z [INFO] server-e42861cd-4a11-860f-9cf8-f69f06cd95c0: aborting pipeline replication: peer="{Voter server-51497266-e076-d678-05fa-8e87d0870285 51497266-e076-d678-05fa-8e87d0870285}" +2025-01-28T14:55:49.808Z [ERROR] server-e42861cd-4a11-860f-9cf8-f69f06cd95c0: failed to heartbeat to: peer=51497266-e076-d678-05fa-8e87d0870285 backoff time=10ms error="failed to connect to peer: 51497266-e076-d678-05fa-8e87d0870285" +2025-01-28T14:55:49.817Z [ERROR] server-e42861cd-4a11-860f-9cf8-f69f06cd95c0: failed to appendEntries to: peer="{Voter server-51497266-e076-d678-05fa-8e87d0870285 51497266-e076-d678-05fa-8e87d0870285}" error="failed to connect to peer: 51497266-e076-d678-05fa-8e87d0870285" +2025-01-28T14:55:49.828Z [ERROR] server-e42861cd-4a11-860f-9cf8-f69f06cd95c0: failed to heartbeat to: peer=51497266-e076-d678-05fa-8e87d0870285 backoff time=10ms error="failed to connect to peer: 51497266-e076-d678-05fa-8e87d0870285" +2025-01-28T14:55:49.835Z [ERROR] server-e42861cd-4a11-860f-9cf8-f69f06cd95c0: failed to appendEntries to: peer="{Voter server-51497266-e076-d678-05fa-8e87d0870285 51497266-e076-d678-05fa-8e87d0870285}" error="failed to connect to peer: 51497266-e076-d678-05fa-8e87d0870285" +2025-01-28T14:55:49.847Z [ERROR] server-e42861cd-4a11-860f-9cf8-f69f06cd95c0: failed to heartbeat to: peer=51497266-e076-d678-05fa-8e87d0870285 backoff time=10ms error="failed to connect to peer: 51497266-e076-d678-05fa-8e87d0870285" +2025-01-28T14:55:49.849Z [WARN] server-e42861cd-4a11-860f-9cf8-f69f06cd95c0: failed to contact: server-id=server-51497266-e076-d678-05fa-8e87d0870285 time=50.548611ms +2025-01-28T14:55:49.849Z [WARN] server-e42861cd-4a11-860f-9cf8-f69f06cd95c0: failed to contact quorum of nodes, stepping down +2025-01-28T14:55:49.850Z [INFO] server-e42861cd-4a11-860f-9cf8-f69f06cd95c0: entering follower state: follower="Node at e42861cd-4a11-860f-9cf8-f69f06cd95c0 [Follower]" leader-address= leader-id= +2025-01-28T14:55:49.854Z [ERROR] server-e42861cd-4a11-860f-9cf8-f69f06cd95c0: failed to appendEntries to: peer="{Voter server-51497266-e076-d678-05fa-8e87d0870285 51497266-e076-d678-05fa-8e87d0870285}" error="failed to connect to peer: 51497266-e076-d678-05fa-8e87d0870285" +2025-01-28T14:55:49.883Z [WARN] server-51497266-e076-d678-05fa-8e87d0870285: heartbeat timeout reached, starting election: last-leader-addr=e42861cd-4a11-860f-9cf8-f69f06cd95c0 last-leader-id=server-e42861cd-4a11-860f-9cf8-f69f06cd95c0 +2025-01-28T14:55:49.883Z [INFO] server-51497266-e076-d678-05fa-8e87d0870285: entering candidate state: node="Node at 51497266-e076-d678-05fa-8e87d0870285 [Candidate]" term=3 +2025-01-28T14:55:49.883Z [ERROR] server-51497266-e076-d678-05fa-8e87d0870285: failed to make requestVote RPC: target="{Voter server-e42861cd-4a11-860f-9cf8-f69f06cd95c0 e42861cd-4a11-860f-9cf8-f69f06cd95c0}" error="failed to connect to peer: e42861cd-4a11-860f-9cf8-f69f06cd95c0" term=3 +2025-01-28T14:55:49.907Z [WARN] server-51497266-e076-d678-05fa-8e87d0870285: rejecting vote request since node is not in configuration: from=fc9db554-9723-5936-cfa0-bb7cb96e09a3 +--- PASS: TestRaft_VoteNotGranted_WhenNodeNotInCluster (0.49s) === RUN TestRaft_ClusterCanRegainStability_WhenNonVoterWithHigherTermJoin -2025-01-28T14:48:53.503Z [INFO] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0 Address:6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0} {Suffrage:Voter ID:server-a7075132-ba8a-d443-a1f5-6963bc4ad7af Address:a7075132-ba8a-d443-a1f5-6963bc4ad7af} {Suffrage:Voter ID:server-329b4e6c-bc90-2795-bd62-ace23e50051f Address:329b4e6c-bc90-2795-bd62-ace23e50051f}]" -2025-01-28T14:48:53.504Z [INFO] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: entering follower state: follower="Node at 6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0 [Follower]" leader-address= leader-id= -2025-01-28T14:48:53.505Z [INFO] server-a7075132-ba8a-d443-a1f5-6963bc4ad7af: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0 Address:6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0} {Suffrage:Voter ID:server-a7075132-ba8a-d443-a1f5-6963bc4ad7af Address:a7075132-ba8a-d443-a1f5-6963bc4ad7af} {Suffrage:Voter ID:server-329b4e6c-bc90-2795-bd62-ace23e50051f Address:329b4e6c-bc90-2795-bd62-ace23e50051f}]" -2025-01-28T14:48:53.505Z [INFO] server-a7075132-ba8a-d443-a1f5-6963bc4ad7af: entering follower state: follower="Node at a7075132-ba8a-d443-a1f5-6963bc4ad7af [Follower]" leader-address= leader-id= -2025-01-28T14:48:53.505Z [INFO] server-329b4e6c-bc90-2795-bd62-ace23e50051f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0 Address:6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0} {Suffrage:Voter ID:server-a7075132-ba8a-d443-a1f5-6963bc4ad7af Address:a7075132-ba8a-d443-a1f5-6963bc4ad7af} {Suffrage:Voter ID:server-329b4e6c-bc90-2795-bd62-ace23e50051f Address:329b4e6c-bc90-2795-bd62-ace23e50051f}]" -2025-01-28T14:48:53.506Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:53.506Z [INFO] server-329b4e6c-bc90-2795-bd62-ace23e50051f: entering follower state: follower="Node at 329b4e6c-bc90-2795-bd62-ace23e50051f [Follower]" leader-address= leader-id= -2025-01-28T14:48:53.561Z [WARN] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:53.561Z [INFO] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: entering candidate state: node="Node at 6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0 [Candidate]" term=2 -2025-01-28T14:48:53.562Z [INFO] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: election won: term=2 tally=2 -2025-01-28T14:48:53.562Z [INFO] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: entering leader state: leader="Node at 6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0 [Leader]" -2025-01-28T14:48:53.562Z [INFO] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: added peer, starting replication: peer=server-a7075132-ba8a-d443-a1f5-6963bc4ad7af -2025-01-28T14:48:53.563Z [INFO] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: added peer, starting replication: peer=server-329b4e6c-bc90-2795-bd62-ace23e50051f -2025-01-28T14:48:53.563Z [INFO] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: pipelining replication: peer="{Voter server-a7075132-ba8a-d443-a1f5-6963bc4ad7af a7075132-ba8a-d443-a1f5-6963bc4ad7af}" -2025-01-28T14:48:53.564Z [INFO] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: pipelining replication: peer="{Voter server-329b4e6c-bc90-2795-bd62-ace23e50051f 329b4e6c-bc90-2795-bd62-ace23e50051f}" -2025-01-28T14:48:53.636Z [WARN] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: failed to contact: server-id=server-a7075132-ba8a-d443-a1f5-6963bc4ad7af time=56.097333ms -2025-01-28T14:48:53.668Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:53.563345581 +0000 UTC m=+116.646750606 (1 nodes), 57.170031ms from start of poll, 60.074784ms from cluster start. Timeout at 2025-01-28 14:48:53.668362582 +0000 UTC m=+116.751767273, 105.016667ms after stability -2025-01-28T14:48:53.673Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:48:53.780Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:48:53.6739652 +0000 UTC m=+116.757370224 (2 nodes), 7.333µs from start of poll, 170.694402ms from cluster start. Timeout at 2025-01-28 14:48:53.779468881 +0000 UTC m=+116.862873906, 105.503682ms after stability -2025-01-28T14:48:53.780Z [ERROR] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: failed to pipeline appendEntries: peer="{Voter server-a7075132-ba8a-d443-a1f5-6963bc4ad7af a7075132-ba8a-d443-a1f5-6963bc4ad7af}" error="append pipeline closed" -2025-01-28T14:48:53.780Z [INFO] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: aborting pipeline replication: peer="{Voter server-a7075132-ba8a-d443-a1f5-6963bc4ad7af a7075132-ba8a-d443-a1f5-6963bc4ad7af}" -2025-01-28T14:48:53.782Z [ERROR] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: failed to heartbeat to: peer=a7075132-ba8a-d443-a1f5-6963bc4ad7af backoff time=10ms error="failed to connect to peer: a7075132-ba8a-d443-a1f5-6963bc4ad7af" -2025-01-28T14:48:53.788Z [ERROR] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: failed to appendEntries to: peer="{Voter server-a7075132-ba8a-d443-a1f5-6963bc4ad7af a7075132-ba8a-d443-a1f5-6963bc4ad7af}" error="failed to connect to peer: a7075132-ba8a-d443-a1f5-6963bc4ad7af" -2025-01-28T14:48:53.800Z [ERROR] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: failed to heartbeat to: peer=a7075132-ba8a-d443-a1f5-6963bc4ad7af backoff time=10ms error="failed to connect to peer: a7075132-ba8a-d443-a1f5-6963bc4ad7af" -2025-01-28T14:48:53.807Z [ERROR] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: failed to appendEntries to: peer="{Voter server-a7075132-ba8a-d443-a1f5-6963bc4ad7af a7075132-ba8a-d443-a1f5-6963bc4ad7af}" error="failed to connect to peer: a7075132-ba8a-d443-a1f5-6963bc4ad7af" -2025-01-28T14:48:53.818Z [ERROR] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: failed to heartbeat to: peer=a7075132-ba8a-d443-a1f5-6963bc4ad7af backoff time=10ms error="failed to connect to peer: a7075132-ba8a-d443-a1f5-6963bc4ad7af" -2025-01-28T14:48:53.824Z [ERROR] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: failed to appendEntries to: peer="{Voter server-a7075132-ba8a-d443-a1f5-6963bc4ad7af a7075132-ba8a-d443-a1f5-6963bc4ad7af}" error="failed to connect to peer: a7075132-ba8a-d443-a1f5-6963bc4ad7af" -2025-01-28T14:48:53.831Z [WARN] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: failed to contact: server-id=server-a7075132-ba8a-d443-a1f5-6963bc4ad7af time=57.107696ms -2025-01-28T14:48:53.840Z [ERROR] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: failed to heartbeat to: peer=a7075132-ba8a-d443-a1f5-6963bc4ad7af backoff time=20ms error="failed to connect to peer: a7075132-ba8a-d443-a1f5-6963bc4ad7af" -2025-01-28T14:48:53.849Z [WARN] server-a7075132-ba8a-d443-a1f5-6963bc4ad7af: heartbeat timeout reached, starting election: last-leader-addr=6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0 last-leader-id=server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0 -2025-01-28T14:48:53.849Z [INFO] server-a7075132-ba8a-d443-a1f5-6963bc4ad7af: entering candidate state: node="Node at a7075132-ba8a-d443-a1f5-6963bc4ad7af [Candidate]" term=3 -2025-01-28T14:48:53.849Z [ERROR] server-a7075132-ba8a-d443-a1f5-6963bc4ad7af: failed to make requestVote RPC: target="{Voter server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0 6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0}" error="failed to connect to peer: 6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0" term=3 -2025-01-28T14:48:53.849Z [ERROR] server-a7075132-ba8a-d443-a1f5-6963bc4ad7af: failed to make requestVote RPC: target="{Voter server-329b4e6c-bc90-2795-bd62-ace23e50051f 329b4e6c-bc90-2795-bd62-ace23e50051f}" error="failed to connect to peer: 329b4e6c-bc90-2795-bd62-ace23e50051f" term=3 -2025-01-28T14:48:53.851Z [ERROR] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: failed to appendEntries to: peer="{Voter server-a7075132-ba8a-d443-a1f5-6963bc4ad7af a7075132-ba8a-d443-a1f5-6963bc4ad7af}" error="failed to connect to peer: a7075132-ba8a-d443-a1f5-6963bc4ad7af" -2025-01-28T14:48:53.869Z [ERROR] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: failed to heartbeat to: peer=a7075132-ba8a-d443-a1f5-6963bc4ad7af backoff time=25ms error="failed to connect to peer: a7075132-ba8a-d443-a1f5-6963bc4ad7af" -2025-01-28T14:48:53.880Z [WARN] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: failed to contact: server-id=server-a7075132-ba8a-d443-a1f5-6963bc4ad7af time=106.352825ms -2025-01-28T14:48:53.885Z [INFO] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: updating configuration: command=RemoveServer server-id=server-a7075132-ba8a-d443-a1f5-6963bc4ad7af server-addr= servers="[{Suffrage:Voter ID:server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0 Address:6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0} {Suffrage:Voter ID:server-329b4e6c-bc90-2795-bd62-ace23e50051f Address:329b4e6c-bc90-2795-bd62-ace23e50051f}]" -2025-01-28T14:48:53.886Z [INFO] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: removed peer, stopping replication: peer=server-a7075132-ba8a-d443-a1f5-6963bc4ad7af last-index=3 -2025-01-28T14:48:53.887Z [INFO] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: updating configuration: command=AddNonvoter server-id=server-a7075132-ba8a-d443-a1f5-6963bc4ad7af server-addr=a7075132-ba8a-d443-a1f5-6963bc4ad7af servers="[{Suffrage:Voter ID:server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0 Address:6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0} {Suffrage:Voter ID:server-329b4e6c-bc90-2795-bd62-ace23e50051f Address:329b4e6c-bc90-2795-bd62-ace23e50051f} {Suffrage:Nonvoter ID:server-a7075132-ba8a-d443-a1f5-6963bc4ad7af Address:a7075132-ba8a-d443-a1f5-6963bc4ad7af}]" -2025-01-28T14:48:53.888Z [INFO] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: added peer, starting replication: peer=server-a7075132-ba8a-d443-a1f5-6963bc4ad7af -2025-01-28T14:48:53.888Z [ERROR] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: failed to appendEntries to: peer="{Nonvoter server-a7075132-ba8a-d443-a1f5-6963bc4ad7af a7075132-ba8a-d443-a1f5-6963bc4ad7af}" error="failed to connect to peer: a7075132-ba8a-d443-a1f5-6963bc4ad7af" -2025-01-28T14:48:53.900Z [ERROR] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: peer has newer term, stopping replication: peer="{Voter server-a7075132-ba8a-d443-a1f5-6963bc4ad7af a7075132-ba8a-d443-a1f5-6963bc4ad7af}" -2025-01-28T14:48:53.900Z [INFO] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: entering follower state: follower="Node at 6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0 [Follower]" leader-address= leader-id= -2025-01-28T14:48:53.900Z [INFO] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: aborting pipeline replication: peer="{Voter server-329b4e6c-bc90-2795-bd62-ace23e50051f 329b4e6c-bc90-2795-bd62-ace23e50051f}" -2025-01-28T14:48:53.906Z [ERROR] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: peer has newer term, stopping replication: peer="{Nonvoter server-a7075132-ba8a-d443-a1f5-6963bc4ad7af a7075132-ba8a-d443-a1f5-6963bc4ad7af}" -2025-01-28T14:48:53.923Z [WARN] server-a7075132-ba8a-d443-a1f5-6963bc4ad7af: Election timeout reached, restarting election -2025-01-28T14:48:53.923Z [INFO] server-a7075132-ba8a-d443-a1f5-6963bc4ad7af: entering candidate state: node="Node at a7075132-ba8a-d443-a1f5-6963bc4ad7af [Candidate]" term=23 -2025-01-28T14:48:53.923Z [WARN] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: rejecting vote request since node is not a voter: from=a7075132-ba8a-d443-a1f5-6963bc4ad7af -2025-01-28T14:48:53.924Z [WARN] server-329b4e6c-bc90-2795-bd62-ace23e50051f: rejecting vote request since we have a leader: from=a7075132-ba8a-d443-a1f5-6963bc4ad7af leader=6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0 leader-id=server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0 -2025-01-28T14:48:53.973Z [WARN] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:53.973Z [INFO] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: entering candidate state: node="Node at 6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0 [Candidate]" term=24 -2025-01-28T14:48:53.974Z [INFO] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: election won: term=24 tally=2 -2025-01-28T14:48:53.974Z [INFO] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: entering leader state: leader="Node at 6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0 [Leader]" -2025-01-28T14:48:53.974Z [INFO] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: added peer, starting replication: peer=server-329b4e6c-bc90-2795-bd62-ace23e50051f -2025-01-28T14:48:53.974Z [INFO] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: added peer, starting replication: peer=server-a7075132-ba8a-d443-a1f5-6963bc4ad7af -2025-01-28T14:48:53.975Z [WARN] server-a7075132-ba8a-d443-a1f5-6963bc4ad7af: failed to get previous log: previous-index=4 last-index=2 error="log not found" -2025-01-28T14:48:53.975Z [INFO] server-a7075132-ba8a-d443-a1f5-6963bc4ad7af: entering follower state: follower="Node at a7075132-ba8a-d443-a1f5-6963bc4ad7af [Follower]" leader-address=6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0 leader-id=server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0 -2025-01-28T14:48:53.975Z [INFO] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: pipelining replication: peer="{Voter server-329b4e6c-bc90-2795-bd62-ace23e50051f 329b4e6c-bc90-2795-bd62-ace23e50051f}" -2025-01-28T14:48:53.975Z [WARN] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: appendEntries rejected, sending older logs: peer="{Nonvoter server-a7075132-ba8a-d443-a1f5-6963bc4ad7af a7075132-ba8a-d443-a1f5-6963bc4ad7af}" next=3 -2025-01-28T14:48:53.976Z [INFO] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: pipelining replication: peer="{Nonvoter server-a7075132-ba8a-d443-a1f5-6963bc4ad7af a7075132-ba8a-d443-a1f5-6963bc4ad7af}" -2025-01-28T14:48:54.034Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:54.144Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:54.037902009 +0000 UTC m=+117.121307366 (1 nodes), 3.61544ms from start of poll, 534.631544ms from cluster start. Timeout at 2025-01-28 14:48:54.142919675 +0000 UTC m=+117.226324700, 105.017334ms after stability -2025-01-28T14:48:54.268Z [INFO] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: updating configuration: command=RemoveServer server-id=server-a7075132-ba8a-d443-a1f5-6963bc4ad7af server-addr= servers="[{Suffrage:Voter ID:server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0 Address:6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0} {Suffrage:Voter ID:server-329b4e6c-bc90-2795-bd62-ace23e50051f Address:329b4e6c-bc90-2795-bd62-ace23e50051f}]" -2025-01-28T14:48:54.269Z [INFO] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: removed peer, stopping replication: peer=server-a7075132-ba8a-d443-a1f5-6963bc4ad7af last-index=106 -2025-01-28T14:48:54.269Z [INFO] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: aborting pipeline replication: peer="{Nonvoter server-a7075132-ba8a-d443-a1f5-6963bc4ad7af a7075132-ba8a-d443-a1f5-6963bc4ad7af}" -2025-01-28T14:48:54.270Z [INFO] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: updating configuration: command=AddVoter server-id=server-a7075132-ba8a-d443-a1f5-6963bc4ad7af server-addr=a7075132-ba8a-d443-a1f5-6963bc4ad7af servers="[{Suffrage:Voter ID:server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0 Address:6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0} {Suffrage:Voter ID:server-329b4e6c-bc90-2795-bd62-ace23e50051f Address:329b4e6c-bc90-2795-bd62-ace23e50051f} {Suffrage:Voter ID:server-a7075132-ba8a-d443-a1f5-6963bc4ad7af Address:a7075132-ba8a-d443-a1f5-6963bc4ad7af}]" -2025-01-28T14:48:54.270Z [INFO] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: added peer, starting replication: peer=server-a7075132-ba8a-d443-a1f5-6963bc4ad7af -2025-01-28T14:48:54.270Z [WARN] server-a7075132-ba8a-d443-a1f5-6963bc4ad7af: failed to get previous log: previous-index=107 last-index=106 error="log not found" -2025-01-28T14:48:54.271Z [WARN] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: appendEntries rejected, sending older logs: peer="{Voter server-a7075132-ba8a-d443-a1f5-6963bc4ad7af a7075132-ba8a-d443-a1f5-6963bc4ad7af}" next=107 -2025-01-28T14:48:54.271Z [INFO] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: pipelining replication: peer="{Voter server-a7075132-ba8a-d443-a1f5-6963bc4ad7af a7075132-ba8a-d443-a1f5-6963bc4ad7af}" -2025-01-28T14:48:55.480Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:55.586Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:55.481142205 +0000 UTC m=+118.564546896 (1 nodes), 6µs from start of poll, 1.977871074s from cluster start. Timeout at 2025-01-28 14:48:55.586518216 +0000 UTC m=+118.669923240, 105.376344ms after stability -2025-01-28T14:48:55.587Z [INFO] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: aborting pipeline replication: peer="{Voter server-a7075132-ba8a-d443-a1f5-6963bc4ad7af a7075132-ba8a-d443-a1f5-6963bc4ad7af}" -2025-01-28T14:48:55.587Z [INFO] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: aborting pipeline replication: peer="{Voter server-329b4e6c-bc90-2795-bd62-ace23e50051f 329b4e6c-bc90-2795-bd62-ace23e50051f}" -2025-01-28T14:48:56.087Z [ERROR] server-6544fdc7-bb64-0c48-3ee6-dcb05fc5dca0: failed to heartbeat to: peer=329b4e6c-bc90-2795-bd62-ace23e50051f backoff time=10ms error="command timed out" ---- PASS: TestRaft_ClusterCanRegainStability_WhenNonVoterWithHigherTermJoin (2.60s) +2025-01-28T14:55:49.911Z [INFO] server-c6d0317c-047e-3058-733b-aa9010a436ca: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c6d0317c-047e-3058-733b-aa9010a436ca Address:c6d0317c-047e-3058-733b-aa9010a436ca} {Suffrage:Voter ID:server-dbec255f-f4af-0920-107a-83477fbee278 Address:dbec255f-f4af-0920-107a-83477fbee278} {Suffrage:Voter ID:server-acfb0190-f267-5721-2143-bca1045c67d5 Address:acfb0190-f267-5721-2143-bca1045c67d5}]" +2025-01-28T14:55:49.912Z [INFO] server-c6d0317c-047e-3058-733b-aa9010a436ca: entering follower state: follower="Node at c6d0317c-047e-3058-733b-aa9010a436ca [Follower]" leader-address= leader-id= +2025-01-28T14:55:49.912Z [INFO] server-dbec255f-f4af-0920-107a-83477fbee278: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c6d0317c-047e-3058-733b-aa9010a436ca Address:c6d0317c-047e-3058-733b-aa9010a436ca} {Suffrage:Voter ID:server-dbec255f-f4af-0920-107a-83477fbee278 Address:dbec255f-f4af-0920-107a-83477fbee278} {Suffrage:Voter ID:server-acfb0190-f267-5721-2143-bca1045c67d5 Address:acfb0190-f267-5721-2143-bca1045c67d5}]" +2025-01-28T14:55:49.912Z [INFO] server-dbec255f-f4af-0920-107a-83477fbee278: entering follower state: follower="Node at dbec255f-f4af-0920-107a-83477fbee278 [Follower]" leader-address= leader-id= +2025-01-28T14:55:49.912Z [INFO] server-acfb0190-f267-5721-2143-bca1045c67d5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c6d0317c-047e-3058-733b-aa9010a436ca Address:c6d0317c-047e-3058-733b-aa9010a436ca} {Suffrage:Voter ID:server-dbec255f-f4af-0920-107a-83477fbee278 Address:dbec255f-f4af-0920-107a-83477fbee278} {Suffrage:Voter ID:server-acfb0190-f267-5721-2143-bca1045c67d5 Address:acfb0190-f267-5721-2143-bca1045c67d5}]" +2025-01-28T14:55:49.912Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:49.912Z [INFO] server-acfb0190-f267-5721-2143-bca1045c67d5: entering follower state: follower="Node at acfb0190-f267-5721-2143-bca1045c67d5 [Follower]" leader-address= leader-id= +2025-01-28T14:55:49.974Z [WARN] server-c6d0317c-047e-3058-733b-aa9010a436ca: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:49.974Z [INFO] server-c6d0317c-047e-3058-733b-aa9010a436ca: entering candidate state: node="Node at c6d0317c-047e-3058-733b-aa9010a436ca [Candidate]" term=2 +2025-01-28T14:55:49.974Z [INFO] server-c6d0317c-047e-3058-733b-aa9010a436ca: election won: term=2 tally=2 +2025-01-28T14:55:49.974Z [INFO] server-c6d0317c-047e-3058-733b-aa9010a436ca: entering leader state: leader="Node at c6d0317c-047e-3058-733b-aa9010a436ca [Leader]" +2025-01-28T14:55:49.974Z [INFO] server-c6d0317c-047e-3058-733b-aa9010a436ca: added peer, starting replication: peer=server-dbec255f-f4af-0920-107a-83477fbee278 +2025-01-28T14:55:49.974Z [INFO] server-c6d0317c-047e-3058-733b-aa9010a436ca: added peer, starting replication: peer=server-acfb0190-f267-5721-2143-bca1045c67d5 +2025-01-28T14:55:49.975Z [INFO] server-c6d0317c-047e-3058-733b-aa9010a436ca: pipelining replication: peer="{Voter server-acfb0190-f267-5721-2143-bca1045c67d5 acfb0190-f267-5721-2143-bca1045c67d5}" +2025-01-28T14:55:49.975Z [INFO] server-c6d0317c-047e-3058-733b-aa9010a436ca: pipelining replication: peer="{Voter server-dbec255f-f4af-0920-107a-83477fbee278 dbec255f-f4af-0920-107a-83477fbee278}" +2025-01-28T14:55:50.080Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:49.975026075 +0000 UTC m=+135.649516095 (1 nodes), 62.389835ms from start of poll, 63.293469ms from cluster start. Timeout at 2025-01-28 14:55:50.080033015 +0000 UTC m=+135.754523055, 105.00696ms after stability +2025-01-28T14:55:50.085Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:55:50.191Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:55:50.085581525 +0000 UTC m=+135.760071505 (2 nodes), 2.959µs from start of poll, 173.848879ms from cluster start. Timeout at 2025-01-28 14:55:50.190765028 +0000 UTC m=+135.865255108, 105.183603ms after stability +2025-01-28T14:55:50.197Z [ERROR] server-c6d0317c-047e-3058-733b-aa9010a436ca: failed to heartbeat to: peer=dbec255f-f4af-0920-107a-83477fbee278 backoff time=10ms error="failed to connect to peer: dbec255f-f4af-0920-107a-83477fbee278" +2025-01-28T14:55:50.198Z [ERROR] server-c6d0317c-047e-3058-733b-aa9010a436ca: failed to pipeline appendEntries: peer="{Voter server-dbec255f-f4af-0920-107a-83477fbee278 dbec255f-f4af-0920-107a-83477fbee278}" error="append pipeline closed" +2025-01-28T14:55:50.198Z [INFO] server-c6d0317c-047e-3058-733b-aa9010a436ca: aborting pipeline replication: peer="{Voter server-dbec255f-f4af-0920-107a-83477fbee278 dbec255f-f4af-0920-107a-83477fbee278}" +2025-01-28T14:55:50.206Z [ERROR] server-c6d0317c-047e-3058-733b-aa9010a436ca: failed to appendEntries to: peer="{Voter server-dbec255f-f4af-0920-107a-83477fbee278 dbec255f-f4af-0920-107a-83477fbee278}" error="failed to connect to peer: dbec255f-f4af-0920-107a-83477fbee278" +2025-01-28T14:55:50.215Z [ERROR] server-c6d0317c-047e-3058-733b-aa9010a436ca: failed to heartbeat to: peer=dbec255f-f4af-0920-107a-83477fbee278 backoff time=10ms error="failed to connect to peer: dbec255f-f4af-0920-107a-83477fbee278" +2025-01-28T14:55:50.227Z [ERROR] server-c6d0317c-047e-3058-733b-aa9010a436ca: failed to appendEntries to: peer="{Voter server-dbec255f-f4af-0920-107a-83477fbee278 dbec255f-f4af-0920-107a-83477fbee278}" error="failed to connect to peer: dbec255f-f4af-0920-107a-83477fbee278" +2025-01-28T14:55:50.234Z [ERROR] server-c6d0317c-047e-3058-733b-aa9010a436ca: failed to heartbeat to: peer=dbec255f-f4af-0920-107a-83477fbee278 backoff time=10ms error="failed to connect to peer: dbec255f-f4af-0920-107a-83477fbee278" +2025-01-28T14:55:50.237Z [WARN] server-c6d0317c-047e-3058-733b-aa9010a436ca: failed to contact: server-id=server-dbec255f-f4af-0920-107a-83477fbee278 time=50.365269ms +2025-01-28T14:55:50.244Z [ERROR] server-c6d0317c-047e-3058-733b-aa9010a436ca: failed to appendEntries to: peer="{Voter server-dbec255f-f4af-0920-107a-83477fbee278 dbec255f-f4af-0920-107a-83477fbee278}" error="failed to connect to peer: dbec255f-f4af-0920-107a-83477fbee278" +2025-01-28T14:55:50.254Z [ERROR] server-c6d0317c-047e-3058-733b-aa9010a436ca: failed to heartbeat to: peer=dbec255f-f4af-0920-107a-83477fbee278 backoff time=20ms error="failed to connect to peer: dbec255f-f4af-0920-107a-83477fbee278" +2025-01-28T14:55:50.273Z [ERROR] server-c6d0317c-047e-3058-733b-aa9010a436ca: failed to appendEntries to: peer="{Voter server-dbec255f-f4af-0920-107a-83477fbee278 dbec255f-f4af-0920-107a-83477fbee278}" error="failed to connect to peer: dbec255f-f4af-0920-107a-83477fbee278" +2025-01-28T14:55:50.282Z [ERROR] server-c6d0317c-047e-3058-733b-aa9010a436ca: failed to heartbeat to: peer=dbec255f-f4af-0920-107a-83477fbee278 backoff time=25ms error="failed to connect to peer: dbec255f-f4af-0920-107a-83477fbee278" +2025-01-28T14:55:50.283Z [WARN] server-dbec255f-f4af-0920-107a-83477fbee278: heartbeat timeout reached, starting election: last-leader-addr=c6d0317c-047e-3058-733b-aa9010a436ca last-leader-id=server-c6d0317c-047e-3058-733b-aa9010a436ca +2025-01-28T14:55:50.283Z [WARN] server-c6d0317c-047e-3058-733b-aa9010a436ca: failed to contact: server-id=server-dbec255f-f4af-0920-107a-83477fbee278 time=96.848564ms +2025-01-28T14:55:50.283Z [INFO] server-dbec255f-f4af-0920-107a-83477fbee278: entering candidate state: node="Node at dbec255f-f4af-0920-107a-83477fbee278 [Candidate]" term=3 +2025-01-28T14:55:50.284Z [ERROR] server-dbec255f-f4af-0920-107a-83477fbee278: failed to make requestVote RPC: target="{Voter server-acfb0190-f267-5721-2143-bca1045c67d5 acfb0190-f267-5721-2143-bca1045c67d5}" error="failed to connect to peer: acfb0190-f267-5721-2143-bca1045c67d5" term=3 +2025-01-28T14:55:50.284Z [ERROR] server-dbec255f-f4af-0920-107a-83477fbee278: failed to make requestVote RPC: target="{Voter server-c6d0317c-047e-3058-733b-aa9010a436ca c6d0317c-047e-3058-733b-aa9010a436ca}" error="failed to connect to peer: c6d0317c-047e-3058-733b-aa9010a436ca" term=3 +2025-01-28T14:55:50.296Z [INFO] server-c6d0317c-047e-3058-733b-aa9010a436ca: updating configuration: command=RemoveServer server-id=server-dbec255f-f4af-0920-107a-83477fbee278 server-addr= servers="[{Suffrage:Voter ID:server-c6d0317c-047e-3058-733b-aa9010a436ca Address:c6d0317c-047e-3058-733b-aa9010a436ca} {Suffrage:Voter ID:server-acfb0190-f267-5721-2143-bca1045c67d5 Address:acfb0190-f267-5721-2143-bca1045c67d5}]" +2025-01-28T14:55:50.297Z [INFO] server-c6d0317c-047e-3058-733b-aa9010a436ca: removed peer, stopping replication: peer=server-dbec255f-f4af-0920-107a-83477fbee278 last-index=3 +2025-01-28T14:55:50.297Z [INFO] server-c6d0317c-047e-3058-733b-aa9010a436ca: updating configuration: command=AddNonvoter server-id=server-dbec255f-f4af-0920-107a-83477fbee278 server-addr=dbec255f-f4af-0920-107a-83477fbee278 servers="[{Suffrage:Voter ID:server-c6d0317c-047e-3058-733b-aa9010a436ca Address:c6d0317c-047e-3058-733b-aa9010a436ca} {Suffrage:Voter ID:server-acfb0190-f267-5721-2143-bca1045c67d5 Address:acfb0190-f267-5721-2143-bca1045c67d5} {Suffrage:Nonvoter ID:server-dbec255f-f4af-0920-107a-83477fbee278 Address:dbec255f-f4af-0920-107a-83477fbee278}]" +2025-01-28T14:55:50.297Z [INFO] server-c6d0317c-047e-3058-733b-aa9010a436ca: added peer, starting replication: peer=server-dbec255f-f4af-0920-107a-83477fbee278 +2025-01-28T14:55:50.297Z [ERROR] server-c6d0317c-047e-3058-733b-aa9010a436ca: failed to appendEntries to: peer="{Nonvoter server-dbec255f-f4af-0920-107a-83477fbee278 dbec255f-f4af-0920-107a-83477fbee278}" error="failed to connect to peer: dbec255f-f4af-0920-107a-83477fbee278" +2025-01-28T14:55:50.319Z [ERROR] server-c6d0317c-047e-3058-733b-aa9010a436ca: peer has newer term, stopping replication: peer="{Nonvoter server-dbec255f-f4af-0920-107a-83477fbee278 dbec255f-f4af-0920-107a-83477fbee278}" +2025-01-28T14:55:50.320Z [INFO] server-c6d0317c-047e-3058-733b-aa9010a436ca: entering follower state: follower="Node at c6d0317c-047e-3058-733b-aa9010a436ca [Follower]" leader-address= leader-id= +2025-01-28T14:55:50.320Z [INFO] server-c6d0317c-047e-3058-733b-aa9010a436ca: aborting pipeline replication: peer="{Voter server-acfb0190-f267-5721-2143-bca1045c67d5 acfb0190-f267-5721-2143-bca1045c67d5}" +2025-01-28T14:55:50.323Z [ERROR] server-c6d0317c-047e-3058-733b-aa9010a436ca: peer has newer term, stopping replication: peer="{Voter server-dbec255f-f4af-0920-107a-83477fbee278 dbec255f-f4af-0920-107a-83477fbee278}" +2025-01-28T14:55:50.347Z [WARN] server-dbec255f-f4af-0920-107a-83477fbee278: Election timeout reached, restarting election +2025-01-28T14:55:50.347Z [INFO] server-dbec255f-f4af-0920-107a-83477fbee278: entering candidate state: node="Node at dbec255f-f4af-0920-107a-83477fbee278 [Candidate]" term=23 +2025-01-28T14:55:50.347Z [WARN] server-acfb0190-f267-5721-2143-bca1045c67d5: rejecting vote request since we have a leader: from=dbec255f-f4af-0920-107a-83477fbee278 leader=c6d0317c-047e-3058-733b-aa9010a436ca leader-id=server-c6d0317c-047e-3058-733b-aa9010a436ca +2025-01-28T14:55:50.347Z [WARN] server-c6d0317c-047e-3058-733b-aa9010a436ca: rejecting vote request since node is not a voter: from=dbec255f-f4af-0920-107a-83477fbee278 +2025-01-28T14:55:50.372Z [WARN] server-acfb0190-f267-5721-2143-bca1045c67d5: heartbeat timeout reached, starting election: last-leader-addr=c6d0317c-047e-3058-733b-aa9010a436ca last-leader-id=server-c6d0317c-047e-3058-733b-aa9010a436ca +2025-01-28T14:55:50.372Z [INFO] server-acfb0190-f267-5721-2143-bca1045c67d5: entering candidate state: node="Node at acfb0190-f267-5721-2143-bca1045c67d5 [Candidate]" term=3 +2025-01-28T14:55:50.372Z [INFO] server-acfb0190-f267-5721-2143-bca1045c67d5: entering follower state: follower="Node at acfb0190-f267-5721-2143-bca1045c67d5 [Follower]" leader-address= leader-id= +2025-01-28T14:55:50.394Z [WARN] server-c6d0317c-047e-3058-733b-aa9010a436ca: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:50.394Z [INFO] server-c6d0317c-047e-3058-733b-aa9010a436ca: entering candidate state: node="Node at c6d0317c-047e-3058-733b-aa9010a436ca [Candidate]" term=24 +2025-01-28T14:55:50.395Z [INFO] server-c6d0317c-047e-3058-733b-aa9010a436ca: election won: term=24 tally=2 +2025-01-28T14:55:50.395Z [INFO] server-c6d0317c-047e-3058-733b-aa9010a436ca: entering leader state: leader="Node at c6d0317c-047e-3058-733b-aa9010a436ca [Leader]" +2025-01-28T14:55:50.395Z [INFO] server-c6d0317c-047e-3058-733b-aa9010a436ca: added peer, starting replication: peer=server-acfb0190-f267-5721-2143-bca1045c67d5 +2025-01-28T14:55:50.395Z [INFO] server-c6d0317c-047e-3058-733b-aa9010a436ca: added peer, starting replication: peer=server-dbec255f-f4af-0920-107a-83477fbee278 +2025-01-28T14:55:50.395Z [WARN] server-dbec255f-f4af-0920-107a-83477fbee278: failed to get previous log: previous-index=4 last-index=2 error="log not found" +2025-01-28T14:55:50.395Z [INFO] server-dbec255f-f4af-0920-107a-83477fbee278: entering follower state: follower="Node at dbec255f-f4af-0920-107a-83477fbee278 [Follower]" leader-address=c6d0317c-047e-3058-733b-aa9010a436ca leader-id=server-c6d0317c-047e-3058-733b-aa9010a436ca +2025-01-28T14:55:50.395Z [INFO] server-c6d0317c-047e-3058-733b-aa9010a436ca: pipelining replication: peer="{Voter server-acfb0190-f267-5721-2143-bca1045c67d5 acfb0190-f267-5721-2143-bca1045c67d5}" +2025-01-28T14:55:50.395Z [WARN] server-c6d0317c-047e-3058-733b-aa9010a436ca: appendEntries rejected, sending older logs: peer="{Nonvoter server-dbec255f-f4af-0920-107a-83477fbee278 dbec255f-f4af-0920-107a-83477fbee278}" next=3 +2025-01-28T14:55:50.395Z [INFO] server-c6d0317c-047e-3058-733b-aa9010a436ca: pipelining replication: peer="{Nonvoter server-dbec255f-f4af-0920-107a-83477fbee278 dbec255f-f4af-0920-107a-83477fbee278}" +2025-01-28T14:55:50.404Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:50.509Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:50.404208848 +0000 UTC m=+136.078698828 (1 nodes), 146.907µs from start of poll, 492.476202ms from cluster start. Timeout at 2025-01-28 14:55:50.509421268 +0000 UTC m=+136.183911248, 105.21242ms after stability +2025-01-28T14:55:50.549Z [INFO] server-c6d0317c-047e-3058-733b-aa9010a436ca: updating configuration: command=RemoveServer server-id=server-dbec255f-f4af-0920-107a-83477fbee278 server-addr= servers="[{Suffrage:Voter ID:server-c6d0317c-047e-3058-733b-aa9010a436ca Address:c6d0317c-047e-3058-733b-aa9010a436ca} {Suffrage:Voter ID:server-acfb0190-f267-5721-2143-bca1045c67d5 Address:acfb0190-f267-5721-2143-bca1045c67d5}]" +2025-01-28T14:55:50.549Z [INFO] server-c6d0317c-047e-3058-733b-aa9010a436ca: removed peer, stopping replication: peer=server-dbec255f-f4af-0920-107a-83477fbee278 last-index=106 +2025-01-28T14:55:50.549Z [INFO] server-c6d0317c-047e-3058-733b-aa9010a436ca: aborting pipeline replication: peer="{Nonvoter server-dbec255f-f4af-0920-107a-83477fbee278 dbec255f-f4af-0920-107a-83477fbee278}" +2025-01-28T14:55:50.550Z [INFO] server-c6d0317c-047e-3058-733b-aa9010a436ca: updating configuration: command=AddVoter server-id=server-dbec255f-f4af-0920-107a-83477fbee278 server-addr=dbec255f-f4af-0920-107a-83477fbee278 servers="[{Suffrage:Voter ID:server-c6d0317c-047e-3058-733b-aa9010a436ca Address:c6d0317c-047e-3058-733b-aa9010a436ca} {Suffrage:Voter ID:server-acfb0190-f267-5721-2143-bca1045c67d5 Address:acfb0190-f267-5721-2143-bca1045c67d5} {Suffrage:Voter ID:server-dbec255f-f4af-0920-107a-83477fbee278 Address:dbec255f-f4af-0920-107a-83477fbee278}]" +2025-01-28T14:55:50.551Z [INFO] server-c6d0317c-047e-3058-733b-aa9010a436ca: added peer, starting replication: peer=server-dbec255f-f4af-0920-107a-83477fbee278 +2025-01-28T14:55:50.552Z [WARN] server-dbec255f-f4af-0920-107a-83477fbee278: failed to get previous log: previous-index=107 last-index=106 error="log not found" +2025-01-28T14:55:50.554Z [WARN] server-c6d0317c-047e-3058-733b-aa9010a436ca: appendEntries rejected, sending older logs: peer="{Voter server-dbec255f-f4af-0920-107a-83477fbee278 dbec255f-f4af-0920-107a-83477fbee278}" next=107 +2025-01-28T14:55:50.556Z [INFO] server-c6d0317c-047e-3058-733b-aa9010a436ca: pipelining replication: peer="{Voter server-dbec255f-f4af-0920-107a-83477fbee278 dbec255f-f4af-0920-107a-83477fbee278}" +2025-01-28T14:55:51.651Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:51.758Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:51.65164901 +0000 UTC m=+137.326138990 (1 nodes), 2.36µs from start of poll, 1.739916364s from cluster start. Timeout at 2025-01-28 14:55:51.756866529 +0000 UTC m=+137.431356549, 105.217559ms after stability +2025-01-28T14:55:51.758Z [INFO] server-c6d0317c-047e-3058-733b-aa9010a436ca: aborting pipeline replication: peer="{Voter server-dbec255f-f4af-0920-107a-83477fbee278 dbec255f-f4af-0920-107a-83477fbee278}" +2025-01-28T14:55:51.758Z [INFO] server-c6d0317c-047e-3058-733b-aa9010a436ca: aborting pipeline replication: peer="{Voter server-acfb0190-f267-5721-2143-bca1045c67d5 acfb0190-f267-5721-2143-bca1045c67d5}" +--- PASS: TestRaft_ClusterCanRegainStability_WhenNonVoterWithHigherTermJoin (1.85s) === RUN TestRaft_FollowerRemovalNoElection -2025-01-28T14:48:56.099Z [INFO] server-1422e03a-af23-f87e-e595-2d8e1c9f2b0f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1422e03a-af23-f87e-e595-2d8e1c9f2b0f Address:1422e03a-af23-f87e-e595-2d8e1c9f2b0f} {Suffrage:Voter ID:server-7c127b04-fbe8-a3b1-b64d-38f4c7d74e30 Address:7c127b04-fbe8-a3b1-b64d-38f4c7d74e30} {Suffrage:Voter ID:server-c7c160c6-db20-0af7-e998-5c658a7b2bef Address:c7c160c6-db20-0af7-e998-5c658a7b2bef}]" -2025-01-28T14:48:56.099Z [INFO] server-1422e03a-af23-f87e-e595-2d8e1c9f2b0f: entering follower state: follower="Node at 1422e03a-af23-f87e-e595-2d8e1c9f2b0f [Follower]" leader-address= leader-id= -2025-01-28T14:48:56.100Z [INFO] server-7c127b04-fbe8-a3b1-b64d-38f4c7d74e30: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1422e03a-af23-f87e-e595-2d8e1c9f2b0f Address:1422e03a-af23-f87e-e595-2d8e1c9f2b0f} {Suffrage:Voter ID:server-7c127b04-fbe8-a3b1-b64d-38f4c7d74e30 Address:7c127b04-fbe8-a3b1-b64d-38f4c7d74e30} {Suffrage:Voter ID:server-c7c160c6-db20-0af7-e998-5c658a7b2bef Address:c7c160c6-db20-0af7-e998-5c658a7b2bef}]" -2025-01-28T14:48:56.100Z [INFO] server-7c127b04-fbe8-a3b1-b64d-38f4c7d74e30: entering follower state: follower="Node at 7c127b04-fbe8-a3b1-b64d-38f4c7d74e30 [Follower]" leader-address= leader-id= -2025-01-28T14:48:56.100Z [INFO] server-c7c160c6-db20-0af7-e998-5c658a7b2bef: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1422e03a-af23-f87e-e595-2d8e1c9f2b0f Address:1422e03a-af23-f87e-e595-2d8e1c9f2b0f} {Suffrage:Voter ID:server-7c127b04-fbe8-a3b1-b64d-38f4c7d74e30 Address:7c127b04-fbe8-a3b1-b64d-38f4c7d74e30} {Suffrage:Voter ID:server-c7c160c6-db20-0af7-e998-5c658a7b2bef Address:c7c160c6-db20-0af7-e998-5c658a7b2bef}]" -2025-01-28T14:48:56.100Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:56.100Z [INFO] server-c7c160c6-db20-0af7-e998-5c658a7b2bef: entering follower state: follower="Node at c7c160c6-db20-0af7-e998-5c658a7b2bef [Follower]" leader-address= leader-id= -2025-01-28T14:48:56.243Z [WARN] server-c7c160c6-db20-0af7-e998-5c658a7b2bef: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:56.243Z [INFO] server-c7c160c6-db20-0af7-e998-5c658a7b2bef: entering candidate state: node="Node at c7c160c6-db20-0af7-e998-5c658a7b2bef [Candidate]" term=2 -2025-01-28T14:48:56.244Z [INFO] server-c7c160c6-db20-0af7-e998-5c658a7b2bef: election won: term=2 tally=2 -2025-01-28T14:48:56.245Z [INFO] server-c7c160c6-db20-0af7-e998-5c658a7b2bef: entering leader state: leader="Node at c7c160c6-db20-0af7-e998-5c658a7b2bef [Leader]" -2025-01-28T14:48:56.245Z [INFO] server-c7c160c6-db20-0af7-e998-5c658a7b2bef: added peer, starting replication: peer=server-1422e03a-af23-f87e-e595-2d8e1c9f2b0f -2025-01-28T14:48:56.245Z [INFO] server-c7c160c6-db20-0af7-e998-5c658a7b2bef: added peer, starting replication: peer=server-7c127b04-fbe8-a3b1-b64d-38f4c7d74e30 -2025-01-28T14:48:56.246Z [INFO] server-c7c160c6-db20-0af7-e998-5c658a7b2bef: pipelining replication: peer="{Voter server-1422e03a-af23-f87e-e595-2d8e1c9f2b0f 1422e03a-af23-f87e-e595-2d8e1c9f2b0f}" -2025-01-28T14:48:56.246Z [INFO] server-c7c160c6-db20-0af7-e998-5c658a7b2bef: pipelining replication: peer="{Voter server-7c127b04-fbe8-a3b1-b64d-38f4c7d74e30 7c127b04-fbe8-a3b1-b64d-38f4c7d74e30}" -2025-01-28T14:48:56.451Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:56.245377569 +0000 UTC m=+119.328782593 (1 nodes), 144.437623ms from start of poll, 146.726691ms from cluster start. Timeout at 2025-01-28 14:48:56.450580575 +0000 UTC m=+119.533985599, 205.203006ms after stability -2025-01-28T14:48:56.452Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:56.659Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:56.452209712 +0000 UTC m=+119.535614403 (1 nodes), 7.334µs from start of poll, 353.558501ms from cluster start. Timeout at 2025-01-28 14:48:56.657607056 +0000 UTC m=+119.741012081, 205.397678ms after stability -2025-01-28T14:48:56.664Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:48:56.870Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:48:56.664853694 +0000 UTC m=+119.748258385 (2 nodes), 6.334µs from start of poll, 566.202483ms from cluster start. Timeout at 2025-01-28 14:48:56.870266039 +0000 UTC m=+119.953670730, 205.412345ms after stability -2025-01-28T14:48:56.870Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:57.076Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:56.870728141 +0000 UTC m=+119.954132832 (1 nodes), 5.333µs from start of poll, 772.07693ms from cluster start. Timeout at 2025-01-28 14:48:57.076183822 +0000 UTC m=+120.159589179, 205.456347ms after stability - raft_test.go:2961: [INFO] restarting Node at 1422e03a-af23-f87e-e595-2d8e1c9f2b0f [Follower] -2025-01-28T14:48:57.077Z [INFO] server-1422e03a-af23-f87e-e595-2d8e1c9f2b0f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1422e03a-af23-f87e-e595-2d8e1c9f2b0f Address:1422e03a-af23-f87e-e595-2d8e1c9f2b0f} {Suffrage:Voter ID:server-7c127b04-fbe8-a3b1-b64d-38f4c7d74e30 Address:7c127b04-fbe8-a3b1-b64d-38f4c7d74e30} {Suffrage:Voter ID:server-c7c160c6-db20-0af7-e998-5c658a7b2bef Address:c7c160c6-db20-0af7-e998-5c658a7b2bef}]" -2025-01-28T14:48:57.077Z [INFO] server-1422e03a-af23-f87e-e595-2d8e1c9f2b0f: entering follower state: follower="Node at 1422e03a-af23-f87e-e595-2d8e1c9f2b0f [Follower]" leader-address= leader-id= -2025-01-28T14:48:57.344Z [INFO] server-c7c160c6-db20-0af7-e998-5c658a7b2bef: aborting pipeline replication: peer="{Voter server-7c127b04-fbe8-a3b1-b64d-38f4c7d74e30 7c127b04-fbe8-a3b1-b64d-38f4c7d74e30}" -2025-01-28T14:48:57.726Z [ERROR] server-c7c160c6-db20-0af7-e998-5c658a7b2bef: failed to pipeline appendEntries: peer="{Voter server-1422e03a-af23-f87e-e595-2d8e1c9f2b0f 1422e03a-af23-f87e-e595-2d8e1c9f2b0f}" error="command enqueue timeout" -2025-01-28T14:48:57.726Z [INFO] server-c7c160c6-db20-0af7-e998-5c658a7b2bef: aborting pipeline replication: peer="{Voter server-1422e03a-af23-f87e-e595-2d8e1c9f2b0f 1422e03a-af23-f87e-e595-2d8e1c9f2b0f}" -2025-01-28T14:48:57.843Z [ERROR] server-c7c160c6-db20-0af7-e998-5c658a7b2bef: failed to heartbeat to: peer=7c127b04-fbe8-a3b1-b64d-38f4c7d74e30 backoff time=10ms error="command timed out" -2025-01-28T14:48:57.845Z [ERROR] server-c7c160c6-db20-0af7-e998-5c658a7b2bef: failed to heartbeat to: peer=1422e03a-af23-f87e-e595-2d8e1c9f2b0f backoff time=10ms error="command timed out" ---- PASS: TestRaft_FollowerRemovalNoElection (1.76s) +2025-01-28T14:55:51.762Z [INFO] server-1916c62d-8327-2bb8-b754-91600e9a3b40: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1916c62d-8327-2bb8-b754-91600e9a3b40 Address:1916c62d-8327-2bb8-b754-91600e9a3b40} {Suffrage:Voter ID:server-a06d4b15-6ea8-86ad-16af-df0261e2bfc2 Address:a06d4b15-6ea8-86ad-16af-df0261e2bfc2} {Suffrage:Voter ID:server-c9b312d1-6b96-ac50-ba4d-5d8bff2356d0 Address:c9b312d1-6b96-ac50-ba4d-5d8bff2356d0}]" +2025-01-28T14:55:51.763Z [INFO] server-1916c62d-8327-2bb8-b754-91600e9a3b40: entering follower state: follower="Node at 1916c62d-8327-2bb8-b754-91600e9a3b40 [Follower]" leader-address= leader-id= +2025-01-28T14:55:51.763Z [INFO] server-a06d4b15-6ea8-86ad-16af-df0261e2bfc2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1916c62d-8327-2bb8-b754-91600e9a3b40 Address:1916c62d-8327-2bb8-b754-91600e9a3b40} {Suffrage:Voter ID:server-a06d4b15-6ea8-86ad-16af-df0261e2bfc2 Address:a06d4b15-6ea8-86ad-16af-df0261e2bfc2} {Suffrage:Voter ID:server-c9b312d1-6b96-ac50-ba4d-5d8bff2356d0 Address:c9b312d1-6b96-ac50-ba4d-5d8bff2356d0}]" +2025-01-28T14:55:51.763Z [INFO] server-a06d4b15-6ea8-86ad-16af-df0261e2bfc2: entering follower state: follower="Node at a06d4b15-6ea8-86ad-16af-df0261e2bfc2 [Follower]" leader-address= leader-id= +2025-01-28T14:55:51.763Z [INFO] server-c9b312d1-6b96-ac50-ba4d-5d8bff2356d0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1916c62d-8327-2bb8-b754-91600e9a3b40 Address:1916c62d-8327-2bb8-b754-91600e9a3b40} {Suffrage:Voter ID:server-a06d4b15-6ea8-86ad-16af-df0261e2bfc2 Address:a06d4b15-6ea8-86ad-16af-df0261e2bfc2} {Suffrage:Voter ID:server-c9b312d1-6b96-ac50-ba4d-5d8bff2356d0 Address:c9b312d1-6b96-ac50-ba4d-5d8bff2356d0}]" +2025-01-28T14:55:51.763Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:51.763Z [INFO] server-c9b312d1-6b96-ac50-ba4d-5d8bff2356d0: entering follower state: follower="Node at c9b312d1-6b96-ac50-ba4d-5d8bff2356d0 [Follower]" leader-address= leader-id= +2025-01-28T14:55:51.889Z [WARN] server-c9b312d1-6b96-ac50-ba4d-5d8bff2356d0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:51.889Z [INFO] server-c9b312d1-6b96-ac50-ba4d-5d8bff2356d0: entering candidate state: node="Node at c9b312d1-6b96-ac50-ba4d-5d8bff2356d0 [Candidate]" term=2 +2025-01-28T14:55:51.890Z [INFO] server-c9b312d1-6b96-ac50-ba4d-5d8bff2356d0: election won: term=2 tally=2 +2025-01-28T14:55:51.890Z [INFO] server-c9b312d1-6b96-ac50-ba4d-5d8bff2356d0: entering leader state: leader="Node at c9b312d1-6b96-ac50-ba4d-5d8bff2356d0 [Leader]" +2025-01-28T14:55:51.890Z [INFO] server-c9b312d1-6b96-ac50-ba4d-5d8bff2356d0: added peer, starting replication: peer=server-1916c62d-8327-2bb8-b754-91600e9a3b40 +2025-01-28T14:55:51.890Z [INFO] server-c9b312d1-6b96-ac50-ba4d-5d8bff2356d0: added peer, starting replication: peer=server-a06d4b15-6ea8-86ad-16af-df0261e2bfc2 +2025-01-28T14:55:51.890Z [INFO] server-c9b312d1-6b96-ac50-ba4d-5d8bff2356d0: pipelining replication: peer="{Voter server-1916c62d-8327-2bb8-b754-91600e9a3b40 1916c62d-8327-2bb8-b754-91600e9a3b40}" +2025-01-28T14:55:51.890Z [INFO] server-c9b312d1-6b96-ac50-ba4d-5d8bff2356d0: pipelining replication: peer="{Voter server-a06d4b15-6ea8-86ad-16af-df0261e2bfc2 a06d4b15-6ea8-86ad-16af-df0261e2bfc2}" +2025-01-28T14:55:52.096Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:51.890235523 +0000 UTC m=+137.564725503 (1 nodes), 126.613996ms from start of poll, 127.683775ms from cluster start. Timeout at 2025-01-28 14:55:52.095239522 +0000 UTC m=+137.769729542, 205.004039ms after stability +2025-01-28T14:55:52.096Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:52.302Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:52.09664893 +0000 UTC m=+137.771138890 (1 nodes), 2.119µs from start of poll, 334.097162ms from cluster start. Timeout at 2025-01-28 14:55:52.301913825 +0000 UTC m=+137.976403805, 205.264915ms after stability +2025-01-28T14:55:52.307Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:55:52.513Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:55:52.307877764 +0000 UTC m=+137.982367744 (2 nodes), 2.86µs from start of poll, 545.326016ms from cluster start. Timeout at 2025-01-28 14:55:52.512982894 +0000 UTC m=+138.187472934, 205.10519ms after stability +2025-01-28T14:55:52.513Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:52.719Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:52.513834835 +0000 UTC m=+138.188324835 (1 nodes), 2.08µs from start of poll, 751.283107ms from cluster start. Timeout at 2025-01-28 14:55:52.7189997 +0000 UTC m=+138.393489680, 205.164845ms after stability + raft_test.go:2961: [INFO] restarting Node at 1916c62d-8327-2bb8-b754-91600e9a3b40 [Follower] +2025-01-28T14:55:52.720Z [INFO] server-1916c62d-8327-2bb8-b754-91600e9a3b40: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1916c62d-8327-2bb8-b754-91600e9a3b40 Address:1916c62d-8327-2bb8-b754-91600e9a3b40} {Suffrage:Voter ID:server-a06d4b15-6ea8-86ad-16af-df0261e2bfc2 Address:a06d4b15-6ea8-86ad-16af-df0261e2bfc2} {Suffrage:Voter ID:server-c9b312d1-6b96-ac50-ba4d-5d8bff2356d0 Address:c9b312d1-6b96-ac50-ba4d-5d8bff2356d0}]" +2025-01-28T14:55:52.720Z [INFO] server-1916c62d-8327-2bb8-b754-91600e9a3b40: entering follower state: follower="Node at 1916c62d-8327-2bb8-b754-91600e9a3b40 [Follower]" leader-address= leader-id= +2025-01-28T14:55:52.971Z [INFO] server-c9b312d1-6b96-ac50-ba4d-5d8bff2356d0: aborting pipeline replication: peer="{Voter server-a06d4b15-6ea8-86ad-16af-df0261e2bfc2 a06d4b15-6ea8-86ad-16af-df0261e2bfc2}" +2025-01-28T14:55:53.349Z [ERROR] server-c9b312d1-6b96-ac50-ba4d-5d8bff2356d0: failed to pipeline appendEntries: peer="{Voter server-1916c62d-8327-2bb8-b754-91600e9a3b40 1916c62d-8327-2bb8-b754-91600e9a3b40}" error="command enqueue timeout" +2025-01-28T14:55:53.349Z [INFO] server-c9b312d1-6b96-ac50-ba4d-5d8bff2356d0: aborting pipeline replication: peer="{Voter server-1916c62d-8327-2bb8-b754-91600e9a3b40 1916c62d-8327-2bb8-b754-91600e9a3b40}" +2025-01-28T14:55:53.483Z [ERROR] server-c9b312d1-6b96-ac50-ba4d-5d8bff2356d0: failed to heartbeat to: peer=1916c62d-8327-2bb8-b754-91600e9a3b40 backoff time=10ms error="command timed out" +--- PASS: TestRaft_FollowerRemovalNoElection (1.72s) === RUN TestRaft_VoteWithNoIDNoAddr -2025-01-28T14:48:57.855Z [INFO] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb Address:1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb} {Suffrage:Voter ID:server-25f685d3-7df0-63c4-baa1-426fd42a906a Address:25f685d3-7df0-63c4-baa1-426fd42a906a} {Suffrage:Voter ID:server-a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b Address:a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b}]" -2025-01-28T14:48:57.856Z [INFO] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: entering follower state: follower="Node at 1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb [Follower]" leader-address= leader-id= -2025-01-28T14:48:57.856Z [INFO] server-25f685d3-7df0-63c4-baa1-426fd42a906a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb Address:1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb} {Suffrage:Voter ID:server-25f685d3-7df0-63c4-baa1-426fd42a906a Address:25f685d3-7df0-63c4-baa1-426fd42a906a} {Suffrage:Voter ID:server-a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b Address:a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b}]" -2025-01-28T14:48:57.857Z [INFO] server-25f685d3-7df0-63c4-baa1-426fd42a906a: entering follower state: follower="Node at 25f685d3-7df0-63c4-baa1-426fd42a906a [Follower]" leader-address= leader-id= -2025-01-28T14:48:57.857Z [INFO] server-a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb Address:1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb} {Suffrage:Voter ID:server-25f685d3-7df0-63c4-baa1-426fd42a906a Address:25f685d3-7df0-63c4-baa1-426fd42a906a} {Suffrage:Voter ID:server-a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b Address:a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b}]" -2025-01-28T14:48:57.857Z [INFO] server-a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b: entering follower state: follower="Node at a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b [Follower]" leader-address= leader-id= -2025-01-28T14:48:57.858Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:57.908Z [WARN] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-01-28T14:48:57.908Z [INFO] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: entering candidate state: node="Node at 1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb [Candidate]" term=2 -2025-01-28T14:48:57.910Z [INFO] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: election won: term=2 tally=2 -2025-01-28T14:48:57.910Z [INFO] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: entering leader state: leader="Node at 1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb [Leader]" -2025-01-28T14:48:57.910Z [INFO] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: added peer, starting replication: peer=server-25f685d3-7df0-63c4-baa1-426fd42a906a -2025-01-28T14:48:57.910Z [INFO] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: added peer, starting replication: peer=server-a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b -2025-01-28T14:48:57.910Z [INFO] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: pipelining replication: peer="{Voter server-25f685d3-7df0-63c4-baa1-426fd42a906a 25f685d3-7df0-63c4-baa1-426fd42a906a}" -2025-01-28T14:48:57.911Z [INFO] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: pipelining replication: peer="{Voter server-a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b}" -2025-01-28T14:48:58.015Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:57.910282015 +0000 UTC m=+120.993686707 (1 nodes), 51.984211ms from start of poll, 55.228974ms from cluster start. Timeout at 2025-01-28 14:48:58.015291015 +0000 UTC m=+121.098695706, 105.008999ms after stability -2025-01-28T14:48:58.015Z [INFO] cluster: starting stability test: raft-state=Leader -2025-01-28T14:48:58.122Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:48:58.015713813 +0000 UTC m=+121.099118504 (1 nodes), 6.333µs from start of poll, 160.660771ms from cluster start. Timeout at 2025-01-28 14:48:58.121138826 +0000 UTC m=+121.204544184, 105.42568ms after stability -2025-01-28T14:48:58.127Z [INFO] cluster: starting stability test: raft-state=Follower -2025-01-28T14:48:58.234Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:48:58.128024816 +0000 UTC m=+121.211429840 (2 nodes), 8.334µs from start of poll, 272.972107ms from cluster start. Timeout at 2025-01-28 14:48:58.233462494 +0000 UTC m=+121.316867519, 105.437679ms after stability -2025-01-28T14:48:58.235Z [ERROR] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: failed to heartbeat to: peer=a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b backoff time=10ms error="failed to connect to peer: a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b" -2025-01-28T14:48:58.235Z [ERROR] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: failed to heartbeat to: peer=25f685d3-7df0-63c4-baa1-426fd42a906a backoff time=10ms error="failed to connect to peer: 25f685d3-7df0-63c4-baa1-426fd42a906a" -2025-01-28T14:48:58.239Z [ERROR] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: failed to pipeline appendEntries: peer="{Voter server-a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b}" error="append pipeline closed" -2025-01-28T14:48:58.239Z [ERROR] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: failed to pipeline appendEntries: peer="{Voter server-25f685d3-7df0-63c4-baa1-426fd42a906a 25f685d3-7df0-63c4-baa1-426fd42a906a}" error="append pipeline closed" -2025-01-28T14:48:58.240Z [INFO] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: aborting pipeline replication: peer="{Voter server-25f685d3-7df0-63c4-baa1-426fd42a906a 25f685d3-7df0-63c4-baa1-426fd42a906a}" -2025-01-28T14:48:58.239Z [INFO] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: aborting pipeline replication: peer="{Voter server-a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b}" -2025-01-28T14:48:58.246Z [ERROR] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: failed to appendEntries to: peer="{Voter server-a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b}" error="failed to connect to peer: a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b" -2025-01-28T14:48:58.247Z [ERROR] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: failed to appendEntries to: peer="{Voter server-25f685d3-7df0-63c4-baa1-426fd42a906a 25f685d3-7df0-63c4-baa1-426fd42a906a}" error="failed to connect to peer: 25f685d3-7df0-63c4-baa1-426fd42a906a" -2025-01-28T14:48:58.253Z [ERROR] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: failed to heartbeat to: peer=a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b backoff time=10ms error="failed to connect to peer: a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b" -2025-01-28T14:48:58.255Z [ERROR] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: failed to heartbeat to: peer=25f685d3-7df0-63c4-baa1-426fd42a906a backoff time=10ms error="failed to connect to peer: 25f685d3-7df0-63c4-baa1-426fd42a906a" -2025-01-28T14:48:58.265Z [ERROR] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: failed to appendEntries to: peer="{Voter server-25f685d3-7df0-63c4-baa1-426fd42a906a 25f685d3-7df0-63c4-baa1-426fd42a906a}" error="failed to connect to peer: 25f685d3-7df0-63c4-baa1-426fd42a906a" -2025-01-28T14:48:58.267Z [ERROR] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: failed to appendEntries to: peer="{Voter server-a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b}" error="failed to connect to peer: a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b" -2025-01-28T14:48:58.273Z [ERROR] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: failed to heartbeat to: peer=a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b backoff time=10ms error="failed to connect to peer: a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b" -2025-01-28T14:48:58.273Z [ERROR] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: failed to heartbeat to: peer=25f685d3-7df0-63c4-baa1-426fd42a906a backoff time=10ms error="failed to connect to peer: 25f685d3-7df0-63c4-baa1-426fd42a906a" -2025-01-28T14:48:58.281Z [WARN] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: failed to contact: server-id=server-a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b time=50.596836ms -2025-01-28T14:48:58.282Z [WARN] server-25f685d3-7df0-63c4-baa1-426fd42a906a: heartbeat timeout reached, starting election: last-leader-addr=1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb last-leader-id=server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb -2025-01-28T14:48:58.282Z [INFO] server-25f685d3-7df0-63c4-baa1-426fd42a906a: entering candidate state: node="Node at 25f685d3-7df0-63c4-baa1-426fd42a906a [Candidate]" term=3 -2025-01-28T14:48:58.283Z [WARN] server-a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b: rejecting vote request since we have a leader: from=25f685d3-7df0-63c4-baa1-426fd42a906a leader=1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb leader-id=server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb -2025-01-28T14:48:58.283Z [ERROR] server-25f685d3-7df0-63c4-baa1-426fd42a906a: failed to make requestVote RPC: target="{Voter server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb 1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb}" error="failed to connect to peer: 1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb" term=3 -2025-01-28T14:48:58.283Z [ERROR] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: failed to appendEntries to: peer="{Voter server-a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b}" error="failed to connect to peer: a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b" -2025-01-28T14:48:58.284Z [ERROR] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: failed to appendEntries to: peer="{Voter server-25f685d3-7df0-63c4-baa1-426fd42a906a 25f685d3-7df0-63c4-baa1-426fd42a906a}" error="failed to connect to peer: 25f685d3-7df0-63c4-baa1-426fd42a906a" -2025-01-28T14:48:58.291Z [WARN] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: failed to contact: server-id=server-25f685d3-7df0-63c4-baa1-426fd42a906a time=59.326762ms -2025-01-28T14:48:58.291Z [WARN] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: failed to contact: server-id=server-a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b time=60.889809ms -2025-01-28T14:48:58.291Z [WARN] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: failed to contact quorum of nodes, stepping down -2025-01-28T14:48:58.292Z [INFO] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: entering follower state: follower="Node at 1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb [Follower]" leader-address= leader-id= -2025-01-28T14:48:58.291Z [ERROR] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: failed to heartbeat to: peer=25f685d3-7df0-63c4-baa1-426fd42a906a backoff time=20ms error="failed to connect to peer: 25f685d3-7df0-63c4-baa1-426fd42a906a" -2025-01-28T14:48:58.293Z [ERROR] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: failed to heartbeat to: peer=a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b backoff time=20ms error="failed to connect to peer: a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b" -2025-01-28T14:48:58.297Z [WARN] server-a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b: heartbeat timeout reached, starting election: last-leader-addr=1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb last-leader-id=server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb -2025-01-28T14:48:58.298Z [INFO] server-a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b: entering candidate state: node="Node at a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b [Candidate]" term=3 -2025-01-28T14:48:58.298Z [ERROR] server-a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b: failed to make requestVote RPC: target="{Voter server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb 1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb}" error="failed to connect to peer: 1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb" term=3 -2025-01-28T14:48:58.298Z [INFO] server-25f685d3-7df0-63c4-baa1-426fd42a906a: duplicate requestVote for same term: term=3 -2025-01-28T14:48:58.311Z [ERROR] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: failed to appendEntries to: peer="{Voter server-a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b}" error="failed to connect to peer: a34baadf-8ed7-0dfd-6ba1-a1ebeca5aa7b" -2025-01-28T14:48:58.311Z [ERROR] server-1b49f7fa-dc5d-c142-fa88-f33ab1c3d9eb: failed to appendEntries to: peer="{Voter server-25f685d3-7df0-63c4-baa1-426fd42a906a 25f685d3-7df0-63c4-baa1-426fd42a906a}" error="failed to connect to peer: 25f685d3-7df0-63c4-baa1-426fd42a906a" -2025-01-28T14:48:58.339Z [INFO] server-25f685d3-7df0-63c4-baa1-426fd42a906a: entering follower state: follower="Node at 25f685d3-7df0-63c4-baa1-426fd42a906a [Follower]" leader-address= leader-id= +2025-01-28T14:55:53.486Z [INFO] server-3f95db3c-8f17-8b74-4c5c-2a59bde16fe2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3f95db3c-8f17-8b74-4c5c-2a59bde16fe2 Address:3f95db3c-8f17-8b74-4c5c-2a59bde16fe2} {Suffrage:Voter ID:server-1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a Address:1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a} {Suffrage:Voter ID:server-71e5a17f-d253-b322-3d77-b5e98015a4b0 Address:71e5a17f-d253-b322-3d77-b5e98015a4b0}]" +2025-01-28T14:55:53.487Z [INFO] server-1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3f95db3c-8f17-8b74-4c5c-2a59bde16fe2 Address:3f95db3c-8f17-8b74-4c5c-2a59bde16fe2} {Suffrage:Voter ID:server-1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a Address:1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a} {Suffrage:Voter ID:server-71e5a17f-d253-b322-3d77-b5e98015a4b0 Address:71e5a17f-d253-b322-3d77-b5e98015a4b0}]" +2025-01-28T14:55:53.487Z [INFO] server-3f95db3c-8f17-8b74-4c5c-2a59bde16fe2: entering follower state: follower="Node at 3f95db3c-8f17-8b74-4c5c-2a59bde16fe2 [Follower]" leader-address= leader-id= +2025-01-28T14:55:53.487Z [INFO] server-1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a: entering follower state: follower="Node at 1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a [Follower]" leader-address= leader-id= +2025-01-28T14:55:53.487Z [INFO] server-71e5a17f-d253-b322-3d77-b5e98015a4b0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3f95db3c-8f17-8b74-4c5c-2a59bde16fe2 Address:3f95db3c-8f17-8b74-4c5c-2a59bde16fe2} {Suffrage:Voter ID:server-1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a Address:1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a} {Suffrage:Voter ID:server-71e5a17f-d253-b322-3d77-b5e98015a4b0 Address:71e5a17f-d253-b322-3d77-b5e98015a4b0}]" +2025-01-28T14:55:53.487Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:53.487Z [INFO] server-71e5a17f-d253-b322-3d77-b5e98015a4b0: entering follower state: follower="Node at 71e5a17f-d253-b322-3d77-b5e98015a4b0 [Follower]" leader-address= leader-id= +2025-01-28T14:55:53.540Z [WARN] server-71e5a17f-d253-b322-3d77-b5e98015a4b0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:53.540Z [INFO] server-71e5a17f-d253-b322-3d77-b5e98015a4b0: entering candidate state: node="Node at 71e5a17f-d253-b322-3d77-b5e98015a4b0 [Candidate]" term=2 +2025-01-28T14:55:53.540Z [INFO] server-71e5a17f-d253-b322-3d77-b5e98015a4b0: election won: term=2 tally=2 +2025-01-28T14:55:53.540Z [INFO] server-71e5a17f-d253-b322-3d77-b5e98015a4b0: entering leader state: leader="Node at 71e5a17f-d253-b322-3d77-b5e98015a4b0 [Leader]" +2025-01-28T14:55:53.540Z [INFO] server-71e5a17f-d253-b322-3d77-b5e98015a4b0: added peer, starting replication: peer=server-3f95db3c-8f17-8b74-4c5c-2a59bde16fe2 +2025-01-28T14:55:53.540Z [INFO] server-71e5a17f-d253-b322-3d77-b5e98015a4b0: added peer, starting replication: peer=server-1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a +2025-01-28T14:55:53.541Z [INFO] server-71e5a17f-d253-b322-3d77-b5e98015a4b0: pipelining replication: peer="{Voter server-1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a 1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a}" +2025-01-28T14:55:53.541Z [INFO] server-71e5a17f-d253-b322-3d77-b5e98015a4b0: pipelining replication: peer="{Voter server-3f95db3c-8f17-8b74-4c5c-2a59bde16fe2 3f95db3c-8f17-8b74-4c5c-2a59bde16fe2}" +2025-01-28T14:55:53.647Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:53.540989478 +0000 UTC m=+139.215479498 (1 nodes), 53.198101ms from start of poll, 54.321076ms from cluster start. Timeout at 2025-01-28 14:55:53.645995038 +0000 UTC m=+139.320485198, 105.0057ms after stability +2025-01-28T14:55:53.647Z [INFO] cluster: starting stability test: raft-state=Leader +2025-01-28T14:55:53.752Z [INFO] cluster: stable state for Leader reached at 2025-01-28 14:55:53.647179541 +0000 UTC m=+139.321669541 (1 nodes), 2.32µs from start of poll, 160.511119ms from cluster start. Timeout at 2025-01-28 14:55:53.752367963 +0000 UTC m=+139.426858003, 105.188462ms after stability +2025-01-28T14:55:53.757Z [INFO] cluster: starting stability test: raft-state=Follower +2025-01-28T14:55:53.863Z [INFO] cluster: stable state for Follower reached at 2025-01-28 14:55:53.757755689 +0000 UTC m=+139.432245669 (2 nodes), 3.659µs from start of poll, 271.087247ms from cluster start. Timeout at 2025-01-28 14:55:53.862931413 +0000 UTC m=+139.537421413, 105.175744ms after stability +2025-01-28T14:55:53.863Z [ERROR] server-71e5a17f-d253-b322-3d77-b5e98015a4b0: failed to pipeline appendEntries: peer="{Voter server-3f95db3c-8f17-8b74-4c5c-2a59bde16fe2 3f95db3c-8f17-8b74-4c5c-2a59bde16fe2}" error="append pipeline closed" +2025-01-28T14:55:53.863Z [INFO] server-71e5a17f-d253-b322-3d77-b5e98015a4b0: aborting pipeline replication: peer="{Voter server-3f95db3c-8f17-8b74-4c5c-2a59bde16fe2 3f95db3c-8f17-8b74-4c5c-2a59bde16fe2}" +2025-01-28T14:55:53.863Z [ERROR] server-71e5a17f-d253-b322-3d77-b5e98015a4b0: failed to heartbeat to: peer=1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a backoff time=10ms error="failed to connect to peer: 1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a" +2025-01-28T14:55:53.866Z [ERROR] server-71e5a17f-d253-b322-3d77-b5e98015a4b0: failed to heartbeat to: peer=3f95db3c-8f17-8b74-4c5c-2a59bde16fe2 backoff time=10ms error="failed to connect to peer: 3f95db3c-8f17-8b74-4c5c-2a59bde16fe2" +2025-01-28T14:55:53.870Z [ERROR] server-71e5a17f-d253-b322-3d77-b5e98015a4b0: failed to pipeline appendEntries: peer="{Voter server-1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a 1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a}" error="append pipeline closed" +2025-01-28T14:55:53.870Z [INFO] server-71e5a17f-d253-b322-3d77-b5e98015a4b0: aborting pipeline replication: peer="{Voter server-1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a 1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a}" +2025-01-28T14:55:53.870Z [ERROR] server-71e5a17f-d253-b322-3d77-b5e98015a4b0: failed to appendEntries to: peer="{Voter server-3f95db3c-8f17-8b74-4c5c-2a59bde16fe2 3f95db3c-8f17-8b74-4c5c-2a59bde16fe2}" error="failed to connect to peer: 3f95db3c-8f17-8b74-4c5c-2a59bde16fe2" +2025-01-28T14:55:53.879Z [ERROR] server-71e5a17f-d253-b322-3d77-b5e98015a4b0: failed to appendEntries to: peer="{Voter server-1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a 1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a}" error="failed to connect to peer: 1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a" +2025-01-28T14:55:53.885Z [ERROR] server-71e5a17f-d253-b322-3d77-b5e98015a4b0: failed to heartbeat to: peer=1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a backoff time=10ms error="failed to connect to peer: 1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a" +2025-01-28T14:55:53.886Z [ERROR] server-71e5a17f-d253-b322-3d77-b5e98015a4b0: failed to heartbeat to: peer=3f95db3c-8f17-8b74-4c5c-2a59bde16fe2 backoff time=10ms error="failed to connect to peer: 3f95db3c-8f17-8b74-4c5c-2a59bde16fe2" +2025-01-28T14:55:53.889Z [ERROR] server-71e5a17f-d253-b322-3d77-b5e98015a4b0: failed to appendEntries to: peer="{Voter server-3f95db3c-8f17-8b74-4c5c-2a59bde16fe2 3f95db3c-8f17-8b74-4c5c-2a59bde16fe2}" error="failed to connect to peer: 3f95db3c-8f17-8b74-4c5c-2a59bde16fe2" +2025-01-28T14:55:53.896Z [ERROR] server-71e5a17f-d253-b322-3d77-b5e98015a4b0: failed to appendEntries to: peer="{Voter server-1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a 1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a}" error="failed to connect to peer: 1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a" +2025-01-28T14:55:53.902Z [ERROR] server-71e5a17f-d253-b322-3d77-b5e98015a4b0: failed to heartbeat to: peer=1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a backoff time=10ms error="failed to connect to peer: 1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a" +2025-01-28T14:55:53.905Z [ERROR] server-71e5a17f-d253-b322-3d77-b5e98015a4b0: failed to appendEntries to: peer="{Voter server-3f95db3c-8f17-8b74-4c5c-2a59bde16fe2 3f95db3c-8f17-8b74-4c5c-2a59bde16fe2}" error="failed to connect to peer: 3f95db3c-8f17-8b74-4c5c-2a59bde16fe2" +2025-01-28T14:55:53.906Z [ERROR] server-71e5a17f-d253-b322-3d77-b5e98015a4b0: failed to heartbeat to: peer=3f95db3c-8f17-8b74-4c5c-2a59bde16fe2 backoff time=10ms error="failed to connect to peer: 3f95db3c-8f17-8b74-4c5c-2a59bde16fe2" +2025-01-28T14:55:53.910Z [WARN] server-71e5a17f-d253-b322-3d77-b5e98015a4b0: failed to contact: server-id=server-3f95db3c-8f17-8b74-4c5c-2a59bde16fe2 time=50.746872ms +2025-01-28T14:55:53.910Z [WARN] server-71e5a17f-d253-b322-3d77-b5e98015a4b0: failed to contact: server-id=server-1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a time=50.432002ms +2025-01-28T14:55:53.910Z [WARN] server-71e5a17f-d253-b322-3d77-b5e98015a4b0: failed to contact quorum of nodes, stepping down +2025-01-28T14:55:53.910Z [INFO] server-71e5a17f-d253-b322-3d77-b5e98015a4b0: entering follower state: follower="Node at 71e5a17f-d253-b322-3d77-b5e98015a4b0 [Follower]" leader-address= leader-id= +2025-01-28T14:55:53.914Z [ERROR] server-71e5a17f-d253-b322-3d77-b5e98015a4b0: failed to appendEntries to: peer="{Voter server-1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a 1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a}" error="failed to connect to peer: 1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a" +2025-01-28T14:55:53.940Z [WARN] server-1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a: heartbeat timeout reached, starting election: last-leader-addr=71e5a17f-d253-b322-3d77-b5e98015a4b0 last-leader-id=server-71e5a17f-d253-b322-3d77-b5e98015a4b0 +2025-01-28T14:55:53.941Z [INFO] server-1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a: entering candidate state: node="Node at 1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a [Candidate]" term=3 +2025-01-28T14:55:53.941Z [ERROR] server-1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a: failed to make requestVote RPC: target="{Voter server-71e5a17f-d253-b322-3d77-b5e98015a4b0 71e5a17f-d253-b322-3d77-b5e98015a4b0}" error="failed to connect to peer: 71e5a17f-d253-b322-3d77-b5e98015a4b0" term=3 +2025-01-28T14:55:53.941Z [WARN] server-3f95db3c-8f17-8b74-4c5c-2a59bde16fe2: rejecting vote request since we have a leader: from=1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a leader=71e5a17f-d253-b322-3d77-b5e98015a4b0 leader-id=server-71e5a17f-d253-b322-3d77-b5e98015a4b0 +2025-01-28T14:55:53.946Z [WARN] server-3f95db3c-8f17-8b74-4c5c-2a59bde16fe2: heartbeat timeout reached, starting election: last-leader-addr=71e5a17f-d253-b322-3d77-b5e98015a4b0 last-leader-id=server-71e5a17f-d253-b322-3d77-b5e98015a4b0 +2025-01-28T14:55:53.946Z [INFO] server-3f95db3c-8f17-8b74-4c5c-2a59bde16fe2: entering candidate state: node="Node at 3f95db3c-8f17-8b74-4c5c-2a59bde16fe2 [Candidate]" term=3 +2025-01-28T14:55:53.946Z [ERROR] server-3f95db3c-8f17-8b74-4c5c-2a59bde16fe2: failed to make requestVote RPC: target="{Voter server-71e5a17f-d253-b322-3d77-b5e98015a4b0 71e5a17f-d253-b322-3d77-b5e98015a4b0}" error="failed to connect to peer: 71e5a17f-d253-b322-3d77-b5e98015a4b0" term=3 +2025-01-28T14:55:53.946Z [INFO] server-1111ca03-b3b9-82b4-1d43-82fc4cb0fc8a: duplicate requestVote for same term: term=3 +2025-01-28T14:55:53.968Z [INFO] server-3f95db3c-8f17-8b74-4c5c-2a59bde16fe2: entering follower state: follower="Node at 3f95db3c-8f17-8b74-4c5c-2a59bde16fe2 [Follower]" leader-address= leader-id= --- PASS: TestRaft_VoteWithNoIDNoAddr (0.49s) === RUN TestRaft_runFollower_State_Transition === RUN TestRaft_runFollower_State_Transition/NonVoter -2025-01-28T14:48:58.346Z [INFO] 127.0.0.1:46455 :: starting node: addr=127.0.0.1:46455 -2025-01-28T14:48:58.347Z [INFO] 127.0.0.1:46455 :: initial configuration: index=0 servers=[] -2025-01-28T14:48:58.347Z [INFO] 127.0.0.1:46455 :: entering follower state: follower="Node at 127.0.0.1:46455 [Follower]" leader-address= leader-id= -2025-01-28T14:48:58.408Z [WARN] 127.0.0.1:46455 :: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +2025-01-28T14:55:53.972Z [INFO] 127.0.0.1:33367 :: starting node: addr=127.0.0.1:33367 +2025-01-28T14:55:53.972Z [INFO] 127.0.0.1:33367 :: initial configuration: index=0 servers=[] +2025-01-28T14:55:53.972Z [INFO] 127.0.0.1:33367 :: entering follower state: follower="Node at 127.0.0.1:33367 [Follower]" leader-address= leader-id= +2025-01-28T14:55:54.066Z [WARN] 127.0.0.1:33367 :: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election === RUN TestRaft_runFollower_State_Transition/Voter -2025-01-28T14:48:58.500Z [INFO] 127.0.0.1:39569 :: starting node: addr=127.0.0.1:39569 -2025-01-28T14:48:58.500Z [INFO] 127.0.0.1:39569 :: initial configuration: index=0 servers=[] -2025-01-28T14:48:58.500Z [INFO] 127.0.0.1:39569 :: entering follower state: follower="Node at 127.0.0.1:39569 [Follower]" leader-address= leader-id= -2025-01-28T14:48:58.573Z [WARN] 127.0.0.1:39569 :: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:54.124Z [INFO] 127.0.0.1:43511 :: starting node: addr=127.0.0.1:43511 +2025-01-28T14:55:54.124Z [INFO] 127.0.0.1:43511 :: initial configuration: index=0 servers=[] +2025-01-28T14:55:54.124Z [INFO] 127.0.0.1:43511 :: entering follower state: follower="Node at 127.0.0.1:43511 [Follower]" leader-address= leader-id= +2025-01-28T14:55:54.189Z [WARN] 127.0.0.1:43511 :: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= === RUN TestRaft_runFollower_State_Transition/Not_in_Config -2025-01-28T14:48:58.653Z [INFO] 127.0.0.1:42957 :: starting node: addr=127.0.0.1:42957 -2025-01-28T14:48:58.654Z [INFO] 127.0.0.1:42957 :: initial configuration: index=0 servers=[] -2025-01-28T14:48:58.654Z [INFO] 127.0.0.1:42957 :: entering follower state: follower="Node at 127.0.0.1:42957 [Follower]" leader-address= leader-id= -2025-01-28T14:48:58.714Z [WARN] 127.0.0.1:42957 :: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +2025-01-28T14:55:54.276Z [INFO] 127.0.0.1:44733 :: starting node: addr=127.0.0.1:44733 +2025-01-28T14:55:54.276Z [INFO] 127.0.0.1:44733 :: initial configuration: index=0 servers=[] +2025-01-28T14:55:54.276Z [INFO] 127.0.0.1:44733 :: entering follower state: follower="Node at 127.0.0.1:44733 [Follower]" leader-address= leader-id= +2025-01-28T14:55:54.340Z [WARN] 127.0.0.1:44733 :: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election --- PASS: TestRaft_runFollower_State_Transition (0.46s) --- PASS: TestRaft_runFollower_State_Transition/NonVoter (0.15s) --- PASS: TestRaft_runFollower_State_Transition/Voter (0.15s) --- PASS: TestRaft_runFollower_State_Transition/Not_in_Config (0.15s) === RUN TestRaft_runFollower_ReloadTimeoutConfigs -2025-01-28T14:48:58.807Z [INFO] 127.0.0.1:40327 :: starting node: addr=127.0.0.1:40327 -2025-01-28T14:48:58.807Z [INFO] 127.0.0.1:40327 :: initial configuration: index=0 servers=[] -2025-01-28T14:48:58.807Z [INFO] 127.0.0.1:40327 :: entering follower state: follower="Node at 127.0.0.1:40327 [Follower]" leader-address= leader-id= -2025-01-28T14:48:58.808Z [WARN] 127.0.0.1:40327 :: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2025-01-28T14:55:54.428Z [INFO] 127.0.0.1:45233 :: starting node: addr=127.0.0.1:45233 +2025-01-28T14:55:54.428Z [INFO] 127.0.0.1:45233 :: initial configuration: index=0 servers=[] +2025-01-28T14:55:54.428Z [INFO] 127.0.0.1:45233 :: entering follower state: follower="Node at 127.0.0.1:45233 [Follower]" leader-address= leader-id= +2025-01-28T14:55:54.428Z [WARN] 127.0.0.1:45233 :: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= --- PASS: TestRaft_runFollower_ReloadTimeoutConfigs (0.15s) === RUN TestSaturationMetric === RUN TestSaturationMetric/without_smoothing @@ -4278,13 +4262,13 @@ === RUN TestMax --- PASS: TestMax (0.00s) === RUN TestGenerateUUID ---- PASS: TestGenerateUUID (0.03s) +--- PASS: TestGenerateUUID (0.00s) === RUN TestBackoff --- PASS: TestBackoff (0.00s) === RUN TestOverrideNotifyBool --- PASS: TestOverrideNotifyBool (0.00s) PASS -ok github.com/hashicorp/raft 122.124s +ok github.com/hashicorp/raft 140.280s make[1]: Leaving directory '/build/reproducible-path/golang-github-hashicorp-raft-1.5.0' create-stamp debian/debhelper-build-stamp dh_testroot -O--buildsystem=golang @@ -4312,12 +4296,14 @@ dpkg-buildpackage: info: binary-only upload (no source included) dpkg-genchanges: info: including full source code in upload I: copying local configuration +I: user script /srv/workspace/pbuilder/31365/tmp/hooks/B01_cleanup starting +I: user script /srv/workspace/pbuilder/31365/tmp/hooks/B01_cleanup finished I: unmounting dev/ptmx filesystem I: unmounting dev/pts filesystem I: unmounting dev/shm filesystem I: unmounting proc filesystem I: unmounting sys filesystem I: cleaning the build env -I: removing directory /srv/workspace/pbuilder/5020 and its subdirectories -I: Current time: Tue Jan 28 02:49:37 -12 2025 -I: pbuilder-time-stamp: 1738075777 +I: removing directory /srv/workspace/pbuilder/31365 and its subdirectories +I: Current time: Wed Jan 29 04:56:10 +14 2025 +I: pbuilder-time-stamp: 1738076170