Diff of the two buildlogs: -- --- b1/build.log 2024-06-11 14:29:58.154520517 +0000 +++ b2/build.log 2024-06-11 14:32:47.991288650 +0000 @@ -1,6 +1,6 @@ I: pbuilder: network access will be disabled during build -I: Current time: Tue Jun 11 02:25:29 -12 2024 -I: pbuilder-time-stamp: 1718115929 +I: Current time: Wed Jun 12 04:30:07 +14 2024 +I: pbuilder-time-stamp: 1718116207 I: Building the build Environment I: extracting base tarball [/var/cache/pbuilder/bookworm-reproducible-base.tgz] I: copying local configuration @@ -16,7 +16,7 @@ I: copying [./golang-github-hashicorp-raft_1.1.2.orig.tar.xz] I: copying [./golang-github-hashicorp-raft_1.1.2-1.debian.tar.xz] I: Extracting source -gpgv: Signature made Sat Jan 18 19:24:28 2020 -12 +gpgv: Signature made Sun Jan 19 21:24:28 2020 +14 gpgv: using RSA key 50BC7CF939D20C272A6B065652B6BBD953968D1B gpgv: Can't check signature: No public key dpkg-source: warning: cannot verify inline signature for ./golang-github-hashicorp-raft_1.1.2-1.dsc: no acceptable signature found @@ -31,135 +31,167 @@ dpkg-source: info: applying t-skip--TestRaft_SnapshotRestore_PeerChange.patch I: using fakeroot in build. I: Installing the build-deps -I: user script /srv/workspace/pbuilder/23716/tmp/hooks/D02_print_environment starting +I: user script /srv/workspace/pbuilder/18637/tmp/hooks/D01_modify_environment starting +debug: Running on virt64z. +I: Changing host+domainname to test build reproducibility +I: Adding a custom variable just for the fun of it... +I: Changing /bin/sh to bash +'/bin/sh' -> '/bin/bash' +lrwxrwxrwx 1 root root 9 Jun 12 04:30 /bin/sh -> /bin/bash +I: Setting pbuilder2's login shell to /bin/bash +I: Setting pbuilder2's GECOS to second user,second room,second work-phone,second home-phone,second other +I: user script /srv/workspace/pbuilder/18637/tmp/hooks/D01_modify_environment finished +I: user script /srv/workspace/pbuilder/18637/tmp/hooks/D02_print_environment starting I: set - BUILDDIR='/build/reproducible-path' - BUILDUSERGECOS='first user,first room,first work-phone,first home-phone,first other' - BUILDUSERNAME='pbuilder1' - BUILD_ARCH='armhf' - DEBIAN_FRONTEND='noninteractive' - DEB_BUILD_OPTIONS='buildinfo=+all reproducible=+all parallel=3 ' - DISTRIBUTION='bookworm' - HOME='/root' - HOST_ARCH='armhf' + BASH=/bin/sh + BASHOPTS=checkwinsize:cmdhist:complete_fullquote:extquote:force_fignore:globasciiranges:globskipdots:hostcomplete:interactive_comments:patsub_replacement:progcomp:promptvars:sourcepath + BASH_ALIASES=() + BASH_ARGC=() + BASH_ARGV=() + BASH_CMDS=() + BASH_LINENO=([0]="12" [1]="0") + BASH_LOADABLES_PATH=/usr/local/lib/bash:/usr/lib/bash:/opt/local/lib/bash:/usr/pkg/lib/bash:/opt/pkg/lib/bash:. + BASH_SOURCE=([0]="/tmp/hooks/D02_print_environment" [1]="/tmp/hooks/D02_print_environment") + BASH_VERSINFO=([0]="5" [1]="2" [2]="15" [3]="1" [4]="release" [5]="arm-unknown-linux-gnueabihf") + BASH_VERSION='5.2.15(1)-release' + BUILDDIR=/build/reproducible-path + BUILDUSERGECOS='second user,second room,second work-phone,second home-phone,second other' + BUILDUSERNAME=pbuilder2 + BUILD_ARCH=armhf + DEBIAN_FRONTEND=noninteractive + DEB_BUILD_OPTIONS='buildinfo=+all reproducible=+all parallel=4 ' + DIRSTACK=() + DISTRIBUTION=bookworm + EUID=0 + FUNCNAME=([0]="Echo" [1]="main") + GROUPS=() + HOME=/root + HOSTNAME=i-capture-the-hostname + HOSTTYPE=arm + HOST_ARCH=armhf IFS=' ' - INVOCATION_ID='a7828d5d0b7b4f5d83b02d0142a77b56' - LANG='C' - LANGUAGE='en_US:en' - LC_ALL='C' - MAIL='/var/mail/root' - OPTIND='1' - PATH='/usr/sbin:/usr/bin:/sbin:/bin:/usr/games' - PBCURRENTCOMMANDLINEOPERATION='build' - PBUILDER_OPERATION='build' - PBUILDER_PKGDATADIR='/usr/share/pbuilder' - PBUILDER_PKGLIBDIR='/usr/lib/pbuilder' - PBUILDER_SYSCONFDIR='/etc' - PPID='23716' - PS1='# ' - PS2='> ' + INVOCATION_ID=13f7f702a825411da853c99fbca43e55 + LANG=C + LANGUAGE=it_CH:it + LC_ALL=C + MACHTYPE=arm-unknown-linux-gnueabihf + MAIL=/var/mail/root + OPTERR=1 + OPTIND=1 + OSTYPE=linux-gnueabihf + PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path + PBCURRENTCOMMANDLINEOPERATION=build + PBUILDER_OPERATION=build + PBUILDER_PKGDATADIR=/usr/share/pbuilder + PBUILDER_PKGLIBDIR=/usr/lib/pbuilder + PBUILDER_SYSCONFDIR=/etc + PIPESTATUS=([0]="0") + POSIXLY_CORRECT=y + PPID=18637 PS4='+ ' - PWD='/' - SHELL='/bin/bash' - SHLVL='2' - SUDO_COMMAND='/usr/bin/timeout -k 18.1h 18h /usr/bin/ionice -c 3 /usr/bin/nice /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/r-b-build.avc5eAbN/pbuilderrc_Eg0l --distribution bookworm --hookdir /etc/pbuilder/first-build-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/bookworm-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/r-b-build.avc5eAbN/b1 --logfile b1/build.log golang-github-hashicorp-raft_1.1.2-1.dsc' - SUDO_GID='113' - SUDO_UID='107' - SUDO_USER='jenkins' - TERM='unknown' - TZ='/usr/share/zoneinfo/Etc/GMT+12' - USER='root' - _='/usr/bin/systemd-run' - http_proxy='http://10.0.0.15:3142/' + PWD=/ + SHELL=/bin/bash + SHELLOPTS=braceexpand:errexit:hashall:interactive-comments:posix + SHLVL=3 + SUDO_COMMAND='/usr/bin/timeout -k 24.1h 24h /usr/bin/ionice -c 3 /usr/bin/nice -n 11 /usr/bin/unshare --uts -- /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/r-b-build.avc5eAbN/pbuilderrc_0Ywj --distribution bookworm --hookdir /etc/pbuilder/rebuild-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/bookworm-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/r-b-build.avc5eAbN/b2 --logfile b2/build.log golang-github-hashicorp-raft_1.1.2-1.dsc' + SUDO_GID=110 + SUDO_UID=103 + SUDO_USER=jenkins + TERM=unknown + TZ=/usr/share/zoneinfo/Etc/GMT-14 + UID=0 + USER=root + _='I: set' + http_proxy=http://10.0.0.15:3142/ I: uname -a - Linux ff4a 6.1.0-21-armmp-lpae #1 SMP Debian 6.1.90-1 (2024-05-03) armv7l GNU/Linux + Linux i-capture-the-hostname 6.1.0-21-arm64 #1 SMP Debian 6.1.90-1 (2024-05-03) aarch64 GNU/Linux I: ls -l /bin total 4964 - -rwxr-xr-x 1 root root 838488 Apr 23 2023 bash - -rwxr-xr-x 3 root root 67144 Sep 18 2022 bunzip2 - -rwxr-xr-x 3 root root 67144 Sep 18 2022 bzcat - lrwxrwxrwx 1 root root 6 Sep 18 2022 bzcmp -> bzdiff - -rwxr-xr-x 1 root root 2225 Sep 18 2022 bzdiff - lrwxrwxrwx 1 root root 6 Sep 18 2022 bzegrep -> bzgrep - -rwxr-xr-x 1 root root 4893 Nov 27 2021 bzexe - lrwxrwxrwx 1 root root 6 Sep 18 2022 bzfgrep -> bzgrep - -rwxr-xr-x 1 root root 3775 Sep 18 2022 bzgrep - -rwxr-xr-x 3 root root 67144 Sep 18 2022 bzip2 - -rwxr-xr-x 1 root root 67112 Sep 18 2022 bzip2recover - lrwxrwxrwx 1 root root 6 Sep 18 2022 bzless -> bzmore - -rwxr-xr-x 1 root root 1297 Sep 18 2022 bzmore - -rwxr-xr-x 1 root root 67632 Sep 20 2022 cat - -rwxr-xr-x 1 root root 67676 Sep 20 2022 chgrp - -rwxr-xr-x 1 root root 67644 Sep 20 2022 chmod - -rwxr-xr-x 1 root root 67684 Sep 20 2022 chown - -rwxr-xr-x 1 root root 133532 Sep 20 2022 cp - -rwxr-xr-x 1 root root 132868 Jan 5 2023 dash - -rwxr-xr-x 1 root root 133220 Sep 20 2022 date - -rwxr-xr-x 1 root root 67732 Sep 20 2022 dd - -rwxr-xr-x 1 root root 68104 Sep 20 2022 df - -rwxr-xr-x 1 root root 133632 Sep 20 2022 dir - -rwxr-xr-x 1 root root 59128 Mar 22 2023 dmesg - lrwxrwxrwx 1 root root 8 Dec 19 2022 dnsdomainname -> hostname - lrwxrwxrwx 1 root root 8 Dec 19 2022 domainname -> hostname - -rwxr-xr-x 1 root root 67560 Sep 20 2022 echo - -rwxr-xr-x 1 root root 41 Jan 24 2023 egrep - -rwxr-xr-x 1 root root 67548 Sep 20 2022 false - -rwxr-xr-x 1 root root 41 Jan 24 2023 fgrep - -rwxr-xr-x 1 root root 55748 Mar 22 2023 findmnt - -rwsr-xr-x 1 root root 26208 Mar 22 2023 fusermount - -rwxr-xr-x 1 root root 128608 Jan 24 2023 grep - -rwxr-xr-x 2 root root 2346 Apr 9 2022 gunzip - -rwxr-xr-x 1 root root 6447 Apr 9 2022 gzexe - -rwxr-xr-x 1 root root 64220 Apr 9 2022 gzip - -rwxr-xr-x 1 root root 67032 Dec 19 2022 hostname - -rwxr-xr-x 1 root root 67720 Sep 20 2022 ln - -rwxr-xr-x 1 root root 35132 Mar 22 2023 login - -rwxr-xr-x 1 root root 133632 Sep 20 2022 ls - -rwxr-xr-x 1 root root 136808 Mar 22 2023 lsblk - -rwxr-xr-x 1 root root 67800 Sep 20 2022 mkdir - -rwxr-xr-x 1 root root 67764 Sep 20 2022 mknod - -rwxr-xr-x 1 root root 67596 Sep 20 2022 mktemp - -rwxr-xr-x 1 root root 38504 Mar 22 2023 more - -rwsr-xr-x 1 root root 38496 Mar 22 2023 mount - -rwxr-xr-x 1 root root 9824 Mar 22 2023 mountpoint - -rwxr-xr-x 1 root root 133532 Sep 20 2022 mv - lrwxrwxrwx 1 root root 8 Dec 19 2022 nisdomainname -> hostname - lrwxrwxrwx 1 root root 14 Apr 2 2023 pidof -> /sbin/killall5 - -rwxr-xr-x 1 root root 67608 Sep 20 2022 pwd - lrwxrwxrwx 1 root root 4 Apr 23 2023 rbash -> bash - -rwxr-xr-x 1 root root 67600 Sep 20 2022 readlink - -rwxr-xr-x 1 root root 67672 Sep 20 2022 rm - -rwxr-xr-x 1 root root 67600 Sep 20 2022 rmdir - -rwxr-xr-x 1 root root 14152 Jul 28 2023 run-parts - -rwxr-xr-x 1 root root 133372 Jan 5 2023 sed - lrwxrwxrwx 1 root root 4 Jan 5 2023 sh -> dash - -rwxr-xr-x 1 root root 67584 Sep 20 2022 sleep - -rwxr-xr-x 1 root root 67644 Sep 20 2022 stty - -rwsr-xr-x 1 root root 50800 Mar 22 2023 su - -rwxr-xr-x 1 root root 67584 Sep 20 2022 sync - -rwxr-xr-x 1 root root 336764 Jan 19 21:27 tar - -rwxr-xr-x 1 root root 9800 Jul 28 2023 tempfile - -rwxr-xr-x 1 root root 133224 Sep 20 2022 touch - -rwxr-xr-x 1 root root 67548 Sep 20 2022 true - -rwxr-xr-x 1 root root 9768 Mar 22 2023 ulockmgr_server - -rwsr-xr-x 1 root root 22108 Mar 22 2023 umount - -rwxr-xr-x 1 root root 67572 Sep 20 2022 uname - -rwxr-xr-x 2 root root 2346 Apr 9 2022 uncompress - -rwxr-xr-x 1 root root 133632 Sep 20 2022 vdir - -rwxr-xr-x 1 root root 42608 Mar 22 2023 wdctl - lrwxrwxrwx 1 root root 8 Dec 19 2022 ypdomainname -> hostname - -rwxr-xr-x 1 root root 1984 Apr 9 2022 zcat - -rwxr-xr-x 1 root root 1678 Apr 9 2022 zcmp - -rwxr-xr-x 1 root root 6460 Apr 9 2022 zdiff - -rwxr-xr-x 1 root root 29 Apr 9 2022 zegrep - -rwxr-xr-x 1 root root 29 Apr 9 2022 zfgrep - -rwxr-xr-x 1 root root 2081 Apr 9 2022 zforce - -rwxr-xr-x 1 root root 8103 Apr 9 2022 zgrep - -rwxr-xr-x 1 root root 2206 Apr 9 2022 zless - -rwxr-xr-x 1 root root 1842 Apr 9 2022 zmore - -rwxr-xr-x 1 root root 4577 Apr 9 2022 znew -I: user script /srv/workspace/pbuilder/23716/tmp/hooks/D02_print_environment finished + -rwxr-xr-x 1 root root 838488 Apr 24 2023 bash + -rwxr-xr-x 3 root root 67144 Sep 19 2022 bunzip2 + -rwxr-xr-x 3 root root 67144 Sep 19 2022 bzcat + lrwxrwxrwx 1 root root 6 Sep 19 2022 bzcmp -> bzdiff + -rwxr-xr-x 1 root root 2225 Sep 19 2022 bzdiff + lrwxrwxrwx 1 root root 6 Sep 19 2022 bzegrep -> bzgrep + -rwxr-xr-x 1 root root 4893 Nov 28 2021 bzexe + lrwxrwxrwx 1 root root 6 Sep 19 2022 bzfgrep -> bzgrep + -rwxr-xr-x 1 root root 3775 Sep 19 2022 bzgrep + -rwxr-xr-x 3 root root 67144 Sep 19 2022 bzip2 + -rwxr-xr-x 1 root root 67112 Sep 19 2022 bzip2recover + lrwxrwxrwx 1 root root 6 Sep 19 2022 bzless -> bzmore + -rwxr-xr-x 1 root root 1297 Sep 19 2022 bzmore + -rwxr-xr-x 1 root root 67632 Sep 21 2022 cat + -rwxr-xr-x 1 root root 67676 Sep 21 2022 chgrp + -rwxr-xr-x 1 root root 67644 Sep 21 2022 chmod + -rwxr-xr-x 1 root root 67684 Sep 21 2022 chown + -rwxr-xr-x 1 root root 133532 Sep 21 2022 cp + -rwxr-xr-x 1 root root 132868 Jan 6 2023 dash + -rwxr-xr-x 1 root root 133220 Sep 21 2022 date + -rwxr-xr-x 1 root root 67732 Sep 21 2022 dd + -rwxr-xr-x 1 root root 68104 Sep 21 2022 df + -rwxr-xr-x 1 root root 133632 Sep 21 2022 dir + -rwxr-xr-x 1 root root 59128 Mar 23 2023 dmesg + lrwxrwxrwx 1 root root 8 Dec 20 2022 dnsdomainname -> hostname + lrwxrwxrwx 1 root root 8 Dec 20 2022 domainname -> hostname + -rwxr-xr-x 1 root root 67560 Sep 21 2022 echo + -rwxr-xr-x 1 root root 41 Jan 25 2023 egrep + -rwxr-xr-x 1 root root 67548 Sep 21 2022 false + -rwxr-xr-x 1 root root 41 Jan 25 2023 fgrep + -rwxr-xr-x 1 root root 55748 Mar 23 2023 findmnt + -rwsr-xr-x 1 root root 26208 Mar 23 2023 fusermount + -rwxr-xr-x 1 root root 128608 Jan 25 2023 grep + -rwxr-xr-x 2 root root 2346 Apr 10 2022 gunzip + -rwxr-xr-x 1 root root 6447 Apr 10 2022 gzexe + -rwxr-xr-x 1 root root 64220 Apr 10 2022 gzip + -rwxr-xr-x 1 root root 67032 Dec 20 2022 hostname + -rwxr-xr-x 1 root root 67720 Sep 21 2022 ln + -rwxr-xr-x 1 root root 35132 Mar 23 2023 login + -rwxr-xr-x 1 root root 133632 Sep 21 2022 ls + -rwxr-xr-x 1 root root 136808 Mar 23 2023 lsblk + -rwxr-xr-x 1 root root 67800 Sep 21 2022 mkdir + -rwxr-xr-x 1 root root 67764 Sep 21 2022 mknod + -rwxr-xr-x 1 root root 67596 Sep 21 2022 mktemp + -rwxr-xr-x 1 root root 38504 Mar 23 2023 more + -rwsr-xr-x 1 root root 38496 Mar 23 2023 mount + -rwxr-xr-x 1 root root 9824 Mar 23 2023 mountpoint + -rwxr-xr-x 1 root root 133532 Sep 21 2022 mv + lrwxrwxrwx 1 root root 8 Dec 20 2022 nisdomainname -> hostname + lrwxrwxrwx 1 root root 14 Apr 3 2023 pidof -> /sbin/killall5 + -rwxr-xr-x 1 root root 67608 Sep 21 2022 pwd + lrwxrwxrwx 1 root root 4 Apr 24 2023 rbash -> bash + -rwxr-xr-x 1 root root 67600 Sep 21 2022 readlink + -rwxr-xr-x 1 root root 67672 Sep 21 2022 rm + -rwxr-xr-x 1 root root 67600 Sep 21 2022 rmdir + -rwxr-xr-x 1 root root 14152 Jul 29 2023 run-parts + -rwxr-xr-x 1 root root 133372 Jan 6 2023 sed + lrwxrwxrwx 1 root root 9 Jun 12 04:30 sh -> /bin/bash + -rwxr-xr-x 1 root root 67584 Sep 21 2022 sleep + -rwxr-xr-x 1 root root 67644 Sep 21 2022 stty + -rwsr-xr-x 1 root root 50800 Mar 23 2023 su + -rwxr-xr-x 1 root root 67584 Sep 21 2022 sync + -rwxr-xr-x 1 root root 336764 Jan 20 23:27 tar + -rwxr-xr-x 1 root root 9800 Jul 29 2023 tempfile + -rwxr-xr-x 1 root root 133224 Sep 21 2022 touch + -rwxr-xr-x 1 root root 67548 Sep 21 2022 true + -rwxr-xr-x 1 root root 9768 Mar 23 2023 ulockmgr_server + -rwsr-xr-x 1 root root 22108 Mar 23 2023 umount + -rwxr-xr-x 1 root root 67572 Sep 21 2022 uname + -rwxr-xr-x 2 root root 2346 Apr 10 2022 uncompress + -rwxr-xr-x 1 root root 133632 Sep 21 2022 vdir + -rwxr-xr-x 1 root root 42608 Mar 23 2023 wdctl + lrwxrwxrwx 1 root root 8 Dec 20 2022 ypdomainname -> hostname + -rwxr-xr-x 1 root root 1984 Apr 10 2022 zcat + -rwxr-xr-x 1 root root 1678 Apr 10 2022 zcmp + -rwxr-xr-x 1 root root 6460 Apr 10 2022 zdiff + -rwxr-xr-x 1 root root 29 Apr 10 2022 zegrep + -rwxr-xr-x 1 root root 29 Apr 10 2022 zfgrep + -rwxr-xr-x 1 root root 2081 Apr 10 2022 zforce + -rwxr-xr-x 1 root root 8103 Apr 10 2022 zgrep + -rwxr-xr-x 1 root root 2206 Apr 10 2022 zless + -rwxr-xr-x 1 root root 1842 Apr 10 2022 zmore + -rwxr-xr-x 1 root root 4577 Apr 10 2022 znew +I: user script /srv/workspace/pbuilder/18637/tmp/hooks/D02_print_environment finished -> Attempting to satisfy build-dependencies -> Creating pbuilder-satisfydepends-dummy package Package: pbuilder-satisfydepends-dummy @@ -303,7 +335,7 @@ Get: 96 http://deb.debian.org/debian bookworm/main armhf golang-gopkg-check.v1-dev all 0.0+git20200902.038fdea-1 [32.1 kB] Get: 97 http://deb.debian.org/debian bookworm/main armhf golang-gopkg-mgo.v2-dev all 2016.08.01-7 [316 kB] Get: 98 http://deb.debian.org/debian bookworm/main armhf golang-github-hashicorp-go-msgpack-dev all 0.5.5-1 [43.3 kB] -Fetched 112 MB in 7s (17.1 MB/s) +Fetched 112 MB in 5s (23.0 MB/s) debconf: delaying package configuration, since apt-utils is not installed Selecting previously unselected package sensible-utils. (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 19288 files and directories currently installed.) @@ -715,7 +747,11 @@ fakeroot is already the newest version (1.31-1.2). 0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded. I: Building the package -I: Running cd /build/reproducible-path/golang-github-hashicorp-raft-1.1.2/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" HOME="/nonexistent/first-build" dpkg-buildpackage -us -uc -b && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" HOME="/nonexistent/first-build" dpkg-genchanges -S > ../golang-github-hashicorp-raft_1.1.2-1_source.changes +I: user script /srv/workspace/pbuilder/18637/tmp/hooks/A99_set_merged_usr starting +Not re-configuring usrmerge for bookworm +I: user script /srv/workspace/pbuilder/18637/tmp/hooks/A99_set_merged_usr finished +hostname: Name or service not known +I: Running cd /build/reproducible-path/golang-github-hashicorp-raft-1.1.2/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" HOME="/nonexistent/second-build" dpkg-buildpackage -us -uc -b && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" HOME="/nonexistent/second-build" dpkg-genchanges -S > ../golang-github-hashicorp-raft_1.1.2-1_source.changes dpkg-buildpackage: info: source package golang-github-hashicorp-raft dpkg-buildpackage: info: source version 1.1.2-1 dpkg-buildpackage: info: source distribution unstable @@ -733,160 +769,160 @@ dh_autoreconf -O--buildsystem=golang dh_auto_configure -O--buildsystem=golang dh_auto_build -O--buildsystem=golang - cd obj-arm-linux-gnueabihf && go install -trimpath -v -p 3 github.com/hashicorp/raft github.com/hashicorp/raft/bench -internal/unsafeheader + cd obj-arm-linux-gnueabihf && go install -trimpath -v -p 4 github.com/hashicorp/raft github.com/hashicorp/raft/bench internal/goarch -internal/cpu internal/goexperiment -internal/abi +internal/unsafeheader internal/goos +internal/cpu runtime/internal/math +internal/abi runtime/internal/sys runtime/internal/syscall -internal/bytealg -runtime/internal/atomic internal/race sync/atomic -unicode unicode/utf8 +unicode +internal/bytealg +runtime/internal/atomic container/list -runtime internal/itoa math/bits -math crypto/internal/subtle crypto/subtle crypto/internal/boring/sig +math encoding unicode/utf16 +runtime internal/nettrace vendor/golang.org/x/crypto/cryptobyte/asn1 vendor/golang.org/x/crypto/internal/subtle internal/reflectlite sync internal/testlog -math/rand internal/singleflight -runtime/cgo +math/rand internal/sysinfo +runtime/cgo errors sort io -bytes -strings internal/oserror syscall -bufio -hash strconv -time +bytes +strings +hash crypto reflect +bufio +time crypto/internal/randutil internal/syscall/unix internal/safefilepath internal/syscall/execenv path github.com/hashicorp/golang-lru/simplelru +vendor/golang.org/x/net/dns/dnsmessage github.com/hashicorp/go-immutable-radix +hash/crc32 context internal/poll io/fs -vendor/golang.org/x/net/dns/dnsmessage -os -hash/crc32 crypto/internal/nistec/fiat -internal/godebug -internal/intern -net/netip +crypto/rc4 +vendor/golang.org/x/text/transform +os encoding/binary -crypto/cipher internal/fmtsort -fmt +net/http/internal/ascii +regexp/syntax +crypto/cipher crypto/internal/boring +fmt encoding/base64 -net +internal/godebug +internal/intern +crypto/aes +net/netip +crypto/des +crypto/sha512 +crypto/internal/edwards25519/field math/big encoding/json log +net compress/flate -crypto/rand -crypto/aes -crypto/des compress/gzip crypto/internal/nistec -crypto/internal/boring/bbig -crypto/sha512 -encoding/asn1 -crypto/elliptic -vendor/golang.org/x/crypto/cryptobyte -crypto/internal/edwards25519/field crypto/internal/edwards25519 -crypto/ecdsa crypto/hmac crypto/md5 -crypto/rc4 -crypto/rsa +crypto/rand +crypto/elliptic +crypto/internal/boring/bbig +encoding/asn1 crypto/ed25519 +crypto/rsa crypto/sha1 crypto/sha256 crypto/dsa +vendor/golang.org/x/crypto/cryptobyte encoding/hex encoding/pem -net/url crypto/x509/pkix +net/url path/filepath +crypto/ecdsa vendor/golang.org/x/crypto/chacha20 vendor/golang.org/x/crypto/internal/poly1305 -crypto/x509 vendor/golang.org/x/crypto/curve25519/internal/field -vendor/golang.org/x/crypto/chacha20poly1305 -vendor/golang.org/x/crypto/curve25519 vendor/golang.org/x/crypto/hkdf -vendor/golang.org/x/text/transform +vendor/golang.org/x/crypto/chacha20poly1305 vendor/golang.org/x/text/unicode/bidi +vendor/golang.org/x/crypto/curve25519 vendor/golang.org/x/text/unicode/norm -vendor/golang.org/x/text/secure/bidirule -net/textproto vendor/golang.org/x/net/http2/hpack -crypto/tls mime -vendor/golang.org/x/net/idna +vendor/golang.org/x/text/secure/bidirule mime/quotedprintable -mime/multipart net/http/internal -net/http/internal/ascii os/signal -vendor/golang.org/x/net/http/httpguts -vendor/golang.org/x/net/http/httpproxy golang.org/x/sys/unix io/ioutil -regexp/syntax -net/http/httptrace regexp -net/http +vendor/golang.org/x/net/idna encoding/gob -github.com/mattn/go-isatty -github.com/mattn/go-colorable -github.com/fatih/color -github.com/hashicorp/go-hclog go/token html text/template/parse +crypto/x509 +net/textproto +vendor/golang.org/x/net/http/httpguts +vendor/golang.org/x/net/http/httpproxy +mime/multipart +github.com/mattn/go-isatty +text/template +github.com/mattn/go-colorable hash/crc64 +github.com/fatih/color flag +github.com/hashicorp/go-hclog +crypto/tls runtime/debug runtime/trace testing -text/template html/template +net/http/httptrace +net/http net/rpc github.com/armon/go-metrics github.com/hashicorp/go-msgpack/codec github.com/hashicorp/raft github.com/hashicorp/raft/bench dh_auto_test -O--buildsystem=golang - cd obj-arm-linux-gnueabihf && go test -vet=off -v -p 3 github.com/hashicorp/raft github.com/hashicorp/raft/bench + cd obj-arm-linux-gnueabihf && go test -vet=off -v -p 4 github.com/hashicorp/raft github.com/hashicorp/raft/bench === RUN TestCommitment_setVoters --- PASS: TestCommitment_setVoters (0.00s) === RUN TestCommitment_match_max @@ -928,33 +964,33 @@ === RUN TestFileSnapshotSinkImpl --- PASS: TestFileSnapshotSinkImpl (0.00s) === RUN TestFileSS_CreateSnapshotMissingParentDir - testing.go:153: 2024-06-11T02:29:09.019-1200 [INFO] creating new snapshot: path=/tmp/raft1693204611/raft49717673/snapshots/3-10-1718116149019.tmp ---- PASS: TestFileSS_CreateSnapshotMissingParentDir (0.05s) + testing.go:153: 2024-06-12T04:32:23.101+1400 [INFO] creating new snapshot: path=/tmp/raft2739472815/raft441183824/snapshots/3-10-1718116343101.tmp +--- PASS: TestFileSS_CreateSnapshotMissingParentDir (0.01s) === RUN TestFileSS_CreateSnapshot - testing.go:153: 2024-06-11T02:29:09.066-1200 [INFO] creating new snapshot: path=/tmp/raft1648071617/snapshots/3-10-1718116149066.tmp - testing.go:153: 2024-06-11T02:29:09.133-1200 [WARN] found temporary snapshot: name=3-10-1718116149066.tmp ---- PASS: TestFileSS_CreateSnapshot (0.19s) + testing.go:153: 2024-06-12T04:32:23.111+1400 [INFO] creating new snapshot: path=/tmp/raft350919335/snapshots/3-10-1718116343111.tmp + testing.go:153: 2024-06-12T04:32:23.115+1400 [WARN] found temporary snapshot: name=3-10-1718116343111.tmp +--- PASS: TestFileSS_CreateSnapshot (0.02s) === RUN TestFileSS_CancelSnapshot - testing.go:153: 2024-06-11T02:29:09.256-1200 [INFO] creating new snapshot: path=/tmp/raft40559523/snapshots/3-10-1718116149256.tmp ---- PASS: TestFileSS_CancelSnapshot (0.06s) + testing.go:153: 2024-06-12T04:32:23.126+1400 [INFO] creating new snapshot: path=/tmp/raft702078745/snapshots/3-10-1718116343126.tmp +--- PASS: TestFileSS_CancelSnapshot (0.01s) === RUN TestFileSS_Retention - testing.go:153: 2024-06-11T02:29:09.312-1200 [INFO] creating new snapshot: path=/tmp/raft2607884383/snapshots/3-10-1718116149312.tmp - testing.go:153: 2024-06-11T02:29:09.429-1200 [INFO] creating new snapshot: path=/tmp/raft2607884383/snapshots/3-11-1718116149429.tmp - testing.go:153: 2024-06-11T02:29:09.652-1200 [INFO] creating new snapshot: path=/tmp/raft2607884383/snapshots/3-12-1718116149652.tmp - testing.go:153: 2024-06-11T02:29:09.674-1200 [INFO] reaping snapshot: path=/tmp/raft2607884383/snapshots/3-10-1718116149312 - testing.go:153: 2024-06-11T02:29:09.674-1200 [INFO] creating new snapshot: path=/tmp/raft2607884383/snapshots/3-13-1718116149674.tmp - testing.go:153: 2024-06-11T02:29:09.697-1200 [INFO] reaping snapshot: path=/tmp/raft2607884383/snapshots/3-11-1718116149429 - testing.go:153: 2024-06-11T02:29:09.698-1200 [INFO] creating new snapshot: path=/tmp/raft2607884383/snapshots/3-14-1718116149698.tmp - testing.go:153: 2024-06-11T02:29:09.722-1200 [INFO] reaping snapshot: path=/tmp/raft2607884383/snapshots/3-12-1718116149652 ---- PASS: TestFileSS_Retention (0.41s) + testing.go:153: 2024-06-12T04:32:23.134+1400 [INFO] creating new snapshot: path=/tmp/raft2677625755/snapshots/3-10-1718116343134.tmp + testing.go:153: 2024-06-12T04:32:23.148+1400 [INFO] creating new snapshot: path=/tmp/raft2677625755/snapshots/3-11-1718116343148.tmp + testing.go:153: 2024-06-12T04:32:23.161+1400 [INFO] creating new snapshot: path=/tmp/raft2677625755/snapshots/3-12-1718116343161.tmp + testing.go:153: 2024-06-12T04:32:23.175+1400 [INFO] reaping snapshot: path=/tmp/raft2677625755/snapshots/3-10-1718116343134 + testing.go:153: 2024-06-12T04:32:23.175+1400 [INFO] creating new snapshot: path=/tmp/raft2677625755/snapshots/3-13-1718116343175.tmp + testing.go:153: 2024-06-12T04:32:23.188+1400 [INFO] reaping snapshot: path=/tmp/raft2677625755/snapshots/3-11-1718116343148 + testing.go:153: 2024-06-12T04:32:23.188+1400 [INFO] creating new snapshot: path=/tmp/raft2677625755/snapshots/3-14-1718116343188.tmp + testing.go:153: 2024-06-12T04:32:23.202+1400 [INFO] reaping snapshot: path=/tmp/raft2677625755/snapshots/3-12-1718116343161 +--- PASS: TestFileSS_Retention (0.07s) === RUN TestFileSS_BadPerm --- PASS: TestFileSS_BadPerm (0.00s) === RUN TestFileSS_MissingParentDir --- PASS: TestFileSS_MissingParentDir (0.00s) === RUN TestFileSS_Ordering - testing.go:153: 2024-06-11T02:29:09.729-1200 [INFO] creating new snapshot: path=/tmp/raft2744311718/snapshots/5-130350-1718116149729.tmp - testing.go:153: 2024-06-11T02:29:09.752-1200 [INFO] creating new snapshot: path=/tmp/raft2744311718/snapshots/36-204917-1718116149752.tmp ---- PASS: TestFileSS_Ordering (0.05s) + testing.go:153: 2024-06-12T04:32:23.206+1400 [INFO] creating new snapshot: path=/tmp/raft3603578058/snapshots/5-130350-1718116343206.tmp + testing.go:153: 2024-06-12T04:32:23.218+1400 [INFO] creating new snapshot: path=/tmp/raft3603578058/snapshots/36-204917-1718116343218.tmp +--- PASS: TestFileSS_Ordering (0.03s) === RUN TestDeferFutureSuccess --- PASS: TestDeferFutureSuccess (0.00s) === RUN TestDeferFutureError @@ -999,29 +1035,29 @@ === RUN TestNetworkTransport_EncodeDecode_AddressProvider --- PASS: TestNetworkTransport_EncodeDecode_AddressProvider (0.00s) === RUN TestNetworkTransport_PooledConn ---- PASS: TestNetworkTransport_PooledConn (0.04s) +--- PASS: TestNetworkTransport_PooledConn (0.00s) === RUN TestNetworkTransport_ListenBackoff - net_transport_test.go:718: countingWriter: 2024-06-11T02:29:09.839-1200 [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:718: countingWriter: 2024-06-12T04:32:23.257+1400 [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:718: countingWriter: 2024-06-11T02:29:09.845-1200 [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:718: countingWriter: 2024-06-12T04:32:23.263+1400 [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:718: countingWriter: 2024-06-11T02:29:09.856-1200 [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:718: countingWriter: 2024-06-12T04:32:23.275+1400 [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:718: countingWriter: 2024-06-11T02:29:09.876-1200 [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:718: countingWriter: 2024-06-12T04:32:23.296+1400 [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:718: countingWriter: 2024-06-11T02:29:09.917-1200 [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:718: countingWriter: 2024-06-12T04:32:23.336+1400 [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:718: countingWriter: 2024-06-11T02:29:09.997-1200 [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:718: countingWriter: 2024-06-12T04:32:23.417+1400 [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:718: countingWriter: 2024-06-11T02:29:10.158-1200 [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:718: countingWriter: 2024-06-12T04:32:23.579+1400 [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:718: countingWriter: 2024-06-11T02:29:10.479-1200 [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:718: countingWriter: 2024-06-12T04:32:23.900+1400 [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:718: countingWriter: 2024-06-11T02:29:11.120-1200 [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:718: countingWriter: 2024-06-12T04:32:24.543+1400 [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:718: countingWriter: 2024-06-11T02:29:12.124-1200 [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:718: countingWriter: 2024-06-12T04:32:25.547+1400 [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:718: countingWriter: 2024-06-11T02:29:13.129-1200 [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:718: countingWriter: 2024-06-12T04:32:26.551+1400 [ERROR] test: failed to accept connection: error="intentional error in test" --- PASS: TestNetworkTransport_ListenBackoff (4.00s) === RUN TestPeersJSON_BadConfiguration @@ -1031,2962 +1067,1267 @@ === RUN TestPeersJSON_ReadConfigJSON --- PASS: TestPeersJSON_ReadConfigJSON (0.00s) === RUN TestRaft_StartStop -testLoggerAdapter verbose: server-7c309cb1-b7df-de98-9648-f8d3db5ba412: 2024-06-11T02:29:13.847-1200 [INFO] server-7c309cb1-b7df-de98-9648-f8d3db5ba412: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7c309cb1-b7df-de98-9648-f8d3db5ba412 Address:7c309cb1-b7df-de98-9648-f8d3db5ba412}]" - testing.go:149: server-7c309cb1-b7df-de98-9648-f8d3db5ba412: 2024-06-11T02:29:13.847-1200 [INFO] server-7c309cb1-b7df-de98-9648-f8d3db5ba412: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7c309cb1-b7df-de98-9648-f8d3db5ba412 Address:7c309cb1-b7df-de98-9648-f8d3db5ba412}]" +testLoggerAdapter verbose: server-0eae6cf0-ea81-0a58-e22b-5697f6c319fc: 2024-06-12T04:32:27.263+1400 [INFO] server-0eae6cf0-ea81-0a58-e22b-5697f6c319fc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0eae6cf0-ea81-0a58-e22b-5697f6c319fc Address:0eae6cf0-ea81-0a58-e22b-5697f6c319fc}]" + testing.go:149: server-0eae6cf0-ea81-0a58-e22b-5697f6c319fc: 2024-06-12T04:32:27.263+1400 [INFO] server-0eae6cf0-ea81-0a58-e22b-5697f6c319fc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0eae6cf0-ea81-0a58-e22b-5697f6c319fc Address:0eae6cf0-ea81-0a58-e22b-5697f6c319fc}]" --- PASS: TestRaft_StartStop (0.00s) === RUN TestRaft_AfterShutdown -testLoggerAdapter verbose: server-0cb30a7b-71e0-18bc-de55-03192fc46cac: 2024-06-11T02:29:13.850-1200 [INFO] server-0cb30a7b-71e0-18bc-de55-03192fc46cac: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0cb30a7b-71e0-18bc-de55-03192fc46cac Address:0cb30a7b-71e0-18bc-de55-03192fc46cac}]" - testing.go:149: server-0cb30a7b-71e0-18bc-de55-03192fc46cac: 2024-06-11T02:29:13.850-1200 [INFO] server-0cb30a7b-71e0-18bc-de55-03192fc46cac: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0cb30a7b-71e0-18bc-de55-03192fc46cac Address:0cb30a7b-71e0-18bc-de55-03192fc46cac}]" +testLoggerAdapter verbose: server-eeb511d0-5d7f-63c2-affe-091a8d78d2d2: 2024-06-12T04:32:27.264+1400 [INFO] server-eeb511d0-5d7f-63c2-affe-091a8d78d2d2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eeb511d0-5d7f-63c2-affe-091a8d78d2d2 Address:eeb511d0-5d7f-63c2-affe-091a8d78d2d2}]" + testing.go:149: server-eeb511d0-5d7f-63c2-affe-091a8d78d2d2: 2024-06-12T04:32:27.264+1400 [INFO] server-eeb511d0-5d7f-63c2-affe-091a8d78d2d2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eeb511d0-5d7f-63c2-affe-091a8d78d2d2 Address:eeb511d0-5d7f-63c2-affe-091a8d78d2d2}]" --- PASS: TestRaft_AfterShutdown (0.00s) === RUN TestRaft_LiveBootstrap -testLoggerAdapter verbose: server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: 2024-06-11T02:29:13.854-1200 [INFO] server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: initial configuration: index=0 servers=[] - testing.go:149: server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: 2024-06-11T02:29:13.854-1200 [INFO] server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: initial configuration: index=0 servers=[] -testLoggerAdapter verbose: server-16c6a1eb-b02a-2af2-8153-05f57ddbdc39: 2024-06-11T02:29:13.855-1200 [INFO] server-16c6a1eb-b02a-2af2-8153-05f57ddbdc39: initial configuration: index=0 servers=[] -testLoggerAdapter verbose: server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: 2024-06-11T02:29:13.855-1200 [INFO] server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: entering follower state: follower="Node at 2d1ce080-8d2e-52bd-b4de-3e4fea798856 [Follower]" leader= - testing.go:149: server-16c6a1eb-b02a-2af2-8153-05f57ddbdc39: 2024-06-11T02:29:13.855-1200 [INFO] server-16c6a1eb-b02a-2af2-8153-05f57ddbdc39: initial configuration: index=0 servers=[] -testLoggerAdapter verbose: server-16c6a1eb-b02a-2af2-8153-05f57ddbdc39: 2024-06-11T02:29:13.855-1200 [INFO] server-16c6a1eb-b02a-2af2-8153-05f57ddbdc39: entering follower state: follower="Node at 16c6a1eb-b02a-2af2-8153-05f57ddbdc39 [Follower]" leader= - testing.go:149: server-16c6a1eb-b02a-2af2-8153-05f57ddbdc39: 2024-06-11T02:29:13.855-1200 [INFO] server-16c6a1eb-b02a-2af2-8153-05f57ddbdc39: entering follower state: follower="Node at 16c6a1eb-b02a-2af2-8153-05f57ddbdc39 [Follower]" leader= -testLoggerAdapter verbose: server-caa30d7e-83d6-c6d4-ef9e-d18a70c17754: 2024-06-11T02:29:13.855-1200 [INFO] server-caa30d7e-83d6-c6d4-ef9e-d18a70c17754: initial configuration: index=0 servers=[] - testing.go:149: server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: 2024-06-11T02:29:13.855-1200 [INFO] server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: entering follower state: follower="Node at 2d1ce080-8d2e-52bd-b4de-3e4fea798856 [Follower]" leader= - testing.go:149: server-caa30d7e-83d6-c6d4-ef9e-d18a70c17754: 2024-06-11T02:29:13.855-1200 [INFO] server-caa30d7e-83d6-c6d4-ef9e-d18a70c17754: initial configuration: index=0 servers=[] -testLoggerAdapter verbose: server-caa30d7e-83d6-c6d4-ef9e-d18a70c17754: 2024-06-11T02:29:13.855-1200 [INFO] server-caa30d7e-83d6-c6d4-ef9e-d18a70c17754: entering follower state: follower="Node at caa30d7e-83d6-c6d4-ef9e-d18a70c17754 [Follower]" leader= - testing.go:149: server-caa30d7e-83d6-c6d4-ef9e-d18a70c17754: 2024-06-11T02:29:13.855-1200 [INFO] server-caa30d7e-83d6-c6d4-ef9e-d18a70c17754: entering follower state: follower="Node at caa30d7e-83d6-c6d4-ef9e-d18a70c17754 [Follower]" leader= -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:13.855-1200 [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2024-06-11T02:29:13.855-1200 [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: server-caa30d7e-83d6-c6d4-ef9e-d18a70c17754: 2024-06-11T02:29:13.929-1200 [WARN] server-caa30d7e-83d6-c6d4-ef9e-d18a70c17754: no known peers, aborting election - testing.go:149: server-caa30d7e-83d6-c6d4-ef9e-d18a70c17754: 2024-06-11T02:29:13.929-1200 [WARN] server-caa30d7e-83d6-c6d4-ef9e-d18a70c17754: no known peers, aborting election -testLoggerAdapter verbose: server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: 2024-06-11T02:29:13.935-1200 [WARN] server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: 2024-06-11T02:29:13.935-1200 [WARN] server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: 2024-06-11T02:29:13.936-1200 [INFO] server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: entering candidate state: node="Node at 2d1ce080-8d2e-52bd-b4de-3e4fea798856 [Candidate]" term=2 - testing.go:149: server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: 2024-06-11T02:29:13.936-1200 [INFO] server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: entering candidate state: node="Node at 2d1ce080-8d2e-52bd-b4de-3e4fea798856 [Candidate]" term=2 -testLoggerAdapter verbose: server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: 2024-06-11T02:29:13.936-1200 [INFO] server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: election won: tally=2 - testing.go:149: server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: 2024-06-11T02:29:13.936-1200 [INFO] server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: election won: tally=2 -testLoggerAdapter verbose: server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: 2024-06-11T02:29:13.936-1200 [INFO] server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: entering leader state: leader="Node at 2d1ce080-8d2e-52bd-b4de-3e4fea798856 [Leader]" - testing.go:149: server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: 2024-06-11T02:29:13.936-1200 [INFO] server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: entering leader state: leader="Node at 2d1ce080-8d2e-52bd-b4de-3e4fea798856 [Leader]" -testLoggerAdapter verbose: server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: 2024-06-11T02:29:13.936-1200 [INFO] server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: added peer, starting replication: peer=server-16c6a1eb-b02a-2af2-8153-05f57ddbdc39 - testing.go:149: server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: 2024-06-11T02:29:13.936-1200 [INFO] server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: added peer, starting replication: peer=server-16c6a1eb-b02a-2af2-8153-05f57ddbdc39 -testLoggerAdapter verbose: server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: 2024-06-11T02:29:13.936-1200 [INFO] server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: added peer, starting replication: peer=server-caa30d7e-83d6-c6d4-ef9e-d18a70c17754 - testing.go:149: server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: 2024-06-11T02:29:13.936-1200 [INFO] server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: added peer, starting replication: peer=server-caa30d7e-83d6-c6d4-ef9e-d18a70c17754 -testLoggerAdapter verbose: server-caa30d7e-83d6-c6d4-ef9e-d18a70c17754: 2024-06-11T02:29:13.936-1200 [WARN] server-caa30d7e-83d6-c6d4-ef9e-d18a70c17754: failed to get previous log: previous-index=1 last-index=0 error="log not found" - testing.go:149: server-caa30d7e-83d6-c6d4-ef9e-d18a70c17754: 2024-06-11T02:29:13.936-1200 [WARN] server-caa30d7e-83d6-c6d4-ef9e-d18a70c17754: failed to get previous log: previous-index=1 last-index=0 error="log not found" -testLoggerAdapter verbose: server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: 2024-06-11T02:29:13.936-1200 [WARN] server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: appendEntries rejected, sending older logs: peer="{Voter server-caa30d7e-83d6-c6d4-ef9e-d18a70c17754 caa30d7e-83d6-c6d4-ef9e-d18a70c17754}" next=1 - testing.go:149: server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: 2024-06-11T02:29:13.936-1200 [WARN] server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: appendEntries rejected, sending older logs: peer="{Voter server-caa30d7e-83d6-c6d4-ef9e-d18a70c17754 caa30d7e-83d6-c6d4-ef9e-d18a70c17754}" next=1 -testLoggerAdapter verbose: server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: 2024-06-11T02:29:13.937-1200 [INFO] server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: pipelining replication: peer="{Voter server-caa30d7e-83d6-c6d4-ef9e-d18a70c17754 caa30d7e-83d6-c6d4-ef9e-d18a70c17754}" - testing.go:149: server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: 2024-06-11T02:29:13.937-1200 [INFO] server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: pipelining replication: peer="{Voter server-caa30d7e-83d6-c6d4-ef9e-d18a70c17754 caa30d7e-83d6-c6d4-ef9e-d18a70c17754}" -testLoggerAdapter verbose: server-16c6a1eb-b02a-2af2-8153-05f57ddbdc39: 2024-06-11T02:29:13.937-1200 [WARN] server-16c6a1eb-b02a-2af2-8153-05f57ddbdc39: failed to get previous log: previous-index=1 last-index=0 error="log not found" - testing.go:149: server-16c6a1eb-b02a-2af2-8153-05f57ddbdc39: 2024-06-11T02:29:13.937-1200 [WARN] server-16c6a1eb-b02a-2af2-8153-05f57ddbdc39: failed to get previous log: previous-index=1 last-index=0 error="log not found" -testLoggerAdapter verbose: server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: 2024-06-11T02:29:13.937-1200 [WARN] server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: appendEntries rejected, sending older logs: peer="{Voter server-16c6a1eb-b02a-2af2-8153-05f57ddbdc39 16c6a1eb-b02a-2af2-8153-05f57ddbdc39}" next=1 - testing.go:149: server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: 2024-06-11T02:29:13.937-1200 [WARN] server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: appendEntries rejected, sending older logs: peer="{Voter server-16c6a1eb-b02a-2af2-8153-05f57ddbdc39 16c6a1eb-b02a-2af2-8153-05f57ddbdc39}" next=1 -testLoggerAdapter verbose: server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: 2024-06-11T02:29:13.937-1200 [INFO] server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: pipelining replication: peer="{Voter server-16c6a1eb-b02a-2af2-8153-05f57ddbdc39 16c6a1eb-b02a-2af2-8153-05f57ddbdc39}" - testing.go:149: server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: 2024-06-11T02:29:13.937-1200 [INFO] server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: pipelining replication: peer="{Voter server-16c6a1eb-b02a-2af2-8153-05f57ddbdc39 16c6a1eb-b02a-2af2-8153-05f57ddbdc39}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:14.044-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:13.937955276 -1200 -12 m=+4.933327817 (2 nodes), 81.77392ms from start of poll, 83.353557ms from cluster start. Timeout at 2024-06-11 02:29:14.044142403 -1200 -12 m=+5.039515236, 106.187419ms after stability - testing.go:149: cluster: 2024-06-11T02:29:14.044-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:13.937955276 -1200 -12 m=+4.933327817 (2 nodes), 81.77392ms from start of poll, 83.353557ms from cluster start. Timeout at 2024-06-11 02:29:14.044142403 -1200 -12 m=+5.039515236, 106.187419ms after stability -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:14.044-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:14.044-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:14.150-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:14.04457552 -1200 -12 m=+5.039948062 (1 nodes), 3.209µs from start of poll, 189.973802ms from cluster start. Timeout at 2024-06-11 02:29:14.150648316 -1200 -12 m=+5.146020858, 106.072796ms after stability - testing.go:149: cluster: 2024-06-11T02:29:14.150-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:14.04457552 -1200 -12 m=+5.039948062 (1 nodes), 3.209µs from start of poll, 189.973802ms from cluster start. Timeout at 2024-06-11 02:29:14.150648316 -1200 -12 m=+5.146020858, 106.072796ms after stability -testLoggerAdapter verbose: server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: 2024-06-11T02:29:14.163-1200 [INFO] server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: aborting pipeline replication: peer="{Voter server-caa30d7e-83d6-c6d4-ef9e-d18a70c17754 caa30d7e-83d6-c6d4-ef9e-d18a70c17754}" - testing.go:149: server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: 2024-06-11T02:29:14.163-1200 [INFO] server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: aborting pipeline replication: peer="{Voter server-caa30d7e-83d6-c6d4-ef9e-d18a70c17754 caa30d7e-83d6-c6d4-ef9e-d18a70c17754}" -testLoggerAdapter verbose: server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: 2024-06-11T02:29:14.163-1200 [INFO] server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: aborting pipeline replication: peer="{Voter server-16c6a1eb-b02a-2af2-8153-05f57ddbdc39 16c6a1eb-b02a-2af2-8153-05f57ddbdc39}" - testing.go:149: server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: 2024-06-11T02:29:14.163-1200 [INFO] server-2d1ce080-8d2e-52bd-b4de-3e4fea798856: aborting pipeline replication: peer="{Voter server-16c6a1eb-b02a-2af2-8153-05f57ddbdc39 16c6a1eb-b02a-2af2-8153-05f57ddbdc39}" ---- PASS: TestRaft_LiveBootstrap (0.31s) +testLoggerAdapter verbose: server-9959a38a-d08f-ce06-fc11-6245c7647547: 2024-06-12T04:32:27.266+1400 [INFO] server-9959a38a-d08f-ce06-fc11-6245c7647547: initial configuration: index=0 servers=[] + testing.go:149: server-9959a38a-d08f-ce06-fc11-6245c7647547: 2024-06-12T04:32:27.266+1400 [INFO] server-9959a38a-d08f-ce06-fc11-6245c7647547: initial configuration: index=0 servers=[] +testLoggerAdapter verbose: server-3cd6135d-68ac-7c7b-1f7f-06c68177f1a3: 2024-06-12T04:32:27.266+1400 [INFO] server-3cd6135d-68ac-7c7b-1f7f-06c68177f1a3: initial configuration: index=0 servers=[] + testing.go:149: server-3cd6135d-68ac-7c7b-1f7f-06c68177f1a3: 2024-06-12T04:32:27.266+1400 [INFO] server-3cd6135d-68ac-7c7b-1f7f-06c68177f1a3: initial configuration: index=0 servers=[] +testLoggerAdapter verbose: server-524b994e-d4c7-6910-bf4d-515087b45e85: 2024-06-12T04:32:27.266+1400 [INFO] server-524b994e-d4c7-6910-bf4d-515087b45e85: initial configuration: index=0 servers=[] + testing.go:149: server-524b994e-d4c7-6910-bf4d-515087b45e85: 2024-06-12T04:32:27.266+1400 [INFO] server-524b994e-d4c7-6910-bf4d-515087b45e85: initial configuration: index=0 servers=[] +testLoggerAdapter verbose: server-9959a38a-d08f-ce06-fc11-6245c7647547: 2024-06-12T04:32:27.266+1400 [INFO] server-9959a38a-d08f-ce06-fc11-6245c7647547: entering follower state: follower="Node at 9959a38a-d08f-ce06-fc11-6245c7647547 [Follower]" leader= + testing.go:149: server-9959a38a-d08f-ce06-fc11-6245c7647547: 2024-06-12T04:32:27.266+1400 [INFO] server-9959a38a-d08f-ce06-fc11-6245c7647547: entering follower state: follower="Node at 9959a38a-d08f-ce06-fc11-6245c7647547 [Follower]" leader= +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:27.266+1400 [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-06-12T04:32:27.266+1400 [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: server-3cd6135d-68ac-7c7b-1f7f-06c68177f1a3: 2024-06-12T04:32:27.266+1400 [INFO] server-3cd6135d-68ac-7c7b-1f7f-06c68177f1a3: entering follower state: follower="Node at 3cd6135d-68ac-7c7b-1f7f-06c68177f1a3 [Follower]" leader= + testing.go:149: server-3cd6135d-68ac-7c7b-1f7f-06c68177f1a3: 2024-06-12T04:32:27.266+1400 [INFO] server-3cd6135d-68ac-7c7b-1f7f-06c68177f1a3: entering follower state: follower="Node at 3cd6135d-68ac-7c7b-1f7f-06c68177f1a3 [Follower]" leader= +testLoggerAdapter verbose: server-524b994e-d4c7-6910-bf4d-515087b45e85: 2024-06-12T04:32:27.266+1400 [INFO] server-524b994e-d4c7-6910-bf4d-515087b45e85: entering follower state: follower="Node at 524b994e-d4c7-6910-bf4d-515087b45e85 [Follower]" leader= + testing.go:149: server-524b994e-d4c7-6910-bf4d-515087b45e85: 2024-06-12T04:32:27.266+1400 [INFO] server-524b994e-d4c7-6910-bf4d-515087b45e85: entering follower state: follower="Node at 524b994e-d4c7-6910-bf4d-515087b45e85 [Follower]" leader= +testLoggerAdapter verbose: server-524b994e-d4c7-6910-bf4d-515087b45e85: 2024-06-12T04:32:27.320+1400 [WARN] server-524b994e-d4c7-6910-bf4d-515087b45e85: no known peers, aborting election + testing.go:149: server-524b994e-d4c7-6910-bf4d-515087b45e85: 2024-06-12T04:32:27.320+1400 [WARN] server-524b994e-d4c7-6910-bf4d-515087b45e85: no known peers, aborting election +testLoggerAdapter verbose: server-3cd6135d-68ac-7c7b-1f7f-06c68177f1a3: 2024-06-12T04:32:27.327+1400 [WARN] server-3cd6135d-68ac-7c7b-1f7f-06c68177f1a3: no known peers, aborting election + testing.go:149: server-3cd6135d-68ac-7c7b-1f7f-06c68177f1a3: 2024-06-12T04:32:27.327+1400 [WARN] server-3cd6135d-68ac-7c7b-1f7f-06c68177f1a3: no known peers, aborting election +testLoggerAdapter verbose: server-9959a38a-d08f-ce06-fc11-6245c7647547: 2024-06-12T04:32:27.329+1400 [WARN] server-9959a38a-d08f-ce06-fc11-6245c7647547: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-9959a38a-d08f-ce06-fc11-6245c7647547: 2024-06-12T04:32:27.329+1400 [WARN] server-9959a38a-d08f-ce06-fc11-6245c7647547: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-9959a38a-d08f-ce06-fc11-6245c7647547: 2024-06-12T04:32:27.329+1400 [INFO] server-9959a38a-d08f-ce06-fc11-6245c7647547: entering candidate state: node="Node at 9959a38a-d08f-ce06-fc11-6245c7647547 [Candidate]" term=2 + testing.go:149: server-9959a38a-d08f-ce06-fc11-6245c7647547: 2024-06-12T04:32:27.329+1400 [INFO] server-9959a38a-d08f-ce06-fc11-6245c7647547: entering candidate state: node="Node at 9959a38a-d08f-ce06-fc11-6245c7647547 [Candidate]" term=2 +testLoggerAdapter verbose: server-9959a38a-d08f-ce06-fc11-6245c7647547: 2024-06-12T04:32:27.329+1400 [INFO] server-9959a38a-d08f-ce06-fc11-6245c7647547: election won: tally=2 + testing.go:149: server-9959a38a-d08f-ce06-fc11-6245c7647547: 2024-06-12T04:32:27.329+1400 [INFO] server-9959a38a-d08f-ce06-fc11-6245c7647547: election won: tally=2 +testLoggerAdapter verbose: server-9959a38a-d08f-ce06-fc11-6245c7647547: 2024-06-12T04:32:27.329+1400 [INFO] server-9959a38a-d08f-ce06-fc11-6245c7647547: entering leader state: leader="Node at 9959a38a-d08f-ce06-fc11-6245c7647547 [Leader]" + testing.go:149: server-9959a38a-d08f-ce06-fc11-6245c7647547: 2024-06-12T04:32:27.329+1400 [INFO] server-9959a38a-d08f-ce06-fc11-6245c7647547: entering leader state: leader="Node at 9959a38a-d08f-ce06-fc11-6245c7647547 [Leader]" +testLoggerAdapter verbose: server-9959a38a-d08f-ce06-fc11-6245c7647547: 2024-06-12T04:32:27.329+1400 [INFO] server-9959a38a-d08f-ce06-fc11-6245c7647547: added peer, starting replication: peer=server-3cd6135d-68ac-7c7b-1f7f-06c68177f1a3 + testing.go:149: server-9959a38a-d08f-ce06-fc11-6245c7647547: 2024-06-12T04:32:27.329+1400 [INFO] server-9959a38a-d08f-ce06-fc11-6245c7647547: added peer, starting replication: peer=server-3cd6135d-68ac-7c7b-1f7f-06c68177f1a3 +testLoggerAdapter verbose: server-9959a38a-d08f-ce06-fc11-6245c7647547: 2024-06-12T04:32:27.329+1400 [INFO] server-9959a38a-d08f-ce06-fc11-6245c7647547: added peer, starting replication: peer=server-524b994e-d4c7-6910-bf4d-515087b45e85 + testing.go:149: server-9959a38a-d08f-ce06-fc11-6245c7647547: 2024-06-12T04:32:27.329+1400 [INFO] server-9959a38a-d08f-ce06-fc11-6245c7647547: added peer, starting replication: peer=server-524b994e-d4c7-6910-bf4d-515087b45e85 +testLoggerAdapter verbose: server-524b994e-d4c7-6910-bf4d-515087b45e85: 2024-06-12T04:32:27.329+1400 [WARN] server-524b994e-d4c7-6910-bf4d-515087b45e85: failed to get previous log: previous-index=1 last-index=0 error="log not found" + testing.go:149: server-524b994e-d4c7-6910-bf4d-515087b45e85: 2024-06-12T04:32:27.329+1400 [WARN] server-524b994e-d4c7-6910-bf4d-515087b45e85: failed to get previous log: previous-index=1 last-index=0 error="log not found" +testLoggerAdapter verbose: server-9959a38a-d08f-ce06-fc11-6245c7647547: 2024-06-12T04:32:27.329+1400 [WARN] server-9959a38a-d08f-ce06-fc11-6245c7647547: appendEntries rejected, sending older logs: peer="{Voter server-524b994e-d4c7-6910-bf4d-515087b45e85 524b994e-d4c7-6910-bf4d-515087b45e85}" next=1 + testing.go:149: server-9959a38a-d08f-ce06-fc11-6245c7647547: 2024-06-12T04:32:27.329+1400 [WARN] server-9959a38a-d08f-ce06-fc11-6245c7647547: appendEntries rejected, sending older logs: peer="{Voter server-524b994e-d4c7-6910-bf4d-515087b45e85 524b994e-d4c7-6910-bf4d-515087b45e85}" next=1 +testLoggerAdapter verbose: server-9959a38a-d08f-ce06-fc11-6245c7647547: 2024-06-12T04:32:27.329+1400 [INFO] server-9959a38a-d08f-ce06-fc11-6245c7647547: pipelining replication: peer="{Voter server-524b994e-d4c7-6910-bf4d-515087b45e85 524b994e-d4c7-6910-bf4d-515087b45e85}" + testing.go:149: server-9959a38a-d08f-ce06-fc11-6245c7647547: 2024-06-12T04:32:27.329+1400 [INFO] server-9959a38a-d08f-ce06-fc11-6245c7647547: pipelining replication: peer="{Voter server-524b994e-d4c7-6910-bf4d-515087b45e85 524b994e-d4c7-6910-bf4d-515087b45e85}" +testLoggerAdapter verbose: server-3cd6135d-68ac-7c7b-1f7f-06c68177f1a3: 2024-06-12T04:32:27.329+1400 [WARN] server-3cd6135d-68ac-7c7b-1f7f-06c68177f1a3: failed to get previous log: previous-index=1 last-index=0 error="log not found" + testing.go:149: server-3cd6135d-68ac-7c7b-1f7f-06c68177f1a3: 2024-06-12T04:32:27.329+1400 [WARN] server-3cd6135d-68ac-7c7b-1f7f-06c68177f1a3: failed to get previous log: previous-index=1 last-index=0 error="log not found" +testLoggerAdapter verbose: server-9959a38a-d08f-ce06-fc11-6245c7647547: 2024-06-12T04:32:27.329+1400 [WARN] server-9959a38a-d08f-ce06-fc11-6245c7647547: appendEntries rejected, sending older logs: peer="{Voter server-3cd6135d-68ac-7c7b-1f7f-06c68177f1a3 3cd6135d-68ac-7c7b-1f7f-06c68177f1a3}" next=1 + testing.go:149: server-9959a38a-d08f-ce06-fc11-6245c7647547: 2024-06-12T04:32:27.329+1400 [WARN] server-9959a38a-d08f-ce06-fc11-6245c7647547: appendEntries rejected, sending older logs: peer="{Voter server-3cd6135d-68ac-7c7b-1f7f-06c68177f1a3 3cd6135d-68ac-7c7b-1f7f-06c68177f1a3}" next=1 +testLoggerAdapter verbose: server-9959a38a-d08f-ce06-fc11-6245c7647547: 2024-06-12T04:32:27.330+1400 [INFO] server-9959a38a-d08f-ce06-fc11-6245c7647547: pipelining replication: peer="{Voter server-3cd6135d-68ac-7c7b-1f7f-06c68177f1a3 3cd6135d-68ac-7c7b-1f7f-06c68177f1a3}" + testing.go:149: server-9959a38a-d08f-ce06-fc11-6245c7647547: 2024-06-12T04:32:27.330+1400 [INFO] server-9959a38a-d08f-ce06-fc11-6245c7647547: pipelining replication: peer="{Voter server-3cd6135d-68ac-7c7b-1f7f-06c68177f1a3 3cd6135d-68ac-7c7b-1f7f-06c68177f1a3}" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:27.436+1400 [INFO] stable state for Follower reached at 2024-06-12 04:32:27.330109675 +1400 +14 m=+4.237628254 (2 nodes), 63.525564ms from start of poll, 64.099384ms from cluster start. Timeout at 2024-06-12 04:32:27.436099279 +1400 +14 m=+4.343617898, 105.989644ms after stability + testing.go:149: cluster: 2024-06-12T04:32:27.436+1400 [INFO] stable state for Follower reached at 2024-06-12 04:32:27.330109675 +1400 +14 m=+4.237628254 (2 nodes), 63.525564ms from start of poll, 64.099384ms from cluster start. Timeout at 2024-06-12 04:32:27.436099279 +1400 +14 m=+4.343617898, 105.989644ms after stability +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:27.436+1400 [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-06-12T04:32:27.436+1400 [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:27.541+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:27.436316431 +1400 +14 m=+4.343835010 (1 nodes), 1.48µs from start of poll, 170.30614ms from cluster start. Timeout at 2024-06-12 04:32:27.541465985 +1400 +14 m=+4.448984564, 105.149554ms after stability + testing.go:149: cluster: 2024-06-12T04:32:27.541+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:27.436316431 +1400 +14 m=+4.343835010 (1 nodes), 1.48µs from start of poll, 170.30614ms from cluster start. Timeout at 2024-06-12 04:32:27.541465985 +1400 +14 m=+4.448984564, 105.149554ms after stability +testLoggerAdapter verbose: server-9959a38a-d08f-ce06-fc11-6245c7647547: 2024-06-12T04:32:27.553+1400 [INFO] server-9959a38a-d08f-ce06-fc11-6245c7647547: aborting pipeline replication: peer="{Voter server-524b994e-d4c7-6910-bf4d-515087b45e85 524b994e-d4c7-6910-bf4d-515087b45e85}" + testing.go:149: server-9959a38a-d08f-ce06-fc11-6245c7647547: 2024-06-12T04:32:27.553+1400 [INFO] server-9959a38a-d08f-ce06-fc11-6245c7647547: aborting pipeline replication: peer="{Voter server-524b994e-d4c7-6910-bf4d-515087b45e85 524b994e-d4c7-6910-bf4d-515087b45e85}" +testLoggerAdapter verbose: server-9959a38a-d08f-ce06-fc11-6245c7647547: 2024-06-12T04:32:27.553+1400 [INFO] server-9959a38a-d08f-ce06-fc11-6245c7647547: aborting pipeline replication: peer="{Voter server-3cd6135d-68ac-7c7b-1f7f-06c68177f1a3 3cd6135d-68ac-7c7b-1f7f-06c68177f1a3}" + testing.go:149: server-9959a38a-d08f-ce06-fc11-6245c7647547: 2024-06-12T04:32:27.553+1400 [INFO] server-9959a38a-d08f-ce06-fc11-6245c7647547: aborting pipeline replication: peer="{Voter server-3cd6135d-68ac-7c7b-1f7f-06c68177f1a3 3cd6135d-68ac-7c7b-1f7f-06c68177f1a3}" +--- PASS: TestRaft_LiveBootstrap (0.29s) === RUN TestRaft_RecoverCluster_NoState -testLoggerAdapter verbose: server-2e47ae3c-a349-9128-bc3c-c8661ae1e02a: 2024-06-11T02:29:14.167-1200 [INFO] server-2e47ae3c-a349-9128-bc3c-c8661ae1e02a: initial configuration: index=0 servers=[] - testing.go:149: server-2e47ae3c-a349-9128-bc3c-c8661ae1e02a: 2024-06-11T02:29:14.167-1200 [INFO] server-2e47ae3c-a349-9128-bc3c-c8661ae1e02a: initial configuration: index=0 servers=[] +testLoggerAdapter verbose: server-3cca2780-e302-a7ca-0212-14e74e467f4e: 2024-06-12T04:32:27.554+1400 [INFO] server-3cca2780-e302-a7ca-0212-14e74e467f4e: initial configuration: index=0 servers=[] + testing.go:149: server-3cca2780-e302-a7ca-0212-14e74e467f4e: 2024-06-12T04:32:27.554+1400 [INFO] server-3cca2780-e302-a7ca-0212-14e74e467f4e: initial configuration: index=0 servers=[] --- PASS: TestRaft_RecoverCluster_NoState (0.00s) === RUN TestRaft_RecoverCluster raft_test.go:119: Disabled unreliable test --- SKIP: TestRaft_RecoverCluster (0.00s) === RUN TestRaft_HasExistingState -testLoggerAdapter verbose: server-72b390b8-3696-f176-aa93-104527b78e86: 2024-06-11T02:29:14.171-1200 [INFO] server-72b390b8-3696-f176-aa93-104527b78e86: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-72b390b8-3696-f176-aa93-104527b78e86 Address:72b390b8-3696-f176-aa93-104527b78e86} {Suffrage:Voter ID:server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a Address:aafc7aef-9e51-95f0-348a-3b4c0ce1369a}]" - testing.go:149: server-72b390b8-3696-f176-aa93-104527b78e86: 2024-06-11T02:29:14.171-1200 [INFO] server-72b390b8-3696-f176-aa93-104527b78e86: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-72b390b8-3696-f176-aa93-104527b78e86 Address:72b390b8-3696-f176-aa93-104527b78e86} {Suffrage:Voter ID:server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a Address:aafc7aef-9e51-95f0-348a-3b4c0ce1369a}]" -testLoggerAdapter verbose: server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: 2024-06-11T02:29:14.171-1200 [INFO] server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-72b390b8-3696-f176-aa93-104527b78e86 Address:72b390b8-3696-f176-aa93-104527b78e86} {Suffrage:Voter ID:server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a Address:aafc7aef-9e51-95f0-348a-3b4c0ce1369a}]" - testing.go:149: server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: 2024-06-11T02:29:14.171-1200 [INFO] server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-72b390b8-3696-f176-aa93-104527b78e86 Address:72b390b8-3696-f176-aa93-104527b78e86} {Suffrage:Voter ID:server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a Address:aafc7aef-9e51-95f0-348a-3b4c0ce1369a}]" -testLoggerAdapter verbose: server-c5d021a2-7570-3d80-8b4c-49cf21b70a89: 2024-06-11T02:29:14.172-1200 [INFO] server-c5d021a2-7570-3d80-8b4c-49cf21b70a89: initial configuration: index=0 servers=[] - testing.go:149: server-c5d021a2-7570-3d80-8b4c-49cf21b70a89: 2024-06-11T02:29:14.172-1200 [INFO] server-c5d021a2-7570-3d80-8b4c-49cf21b70a89: initial configuration: index=0 servers=[] -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:14.173-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:14.173-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-72b390b8-3696-f176-aa93-104527b78e86: 2024-06-11T02:29:14.173-1200 [INFO] server-72b390b8-3696-f176-aa93-104527b78e86: entering follower state: follower="Node at 72b390b8-3696-f176-aa93-104527b78e86 [Follower]" leader= - testing.go:149: server-72b390b8-3696-f176-aa93-104527b78e86: 2024-06-11T02:29:14.173-1200 [INFO] server-72b390b8-3696-f176-aa93-104527b78e86: entering follower state: follower="Node at 72b390b8-3696-f176-aa93-104527b78e86 [Follower]" leader= -testLoggerAdapter verbose: server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: 2024-06-11T02:29:14.173-1200 [INFO] server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: entering follower state: follower="Node at aafc7aef-9e51-95f0-348a-3b4c0ce1369a [Follower]" leader= - testing.go:149: server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: 2024-06-11T02:29:14.173-1200 [INFO] server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: entering follower state: follower="Node at aafc7aef-9e51-95f0-348a-3b4c0ce1369a [Follower]" leader= -testLoggerAdapter verbose: server-c5d021a2-7570-3d80-8b4c-49cf21b70a89: 2024-06-11T02:29:14.173-1200 [INFO] server-c5d021a2-7570-3d80-8b4c-49cf21b70a89: entering follower state: follower="Node at c5d021a2-7570-3d80-8b4c-49cf21b70a89 [Follower]" leader= - testing.go:149: server-c5d021a2-7570-3d80-8b4c-49cf21b70a89: 2024-06-11T02:29:14.173-1200 [INFO] server-c5d021a2-7570-3d80-8b4c-49cf21b70a89: entering follower state: follower="Node at c5d021a2-7570-3d80-8b4c-49cf21b70a89 [Follower]" leader= -testLoggerAdapter verbose: server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: 2024-06-11T02:29:14.236-1200 [WARN] server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: 2024-06-11T02:29:14.236-1200 [WARN] server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: 2024-06-11T02:29:14.236-1200 [INFO] server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: entering candidate state: node="Node at aafc7aef-9e51-95f0-348a-3b4c0ce1369a [Candidate]" term=2 - testing.go:149: server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: 2024-06-11T02:29:14.236-1200 [INFO] server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: entering candidate state: node="Node at aafc7aef-9e51-95f0-348a-3b4c0ce1369a [Candidate]" term=2 -testLoggerAdapter verbose: server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: 2024-06-11T02:29:14.236-1200 [INFO] server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: election won: tally=2 - testing.go:149: server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: 2024-06-11T02:29:14.236-1200 [INFO] server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: election won: tally=2 -testLoggerAdapter verbose: server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: 2024-06-11T02:29:14.236-1200 [INFO] server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: entering leader state: leader="Node at aafc7aef-9e51-95f0-348a-3b4c0ce1369a [Leader]" - testing.go:149: server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: 2024-06-11T02:29:14.236-1200 [INFO] server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: entering leader state: leader="Node at aafc7aef-9e51-95f0-348a-3b4c0ce1369a [Leader]" -testLoggerAdapter verbose: server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: 2024-06-11T02:29:14.236-1200 [INFO] server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: added peer, starting replication: peer=server-72b390b8-3696-f176-aa93-104527b78e86 - testing.go:149: server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: 2024-06-11T02:29:14.236-1200 [INFO] server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: added peer, starting replication: peer=server-72b390b8-3696-f176-aa93-104527b78e86 -testLoggerAdapter verbose: server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: 2024-06-11T02:29:14.237-1200 [INFO] server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: pipelining replication: peer="{Voter server-72b390b8-3696-f176-aa93-104527b78e86 72b390b8-3696-f176-aa93-104527b78e86}" - testing.go:149: server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: 2024-06-11T02:29:14.237-1200 [INFO] server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: pipelining replication: peer="{Voter server-72b390b8-3696-f176-aa93-104527b78e86 72b390b8-3696-f176-aa93-104527b78e86}" -testLoggerAdapter verbose: server-c5d021a2-7570-3d80-8b4c-49cf21b70a89: 2024-06-11T02:29:14.264-1200 [WARN] server-c5d021a2-7570-3d80-8b4c-49cf21b70a89: no known peers, aborting election - testing.go:149: server-c5d021a2-7570-3d80-8b4c-49cf21b70a89: 2024-06-11T02:29:14.264-1200 [WARN] server-c5d021a2-7570-3d80-8b4c-49cf21b70a89: no known peers, aborting election -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:14.343-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:14.237354227 -1200 -12 m=+5.232726768 (1 nodes), 64.028379ms from start of poll, 66.67637ms from cluster start. Timeout at 2024-06-11 02:29:14.343303942 -1200 -12 m=+5.338676483, 105.949715ms after stability - testing.go:149: cluster: 2024-06-11T02:29:14.343-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:14.237354227 -1200 -12 m=+5.232726768 (1 nodes), 64.028379ms from start of poll, 66.67637ms from cluster start. Timeout at 2024-06-11 02:29:14.343303942 -1200 -12 m=+5.338676483, 105.949715ms after stability -testLoggerAdapter verbose: server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: 2024-06-11T02:29:14.343-1200 [INFO] server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: updating configuration: command=AddStaging server-id=server-c5d021a2-7570-3d80-8b4c-49cf21b70a89 server-addr=c5d021a2-7570-3d80-8b4c-49cf21b70a89 servers="[{Suffrage:Voter ID:server-72b390b8-3696-f176-aa93-104527b78e86 Address:72b390b8-3696-f176-aa93-104527b78e86} {Suffrage:Voter ID:server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a Address:aafc7aef-9e51-95f0-348a-3b4c0ce1369a} {Suffrage:Voter ID:server-c5d021a2-7570-3d80-8b4c-49cf21b70a89 Address:c5d021a2-7570-3d80-8b4c-49cf21b70a89}]" - testing.go:149: server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: 2024-06-11T02:29:14.343-1200 [INFO] server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: updating configuration: command=AddStaging server-id=server-c5d021a2-7570-3d80-8b4c-49cf21b70a89 server-addr=c5d021a2-7570-3d80-8b4c-49cf21b70a89 servers="[{Suffrage:Voter ID:server-72b390b8-3696-f176-aa93-104527b78e86 Address:72b390b8-3696-f176-aa93-104527b78e86} {Suffrage:Voter ID:server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a Address:aafc7aef-9e51-95f0-348a-3b4c0ce1369a} {Suffrage:Voter ID:server-c5d021a2-7570-3d80-8b4c-49cf21b70a89 Address:c5d021a2-7570-3d80-8b4c-49cf21b70a89}]" -testLoggerAdapter verbose: server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: 2024-06-11T02:29:14.343-1200 [INFO] server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: added peer, starting replication: peer=server-c5d021a2-7570-3d80-8b4c-49cf21b70a89 - testing.go:149: server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: 2024-06-11T02:29:14.343-1200 [INFO] server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: added peer, starting replication: peer=server-c5d021a2-7570-3d80-8b4c-49cf21b70a89 -testLoggerAdapter verbose: server-c5d021a2-7570-3d80-8b4c-49cf21b70a89: 2024-06-11T02:29:14.344-1200 [WARN] server-c5d021a2-7570-3d80-8b4c-49cf21b70a89: failed to get previous log: previous-index=3 last-index=0 error="log not found" - testing.go:149: server-c5d021a2-7570-3d80-8b4c-49cf21b70a89: 2024-06-11T02:29:14.344-1200 [WARN] server-c5d021a2-7570-3d80-8b4c-49cf21b70a89: failed to get previous log: previous-index=3 last-index=0 error="log not found" -testLoggerAdapter verbose: server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: 2024-06-11T02:29:14.344-1200 [WARN] server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: appendEntries rejected, sending older logs: peer="{Voter server-c5d021a2-7570-3d80-8b4c-49cf21b70a89 c5d021a2-7570-3d80-8b4c-49cf21b70a89}" next=1 - testing.go:149: server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: 2024-06-11T02:29:14.344-1200 [WARN] server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: appendEntries rejected, sending older logs: peer="{Voter server-c5d021a2-7570-3d80-8b4c-49cf21b70a89 c5d021a2-7570-3d80-8b4c-49cf21b70a89}" next=1 -testLoggerAdapter verbose: server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: 2024-06-11T02:29:14.344-1200 [INFO] server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: pipelining replication: peer="{Voter server-c5d021a2-7570-3d80-8b4c-49cf21b70a89 c5d021a2-7570-3d80-8b4c-49cf21b70a89}" - testing.go:149: server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: 2024-06-11T02:29:14.344-1200 [INFO] server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: pipelining replication: peer="{Voter server-c5d021a2-7570-3d80-8b4c-49cf21b70a89 c5d021a2-7570-3d80-8b4c-49cf21b70a89}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:14.345-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:14.345-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:14.450-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:14.345451735 -1200 -12 m=+5.340824276 (1 nodes), 2.625µs from start of poll, 174.773878ms from cluster start. Timeout at 2024-06-11 02:29:14.450471634 -1200 -12 m=+5.445844176, 105.0199ms after stability - testing.go:149: cluster: 2024-06-11T02:29:14.450-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:14.345451735 -1200 -12 m=+5.340824276 (1 nodes), 2.625µs from start of poll, 174.773878ms from cluster start. Timeout at 2024-06-11 02:29:14.450471634 -1200 -12 m=+5.445844176, 105.0199ms after stability -testLoggerAdapter verbose: server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: 2024-06-11T02:29:14.450-1200 [INFO] server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: aborting pipeline replication: peer="{Voter server-c5d021a2-7570-3d80-8b4c-49cf21b70a89 c5d021a2-7570-3d80-8b4c-49cf21b70a89}" - testing.go:149: server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: 2024-06-11T02:29:14.450-1200 [INFO] server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: aborting pipeline replication: peer="{Voter server-c5d021a2-7570-3d80-8b4c-49cf21b70a89 c5d021a2-7570-3d80-8b4c-49cf21b70a89}" -testLoggerAdapter verbose: server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: 2024-06-11T02:29:14.450-1200 [INFO] server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: aborting pipeline replication: peer="{Voter server-72b390b8-3696-f176-aa93-104527b78e86 72b390b8-3696-f176-aa93-104527b78e86}" - testing.go:149: server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: 2024-06-11T02:29:14.450-1200 [INFO] server-aafc7aef-9e51-95f0-348a-3b4c0ce1369a: aborting pipeline replication: peer="{Voter server-72b390b8-3696-f176-aa93-104527b78e86 72b390b8-3696-f176-aa93-104527b78e86}" ---- PASS: TestRaft_HasExistingState (0.29s) +testLoggerAdapter verbose: server-458cf6b8-fd2f-3e20-0e83-adcf4b9f63b8: 2024-06-12T04:32:27.556+1400 [INFO] server-458cf6b8-fd2f-3e20-0e83-adcf4b9f63b8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-458cf6b8-fd2f-3e20-0e83-adcf4b9f63b8 Address:458cf6b8-fd2f-3e20-0e83-adcf4b9f63b8} {Suffrage:Voter ID:server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09 Address:edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09}]" + testing.go:149: server-458cf6b8-fd2f-3e20-0e83-adcf4b9f63b8: 2024-06-12T04:32:27.556+1400 [INFO] server-458cf6b8-fd2f-3e20-0e83-adcf4b9f63b8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-458cf6b8-fd2f-3e20-0e83-adcf4b9f63b8 Address:458cf6b8-fd2f-3e20-0e83-adcf4b9f63b8} {Suffrage:Voter ID:server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09 Address:edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09}]" +testLoggerAdapter verbose: server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: 2024-06-12T04:32:27.556+1400 [INFO] server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-458cf6b8-fd2f-3e20-0e83-adcf4b9f63b8 Address:458cf6b8-fd2f-3e20-0e83-adcf4b9f63b8} {Suffrage:Voter ID:server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09 Address:edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09}]" + testing.go:149: server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: 2024-06-12T04:32:27.556+1400 [INFO] server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-458cf6b8-fd2f-3e20-0e83-adcf4b9f63b8 Address:458cf6b8-fd2f-3e20-0e83-adcf4b9f63b8} {Suffrage:Voter ID:server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09 Address:edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09}]" +testLoggerAdapter verbose: server-a478c75e-aa84-b1b6-2c10-4f09a6bcd070: 2024-06-12T04:32:27.556+1400 [INFO] server-a478c75e-aa84-b1b6-2c10-4f09a6bcd070: initial configuration: index=0 servers=[] + testing.go:149: server-a478c75e-aa84-b1b6-2c10-4f09a6bcd070: 2024-06-12T04:32:27.556+1400 [INFO] server-a478c75e-aa84-b1b6-2c10-4f09a6bcd070: initial configuration: index=0 servers=[] +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:27.557+1400 [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-06-12T04:32:27.557+1400 [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-458cf6b8-fd2f-3e20-0e83-adcf4b9f63b8: 2024-06-12T04:32:27.557+1400 [INFO] server-458cf6b8-fd2f-3e20-0e83-adcf4b9f63b8: entering follower state: follower="Node at 458cf6b8-fd2f-3e20-0e83-adcf4b9f63b8 [Follower]" leader= + testing.go:149: server-458cf6b8-fd2f-3e20-0e83-adcf4b9f63b8: 2024-06-12T04:32:27.557+1400 [INFO] server-458cf6b8-fd2f-3e20-0e83-adcf4b9f63b8: entering follower state: follower="Node at 458cf6b8-fd2f-3e20-0e83-adcf4b9f63b8 [Follower]" leader= +testLoggerAdapter verbose: server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: 2024-06-12T04:32:27.557+1400 [INFO] server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: entering follower state: follower="Node at edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09 [Follower]" leader= + testing.go:149: server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: 2024-06-12T04:32:27.557+1400 [INFO] server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: entering follower state: follower="Node at edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09 [Follower]" leader= +testLoggerAdapter verbose: server-a478c75e-aa84-b1b6-2c10-4f09a6bcd070: 2024-06-12T04:32:27.557+1400 [INFO] server-a478c75e-aa84-b1b6-2c10-4f09a6bcd070: entering follower state: follower="Node at a478c75e-aa84-b1b6-2c10-4f09a6bcd070 [Follower]" leader= + testing.go:149: server-a478c75e-aa84-b1b6-2c10-4f09a6bcd070: 2024-06-12T04:32:27.557+1400 [INFO] server-a478c75e-aa84-b1b6-2c10-4f09a6bcd070: entering follower state: follower="Node at a478c75e-aa84-b1b6-2c10-4f09a6bcd070 [Follower]" leader= +testLoggerAdapter verbose: server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: 2024-06-12T04:32:27.624+1400 [WARN] server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: 2024-06-12T04:32:27.624+1400 [WARN] server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: 2024-06-12T04:32:27.624+1400 [INFO] server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: entering candidate state: node="Node at edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09 [Candidate]" term=2 + testing.go:149: server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: 2024-06-12T04:32:27.624+1400 [INFO] server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: entering candidate state: node="Node at edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09 [Candidate]" term=2 +testLoggerAdapter verbose: server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: 2024-06-12T04:32:27.624+1400 [INFO] server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: election won: tally=2 + testing.go:149: server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: 2024-06-12T04:32:27.624+1400 [INFO] server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: election won: tally=2 +testLoggerAdapter verbose: server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: 2024-06-12T04:32:27.624+1400 [INFO] server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: entering leader state: leader="Node at edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09 [Leader]" + testing.go:149: server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: 2024-06-12T04:32:27.624+1400 [INFO] server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: entering leader state: leader="Node at edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09 [Leader]" +testLoggerAdapter verbose: server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: 2024-06-12T04:32:27.624+1400 [INFO] server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: added peer, starting replication: peer=server-458cf6b8-fd2f-3e20-0e83-adcf4b9f63b8 + testing.go:149: server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: 2024-06-12T04:32:27.624+1400 [INFO] server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: added peer, starting replication: peer=server-458cf6b8-fd2f-3e20-0e83-adcf4b9f63b8 +testLoggerAdapter verbose: server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: 2024-06-12T04:32:27.624+1400 [INFO] server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: pipelining replication: peer="{Voter server-458cf6b8-fd2f-3e20-0e83-adcf4b9f63b8 458cf6b8-fd2f-3e20-0e83-adcf4b9f63b8}" + testing.go:149: server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: 2024-06-12T04:32:27.624+1400 [INFO] server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: pipelining replication: peer="{Voter server-458cf6b8-fd2f-3e20-0e83-adcf4b9f63b8 458cf6b8-fd2f-3e20-0e83-adcf4b9f63b8}" +testLoggerAdapter verbose: server-a478c75e-aa84-b1b6-2c10-4f09a6bcd070: 2024-06-12T04:32:27.639+1400 [WARN] server-a478c75e-aa84-b1b6-2c10-4f09a6bcd070: no known peers, aborting election + testing.go:149: server-a478c75e-aa84-b1b6-2c10-4f09a6bcd070: 2024-06-12T04:32:27.639+1400 [WARN] server-a478c75e-aa84-b1b6-2c10-4f09a6bcd070: no known peers, aborting election +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:27.730+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:27.624694284 +1400 +14 m=+4.532212863 (1 nodes), 67.598459ms from start of poll, 68.553985ms from cluster start. Timeout at 2024-06-12 04:32:27.729974352 +1400 +14 m=+4.637492931, 105.280068ms after stability + testing.go:149: cluster: 2024-06-12T04:32:27.730+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:27.624694284 +1400 +14 m=+4.532212863 (1 nodes), 67.598459ms from start of poll, 68.553985ms from cluster start. Timeout at 2024-06-12 04:32:27.729974352 +1400 +14 m=+4.637492931, 105.280068ms after stability +testLoggerAdapter verbose: server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: 2024-06-12T04:32:27.730+1400 [INFO] server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: updating configuration: command=AddStaging server-id=server-a478c75e-aa84-b1b6-2c10-4f09a6bcd070 server-addr=a478c75e-aa84-b1b6-2c10-4f09a6bcd070 servers="[{Suffrage:Voter ID:server-458cf6b8-fd2f-3e20-0e83-adcf4b9f63b8 Address:458cf6b8-fd2f-3e20-0e83-adcf4b9f63b8} {Suffrage:Voter ID:server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09 Address:edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09} {Suffrage:Voter ID:server-a478c75e-aa84-b1b6-2c10-4f09a6bcd070 Address:a478c75e-aa84-b1b6-2c10-4f09a6bcd070}]" + testing.go:149: server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: 2024-06-12T04:32:27.730+1400 [INFO] server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: updating configuration: command=AddStaging server-id=server-a478c75e-aa84-b1b6-2c10-4f09a6bcd070 server-addr=a478c75e-aa84-b1b6-2c10-4f09a6bcd070 servers="[{Suffrage:Voter ID:server-458cf6b8-fd2f-3e20-0e83-adcf4b9f63b8 Address:458cf6b8-fd2f-3e20-0e83-adcf4b9f63b8} {Suffrage:Voter ID:server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09 Address:edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09} {Suffrage:Voter ID:server-a478c75e-aa84-b1b6-2c10-4f09a6bcd070 Address:a478c75e-aa84-b1b6-2c10-4f09a6bcd070}]" +testLoggerAdapter verbose: server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: 2024-06-12T04:32:27.730+1400 [INFO] server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: added peer, starting replication: peer=server-a478c75e-aa84-b1b6-2c10-4f09a6bcd070 + testing.go:149: server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: 2024-06-12T04:32:27.730+1400 [INFO] server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: added peer, starting replication: peer=server-a478c75e-aa84-b1b6-2c10-4f09a6bcd070 +testLoggerAdapter verbose: server-a478c75e-aa84-b1b6-2c10-4f09a6bcd070: 2024-06-12T04:32:27.730+1400 [WARN] server-a478c75e-aa84-b1b6-2c10-4f09a6bcd070: failed to get previous log: previous-index=3 last-index=0 error="log not found" + testing.go:149: server-a478c75e-aa84-b1b6-2c10-4f09a6bcd070: 2024-06-12T04:32:27.730+1400 [WARN] server-a478c75e-aa84-b1b6-2c10-4f09a6bcd070: failed to get previous log: previous-index=3 last-index=0 error="log not found" +testLoggerAdapter verbose: server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: 2024-06-12T04:32:27.730+1400 [WARN] server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: appendEntries rejected, sending older logs: peer="{Voter server-a478c75e-aa84-b1b6-2c10-4f09a6bcd070 a478c75e-aa84-b1b6-2c10-4f09a6bcd070}" next=1 + testing.go:149: server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: 2024-06-12T04:32:27.730+1400 [WARN] server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: appendEntries rejected, sending older logs: peer="{Voter server-a478c75e-aa84-b1b6-2c10-4f09a6bcd070 a478c75e-aa84-b1b6-2c10-4f09a6bcd070}" next=1 +testLoggerAdapter verbose: server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: 2024-06-12T04:32:27.731+1400 [INFO] server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: pipelining replication: peer="{Voter server-a478c75e-aa84-b1b6-2c10-4f09a6bcd070 a478c75e-aa84-b1b6-2c10-4f09a6bcd070}" + testing.go:149: server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: 2024-06-12T04:32:27.731+1400 [INFO] server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: pipelining replication: peer="{Voter server-a478c75e-aa84-b1b6-2c10-4f09a6bcd070 a478c75e-aa84-b1b6-2c10-4f09a6bcd070}" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:27.731+1400 [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-06-12T04:32:27.731+1400 [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:27.836+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:27.731392342 +1400 +14 m=+4.638910921 (1 nodes), 1.32µs from start of poll, 175.252043ms from cluster start. Timeout at 2024-06-12 04:32:27.836715969 +1400 +14 m=+4.744234548, 105.323627ms after stability + testing.go:149: cluster: 2024-06-12T04:32:27.836+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:27.731392342 +1400 +14 m=+4.638910921 (1 nodes), 1.32µs from start of poll, 175.252043ms from cluster start. Timeout at 2024-06-12 04:32:27.836715969 +1400 +14 m=+4.744234548, 105.323627ms after stability +testLoggerAdapter verbose: server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: 2024-06-12T04:32:27.836+1400 [INFO] server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: aborting pipeline replication: peer="{Voter server-a478c75e-aa84-b1b6-2c10-4f09a6bcd070 a478c75e-aa84-b1b6-2c10-4f09a6bcd070}" + testing.go:149: server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: 2024-06-12T04:32:27.836+1400 [INFO] server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: aborting pipeline replication: peer="{Voter server-a478c75e-aa84-b1b6-2c10-4f09a6bcd070 a478c75e-aa84-b1b6-2c10-4f09a6bcd070}" +testLoggerAdapter verbose: server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: 2024-06-12T04:32:27.836+1400 [INFO] server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: aborting pipeline replication: peer="{Voter server-458cf6b8-fd2f-3e20-0e83-adcf4b9f63b8 458cf6b8-fd2f-3e20-0e83-adcf4b9f63b8}" + testing.go:149: server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: 2024-06-12T04:32:27.836+1400 [INFO] server-edc3da8f-77a9-1a3d-b6e9-cf6da7a7ac09: aborting pipeline replication: peer="{Voter server-458cf6b8-fd2f-3e20-0e83-adcf4b9f63b8 458cf6b8-fd2f-3e20-0e83-adcf4b9f63b8}" +--- PASS: TestRaft_HasExistingState (0.28s) === RUN TestRaft_SingleNode -testLoggerAdapter verbose: server-ef9f4f0a-14af-1a6f-2693-cfbf5199c4ed: 2024-06-11T02:29:14.457-1200 [INFO] server-ef9f4f0a-14af-1a6f-2693-cfbf5199c4ed: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ef9f4f0a-14af-1a6f-2693-cfbf5199c4ed Address:ef9f4f0a-14af-1a6f-2693-cfbf5199c4ed}]" - testing.go:149: server-ef9f4f0a-14af-1a6f-2693-cfbf5199c4ed: 2024-06-11T02:29:14.457-1200 [INFO] server-ef9f4f0a-14af-1a6f-2693-cfbf5199c4ed: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ef9f4f0a-14af-1a6f-2693-cfbf5199c4ed Address:ef9f4f0a-14af-1a6f-2693-cfbf5199c4ed}]" -testLoggerAdapter verbose: server-ef9f4f0a-14af-1a6f-2693-cfbf5199c4ed: 2024-06-11T02:29:14.458-1200 [INFO] server-ef9f4f0a-14af-1a6f-2693-cfbf5199c4ed: entering follower state: follower="Node at ef9f4f0a-14af-1a6f-2693-cfbf5199c4ed [Follower]" leader= - testing.go:149: server-ef9f4f0a-14af-1a6f-2693-cfbf5199c4ed: 2024-06-11T02:29:14.458-1200 [INFO] server-ef9f4f0a-14af-1a6f-2693-cfbf5199c4ed: entering follower state: follower="Node at ef9f4f0a-14af-1a6f-2693-cfbf5199c4ed [Follower]" leader= -testLoggerAdapter verbose: server-ef9f4f0a-14af-1a6f-2693-cfbf5199c4ed: 2024-06-11T02:29:14.515-1200 [WARN] server-ef9f4f0a-14af-1a6f-2693-cfbf5199c4ed: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-ef9f4f0a-14af-1a6f-2693-cfbf5199c4ed: 2024-06-11T02:29:14.515-1200 [WARN] server-ef9f4f0a-14af-1a6f-2693-cfbf5199c4ed: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-ef9f4f0a-14af-1a6f-2693-cfbf5199c4ed: 2024-06-11T02:29:14.515-1200 [INFO] server-ef9f4f0a-14af-1a6f-2693-cfbf5199c4ed: entering candidate state: node="Node at ef9f4f0a-14af-1a6f-2693-cfbf5199c4ed [Candidate]" term=2 - testing.go:149: server-ef9f4f0a-14af-1a6f-2693-cfbf5199c4ed: 2024-06-11T02:29:14.515-1200 [INFO] server-ef9f4f0a-14af-1a6f-2693-cfbf5199c4ed: entering candidate state: node="Node at ef9f4f0a-14af-1a6f-2693-cfbf5199c4ed [Candidate]" term=2 -testLoggerAdapter verbose: server-ef9f4f0a-14af-1a6f-2693-cfbf5199c4ed: 2024-06-11T02:29:14.516-1200 [INFO] server-ef9f4f0a-14af-1a6f-2693-cfbf5199c4ed: election won: tally=1 - testing.go:149: server-ef9f4f0a-14af-1a6f-2693-cfbf5199c4ed: 2024-06-11T02:29:14.516-1200 [INFO] server-ef9f4f0a-14af-1a6f-2693-cfbf5199c4ed: election won: tally=1 -testLoggerAdapter verbose: server-ef9f4f0a-14af-1a6f-2693-cfbf5199c4ed: 2024-06-11T02:29:14.516-1200 [INFO] server-ef9f4f0a-14af-1a6f-2693-cfbf5199c4ed: entering leader state: leader="Node at ef9f4f0a-14af-1a6f-2693-cfbf5199c4ed [Leader]" - testing.go:149: server-ef9f4f0a-14af-1a6f-2693-cfbf5199c4ed: 2024-06-11T02:29:14.516-1200 [INFO] server-ef9f4f0a-14af-1a6f-2693-cfbf5199c4ed: entering leader state: leader="Node at ef9f4f0a-14af-1a6f-2693-cfbf5199c4ed [Leader]" ---- PASS: TestRaft_SingleNode (0.06s) +testLoggerAdapter verbose: server-0704f701-febd-f2a2-f84b-7c6907c370da: 2024-06-12T04:32:27.838+1400 [INFO] server-0704f701-febd-f2a2-f84b-7c6907c370da: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0704f701-febd-f2a2-f84b-7c6907c370da Address:0704f701-febd-f2a2-f84b-7c6907c370da}]" + testing.go:149: server-0704f701-febd-f2a2-f84b-7c6907c370da: 2024-06-12T04:32:27.838+1400 [INFO] server-0704f701-febd-f2a2-f84b-7c6907c370da: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0704f701-febd-f2a2-f84b-7c6907c370da Address:0704f701-febd-f2a2-f84b-7c6907c370da}]" +testLoggerAdapter verbose: server-0704f701-febd-f2a2-f84b-7c6907c370da: 2024-06-12T04:32:27.838+1400 [INFO] server-0704f701-febd-f2a2-f84b-7c6907c370da: entering follower state: follower="Node at 0704f701-febd-f2a2-f84b-7c6907c370da [Follower]" leader= + testing.go:149: server-0704f701-febd-f2a2-f84b-7c6907c370da: 2024-06-12T04:32:27.838+1400 [INFO] server-0704f701-febd-f2a2-f84b-7c6907c370da: entering follower state: follower="Node at 0704f701-febd-f2a2-f84b-7c6907c370da [Follower]" leader= +testLoggerAdapter verbose: server-0704f701-febd-f2a2-f84b-7c6907c370da: 2024-06-12T04:32:27.903+1400 [WARN] server-0704f701-febd-f2a2-f84b-7c6907c370da: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-0704f701-febd-f2a2-f84b-7c6907c370da: 2024-06-12T04:32:27.903+1400 [WARN] server-0704f701-febd-f2a2-f84b-7c6907c370da: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-0704f701-febd-f2a2-f84b-7c6907c370da: 2024-06-12T04:32:27.903+1400 [INFO] server-0704f701-febd-f2a2-f84b-7c6907c370da: entering candidate state: node="Node at 0704f701-febd-f2a2-f84b-7c6907c370da [Candidate]" term=2 + testing.go:149: server-0704f701-febd-f2a2-f84b-7c6907c370da: 2024-06-12T04:32:27.903+1400 [INFO] server-0704f701-febd-f2a2-f84b-7c6907c370da: entering candidate state: node="Node at 0704f701-febd-f2a2-f84b-7c6907c370da [Candidate]" term=2 +testLoggerAdapter verbose: server-0704f701-febd-f2a2-f84b-7c6907c370da: 2024-06-12T04:32:27.904+1400 [INFO] server-0704f701-febd-f2a2-f84b-7c6907c370da: election won: tally=1 + testing.go:149: server-0704f701-febd-f2a2-f84b-7c6907c370da: 2024-06-12T04:32:27.904+1400 [INFO] server-0704f701-febd-f2a2-f84b-7c6907c370da: election won: tally=1 +testLoggerAdapter verbose: server-0704f701-febd-f2a2-f84b-7c6907c370da: 2024-06-12T04:32:27.904+1400 [INFO] server-0704f701-febd-f2a2-f84b-7c6907c370da: entering leader state: leader="Node at 0704f701-febd-f2a2-f84b-7c6907c370da [Leader]" + testing.go:149: server-0704f701-febd-f2a2-f84b-7c6907c370da: 2024-06-12T04:32:27.904+1400 [INFO] server-0704f701-febd-f2a2-f84b-7c6907c370da: entering leader state: leader="Node at 0704f701-febd-f2a2-f84b-7c6907c370da [Leader]" +--- PASS: TestRaft_SingleNode (0.07s) === RUN TestRaft_TripleNode -testLoggerAdapter verbose: server-95a75c5c-5a17-6333-c5ce-d95aaff4dfe0: 2024-06-11T02:29:14.519-1200 [INFO] server-95a75c5c-5a17-6333-c5ce-d95aaff4dfe0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-95a75c5c-5a17-6333-c5ce-d95aaff4dfe0 Address:95a75c5c-5a17-6333-c5ce-d95aaff4dfe0} {Suffrage:Voter ID:server-efad6d4c-fc20-8f16-c58d-e3551214c4b7 Address:efad6d4c-fc20-8f16-c58d-e3551214c4b7} {Suffrage:Voter ID:server-7f42c859-dbbb-9392-4af5-556fe97c0d7d Address:7f42c859-dbbb-9392-4af5-556fe97c0d7d}]" - testing.go:149: server-95a75c5c-5a17-6333-c5ce-d95aaff4dfe0: 2024-06-11T02:29:14.519-1200 [INFO] server-95a75c5c-5a17-6333-c5ce-d95aaff4dfe0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-95a75c5c-5a17-6333-c5ce-d95aaff4dfe0 Address:95a75c5c-5a17-6333-c5ce-d95aaff4dfe0} {Suffrage:Voter ID:server-efad6d4c-fc20-8f16-c58d-e3551214c4b7 Address:efad6d4c-fc20-8f16-c58d-e3551214c4b7} {Suffrage:Voter ID:server-7f42c859-dbbb-9392-4af5-556fe97c0d7d Address:7f42c859-dbbb-9392-4af5-556fe97c0d7d}]" -testLoggerAdapter verbose: server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: 2024-06-11T02:29:14.520-1200 [INFO] server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-95a75c5c-5a17-6333-c5ce-d95aaff4dfe0 Address:95a75c5c-5a17-6333-c5ce-d95aaff4dfe0} {Suffrage:Voter ID:server-efad6d4c-fc20-8f16-c58d-e3551214c4b7 Address:efad6d4c-fc20-8f16-c58d-e3551214c4b7} {Suffrage:Voter ID:server-7f42c859-dbbb-9392-4af5-556fe97c0d7d Address:7f42c859-dbbb-9392-4af5-556fe97c0d7d}]" - testing.go:149: server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: 2024-06-11T02:29:14.520-1200 [INFO] server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-95a75c5c-5a17-6333-c5ce-d95aaff4dfe0 Address:95a75c5c-5a17-6333-c5ce-d95aaff4dfe0} {Suffrage:Voter ID:server-efad6d4c-fc20-8f16-c58d-e3551214c4b7 Address:efad6d4c-fc20-8f16-c58d-e3551214c4b7} {Suffrage:Voter ID:server-7f42c859-dbbb-9392-4af5-556fe97c0d7d Address:7f42c859-dbbb-9392-4af5-556fe97c0d7d}]" -testLoggerAdapter verbose: server-7f42c859-dbbb-9392-4af5-556fe97c0d7d: 2024-06-11T02:29:14.520-1200 [INFO] server-7f42c859-dbbb-9392-4af5-556fe97c0d7d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-95a75c5c-5a17-6333-c5ce-d95aaff4dfe0 Address:95a75c5c-5a17-6333-c5ce-d95aaff4dfe0} {Suffrage:Voter ID:server-efad6d4c-fc20-8f16-c58d-e3551214c4b7 Address:efad6d4c-fc20-8f16-c58d-e3551214c4b7} {Suffrage:Voter ID:server-7f42c859-dbbb-9392-4af5-556fe97c0d7d Address:7f42c859-dbbb-9392-4af5-556fe97c0d7d}]" - testing.go:149: server-7f42c859-dbbb-9392-4af5-556fe97c0d7d: 2024-06-11T02:29:14.520-1200 [INFO] server-7f42c859-dbbb-9392-4af5-556fe97c0d7d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-95a75c5c-5a17-6333-c5ce-d95aaff4dfe0 Address:95a75c5c-5a17-6333-c5ce-d95aaff4dfe0} {Suffrage:Voter ID:server-efad6d4c-fc20-8f16-c58d-e3551214c4b7 Address:efad6d4c-fc20-8f16-c58d-e3551214c4b7} {Suffrage:Voter ID:server-7f42c859-dbbb-9392-4af5-556fe97c0d7d Address:7f42c859-dbbb-9392-4af5-556fe97c0d7d}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:14.520-1200 [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2024-06-11T02:29:14.520-1200 [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: server-95a75c5c-5a17-6333-c5ce-d95aaff4dfe0: 2024-06-11T02:29:14.520-1200 [INFO] server-95a75c5c-5a17-6333-c5ce-d95aaff4dfe0: entering follower state: follower="Node at 95a75c5c-5a17-6333-c5ce-d95aaff4dfe0 [Follower]" leader= - testing.go:149: server-95a75c5c-5a17-6333-c5ce-d95aaff4dfe0: 2024-06-11T02:29:14.520-1200 [INFO] server-95a75c5c-5a17-6333-c5ce-d95aaff4dfe0: entering follower state: follower="Node at 95a75c5c-5a17-6333-c5ce-d95aaff4dfe0 [Follower]" leader= -testLoggerAdapter verbose: server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: 2024-06-11T02:29:14.520-1200 [INFO] server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: entering follower state: follower="Node at efad6d4c-fc20-8f16-c58d-e3551214c4b7 [Follower]" leader= - testing.go:149: server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: 2024-06-11T02:29:14.520-1200 [INFO] server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: entering follower state: follower="Node at efad6d4c-fc20-8f16-c58d-e3551214c4b7 [Follower]" leader= -testLoggerAdapter verbose: server-7f42c859-dbbb-9392-4af5-556fe97c0d7d: 2024-06-11T02:29:14.521-1200 [INFO] server-7f42c859-dbbb-9392-4af5-556fe97c0d7d: entering follower state: follower="Node at 7f42c859-dbbb-9392-4af5-556fe97c0d7d [Follower]" leader= - testing.go:149: server-7f42c859-dbbb-9392-4af5-556fe97c0d7d: 2024-06-11T02:29:14.521-1200 [INFO] server-7f42c859-dbbb-9392-4af5-556fe97c0d7d: entering follower state: follower="Node at 7f42c859-dbbb-9392-4af5-556fe97c0d7d [Follower]" leader= -testLoggerAdapter verbose: server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: 2024-06-11T02:29:14.587-1200 [WARN] server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: 2024-06-11T02:29:14.587-1200 [WARN] server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: 2024-06-11T02:29:14.587-1200 [INFO] server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: entering candidate state: node="Node at efad6d4c-fc20-8f16-c58d-e3551214c4b7 [Candidate]" term=2 - testing.go:149: server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: 2024-06-11T02:29:14.587-1200 [INFO] server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: entering candidate state: node="Node at efad6d4c-fc20-8f16-c58d-e3551214c4b7 [Candidate]" term=2 -testLoggerAdapter verbose: server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: 2024-06-11T02:29:14.587-1200 [INFO] server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: election won: tally=2 - testing.go:149: server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: 2024-06-11T02:29:14.587-1200 [INFO] server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: election won: tally=2 -testLoggerAdapter verbose: server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: 2024-06-11T02:29:14.587-1200 [INFO] server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: entering leader state: leader="Node at efad6d4c-fc20-8f16-c58d-e3551214c4b7 [Leader]" - testing.go:149: server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: 2024-06-11T02:29:14.587-1200 [INFO] server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: entering leader state: leader="Node at efad6d4c-fc20-8f16-c58d-e3551214c4b7 [Leader]" -testLoggerAdapter verbose: server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: 2024-06-11T02:29:14.587-1200 [INFO] server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: added peer, starting replication: peer=server-95a75c5c-5a17-6333-c5ce-d95aaff4dfe0 - testing.go:149: server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: 2024-06-11T02:29:14.587-1200 [INFO] server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: added peer, starting replication: peer=server-95a75c5c-5a17-6333-c5ce-d95aaff4dfe0 -testLoggerAdapter verbose: server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: 2024-06-11T02:29:14.587-1200 [INFO] server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: added peer, starting replication: peer=server-7f42c859-dbbb-9392-4af5-556fe97c0d7d - testing.go:149: server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: 2024-06-11T02:29:14.587-1200 [INFO] server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: added peer, starting replication: peer=server-7f42c859-dbbb-9392-4af5-556fe97c0d7d -testLoggerAdapter verbose: server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: 2024-06-11T02:29:14.588-1200 [INFO] server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: pipelining replication: peer="{Voter server-7f42c859-dbbb-9392-4af5-556fe97c0d7d 7f42c859-dbbb-9392-4af5-556fe97c0d7d}" - testing.go:149: server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: 2024-06-11T02:29:14.588-1200 [INFO] server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: pipelining replication: peer="{Voter server-7f42c859-dbbb-9392-4af5-556fe97c0d7d 7f42c859-dbbb-9392-4af5-556fe97c0d7d}" -testLoggerAdapter verbose: server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: 2024-06-11T02:29:14.588-1200 [INFO] server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: pipelining replication: peer="{Voter server-95a75c5c-5a17-6333-c5ce-d95aaff4dfe0 95a75c5c-5a17-6333-c5ce-d95aaff4dfe0}" - testing.go:149: server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: 2024-06-11T02:29:14.588-1200 [INFO] server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: pipelining replication: peer="{Voter server-95a75c5c-5a17-6333-c5ce-d95aaff4dfe0 95a75c5c-5a17-6333-c5ce-d95aaff4dfe0}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:14.694-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:14.588810615 -1200 -12 m=+5.584183448 (2 nodes), 67.979513ms from start of poll, 69.90856ms from cluster start. Timeout at 2024-06-11 02:29:14.694436586 -1200 -12 m=+5.689809420, 105.625972ms after stability - testing.go:149: cluster: 2024-06-11T02:29:14.694-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:14.588810615 -1200 -12 m=+5.584183448 (2 nodes), 67.979513ms from start of poll, 69.90856ms from cluster start. Timeout at 2024-06-11 02:29:14.694436586 -1200 -12 m=+5.689809420, 105.625972ms after stability -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:14.694-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:14.694-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:14.799-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:14.694887203 -1200 -12 m=+5.690259744 (1 nodes), 3.208µs from start of poll, 175.984856ms from cluster start. Timeout at 2024-06-11 02:29:14.799902436 -1200 -12 m=+5.795274977, 105.015233ms after stability - testing.go:149: cluster: 2024-06-11T02:29:14.799-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:14.694887203 -1200 -12 m=+5.690259744 (1 nodes), 3.208µs from start of poll, 175.984856ms from cluster start. Timeout at 2024-06-11 02:29:14.799902436 -1200 -12 m=+5.795274977, 105.015233ms after stability -testLoggerAdapter verbose: server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: 2024-06-11T02:29:14.811-1200 [INFO] server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: aborting pipeline replication: peer="{Voter server-7f42c859-dbbb-9392-4af5-556fe97c0d7d 7f42c859-dbbb-9392-4af5-556fe97c0d7d}" - testing.go:149: server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: 2024-06-11T02:29:14.811-1200 [INFO] server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: aborting pipeline replication: peer="{Voter server-7f42c859-dbbb-9392-4af5-556fe97c0d7d 7f42c859-dbbb-9392-4af5-556fe97c0d7d}" -testLoggerAdapter verbose: server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: 2024-06-11T02:29:14.812-1200 [INFO] server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: aborting pipeline replication: peer="{Voter server-95a75c5c-5a17-6333-c5ce-d95aaff4dfe0 95a75c5c-5a17-6333-c5ce-d95aaff4dfe0}" - testing.go:149: server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: 2024-06-11T02:29:14.812-1200 [INFO] server-efad6d4c-fc20-8f16-c58d-e3551214c4b7: aborting pipeline replication: peer="{Voter server-95a75c5c-5a17-6333-c5ce-d95aaff4dfe0 95a75c5c-5a17-6333-c5ce-d95aaff4dfe0}" ---- PASS: TestRaft_TripleNode (0.30s) +testLoggerAdapter verbose: server-8a314442-5589-2def-73fa-76ef09960360: 2024-06-12T04:32:27.905+1400 [INFO] server-8a314442-5589-2def-73fa-76ef09960360: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8a314442-5589-2def-73fa-76ef09960360 Address:8a314442-5589-2def-73fa-76ef09960360} {Suffrage:Voter ID:server-b0aadc71-d749-6d6c-4fb2-82d3df3ba0f3 Address:b0aadc71-d749-6d6c-4fb2-82d3df3ba0f3} {Suffrage:Voter ID:server-021c45a8-3e1d-5a00-fe44-4e8eb773ffb6 Address:021c45a8-3e1d-5a00-fe44-4e8eb773ffb6}]" + testing.go:149: server-8a314442-5589-2def-73fa-76ef09960360: 2024-06-12T04:32:27.905+1400 [INFO] server-8a314442-5589-2def-73fa-76ef09960360: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8a314442-5589-2def-73fa-76ef09960360 Address:8a314442-5589-2def-73fa-76ef09960360} {Suffrage:Voter ID:server-b0aadc71-d749-6d6c-4fb2-82d3df3ba0f3 Address:b0aadc71-d749-6d6c-4fb2-82d3df3ba0f3} {Suffrage:Voter ID:server-021c45a8-3e1d-5a00-fe44-4e8eb773ffb6 Address:021c45a8-3e1d-5a00-fe44-4e8eb773ffb6}]" +testLoggerAdapter verbose: server-b0aadc71-d749-6d6c-4fb2-82d3df3ba0f3: 2024-06-12T04:32:27.905+1400 [INFO] server-b0aadc71-d749-6d6c-4fb2-82d3df3ba0f3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8a314442-5589-2def-73fa-76ef09960360 Address:8a314442-5589-2def-73fa-76ef09960360} {Suffrage:Voter ID:server-b0aadc71-d749-6d6c-4fb2-82d3df3ba0f3 Address:b0aadc71-d749-6d6c-4fb2-82d3df3ba0f3} {Suffrage:Voter ID:server-021c45a8-3e1d-5a00-fe44-4e8eb773ffb6 Address:021c45a8-3e1d-5a00-fe44-4e8eb773ffb6}]" + testing.go:149: server-b0aadc71-d749-6d6c-4fb2-82d3df3ba0f3: 2024-06-12T04:32:27.905+1400 [INFO] server-b0aadc71-d749-6d6c-4fb2-82d3df3ba0f3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8a314442-5589-2def-73fa-76ef09960360 Address:8a314442-5589-2def-73fa-76ef09960360} {Suffrage:Voter ID:server-b0aadc71-d749-6d6c-4fb2-82d3df3ba0f3 Address:b0aadc71-d749-6d6c-4fb2-82d3df3ba0f3} {Suffrage:Voter ID:server-021c45a8-3e1d-5a00-fe44-4e8eb773ffb6 Address:021c45a8-3e1d-5a00-fe44-4e8eb773ffb6}]" +testLoggerAdapter verbose: server-021c45a8-3e1d-5a00-fe44-4e8eb773ffb6: 2024-06-12T04:32:27.906+1400 [INFO] server-021c45a8-3e1d-5a00-fe44-4e8eb773ffb6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8a314442-5589-2def-73fa-76ef09960360 Address:8a314442-5589-2def-73fa-76ef09960360} {Suffrage:Voter ID:server-b0aadc71-d749-6d6c-4fb2-82d3df3ba0f3 Address:b0aadc71-d749-6d6c-4fb2-82d3df3ba0f3} {Suffrage:Voter ID:server-021c45a8-3e1d-5a00-fe44-4e8eb773ffb6 Address:021c45a8-3e1d-5a00-fe44-4e8eb773ffb6}]" + testing.go:149: server-021c45a8-3e1d-5a00-fe44-4e8eb773ffb6: 2024-06-12T04:32:27.906+1400 [INFO] server-021c45a8-3e1d-5a00-fe44-4e8eb773ffb6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8a314442-5589-2def-73fa-76ef09960360 Address:8a314442-5589-2def-73fa-76ef09960360} {Suffrage:Voter ID:server-b0aadc71-d749-6d6c-4fb2-82d3df3ba0f3 Address:b0aadc71-d749-6d6c-4fb2-82d3df3ba0f3} {Suffrage:Voter ID:server-021c45a8-3e1d-5a00-fe44-4e8eb773ffb6 Address:021c45a8-3e1d-5a00-fe44-4e8eb773ffb6}]" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:27.906+1400 [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-06-12T04:32:27.906+1400 [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: server-8a314442-5589-2def-73fa-76ef09960360: 2024-06-12T04:32:27.906+1400 [INFO] server-8a314442-5589-2def-73fa-76ef09960360: entering follower state: follower="Node at 8a314442-5589-2def-73fa-76ef09960360 [Follower]" leader= + testing.go:149: server-8a314442-5589-2def-73fa-76ef09960360: 2024-06-12T04:32:27.906+1400 [INFO] server-8a314442-5589-2def-73fa-76ef09960360: entering follower state: follower="Node at 8a314442-5589-2def-73fa-76ef09960360 [Follower]" leader= +testLoggerAdapter verbose: server-b0aadc71-d749-6d6c-4fb2-82d3df3ba0f3: 2024-06-12T04:32:27.906+1400 [INFO] server-b0aadc71-d749-6d6c-4fb2-82d3df3ba0f3: entering follower state: follower="Node at b0aadc71-d749-6d6c-4fb2-82d3df3ba0f3 [Follower]" leader= + testing.go:149: server-b0aadc71-d749-6d6c-4fb2-82d3df3ba0f3: 2024-06-12T04:32:27.906+1400 [INFO] server-b0aadc71-d749-6d6c-4fb2-82d3df3ba0f3: entering follower state: follower="Node at b0aadc71-d749-6d6c-4fb2-82d3df3ba0f3 [Follower]" leader= +testLoggerAdapter verbose: server-021c45a8-3e1d-5a00-fe44-4e8eb773ffb6: 2024-06-12T04:32:27.906+1400 [INFO] server-021c45a8-3e1d-5a00-fe44-4e8eb773ffb6: entering follower state: follower="Node at 021c45a8-3e1d-5a00-fe44-4e8eb773ffb6 [Follower]" leader= + testing.go:149: server-021c45a8-3e1d-5a00-fe44-4e8eb773ffb6: 2024-06-12T04:32:27.906+1400 [INFO] server-021c45a8-3e1d-5a00-fe44-4e8eb773ffb6: entering follower state: follower="Node at 021c45a8-3e1d-5a00-fe44-4e8eb773ffb6 [Follower]" leader= +testLoggerAdapter verbose: server-8a314442-5589-2def-73fa-76ef09960360: 2024-06-12T04:32:27.966+1400 [WARN] server-8a314442-5589-2def-73fa-76ef09960360: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-8a314442-5589-2def-73fa-76ef09960360: 2024-06-12T04:32:27.966+1400 [WARN] server-8a314442-5589-2def-73fa-76ef09960360: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-8a314442-5589-2def-73fa-76ef09960360: 2024-06-12T04:32:27.966+1400 [INFO] server-8a314442-5589-2def-73fa-76ef09960360: entering candidate state: node="Node at 8a314442-5589-2def-73fa-76ef09960360 [Candidate]" term=2 + testing.go:149: server-8a314442-5589-2def-73fa-76ef09960360: 2024-06-12T04:32:27.966+1400 [INFO] server-8a314442-5589-2def-73fa-76ef09960360: entering candidate state: node="Node at 8a314442-5589-2def-73fa-76ef09960360 [Candidate]" term=2 +testLoggerAdapter verbose: server-8a314442-5589-2def-73fa-76ef09960360: 2024-06-12T04:32:27.966+1400 [INFO] server-8a314442-5589-2def-73fa-76ef09960360: election won: tally=2 + testing.go:149: server-8a314442-5589-2def-73fa-76ef09960360: 2024-06-12T04:32:27.966+1400 [INFO] server-8a314442-5589-2def-73fa-76ef09960360: election won: tally=2 +testLoggerAdapter verbose: server-8a314442-5589-2def-73fa-76ef09960360: 2024-06-12T04:32:27.966+1400 [INFO] server-8a314442-5589-2def-73fa-76ef09960360: entering leader state: leader="Node at 8a314442-5589-2def-73fa-76ef09960360 [Leader]" + testing.go:149: server-8a314442-5589-2def-73fa-76ef09960360: 2024-06-12T04:32:27.966+1400 [INFO] server-8a314442-5589-2def-73fa-76ef09960360: entering leader state: leader="Node at 8a314442-5589-2def-73fa-76ef09960360 [Leader]" +testLoggerAdapter verbose: server-8a314442-5589-2def-73fa-76ef09960360: 2024-06-12T04:32:27.966+1400 [INFO] server-8a314442-5589-2def-73fa-76ef09960360: added peer, starting replication: peer=server-b0aadc71-d749-6d6c-4fb2-82d3df3ba0f3 + testing.go:149: server-8a314442-5589-2def-73fa-76ef09960360: 2024-06-12T04:32:27.966+1400 [INFO] server-8a314442-5589-2def-73fa-76ef09960360: added peer, starting replication: peer=server-b0aadc71-d749-6d6c-4fb2-82d3df3ba0f3 +testLoggerAdapter verbose: server-8a314442-5589-2def-73fa-76ef09960360: 2024-06-12T04:32:27.966+1400 [INFO] server-8a314442-5589-2def-73fa-76ef09960360: added peer, starting replication: peer=server-021c45a8-3e1d-5a00-fe44-4e8eb773ffb6 + testing.go:149: server-8a314442-5589-2def-73fa-76ef09960360: 2024-06-12T04:32:27.966+1400 [INFO] server-8a314442-5589-2def-73fa-76ef09960360: added peer, starting replication: peer=server-021c45a8-3e1d-5a00-fe44-4e8eb773ffb6 +testLoggerAdapter verbose: server-8a314442-5589-2def-73fa-76ef09960360: 2024-06-12T04:32:27.966+1400 [INFO] server-8a314442-5589-2def-73fa-76ef09960360: pipelining replication: peer="{Voter server-021c45a8-3e1d-5a00-fe44-4e8eb773ffb6 021c45a8-3e1d-5a00-fe44-4e8eb773ffb6}" + testing.go:149: server-8a314442-5589-2def-73fa-76ef09960360: 2024-06-12T04:32:27.966+1400 [INFO] server-8a314442-5589-2def-73fa-76ef09960360: pipelining replication: peer="{Voter server-021c45a8-3e1d-5a00-fe44-4e8eb773ffb6 021c45a8-3e1d-5a00-fe44-4e8eb773ffb6}" +testLoggerAdapter verbose: server-8a314442-5589-2def-73fa-76ef09960360: 2024-06-12T04:32:27.966+1400 [INFO] server-8a314442-5589-2def-73fa-76ef09960360: pipelining replication: peer="{Voter server-b0aadc71-d749-6d6c-4fb2-82d3df3ba0f3 b0aadc71-d749-6d6c-4fb2-82d3df3ba0f3}" + testing.go:149: server-8a314442-5589-2def-73fa-76ef09960360: 2024-06-12T04:32:27.966+1400 [INFO] server-8a314442-5589-2def-73fa-76ef09960360: pipelining replication: peer="{Voter server-b0aadc71-d749-6d6c-4fb2-82d3df3ba0f3 b0aadc71-d749-6d6c-4fb2-82d3df3ba0f3}" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:28.072+1400 [INFO] stable state for Follower reached at 2024-06-12 04:32:27.966869347 +1400 +14 m=+4.874387926 (2 nodes), 60.618269ms from start of poll, 61.299644ms from cluster start. Timeout at 2024-06-12 04:32:28.072165216 +1400 +14 m=+4.979683835, 105.295909ms after stability + testing.go:149: cluster: 2024-06-12T04:32:28.072+1400 [INFO] stable state for Follower reached at 2024-06-12 04:32:27.966869347 +1400 +14 m=+4.874387926 (2 nodes), 60.618269ms from start of poll, 61.299644ms from cluster start. Timeout at 2024-06-12 04:32:28.072165216 +1400 +14 m=+4.979683835, 105.295909ms after stability +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:28.072+1400 [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-06-12T04:32:28.072+1400 [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:28.177+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:28.072362169 +1400 +14 m=+4.979880748 (1 nodes), 1.52µs from start of poll, 166.792466ms from cluster start. Timeout at 2024-06-12 04:32:28.177877549 +1400 +14 m=+5.085396128, 105.51538ms after stability + testing.go:149: cluster: 2024-06-12T04:32:28.177+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:28.072362169 +1400 +14 m=+4.979880748 (1 nodes), 1.52µs from start of poll, 166.792466ms from cluster start. Timeout at 2024-06-12 04:32:28.177877549 +1400 +14 m=+5.085396128, 105.51538ms after stability +testLoggerAdapter verbose: server-8a314442-5589-2def-73fa-76ef09960360: 2024-06-12T04:32:28.189+1400 [INFO] server-8a314442-5589-2def-73fa-76ef09960360: aborting pipeline replication: peer="{Voter server-021c45a8-3e1d-5a00-fe44-4e8eb773ffb6 021c45a8-3e1d-5a00-fe44-4e8eb773ffb6}" + testing.go:149: server-8a314442-5589-2def-73fa-76ef09960360: 2024-06-12T04:32:28.189+1400 [INFO] server-8a314442-5589-2def-73fa-76ef09960360: aborting pipeline replication: peer="{Voter server-021c45a8-3e1d-5a00-fe44-4e8eb773ffb6 021c45a8-3e1d-5a00-fe44-4e8eb773ffb6}" +testLoggerAdapter verbose: server-8a314442-5589-2def-73fa-76ef09960360: 2024-06-12T04:32:28.189+1400 [INFO] server-8a314442-5589-2def-73fa-76ef09960360: aborting pipeline replication: peer="{Voter server-b0aadc71-d749-6d6c-4fb2-82d3df3ba0f3 b0aadc71-d749-6d6c-4fb2-82d3df3ba0f3}" + testing.go:149: server-8a314442-5589-2def-73fa-76ef09960360: 2024-06-12T04:32:28.189+1400 [INFO] server-8a314442-5589-2def-73fa-76ef09960360: aborting pipeline replication: peer="{Voter server-b0aadc71-d749-6d6c-4fb2-82d3df3ba0f3 b0aadc71-d749-6d6c-4fb2-82d3df3ba0f3}" +--- PASS: TestRaft_TripleNode (0.29s) === RUN TestRaft_LeaderFail -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:14.819-1200 [INFO] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5a06e4a1-a619-73f1-3519-d47fddb9f320 Address:5a06e4a1-a619-73f1-3519-d47fddb9f320} {Suffrage:Voter ID:server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f Address:e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f} {Suffrage:Voter ID:server-9f3e2a6d-8959-b6de-e3e0-8487352c715f Address:9f3e2a6d-8959-b6de-e3e0-8487352c715f}]" - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:14.819-1200 [INFO] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5a06e4a1-a619-73f1-3519-d47fddb9f320 Address:5a06e4a1-a619-73f1-3519-d47fddb9f320} {Suffrage:Voter ID:server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f Address:e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f} {Suffrage:Voter ID:server-9f3e2a6d-8959-b6de-e3e0-8487352c715f Address:9f3e2a6d-8959-b6de-e3e0-8487352c715f}]" -testLoggerAdapter verbose: server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: 2024-06-11T02:29:14.819-1200 [INFO] server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5a06e4a1-a619-73f1-3519-d47fddb9f320 Address:5a06e4a1-a619-73f1-3519-d47fddb9f320} {Suffrage:Voter ID:server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f Address:e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f} {Suffrage:Voter ID:server-9f3e2a6d-8959-b6de-e3e0-8487352c715f Address:9f3e2a6d-8959-b6de-e3e0-8487352c715f}]" - testing.go:149: server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: 2024-06-11T02:29:14.819-1200 [INFO] server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5a06e4a1-a619-73f1-3519-d47fddb9f320 Address:5a06e4a1-a619-73f1-3519-d47fddb9f320} {Suffrage:Voter ID:server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f Address:e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f} {Suffrage:Voter ID:server-9f3e2a6d-8959-b6de-e3e0-8487352c715f Address:9f3e2a6d-8959-b6de-e3e0-8487352c715f}]" -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:14.820-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5a06e4a1-a619-73f1-3519-d47fddb9f320 Address:5a06e4a1-a619-73f1-3519-d47fddb9f320} {Suffrage:Voter ID:server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f Address:e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f} {Suffrage:Voter ID:server-9f3e2a6d-8959-b6de-e3e0-8487352c715f Address:9f3e2a6d-8959-b6de-e3e0-8487352c715f}]" - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:14.820-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5a06e4a1-a619-73f1-3519-d47fddb9f320 Address:5a06e4a1-a619-73f1-3519-d47fddb9f320} {Suffrage:Voter ID:server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f Address:e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f} {Suffrage:Voter ID:server-9f3e2a6d-8959-b6de-e3e0-8487352c715f Address:9f3e2a6d-8959-b6de-e3e0-8487352c715f}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:14.820-1200 [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2024-06-11T02:29:14.820-1200 [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:14.820-1200 [INFO] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: entering follower state: follower="Node at 5a06e4a1-a619-73f1-3519-d47fddb9f320 [Follower]" leader= - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:14.820-1200 [INFO] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: entering follower state: follower="Node at 5a06e4a1-a619-73f1-3519-d47fddb9f320 [Follower]" leader= -testLoggerAdapter verbose: server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: 2024-06-11T02:29:14.820-1200 [INFO] server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: entering follower state: follower="Node at e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f [Follower]" leader= - testing.go:149: server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: 2024-06-11T02:29:14.820-1200 [INFO] server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: entering follower state: follower="Node at e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f [Follower]" leader= -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:14.820-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: entering follower state: follower="Node at 9f3e2a6d-8959-b6de-e3e0-8487352c715f [Follower]" leader= - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:14.820-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: entering follower state: follower="Node at 9f3e2a6d-8959-b6de-e3e0-8487352c715f [Follower]" leader= -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:14.891-1200 [WARN] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:14.891-1200 [WARN] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:14.892-1200 [INFO] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: entering candidate state: node="Node at 5a06e4a1-a619-73f1-3519-d47fddb9f320 [Candidate]" term=2 - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:14.892-1200 [INFO] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: entering candidate state: node="Node at 5a06e4a1-a619-73f1-3519-d47fddb9f320 [Candidate]" term=2 -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:14.892-1200 [INFO] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: election won: tally=2 - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:14.892-1200 [INFO] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: election won: tally=2 -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:14.892-1200 [INFO] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: entering leader state: leader="Node at 5a06e4a1-a619-73f1-3519-d47fddb9f320 [Leader]" - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:14.892-1200 [INFO] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: entering leader state: leader="Node at 5a06e4a1-a619-73f1-3519-d47fddb9f320 [Leader]" -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:14.892-1200 [INFO] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: added peer, starting replication: peer=server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:14.892-1200 [INFO] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: added peer, starting replication: peer=server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:14.892-1200 [INFO] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: added peer, starting replication: peer=server-9f3e2a6d-8959-b6de-e3e0-8487352c715f - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:14.892-1200 [INFO] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: added peer, starting replication: peer=server-9f3e2a6d-8959-b6de-e3e0-8487352c715f -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:14.893-1200 [INFO] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: pipelining replication: peer="{Voter server-9f3e2a6d-8959-b6de-e3e0-8487352c715f 9f3e2a6d-8959-b6de-e3e0-8487352c715f}" - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:14.893-1200 [INFO] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: pipelining replication: peer="{Voter server-9f3e2a6d-8959-b6de-e3e0-8487352c715f 9f3e2a6d-8959-b6de-e3e0-8487352c715f}" -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:14.893-1200 [INFO] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: pipelining replication: peer="{Voter server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f}" - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:14.893-1200 [INFO] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: pipelining replication: peer="{Voter server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:14.999-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:14.893795461 -1200 -12 m=+5.889168003 (2 nodes), 73.49541ms from start of poll, 74.97909ms from cluster start. Timeout at 2024-06-11 02:29:14.999183145 -1200 -12 m=+5.994555978, 105.387975ms after stability - testing.go:149: cluster: 2024-06-11T02:29:14.999-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:14.893795461 -1200 -12 m=+5.889168003 (2 nodes), 73.49541ms from start of poll, 74.97909ms from cluster start. Timeout at 2024-06-11 02:29:14.999183145 -1200 -12 m=+5.994555978, 105.387975ms after stability -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:14.999-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:14.999-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:15.105-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:14.999467223 -1200 -12 m=+5.994839765 (1 nodes), 2.625µs from start of poll, 180.650852ms from cluster start. Timeout at 2024-06-11 02:29:15.105085903 -1200 -12 m=+6.100458444, 105.618679ms after stability - testing.go:149: cluster: 2024-06-11T02:29:15.105-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:14.999467223 -1200 -12 m=+5.994839765 (1 nodes), 2.625µs from start of poll, 180.650852ms from cluster start. Timeout at 2024-06-11 02:29:15.105085903 -1200 -12 m=+6.100458444, 105.618679ms after stability - raft_test.go:336: [INFO] Disconnecting Node at 5a06e4a1-a619-73f1-3519-d47fddb9f320 [Leader] -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.118-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to heartbeat to: peer=9f3e2a6d-8959-b6de-e3e0-8487352c715f error="failed to connect to peer: 9f3e2a6d-8959-b6de-e3e0-8487352c715f" - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.118-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to heartbeat to: peer=9f3e2a6d-8959-b6de-e3e0-8487352c715f error="failed to connect to peer: 9f3e2a6d-8959-b6de-e3e0-8487352c715f" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:15.123-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:15.123-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.123-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to heartbeat to: peer=e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f error="failed to connect to peer: e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f" - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.123-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to heartbeat to: peer=e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f error="failed to connect to peer: e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f" -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.123-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to pipeline appendEntries: peer="{Voter server-9f3e2a6d-8959-b6de-e3e0-8487352c715f 9f3e2a6d-8959-b6de-e3e0-8487352c715f}" error="append pipeline closed" - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.123-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to pipeline appendEntries: peer="{Voter server-9f3e2a6d-8959-b6de-e3e0-8487352c715f 9f3e2a6d-8959-b6de-e3e0-8487352c715f}" error="append pipeline closed" -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.123-1200 [INFO] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: aborting pipeline replication: peer="{Voter server-9f3e2a6d-8959-b6de-e3e0-8487352c715f 9f3e2a6d-8959-b6de-e3e0-8487352c715f}" - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.123-1200 [INFO] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: aborting pipeline replication: peer="{Voter server-9f3e2a6d-8959-b6de-e3e0-8487352c715f 9f3e2a6d-8959-b6de-e3e0-8487352c715f}" -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.126-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to pipeline appendEntries: peer="{Voter server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f}" error="append pipeline closed" - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.126-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to pipeline appendEntries: peer="{Voter server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f}" error="append pipeline closed" -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.127-1200 [INFO] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: aborting pipeline replication: peer="{Voter server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f}" - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.127-1200 [INFO] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: aborting pipeline replication: peer="{Voter server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f}" -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.132-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to appendEntries to: peer="{Voter server-9f3e2a6d-8959-b6de-e3e0-8487352c715f 9f3e2a6d-8959-b6de-e3e0-8487352c715f}" error="failed to connect to peer: 9f3e2a6d-8959-b6de-e3e0-8487352c715f" - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.132-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to appendEntries to: peer="{Voter server-9f3e2a6d-8959-b6de-e3e0-8487352c715f 9f3e2a6d-8959-b6de-e3e0-8487352c715f}" error="failed to connect to peer: 9f3e2a6d-8959-b6de-e3e0-8487352c715f" -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.135-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to appendEntries to: peer="{Voter server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f}" error="failed to connect to peer: e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f" - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.135-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to appendEntries to: peer="{Voter server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f}" error="failed to connect to peer: e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f" -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.138-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to heartbeat to: peer=9f3e2a6d-8959-b6de-e3e0-8487352c715f error="failed to connect to peer: 9f3e2a6d-8959-b6de-e3e0-8487352c715f" - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.138-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to heartbeat to: peer=9f3e2a6d-8959-b6de-e3e0-8487352c715f error="failed to connect to peer: 9f3e2a6d-8959-b6de-e3e0-8487352c715f" -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.143-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to heartbeat to: peer=e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f error="failed to connect to peer: e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f" - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.143-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to heartbeat to: peer=e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f error="failed to connect to peer: e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f" -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.150-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to appendEntries to: peer="{Voter server-9f3e2a6d-8959-b6de-e3e0-8487352c715f 9f3e2a6d-8959-b6de-e3e0-8487352c715f}" error="failed to connect to peer: 9f3e2a6d-8959-b6de-e3e0-8487352c715f" - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.150-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to appendEntries to: peer="{Voter server-9f3e2a6d-8959-b6de-e3e0-8487352c715f 9f3e2a6d-8959-b6de-e3e0-8487352c715f}" error="failed to connect to peer: 9f3e2a6d-8959-b6de-e3e0-8487352c715f" -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.154-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to appendEntries to: peer="{Voter server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f}" error="failed to connect to peer: e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f" - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.154-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to appendEntries to: peer="{Voter server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f}" error="failed to connect to peer: e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f" -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.156-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to heartbeat to: peer=9f3e2a6d-8959-b6de-e3e0-8487352c715f error="failed to connect to peer: 9f3e2a6d-8959-b6de-e3e0-8487352c715f" - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.156-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to heartbeat to: peer=9f3e2a6d-8959-b6de-e3e0-8487352c715f error="failed to connect to peer: 9f3e2a6d-8959-b6de-e3e0-8487352c715f" -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.164-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to heartbeat to: peer=e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f error="failed to connect to peer: e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f" - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.164-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to heartbeat to: peer=e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f error="failed to connect to peer: e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f" -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.167-1200 [WARN] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to contact: server-id=server-9f3e2a6d-8959-b6de-e3e0-8487352c715f time=51.040123ms - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.167-1200 [WARN] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to contact: server-id=server-9f3e2a6d-8959-b6de-e3e0-8487352c715f time=51.040123ms -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.168-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to appendEntries to: peer="{Voter server-9f3e2a6d-8959-b6de-e3e0-8487352c715f 9f3e2a6d-8959-b6de-e3e0-8487352c715f}" error="failed to connect to peer: 9f3e2a6d-8959-b6de-e3e0-8487352c715f" - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.168-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to appendEntries to: peer="{Voter server-9f3e2a6d-8959-b6de-e3e0-8487352c715f 9f3e2a6d-8959-b6de-e3e0-8487352c715f}" error="failed to connect to peer: 9f3e2a6d-8959-b6de-e3e0-8487352c715f" -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.175-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to appendEntries to: peer="{Voter server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f}" error="failed to connect to peer: e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f" - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.175-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to appendEntries to: peer="{Voter server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f}" error="failed to connect to peer: e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f" -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.178-1200 [WARN] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to contact: server-id=server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f time=60.97177ms - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.178-1200 [WARN] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to contact: server-id=server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f time=60.97177ms -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.178-1200 [WARN] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to contact: server-id=server-9f3e2a6d-8959-b6de-e3e0-8487352c715f time=62.135789ms - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.178-1200 [WARN] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to contact: server-id=server-9f3e2a6d-8959-b6de-e3e0-8487352c715f time=62.135789ms -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.178-1200 [WARN] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to contact quorum of nodes, stepping down - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.178-1200 [WARN] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to contact quorum of nodes, stepping down -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.178-1200 [INFO] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: entering follower state: follower="Node at 5a06e4a1-a619-73f1-3519-d47fddb9f320 [Follower]" leader= - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.178-1200 [INFO] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: entering follower state: follower="Node at 5a06e4a1-a619-73f1-3519-d47fddb9f320 [Follower]" leader= -testLoggerAdapter verbose: server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: 2024-06-11T02:29:15.215-1200 [WARN] server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: heartbeat timeout reached, starting election: last-leader=5a06e4a1-a619-73f1-3519-d47fddb9f320 - testing.go:149: server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: 2024-06-11T02:29:15.215-1200 [WARN] server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: heartbeat timeout reached, starting election: last-leader=5a06e4a1-a619-73f1-3519-d47fddb9f320 -testLoggerAdapter verbose: server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: 2024-06-11T02:29:15.215-1200 [INFO] server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: entering candidate state: node="Node at e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f [Candidate]" term=3 - testing.go:149: server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: 2024-06-11T02:29:15.215-1200 [INFO] server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: entering candidate state: node="Node at e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f [Candidate]" term=3 -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.216-1200 [WARN] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: rejecting vote request since we have a leader: from=e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f leader=5a06e4a1-a619-73f1-3519-d47fddb9f320 - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.216-1200 [WARN] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: rejecting vote request since we have a leader: from=e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f leader=5a06e4a1-a619-73f1-3519-d47fddb9f320 -testLoggerAdapter verbose: server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: 2024-06-11T02:29:15.216-1200 [ERROR] server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: failed to make requestVote RPC: target="{Voter server-5a06e4a1-a619-73f1-3519-d47fddb9f320 5a06e4a1-a619-73f1-3519-d47fddb9f320}" error="failed to connect to peer: 5a06e4a1-a619-73f1-3519-d47fddb9f320" - testing.go:149: server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: 2024-06-11T02:29:15.216-1200 [ERROR] server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: failed to make requestVote RPC: target="{Voter server-5a06e4a1-a619-73f1-3519-d47fddb9f320 5a06e4a1-a619-73f1-3519-d47fddb9f320}" error="failed to connect to peer: 5a06e4a1-a619-73f1-3519-d47fddb9f320" -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.216-1200 [WARN] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: heartbeat timeout reached, starting election: last-leader=5a06e4a1-a619-73f1-3519-d47fddb9f320 - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.216-1200 [WARN] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: heartbeat timeout reached, starting election: last-leader=5a06e4a1-a619-73f1-3519-d47fddb9f320 -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.216-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: entering candidate state: node="Node at 9f3e2a6d-8959-b6de-e3e0-8487352c715f [Candidate]" term=3 - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.216-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: entering candidate state: node="Node at 9f3e2a6d-8959-b6de-e3e0-8487352c715f [Candidate]" term=3 -testLoggerAdapter verbose: server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: 2024-06-11T02:29:15.216-1200 [INFO] server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: duplicate requestVote for same term: term=3 - testing.go:149: server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: 2024-06-11T02:29:15.216-1200 [INFO] server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: duplicate requestVote for same term: term=3 -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.216-1200 [ERROR] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: failed to make requestVote RPC: target="{Voter server-5a06e4a1-a619-73f1-3519-d47fddb9f320 5a06e4a1-a619-73f1-3519-d47fddb9f320}" error="failed to connect to peer: 5a06e4a1-a619-73f1-3519-d47fddb9f320" - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.216-1200 [ERROR] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: failed to make requestVote RPC: target="{Voter server-5a06e4a1-a619-73f1-3519-d47fddb9f320 5a06e4a1-a619-73f1-3519-d47fddb9f320}" error="failed to connect to peer: 5a06e4a1-a619-73f1-3519-d47fddb9f320" -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.242-1200 [WARN] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.242-1200 [WARN] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.242-1200 [INFO] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: entering candidate state: node="Node at 5a06e4a1-a619-73f1-3519-d47fddb9f320 [Candidate]" term=3 - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.242-1200 [INFO] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: entering candidate state: node="Node at 5a06e4a1-a619-73f1-3519-d47fddb9f320 [Candidate]" term=3 -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.242-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to make requestVote RPC: target="{Voter server-9f3e2a6d-8959-b6de-e3e0-8487352c715f 9f3e2a6d-8959-b6de-e3e0-8487352c715f}" error="failed to connect to peer: 9f3e2a6d-8959-b6de-e3e0-8487352c715f" - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.242-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to make requestVote RPC: target="{Voter server-9f3e2a6d-8959-b6de-e3e0-8487352c715f 9f3e2a6d-8959-b6de-e3e0-8487352c715f}" error="failed to connect to peer: 9f3e2a6d-8959-b6de-e3e0-8487352c715f" -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.243-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to make requestVote RPC: target="{Voter server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f}" error="failed to connect to peer: e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f" - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.243-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to make requestVote RPC: target="{Voter server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f}" error="failed to connect to peer: e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f" -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.305-1200 [WARN] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: Election timeout reached, restarting election - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.305-1200 [WARN] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: Election timeout reached, restarting election -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.305-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: entering candidate state: node="Node at 9f3e2a6d-8959-b6de-e3e0-8487352c715f [Candidate]" term=4 - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.305-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: entering candidate state: node="Node at 9f3e2a6d-8959-b6de-e3e0-8487352c715f [Candidate]" term=4 -testLoggerAdapter verbose: server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: 2024-06-11T02:29:15.305-1200 [INFO] server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: entering follower state: follower="Node at e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f [Follower]" leader= - testing.go:149: server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: 2024-06-11T02:29:15.305-1200 [INFO] server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: entering follower state: follower="Node at e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f [Follower]" leader= -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.305-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: election won: tally=2 - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.305-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: election won: tally=2 -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.305-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: entering leader state: leader="Node at 9f3e2a6d-8959-b6de-e3e0-8487352c715f [Leader]" - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.305-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: entering leader state: leader="Node at 9f3e2a6d-8959-b6de-e3e0-8487352c715f [Leader]" -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.305-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: added peer, starting replication: peer=server-5a06e4a1-a619-73f1-3519-d47fddb9f320 - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.305-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: added peer, starting replication: peer=server-5a06e4a1-a619-73f1-3519-d47fddb9f320 -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.305-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: added peer, starting replication: peer=server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.305-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: added peer, starting replication: peer=server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.306-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: pipelining replication: peer="{Voter server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f}" - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.306-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: pipelining replication: peer="{Voter server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f}" -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.306-1200 [ERROR] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: failed to make requestVote RPC: target="{Voter server-5a06e4a1-a619-73f1-3519-d47fddb9f320 5a06e4a1-a619-73f1-3519-d47fddb9f320}" error="failed to connect to peer: 5a06e4a1-a619-73f1-3519-d47fddb9f320" - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.306-1200 [ERROR] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: failed to make requestVote RPC: target="{Voter server-5a06e4a1-a619-73f1-3519-d47fddb9f320 5a06e4a1-a619-73f1-3519-d47fddb9f320}" error="failed to connect to peer: 5a06e4a1-a619-73f1-3519-d47fddb9f320" -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.306-1200 [ERROR] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: failed to appendEntries to: peer="{Voter server-5a06e4a1-a619-73f1-3519-d47fddb9f320 5a06e4a1-a619-73f1-3519-d47fddb9f320}" error="failed to connect to peer: 5a06e4a1-a619-73f1-3519-d47fddb9f320" - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.306-1200 [ERROR] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: failed to appendEntries to: peer="{Voter server-5a06e4a1-a619-73f1-3519-d47fddb9f320 5a06e4a1-a619-73f1-3519-d47fddb9f320}" error="failed to connect to peer: 5a06e4a1-a619-73f1-3519-d47fddb9f320" -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.315-1200 [ERROR] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: failed to heartbeat to: peer=5a06e4a1-a619-73f1-3519-d47fddb9f320 error="failed to connect to peer: 5a06e4a1-a619-73f1-3519-d47fddb9f320" - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.315-1200 [ERROR] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: failed to heartbeat to: peer=5a06e4a1-a619-73f1-3519-d47fddb9f320 error="failed to connect to peer: 5a06e4a1-a619-73f1-3519-d47fddb9f320" -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.318-1200 [WARN] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: Election timeout reached, restarting election - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.318-1200 [WARN] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: Election timeout reached, restarting election -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.318-1200 [INFO] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: entering candidate state: node="Node at 5a06e4a1-a619-73f1-3519-d47fddb9f320 [Candidate]" term=4 - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.318-1200 [INFO] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: entering candidate state: node="Node at 5a06e4a1-a619-73f1-3519-d47fddb9f320 [Candidate]" term=4 -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.318-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to make requestVote RPC: target="{Voter server-9f3e2a6d-8959-b6de-e3e0-8487352c715f 9f3e2a6d-8959-b6de-e3e0-8487352c715f}" error="failed to connect to peer: 9f3e2a6d-8959-b6de-e3e0-8487352c715f" - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.318-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to make requestVote RPC: target="{Voter server-9f3e2a6d-8959-b6de-e3e0-8487352c715f 9f3e2a6d-8959-b6de-e3e0-8487352c715f}" error="failed to connect to peer: 9f3e2a6d-8959-b6de-e3e0-8487352c715f" -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.318-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to make requestVote RPC: target="{Voter server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f}" error="failed to connect to peer: e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f" - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.318-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to make requestVote RPC: target="{Voter server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f}" error="failed to connect to peer: e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f" -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.324-1200 [ERROR] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: failed to appendEntries to: peer="{Voter server-5a06e4a1-a619-73f1-3519-d47fddb9f320 5a06e4a1-a619-73f1-3519-d47fddb9f320}" error="failed to connect to peer: 5a06e4a1-a619-73f1-3519-d47fddb9f320" - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.324-1200 [ERROR] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: failed to appendEntries to: peer="{Voter server-5a06e4a1-a619-73f1-3519-d47fddb9f320 5a06e4a1-a619-73f1-3519-d47fddb9f320}" error="failed to connect to peer: 5a06e4a1-a619-73f1-3519-d47fddb9f320" -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.335-1200 [ERROR] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: failed to heartbeat to: peer=5a06e4a1-a619-73f1-3519-d47fddb9f320 error="failed to connect to peer: 5a06e4a1-a619-73f1-3519-d47fddb9f320" - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.335-1200 [ERROR] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: failed to heartbeat to: peer=5a06e4a1-a619-73f1-3519-d47fddb9f320 error="failed to connect to peer: 5a06e4a1-a619-73f1-3519-d47fddb9f320" -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.343-1200 [ERROR] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: failed to appendEntries to: peer="{Voter server-5a06e4a1-a619-73f1-3519-d47fddb9f320 5a06e4a1-a619-73f1-3519-d47fddb9f320}" error="failed to connect to peer: 5a06e4a1-a619-73f1-3519-d47fddb9f320" - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.343-1200 [ERROR] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: failed to appendEntries to: peer="{Voter server-5a06e4a1-a619-73f1-3519-d47fddb9f320 5a06e4a1-a619-73f1-3519-d47fddb9f320}" error="failed to connect to peer: 5a06e4a1-a619-73f1-3519-d47fddb9f320" -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.351-1200 [ERROR] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: failed to heartbeat to: peer=5a06e4a1-a619-73f1-3519-d47fddb9f320 error="failed to connect to peer: 5a06e4a1-a619-73f1-3519-d47fddb9f320" - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.351-1200 [ERROR] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: failed to heartbeat to: peer=5a06e4a1-a619-73f1-3519-d47fddb9f320 error="failed to connect to peer: 5a06e4a1-a619-73f1-3519-d47fddb9f320" -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.356-1200 [WARN] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: failed to contact: server-id=server-5a06e4a1-a619-73f1-3519-d47fddb9f320 time=50.406636ms - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.356-1200 [WARN] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: failed to contact: server-id=server-5a06e4a1-a619-73f1-3519-d47fddb9f320 time=50.406636ms -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.370-1200 [ERROR] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: failed to appendEntries to: peer="{Voter server-5a06e4a1-a619-73f1-3519-d47fddb9f320 5a06e4a1-a619-73f1-3519-d47fddb9f320}" error="failed to connect to peer: 5a06e4a1-a619-73f1-3519-d47fddb9f320" - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.370-1200 [ERROR] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: failed to appendEntries to: peer="{Voter server-5a06e4a1-a619-73f1-3519-d47fddb9f320 5a06e4a1-a619-73f1-3519-d47fddb9f320}" error="failed to connect to peer: 5a06e4a1-a619-73f1-3519-d47fddb9f320" -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.381-1200 [ERROR] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: failed to heartbeat to: peer=5a06e4a1-a619-73f1-3519-d47fddb9f320 error="failed to connect to peer: 5a06e4a1-a619-73f1-3519-d47fddb9f320" - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.381-1200 [ERROR] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: failed to heartbeat to: peer=5a06e4a1-a619-73f1-3519-d47fddb9f320 error="failed to connect to peer: 5a06e4a1-a619-73f1-3519-d47fddb9f320" -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.403-1200 [WARN] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: failed to contact: server-id=server-5a06e4a1-a619-73f1-3519-d47fddb9f320 time=97.22088ms - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.403-1200 [WARN] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: failed to contact: server-id=server-5a06e4a1-a619-73f1-3519-d47fddb9f320 time=97.22088ms -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.412-1200 [WARN] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: Election timeout reached, restarting election - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.412-1200 [WARN] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: Election timeout reached, restarting election -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.412-1200 [INFO] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: entering candidate state: node="Node at 5a06e4a1-a619-73f1-3519-d47fddb9f320 [Candidate]" term=5 - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.412-1200 [INFO] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: entering candidate state: node="Node at 5a06e4a1-a619-73f1-3519-d47fddb9f320 [Candidate]" term=5 -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.412-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to make requestVote RPC: target="{Voter server-9f3e2a6d-8959-b6de-e3e0-8487352c715f 9f3e2a6d-8959-b6de-e3e0-8487352c715f}" error="failed to connect to peer: 9f3e2a6d-8959-b6de-e3e0-8487352c715f" - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.412-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to make requestVote RPC: target="{Voter server-9f3e2a6d-8959-b6de-e3e0-8487352c715f 9f3e2a6d-8959-b6de-e3e0-8487352c715f}" error="failed to connect to peer: 9f3e2a6d-8959-b6de-e3e0-8487352c715f" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:15.412-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:15.306413199 -1200 -12 m=+6.301785740 (1 nodes), 182.9576ms from start of poll, 487.596827ms from cluster start. Timeout at 2024-06-11 02:29:15.412040337 -1200 -12 m=+6.407412878, 105.627138ms after stability - testing.go:149: cluster: 2024-06-11T02:29:15.412-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:15.306413199 -1200 -12 m=+6.301785740 (1 nodes), 182.9576ms from start of poll, 487.596827ms from cluster start. Timeout at 2024-06-11 02:29:15.412040337 -1200 -12 m=+6.407412878, 105.627138ms after stability -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.412-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to make requestVote RPC: target="{Voter server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f}" error="failed to connect to peer: e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f" - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.412-1200 [ERROR] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to make requestVote RPC: target="{Voter server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f}" error="failed to connect to peer: e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f" - raft_test.go:371: [INFO] Reconnecting Node at 5a06e4a1-a619-73f1-3519-d47fddb9f320 [Candidate] -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.422-1200 [ERROR] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: peer has newer term, stopping replication: peer="{Voter server-5a06e4a1-a619-73f1-3519-d47fddb9f320 5a06e4a1-a619-73f1-3519-d47fddb9f320}" - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.422-1200 [ERROR] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: peer has newer term, stopping replication: peer="{Voter server-5a06e4a1-a619-73f1-3519-d47fddb9f320 5a06e4a1-a619-73f1-3519-d47fddb9f320}" -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.422-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: entering follower state: follower="Node at 9f3e2a6d-8959-b6de-e3e0-8487352c715f [Follower]" leader= - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.422-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: entering follower state: follower="Node at 9f3e2a6d-8959-b6de-e3e0-8487352c715f [Follower]" leader= -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.422-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: aborting pipeline replication: peer="{Voter server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f}" - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.422-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: aborting pipeline replication: peer="{Voter server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f}" -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.486-1200 [WARN] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: Election timeout reached, restarting election - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.486-1200 [WARN] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: Election timeout reached, restarting election -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.486-1200 [INFO] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: entering candidate state: node="Node at 5a06e4a1-a619-73f1-3519-d47fddb9f320 [Candidate]" term=6 - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.486-1200 [INFO] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: entering candidate state: node="Node at 5a06e4a1-a619-73f1-3519-d47fddb9f320 [Candidate]" term=6 -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.487-1200 [WARN] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: rejecting vote request since our last term is greater: candidate=5a06e4a1-a619-73f1-3519-d47fddb9f320 last-term=4 last-candidate-term=2 - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.487-1200 [WARN] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: rejecting vote request since our last term is greater: candidate=5a06e4a1-a619-73f1-3519-d47fddb9f320 last-term=4 last-candidate-term=2 -testLoggerAdapter verbose: server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: 2024-06-11T02:29:15.487-1200 [WARN] server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: rejecting vote request since we have a leader: from=5a06e4a1-a619-73f1-3519-d47fddb9f320 leader=9f3e2a6d-8959-b6de-e3e0-8487352c715f - testing.go:149: server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: 2024-06-11T02:29:15.487-1200 [WARN] server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: rejecting vote request since we have a leader: from=5a06e4a1-a619-73f1-3519-d47fddb9f320 leader=9f3e2a6d-8959-b6de-e3e0-8487352c715f -testLoggerAdapter verbose: server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: 2024-06-11T02:29:15.487-1200 [WARN] server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: heartbeat timeout reached, starting election: last-leader=9f3e2a6d-8959-b6de-e3e0-8487352c715f - testing.go:149: server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: 2024-06-11T02:29:15.487-1200 [WARN] server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: heartbeat timeout reached, starting election: last-leader=9f3e2a6d-8959-b6de-e3e0-8487352c715f -testLoggerAdapter verbose: server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: 2024-06-11T02:29:15.487-1200 [INFO] server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: entering candidate state: node="Node at e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f [Candidate]" term=5 - testing.go:149: server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: 2024-06-11T02:29:15.487-1200 [INFO] server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: entering candidate state: node="Node at e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f [Candidate]" term=5 -testLoggerAdapter verbose: server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: 2024-06-11T02:29:15.488-1200 [INFO] server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: entering follower state: follower="Node at e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f [Follower]" leader= - testing.go:149: server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: 2024-06-11T02:29:15.488-1200 [INFO] server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f: entering follower state: follower="Node at e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f [Follower]" leader= -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.505-1200 [WARN] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.505-1200 [WARN] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.506-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: entering candidate state: node="Node at 9f3e2a6d-8959-b6de-e3e0-8487352c715f [Candidate]" term=7 - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.506-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: entering candidate state: node="Node at 9f3e2a6d-8959-b6de-e3e0-8487352c715f [Candidate]" term=7 -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.506-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: election won: tally=2 - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.506-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: election won: tally=2 -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.506-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: entering leader state: leader="Node at 9f3e2a6d-8959-b6de-e3e0-8487352c715f [Leader]" - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.506-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: entering leader state: leader="Node at 9f3e2a6d-8959-b6de-e3e0-8487352c715f [Leader]" -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.506-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: added peer, starting replication: peer=server-5a06e4a1-a619-73f1-3519-d47fddb9f320 - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.506-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: added peer, starting replication: peer=server-5a06e4a1-a619-73f1-3519-d47fddb9f320 -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.506-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: added peer, starting replication: peer=server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.506-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: added peer, starting replication: peer=server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.506-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: pipelining replication: peer="{Voter server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f}" - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.506-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: pipelining replication: peer="{Voter server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f}" -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.507-1200 [INFO] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: entering follower state: follower="Node at 5a06e4a1-a619-73f1-3519-d47fddb9f320 [Follower]" leader= - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.507-1200 [INFO] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: entering follower state: follower="Node at 5a06e4a1-a619-73f1-3519-d47fddb9f320 [Follower]" leader= -testLoggerAdapter verbose: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.507-1200 [WARN] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to get previous log: previous-index=5 last-index=3 error="log not found" - testing.go:149: server-5a06e4a1-a619-73f1-3519-d47fddb9f320: 2024-06-11T02:29:15.507-1200 [WARN] server-5a06e4a1-a619-73f1-3519-d47fddb9f320: failed to get previous log: previous-index=5 last-index=3 error="log not found" -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.507-1200 [WARN] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: appendEntries rejected, sending older logs: peer="{Voter server-5a06e4a1-a619-73f1-3519-d47fddb9f320 5a06e4a1-a619-73f1-3519-d47fddb9f320}" next=4 - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.507-1200 [WARN] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: appendEntries rejected, sending older logs: peer="{Voter server-5a06e4a1-a619-73f1-3519-d47fddb9f320 5a06e4a1-a619-73f1-3519-d47fddb9f320}" next=4 -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.507-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: pipelining replication: peer="{Voter server-5a06e4a1-a619-73f1-3519-d47fddb9f320 5a06e4a1-a619-73f1-3519-d47fddb9f320}" - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.507-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: pipelining replication: peer="{Voter server-5a06e4a1-a619-73f1-3519-d47fddb9f320 5a06e4a1-a619-73f1-3519-d47fddb9f320}" -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.514-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: aborting pipeline replication: peer="{Voter server-5a06e4a1-a619-73f1-3519-d47fddb9f320 5a06e4a1-a619-73f1-3519-d47fddb9f320}" - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.514-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: aborting pipeline replication: peer="{Voter server-5a06e4a1-a619-73f1-3519-d47fddb9f320 5a06e4a1-a619-73f1-3519-d47fddb9f320}" -testLoggerAdapter verbose: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.514-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: aborting pipeline replication: peer="{Voter server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f}" - testing.go:149: server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: 2024-06-11T02:29:15.514-1200 [INFO] server-9f3e2a6d-8959-b6de-e3e0-8487352c715f: aborting pipeline replication: peer="{Voter server-e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f e5cf5edb-c6cf-7e6c-8c81-b64d99aa911f}" +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.192+1400 [INFO] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1b67a0c6-3830-c6e6-3008-ce525ea6796b Address:1b67a0c6-3830-c6e6-3008-ce525ea6796b} {Suffrage:Voter ID:server-b98bac58-4a84-edd5-1003-a3d0df30dc2c Address:b98bac58-4a84-edd5-1003-a3d0df30dc2c} {Suffrage:Voter ID:server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6 Address:e07a9304-4edc-3a14-d254-66c4a5bfcaf6}]" + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.192+1400 [INFO] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1b67a0c6-3830-c6e6-3008-ce525ea6796b Address:1b67a0c6-3830-c6e6-3008-ce525ea6796b} {Suffrage:Voter ID:server-b98bac58-4a84-edd5-1003-a3d0df30dc2c Address:b98bac58-4a84-edd5-1003-a3d0df30dc2c} {Suffrage:Voter ID:server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6 Address:e07a9304-4edc-3a14-d254-66c4a5bfcaf6}]" +testLoggerAdapter verbose: server-b98bac58-4a84-edd5-1003-a3d0df30dc2c: 2024-06-12T04:32:28.192+1400 [INFO] server-b98bac58-4a84-edd5-1003-a3d0df30dc2c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1b67a0c6-3830-c6e6-3008-ce525ea6796b Address:1b67a0c6-3830-c6e6-3008-ce525ea6796b} {Suffrage:Voter ID:server-b98bac58-4a84-edd5-1003-a3d0df30dc2c Address:b98bac58-4a84-edd5-1003-a3d0df30dc2c} {Suffrage:Voter ID:server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6 Address:e07a9304-4edc-3a14-d254-66c4a5bfcaf6}]" + testing.go:149: server-b98bac58-4a84-edd5-1003-a3d0df30dc2c: 2024-06-12T04:32:28.192+1400 [INFO] server-b98bac58-4a84-edd5-1003-a3d0df30dc2c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1b67a0c6-3830-c6e6-3008-ce525ea6796b Address:1b67a0c6-3830-c6e6-3008-ce525ea6796b} {Suffrage:Voter ID:server-b98bac58-4a84-edd5-1003-a3d0df30dc2c Address:b98bac58-4a84-edd5-1003-a3d0df30dc2c} {Suffrage:Voter ID:server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6 Address:e07a9304-4edc-3a14-d254-66c4a5bfcaf6}]" +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.192+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1b67a0c6-3830-c6e6-3008-ce525ea6796b Address:1b67a0c6-3830-c6e6-3008-ce525ea6796b} {Suffrage:Voter ID:server-b98bac58-4a84-edd5-1003-a3d0df30dc2c Address:b98bac58-4a84-edd5-1003-a3d0df30dc2c} {Suffrage:Voter ID:server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6 Address:e07a9304-4edc-3a14-d254-66c4a5bfcaf6}]" + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.192+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1b67a0c6-3830-c6e6-3008-ce525ea6796b Address:1b67a0c6-3830-c6e6-3008-ce525ea6796b} {Suffrage:Voter ID:server-b98bac58-4a84-edd5-1003-a3d0df30dc2c Address:b98bac58-4a84-edd5-1003-a3d0df30dc2c} {Suffrage:Voter ID:server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6 Address:e07a9304-4edc-3a14-d254-66c4a5bfcaf6}]" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:28.192+1400 [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-06-12T04:32:28.192+1400 [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.192+1400 [INFO] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: entering follower state: follower="Node at 1b67a0c6-3830-c6e6-3008-ce525ea6796b [Follower]" leader= + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.192+1400 [INFO] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: entering follower state: follower="Node at 1b67a0c6-3830-c6e6-3008-ce525ea6796b [Follower]" leader= +testLoggerAdapter verbose: server-b98bac58-4a84-edd5-1003-a3d0df30dc2c: 2024-06-12T04:32:28.192+1400 [INFO] server-b98bac58-4a84-edd5-1003-a3d0df30dc2c: entering follower state: follower="Node at b98bac58-4a84-edd5-1003-a3d0df30dc2c [Follower]" leader= + testing.go:149: server-b98bac58-4a84-edd5-1003-a3d0df30dc2c: 2024-06-12T04:32:28.192+1400 [INFO] server-b98bac58-4a84-edd5-1003-a3d0df30dc2c: entering follower state: follower="Node at b98bac58-4a84-edd5-1003-a3d0df30dc2c [Follower]" leader= +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.193+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: entering follower state: follower="Node at e07a9304-4edc-3a14-d254-66c4a5bfcaf6 [Follower]" leader= + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.193+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: entering follower state: follower="Node at e07a9304-4edc-3a14-d254-66c4a5bfcaf6 [Follower]" leader= +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.271+1400 [WARN] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.271+1400 [WARN] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.271+1400 [INFO] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: entering candidate state: node="Node at 1b67a0c6-3830-c6e6-3008-ce525ea6796b [Candidate]" term=2 + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.271+1400 [INFO] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: entering candidate state: node="Node at 1b67a0c6-3830-c6e6-3008-ce525ea6796b [Candidate]" term=2 +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.271+1400 [INFO] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: election won: tally=2 + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.271+1400 [INFO] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: election won: tally=2 +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.271+1400 [INFO] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: entering leader state: leader="Node at 1b67a0c6-3830-c6e6-3008-ce525ea6796b [Leader]" + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.271+1400 [INFO] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: entering leader state: leader="Node at 1b67a0c6-3830-c6e6-3008-ce525ea6796b [Leader]" +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.271+1400 [INFO] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: added peer, starting replication: peer=server-b98bac58-4a84-edd5-1003-a3d0df30dc2c + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.271+1400 [INFO] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: added peer, starting replication: peer=server-b98bac58-4a84-edd5-1003-a3d0df30dc2c +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.271+1400 [INFO] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: added peer, starting replication: peer=server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6 + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.271+1400 [INFO] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: added peer, starting replication: peer=server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6 +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.272+1400 [INFO] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: pipelining replication: peer="{Voter server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6 e07a9304-4edc-3a14-d254-66c4a5bfcaf6}" + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.272+1400 [INFO] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: pipelining replication: peer="{Voter server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6 e07a9304-4edc-3a14-d254-66c4a5bfcaf6}" +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.272+1400 [INFO] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: pipelining replication: peer="{Voter server-b98bac58-4a84-edd5-1003-a3d0df30dc2c b98bac58-4a84-edd5-1003-a3d0df30dc2c}" + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.272+1400 [INFO] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: pipelining replication: peer="{Voter server-b98bac58-4a84-edd5-1003-a3d0df30dc2c b98bac58-4a84-edd5-1003-a3d0df30dc2c}" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:28.378+1400 [INFO] stable state for Follower reached at 2024-06-12 04:32:28.272239609 +1400 +14 m=+5.179758188 (2 nodes), 79.395196ms from start of poll, 80.052532ms from cluster start. Timeout at 2024-06-12 04:32:28.377977582 +1400 +14 m=+5.285496161, 105.737973ms after stability + testing.go:149: cluster: 2024-06-12T04:32:28.378+1400 [INFO] stable state for Follower reached at 2024-06-12 04:32:28.272239609 +1400 +14 m=+5.179758188 (2 nodes), 79.395196ms from start of poll, 80.052532ms from cluster start. Timeout at 2024-06-12 04:32:28.377977582 +1400 +14 m=+5.285496161, 105.737973ms after stability +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:28.378+1400 [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-06-12T04:32:28.378+1400 [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:28.483+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:28.378172175 +1400 +14 m=+5.285690754 (1 nodes), 1.64µs from start of poll, 185.985098ms from cluster start. Timeout at 2024-06-12 04:32:28.48326889 +1400 +14 m=+5.390787509, 105.096755ms after stability + testing.go:149: cluster: 2024-06-12T04:32:28.483+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:28.378172175 +1400 +14 m=+5.285690754 (1 nodes), 1.64µs from start of poll, 185.985098ms from cluster start. Timeout at 2024-06-12 04:32:28.48326889 +1400 +14 m=+5.390787509, 105.096755ms after stability + raft_test.go:336: [INFO] Disconnecting Node at 1b67a0c6-3830-c6e6-3008-ce525ea6796b [Leader] +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:28.500+1400 [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-06-12T04:32:28.500+1400 [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.502+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to pipeline appendEntries: peer="{Voter server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6 e07a9304-4edc-3a14-d254-66c4a5bfcaf6}" error="append pipeline closed" + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.502+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to pipeline appendEntries: peer="{Voter server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6 e07a9304-4edc-3a14-d254-66c4a5bfcaf6}" error="append pipeline closed" +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.502+1400 [INFO] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: aborting pipeline replication: peer="{Voter server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6 e07a9304-4edc-3a14-d254-66c4a5bfcaf6}" + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.502+1400 [INFO] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: aborting pipeline replication: peer="{Voter server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6 e07a9304-4edc-3a14-d254-66c4a5bfcaf6}" +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.502+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to heartbeat to: peer=b98bac58-4a84-edd5-1003-a3d0df30dc2c error="failed to connect to peer: b98bac58-4a84-edd5-1003-a3d0df30dc2c" + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.502+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to heartbeat to: peer=b98bac58-4a84-edd5-1003-a3d0df30dc2c error="failed to connect to peer: b98bac58-4a84-edd5-1003-a3d0df30dc2c" +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.503+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to pipeline appendEntries: peer="{Voter server-b98bac58-4a84-edd5-1003-a3d0df30dc2c b98bac58-4a84-edd5-1003-a3d0df30dc2c}" error="append pipeline closed" + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.503+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to pipeline appendEntries: peer="{Voter server-b98bac58-4a84-edd5-1003-a3d0df30dc2c b98bac58-4a84-edd5-1003-a3d0df30dc2c}" error="append pipeline closed" +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.503+1400 [INFO] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: aborting pipeline replication: peer="{Voter server-b98bac58-4a84-edd5-1003-a3d0df30dc2c b98bac58-4a84-edd5-1003-a3d0df30dc2c}" + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.503+1400 [INFO] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: aborting pipeline replication: peer="{Voter server-b98bac58-4a84-edd5-1003-a3d0df30dc2c b98bac58-4a84-edd5-1003-a3d0df30dc2c}" +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.505+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to heartbeat to: peer=e07a9304-4edc-3a14-d254-66c4a5bfcaf6 error="failed to connect to peer: e07a9304-4edc-3a14-d254-66c4a5bfcaf6" + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.505+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to heartbeat to: peer=e07a9304-4edc-3a14-d254-66c4a5bfcaf6 error="failed to connect to peer: e07a9304-4edc-3a14-d254-66c4a5bfcaf6" +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.510+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to appendEntries to: peer="{Voter server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6 e07a9304-4edc-3a14-d254-66c4a5bfcaf6}" error="failed to connect to peer: e07a9304-4edc-3a14-d254-66c4a5bfcaf6" + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.510+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to appendEntries to: peer="{Voter server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6 e07a9304-4edc-3a14-d254-66c4a5bfcaf6}" error="failed to connect to peer: e07a9304-4edc-3a14-d254-66c4a5bfcaf6" +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.510+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to appendEntries to: peer="{Voter server-b98bac58-4a84-edd5-1003-a3d0df30dc2c b98bac58-4a84-edd5-1003-a3d0df30dc2c}" error="failed to connect to peer: b98bac58-4a84-edd5-1003-a3d0df30dc2c" + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.510+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to appendEntries to: peer="{Voter server-b98bac58-4a84-edd5-1003-a3d0df30dc2c b98bac58-4a84-edd5-1003-a3d0df30dc2c}" error="failed to connect to peer: b98bac58-4a84-edd5-1003-a3d0df30dc2c" +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.520+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to heartbeat to: peer=b98bac58-4a84-edd5-1003-a3d0df30dc2c error="failed to connect to peer: b98bac58-4a84-edd5-1003-a3d0df30dc2c" + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.520+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to heartbeat to: peer=b98bac58-4a84-edd5-1003-a3d0df30dc2c error="failed to connect to peer: b98bac58-4a84-edd5-1003-a3d0df30dc2c" +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.524+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to heartbeat to: peer=e07a9304-4edc-3a14-d254-66c4a5bfcaf6 error="failed to connect to peer: e07a9304-4edc-3a14-d254-66c4a5bfcaf6" + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.524+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to heartbeat to: peer=e07a9304-4edc-3a14-d254-66c4a5bfcaf6 error="failed to connect to peer: e07a9304-4edc-3a14-d254-66c4a5bfcaf6" +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.526+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to appendEntries to: peer="{Voter server-b98bac58-4a84-edd5-1003-a3d0df30dc2c b98bac58-4a84-edd5-1003-a3d0df30dc2c}" error="failed to connect to peer: b98bac58-4a84-edd5-1003-a3d0df30dc2c" + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.526+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to appendEntries to: peer="{Voter server-b98bac58-4a84-edd5-1003-a3d0df30dc2c b98bac58-4a84-edd5-1003-a3d0df30dc2c}" error="failed to connect to peer: b98bac58-4a84-edd5-1003-a3d0df30dc2c" +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.530+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to appendEntries to: peer="{Voter server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6 e07a9304-4edc-3a14-d254-66c4a5bfcaf6}" error="failed to connect to peer: e07a9304-4edc-3a14-d254-66c4a5bfcaf6" + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.530+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to appendEntries to: peer="{Voter server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6 e07a9304-4edc-3a14-d254-66c4a5bfcaf6}" error="failed to connect to peer: e07a9304-4edc-3a14-d254-66c4a5bfcaf6" +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.539+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to heartbeat to: peer=b98bac58-4a84-edd5-1003-a3d0df30dc2c error="failed to connect to peer: b98bac58-4a84-edd5-1003-a3d0df30dc2c" + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.539+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to heartbeat to: peer=b98bac58-4a84-edd5-1003-a3d0df30dc2c error="failed to connect to peer: b98bac58-4a84-edd5-1003-a3d0df30dc2c" +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.541+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to heartbeat to: peer=e07a9304-4edc-3a14-d254-66c4a5bfcaf6 error="failed to connect to peer: e07a9304-4edc-3a14-d254-66c4a5bfcaf6" + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.541+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to heartbeat to: peer=e07a9304-4edc-3a14-d254-66c4a5bfcaf6 error="failed to connect to peer: e07a9304-4edc-3a14-d254-66c4a5bfcaf6" +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.544+1400 [WARN] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to contact: server-id=server-b98bac58-4a84-edd5-1003-a3d0df30dc2c time=50.892417ms + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.544+1400 [WARN] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to contact: server-id=server-b98bac58-4a84-edd5-1003-a3d0df30dc2c time=50.892417ms +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.544+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to appendEntries to: peer="{Voter server-b98bac58-4a84-edd5-1003-a3d0df30dc2c b98bac58-4a84-edd5-1003-a3d0df30dc2c}" error="failed to connect to peer: b98bac58-4a84-edd5-1003-a3d0df30dc2c" + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.544+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to appendEntries to: peer="{Voter server-b98bac58-4a84-edd5-1003-a3d0df30dc2c b98bac58-4a84-edd5-1003-a3d0df30dc2c}" error="failed to connect to peer: b98bac58-4a84-edd5-1003-a3d0df30dc2c" +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.546+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to appendEntries to: peer="{Voter server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6 e07a9304-4edc-3a14-d254-66c4a5bfcaf6}" error="failed to connect to peer: e07a9304-4edc-3a14-d254-66c4a5bfcaf6" + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.546+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to appendEntries to: peer="{Voter server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6 e07a9304-4edc-3a14-d254-66c4a5bfcaf6}" error="failed to connect to peer: e07a9304-4edc-3a14-d254-66c4a5bfcaf6" +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.555+1400 [WARN] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to contact: server-id=server-b98bac58-4a84-edd5-1003-a3d0df30dc2c time=61.990219ms + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.555+1400 [WARN] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to contact: server-id=server-b98bac58-4a84-edd5-1003-a3d0df30dc2c time=61.990219ms +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.555+1400 [WARN] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to contact: server-id=server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6 time=60.83502ms + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.555+1400 [WARN] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to contact: server-id=server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6 time=60.83502ms +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.555+1400 [WARN] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to contact quorum of nodes, stepping down + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.555+1400 [WARN] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to contact quorum of nodes, stepping down +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.556+1400 [INFO] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: entering follower state: follower="Node at 1b67a0c6-3830-c6e6-3008-ce525ea6796b [Follower]" leader= + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.556+1400 [INFO] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: entering follower state: follower="Node at 1b67a0c6-3830-c6e6-3008-ce525ea6796b [Follower]" leader= +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.568+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to heartbeat to: peer=b98bac58-4a84-edd5-1003-a3d0df30dc2c error="failed to connect to peer: b98bac58-4a84-edd5-1003-a3d0df30dc2c" + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.568+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to heartbeat to: peer=b98bac58-4a84-edd5-1003-a3d0df30dc2c error="failed to connect to peer: b98bac58-4a84-edd5-1003-a3d0df30dc2c" +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.569+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to heartbeat to: peer=e07a9304-4edc-3a14-d254-66c4a5bfcaf6 error="failed to connect to peer: e07a9304-4edc-3a14-d254-66c4a5bfcaf6" + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.569+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to heartbeat to: peer=e07a9304-4edc-3a14-d254-66c4a5bfcaf6 error="failed to connect to peer: e07a9304-4edc-3a14-d254-66c4a5bfcaf6" +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.575+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to appendEntries to: peer="{Voter server-b98bac58-4a84-edd5-1003-a3d0df30dc2c b98bac58-4a84-edd5-1003-a3d0df30dc2c}" error="failed to connect to peer: b98bac58-4a84-edd5-1003-a3d0df30dc2c" + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.575+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to appendEntries to: peer="{Voter server-b98bac58-4a84-edd5-1003-a3d0df30dc2c b98bac58-4a84-edd5-1003-a3d0df30dc2c}" error="failed to connect to peer: b98bac58-4a84-edd5-1003-a3d0df30dc2c" +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.576+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to appendEntries to: peer="{Voter server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6 e07a9304-4edc-3a14-d254-66c4a5bfcaf6}" error="failed to connect to peer: e07a9304-4edc-3a14-d254-66c4a5bfcaf6" + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.576+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to appendEntries to: peer="{Voter server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6 e07a9304-4edc-3a14-d254-66c4a5bfcaf6}" error="failed to connect to peer: e07a9304-4edc-3a14-d254-66c4a5bfcaf6" +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.601+1400 [WARN] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: heartbeat timeout reached, starting election: last-leader=1b67a0c6-3830-c6e6-3008-ce525ea6796b + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.601+1400 [WARN] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: heartbeat timeout reached, starting election: last-leader=1b67a0c6-3830-c6e6-3008-ce525ea6796b +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.601+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: entering candidate state: node="Node at e07a9304-4edc-3a14-d254-66c4a5bfcaf6 [Candidate]" term=3 + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.601+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: entering candidate state: node="Node at e07a9304-4edc-3a14-d254-66c4a5bfcaf6 [Candidate]" term=3 +testLoggerAdapter verbose: server-b98bac58-4a84-edd5-1003-a3d0df30dc2c: 2024-06-12T04:32:28.601+1400 [WARN] server-b98bac58-4a84-edd5-1003-a3d0df30dc2c: rejecting vote request since we have a leader: from=e07a9304-4edc-3a14-d254-66c4a5bfcaf6 leader=1b67a0c6-3830-c6e6-3008-ce525ea6796b + testing.go:149: server-b98bac58-4a84-edd5-1003-a3d0df30dc2c: 2024-06-12T04:32:28.601+1400 [WARN] server-b98bac58-4a84-edd5-1003-a3d0df30dc2c: rejecting vote request since we have a leader: from=e07a9304-4edc-3a14-d254-66c4a5bfcaf6 leader=1b67a0c6-3830-c6e6-3008-ce525ea6796b +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.601+1400 [ERROR] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: failed to make requestVote RPC: target="{Voter server-1b67a0c6-3830-c6e6-3008-ce525ea6796b 1b67a0c6-3830-c6e6-3008-ce525ea6796b}" error="failed to connect to peer: 1b67a0c6-3830-c6e6-3008-ce525ea6796b" + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.601+1400 [ERROR] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: failed to make requestVote RPC: target="{Voter server-1b67a0c6-3830-c6e6-3008-ce525ea6796b 1b67a0c6-3830-c6e6-3008-ce525ea6796b}" error="failed to connect to peer: 1b67a0c6-3830-c6e6-3008-ce525ea6796b" +testLoggerAdapter verbose: server-b98bac58-4a84-edd5-1003-a3d0df30dc2c: 2024-06-12T04:32:28.625+1400 [WARN] server-b98bac58-4a84-edd5-1003-a3d0df30dc2c: heartbeat timeout reached, starting election: last-leader=1b67a0c6-3830-c6e6-3008-ce525ea6796b + testing.go:149: server-b98bac58-4a84-edd5-1003-a3d0df30dc2c: 2024-06-12T04:32:28.625+1400 [WARN] server-b98bac58-4a84-edd5-1003-a3d0df30dc2c: heartbeat timeout reached, starting election: last-leader=1b67a0c6-3830-c6e6-3008-ce525ea6796b +testLoggerAdapter verbose: server-b98bac58-4a84-edd5-1003-a3d0df30dc2c: 2024-06-12T04:32:28.625+1400 [INFO] server-b98bac58-4a84-edd5-1003-a3d0df30dc2c: entering candidate state: node="Node at b98bac58-4a84-edd5-1003-a3d0df30dc2c [Candidate]" term=3 + testing.go:149: server-b98bac58-4a84-edd5-1003-a3d0df30dc2c: 2024-06-12T04:32:28.625+1400 [INFO] server-b98bac58-4a84-edd5-1003-a3d0df30dc2c: entering candidate state: node="Node at b98bac58-4a84-edd5-1003-a3d0df30dc2c [Candidate]" term=3 +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.625+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: duplicate requestVote for same term: term=3 + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.625+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: duplicate requestVote for same term: term=3 +testLoggerAdapter verbose: server-b98bac58-4a84-edd5-1003-a3d0df30dc2c: 2024-06-12T04:32:28.625+1400 [ERROR] server-b98bac58-4a84-edd5-1003-a3d0df30dc2c: failed to make requestVote RPC: target="{Voter server-1b67a0c6-3830-c6e6-3008-ce525ea6796b 1b67a0c6-3830-c6e6-3008-ce525ea6796b}" error="failed to connect to peer: 1b67a0c6-3830-c6e6-3008-ce525ea6796b" + testing.go:149: server-b98bac58-4a84-edd5-1003-a3d0df30dc2c: 2024-06-12T04:32:28.625+1400 [ERROR] server-b98bac58-4a84-edd5-1003-a3d0df30dc2c: failed to make requestVote RPC: target="{Voter server-1b67a0c6-3830-c6e6-3008-ce525ea6796b 1b67a0c6-3830-c6e6-3008-ce525ea6796b}" error="failed to connect to peer: 1b67a0c6-3830-c6e6-3008-ce525ea6796b" +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.632+1400 [WARN] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.632+1400 [WARN] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.632+1400 [INFO] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: entering candidate state: node="Node at 1b67a0c6-3830-c6e6-3008-ce525ea6796b [Candidate]" term=3 + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.632+1400 [INFO] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: entering candidate state: node="Node at 1b67a0c6-3830-c6e6-3008-ce525ea6796b [Candidate]" term=3 +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.632+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to make requestVote RPC: target="{Voter server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6 e07a9304-4edc-3a14-d254-66c4a5bfcaf6}" error="failed to connect to peer: e07a9304-4edc-3a14-d254-66c4a5bfcaf6" + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.632+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to make requestVote RPC: target="{Voter server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6 e07a9304-4edc-3a14-d254-66c4a5bfcaf6}" error="failed to connect to peer: e07a9304-4edc-3a14-d254-66c4a5bfcaf6" +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.632+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to make requestVote RPC: target="{Voter server-b98bac58-4a84-edd5-1003-a3d0df30dc2c b98bac58-4a84-edd5-1003-a3d0df30dc2c}" error="failed to connect to peer: b98bac58-4a84-edd5-1003-a3d0df30dc2c" + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.632+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to make requestVote RPC: target="{Voter server-b98bac58-4a84-edd5-1003-a3d0df30dc2c b98bac58-4a84-edd5-1003-a3d0df30dc2c}" error="failed to connect to peer: b98bac58-4a84-edd5-1003-a3d0df30dc2c" +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.693+1400 [WARN] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: Election timeout reached, restarting election + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.693+1400 [WARN] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: Election timeout reached, restarting election +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.693+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: entering candidate state: node="Node at e07a9304-4edc-3a14-d254-66c4a5bfcaf6 [Candidate]" term=4 + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.693+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: entering candidate state: node="Node at e07a9304-4edc-3a14-d254-66c4a5bfcaf6 [Candidate]" term=4 +testLoggerAdapter verbose: server-b98bac58-4a84-edd5-1003-a3d0df30dc2c: 2024-06-12T04:32:28.693+1400 [INFO] server-b98bac58-4a84-edd5-1003-a3d0df30dc2c: entering follower state: follower="Node at b98bac58-4a84-edd5-1003-a3d0df30dc2c [Follower]" leader= + testing.go:149: server-b98bac58-4a84-edd5-1003-a3d0df30dc2c: 2024-06-12T04:32:28.693+1400 [INFO] server-b98bac58-4a84-edd5-1003-a3d0df30dc2c: entering follower state: follower="Node at b98bac58-4a84-edd5-1003-a3d0df30dc2c [Follower]" leader= +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.693+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: election won: tally=2 + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.693+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: election won: tally=2 +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.693+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: entering leader state: leader="Node at e07a9304-4edc-3a14-d254-66c4a5bfcaf6 [Leader]" + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.693+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: entering leader state: leader="Node at e07a9304-4edc-3a14-d254-66c4a5bfcaf6 [Leader]" +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.693+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: added peer, starting replication: peer=server-1b67a0c6-3830-c6e6-3008-ce525ea6796b + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.693+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: added peer, starting replication: peer=server-1b67a0c6-3830-c6e6-3008-ce525ea6796b +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.693+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: added peer, starting replication: peer=server-b98bac58-4a84-edd5-1003-a3d0df30dc2c + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.693+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: added peer, starting replication: peer=server-b98bac58-4a84-edd5-1003-a3d0df30dc2c +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.693+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: pipelining replication: peer="{Voter server-b98bac58-4a84-edd5-1003-a3d0df30dc2c b98bac58-4a84-edd5-1003-a3d0df30dc2c}" + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.693+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: pipelining replication: peer="{Voter server-b98bac58-4a84-edd5-1003-a3d0df30dc2c b98bac58-4a84-edd5-1003-a3d0df30dc2c}" +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.693+1400 [ERROR] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: failed to make requestVote RPC: target="{Voter server-1b67a0c6-3830-c6e6-3008-ce525ea6796b 1b67a0c6-3830-c6e6-3008-ce525ea6796b}" error="failed to connect to peer: 1b67a0c6-3830-c6e6-3008-ce525ea6796b" + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.693+1400 [ERROR] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: failed to make requestVote RPC: target="{Voter server-1b67a0c6-3830-c6e6-3008-ce525ea6796b 1b67a0c6-3830-c6e6-3008-ce525ea6796b}" error="failed to connect to peer: 1b67a0c6-3830-c6e6-3008-ce525ea6796b" +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.694+1400 [ERROR] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: failed to appendEntries to: peer="{Voter server-1b67a0c6-3830-c6e6-3008-ce525ea6796b 1b67a0c6-3830-c6e6-3008-ce525ea6796b}" error="failed to connect to peer: 1b67a0c6-3830-c6e6-3008-ce525ea6796b" + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.694+1400 [ERROR] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: failed to appendEntries to: peer="{Voter server-1b67a0c6-3830-c6e6-3008-ce525ea6796b 1b67a0c6-3830-c6e6-3008-ce525ea6796b}" error="failed to connect to peer: 1b67a0c6-3830-c6e6-3008-ce525ea6796b" +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.698+1400 [WARN] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: Election timeout reached, restarting election + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.698+1400 [WARN] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: Election timeout reached, restarting election +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.698+1400 [INFO] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: entering candidate state: node="Node at 1b67a0c6-3830-c6e6-3008-ce525ea6796b [Candidate]" term=4 + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.698+1400 [INFO] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: entering candidate state: node="Node at 1b67a0c6-3830-c6e6-3008-ce525ea6796b [Candidate]" term=4 +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.698+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to make requestVote RPC: target="{Voter server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6 e07a9304-4edc-3a14-d254-66c4a5bfcaf6}" error="failed to connect to peer: e07a9304-4edc-3a14-d254-66c4a5bfcaf6" + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.698+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to make requestVote RPC: target="{Voter server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6 e07a9304-4edc-3a14-d254-66c4a5bfcaf6}" error="failed to connect to peer: e07a9304-4edc-3a14-d254-66c4a5bfcaf6" +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.698+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to make requestVote RPC: target="{Voter server-b98bac58-4a84-edd5-1003-a3d0df30dc2c b98bac58-4a84-edd5-1003-a3d0df30dc2c}" error="failed to connect to peer: b98bac58-4a84-edd5-1003-a3d0df30dc2c" + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.698+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to make requestVote RPC: target="{Voter server-b98bac58-4a84-edd5-1003-a3d0df30dc2c b98bac58-4a84-edd5-1003-a3d0df30dc2c}" error="failed to connect to peer: b98bac58-4a84-edd5-1003-a3d0df30dc2c" +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.703+1400 [ERROR] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: failed to heartbeat to: peer=1b67a0c6-3830-c6e6-3008-ce525ea6796b error="failed to connect to peer: 1b67a0c6-3830-c6e6-3008-ce525ea6796b" + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.703+1400 [ERROR] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: failed to heartbeat to: peer=1b67a0c6-3830-c6e6-3008-ce525ea6796b error="failed to connect to peer: 1b67a0c6-3830-c6e6-3008-ce525ea6796b" +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.712+1400 [ERROR] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: failed to appendEntries to: peer="{Voter server-1b67a0c6-3830-c6e6-3008-ce525ea6796b 1b67a0c6-3830-c6e6-3008-ce525ea6796b}" error="failed to connect to peer: 1b67a0c6-3830-c6e6-3008-ce525ea6796b" + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.712+1400 [ERROR] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: failed to appendEntries to: peer="{Voter server-1b67a0c6-3830-c6e6-3008-ce525ea6796b 1b67a0c6-3830-c6e6-3008-ce525ea6796b}" error="failed to connect to peer: 1b67a0c6-3830-c6e6-3008-ce525ea6796b" +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.721+1400 [ERROR] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: failed to heartbeat to: peer=1b67a0c6-3830-c6e6-3008-ce525ea6796b error="failed to connect to peer: 1b67a0c6-3830-c6e6-3008-ce525ea6796b" + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.721+1400 [ERROR] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: failed to heartbeat to: peer=1b67a0c6-3830-c6e6-3008-ce525ea6796b error="failed to connect to peer: 1b67a0c6-3830-c6e6-3008-ce525ea6796b" +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.731+1400 [ERROR] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: failed to appendEntries to: peer="{Voter server-1b67a0c6-3830-c6e6-3008-ce525ea6796b 1b67a0c6-3830-c6e6-3008-ce525ea6796b}" error="failed to connect to peer: 1b67a0c6-3830-c6e6-3008-ce525ea6796b" + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.731+1400 [ERROR] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: failed to appendEntries to: peer="{Voter server-1b67a0c6-3830-c6e6-3008-ce525ea6796b 1b67a0c6-3830-c6e6-3008-ce525ea6796b}" error="failed to connect to peer: 1b67a0c6-3830-c6e6-3008-ce525ea6796b" +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.740+1400 [ERROR] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: failed to heartbeat to: peer=1b67a0c6-3830-c6e6-3008-ce525ea6796b error="failed to connect to peer: 1b67a0c6-3830-c6e6-3008-ce525ea6796b" + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.740+1400 [ERROR] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: failed to heartbeat to: peer=1b67a0c6-3830-c6e6-3008-ce525ea6796b error="failed to connect to peer: 1b67a0c6-3830-c6e6-3008-ce525ea6796b" +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.744+1400 [WARN] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: failed to contact: server-id=server-1b67a0c6-3830-c6e6-3008-ce525ea6796b time=50.422034ms + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.744+1400 [WARN] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: failed to contact: server-id=server-1b67a0c6-3830-c6e6-3008-ce525ea6796b time=50.422034ms +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.754+1400 [WARN] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: Election timeout reached, restarting election + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.754+1400 [WARN] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: Election timeout reached, restarting election +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.754+1400 [INFO] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: entering candidate state: node="Node at 1b67a0c6-3830-c6e6-3008-ce525ea6796b [Candidate]" term=5 + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.754+1400 [INFO] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: entering candidate state: node="Node at 1b67a0c6-3830-c6e6-3008-ce525ea6796b [Candidate]" term=5 +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.754+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to make requestVote RPC: target="{Voter server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6 e07a9304-4edc-3a14-d254-66c4a5bfcaf6}" error="failed to connect to peer: e07a9304-4edc-3a14-d254-66c4a5bfcaf6" + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.754+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to make requestVote RPC: target="{Voter server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6 e07a9304-4edc-3a14-d254-66c4a5bfcaf6}" error="failed to connect to peer: e07a9304-4edc-3a14-d254-66c4a5bfcaf6" +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.754+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to make requestVote RPC: target="{Voter server-b98bac58-4a84-edd5-1003-a3d0df30dc2c b98bac58-4a84-edd5-1003-a3d0df30dc2c}" error="failed to connect to peer: b98bac58-4a84-edd5-1003-a3d0df30dc2c" + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.754+1400 [ERROR] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to make requestVote RPC: target="{Voter server-b98bac58-4a84-edd5-1003-a3d0df30dc2c b98bac58-4a84-edd5-1003-a3d0df30dc2c}" error="failed to connect to peer: b98bac58-4a84-edd5-1003-a3d0df30dc2c" +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.761+1400 [ERROR] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: failed to appendEntries to: peer="{Voter server-1b67a0c6-3830-c6e6-3008-ce525ea6796b 1b67a0c6-3830-c6e6-3008-ce525ea6796b}" error="failed to connect to peer: 1b67a0c6-3830-c6e6-3008-ce525ea6796b" + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.761+1400 [ERROR] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: failed to appendEntries to: peer="{Voter server-1b67a0c6-3830-c6e6-3008-ce525ea6796b 1b67a0c6-3830-c6e6-3008-ce525ea6796b}" error="failed to connect to peer: 1b67a0c6-3830-c6e6-3008-ce525ea6796b" +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.770+1400 [ERROR] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: failed to heartbeat to: peer=1b67a0c6-3830-c6e6-3008-ce525ea6796b error="failed to connect to peer: 1b67a0c6-3830-c6e6-3008-ce525ea6796b" + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.770+1400 [ERROR] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: failed to heartbeat to: peer=1b67a0c6-3830-c6e6-3008-ce525ea6796b error="failed to connect to peer: 1b67a0c6-3830-c6e6-3008-ce525ea6796b" +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.793+1400 [WARN] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: failed to contact: server-id=server-1b67a0c6-3830-c6e6-3008-ce525ea6796b time=99.503796ms + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.793+1400 [WARN] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: failed to contact: server-id=server-1b67a0c6-3830-c6e6-3008-ce525ea6796b time=99.503796ms +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:28.800+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:28.694086379 +1400 +14 m=+5.601604998 (1 nodes), 193.316756ms from start of poll, 501.899342ms from cluster start. Timeout at 2024-06-12 04:32:28.800059463 +1400 +14 m=+5.707578082, 105.973084ms after stability + testing.go:149: cluster: 2024-06-12T04:32:28.800+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:28.694086379 +1400 +14 m=+5.601604998 (1 nodes), 193.316756ms from start of poll, 501.899342ms from cluster start. Timeout at 2024-06-12 04:32:28.800059463 +1400 +14 m=+5.707578082, 105.973084ms after stability + raft_test.go:371: [INFO] Reconnecting Node at 1b67a0c6-3830-c6e6-3008-ce525ea6796b [Candidate] +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.810+1400 [ERROR] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: peer has newer term, stopping replication: peer="{Voter server-1b67a0c6-3830-c6e6-3008-ce525ea6796b 1b67a0c6-3830-c6e6-3008-ce525ea6796b}" + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.810+1400 [ERROR] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: peer has newer term, stopping replication: peer="{Voter server-1b67a0c6-3830-c6e6-3008-ce525ea6796b 1b67a0c6-3830-c6e6-3008-ce525ea6796b}" +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.810+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: entering follower state: follower="Node at e07a9304-4edc-3a14-d254-66c4a5bfcaf6 [Follower]" leader= + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.810+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: entering follower state: follower="Node at e07a9304-4edc-3a14-d254-66c4a5bfcaf6 [Follower]" leader= +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.810+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: aborting pipeline replication: peer="{Voter server-b98bac58-4a84-edd5-1003-a3d0df30dc2c b98bac58-4a84-edd5-1003-a3d0df30dc2c}" + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.810+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: aborting pipeline replication: peer="{Voter server-b98bac58-4a84-edd5-1003-a3d0df30dc2c b98bac58-4a84-edd5-1003-a3d0df30dc2c}" +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.821+1400 [WARN] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: Election timeout reached, restarting election + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.821+1400 [WARN] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: Election timeout reached, restarting election +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.821+1400 [INFO] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: entering candidate state: node="Node at 1b67a0c6-3830-c6e6-3008-ce525ea6796b [Candidate]" term=6 + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.821+1400 [INFO] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: entering candidate state: node="Node at 1b67a0c6-3830-c6e6-3008-ce525ea6796b [Candidate]" term=6 +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.821+1400 [WARN] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: rejecting vote request since our last term is greater: candidate=1b67a0c6-3830-c6e6-3008-ce525ea6796b last-term=4 last-candidate-term=2 + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.821+1400 [WARN] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: rejecting vote request since our last term is greater: candidate=1b67a0c6-3830-c6e6-3008-ce525ea6796b last-term=4 last-candidate-term=2 +testLoggerAdapter verbose: server-b98bac58-4a84-edd5-1003-a3d0df30dc2c: 2024-06-12T04:32:28.821+1400 [WARN] server-b98bac58-4a84-edd5-1003-a3d0df30dc2c: rejecting vote request since we have a leader: from=1b67a0c6-3830-c6e6-3008-ce525ea6796b leader=e07a9304-4edc-3a14-d254-66c4a5bfcaf6 + testing.go:149: server-b98bac58-4a84-edd5-1003-a3d0df30dc2c: 2024-06-12T04:32:28.821+1400 [WARN] server-b98bac58-4a84-edd5-1003-a3d0df30dc2c: rejecting vote request since we have a leader: from=1b67a0c6-3830-c6e6-3008-ce525ea6796b leader=e07a9304-4edc-3a14-d254-66c4a5bfcaf6 +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.886+1400 [WARN] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.886+1400 [WARN] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.886+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: entering candidate state: node="Node at e07a9304-4edc-3a14-d254-66c4a5bfcaf6 [Candidate]" term=7 + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.886+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: entering candidate state: node="Node at e07a9304-4edc-3a14-d254-66c4a5bfcaf6 [Candidate]" term=7 +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.886+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: election won: tally=2 + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.886+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: election won: tally=2 +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.886+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: entering leader state: leader="Node at e07a9304-4edc-3a14-d254-66c4a5bfcaf6 [Leader]" + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.886+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: entering leader state: leader="Node at e07a9304-4edc-3a14-d254-66c4a5bfcaf6 [Leader]" +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.886+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: added peer, starting replication: peer=server-1b67a0c6-3830-c6e6-3008-ce525ea6796b + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.886+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: added peer, starting replication: peer=server-1b67a0c6-3830-c6e6-3008-ce525ea6796b +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.886+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: added peer, starting replication: peer=server-b98bac58-4a84-edd5-1003-a3d0df30dc2c + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.886+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: added peer, starting replication: peer=server-b98bac58-4a84-edd5-1003-a3d0df30dc2c +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.887+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: pipelining replication: peer="{Voter server-b98bac58-4a84-edd5-1003-a3d0df30dc2c b98bac58-4a84-edd5-1003-a3d0df30dc2c}" + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.887+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: pipelining replication: peer="{Voter server-b98bac58-4a84-edd5-1003-a3d0df30dc2c b98bac58-4a84-edd5-1003-a3d0df30dc2c}" +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.887+1400 [INFO] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: entering follower state: follower="Node at 1b67a0c6-3830-c6e6-3008-ce525ea6796b [Follower]" leader= + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.887+1400 [INFO] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: entering follower state: follower="Node at 1b67a0c6-3830-c6e6-3008-ce525ea6796b [Follower]" leader= +testLoggerAdapter verbose: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.887+1400 [WARN] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to get previous log: previous-index=5 last-index=3 error="log not found" + testing.go:149: server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: 2024-06-12T04:32:28.887+1400 [WARN] server-1b67a0c6-3830-c6e6-3008-ce525ea6796b: failed to get previous log: previous-index=5 last-index=3 error="log not found" +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.887+1400 [WARN] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: appendEntries rejected, sending older logs: peer="{Voter server-1b67a0c6-3830-c6e6-3008-ce525ea6796b 1b67a0c6-3830-c6e6-3008-ce525ea6796b}" next=4 + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.887+1400 [WARN] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: appendEntries rejected, sending older logs: peer="{Voter server-1b67a0c6-3830-c6e6-3008-ce525ea6796b 1b67a0c6-3830-c6e6-3008-ce525ea6796b}" next=4 +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.887+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: pipelining replication: peer="{Voter server-1b67a0c6-3830-c6e6-3008-ce525ea6796b 1b67a0c6-3830-c6e6-3008-ce525ea6796b}" + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.887+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: pipelining replication: peer="{Voter server-1b67a0c6-3830-c6e6-3008-ce525ea6796b 1b67a0c6-3830-c6e6-3008-ce525ea6796b}" +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.892+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: aborting pipeline replication: peer="{Voter server-b98bac58-4a84-edd5-1003-a3d0df30dc2c b98bac58-4a84-edd5-1003-a3d0df30dc2c}" + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.892+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: aborting pipeline replication: peer="{Voter server-b98bac58-4a84-edd5-1003-a3d0df30dc2c b98bac58-4a84-edd5-1003-a3d0df30dc2c}" +testLoggerAdapter verbose: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.892+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: aborting pipeline replication: peer="{Voter server-1b67a0c6-3830-c6e6-3008-ce525ea6796b 1b67a0c6-3830-c6e6-3008-ce525ea6796b}" + testing.go:149: server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: 2024-06-12T04:32:28.892+1400 [INFO] server-e07a9304-4edc-3a14-d254-66c4a5bfcaf6: aborting pipeline replication: peer="{Voter server-1b67a0c6-3830-c6e6-3008-ce525ea6796b 1b67a0c6-3830-c6e6-3008-ce525ea6796b}" --- PASS: TestRaft_LeaderFail (0.70s) === RUN TestRaft_BehindFollower -testLoggerAdapter verbose: server-0566346b-4394-87de-0c3c-f1ae0f30ab5c: 2024-06-11T02:29:15.520-1200 [INFO] server-0566346b-4394-87de-0c3c-f1ae0f30ab5c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0566346b-4394-87de-0c3c-f1ae0f30ab5c Address:0566346b-4394-87de-0c3c-f1ae0f30ab5c} {Suffrage:Voter ID:server-1cedac6b-c5a5-15ff-f932-7781e717b985 Address:1cedac6b-c5a5-15ff-f932-7781e717b985} {Suffrage:Voter ID:server-9e65cb0c-900c-14cc-4d41-42fd87efaa99 Address:9e65cb0c-900c-14cc-4d41-42fd87efaa99}]" - testing.go:149: server-0566346b-4394-87de-0c3c-f1ae0f30ab5c: 2024-06-11T02:29:15.520-1200 [INFO] server-0566346b-4394-87de-0c3c-f1ae0f30ab5c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0566346b-4394-87de-0c3c-f1ae0f30ab5c Address:0566346b-4394-87de-0c3c-f1ae0f30ab5c} {Suffrage:Voter ID:server-1cedac6b-c5a5-15ff-f932-7781e717b985 Address:1cedac6b-c5a5-15ff-f932-7781e717b985} {Suffrage:Voter ID:server-9e65cb0c-900c-14cc-4d41-42fd87efaa99 Address:9e65cb0c-900c-14cc-4d41-42fd87efaa99}]" -testLoggerAdapter verbose: server-1cedac6b-c5a5-15ff-f932-7781e717b985: 2024-06-11T02:29:15.520-1200 [INFO] server-1cedac6b-c5a5-15ff-f932-7781e717b985: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0566346b-4394-87de-0c3c-f1ae0f30ab5c Address:0566346b-4394-87de-0c3c-f1ae0f30ab5c} {Suffrage:Voter ID:server-1cedac6b-c5a5-15ff-f932-7781e717b985 Address:1cedac6b-c5a5-15ff-f932-7781e717b985} {Suffrage:Voter ID:server-9e65cb0c-900c-14cc-4d41-42fd87efaa99 Address:9e65cb0c-900c-14cc-4d41-42fd87efaa99}]" - testing.go:149: server-1cedac6b-c5a5-15ff-f932-7781e717b985: 2024-06-11T02:29:15.520-1200 [INFO] server-1cedac6b-c5a5-15ff-f932-7781e717b985: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0566346b-4394-87de-0c3c-f1ae0f30ab5c Address:0566346b-4394-87de-0c3c-f1ae0f30ab5c} {Suffrage:Voter ID:server-1cedac6b-c5a5-15ff-f932-7781e717b985 Address:1cedac6b-c5a5-15ff-f932-7781e717b985} {Suffrage:Voter ID:server-9e65cb0c-900c-14cc-4d41-42fd87efaa99 Address:9e65cb0c-900c-14cc-4d41-42fd87efaa99}]" -testLoggerAdapter verbose: server-9e65cb0c-900c-14cc-4d41-42fd87efaa99: 2024-06-11T02:29:15.521-1200 [INFO] server-9e65cb0c-900c-14cc-4d41-42fd87efaa99: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0566346b-4394-87de-0c3c-f1ae0f30ab5c Address:0566346b-4394-87de-0c3c-f1ae0f30ab5c} {Suffrage:Voter ID:server-1cedac6b-c5a5-15ff-f932-7781e717b985 Address:1cedac6b-c5a5-15ff-f932-7781e717b985} {Suffrage:Voter ID:server-9e65cb0c-900c-14cc-4d41-42fd87efaa99 Address:9e65cb0c-900c-14cc-4d41-42fd87efaa99}]" - testing.go:149: server-9e65cb0c-900c-14cc-4d41-42fd87efaa99: 2024-06-11T02:29:15.521-1200 [INFO] server-9e65cb0c-900c-14cc-4d41-42fd87efaa99: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0566346b-4394-87de-0c3c-f1ae0f30ab5c Address:0566346b-4394-87de-0c3c-f1ae0f30ab5c} {Suffrage:Voter ID:server-1cedac6b-c5a5-15ff-f932-7781e717b985 Address:1cedac6b-c5a5-15ff-f932-7781e717b985} {Suffrage:Voter ID:server-9e65cb0c-900c-14cc-4d41-42fd87efaa99 Address:9e65cb0c-900c-14cc-4d41-42fd87efaa99}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:15.521-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:15.521-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-0566346b-4394-87de-0c3c-f1ae0f30ab5c: 2024-06-11T02:29:15.521-1200 [INFO] server-0566346b-4394-87de-0c3c-f1ae0f30ab5c: entering follower state: follower="Node at 0566346b-4394-87de-0c3c-f1ae0f30ab5c [Follower]" leader= - testing.go:149: server-0566346b-4394-87de-0c3c-f1ae0f30ab5c: 2024-06-11T02:29:15.521-1200 [INFO] server-0566346b-4394-87de-0c3c-f1ae0f30ab5c: entering follower state: follower="Node at 0566346b-4394-87de-0c3c-f1ae0f30ab5c [Follower]" leader= -testLoggerAdapter verbose: server-1cedac6b-c5a5-15ff-f932-7781e717b985: 2024-06-11T02:29:15.521-1200 [INFO] server-1cedac6b-c5a5-15ff-f932-7781e717b985: entering follower state: follower="Node at 1cedac6b-c5a5-15ff-f932-7781e717b985 [Follower]" leader= - testing.go:149: server-1cedac6b-c5a5-15ff-f932-7781e717b985: 2024-06-11T02:29:15.521-1200 [INFO] server-1cedac6b-c5a5-15ff-f932-7781e717b985: entering follower state: follower="Node at 1cedac6b-c5a5-15ff-f932-7781e717b985 [Follower]" leader= -testLoggerAdapter verbose: server-9e65cb0c-900c-14cc-4d41-42fd87efaa99: 2024-06-11T02:29:15.522-1200 [INFO] server-9e65cb0c-900c-14cc-4d41-42fd87efaa99: entering follower state: follower="Node at 9e65cb0c-900c-14cc-4d41-42fd87efaa99 [Follower]" leader= - testing.go:149: server-9e65cb0c-900c-14cc-4d41-42fd87efaa99: 2024-06-11T02:29:15.522-1200 [INFO] server-9e65cb0c-900c-14cc-4d41-42fd87efaa99: entering follower state: follower="Node at 9e65cb0c-900c-14cc-4d41-42fd87efaa99 [Follower]" leader= -testLoggerAdapter verbose: server-1cedac6b-c5a5-15ff-f932-7781e717b985: 2024-06-11T02:29:15.589-1200 [WARN] server-1cedac6b-c5a5-15ff-f932-7781e717b985: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-1cedac6b-c5a5-15ff-f932-7781e717b985: 2024-06-11T02:29:15.589-1200 [WARN] server-1cedac6b-c5a5-15ff-f932-7781e717b985: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-1cedac6b-c5a5-15ff-f932-7781e717b985: 2024-06-11T02:29:15.589-1200 [INFO] server-1cedac6b-c5a5-15ff-f932-7781e717b985: entering candidate state: node="Node at 1cedac6b-c5a5-15ff-f932-7781e717b985 [Candidate]" term=2 - testing.go:149: server-1cedac6b-c5a5-15ff-f932-7781e717b985: 2024-06-11T02:29:15.589-1200 [INFO] server-1cedac6b-c5a5-15ff-f932-7781e717b985: entering candidate state: node="Node at 1cedac6b-c5a5-15ff-f932-7781e717b985 [Candidate]" term=2 -testLoggerAdapter verbose: server-1cedac6b-c5a5-15ff-f932-7781e717b985: 2024-06-11T02:29:15.589-1200 [INFO] server-1cedac6b-c5a5-15ff-f932-7781e717b985: election won: tally=2 - testing.go:149: server-1cedac6b-c5a5-15ff-f932-7781e717b985: 2024-06-11T02:29:15.589-1200 [INFO] server-1cedac6b-c5a5-15ff-f932-7781e717b985: election won: tally=2 -testLoggerAdapter verbose: server-1cedac6b-c5a5-15ff-f932-7781e717b985: 2024-06-11T02:29:15.589-1200 [INFO] server-1cedac6b-c5a5-15ff-f932-7781e717b985: entering leader state: leader="Node at 1cedac6b-c5a5-15ff-f932-7781e717b985 [Leader]" - testing.go:149: server-1cedac6b-c5a5-15ff-f932-7781e717b985: 2024-06-11T02:29:15.589-1200 [INFO] server-1cedac6b-c5a5-15ff-f932-7781e717b985: entering leader state: leader="Node at 1cedac6b-c5a5-15ff-f932-7781e717b985 [Leader]" -testLoggerAdapter verbose: server-1cedac6b-c5a5-15ff-f932-7781e717b985: 2024-06-11T02:29:15.589-1200 [INFO] server-1cedac6b-c5a5-15ff-f932-7781e717b985: added peer, starting replication: peer=server-0566346b-4394-87de-0c3c-f1ae0f30ab5c - testing.go:149: server-1cedac6b-c5a5-15ff-f932-7781e717b985: 2024-06-11T02:29:15.589-1200 [INFO] server-1cedac6b-c5a5-15ff-f932-7781e717b985: added peer, starting replication: peer=server-0566346b-4394-87de-0c3c-f1ae0f30ab5c -testLoggerAdapter verbose: server-1cedac6b-c5a5-15ff-f932-7781e717b985: 2024-06-11T02:29:15.589-1200 [INFO] server-1cedac6b-c5a5-15ff-f932-7781e717b985: added peer, starting replication: peer=server-9e65cb0c-900c-14cc-4d41-42fd87efaa99 - testing.go:149: server-1cedac6b-c5a5-15ff-f932-7781e717b985: 2024-06-11T02:29:15.589-1200 [INFO] server-1cedac6b-c5a5-15ff-f932-7781e717b985: added peer, starting replication: peer=server-9e65cb0c-900c-14cc-4d41-42fd87efaa99 -testLoggerAdapter verbose: server-1cedac6b-c5a5-15ff-f932-7781e717b985: 2024-06-11T02:29:15.590-1200 [INFO] server-1cedac6b-c5a5-15ff-f932-7781e717b985: pipelining replication: peer="{Voter server-9e65cb0c-900c-14cc-4d41-42fd87efaa99 9e65cb0c-900c-14cc-4d41-42fd87efaa99}" - testing.go:149: server-1cedac6b-c5a5-15ff-f932-7781e717b985: 2024-06-11T02:29:15.590-1200 [INFO] server-1cedac6b-c5a5-15ff-f932-7781e717b985: pipelining replication: peer="{Voter server-9e65cb0c-900c-14cc-4d41-42fd87efaa99 9e65cb0c-900c-14cc-4d41-42fd87efaa99}" -testLoggerAdapter verbose: server-1cedac6b-c5a5-15ff-f932-7781e717b985: 2024-06-11T02:29:15.590-1200 [INFO] server-1cedac6b-c5a5-15ff-f932-7781e717b985: pipelining replication: peer="{Voter server-0566346b-4394-87de-0c3c-f1ae0f30ab5c 0566346b-4394-87de-0c3c-f1ae0f30ab5c}" - testing.go:149: server-1cedac6b-c5a5-15ff-f932-7781e717b985: 2024-06-11T02:29:15.590-1200 [INFO] server-1cedac6b-c5a5-15ff-f932-7781e717b985: pipelining replication: peer="{Voter server-0566346b-4394-87de-0c3c-f1ae0f30ab5c 0566346b-4394-87de-0c3c-f1ae0f30ab5c}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:15.696-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:15.590609602 -1200 -12 m=+6.585982144 (1 nodes), 68.851288ms from start of poll, 70.740961ms from cluster start. Timeout at 2024-06-11 02:29:15.695926413 -1200 -12 m=+6.691298954, 105.31681ms after stability - testing.go:149: cluster: 2024-06-11T02:29:15.696-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:15.590609602 -1200 -12 m=+6.585982144 (1 nodes), 68.851288ms from start of poll, 70.740961ms from cluster start. Timeout at 2024-06-11 02:29:15.695926413 -1200 -12 m=+6.691298954, 105.31681ms after stability -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:15.696-1200 [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2024-06-11T02:29:15.696-1200 [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:15.801-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:15.696180741 -1200 -12 m=+6.691553283 (2 nodes), 2.917µs from start of poll, 176.3121ms from cluster start. Timeout at 2024-06-11 02:29:15.801266848 -1200 -12 m=+6.796639681, 105.086398ms after stability - testing.go:149: cluster: 2024-06-11T02:29:15.801-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:15.696180741 -1200 -12 m=+6.691553283 (2 nodes), 2.917µs from start of poll, 176.3121ms from cluster start. Timeout at 2024-06-11 02:29:15.801266848 -1200 -12 m=+6.796639681, 105.086398ms after stability -testLoggerAdapter verbose: server-1cedac6b-c5a5-15ff-f932-7781e717b985: 2024-06-11T02:29:15.801-1200 [ERROR] server-1cedac6b-c5a5-15ff-f932-7781e717b985: failed to pipeline appendEntries: peer="{Voter server-0566346b-4394-87de-0c3c-f1ae0f30ab5c 0566346b-4394-87de-0c3c-f1ae0f30ab5c}" error="append pipeline closed" - testing.go:149: server-1cedac6b-c5a5-15ff-f932-7781e717b985: 2024-06-11T02:29:15.801-1200 [ERROR] server-1cedac6b-c5a5-15ff-f932-7781e717b985: failed to pipeline appendEntries: peer="{Voter server-0566346b-4394-87de-0c3c-f1ae0f30ab5c 0566346b-4394-87de-0c3c-f1ae0f30ab5c}" error="append pipeline closed" -testLoggerAdapter verbose: server-1cedac6b-c5a5-15ff-f932-7781e717b985: 2024-06-11T02:29:15.801-1200 [INFO] server-1cedac6b-c5a5-15ff-f932-7781e717b985: aborting pipeline replication: peer="{Voter server-0566346b-4394-87de-0c3c-f1ae0f30ab5c 0566346b-4394-87de-0c3c-f1ae0f30ab5c}" - testing.go:149: server-1cedac6b-c5a5-15ff-f932-7781e717b985: 2024-06-11T02:29:15.801-1200 [INFO] server-1cedac6b-c5a5-15ff-f932-7781e717b985: aborting pipeline replication: peer="{Voter server-0566346b-4394-87de-0c3c-f1ae0f30ab5c 0566346b-4394-87de-0c3c-f1ae0f30ab5c}" -testLoggerAdapter verbose: server-1cedac6b-c5a5-15ff-f932-7781e717b985: 2024-06-11T02:29:15.802-1200 [ERROR] server-1cedac6b-c5a5-15ff-f932-7781e717b985: failed to appendEntries to: peer="{Voter server-0566346b-4394-87de-0c3c-f1ae0f30ab5c 0566346b-4394-87de-0c3c-f1ae0f30ab5c}" error="failed to connect to peer: 0566346b-4394-87de-0c3c-f1ae0f30ab5c" - testing.go:149: server-1cedac6b-c5a5-15ff-f932-7781e717b985: 2024-06-11T02:29:15.802-1200 [ERROR] server-1cedac6b-c5a5-15ff-f932-7781e717b985: failed to appendEntries to: peer="{Voter server-0566346b-4394-87de-0c3c-f1ae0f30ab5c 0566346b-4394-87de-0c3c-f1ae0f30ab5c}" error="failed to connect to peer: 0566346b-4394-87de-0c3c-f1ae0f30ab5c" -testLoggerAdapter verbose: server-1cedac6b-c5a5-15ff-f932-7781e717b985: 2024-06-11T02:29:15.807-1200 [ERROR] server-1cedac6b-c5a5-15ff-f932-7781e717b985: failed to heartbeat to: peer=0566346b-4394-87de-0c3c-f1ae0f30ab5c error="failed to connect to peer: 0566346b-4394-87de-0c3c-f1ae0f30ab5c" - testing.go:149: server-1cedac6b-c5a5-15ff-f932-7781e717b985: 2024-06-11T02:29:15.807-1200 [ERROR] server-1cedac6b-c5a5-15ff-f932-7781e717b985: failed to heartbeat to: peer=0566346b-4394-87de-0c3c-f1ae0f30ab5c error="failed to connect to peer: 0566346b-4394-87de-0c3c-f1ae0f30ab5c" +testLoggerAdapter verbose: server-69be2739-89bd-c296-c04b-dfebbfb903ad: 2024-06-12T04:32:28.894+1400 [INFO] server-69be2739-89bd-c296-c04b-dfebbfb903ad: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-69be2739-89bd-c296-c04b-dfebbfb903ad Address:69be2739-89bd-c296-c04b-dfebbfb903ad} {Suffrage:Voter ID:server-48c20aa7-07e6-d893-c52d-be0ff43926c7 Address:48c20aa7-07e6-d893-c52d-be0ff43926c7} {Suffrage:Voter ID:server-97313dad-c6e1-76ce-4f67-c599c5c35287 Address:97313dad-c6e1-76ce-4f67-c599c5c35287}]" + testing.go:149: server-69be2739-89bd-c296-c04b-dfebbfb903ad: 2024-06-12T04:32:28.894+1400 [INFO] server-69be2739-89bd-c296-c04b-dfebbfb903ad: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-69be2739-89bd-c296-c04b-dfebbfb903ad Address:69be2739-89bd-c296-c04b-dfebbfb903ad} {Suffrage:Voter ID:server-48c20aa7-07e6-d893-c52d-be0ff43926c7 Address:48c20aa7-07e6-d893-c52d-be0ff43926c7} {Suffrage:Voter ID:server-97313dad-c6e1-76ce-4f67-c599c5c35287 Address:97313dad-c6e1-76ce-4f67-c599c5c35287}]" +testLoggerAdapter verbose: server-48c20aa7-07e6-d893-c52d-be0ff43926c7: 2024-06-12T04:32:28.895+1400 [INFO] server-48c20aa7-07e6-d893-c52d-be0ff43926c7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-69be2739-89bd-c296-c04b-dfebbfb903ad Address:69be2739-89bd-c296-c04b-dfebbfb903ad} {Suffrage:Voter ID:server-48c20aa7-07e6-d893-c52d-be0ff43926c7 Address:48c20aa7-07e6-d893-c52d-be0ff43926c7} {Suffrage:Voter ID:server-97313dad-c6e1-76ce-4f67-c599c5c35287 Address:97313dad-c6e1-76ce-4f67-c599c5c35287}]" + testing.go:149: server-48c20aa7-07e6-d893-c52d-be0ff43926c7: 2024-06-12T04:32:28.895+1400 [INFO] server-48c20aa7-07e6-d893-c52d-be0ff43926c7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-69be2739-89bd-c296-c04b-dfebbfb903ad Address:69be2739-89bd-c296-c04b-dfebbfb903ad} {Suffrage:Voter ID:server-48c20aa7-07e6-d893-c52d-be0ff43926c7 Address:48c20aa7-07e6-d893-c52d-be0ff43926c7} {Suffrage:Voter ID:server-97313dad-c6e1-76ce-4f67-c599c5c35287 Address:97313dad-c6e1-76ce-4f67-c599c5c35287}]" +testLoggerAdapter verbose: server-97313dad-c6e1-76ce-4f67-c599c5c35287: 2024-06-12T04:32:28.895+1400 [INFO] server-97313dad-c6e1-76ce-4f67-c599c5c35287: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-69be2739-89bd-c296-c04b-dfebbfb903ad Address:69be2739-89bd-c296-c04b-dfebbfb903ad} {Suffrage:Voter ID:server-48c20aa7-07e6-d893-c52d-be0ff43926c7 Address:48c20aa7-07e6-d893-c52d-be0ff43926c7} {Suffrage:Voter ID:server-97313dad-c6e1-76ce-4f67-c599c5c35287 Address:97313dad-c6e1-76ce-4f67-c599c5c35287}]" + testing.go:149: server-97313dad-c6e1-76ce-4f67-c599c5c35287: 2024-06-12T04:32:28.895+1400 [INFO] server-97313dad-c6e1-76ce-4f67-c599c5c35287: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-69be2739-89bd-c296-c04b-dfebbfb903ad Address:69be2739-89bd-c296-c04b-dfebbfb903ad} {Suffrage:Voter ID:server-48c20aa7-07e6-d893-c52d-be0ff43926c7 Address:48c20aa7-07e6-d893-c52d-be0ff43926c7} {Suffrage:Voter ID:server-97313dad-c6e1-76ce-4f67-c599c5c35287 Address:97313dad-c6e1-76ce-4f67-c599c5c35287}]" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:28.895+1400 [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-06-12T04:32:28.895+1400 [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-69be2739-89bd-c296-c04b-dfebbfb903ad: 2024-06-12T04:32:28.895+1400 [INFO] server-69be2739-89bd-c296-c04b-dfebbfb903ad: entering follower state: follower="Node at 69be2739-89bd-c296-c04b-dfebbfb903ad [Follower]" leader= + testing.go:149: server-69be2739-89bd-c296-c04b-dfebbfb903ad: 2024-06-12T04:32:28.895+1400 [INFO] server-69be2739-89bd-c296-c04b-dfebbfb903ad: entering follower state: follower="Node at 69be2739-89bd-c296-c04b-dfebbfb903ad [Follower]" leader= +testLoggerAdapter verbose: server-48c20aa7-07e6-d893-c52d-be0ff43926c7: 2024-06-12T04:32:28.895+1400 [INFO] server-48c20aa7-07e6-d893-c52d-be0ff43926c7: entering follower state: follower="Node at 48c20aa7-07e6-d893-c52d-be0ff43926c7 [Follower]" leader= + testing.go:149: server-48c20aa7-07e6-d893-c52d-be0ff43926c7: 2024-06-12T04:32:28.895+1400 [INFO] server-48c20aa7-07e6-d893-c52d-be0ff43926c7: entering follower state: follower="Node at 48c20aa7-07e6-d893-c52d-be0ff43926c7 [Follower]" leader= +testLoggerAdapter verbose: server-97313dad-c6e1-76ce-4f67-c599c5c35287: 2024-06-12T04:32:28.895+1400 [INFO] server-97313dad-c6e1-76ce-4f67-c599c5c35287: entering follower state: follower="Node at 97313dad-c6e1-76ce-4f67-c599c5c35287 [Follower]" leader= + testing.go:149: server-97313dad-c6e1-76ce-4f67-c599c5c35287: 2024-06-12T04:32:28.895+1400 [INFO] server-97313dad-c6e1-76ce-4f67-c599c5c35287: entering follower state: follower="Node at 97313dad-c6e1-76ce-4f67-c599c5c35287 [Follower]" leader= +testLoggerAdapter verbose: server-97313dad-c6e1-76ce-4f67-c599c5c35287: 2024-06-12T04:32:28.980+1400 [WARN] server-97313dad-c6e1-76ce-4f67-c599c5c35287: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-97313dad-c6e1-76ce-4f67-c599c5c35287: 2024-06-12T04:32:28.980+1400 [WARN] server-97313dad-c6e1-76ce-4f67-c599c5c35287: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-97313dad-c6e1-76ce-4f67-c599c5c35287: 2024-06-12T04:32:28.980+1400 [INFO] server-97313dad-c6e1-76ce-4f67-c599c5c35287: entering candidate state: node="Node at 97313dad-c6e1-76ce-4f67-c599c5c35287 [Candidate]" term=2 + testing.go:149: server-97313dad-c6e1-76ce-4f67-c599c5c35287: 2024-06-12T04:32:28.980+1400 [INFO] server-97313dad-c6e1-76ce-4f67-c599c5c35287: entering candidate state: node="Node at 97313dad-c6e1-76ce-4f67-c599c5c35287 [Candidate]" term=2 +testLoggerAdapter verbose: server-97313dad-c6e1-76ce-4f67-c599c5c35287: 2024-06-12T04:32:28.980+1400 [INFO] server-97313dad-c6e1-76ce-4f67-c599c5c35287: election won: tally=2 + testing.go:149: server-97313dad-c6e1-76ce-4f67-c599c5c35287: 2024-06-12T04:32:28.980+1400 [INFO] server-97313dad-c6e1-76ce-4f67-c599c5c35287: election won: tally=2 +testLoggerAdapter verbose: server-97313dad-c6e1-76ce-4f67-c599c5c35287: 2024-06-12T04:32:28.980+1400 [INFO] server-97313dad-c6e1-76ce-4f67-c599c5c35287: entering leader state: leader="Node at 97313dad-c6e1-76ce-4f67-c599c5c35287 [Leader]" + testing.go:149: server-97313dad-c6e1-76ce-4f67-c599c5c35287: 2024-06-12T04:32:28.980+1400 [INFO] server-97313dad-c6e1-76ce-4f67-c599c5c35287: entering leader state: leader="Node at 97313dad-c6e1-76ce-4f67-c599c5c35287 [Leader]" +testLoggerAdapter verbose: server-97313dad-c6e1-76ce-4f67-c599c5c35287: 2024-06-12T04:32:28.980+1400 [INFO] server-97313dad-c6e1-76ce-4f67-c599c5c35287: added peer, starting replication: peer=server-69be2739-89bd-c296-c04b-dfebbfb903ad + testing.go:149: server-97313dad-c6e1-76ce-4f67-c599c5c35287: 2024-06-12T04:32:28.980+1400 [INFO] server-97313dad-c6e1-76ce-4f67-c599c5c35287: added peer, starting replication: peer=server-69be2739-89bd-c296-c04b-dfebbfb903ad +testLoggerAdapter verbose: server-97313dad-c6e1-76ce-4f67-c599c5c35287: 2024-06-12T04:32:28.981+1400 [INFO] server-97313dad-c6e1-76ce-4f67-c599c5c35287: added peer, starting replication: peer=server-48c20aa7-07e6-d893-c52d-be0ff43926c7 + testing.go:149: server-97313dad-c6e1-76ce-4f67-c599c5c35287: 2024-06-12T04:32:28.981+1400 [INFO] server-97313dad-c6e1-76ce-4f67-c599c5c35287: added peer, starting replication: peer=server-48c20aa7-07e6-d893-c52d-be0ff43926c7 +testLoggerAdapter verbose: server-97313dad-c6e1-76ce-4f67-c599c5c35287: 2024-06-12T04:32:28.981+1400 [INFO] server-97313dad-c6e1-76ce-4f67-c599c5c35287: pipelining replication: peer="{Voter server-48c20aa7-07e6-d893-c52d-be0ff43926c7 48c20aa7-07e6-d893-c52d-be0ff43926c7}" + testing.go:149: server-97313dad-c6e1-76ce-4f67-c599c5c35287: 2024-06-12T04:32:28.981+1400 [INFO] server-97313dad-c6e1-76ce-4f67-c599c5c35287: pipelining replication: peer="{Voter server-48c20aa7-07e6-d893-c52d-be0ff43926c7 48c20aa7-07e6-d893-c52d-be0ff43926c7}" +testLoggerAdapter verbose: server-97313dad-c6e1-76ce-4f67-c599c5c35287: 2024-06-12T04:32:28.981+1400 [INFO] server-97313dad-c6e1-76ce-4f67-c599c5c35287: pipelining replication: peer="{Voter server-69be2739-89bd-c296-c04b-dfebbfb903ad 69be2739-89bd-c296-c04b-dfebbfb903ad}" + testing.go:149: server-97313dad-c6e1-76ce-4f67-c599c5c35287: 2024-06-12T04:32:28.981+1400 [INFO] server-97313dad-c6e1-76ce-4f67-c599c5c35287: pipelining replication: peer="{Voter server-69be2739-89bd-c296-c04b-dfebbfb903ad 69be2739-89bd-c296-c04b-dfebbfb903ad}" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:29.086+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:28.981399568 +1400 +14 m=+5.888918147 (1 nodes), 85.900323ms from start of poll, 86.606098ms from cluster start. Timeout at 2024-06-12 04:32:29.086414366 +1400 +14 m=+5.993932985, 105.014838ms after stability + testing.go:149: cluster: 2024-06-12T04:32:29.086+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:28.981399568 +1400 +14 m=+5.888918147 (1 nodes), 85.900323ms from start of poll, 86.606098ms from cluster start. Timeout at 2024-06-12 04:32:29.086414366 +1400 +14 m=+5.993932985, 105.014838ms after stability +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:29.086+1400 [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-06-12T04:32:29.086+1400 [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:29.192+1400 [INFO] stable state for Follower reached at 2024-06-12 04:32:29.086612239 +1400 +14 m=+5.994130818 (2 nodes), 1.72µs from start of poll, 191.818769ms from cluster start. Timeout at 2024-06-12 04:32:29.192657641 +1400 +14 m=+6.100176260, 106.045442ms after stability + testing.go:149: cluster: 2024-06-12T04:32:29.192+1400 [INFO] stable state for Follower reached at 2024-06-12 04:32:29.086612239 +1400 +14 m=+5.994130818 (2 nodes), 1.72µs from start of poll, 191.818769ms from cluster start. Timeout at 2024-06-12 04:32:29.192657641 +1400 +14 m=+6.100176260, 106.045442ms after stability +testLoggerAdapter verbose: server-97313dad-c6e1-76ce-4f67-c599c5c35287: 2024-06-12T04:32:29.192+1400 [ERROR] server-97313dad-c6e1-76ce-4f67-c599c5c35287: failed to pipeline appendEntries: peer="{Voter server-69be2739-89bd-c296-c04b-dfebbfb903ad 69be2739-89bd-c296-c04b-dfebbfb903ad}" error="append pipeline closed" + testing.go:149: server-97313dad-c6e1-76ce-4f67-c599c5c35287: 2024-06-12T04:32:29.192+1400 [ERROR] server-97313dad-c6e1-76ce-4f67-c599c5c35287: failed to pipeline appendEntries: peer="{Voter server-69be2739-89bd-c296-c04b-dfebbfb903ad 69be2739-89bd-c296-c04b-dfebbfb903ad}" error="append pipeline closed" +testLoggerAdapter verbose: server-97313dad-c6e1-76ce-4f67-c599c5c35287: 2024-06-12T04:32:29.192+1400 [INFO] server-97313dad-c6e1-76ce-4f67-c599c5c35287: aborting pipeline replication: peer="{Voter server-69be2739-89bd-c296-c04b-dfebbfb903ad 69be2739-89bd-c296-c04b-dfebbfb903ad}" + testing.go:149: server-97313dad-c6e1-76ce-4f67-c599c5c35287: 2024-06-12T04:32:29.192+1400 [INFO] server-97313dad-c6e1-76ce-4f67-c599c5c35287: aborting pipeline replication: peer="{Voter server-69be2739-89bd-c296-c04b-dfebbfb903ad 69be2739-89bd-c296-c04b-dfebbfb903ad}" +testLoggerAdapter verbose: server-97313dad-c6e1-76ce-4f67-c599c5c35287: 2024-06-12T04:32:29.193+1400 [ERROR] server-97313dad-c6e1-76ce-4f67-c599c5c35287: failed to appendEntries to: peer="{Voter server-69be2739-89bd-c296-c04b-dfebbfb903ad 69be2739-89bd-c296-c04b-dfebbfb903ad}" error="failed to connect to peer: 69be2739-89bd-c296-c04b-dfebbfb903ad" + testing.go:149: server-97313dad-c6e1-76ce-4f67-c599c5c35287: 2024-06-12T04:32:29.193+1400 [ERROR] server-97313dad-c6e1-76ce-4f67-c599c5c35287: failed to appendEntries to: peer="{Voter server-69be2739-89bd-c296-c04b-dfebbfb903ad 69be2739-89bd-c296-c04b-dfebbfb903ad}" error="failed to connect to peer: 69be2739-89bd-c296-c04b-dfebbfb903ad" raft_test.go:420: [INFO] Finished apply without behind follower -testLoggerAdapter verbose: server-1cedac6b-c5a5-15ff-f932-7781e717b985: 2024-06-11T02:29:15.812-1200 [INFO] server-1cedac6b-c5a5-15ff-f932-7781e717b985: pipelining replication: peer="{Voter server-0566346b-4394-87de-0c3c-f1ae0f30ab5c 0566346b-4394-87de-0c3c-f1ae0f30ab5c}" - testing.go:149: server-1cedac6b-c5a5-15ff-f932-7781e717b985: 2024-06-11T02:29:15.812-1200 [INFO] server-1cedac6b-c5a5-15ff-f932-7781e717b985: pipelining replication: peer="{Voter server-0566346b-4394-87de-0c3c-f1ae0f30ab5c 0566346b-4394-87de-0c3c-f1ae0f30ab5c}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:15.822-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:15.822-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:15.928-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:15.822771901 -1200 -12 m=+6.818144443 (1 nodes), 3.791µs from start of poll, 302.90326ms from cluster start. Timeout at 2024-06-11 02:29:15.92843783 -1200 -12 m=+6.923810664, 105.666221ms after stability - testing.go:149: cluster: 2024-06-11T02:29:15.928-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:15.822771901 -1200 -12 m=+6.818144443 (1 nodes), 3.791µs from start of poll, 302.90326ms from cluster start. Timeout at 2024-06-11 02:29:15.92843783 -1200 -12 m=+6.923810664, 105.666221ms after stability -testLoggerAdapter verbose: server-1cedac6b-c5a5-15ff-f932-7781e717b985: 2024-06-11T02:29:15.928-1200 [INFO] server-1cedac6b-c5a5-15ff-f932-7781e717b985: aborting pipeline replication: peer="{Voter server-9e65cb0c-900c-14cc-4d41-42fd87efaa99 9e65cb0c-900c-14cc-4d41-42fd87efaa99}" - testing.go:149: server-1cedac6b-c5a5-15ff-f932-7781e717b985: 2024-06-11T02:29:15.928-1200 [INFO] server-1cedac6b-c5a5-15ff-f932-7781e717b985: aborting pipeline replication: peer="{Voter server-9e65cb0c-900c-14cc-4d41-42fd87efaa99 9e65cb0c-900c-14cc-4d41-42fd87efaa99}" -testLoggerAdapter verbose: server-1cedac6b-c5a5-15ff-f932-7781e717b985: 2024-06-11T02:29:15.928-1200 [INFO] server-1cedac6b-c5a5-15ff-f932-7781e717b985: aborting pipeline replication: peer="{Voter server-0566346b-4394-87de-0c3c-f1ae0f30ab5c 0566346b-4394-87de-0c3c-f1ae0f30ab5c}" - testing.go:149: server-1cedac6b-c5a5-15ff-f932-7781e717b985: 2024-06-11T02:29:15.928-1200 [INFO] server-1cedac6b-c5a5-15ff-f932-7781e717b985: aborting pipeline replication: peer="{Voter server-0566346b-4394-87de-0c3c-f1ae0f30ab5c 0566346b-4394-87de-0c3c-f1ae0f30ab5c}" ---- PASS: TestRaft_BehindFollower (0.41s) +testLoggerAdapter verbose: server-97313dad-c6e1-76ce-4f67-c599c5c35287: 2024-06-12T04:32:29.203+1400 [INFO] server-97313dad-c6e1-76ce-4f67-c599c5c35287: pipelining replication: peer="{Voter server-69be2739-89bd-c296-c04b-dfebbfb903ad 69be2739-89bd-c296-c04b-dfebbfb903ad}" + testing.go:149: server-97313dad-c6e1-76ce-4f67-c599c5c35287: 2024-06-12T04:32:29.203+1400 [INFO] server-97313dad-c6e1-76ce-4f67-c599c5c35287: pipelining replication: peer="{Voter server-69be2739-89bd-c296-c04b-dfebbfb903ad 69be2739-89bd-c296-c04b-dfebbfb903ad}" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:29.213+1400 [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-06-12T04:32:29.213+1400 [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:29.319+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:29.213744845 +1400 +14 m=+6.121263464 (1 nodes), 1.44µs from start of poll, 318.951415ms from cluster start. Timeout at 2024-06-12 04:32:29.319592134 +1400 +14 m=+6.227110753, 105.847289ms after stability + testing.go:149: cluster: 2024-06-12T04:32:29.319+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:29.213744845 +1400 +14 m=+6.121263464 (1 nodes), 1.44µs from start of poll, 318.951415ms from cluster start. Timeout at 2024-06-12 04:32:29.319592134 +1400 +14 m=+6.227110753, 105.847289ms after stability +testLoggerAdapter verbose: server-97313dad-c6e1-76ce-4f67-c599c5c35287: 2024-06-12T04:32:29.319+1400 [INFO] server-97313dad-c6e1-76ce-4f67-c599c5c35287: aborting pipeline replication: peer="{Voter server-48c20aa7-07e6-d893-c52d-be0ff43926c7 48c20aa7-07e6-d893-c52d-be0ff43926c7}" + testing.go:149: server-97313dad-c6e1-76ce-4f67-c599c5c35287: 2024-06-12T04:32:29.319+1400 [INFO] server-97313dad-c6e1-76ce-4f67-c599c5c35287: aborting pipeline replication: peer="{Voter server-48c20aa7-07e6-d893-c52d-be0ff43926c7 48c20aa7-07e6-d893-c52d-be0ff43926c7}" +testLoggerAdapter verbose: server-97313dad-c6e1-76ce-4f67-c599c5c35287: 2024-06-12T04:32:29.319+1400 [INFO] server-97313dad-c6e1-76ce-4f67-c599c5c35287: aborting pipeline replication: peer="{Voter server-69be2739-89bd-c296-c04b-dfebbfb903ad 69be2739-89bd-c296-c04b-dfebbfb903ad}" + testing.go:149: server-97313dad-c6e1-76ce-4f67-c599c5c35287: 2024-06-12T04:32:29.319+1400 [INFO] server-97313dad-c6e1-76ce-4f67-c599c5c35287: aborting pipeline replication: peer="{Voter server-69be2739-89bd-c296-c04b-dfebbfb903ad 69be2739-89bd-c296-c04b-dfebbfb903ad}" +--- PASS: TestRaft_BehindFollower (0.43s) === RUN TestRaft_ApplyNonLeader -testLoggerAdapter verbose: server-4f55793f-3fd9-8b05-b3ae-1043d03468cd: 2024-06-11T02:29:15.934-1200 [INFO] server-4f55793f-3fd9-8b05-b3ae-1043d03468cd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4f55793f-3fd9-8b05-b3ae-1043d03468cd Address:4f55793f-3fd9-8b05-b3ae-1043d03468cd} {Suffrage:Voter ID:server-e4b41429-0c64-9352-0320-fd66b8f16c54 Address:e4b41429-0c64-9352-0320-fd66b8f16c54} {Suffrage:Voter ID:server-7f31fce5-1703-40b7-4515-df507e00075b Address:7f31fce5-1703-40b7-4515-df507e00075b}]" - testing.go:149: server-4f55793f-3fd9-8b05-b3ae-1043d03468cd: 2024-06-11T02:29:15.934-1200 [INFO] server-4f55793f-3fd9-8b05-b3ae-1043d03468cd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4f55793f-3fd9-8b05-b3ae-1043d03468cd Address:4f55793f-3fd9-8b05-b3ae-1043d03468cd} {Suffrage:Voter ID:server-e4b41429-0c64-9352-0320-fd66b8f16c54 Address:e4b41429-0c64-9352-0320-fd66b8f16c54} {Suffrage:Voter ID:server-7f31fce5-1703-40b7-4515-df507e00075b Address:7f31fce5-1703-40b7-4515-df507e00075b}]" -testLoggerAdapter verbose: server-4f55793f-3fd9-8b05-b3ae-1043d03468cd: 2024-06-11T02:29:15.935-1200 [INFO] server-4f55793f-3fd9-8b05-b3ae-1043d03468cd: entering follower state: follower="Node at 4f55793f-3fd9-8b05-b3ae-1043d03468cd [Follower]" leader= - testing.go:149: server-4f55793f-3fd9-8b05-b3ae-1043d03468cd: 2024-06-11T02:29:15.935-1200 [INFO] server-4f55793f-3fd9-8b05-b3ae-1043d03468cd: entering follower state: follower="Node at 4f55793f-3fd9-8b05-b3ae-1043d03468cd [Follower]" leader= -testLoggerAdapter verbose: server-e4b41429-0c64-9352-0320-fd66b8f16c54: 2024-06-11T02:29:15.935-1200 [INFO] server-e4b41429-0c64-9352-0320-fd66b8f16c54: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4f55793f-3fd9-8b05-b3ae-1043d03468cd Address:4f55793f-3fd9-8b05-b3ae-1043d03468cd} {Suffrage:Voter ID:server-e4b41429-0c64-9352-0320-fd66b8f16c54 Address:e4b41429-0c64-9352-0320-fd66b8f16c54} {Suffrage:Voter ID:server-7f31fce5-1703-40b7-4515-df507e00075b Address:7f31fce5-1703-40b7-4515-df507e00075b}]" - testing.go:149: server-e4b41429-0c64-9352-0320-fd66b8f16c54: 2024-06-11T02:29:15.935-1200 [INFO] server-e4b41429-0c64-9352-0320-fd66b8f16c54: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4f55793f-3fd9-8b05-b3ae-1043d03468cd Address:4f55793f-3fd9-8b05-b3ae-1043d03468cd} {Suffrage:Voter ID:server-e4b41429-0c64-9352-0320-fd66b8f16c54 Address:e4b41429-0c64-9352-0320-fd66b8f16c54} {Suffrage:Voter ID:server-7f31fce5-1703-40b7-4515-df507e00075b Address:7f31fce5-1703-40b7-4515-df507e00075b}]" -testLoggerAdapter verbose: server-e4b41429-0c64-9352-0320-fd66b8f16c54: 2024-06-11T02:29:15.935-1200 [INFO] server-e4b41429-0c64-9352-0320-fd66b8f16c54: entering follower state: follower="Node at e4b41429-0c64-9352-0320-fd66b8f16c54 [Follower]" leader= - testing.go:149: server-e4b41429-0c64-9352-0320-fd66b8f16c54: 2024-06-11T02:29:15.935-1200 [INFO] server-e4b41429-0c64-9352-0320-fd66b8f16c54: entering follower state: follower="Node at e4b41429-0c64-9352-0320-fd66b8f16c54 [Follower]" leader= -testLoggerAdapter verbose: server-7f31fce5-1703-40b7-4515-df507e00075b: 2024-06-11T02:29:15.936-1200 [INFO] server-7f31fce5-1703-40b7-4515-df507e00075b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4f55793f-3fd9-8b05-b3ae-1043d03468cd Address:4f55793f-3fd9-8b05-b3ae-1043d03468cd} {Suffrage:Voter ID:server-e4b41429-0c64-9352-0320-fd66b8f16c54 Address:e4b41429-0c64-9352-0320-fd66b8f16c54} {Suffrage:Voter ID:server-7f31fce5-1703-40b7-4515-df507e00075b Address:7f31fce5-1703-40b7-4515-df507e00075b}]" - testing.go:149: server-7f31fce5-1703-40b7-4515-df507e00075b: 2024-06-11T02:29:15.936-1200 [INFO] server-7f31fce5-1703-40b7-4515-df507e00075b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4f55793f-3fd9-8b05-b3ae-1043d03468cd Address:4f55793f-3fd9-8b05-b3ae-1043d03468cd} {Suffrage:Voter ID:server-e4b41429-0c64-9352-0320-fd66b8f16c54 Address:e4b41429-0c64-9352-0320-fd66b8f16c54} {Suffrage:Voter ID:server-7f31fce5-1703-40b7-4515-df507e00075b Address:7f31fce5-1703-40b7-4515-df507e00075b}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:15.936-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:15.936-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-7f31fce5-1703-40b7-4515-df507e00075b: 2024-06-11T02:29:15.936-1200 [INFO] server-7f31fce5-1703-40b7-4515-df507e00075b: entering follower state: follower="Node at 7f31fce5-1703-40b7-4515-df507e00075b [Follower]" leader= - testing.go:149: server-7f31fce5-1703-40b7-4515-df507e00075b: 2024-06-11T02:29:15.936-1200 [INFO] server-7f31fce5-1703-40b7-4515-df507e00075b: entering follower state: follower="Node at 7f31fce5-1703-40b7-4515-df507e00075b [Follower]" leader= -testLoggerAdapter verbose: server-7f31fce5-1703-40b7-4515-df507e00075b: 2024-06-11T02:29:15.991-1200 [WARN] server-7f31fce5-1703-40b7-4515-df507e00075b: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-7f31fce5-1703-40b7-4515-df507e00075b: 2024-06-11T02:29:15.991-1200 [WARN] server-7f31fce5-1703-40b7-4515-df507e00075b: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-7f31fce5-1703-40b7-4515-df507e00075b: 2024-06-11T02:29:15.991-1200 [INFO] server-7f31fce5-1703-40b7-4515-df507e00075b: entering candidate state: node="Node at 7f31fce5-1703-40b7-4515-df507e00075b [Candidate]" term=2 - testing.go:149: server-7f31fce5-1703-40b7-4515-df507e00075b: 2024-06-11T02:29:15.991-1200 [INFO] server-7f31fce5-1703-40b7-4515-df507e00075b: entering candidate state: node="Node at 7f31fce5-1703-40b7-4515-df507e00075b [Candidate]" term=2 -testLoggerAdapter verbose: server-7f31fce5-1703-40b7-4515-df507e00075b: 2024-06-11T02:29:15.991-1200 [INFO] server-7f31fce5-1703-40b7-4515-df507e00075b: election won: tally=2 - testing.go:149: server-7f31fce5-1703-40b7-4515-df507e00075b: 2024-06-11T02:29:15.991-1200 [INFO] server-7f31fce5-1703-40b7-4515-df507e00075b: election won: tally=2 -testLoggerAdapter verbose: server-7f31fce5-1703-40b7-4515-df507e00075b: 2024-06-11T02:29:15.991-1200 [INFO] server-7f31fce5-1703-40b7-4515-df507e00075b: entering leader state: leader="Node at 7f31fce5-1703-40b7-4515-df507e00075b [Leader]" - testing.go:149: server-7f31fce5-1703-40b7-4515-df507e00075b: 2024-06-11T02:29:15.991-1200 [INFO] server-7f31fce5-1703-40b7-4515-df507e00075b: entering leader state: leader="Node at 7f31fce5-1703-40b7-4515-df507e00075b [Leader]" -testLoggerAdapter verbose: server-7f31fce5-1703-40b7-4515-df507e00075b: 2024-06-11T02:29:15.991-1200 [INFO] server-7f31fce5-1703-40b7-4515-df507e00075b: added peer, starting replication: peer=server-4f55793f-3fd9-8b05-b3ae-1043d03468cd - testing.go:149: server-7f31fce5-1703-40b7-4515-df507e00075b: 2024-06-11T02:29:15.991-1200 [INFO] server-7f31fce5-1703-40b7-4515-df507e00075b: added peer, starting replication: peer=server-4f55793f-3fd9-8b05-b3ae-1043d03468cd -testLoggerAdapter verbose: server-7f31fce5-1703-40b7-4515-df507e00075b: 2024-06-11T02:29:15.991-1200 [INFO] server-7f31fce5-1703-40b7-4515-df507e00075b: added peer, starting replication: peer=server-e4b41429-0c64-9352-0320-fd66b8f16c54 - testing.go:149: server-7f31fce5-1703-40b7-4515-df507e00075b: 2024-06-11T02:29:15.991-1200 [INFO] server-7f31fce5-1703-40b7-4515-df507e00075b: added peer, starting replication: peer=server-e4b41429-0c64-9352-0320-fd66b8f16c54 -testLoggerAdapter verbose: server-7f31fce5-1703-40b7-4515-df507e00075b: 2024-06-11T02:29:15.991-1200 [INFO] server-7f31fce5-1703-40b7-4515-df507e00075b: pipelining replication: peer="{Voter server-e4b41429-0c64-9352-0320-fd66b8f16c54 e4b41429-0c64-9352-0320-fd66b8f16c54}" - testing.go:149: server-7f31fce5-1703-40b7-4515-df507e00075b: 2024-06-11T02:29:15.991-1200 [INFO] server-7f31fce5-1703-40b7-4515-df507e00075b: pipelining replication: peer="{Voter server-e4b41429-0c64-9352-0320-fd66b8f16c54 e4b41429-0c64-9352-0320-fd66b8f16c54}" -testLoggerAdapter verbose: server-7f31fce5-1703-40b7-4515-df507e00075b: 2024-06-11T02:29:15.992-1200 [INFO] server-7f31fce5-1703-40b7-4515-df507e00075b: pipelining replication: peer="{Voter server-4f55793f-3fd9-8b05-b3ae-1043d03468cd 4f55793f-3fd9-8b05-b3ae-1043d03468cd}" - testing.go:149: server-7f31fce5-1703-40b7-4515-df507e00075b: 2024-06-11T02:29:15.992-1200 [INFO] server-7f31fce5-1703-40b7-4515-df507e00075b: pipelining replication: peer="{Voter server-4f55793f-3fd9-8b05-b3ae-1043d03468cd 4f55793f-3fd9-8b05-b3ae-1043d03468cd}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:16.098-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:15.992496834 -1200 -12 m=+6.987869667 (1 nodes), 55.723618ms from start of poll, 58.298403ms from cluster start. Timeout at 2024-06-11 02:29:16.09839405 -1200 -12 m=+7.093767175, 105.897508ms after stability - testing.go:149: cluster: 2024-06-11T02:29:16.098-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:15.992496834 -1200 -12 m=+6.987869667 (1 nodes), 55.723618ms from start of poll, 58.298403ms from cluster start. Timeout at 2024-06-11 02:29:16.09839405 -1200 -12 m=+7.093767175, 105.897508ms after stability -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:16.098-1200 [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2024-06-11T02:29:16.098-1200 [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:16.204-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:16.098868874 -1200 -12 m=+7.094241416 (2 nodes), 5.25µs from start of poll, 164.670152ms from cluster start. Timeout at 2024-06-11 02:29:16.204286016 -1200 -12 m=+7.199658558, 105.417142ms after stability - testing.go:149: cluster: 2024-06-11T02:29:16.204-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:16.098868874 -1200 -12 m=+7.094241416 (2 nodes), 5.25µs from start of poll, 164.670152ms from cluster start. Timeout at 2024-06-11 02:29:16.204286016 -1200 -12 m=+7.199658558, 105.417142ms after stability -testLoggerAdapter verbose: server-7f31fce5-1703-40b7-4515-df507e00075b: 2024-06-11T02:29:16.204-1200 [INFO] server-7f31fce5-1703-40b7-4515-df507e00075b: aborting pipeline replication: peer="{Voter server-e4b41429-0c64-9352-0320-fd66b8f16c54 e4b41429-0c64-9352-0320-fd66b8f16c54}" - testing.go:149: server-7f31fce5-1703-40b7-4515-df507e00075b: 2024-06-11T02:29:16.204-1200 [INFO] server-7f31fce5-1703-40b7-4515-df507e00075b: aborting pipeline replication: peer="{Voter server-e4b41429-0c64-9352-0320-fd66b8f16c54 e4b41429-0c64-9352-0320-fd66b8f16c54}" -testLoggerAdapter verbose: server-7f31fce5-1703-40b7-4515-df507e00075b: 2024-06-11T02:29:16.204-1200 [INFO] server-7f31fce5-1703-40b7-4515-df507e00075b: aborting pipeline replication: peer="{Voter server-4f55793f-3fd9-8b05-b3ae-1043d03468cd 4f55793f-3fd9-8b05-b3ae-1043d03468cd}" - testing.go:149: server-7f31fce5-1703-40b7-4515-df507e00075b: 2024-06-11T02:29:16.204-1200 [INFO] server-7f31fce5-1703-40b7-4515-df507e00075b: aborting pipeline replication: peer="{Voter server-4f55793f-3fd9-8b05-b3ae-1043d03468cd 4f55793f-3fd9-8b05-b3ae-1043d03468cd}" +testLoggerAdapter verbose: server-c09d30a2-a922-f086-95e2-c9d01912bed3: 2024-06-12T04:32:29.321+1400 [INFO] server-c09d30a2-a922-f086-95e2-c9d01912bed3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c09d30a2-a922-f086-95e2-c9d01912bed3 Address:c09d30a2-a922-f086-95e2-c9d01912bed3} {Suffrage:Voter ID:server-df5d237b-3e52-babb-d245-bbd701516719 Address:df5d237b-3e52-babb-d245-bbd701516719} {Suffrage:Voter ID:server-3b40fe9f-29c8-e8ac-af9d-c55d7b09c2f6 Address:3b40fe9f-29c8-e8ac-af9d-c55d7b09c2f6}]" + testing.go:149: server-c09d30a2-a922-f086-95e2-c9d01912bed3: 2024-06-12T04:32:29.321+1400 [INFO] server-c09d30a2-a922-f086-95e2-c9d01912bed3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c09d30a2-a922-f086-95e2-c9d01912bed3 Address:c09d30a2-a922-f086-95e2-c9d01912bed3} {Suffrage:Voter ID:server-df5d237b-3e52-babb-d245-bbd701516719 Address:df5d237b-3e52-babb-d245-bbd701516719} {Suffrage:Voter ID:server-3b40fe9f-29c8-e8ac-af9d-c55d7b09c2f6 Address:3b40fe9f-29c8-e8ac-af9d-c55d7b09c2f6}]" +testLoggerAdapter verbose: server-df5d237b-3e52-babb-d245-bbd701516719: 2024-06-12T04:32:29.322+1400 [INFO] server-df5d237b-3e52-babb-d245-bbd701516719: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c09d30a2-a922-f086-95e2-c9d01912bed3 Address:c09d30a2-a922-f086-95e2-c9d01912bed3} {Suffrage:Voter ID:server-df5d237b-3e52-babb-d245-bbd701516719 Address:df5d237b-3e52-babb-d245-bbd701516719} {Suffrage:Voter ID:server-3b40fe9f-29c8-e8ac-af9d-c55d7b09c2f6 Address:3b40fe9f-29c8-e8ac-af9d-c55d7b09c2f6}]" + testing.go:149: server-df5d237b-3e52-babb-d245-bbd701516719: 2024-06-12T04:32:29.322+1400 [INFO] server-df5d237b-3e52-babb-d245-bbd701516719: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c09d30a2-a922-f086-95e2-c9d01912bed3 Address:c09d30a2-a922-f086-95e2-c9d01912bed3} {Suffrage:Voter ID:server-df5d237b-3e52-babb-d245-bbd701516719 Address:df5d237b-3e52-babb-d245-bbd701516719} {Suffrage:Voter ID:server-3b40fe9f-29c8-e8ac-af9d-c55d7b09c2f6 Address:3b40fe9f-29c8-e8ac-af9d-c55d7b09c2f6}]" +testLoggerAdapter verbose: server-3b40fe9f-29c8-e8ac-af9d-c55d7b09c2f6: 2024-06-12T04:32:29.322+1400 [INFO] server-3b40fe9f-29c8-e8ac-af9d-c55d7b09c2f6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c09d30a2-a922-f086-95e2-c9d01912bed3 Address:c09d30a2-a922-f086-95e2-c9d01912bed3} {Suffrage:Voter ID:server-df5d237b-3e52-babb-d245-bbd701516719 Address:df5d237b-3e52-babb-d245-bbd701516719} {Suffrage:Voter ID:server-3b40fe9f-29c8-e8ac-af9d-c55d7b09c2f6 Address:3b40fe9f-29c8-e8ac-af9d-c55d7b09c2f6}]" + testing.go:149: server-3b40fe9f-29c8-e8ac-af9d-c55d7b09c2f6: 2024-06-12T04:32:29.322+1400 [INFO] server-3b40fe9f-29c8-e8ac-af9d-c55d7b09c2f6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c09d30a2-a922-f086-95e2-c9d01912bed3 Address:c09d30a2-a922-f086-95e2-c9d01912bed3} {Suffrage:Voter ID:server-df5d237b-3e52-babb-d245-bbd701516719 Address:df5d237b-3e52-babb-d245-bbd701516719} {Suffrage:Voter ID:server-3b40fe9f-29c8-e8ac-af9d-c55d7b09c2f6 Address:3b40fe9f-29c8-e8ac-af9d-c55d7b09c2f6}]" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:29.322+1400 [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-06-12T04:32:29.322+1400 [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-c09d30a2-a922-f086-95e2-c9d01912bed3: 2024-06-12T04:32:29.322+1400 [INFO] server-c09d30a2-a922-f086-95e2-c9d01912bed3: entering follower state: follower="Node at c09d30a2-a922-f086-95e2-c9d01912bed3 [Follower]" leader= + testing.go:149: server-c09d30a2-a922-f086-95e2-c9d01912bed3: 2024-06-12T04:32:29.322+1400 [INFO] server-c09d30a2-a922-f086-95e2-c9d01912bed3: entering follower state: follower="Node at c09d30a2-a922-f086-95e2-c9d01912bed3 [Follower]" leader= +testLoggerAdapter verbose: server-3b40fe9f-29c8-e8ac-af9d-c55d7b09c2f6: 2024-06-12T04:32:29.322+1400 [INFO] server-3b40fe9f-29c8-e8ac-af9d-c55d7b09c2f6: entering follower state: follower="Node at 3b40fe9f-29c8-e8ac-af9d-c55d7b09c2f6 [Follower]" leader= + testing.go:149: server-3b40fe9f-29c8-e8ac-af9d-c55d7b09c2f6: 2024-06-12T04:32:29.322+1400 [INFO] server-3b40fe9f-29c8-e8ac-af9d-c55d7b09c2f6: entering follower state: follower="Node at 3b40fe9f-29c8-e8ac-af9d-c55d7b09c2f6 [Follower]" leader= +testLoggerAdapter verbose: server-df5d237b-3e52-babb-d245-bbd701516719: 2024-06-12T04:32:29.322+1400 [INFO] server-df5d237b-3e52-babb-d245-bbd701516719: entering follower state: follower="Node at df5d237b-3e52-babb-d245-bbd701516719 [Follower]" leader= + testing.go:149: server-df5d237b-3e52-babb-d245-bbd701516719: 2024-06-12T04:32:29.322+1400 [INFO] server-df5d237b-3e52-babb-d245-bbd701516719: entering follower state: follower="Node at df5d237b-3e52-babb-d245-bbd701516719 [Follower]" leader= +testLoggerAdapter verbose: server-c09d30a2-a922-f086-95e2-c9d01912bed3: 2024-06-12T04:32:29.387+1400 [WARN] server-c09d30a2-a922-f086-95e2-c9d01912bed3: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-c09d30a2-a922-f086-95e2-c9d01912bed3: 2024-06-12T04:32:29.387+1400 [WARN] server-c09d30a2-a922-f086-95e2-c9d01912bed3: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-c09d30a2-a922-f086-95e2-c9d01912bed3: 2024-06-12T04:32:29.387+1400 [INFO] server-c09d30a2-a922-f086-95e2-c9d01912bed3: entering candidate state: node="Node at c09d30a2-a922-f086-95e2-c9d01912bed3 [Candidate]" term=2 + testing.go:149: server-c09d30a2-a922-f086-95e2-c9d01912bed3: 2024-06-12T04:32:29.387+1400 [INFO] server-c09d30a2-a922-f086-95e2-c9d01912bed3: entering candidate state: node="Node at c09d30a2-a922-f086-95e2-c9d01912bed3 [Candidate]" term=2 +testLoggerAdapter verbose: server-c09d30a2-a922-f086-95e2-c9d01912bed3: 2024-06-12T04:32:29.387+1400 [INFO] server-c09d30a2-a922-f086-95e2-c9d01912bed3: election won: tally=2 + testing.go:149: server-c09d30a2-a922-f086-95e2-c9d01912bed3: 2024-06-12T04:32:29.387+1400 [INFO] server-c09d30a2-a922-f086-95e2-c9d01912bed3: election won: tally=2 +testLoggerAdapter verbose: server-c09d30a2-a922-f086-95e2-c9d01912bed3: 2024-06-12T04:32:29.387+1400 [INFO] server-c09d30a2-a922-f086-95e2-c9d01912bed3: entering leader state: leader="Node at c09d30a2-a922-f086-95e2-c9d01912bed3 [Leader]" + testing.go:149: server-c09d30a2-a922-f086-95e2-c9d01912bed3: 2024-06-12T04:32:29.387+1400 [INFO] server-c09d30a2-a922-f086-95e2-c9d01912bed3: entering leader state: leader="Node at c09d30a2-a922-f086-95e2-c9d01912bed3 [Leader]" +testLoggerAdapter verbose: server-c09d30a2-a922-f086-95e2-c9d01912bed3: 2024-06-12T04:32:29.387+1400 [INFO] server-c09d30a2-a922-f086-95e2-c9d01912bed3: added peer, starting replication: peer=server-df5d237b-3e52-babb-d245-bbd701516719 + testing.go:149: server-c09d30a2-a922-f086-95e2-c9d01912bed3: 2024-06-12T04:32:29.387+1400 [INFO] server-c09d30a2-a922-f086-95e2-c9d01912bed3: added peer, starting replication: peer=server-df5d237b-3e52-babb-d245-bbd701516719 +testLoggerAdapter verbose: server-c09d30a2-a922-f086-95e2-c9d01912bed3: 2024-06-12T04:32:29.387+1400 [INFO] server-c09d30a2-a922-f086-95e2-c9d01912bed3: added peer, starting replication: peer=server-3b40fe9f-29c8-e8ac-af9d-c55d7b09c2f6 + testing.go:149: server-c09d30a2-a922-f086-95e2-c9d01912bed3: 2024-06-12T04:32:29.387+1400 [INFO] server-c09d30a2-a922-f086-95e2-c9d01912bed3: added peer, starting replication: peer=server-3b40fe9f-29c8-e8ac-af9d-c55d7b09c2f6 +testLoggerAdapter verbose: server-c09d30a2-a922-f086-95e2-c9d01912bed3: 2024-06-12T04:32:29.387+1400 [INFO] server-c09d30a2-a922-f086-95e2-c9d01912bed3: pipelining replication: peer="{Voter server-df5d237b-3e52-babb-d245-bbd701516719 df5d237b-3e52-babb-d245-bbd701516719}" + testing.go:149: server-c09d30a2-a922-f086-95e2-c9d01912bed3: 2024-06-12T04:32:29.387+1400 [INFO] server-c09d30a2-a922-f086-95e2-c9d01912bed3: pipelining replication: peer="{Voter server-df5d237b-3e52-babb-d245-bbd701516719 df5d237b-3e52-babb-d245-bbd701516719}" +testLoggerAdapter verbose: server-c09d30a2-a922-f086-95e2-c9d01912bed3: 2024-06-12T04:32:29.388+1400 [INFO] server-c09d30a2-a922-f086-95e2-c9d01912bed3: pipelining replication: peer="{Voter server-3b40fe9f-29c8-e8ac-af9d-c55d7b09c2f6 3b40fe9f-29c8-e8ac-af9d-c55d7b09c2f6}" + testing.go:149: server-c09d30a2-a922-f086-95e2-c9d01912bed3: 2024-06-12T04:32:29.388+1400 [INFO] server-c09d30a2-a922-f086-95e2-c9d01912bed3: pipelining replication: peer="{Voter server-3b40fe9f-29c8-e8ac-af9d-c55d7b09c2f6 3b40fe9f-29c8-e8ac-af9d-c55d7b09c2f6}" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:29.493+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:29.38781417 +1400 +14 m=+6.295332749 (1 nodes), 64.989352ms from start of poll, 66.110952ms from cluster start. Timeout at 2024-06-12 04:32:29.493551783 +1400 +14 m=+6.401070362, 105.737613ms after stability + testing.go:149: cluster: 2024-06-12T04:32:29.493+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:29.38781417 +1400 +14 m=+6.295332749 (1 nodes), 64.989352ms from start of poll, 66.110952ms from cluster start. Timeout at 2024-06-12 04:32:29.493551783 +1400 +14 m=+6.401070362, 105.737613ms after stability +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:29.493+1400 [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-06-12T04:32:29.493+1400 [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:29.599+1400 [INFO] stable state for Follower reached at 2024-06-12 04:32:29.493748656 +1400 +14 m=+6.401267235 (2 nodes), 2.12µs from start of poll, 172.045438ms from cluster start. Timeout at 2024-06-12 04:32:29.599256717 +1400 +14 m=+6.506775336, 105.508101ms after stability + testing.go:149: cluster: 2024-06-12T04:32:29.599+1400 [INFO] stable state for Follower reached at 2024-06-12 04:32:29.493748656 +1400 +14 m=+6.401267235 (2 nodes), 2.12µs from start of poll, 172.045438ms from cluster start. Timeout at 2024-06-12 04:32:29.599256717 +1400 +14 m=+6.506775336, 105.508101ms after stability +testLoggerAdapter verbose: server-c09d30a2-a922-f086-95e2-c9d01912bed3: 2024-06-12T04:32:29.599+1400 [INFO] server-c09d30a2-a922-f086-95e2-c9d01912bed3: aborting pipeline replication: peer="{Voter server-df5d237b-3e52-babb-d245-bbd701516719 df5d237b-3e52-babb-d245-bbd701516719}" + testing.go:149: server-c09d30a2-a922-f086-95e2-c9d01912bed3: 2024-06-12T04:32:29.599+1400 [INFO] server-c09d30a2-a922-f086-95e2-c9d01912bed3: aborting pipeline replication: peer="{Voter server-df5d237b-3e52-babb-d245-bbd701516719 df5d237b-3e52-babb-d245-bbd701516719}" +testLoggerAdapter verbose: server-c09d30a2-a922-f086-95e2-c9d01912bed3: 2024-06-12T04:32:29.599+1400 [INFO] server-c09d30a2-a922-f086-95e2-c9d01912bed3: aborting pipeline replication: peer="{Voter server-3b40fe9f-29c8-e8ac-af9d-c55d7b09c2f6 3b40fe9f-29c8-e8ac-af9d-c55d7b09c2f6}" + testing.go:149: server-c09d30a2-a922-f086-95e2-c9d01912bed3: 2024-06-12T04:32:29.599+1400 [INFO] server-c09d30a2-a922-f086-95e2-c9d01912bed3: aborting pipeline replication: peer="{Voter server-3b40fe9f-29c8-e8ac-af9d-c55d7b09c2f6 3b40fe9f-29c8-e8ac-af9d-c55d7b09c2f6}" --- PASS: TestRaft_ApplyNonLeader (0.28s) === RUN TestRaft_ApplyConcurrent -testLoggerAdapter verbose: server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: 2024-06-11T02:29:16.210-1200 [INFO] server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f Address:09059b36-ce8a-ac6c-80d7-59dd1fb3143f} {Suffrage:Voter ID:server-49161ecf-4b53-6dd2-d29f-82d84323ab3a Address:49161ecf-4b53-6dd2-d29f-82d84323ab3a} {Suffrage:Voter ID:server-15a6f5c3-a064-69ef-4b8b-f537c2a7525f Address:15a6f5c3-a064-69ef-4b8b-f537c2a7525f}]" - testing.go:149: server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: 2024-06-11T02:29:16.210-1200 [INFO] server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f Address:09059b36-ce8a-ac6c-80d7-59dd1fb3143f} {Suffrage:Voter ID:server-49161ecf-4b53-6dd2-d29f-82d84323ab3a Address:49161ecf-4b53-6dd2-d29f-82d84323ab3a} {Suffrage:Voter ID:server-15a6f5c3-a064-69ef-4b8b-f537c2a7525f Address:15a6f5c3-a064-69ef-4b8b-f537c2a7525f}]" -testLoggerAdapter verbose: server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: 2024-06-11T02:29:16.210-1200 [INFO] server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: entering follower state: follower="Node at 09059b36-ce8a-ac6c-80d7-59dd1fb3143f [Follower]" leader= - testing.go:149: server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: 2024-06-11T02:29:16.210-1200 [INFO] server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: entering follower state: follower="Node at 09059b36-ce8a-ac6c-80d7-59dd1fb3143f [Follower]" leader= -testLoggerAdapter verbose: server-49161ecf-4b53-6dd2-d29f-82d84323ab3a: 2024-06-11T02:29:16.210-1200 [INFO] server-49161ecf-4b53-6dd2-d29f-82d84323ab3a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f Address:09059b36-ce8a-ac6c-80d7-59dd1fb3143f} {Suffrage:Voter ID:server-49161ecf-4b53-6dd2-d29f-82d84323ab3a Address:49161ecf-4b53-6dd2-d29f-82d84323ab3a} {Suffrage:Voter ID:server-15a6f5c3-a064-69ef-4b8b-f537c2a7525f Address:15a6f5c3-a064-69ef-4b8b-f537c2a7525f}]" - testing.go:149: server-49161ecf-4b53-6dd2-d29f-82d84323ab3a: 2024-06-11T02:29:16.210-1200 [INFO] server-49161ecf-4b53-6dd2-d29f-82d84323ab3a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f Address:09059b36-ce8a-ac6c-80d7-59dd1fb3143f} {Suffrage:Voter ID:server-49161ecf-4b53-6dd2-d29f-82d84323ab3a Address:49161ecf-4b53-6dd2-d29f-82d84323ab3a} {Suffrage:Voter ID:server-15a6f5c3-a064-69ef-4b8b-f537c2a7525f Address:15a6f5c3-a064-69ef-4b8b-f537c2a7525f}]" -testLoggerAdapter verbose: server-49161ecf-4b53-6dd2-d29f-82d84323ab3a: 2024-06-11T02:29:16.211-1200 [INFO] server-49161ecf-4b53-6dd2-d29f-82d84323ab3a: entering follower state: follower="Node at 49161ecf-4b53-6dd2-d29f-82d84323ab3a [Follower]" leader= - testing.go:149: server-49161ecf-4b53-6dd2-d29f-82d84323ab3a: 2024-06-11T02:29:16.211-1200 [INFO] server-49161ecf-4b53-6dd2-d29f-82d84323ab3a: entering follower state: follower="Node at 49161ecf-4b53-6dd2-d29f-82d84323ab3a [Follower]" leader= -testLoggerAdapter verbose: server-15a6f5c3-a064-69ef-4b8b-f537c2a7525f: 2024-06-11T02:29:16.211-1200 [INFO] server-15a6f5c3-a064-69ef-4b8b-f537c2a7525f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f Address:09059b36-ce8a-ac6c-80d7-59dd1fb3143f} {Suffrage:Voter ID:server-49161ecf-4b53-6dd2-d29f-82d84323ab3a Address:49161ecf-4b53-6dd2-d29f-82d84323ab3a} {Suffrage:Voter ID:server-15a6f5c3-a064-69ef-4b8b-f537c2a7525f Address:15a6f5c3-a064-69ef-4b8b-f537c2a7525f}]" - testing.go:149: server-15a6f5c3-a064-69ef-4b8b-f537c2a7525f: 2024-06-11T02:29:16.211-1200 [INFO] server-15a6f5c3-a064-69ef-4b8b-f537c2a7525f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f Address:09059b36-ce8a-ac6c-80d7-59dd1fb3143f} {Suffrage:Voter ID:server-49161ecf-4b53-6dd2-d29f-82d84323ab3a Address:49161ecf-4b53-6dd2-d29f-82d84323ab3a} {Suffrage:Voter ID:server-15a6f5c3-a064-69ef-4b8b-f537c2a7525f Address:15a6f5c3-a064-69ef-4b8b-f537c2a7525f}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:16.211-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:16.211-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-15a6f5c3-a064-69ef-4b8b-f537c2a7525f: 2024-06-11T02:29:16.211-1200 [INFO] server-15a6f5c3-a064-69ef-4b8b-f537c2a7525f: entering follower state: follower="Node at 15a6f5c3-a064-69ef-4b8b-f537c2a7525f [Follower]" leader= - testing.go:149: server-15a6f5c3-a064-69ef-4b8b-f537c2a7525f: 2024-06-11T02:29:16.211-1200 [INFO] server-15a6f5c3-a064-69ef-4b8b-f537c2a7525f: entering follower state: follower="Node at 15a6f5c3-a064-69ef-4b8b-f537c2a7525f [Follower]" leader= -testLoggerAdapter verbose: server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: 2024-06-11T02:29:16.321-1200 [WARN] server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: 2024-06-11T02:29:16.321-1200 [WARN] server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: 2024-06-11T02:29:16.322-1200 [INFO] server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: entering candidate state: node="Node at 09059b36-ce8a-ac6c-80d7-59dd1fb3143f [Candidate]" term=2 - testing.go:149: server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: 2024-06-11T02:29:16.322-1200 [INFO] server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: entering candidate state: node="Node at 09059b36-ce8a-ac6c-80d7-59dd1fb3143f [Candidate]" term=2 -testLoggerAdapter verbose: server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: 2024-06-11T02:29:16.322-1200 [INFO] server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: election won: tally=2 - testing.go:149: server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: 2024-06-11T02:29:16.322-1200 [INFO] server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: election won: tally=2 -testLoggerAdapter verbose: server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: 2024-06-11T02:29:16.322-1200 [INFO] server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: entering leader state: leader="Node at 09059b36-ce8a-ac6c-80d7-59dd1fb3143f [Leader]" - testing.go:149: server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: 2024-06-11T02:29:16.322-1200 [INFO] server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: entering leader state: leader="Node at 09059b36-ce8a-ac6c-80d7-59dd1fb3143f [Leader]" -testLoggerAdapter verbose: server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: 2024-06-11T02:29:16.322-1200 [INFO] server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: added peer, starting replication: peer=server-49161ecf-4b53-6dd2-d29f-82d84323ab3a - testing.go:149: server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: 2024-06-11T02:29:16.322-1200 [INFO] server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: added peer, starting replication: peer=server-49161ecf-4b53-6dd2-d29f-82d84323ab3a -testLoggerAdapter verbose: server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: 2024-06-11T02:29:16.322-1200 [INFO] server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: added peer, starting replication: peer=server-15a6f5c3-a064-69ef-4b8b-f537c2a7525f - testing.go:149: server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: 2024-06-11T02:29:16.322-1200 [INFO] server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: added peer, starting replication: peer=server-15a6f5c3-a064-69ef-4b8b-f537c2a7525f -testLoggerAdapter verbose: server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: 2024-06-11T02:29:16.322-1200 [INFO] server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: pipelining replication: peer="{Voter server-15a6f5c3-a064-69ef-4b8b-f537c2a7525f 15a6f5c3-a064-69ef-4b8b-f537c2a7525f}" - testing.go:149: server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: 2024-06-11T02:29:16.322-1200 [INFO] server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: pipelining replication: peer="{Voter server-15a6f5c3-a064-69ef-4b8b-f537c2a7525f 15a6f5c3-a064-69ef-4b8b-f537c2a7525f}" -testLoggerAdapter verbose: server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: 2024-06-11T02:29:16.323-1200 [INFO] server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: pipelining replication: peer="{Voter server-49161ecf-4b53-6dd2-d29f-82d84323ab3a 49161ecf-4b53-6dd2-d29f-82d84323ab3a}" - testing.go:149: server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: 2024-06-11T02:29:16.323-1200 [INFO] server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: pipelining replication: peer="{Voter server-49161ecf-4b53-6dd2-d29f-82d84323ab3a 49161ecf-4b53-6dd2-d29f-82d84323ab3a}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:16.528-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:16.323227487 -1200 -12 m=+7.318600029 (1 nodes), 111.420696ms from start of poll, 113.368409ms from cluster start. Timeout at 2024-06-11 02:29:16.528506209 -1200 -12 m=+7.523879042, 205.279013ms after stability - testing.go:149: cluster: 2024-06-11T02:29:16.528-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:16.323227487 -1200 -12 m=+7.318600029 (1 nodes), 111.420696ms from start of poll, 113.368409ms from cluster start. Timeout at 2024-06-11 02:29:16.528506209 -1200 -12 m=+7.523879042, 205.279013ms after stability -testLoggerAdapter verbose: server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: 2024-06-11T02:29:16.547-1200 [INFO] server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: aborting pipeline replication: peer="{Voter server-15a6f5c3-a064-69ef-4b8b-f537c2a7525f 15a6f5c3-a064-69ef-4b8b-f537c2a7525f}" - testing.go:149: server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: 2024-06-11T02:29:16.547-1200 [INFO] server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: aborting pipeline replication: peer="{Voter server-15a6f5c3-a064-69ef-4b8b-f537c2a7525f 15a6f5c3-a064-69ef-4b8b-f537c2a7525f}" -testLoggerAdapter verbose: server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: 2024-06-11T02:29:16.548-1200 [INFO] server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: aborting pipeline replication: peer="{Voter server-49161ecf-4b53-6dd2-d29f-82d84323ab3a 49161ecf-4b53-6dd2-d29f-82d84323ab3a}" - testing.go:149: server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: 2024-06-11T02:29:16.548-1200 [INFO] server-09059b36-ce8a-ac6c-80d7-59dd1fb3143f: aborting pipeline replication: peer="{Voter server-49161ecf-4b53-6dd2-d29f-82d84323ab3a 49161ecf-4b53-6dd2-d29f-82d84323ab3a}" +testLoggerAdapter verbose: server-725f2e99-44df-9a1d-09a4-87a804953151: 2024-06-12T04:32:29.602+1400 [INFO] server-725f2e99-44df-9a1d-09a4-87a804953151: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-725f2e99-44df-9a1d-09a4-87a804953151 Address:725f2e99-44df-9a1d-09a4-87a804953151} {Suffrage:Voter ID:server-7c0e0a89-0af6-24b8-0327-bef42f52a8bb Address:7c0e0a89-0af6-24b8-0327-bef42f52a8bb} {Suffrage:Voter ID:server-68002a6e-c0fd-f70a-0a4d-d065e1609f17 Address:68002a6e-c0fd-f70a-0a4d-d065e1609f17}]" + testing.go:149: server-725f2e99-44df-9a1d-09a4-87a804953151: 2024-06-12T04:32:29.602+1400 [INFO] server-725f2e99-44df-9a1d-09a4-87a804953151: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-725f2e99-44df-9a1d-09a4-87a804953151 Address:725f2e99-44df-9a1d-09a4-87a804953151} {Suffrage:Voter ID:server-7c0e0a89-0af6-24b8-0327-bef42f52a8bb Address:7c0e0a89-0af6-24b8-0327-bef42f52a8bb} {Suffrage:Voter ID:server-68002a6e-c0fd-f70a-0a4d-d065e1609f17 Address:68002a6e-c0fd-f70a-0a4d-d065e1609f17}]" +testLoggerAdapter verbose: server-7c0e0a89-0af6-24b8-0327-bef42f52a8bb: 2024-06-12T04:32:29.603+1400 [INFO] server-7c0e0a89-0af6-24b8-0327-bef42f52a8bb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-725f2e99-44df-9a1d-09a4-87a804953151 Address:725f2e99-44df-9a1d-09a4-87a804953151} {Suffrage:Voter ID:server-7c0e0a89-0af6-24b8-0327-bef42f52a8bb Address:7c0e0a89-0af6-24b8-0327-bef42f52a8bb} {Suffrage:Voter ID:server-68002a6e-c0fd-f70a-0a4d-d065e1609f17 Address:68002a6e-c0fd-f70a-0a4d-d065e1609f17}]" + testing.go:149: server-7c0e0a89-0af6-24b8-0327-bef42f52a8bb: 2024-06-12T04:32:29.603+1400 [INFO] server-7c0e0a89-0af6-24b8-0327-bef42f52a8bb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-725f2e99-44df-9a1d-09a4-87a804953151 Address:725f2e99-44df-9a1d-09a4-87a804953151} {Suffrage:Voter ID:server-7c0e0a89-0af6-24b8-0327-bef42f52a8bb Address:7c0e0a89-0af6-24b8-0327-bef42f52a8bb} {Suffrage:Voter ID:server-68002a6e-c0fd-f70a-0a4d-d065e1609f17 Address:68002a6e-c0fd-f70a-0a4d-d065e1609f17}]" +testLoggerAdapter verbose: server-68002a6e-c0fd-f70a-0a4d-d065e1609f17: 2024-06-12T04:32:29.603+1400 [INFO] server-68002a6e-c0fd-f70a-0a4d-d065e1609f17: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-725f2e99-44df-9a1d-09a4-87a804953151 Address:725f2e99-44df-9a1d-09a4-87a804953151} {Suffrage:Voter ID:server-7c0e0a89-0af6-24b8-0327-bef42f52a8bb Address:7c0e0a89-0af6-24b8-0327-bef42f52a8bb} {Suffrage:Voter ID:server-68002a6e-c0fd-f70a-0a4d-d065e1609f17 Address:68002a6e-c0fd-f70a-0a4d-d065e1609f17}]" + testing.go:149: server-68002a6e-c0fd-f70a-0a4d-d065e1609f17: 2024-06-12T04:32:29.603+1400 [INFO] server-68002a6e-c0fd-f70a-0a4d-d065e1609f17: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-725f2e99-44df-9a1d-09a4-87a804953151 Address:725f2e99-44df-9a1d-09a4-87a804953151} {Suffrage:Voter ID:server-7c0e0a89-0af6-24b8-0327-bef42f52a8bb Address:7c0e0a89-0af6-24b8-0327-bef42f52a8bb} {Suffrage:Voter ID:server-68002a6e-c0fd-f70a-0a4d-d065e1609f17 Address:68002a6e-c0fd-f70a-0a4d-d065e1609f17}]" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:29.603+1400 [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-06-12T04:32:29.603+1400 [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-7c0e0a89-0af6-24b8-0327-bef42f52a8bb: 2024-06-12T04:32:29.603+1400 [INFO] server-7c0e0a89-0af6-24b8-0327-bef42f52a8bb: entering follower state: follower="Node at 7c0e0a89-0af6-24b8-0327-bef42f52a8bb [Follower]" leader= + testing.go:149: server-7c0e0a89-0af6-24b8-0327-bef42f52a8bb: 2024-06-12T04:32:29.603+1400 [INFO] server-7c0e0a89-0af6-24b8-0327-bef42f52a8bb: entering follower state: follower="Node at 7c0e0a89-0af6-24b8-0327-bef42f52a8bb [Follower]" leader= +testLoggerAdapter verbose: server-725f2e99-44df-9a1d-09a4-87a804953151: 2024-06-12T04:32:29.603+1400 [INFO] server-725f2e99-44df-9a1d-09a4-87a804953151: entering follower state: follower="Node at 725f2e99-44df-9a1d-09a4-87a804953151 [Follower]" leader= + testing.go:149: server-725f2e99-44df-9a1d-09a4-87a804953151: 2024-06-12T04:32:29.603+1400 [INFO] server-725f2e99-44df-9a1d-09a4-87a804953151: entering follower state: follower="Node at 725f2e99-44df-9a1d-09a4-87a804953151 [Follower]" leader= +testLoggerAdapter verbose: server-68002a6e-c0fd-f70a-0a4d-d065e1609f17: 2024-06-12T04:32:29.604+1400 [INFO] server-68002a6e-c0fd-f70a-0a4d-d065e1609f17: entering follower state: follower="Node at 68002a6e-c0fd-f70a-0a4d-d065e1609f17 [Follower]" leader= + testing.go:149: server-68002a6e-c0fd-f70a-0a4d-d065e1609f17: 2024-06-12T04:32:29.604+1400 [INFO] server-68002a6e-c0fd-f70a-0a4d-d065e1609f17: entering follower state: follower="Node at 68002a6e-c0fd-f70a-0a4d-d065e1609f17 [Follower]" leader= +testLoggerAdapter verbose: server-725f2e99-44df-9a1d-09a4-87a804953151: 2024-06-12T04:32:29.713+1400 [WARN] server-725f2e99-44df-9a1d-09a4-87a804953151: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-725f2e99-44df-9a1d-09a4-87a804953151: 2024-06-12T04:32:29.713+1400 [WARN] server-725f2e99-44df-9a1d-09a4-87a804953151: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-725f2e99-44df-9a1d-09a4-87a804953151: 2024-06-12T04:32:29.713+1400 [INFO] server-725f2e99-44df-9a1d-09a4-87a804953151: entering candidate state: node="Node at 725f2e99-44df-9a1d-09a4-87a804953151 [Candidate]" term=2 + testing.go:149: server-725f2e99-44df-9a1d-09a4-87a804953151: 2024-06-12T04:32:29.713+1400 [INFO] server-725f2e99-44df-9a1d-09a4-87a804953151: entering candidate state: node="Node at 725f2e99-44df-9a1d-09a4-87a804953151 [Candidate]" term=2 +testLoggerAdapter verbose: server-725f2e99-44df-9a1d-09a4-87a804953151: 2024-06-12T04:32:29.713+1400 [INFO] server-725f2e99-44df-9a1d-09a4-87a804953151: election won: tally=2 + testing.go:149: server-725f2e99-44df-9a1d-09a4-87a804953151: 2024-06-12T04:32:29.713+1400 [INFO] server-725f2e99-44df-9a1d-09a4-87a804953151: election won: tally=2 +testLoggerAdapter verbose: server-725f2e99-44df-9a1d-09a4-87a804953151: 2024-06-12T04:32:29.713+1400 [INFO] server-725f2e99-44df-9a1d-09a4-87a804953151: entering leader state: leader="Node at 725f2e99-44df-9a1d-09a4-87a804953151 [Leader]" + testing.go:149: server-725f2e99-44df-9a1d-09a4-87a804953151: 2024-06-12T04:32:29.713+1400 [INFO] server-725f2e99-44df-9a1d-09a4-87a804953151: entering leader state: leader="Node at 725f2e99-44df-9a1d-09a4-87a804953151 [Leader]" +testLoggerAdapter verbose: server-725f2e99-44df-9a1d-09a4-87a804953151: 2024-06-12T04:32:29.713+1400 [INFO] server-725f2e99-44df-9a1d-09a4-87a804953151: added peer, starting replication: peer=server-7c0e0a89-0af6-24b8-0327-bef42f52a8bb + testing.go:149: server-725f2e99-44df-9a1d-09a4-87a804953151: 2024-06-12T04:32:29.713+1400 [INFO] server-725f2e99-44df-9a1d-09a4-87a804953151: added peer, starting replication: peer=server-7c0e0a89-0af6-24b8-0327-bef42f52a8bb +testLoggerAdapter verbose: server-725f2e99-44df-9a1d-09a4-87a804953151: 2024-06-12T04:32:29.713+1400 [INFO] server-725f2e99-44df-9a1d-09a4-87a804953151: added peer, starting replication: peer=server-68002a6e-c0fd-f70a-0a4d-d065e1609f17 + testing.go:149: server-725f2e99-44df-9a1d-09a4-87a804953151: 2024-06-12T04:32:29.713+1400 [INFO] server-725f2e99-44df-9a1d-09a4-87a804953151: added peer, starting replication: peer=server-68002a6e-c0fd-f70a-0a4d-d065e1609f17 +testLoggerAdapter verbose: server-725f2e99-44df-9a1d-09a4-87a804953151: 2024-06-12T04:32:29.713+1400 [INFO] server-725f2e99-44df-9a1d-09a4-87a804953151: pipelining replication: peer="{Voter server-68002a6e-c0fd-f70a-0a4d-d065e1609f17 68002a6e-c0fd-f70a-0a4d-d065e1609f17}" + testing.go:149: server-725f2e99-44df-9a1d-09a4-87a804953151: 2024-06-12T04:32:29.713+1400 [INFO] server-725f2e99-44df-9a1d-09a4-87a804953151: pipelining replication: peer="{Voter server-68002a6e-c0fd-f70a-0a4d-d065e1609f17 68002a6e-c0fd-f70a-0a4d-d065e1609f17}" +testLoggerAdapter verbose: server-725f2e99-44df-9a1d-09a4-87a804953151: 2024-06-12T04:32:29.713+1400 [INFO] server-725f2e99-44df-9a1d-09a4-87a804953151: pipelining replication: peer="{Voter server-7c0e0a89-0af6-24b8-0327-bef42f52a8bb 7c0e0a89-0af6-24b8-0327-bef42f52a8bb}" + testing.go:149: server-725f2e99-44df-9a1d-09a4-87a804953151: 2024-06-12T04:32:29.713+1400 [INFO] server-725f2e99-44df-9a1d-09a4-87a804953151: pipelining replication: peer="{Voter server-7c0e0a89-0af6-24b8-0327-bef42f52a8bb 7c0e0a89-0af6-24b8-0327-bef42f52a8bb}" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:29.919+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:29.713865931 +1400 +14 m=+6.621384550 (1 nodes), 110.413245ms from start of poll, 111.325972ms from cluster start. Timeout at 2024-06-12 04:32:29.919301213 +1400 +14 m=+6.826819832, 205.435282ms after stability + testing.go:149: cluster: 2024-06-12T04:32:29.919+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:29.713865931 +1400 +14 m=+6.621384550 (1 nodes), 110.413245ms from start of poll, 111.325972ms from cluster start. Timeout at 2024-06-12 04:32:29.919301213 +1400 +14 m=+6.826819832, 205.435282ms after stability +testLoggerAdapter verbose: server-725f2e99-44df-9a1d-09a4-87a804953151: 2024-06-12T04:32:29.935+1400 [INFO] server-725f2e99-44df-9a1d-09a4-87a804953151: aborting pipeline replication: peer="{Voter server-68002a6e-c0fd-f70a-0a4d-d065e1609f17 68002a6e-c0fd-f70a-0a4d-d065e1609f17}" + testing.go:149: server-725f2e99-44df-9a1d-09a4-87a804953151: 2024-06-12T04:32:29.935+1400 [INFO] server-725f2e99-44df-9a1d-09a4-87a804953151: aborting pipeline replication: peer="{Voter server-68002a6e-c0fd-f70a-0a4d-d065e1609f17 68002a6e-c0fd-f70a-0a4d-d065e1609f17}" +testLoggerAdapter verbose: server-725f2e99-44df-9a1d-09a4-87a804953151: 2024-06-12T04:32:29.935+1400 [INFO] server-725f2e99-44df-9a1d-09a4-87a804953151: aborting pipeline replication: peer="{Voter server-7c0e0a89-0af6-24b8-0327-bef42f52a8bb 7c0e0a89-0af6-24b8-0327-bef42f52a8bb}" + testing.go:149: server-725f2e99-44df-9a1d-09a4-87a804953151: 2024-06-12T04:32:29.935+1400 [INFO] server-725f2e99-44df-9a1d-09a4-87a804953151: aborting pipeline replication: peer="{Voter server-7c0e0a89-0af6-24b8-0327-bef42f52a8bb 7c0e0a89-0af6-24b8-0327-bef42f52a8bb}" --- PASS: TestRaft_ApplyConcurrent (0.34s) === RUN TestRaft_ApplyConcurrent_Timeout -testLoggerAdapter verbose: server-b80ca9f3-42df-06f1-5151-04075f4cbeb1: 2024-06-11T02:29:16.550-1200 [INFO] server-b80ca9f3-42df-06f1-5151-04075f4cbeb1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b80ca9f3-42df-06f1-5151-04075f4cbeb1 Address:b80ca9f3-42df-06f1-5151-04075f4cbeb1}]" - testing.go:149: server-b80ca9f3-42df-06f1-5151-04075f4cbeb1: 2024-06-11T02:29:16.550-1200 [INFO] server-b80ca9f3-42df-06f1-5151-04075f4cbeb1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b80ca9f3-42df-06f1-5151-04075f4cbeb1 Address:b80ca9f3-42df-06f1-5151-04075f4cbeb1}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:16.550-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:16.550-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-b80ca9f3-42df-06f1-5151-04075f4cbeb1: 2024-06-11T02:29:16.550-1200 [INFO] server-b80ca9f3-42df-06f1-5151-04075f4cbeb1: entering follower state: follower="Node at b80ca9f3-42df-06f1-5151-04075f4cbeb1 [Follower]" leader= - testing.go:149: server-b80ca9f3-42df-06f1-5151-04075f4cbeb1: 2024-06-11T02:29:16.550-1200 [INFO] server-b80ca9f3-42df-06f1-5151-04075f4cbeb1: entering follower state: follower="Node at b80ca9f3-42df-06f1-5151-04075f4cbeb1 [Follower]" leader= -testLoggerAdapter verbose: server-b80ca9f3-42df-06f1-5151-04075f4cbeb1: 2024-06-11T02:29:16.671-1200 [WARN] server-b80ca9f3-42df-06f1-5151-04075f4cbeb1: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-b80ca9f3-42df-06f1-5151-04075f4cbeb1: 2024-06-11T02:29:16.671-1200 [WARN] server-b80ca9f3-42df-06f1-5151-04075f4cbeb1: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-b80ca9f3-42df-06f1-5151-04075f4cbeb1: 2024-06-11T02:29:16.671-1200 [INFO] server-b80ca9f3-42df-06f1-5151-04075f4cbeb1: entering candidate state: node="Node at b80ca9f3-42df-06f1-5151-04075f4cbeb1 [Candidate]" term=2 - testing.go:149: server-b80ca9f3-42df-06f1-5151-04075f4cbeb1: 2024-06-11T02:29:16.671-1200 [INFO] server-b80ca9f3-42df-06f1-5151-04075f4cbeb1: entering candidate state: node="Node at b80ca9f3-42df-06f1-5151-04075f4cbeb1 [Candidate]" term=2 -testLoggerAdapter verbose: server-b80ca9f3-42df-06f1-5151-04075f4cbeb1: 2024-06-11T02:29:16.671-1200 [INFO] server-b80ca9f3-42df-06f1-5151-04075f4cbeb1: election won: tally=1 - testing.go:149: server-b80ca9f3-42df-06f1-5151-04075f4cbeb1: 2024-06-11T02:29:16.671-1200 [INFO] server-b80ca9f3-42df-06f1-5151-04075f4cbeb1: election won: tally=1 -testLoggerAdapter verbose: server-b80ca9f3-42df-06f1-5151-04075f4cbeb1: 2024-06-11T02:29:16.671-1200 [INFO] server-b80ca9f3-42df-06f1-5151-04075f4cbeb1: entering leader state: leader="Node at b80ca9f3-42df-06f1-5151-04075f4cbeb1 [Leader]" - testing.go:149: server-b80ca9f3-42df-06f1-5151-04075f4cbeb1: 2024-06-11T02:29:16.671-1200 [INFO] server-b80ca9f3-42df-06f1-5151-04075f4cbeb1: entering leader state: leader="Node at b80ca9f3-42df-06f1-5151-04075f4cbeb1 [Leader]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:16.873-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:16.671861178 -1200 -12 m=+7.667233720 (1 nodes), 120.942558ms from start of poll, 121.407758ms from cluster start. Timeout at 2024-06-11 02:29:16.873223182 -1200 -12 m=+7.868596015, 201.362295ms after stability - testing.go:149: cluster: 2024-06-11T02:29:16.873-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:16.671861178 -1200 -12 m=+7.667233720 (1 nodes), 120.942558ms from start of poll, 121.407758ms from cluster start. Timeout at 2024-06-11 02:29:16.873223182 -1200 -12 m=+7.868596015, 201.362295ms after stability ---- PASS: TestRaft_ApplyConcurrent_Timeout (0.32s) +testLoggerAdapter verbose: server-8a8dab93-58bf-e07a-4145-b749878e27e4: 2024-06-12T04:32:29.937+1400 [INFO] server-8a8dab93-58bf-e07a-4145-b749878e27e4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8a8dab93-58bf-e07a-4145-b749878e27e4 Address:8a8dab93-58bf-e07a-4145-b749878e27e4}]" + testing.go:149: server-8a8dab93-58bf-e07a-4145-b749878e27e4: 2024-06-12T04:32:29.937+1400 [INFO] server-8a8dab93-58bf-e07a-4145-b749878e27e4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8a8dab93-58bf-e07a-4145-b749878e27e4 Address:8a8dab93-58bf-e07a-4145-b749878e27e4}]" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:29.937+1400 [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-06-12T04:32:29.937+1400 [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-8a8dab93-58bf-e07a-4145-b749878e27e4: 2024-06-12T04:32:29.937+1400 [INFO] server-8a8dab93-58bf-e07a-4145-b749878e27e4: entering follower state: follower="Node at 8a8dab93-58bf-e07a-4145-b749878e27e4 [Follower]" leader= + testing.go:149: server-8a8dab93-58bf-e07a-4145-b749878e27e4: 2024-06-12T04:32:29.937+1400 [INFO] server-8a8dab93-58bf-e07a-4145-b749878e27e4: entering follower state: follower="Node at 8a8dab93-58bf-e07a-4145-b749878e27e4 [Follower]" leader= +testLoggerAdapter verbose: server-8a8dab93-58bf-e07a-4145-b749878e27e4: 2024-06-12T04:32:30.077+1400 [WARN] server-8a8dab93-58bf-e07a-4145-b749878e27e4: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-8a8dab93-58bf-e07a-4145-b749878e27e4: 2024-06-12T04:32:30.077+1400 [WARN] server-8a8dab93-58bf-e07a-4145-b749878e27e4: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-8a8dab93-58bf-e07a-4145-b749878e27e4: 2024-06-12T04:32:30.077+1400 [INFO] server-8a8dab93-58bf-e07a-4145-b749878e27e4: entering candidate state: node="Node at 8a8dab93-58bf-e07a-4145-b749878e27e4 [Candidate]" term=2 + testing.go:149: server-8a8dab93-58bf-e07a-4145-b749878e27e4: 2024-06-12T04:32:30.077+1400 [INFO] server-8a8dab93-58bf-e07a-4145-b749878e27e4: entering candidate state: node="Node at 8a8dab93-58bf-e07a-4145-b749878e27e4 [Candidate]" term=2 +testLoggerAdapter verbose: server-8a8dab93-58bf-e07a-4145-b749878e27e4: 2024-06-12T04:32:30.077+1400 [INFO] server-8a8dab93-58bf-e07a-4145-b749878e27e4: election won: tally=1 + testing.go:149: server-8a8dab93-58bf-e07a-4145-b749878e27e4: 2024-06-12T04:32:30.077+1400 [INFO] server-8a8dab93-58bf-e07a-4145-b749878e27e4: election won: tally=1 +testLoggerAdapter verbose: server-8a8dab93-58bf-e07a-4145-b749878e27e4: 2024-06-12T04:32:30.077+1400 [INFO] server-8a8dab93-58bf-e07a-4145-b749878e27e4: entering leader state: leader="Node at 8a8dab93-58bf-e07a-4145-b749878e27e4 [Leader]" + testing.go:149: server-8a8dab93-58bf-e07a-4145-b749878e27e4: 2024-06-12T04:32:30.077+1400 [INFO] server-8a8dab93-58bf-e07a-4145-b749878e27e4: entering leader state: leader="Node at 8a8dab93-58bf-e07a-4145-b749878e27e4 [Leader]" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:30.278+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:30.077638262 +1400 +14 m=+6.985156841 (1 nodes), 140.120022ms from start of poll, 140.393692ms from cluster start. Timeout at 2024-06-12 04:32:30.278763577 +1400 +14 m=+7.186282156, 201.125315ms after stability + testing.go:149: cluster: 2024-06-12T04:32:30.278+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:30.077638262 +1400 +14 m=+6.985156841 (1 nodes), 140.120022ms from start of poll, 140.393692ms from cluster start. Timeout at 2024-06-12 04:32:30.278763577 +1400 +14 m=+7.186282156, 201.125315ms after stability +--- PASS: TestRaft_ApplyConcurrent_Timeout (0.34s) === RUN TestRaft_JoinNode -testLoggerAdapter verbose: server-8eecfc0a-de7d-2d62-39a4-5dbe0928a4b5: 2024-06-11T02:29:16.875-1200 [INFO] server-8eecfc0a-de7d-2d62-39a4-5dbe0928a4b5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8eecfc0a-de7d-2d62-39a4-5dbe0928a4b5 Address:8eecfc0a-de7d-2d62-39a4-5dbe0928a4b5} {Suffrage:Voter ID:server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b Address:85ab5ecc-ba33-9aaf-8494-3984a3b96a7b}]" - testing.go:149: server-8eecfc0a-de7d-2d62-39a4-5dbe0928a4b5: 2024-06-11T02:29:16.875-1200 [INFO] server-8eecfc0a-de7d-2d62-39a4-5dbe0928a4b5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8eecfc0a-de7d-2d62-39a4-5dbe0928a4b5 Address:8eecfc0a-de7d-2d62-39a4-5dbe0928a4b5} {Suffrage:Voter ID:server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b Address:85ab5ecc-ba33-9aaf-8494-3984a3b96a7b}]" -testLoggerAdapter verbose: server-8eecfc0a-de7d-2d62-39a4-5dbe0928a4b5: 2024-06-11T02:29:16.876-1200 [INFO] server-8eecfc0a-de7d-2d62-39a4-5dbe0928a4b5: entering follower state: follower="Node at 8eecfc0a-de7d-2d62-39a4-5dbe0928a4b5 [Follower]" leader= - testing.go:149: server-8eecfc0a-de7d-2d62-39a4-5dbe0928a4b5: 2024-06-11T02:29:16.876-1200 [INFO] server-8eecfc0a-de7d-2d62-39a4-5dbe0928a4b5: entering follower state: follower="Node at 8eecfc0a-de7d-2d62-39a4-5dbe0928a4b5 [Follower]" leader= -testLoggerAdapter verbose: server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: 2024-06-11T02:29:16.876-1200 [INFO] server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8eecfc0a-de7d-2d62-39a4-5dbe0928a4b5 Address:8eecfc0a-de7d-2d62-39a4-5dbe0928a4b5} {Suffrage:Voter ID:server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b Address:85ab5ecc-ba33-9aaf-8494-3984a3b96a7b}]" - testing.go:149: server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: 2024-06-11T02:29:16.876-1200 [INFO] server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8eecfc0a-de7d-2d62-39a4-5dbe0928a4b5 Address:8eecfc0a-de7d-2d62-39a4-5dbe0928a4b5} {Suffrage:Voter ID:server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b Address:85ab5ecc-ba33-9aaf-8494-3984a3b96a7b}]" -testLoggerAdapter verbose: server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: 2024-06-11T02:29:16.876-1200 [INFO] server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: entering follower state: follower="Node at 85ab5ecc-ba33-9aaf-8494-3984a3b96a7b [Follower]" leader= - testing.go:149: server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: 2024-06-11T02:29:16.876-1200 [INFO] server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: entering follower state: follower="Node at 85ab5ecc-ba33-9aaf-8494-3984a3b96a7b [Follower]" leader= -testLoggerAdapter verbose: server-8c35c0fc-6286-4172-95e5-9ba6ec24105e: 2024-06-11T02:29:16.877-1200 [INFO] server-8c35c0fc-6286-4172-95e5-9ba6ec24105e: initial configuration: index=0 servers=[] - testing.go:149: server-8c35c0fc-6286-4172-95e5-9ba6ec24105e: 2024-06-11T02:29:16.877-1200 [INFO] server-8c35c0fc-6286-4172-95e5-9ba6ec24105e: initial configuration: index=0 servers=[] -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:16.877-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-8c35c0fc-6286-4172-95e5-9ba6ec24105e: 2024-06-11T02:29:16.877-1200 [INFO] server-8c35c0fc-6286-4172-95e5-9ba6ec24105e: entering follower state: follower="Node at 8c35c0fc-6286-4172-95e5-9ba6ec24105e [Follower]" leader= - testing.go:149: cluster: 2024-06-11T02:29:16.877-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: server-8c35c0fc-6286-4172-95e5-9ba6ec24105e: 2024-06-11T02:29:16.877-1200 [INFO] server-8c35c0fc-6286-4172-95e5-9ba6ec24105e: entering follower state: follower="Node at 8c35c0fc-6286-4172-95e5-9ba6ec24105e [Follower]" leader= -testLoggerAdapter verbose: server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: 2024-06-11T02:29:16.928-1200 [WARN] server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: 2024-06-11T02:29:16.928-1200 [WARN] server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: 2024-06-11T02:29:16.928-1200 [INFO] server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: entering candidate state: node="Node at 85ab5ecc-ba33-9aaf-8494-3984a3b96a7b [Candidate]" term=2 - testing.go:149: server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: 2024-06-11T02:29:16.928-1200 [INFO] server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: entering candidate state: node="Node at 85ab5ecc-ba33-9aaf-8494-3984a3b96a7b [Candidate]" term=2 -testLoggerAdapter verbose: server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: 2024-06-11T02:29:16.928-1200 [INFO] server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: election won: tally=2 - testing.go:149: server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: 2024-06-11T02:29:16.928-1200 [INFO] server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: election won: tally=2 -testLoggerAdapter verbose: server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: 2024-06-11T02:29:16.928-1200 [INFO] server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: entering leader state: leader="Node at 85ab5ecc-ba33-9aaf-8494-3984a3b96a7b [Leader]" - testing.go:149: server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: 2024-06-11T02:29:16.928-1200 [INFO] server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: entering leader state: leader="Node at 85ab5ecc-ba33-9aaf-8494-3984a3b96a7b [Leader]" -testLoggerAdapter verbose: server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: 2024-06-11T02:29:16.928-1200 [INFO] server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: added peer, starting replication: peer=server-8eecfc0a-de7d-2d62-39a4-5dbe0928a4b5 - testing.go:149: server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: 2024-06-11T02:29:16.928-1200 [INFO] server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: added peer, starting replication: peer=server-8eecfc0a-de7d-2d62-39a4-5dbe0928a4b5 -testLoggerAdapter verbose: server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: 2024-06-11T02:29:16.928-1200 [INFO] server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: pipelining replication: peer="{Voter server-8eecfc0a-de7d-2d62-39a4-5dbe0928a4b5 8eecfc0a-de7d-2d62-39a4-5dbe0928a4b5}" - testing.go:149: server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: 2024-06-11T02:29:16.928-1200 [INFO] server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: pipelining replication: peer="{Voter server-8eecfc0a-de7d-2d62-39a4-5dbe0928a4b5 8eecfc0a-de7d-2d62-39a4-5dbe0928a4b5}" -testLoggerAdapter verbose: server-8c35c0fc-6286-4172-95e5-9ba6ec24105e: 2024-06-11T02:29:16.969-1200 [WARN] server-8c35c0fc-6286-4172-95e5-9ba6ec24105e: no known peers, aborting election - testing.go:149: server-8c35c0fc-6286-4172-95e5-9ba6ec24105e: 2024-06-11T02:29:16.969-1200 [WARN] server-8c35c0fc-6286-4172-95e5-9ba6ec24105e: no known peers, aborting election -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:17.034-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:16.928897801 -1200 -12 m=+7.924270343 (1 nodes), 51.462449ms from start of poll, 53.207457ms from cluster start. Timeout at 2024-06-11 02:29:17.034302402 -1200 -12 m=+8.029674943, 105.4046ms after stability - testing.go:149: cluster: 2024-06-11T02:29:17.034-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:16.928897801 -1200 -12 m=+7.924270343 (1 nodes), 51.462449ms from start of poll, 53.207457ms from cluster start. Timeout at 2024-06-11 02:29:17.034302402 -1200 -12 m=+8.029674943, 105.4046ms after stability -testLoggerAdapter verbose: server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: 2024-06-11T02:29:17.034-1200 [INFO] server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: updating configuration: command=AddStaging server-id=server-8c35c0fc-6286-4172-95e5-9ba6ec24105e server-addr=8c35c0fc-6286-4172-95e5-9ba6ec24105e servers="[{Suffrage:Voter ID:server-8eecfc0a-de7d-2d62-39a4-5dbe0928a4b5 Address:8eecfc0a-de7d-2d62-39a4-5dbe0928a4b5} {Suffrage:Voter ID:server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b Address:85ab5ecc-ba33-9aaf-8494-3984a3b96a7b} {Suffrage:Voter ID:server-8c35c0fc-6286-4172-95e5-9ba6ec24105e Address:8c35c0fc-6286-4172-95e5-9ba6ec24105e}]" - testing.go:149: server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: 2024-06-11T02:29:17.034-1200 [INFO] server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: updating configuration: command=AddStaging server-id=server-8c35c0fc-6286-4172-95e5-9ba6ec24105e server-addr=8c35c0fc-6286-4172-95e5-9ba6ec24105e servers="[{Suffrage:Voter ID:server-8eecfc0a-de7d-2d62-39a4-5dbe0928a4b5 Address:8eecfc0a-de7d-2d62-39a4-5dbe0928a4b5} {Suffrage:Voter ID:server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b Address:85ab5ecc-ba33-9aaf-8494-3984a3b96a7b} {Suffrage:Voter ID:server-8c35c0fc-6286-4172-95e5-9ba6ec24105e Address:8c35c0fc-6286-4172-95e5-9ba6ec24105e}]" -testLoggerAdapter verbose: server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: 2024-06-11T02:29:17.034-1200 [INFO] server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: added peer, starting replication: peer=server-8c35c0fc-6286-4172-95e5-9ba6ec24105e - testing.go:149: server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: 2024-06-11T02:29:17.034-1200 [INFO] server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: added peer, starting replication: peer=server-8c35c0fc-6286-4172-95e5-9ba6ec24105e -testLoggerAdapter verbose: server-8c35c0fc-6286-4172-95e5-9ba6ec24105e: 2024-06-11T02:29:17.034-1200 [WARN] server-8c35c0fc-6286-4172-95e5-9ba6ec24105e: failed to get previous log: previous-index=3 last-index=0 error="log not found" - testing.go:149: server-8c35c0fc-6286-4172-95e5-9ba6ec24105e: 2024-06-11T02:29:17.034-1200 [WARN] server-8c35c0fc-6286-4172-95e5-9ba6ec24105e: failed to get previous log: previous-index=3 last-index=0 error="log not found" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:17.035-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:17.035-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: 2024-06-11T02:29:17.035-1200 [WARN] server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: appendEntries rejected, sending older logs: peer="{Voter server-8c35c0fc-6286-4172-95e5-9ba6ec24105e 8c35c0fc-6286-4172-95e5-9ba6ec24105e}" next=1 - testing.go:149: server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: 2024-06-11T02:29:17.035-1200 [WARN] server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: appendEntries rejected, sending older logs: peer="{Voter server-8c35c0fc-6286-4172-95e5-9ba6ec24105e 8c35c0fc-6286-4172-95e5-9ba6ec24105e}" next=1 -testLoggerAdapter verbose: server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: 2024-06-11T02:29:17.035-1200 [INFO] server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: pipelining replication: peer="{Voter server-8c35c0fc-6286-4172-95e5-9ba6ec24105e 8c35c0fc-6286-4172-95e5-9ba6ec24105e}" - testing.go:149: server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: 2024-06-11T02:29:17.035-1200 [INFO] server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: pipelining replication: peer="{Voter server-8c35c0fc-6286-4172-95e5-9ba6ec24105e 8c35c0fc-6286-4172-95e5-9ba6ec24105e}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:17.141-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:17.035096012 -1200 -12 m=+8.030468554 (1 nodes), 2.625µs from start of poll, 159.405668ms from cluster start. Timeout at 2024-06-11 02:29:17.140981562 -1200 -12 m=+8.136354103, 105.885549ms after stability - testing.go:149: cluster: 2024-06-11T02:29:17.141-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:17.035096012 -1200 -12 m=+8.030468554 (1 nodes), 2.625µs from start of poll, 159.405668ms from cluster start. Timeout at 2024-06-11 02:29:17.140981562 -1200 -12 m=+8.136354103, 105.885549ms after stability -testLoggerAdapter verbose: server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: 2024-06-11T02:29:17.141-1200 [INFO] server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: aborting pipeline replication: peer="{Voter server-8c35c0fc-6286-4172-95e5-9ba6ec24105e 8c35c0fc-6286-4172-95e5-9ba6ec24105e}" - testing.go:149: server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: 2024-06-11T02:29:17.141-1200 [INFO] server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: aborting pipeline replication: peer="{Voter server-8c35c0fc-6286-4172-95e5-9ba6ec24105e 8c35c0fc-6286-4172-95e5-9ba6ec24105e}" -testLoggerAdapter verbose: server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: 2024-06-11T02:29:17.141-1200 [INFO] server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: aborting pipeline replication: peer="{Voter server-8eecfc0a-de7d-2d62-39a4-5dbe0928a4b5 8eecfc0a-de7d-2d62-39a4-5dbe0928a4b5}" - testing.go:149: server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: 2024-06-11T02:29:17.141-1200 [INFO] server-85ab5ecc-ba33-9aaf-8494-3984a3b96a7b: aborting pipeline replication: peer="{Voter server-8eecfc0a-de7d-2d62-39a4-5dbe0928a4b5 8eecfc0a-de7d-2d62-39a4-5dbe0928a4b5}" ---- PASS: TestRaft_JoinNode (0.27s) +testLoggerAdapter verbose: server-ba5ea0b5-da8f-f7a4-c2b0-6369b9df7d9d: 2024-06-12T04:32:30.283+1400 [INFO] server-ba5ea0b5-da8f-f7a4-c2b0-6369b9df7d9d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ba5ea0b5-da8f-f7a4-c2b0-6369b9df7d9d Address:ba5ea0b5-da8f-f7a4-c2b0-6369b9df7d9d} {Suffrage:Voter ID:server-adab6b73-3013-8559-f2d7-22591934104c Address:adab6b73-3013-8559-f2d7-22591934104c}]" + testing.go:149: server-ba5ea0b5-da8f-f7a4-c2b0-6369b9df7d9d: 2024-06-12T04:32:30.283+1400 [INFO] server-ba5ea0b5-da8f-f7a4-c2b0-6369b9df7d9d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ba5ea0b5-da8f-f7a4-c2b0-6369b9df7d9d Address:ba5ea0b5-da8f-f7a4-c2b0-6369b9df7d9d} {Suffrage:Voter ID:server-adab6b73-3013-8559-f2d7-22591934104c Address:adab6b73-3013-8559-f2d7-22591934104c}]" +testLoggerAdapter verbose: server-adab6b73-3013-8559-f2d7-22591934104c: 2024-06-12T04:32:30.284+1400 [INFO] server-adab6b73-3013-8559-f2d7-22591934104c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ba5ea0b5-da8f-f7a4-c2b0-6369b9df7d9d Address:ba5ea0b5-da8f-f7a4-c2b0-6369b9df7d9d} {Suffrage:Voter ID:server-adab6b73-3013-8559-f2d7-22591934104c Address:adab6b73-3013-8559-f2d7-22591934104c}]" + testing.go:149: server-adab6b73-3013-8559-f2d7-22591934104c: 2024-06-12T04:32:30.284+1400 [INFO] server-adab6b73-3013-8559-f2d7-22591934104c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ba5ea0b5-da8f-f7a4-c2b0-6369b9df7d9d Address:ba5ea0b5-da8f-f7a4-c2b0-6369b9df7d9d} {Suffrage:Voter ID:server-adab6b73-3013-8559-f2d7-22591934104c Address:adab6b73-3013-8559-f2d7-22591934104c}]" +testLoggerAdapter verbose: server-d56af706-964d-9927-fc35-25334e2b1781: 2024-06-12T04:32:30.284+1400 [INFO] server-d56af706-964d-9927-fc35-25334e2b1781: initial configuration: index=0 servers=[] + testing.go:149: server-d56af706-964d-9927-fc35-25334e2b1781: 2024-06-12T04:32:30.284+1400 [INFO] server-d56af706-964d-9927-fc35-25334e2b1781: initial configuration: index=0 servers=[] +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:30.284+1400 [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-06-12T04:32:30.284+1400 [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-ba5ea0b5-da8f-f7a4-c2b0-6369b9df7d9d: 2024-06-12T04:32:30.284+1400 [INFO] server-ba5ea0b5-da8f-f7a4-c2b0-6369b9df7d9d: entering follower state: follower="Node at ba5ea0b5-da8f-f7a4-c2b0-6369b9df7d9d [Follower]" leader= + testing.go:149: server-ba5ea0b5-da8f-f7a4-c2b0-6369b9df7d9d: 2024-06-12T04:32:30.284+1400 [INFO] server-ba5ea0b5-da8f-f7a4-c2b0-6369b9df7d9d: entering follower state: follower="Node at ba5ea0b5-da8f-f7a4-c2b0-6369b9df7d9d [Follower]" leader= +testLoggerAdapter verbose: server-adab6b73-3013-8559-f2d7-22591934104c: 2024-06-12T04:32:30.284+1400 [INFO] server-adab6b73-3013-8559-f2d7-22591934104c: entering follower state: follower="Node at adab6b73-3013-8559-f2d7-22591934104c [Follower]" leader= + testing.go:149: server-adab6b73-3013-8559-f2d7-22591934104c: 2024-06-12T04:32:30.284+1400 [INFO] server-adab6b73-3013-8559-f2d7-22591934104c: entering follower state: follower="Node at adab6b73-3013-8559-f2d7-22591934104c [Follower]" leader= +testLoggerAdapter verbose: server-d56af706-964d-9927-fc35-25334e2b1781: 2024-06-12T04:32:30.284+1400 [INFO] server-d56af706-964d-9927-fc35-25334e2b1781: entering follower state: follower="Node at d56af706-964d-9927-fc35-25334e2b1781 [Follower]" leader= + testing.go:149: server-d56af706-964d-9927-fc35-25334e2b1781: 2024-06-12T04:32:30.284+1400 [INFO] server-d56af706-964d-9927-fc35-25334e2b1781: entering follower state: follower="Node at d56af706-964d-9927-fc35-25334e2b1781 [Follower]" leader= +testLoggerAdapter verbose: server-adab6b73-3013-8559-f2d7-22591934104c: 2024-06-12T04:32:30.370+1400 [WARN] server-adab6b73-3013-8559-f2d7-22591934104c: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-adab6b73-3013-8559-f2d7-22591934104c: 2024-06-12T04:32:30.370+1400 [WARN] server-adab6b73-3013-8559-f2d7-22591934104c: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-adab6b73-3013-8559-f2d7-22591934104c: 2024-06-12T04:32:30.370+1400 [INFO] server-adab6b73-3013-8559-f2d7-22591934104c: entering candidate state: node="Node at adab6b73-3013-8559-f2d7-22591934104c [Candidate]" term=2 + testing.go:149: server-adab6b73-3013-8559-f2d7-22591934104c: 2024-06-12T04:32:30.370+1400 [INFO] server-adab6b73-3013-8559-f2d7-22591934104c: entering candidate state: node="Node at adab6b73-3013-8559-f2d7-22591934104c [Candidate]" term=2 +testLoggerAdapter verbose: server-adab6b73-3013-8559-f2d7-22591934104c: 2024-06-12T04:32:30.370+1400 [INFO] server-adab6b73-3013-8559-f2d7-22591934104c: election won: tally=2 + testing.go:149: server-adab6b73-3013-8559-f2d7-22591934104c: 2024-06-12T04:32:30.370+1400 [INFO] server-adab6b73-3013-8559-f2d7-22591934104c: election won: tally=2 +testLoggerAdapter verbose: server-adab6b73-3013-8559-f2d7-22591934104c: 2024-06-12T04:32:30.370+1400 [INFO] server-adab6b73-3013-8559-f2d7-22591934104c: entering leader state: leader="Node at adab6b73-3013-8559-f2d7-22591934104c [Leader]" + testing.go:149: server-adab6b73-3013-8559-f2d7-22591934104c: 2024-06-12T04:32:30.370+1400 [INFO] server-adab6b73-3013-8559-f2d7-22591934104c: entering leader state: leader="Node at adab6b73-3013-8559-f2d7-22591934104c [Leader]" +testLoggerAdapter verbose: server-adab6b73-3013-8559-f2d7-22591934104c: 2024-06-12T04:32:30.370+1400 [INFO] server-adab6b73-3013-8559-f2d7-22591934104c: added peer, starting replication: peer=server-ba5ea0b5-da8f-f7a4-c2b0-6369b9df7d9d + testing.go:149: server-adab6b73-3013-8559-f2d7-22591934104c: 2024-06-12T04:32:30.370+1400 [INFO] server-adab6b73-3013-8559-f2d7-22591934104c: added peer, starting replication: peer=server-ba5ea0b5-da8f-f7a4-c2b0-6369b9df7d9d +testLoggerAdapter verbose: server-adab6b73-3013-8559-f2d7-22591934104c: 2024-06-12T04:32:30.370+1400 [INFO] server-adab6b73-3013-8559-f2d7-22591934104c: pipelining replication: peer="{Voter server-ba5ea0b5-da8f-f7a4-c2b0-6369b9df7d9d ba5ea0b5-da8f-f7a4-c2b0-6369b9df7d9d}" + testing.go:149: server-adab6b73-3013-8559-f2d7-22591934104c: 2024-06-12T04:32:30.370+1400 [INFO] server-adab6b73-3013-8559-f2d7-22591934104c: pipelining replication: peer="{Voter server-ba5ea0b5-da8f-f7a4-c2b0-6369b9df7d9d ba5ea0b5-da8f-f7a4-c2b0-6369b9df7d9d}" +testLoggerAdapter verbose: server-d56af706-964d-9927-fc35-25334e2b1781: 2024-06-12T04:32:30.377+1400 [WARN] server-d56af706-964d-9927-fc35-25334e2b1781: no known peers, aborting election + testing.go:149: server-d56af706-964d-9927-fc35-25334e2b1781: 2024-06-12T04:32:30.377+1400 [WARN] server-d56af706-964d-9927-fc35-25334e2b1781: no known peers, aborting election +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:30.476+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:30.370872918 +1400 +14 m=+7.278391537 (1 nodes), 86.227631ms from start of poll, 87.128599ms from cluster start. Timeout at 2024-06-12 04:32:30.47690308 +1400 +14 m=+7.384421699, 106.030162ms after stability + testing.go:149: cluster: 2024-06-12T04:32:30.476+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:30.370872918 +1400 +14 m=+7.278391537 (1 nodes), 86.227631ms from start of poll, 87.128599ms from cluster start. Timeout at 2024-06-12 04:32:30.47690308 +1400 +14 m=+7.384421699, 106.030162ms after stability +testLoggerAdapter verbose: server-adab6b73-3013-8559-f2d7-22591934104c: 2024-06-12T04:32:30.477+1400 [INFO] server-adab6b73-3013-8559-f2d7-22591934104c: updating configuration: command=AddStaging server-id=server-d56af706-964d-9927-fc35-25334e2b1781 server-addr=d56af706-964d-9927-fc35-25334e2b1781 servers="[{Suffrage:Voter ID:server-ba5ea0b5-da8f-f7a4-c2b0-6369b9df7d9d Address:ba5ea0b5-da8f-f7a4-c2b0-6369b9df7d9d} {Suffrage:Voter ID:server-adab6b73-3013-8559-f2d7-22591934104c Address:adab6b73-3013-8559-f2d7-22591934104c} {Suffrage:Voter ID:server-d56af706-964d-9927-fc35-25334e2b1781 Address:d56af706-964d-9927-fc35-25334e2b1781}]" + testing.go:149: server-adab6b73-3013-8559-f2d7-22591934104c: 2024-06-12T04:32:30.477+1400 [INFO] server-adab6b73-3013-8559-f2d7-22591934104c: updating configuration: command=AddStaging server-id=server-d56af706-964d-9927-fc35-25334e2b1781 server-addr=d56af706-964d-9927-fc35-25334e2b1781 servers="[{Suffrage:Voter ID:server-ba5ea0b5-da8f-f7a4-c2b0-6369b9df7d9d Address:ba5ea0b5-da8f-f7a4-c2b0-6369b9df7d9d} {Suffrage:Voter ID:server-adab6b73-3013-8559-f2d7-22591934104c Address:adab6b73-3013-8559-f2d7-22591934104c} {Suffrage:Voter ID:server-d56af706-964d-9927-fc35-25334e2b1781 Address:d56af706-964d-9927-fc35-25334e2b1781}]" +testLoggerAdapter verbose: server-adab6b73-3013-8559-f2d7-22591934104c: 2024-06-12T04:32:30.477+1400 [INFO] server-adab6b73-3013-8559-f2d7-22591934104c: added peer, starting replication: peer=server-d56af706-964d-9927-fc35-25334e2b1781 + testing.go:149: server-adab6b73-3013-8559-f2d7-22591934104c: 2024-06-12T04:32:30.477+1400 [INFO] server-adab6b73-3013-8559-f2d7-22591934104c: added peer, starting replication: peer=server-d56af706-964d-9927-fc35-25334e2b1781 +testLoggerAdapter verbose: server-d56af706-964d-9927-fc35-25334e2b1781: 2024-06-12T04:32:30.477+1400 [WARN] server-d56af706-964d-9927-fc35-25334e2b1781: failed to get previous log: previous-index=3 last-index=0 error="log not found" + testing.go:149: server-d56af706-964d-9927-fc35-25334e2b1781: 2024-06-12T04:32:30.477+1400 [WARN] server-d56af706-964d-9927-fc35-25334e2b1781: failed to get previous log: previous-index=3 last-index=0 error="log not found" +testLoggerAdapter verbose: server-adab6b73-3013-8559-f2d7-22591934104c: 2024-06-12T04:32:30.477+1400 [WARN] server-adab6b73-3013-8559-f2d7-22591934104c: appendEntries rejected, sending older logs: peer="{Voter server-d56af706-964d-9927-fc35-25334e2b1781 d56af706-964d-9927-fc35-25334e2b1781}" next=1 + testing.go:149: server-adab6b73-3013-8559-f2d7-22591934104c: 2024-06-12T04:32:30.477+1400 [WARN] server-adab6b73-3013-8559-f2d7-22591934104c: appendEntries rejected, sending older logs: peer="{Voter server-d56af706-964d-9927-fc35-25334e2b1781 d56af706-964d-9927-fc35-25334e2b1781}" next=1 +testLoggerAdapter verbose: server-adab6b73-3013-8559-f2d7-22591934104c: 2024-06-12T04:32:30.477+1400 [INFO] server-adab6b73-3013-8559-f2d7-22591934104c: pipelining replication: peer="{Voter server-d56af706-964d-9927-fc35-25334e2b1781 d56af706-964d-9927-fc35-25334e2b1781}" + testing.go:149: server-adab6b73-3013-8559-f2d7-22591934104c: 2024-06-12T04:32:30.477+1400 [INFO] server-adab6b73-3013-8559-f2d7-22591934104c: pipelining replication: peer="{Voter server-d56af706-964d-9927-fc35-25334e2b1781 d56af706-964d-9927-fc35-25334e2b1781}" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:30.477+1400 [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-06-12T04:32:30.477+1400 [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:30.582+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:30.477817807 +1400 +14 m=+7.385336426 (1 nodes), 1.959µs from start of poll, 194.073488ms from cluster start. Timeout at 2024-06-12 04:32:30.582849765 +1400 +14 m=+7.490368344, 105.031918ms after stability + testing.go:149: cluster: 2024-06-12T04:32:30.582+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:30.477817807 +1400 +14 m=+7.385336426 (1 nodes), 1.959µs from start of poll, 194.073488ms from cluster start. Timeout at 2024-06-12 04:32:30.582849765 +1400 +14 m=+7.490368344, 105.031918ms after stability +testLoggerAdapter verbose: server-adab6b73-3013-8559-f2d7-22591934104c: 2024-06-12T04:32:30.583+1400 [INFO] server-adab6b73-3013-8559-f2d7-22591934104c: aborting pipeline replication: peer="{Voter server-d56af706-964d-9927-fc35-25334e2b1781 d56af706-964d-9927-fc35-25334e2b1781}" + testing.go:149: server-adab6b73-3013-8559-f2d7-22591934104c: 2024-06-12T04:32:30.583+1400 [INFO] server-adab6b73-3013-8559-f2d7-22591934104c: aborting pipeline replication: peer="{Voter server-d56af706-964d-9927-fc35-25334e2b1781 d56af706-964d-9927-fc35-25334e2b1781}" +testLoggerAdapter verbose: server-adab6b73-3013-8559-f2d7-22591934104c: 2024-06-12T04:32:30.583+1400 [INFO] server-adab6b73-3013-8559-f2d7-22591934104c: aborting pipeline replication: peer="{Voter server-ba5ea0b5-da8f-f7a4-c2b0-6369b9df7d9d ba5ea0b5-da8f-f7a4-c2b0-6369b9df7d9d}" + testing.go:149: server-adab6b73-3013-8559-f2d7-22591934104c: 2024-06-12T04:32:30.583+1400 [INFO] server-adab6b73-3013-8559-f2d7-22591934104c: aborting pipeline replication: peer="{Voter server-ba5ea0b5-da8f-f7a4-c2b0-6369b9df7d9d ba5ea0b5-da8f-f7a4-c2b0-6369b9df7d9d}" +--- PASS: TestRaft_JoinNode (0.30s) === RUN TestRaft_JoinNode_ConfigStore -testLoggerAdapter verbose: server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: 2024-06-11T02:29:17.144-1200 [INFO] server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb Address:02a5b28a-45a3-7cdf-d1b0-fcab974027eb}]" - testing.go:149: server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: 2024-06-11T02:29:17.144-1200 [INFO] server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb Address:02a5b28a-45a3-7cdf-d1b0-fcab974027eb}]" -testLoggerAdapter verbose: server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: 2024-06-11T02:29:17.144-1200 [INFO] server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: entering follower state: follower="Node at 02a5b28a-45a3-7cdf-d1b0-fcab974027eb [Follower]" leader= - testing.go:149: server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: 2024-06-11T02:29:17.144-1200 [INFO] server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: entering follower state: follower="Node at 02a5b28a-45a3-7cdf-d1b0-fcab974027eb [Follower]" leader= -testLoggerAdapter verbose: server-df93554e-b4aa-ce17-2d9e-7f039aa6e8ce: 2024-06-11T02:29:17.144-1200 [INFO] server-df93554e-b4aa-ce17-2d9e-7f039aa6e8ce: initial configuration: index=0 servers=[] - testing.go:149: server-df93554e-b4aa-ce17-2d9e-7f039aa6e8ce: 2024-06-11T02:29:17.144-1200 [INFO] server-df93554e-b4aa-ce17-2d9e-7f039aa6e8ce: initial configuration: index=0 servers=[] -testLoggerAdapter verbose: server-df93554e-b4aa-ce17-2d9e-7f039aa6e8ce: 2024-06-11T02:29:17.145-1200 [INFO] server-df93554e-b4aa-ce17-2d9e-7f039aa6e8ce: entering follower state: follower="Node at df93554e-b4aa-ce17-2d9e-7f039aa6e8ce [Follower]" leader= - testing.go:149: server-df93554e-b4aa-ce17-2d9e-7f039aa6e8ce: 2024-06-11T02:29:17.145-1200 [INFO] server-df93554e-b4aa-ce17-2d9e-7f039aa6e8ce: entering follower state: follower="Node at df93554e-b4aa-ce17-2d9e-7f039aa6e8ce [Follower]" leader= -testLoggerAdapter verbose: server-80c1161f-7be8-484a-900b-4bab23eae63f: 2024-06-11T02:29:17.145-1200 [INFO] server-80c1161f-7be8-484a-900b-4bab23eae63f: initial configuration: index=0 servers=[] - testing.go:149: server-80c1161f-7be8-484a-900b-4bab23eae63f: 2024-06-11T02:29:17.145-1200 [INFO] server-80c1161f-7be8-484a-900b-4bab23eae63f: initial configuration: index=0 servers=[] -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:17.146-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-80c1161f-7be8-484a-900b-4bab23eae63f: 2024-06-11T02:29:17.146-1200 [INFO] server-80c1161f-7be8-484a-900b-4bab23eae63f: entering follower state: follower="Node at 80c1161f-7be8-484a-900b-4bab23eae63f [Follower]" leader= - testing.go:149: cluster: 2024-06-11T02:29:17.146-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: server-80c1161f-7be8-484a-900b-4bab23eae63f: 2024-06-11T02:29:17.146-1200 [INFO] server-80c1161f-7be8-484a-900b-4bab23eae63f: entering follower state: follower="Node at 80c1161f-7be8-484a-900b-4bab23eae63f [Follower]" leader= -testLoggerAdapter verbose: server-df93554e-b4aa-ce17-2d9e-7f039aa6e8ce: 2024-06-11T02:29:17.200-1200 [WARN] server-df93554e-b4aa-ce17-2d9e-7f039aa6e8ce: no known peers, aborting election - testing.go:149: server-df93554e-b4aa-ce17-2d9e-7f039aa6e8ce: 2024-06-11T02:29:17.200-1200 [WARN] server-df93554e-b4aa-ce17-2d9e-7f039aa6e8ce: no known peers, aborting election -testLoggerAdapter verbose: server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: 2024-06-11T02:29:17.206-1200 [WARN] server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: 2024-06-11T02:29:17.206-1200 [WARN] server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: 2024-06-11T02:29:17.206-1200 [INFO] server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: entering candidate state: node="Node at 02a5b28a-45a3-7cdf-d1b0-fcab974027eb [Candidate]" term=2 - testing.go:149: server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: 2024-06-11T02:29:17.206-1200 [INFO] server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: entering candidate state: node="Node at 02a5b28a-45a3-7cdf-d1b0-fcab974027eb [Candidate]" term=2 -testLoggerAdapter verbose: server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: 2024-06-11T02:29:17.206-1200 [INFO] server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: election won: tally=1 - testing.go:149: server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: 2024-06-11T02:29:17.206-1200 [INFO] server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: election won: tally=1 -testLoggerAdapter verbose: server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: 2024-06-11T02:29:17.206-1200 [INFO] server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: entering leader state: leader="Node at 02a5b28a-45a3-7cdf-d1b0-fcab974027eb [Leader]" - testing.go:149: server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: 2024-06-11T02:29:17.206-1200 [INFO] server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: entering leader state: leader="Node at 02a5b28a-45a3-7cdf-d1b0-fcab974027eb [Leader]" -testLoggerAdapter verbose: server-80c1161f-7be8-484a-900b-4bab23eae63f: 2024-06-11T02:29:17.220-1200 [WARN] server-80c1161f-7be8-484a-900b-4bab23eae63f: no known peers, aborting election - testing.go:149: server-80c1161f-7be8-484a-900b-4bab23eae63f: 2024-06-11T02:29:17.220-1200 [WARN] server-80c1161f-7be8-484a-900b-4bab23eae63f: no known peers, aborting election -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:17.312-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:17.207026486 -1200 -12 m=+8.202399028 (1 nodes), 60.925979ms from start of poll, 63.244102ms from cluster start. Timeout at 2024-06-11 02:29:17.312574292 -1200 -12 m=+8.307946834, 105.547806ms after stability - testing.go:149: cluster: 2024-06-11T02:29:17.312-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:17.207026486 -1200 -12 m=+8.202399028 (1 nodes), 60.925979ms from start of poll, 63.244102ms from cluster start. Timeout at 2024-06-11 02:29:17.312574292 -1200 -12 m=+8.307946834, 105.547806ms after stability -testLoggerAdapter verbose: server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: 2024-06-11T02:29:17.312-1200 [INFO] server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: updating configuration: command=AddStaging server-id=server-df93554e-b4aa-ce17-2d9e-7f039aa6e8ce server-addr=df93554e-b4aa-ce17-2d9e-7f039aa6e8ce servers="[{Suffrage:Voter ID:server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb Address:02a5b28a-45a3-7cdf-d1b0-fcab974027eb} {Suffrage:Voter ID:server-df93554e-b4aa-ce17-2d9e-7f039aa6e8ce Address:df93554e-b4aa-ce17-2d9e-7f039aa6e8ce}]" - testing.go:149: server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: 2024-06-11T02:29:17.312-1200 [INFO] server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: updating configuration: command=AddStaging server-id=server-df93554e-b4aa-ce17-2d9e-7f039aa6e8ce server-addr=df93554e-b4aa-ce17-2d9e-7f039aa6e8ce servers="[{Suffrage:Voter ID:server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb Address:02a5b28a-45a3-7cdf-d1b0-fcab974027eb} {Suffrage:Voter ID:server-df93554e-b4aa-ce17-2d9e-7f039aa6e8ce Address:df93554e-b4aa-ce17-2d9e-7f039aa6e8ce}]" -testLoggerAdapter verbose: server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: 2024-06-11T02:29:17.312-1200 [INFO] server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: added peer, starting replication: peer=server-df93554e-b4aa-ce17-2d9e-7f039aa6e8ce - testing.go:149: server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: 2024-06-11T02:29:17.312-1200 [INFO] server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: added peer, starting replication: peer=server-df93554e-b4aa-ce17-2d9e-7f039aa6e8ce -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:17.313-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:17.313-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-df93554e-b4aa-ce17-2d9e-7f039aa6e8ce: 2024-06-11T02:29:17.313-1200 [WARN] server-df93554e-b4aa-ce17-2d9e-7f039aa6e8ce: failed to get previous log: previous-index=3 last-index=0 error="log not found" - testing.go:149: server-df93554e-b4aa-ce17-2d9e-7f039aa6e8ce: 2024-06-11T02:29:17.313-1200 [WARN] server-df93554e-b4aa-ce17-2d9e-7f039aa6e8ce: failed to get previous log: previous-index=3 last-index=0 error="log not found" -testLoggerAdapter verbose: server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: 2024-06-11T02:29:17.313-1200 [WARN] server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: appendEntries rejected, sending older logs: peer="{Voter server-df93554e-b4aa-ce17-2d9e-7f039aa6e8ce df93554e-b4aa-ce17-2d9e-7f039aa6e8ce}" next=1 - testing.go:149: server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: 2024-06-11T02:29:17.313-1200 [WARN] server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: appendEntries rejected, sending older logs: peer="{Voter server-df93554e-b4aa-ce17-2d9e-7f039aa6e8ce df93554e-b4aa-ce17-2d9e-7f039aa6e8ce}" next=1 -testLoggerAdapter verbose: server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: 2024-06-11T02:29:17.313-1200 [INFO] server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: pipelining replication: peer="{Voter server-df93554e-b4aa-ce17-2d9e-7f039aa6e8ce df93554e-b4aa-ce17-2d9e-7f039aa6e8ce}" - testing.go:149: server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: 2024-06-11T02:29:17.313-1200 [INFO] server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: pipelining replication: peer="{Voter server-df93554e-b4aa-ce17-2d9e-7f039aa6e8ce df93554e-b4aa-ce17-2d9e-7f039aa6e8ce}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:17.419-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:17.31330957 -1200 -12 m=+8.308682111 (1 nodes), 2.624µs from start of poll, 169.527185ms from cluster start. Timeout at 2024-06-11 02:29:17.419221952 -1200 -12 m=+8.414594494, 105.912383ms after stability - testing.go:149: cluster: 2024-06-11T02:29:17.419-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:17.31330957 -1200 -12 m=+8.308682111 (1 nodes), 2.624µs from start of poll, 169.527185ms from cluster start. Timeout at 2024-06-11 02:29:17.419221952 -1200 -12 m=+8.414594494, 105.912383ms after stability -testLoggerAdapter verbose: server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: 2024-06-11T02:29:17.419-1200 [INFO] server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: updating configuration: command=AddStaging server-id=server-80c1161f-7be8-484a-900b-4bab23eae63f server-addr=80c1161f-7be8-484a-900b-4bab23eae63f servers="[{Suffrage:Voter ID:server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb Address:02a5b28a-45a3-7cdf-d1b0-fcab974027eb} {Suffrage:Voter ID:server-df93554e-b4aa-ce17-2d9e-7f039aa6e8ce Address:df93554e-b4aa-ce17-2d9e-7f039aa6e8ce} {Suffrage:Voter ID:server-80c1161f-7be8-484a-900b-4bab23eae63f Address:80c1161f-7be8-484a-900b-4bab23eae63f}]" - testing.go:149: server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: 2024-06-11T02:29:17.419-1200 [INFO] server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: updating configuration: command=AddStaging server-id=server-80c1161f-7be8-484a-900b-4bab23eae63f server-addr=80c1161f-7be8-484a-900b-4bab23eae63f servers="[{Suffrage:Voter ID:server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb Address:02a5b28a-45a3-7cdf-d1b0-fcab974027eb} {Suffrage:Voter ID:server-df93554e-b4aa-ce17-2d9e-7f039aa6e8ce Address:df93554e-b4aa-ce17-2d9e-7f039aa6e8ce} {Suffrage:Voter ID:server-80c1161f-7be8-484a-900b-4bab23eae63f Address:80c1161f-7be8-484a-900b-4bab23eae63f}]" -testLoggerAdapter verbose: server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: 2024-06-11T02:29:17.419-1200 [INFO] server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: added peer, starting replication: peer=server-80c1161f-7be8-484a-900b-4bab23eae63f - testing.go:149: server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: 2024-06-11T02:29:17.419-1200 [INFO] server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: added peer, starting replication: peer=server-80c1161f-7be8-484a-900b-4bab23eae63f -testLoggerAdapter verbose: server-80c1161f-7be8-484a-900b-4bab23eae63f: 2024-06-11T02:29:17.419-1200 [WARN] server-80c1161f-7be8-484a-900b-4bab23eae63f: failed to get previous log: previous-index=4 last-index=0 error="log not found" - testing.go:149: server-80c1161f-7be8-484a-900b-4bab23eae63f: 2024-06-11T02:29:17.419-1200 [WARN] server-80c1161f-7be8-484a-900b-4bab23eae63f: failed to get previous log: previous-index=4 last-index=0 error="log not found" -testLoggerAdapter verbose: server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: 2024-06-11T02:29:17.419-1200 [WARN] server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: appendEntries rejected, sending older logs: peer="{Voter server-80c1161f-7be8-484a-900b-4bab23eae63f 80c1161f-7be8-484a-900b-4bab23eae63f}" next=1 - testing.go:149: server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: 2024-06-11T02:29:17.419-1200 [WARN] server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: appendEntries rejected, sending older logs: peer="{Voter server-80c1161f-7be8-484a-900b-4bab23eae63f 80c1161f-7be8-484a-900b-4bab23eae63f}" next=1 -testLoggerAdapter verbose: server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: 2024-06-11T02:29:17.420-1200 [INFO] server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: pipelining replication: peer="{Voter server-80c1161f-7be8-484a-900b-4bab23eae63f 80c1161f-7be8-484a-900b-4bab23eae63f}" - testing.go:149: server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: 2024-06-11T02:29:17.420-1200 [INFO] server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: pipelining replication: peer="{Voter server-80c1161f-7be8-484a-900b-4bab23eae63f 80c1161f-7be8-484a-900b-4bab23eae63f}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:17.420-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:17.420-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:17.526-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:17.420743257 -1200 -12 m=+8.416115799 (1 nodes), 2.625µs from start of poll, 276.960873ms from cluster start. Timeout at 2024-06-11 02:29:17.526380312 -1200 -12 m=+8.521753437, 105.637638ms after stability - testing.go:149: cluster: 2024-06-11T02:29:17.526-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:17.420743257 -1200 -12 m=+8.416115799 (1 nodes), 2.625µs from start of poll, 276.960873ms from cluster start. Timeout at 2024-06-11 02:29:17.526380312 -1200 -12 m=+8.521753437, 105.637638ms after stability -testLoggerAdapter verbose: server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: 2024-06-11T02:29:17.526-1200 [INFO] server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: aborting pipeline replication: peer="{Voter server-df93554e-b4aa-ce17-2d9e-7f039aa6e8ce df93554e-b4aa-ce17-2d9e-7f039aa6e8ce}" - testing.go:149: server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: 2024-06-11T02:29:17.526-1200 [INFO] server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: aborting pipeline replication: peer="{Voter server-df93554e-b4aa-ce17-2d9e-7f039aa6e8ce df93554e-b4aa-ce17-2d9e-7f039aa6e8ce}" -testLoggerAdapter verbose: server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: 2024-06-11T02:29:17.527-1200 [INFO] server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: aborting pipeline replication: peer="{Voter server-80c1161f-7be8-484a-900b-4bab23eae63f 80c1161f-7be8-484a-900b-4bab23eae63f}" - testing.go:149: server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: 2024-06-11T02:29:17.527-1200 [INFO] server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: aborting pipeline replication: peer="{Voter server-80c1161f-7be8-484a-900b-4bab23eae63f 80c1161f-7be8-484a-900b-4bab23eae63f}" -testLoggerAdapter verbose: server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: 2024-06-11T02:29:17.577-1200 [ERROR] server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: failed to heartbeat to: peer=df93554e-b4aa-ce17-2d9e-7f039aa6e8ce error="command timed out" - testing.go:149: server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: 2024-06-11T02:29:17.577-1200 [ERROR] server-02a5b28a-45a3-7cdf-d1b0-fcab974027eb: failed to heartbeat to: peer=df93554e-b4aa-ce17-2d9e-7f039aa6e8ce error="command timed out" ---- PASS: TestRaft_JoinNode_ConfigStore (0.44s) +testLoggerAdapter verbose: server-745c502e-38cb-1b1e-a943-bc158cafcce8: 2024-06-12T04:32:30.584+1400 [INFO] server-745c502e-38cb-1b1e-a943-bc158cafcce8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-745c502e-38cb-1b1e-a943-bc158cafcce8 Address:745c502e-38cb-1b1e-a943-bc158cafcce8}]" + testing.go:149: server-745c502e-38cb-1b1e-a943-bc158cafcce8: 2024-06-12T04:32:30.584+1400 [INFO] server-745c502e-38cb-1b1e-a943-bc158cafcce8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-745c502e-38cb-1b1e-a943-bc158cafcce8 Address:745c502e-38cb-1b1e-a943-bc158cafcce8}]" +testLoggerAdapter verbose: server-aecede68-0038-d0d2-a0bf-dfc29611e5d7: 2024-06-12T04:32:30.585+1400 [INFO] server-aecede68-0038-d0d2-a0bf-dfc29611e5d7: initial configuration: index=0 servers=[] + testing.go:149: server-aecede68-0038-d0d2-a0bf-dfc29611e5d7: 2024-06-12T04:32:30.585+1400 [INFO] server-aecede68-0038-d0d2-a0bf-dfc29611e5d7: initial configuration: index=0 servers=[] +testLoggerAdapter verbose: server-f3da6f6c-5d28-9436-09ca-c821425fade9: 2024-06-12T04:32:30.585+1400 [INFO] server-f3da6f6c-5d28-9436-09ca-c821425fade9: initial configuration: index=0 servers=[] + testing.go:149: server-f3da6f6c-5d28-9436-09ca-c821425fade9: 2024-06-12T04:32:30.585+1400 [INFO] server-f3da6f6c-5d28-9436-09ca-c821425fade9: initial configuration: index=0 servers=[] +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:30.585+1400 [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-06-12T04:32:30.585+1400 [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-745c502e-38cb-1b1e-a943-bc158cafcce8: 2024-06-12T04:32:30.585+1400 [INFO] server-745c502e-38cb-1b1e-a943-bc158cafcce8: entering follower state: follower="Node at 745c502e-38cb-1b1e-a943-bc158cafcce8 [Follower]" leader= + testing.go:149: server-745c502e-38cb-1b1e-a943-bc158cafcce8: 2024-06-12T04:32:30.585+1400 [INFO] server-745c502e-38cb-1b1e-a943-bc158cafcce8: entering follower state: follower="Node at 745c502e-38cb-1b1e-a943-bc158cafcce8 [Follower]" leader= +testLoggerAdapter verbose: server-aecede68-0038-d0d2-a0bf-dfc29611e5d7: 2024-06-12T04:32:30.585+1400 [INFO] server-aecede68-0038-d0d2-a0bf-dfc29611e5d7: entering follower state: follower="Node at aecede68-0038-d0d2-a0bf-dfc29611e5d7 [Follower]" leader= + testing.go:149: server-aecede68-0038-d0d2-a0bf-dfc29611e5d7: 2024-06-12T04:32:30.585+1400 [INFO] server-aecede68-0038-d0d2-a0bf-dfc29611e5d7: entering follower state: follower="Node at aecede68-0038-d0d2-a0bf-dfc29611e5d7 [Follower]" leader= +testLoggerAdapter verbose: server-f3da6f6c-5d28-9436-09ca-c821425fade9: 2024-06-12T04:32:30.586+1400 [INFO] server-f3da6f6c-5d28-9436-09ca-c821425fade9: entering follower state: follower="Node at f3da6f6c-5d28-9436-09ca-c821425fade9 [Follower]" leader= + testing.go:149: server-f3da6f6c-5d28-9436-09ca-c821425fade9: 2024-06-12T04:32:30.586+1400 [INFO] server-f3da6f6c-5d28-9436-09ca-c821425fade9: entering follower state: follower="Node at f3da6f6c-5d28-9436-09ca-c821425fade9 [Follower]" leader= +testLoggerAdapter verbose: server-745c502e-38cb-1b1e-a943-bc158cafcce8: 2024-06-12T04:32:30.636+1400 [WARN] server-745c502e-38cb-1b1e-a943-bc158cafcce8: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-745c502e-38cb-1b1e-a943-bc158cafcce8: 2024-06-12T04:32:30.636+1400 [WARN] server-745c502e-38cb-1b1e-a943-bc158cafcce8: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-745c502e-38cb-1b1e-a943-bc158cafcce8: 2024-06-12T04:32:30.636+1400 [INFO] server-745c502e-38cb-1b1e-a943-bc158cafcce8: entering candidate state: node="Node at 745c502e-38cb-1b1e-a943-bc158cafcce8 [Candidate]" term=2 + testing.go:149: server-745c502e-38cb-1b1e-a943-bc158cafcce8: 2024-06-12T04:32:30.636+1400 [INFO] server-745c502e-38cb-1b1e-a943-bc158cafcce8: entering candidate state: node="Node at 745c502e-38cb-1b1e-a943-bc158cafcce8 [Candidate]" term=2 +testLoggerAdapter verbose: server-745c502e-38cb-1b1e-a943-bc158cafcce8: 2024-06-12T04:32:30.636+1400 [INFO] server-745c502e-38cb-1b1e-a943-bc158cafcce8: election won: tally=1 + testing.go:149: server-745c502e-38cb-1b1e-a943-bc158cafcce8: 2024-06-12T04:32:30.636+1400 [INFO] server-745c502e-38cb-1b1e-a943-bc158cafcce8: election won: tally=1 +testLoggerAdapter verbose: server-745c502e-38cb-1b1e-a943-bc158cafcce8: 2024-06-12T04:32:30.636+1400 [INFO] server-745c502e-38cb-1b1e-a943-bc158cafcce8: entering leader state: leader="Node at 745c502e-38cb-1b1e-a943-bc158cafcce8 [Leader]" + testing.go:149: server-745c502e-38cb-1b1e-a943-bc158cafcce8: 2024-06-12T04:32:30.636+1400 [INFO] server-745c502e-38cb-1b1e-a943-bc158cafcce8: entering leader state: leader="Node at 745c502e-38cb-1b1e-a943-bc158cafcce8 [Leader]" +testLoggerAdapter verbose: server-f3da6f6c-5d28-9436-09ca-c821425fade9: 2024-06-12T04:32:30.656+1400 [WARN] server-f3da6f6c-5d28-9436-09ca-c821425fade9: no known peers, aborting election + testing.go:149: server-f3da6f6c-5d28-9436-09ca-c821425fade9: 2024-06-12T04:32:30.656+1400 [WARN] server-f3da6f6c-5d28-9436-09ca-c821425fade9: no known peers, aborting election +testLoggerAdapter verbose: server-aecede68-0038-d0d2-a0bf-dfc29611e5d7: 2024-06-12T04:32:30.683+1400 [WARN] server-aecede68-0038-d0d2-a0bf-dfc29611e5d7: no known peers, aborting election + testing.go:149: server-aecede68-0038-d0d2-a0bf-dfc29611e5d7: 2024-06-12T04:32:30.683+1400 [WARN] server-aecede68-0038-d0d2-a0bf-dfc29611e5d7: no known peers, aborting election +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:30.743+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:30.636897349 +1400 +14 m=+7.544415928 (1 nodes), 51.048771ms from start of poll, 52.229449ms from cluster start. Timeout at 2024-06-12 04:32:30.742935311 +1400 +14 m=+7.650453930, 106.038002ms after stability + testing.go:149: cluster: 2024-06-12T04:32:30.743+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:30.636897349 +1400 +14 m=+7.544415928 (1 nodes), 51.048771ms from start of poll, 52.229449ms from cluster start. Timeout at 2024-06-12 04:32:30.742935311 +1400 +14 m=+7.650453930, 106.038002ms after stability +testLoggerAdapter verbose: server-745c502e-38cb-1b1e-a943-bc158cafcce8: 2024-06-12T04:32:30.743+1400 [INFO] server-745c502e-38cb-1b1e-a943-bc158cafcce8: updating configuration: command=AddStaging server-id=server-aecede68-0038-d0d2-a0bf-dfc29611e5d7 server-addr=aecede68-0038-d0d2-a0bf-dfc29611e5d7 servers="[{Suffrage:Voter ID:server-745c502e-38cb-1b1e-a943-bc158cafcce8 Address:745c502e-38cb-1b1e-a943-bc158cafcce8} {Suffrage:Voter ID:server-aecede68-0038-d0d2-a0bf-dfc29611e5d7 Address:aecede68-0038-d0d2-a0bf-dfc29611e5d7}]" + testing.go:149: server-745c502e-38cb-1b1e-a943-bc158cafcce8: 2024-06-12T04:32:30.743+1400 [INFO] server-745c502e-38cb-1b1e-a943-bc158cafcce8: updating configuration: command=AddStaging server-id=server-aecede68-0038-d0d2-a0bf-dfc29611e5d7 server-addr=aecede68-0038-d0d2-a0bf-dfc29611e5d7 servers="[{Suffrage:Voter ID:server-745c502e-38cb-1b1e-a943-bc158cafcce8 Address:745c502e-38cb-1b1e-a943-bc158cafcce8} {Suffrage:Voter ID:server-aecede68-0038-d0d2-a0bf-dfc29611e5d7 Address:aecede68-0038-d0d2-a0bf-dfc29611e5d7}]" +testLoggerAdapter verbose: server-745c502e-38cb-1b1e-a943-bc158cafcce8: 2024-06-12T04:32:30.743+1400 [INFO] server-745c502e-38cb-1b1e-a943-bc158cafcce8: added peer, starting replication: peer=server-aecede68-0038-d0d2-a0bf-dfc29611e5d7 + testing.go:149: server-745c502e-38cb-1b1e-a943-bc158cafcce8: 2024-06-12T04:32:30.743+1400 [INFO] server-745c502e-38cb-1b1e-a943-bc158cafcce8: added peer, starting replication: peer=server-aecede68-0038-d0d2-a0bf-dfc29611e5d7 +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:30.743+1400 [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-06-12T04:32:30.743+1400 [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-aecede68-0038-d0d2-a0bf-dfc29611e5d7: 2024-06-12T04:32:30.743+1400 [WARN] server-aecede68-0038-d0d2-a0bf-dfc29611e5d7: failed to get previous log: previous-index=3 last-index=0 error="log not found" + testing.go:149: server-aecede68-0038-d0d2-a0bf-dfc29611e5d7: 2024-06-12T04:32:30.743+1400 [WARN] server-aecede68-0038-d0d2-a0bf-dfc29611e5d7: failed to get previous log: previous-index=3 last-index=0 error="log not found" +testLoggerAdapter verbose: server-745c502e-38cb-1b1e-a943-bc158cafcce8: 2024-06-12T04:32:30.743+1400 [WARN] server-745c502e-38cb-1b1e-a943-bc158cafcce8: appendEntries rejected, sending older logs: peer="{Voter server-aecede68-0038-d0d2-a0bf-dfc29611e5d7 aecede68-0038-d0d2-a0bf-dfc29611e5d7}" next=1 + testing.go:149: server-745c502e-38cb-1b1e-a943-bc158cafcce8: 2024-06-12T04:32:30.743+1400 [WARN] server-745c502e-38cb-1b1e-a943-bc158cafcce8: appendEntries rejected, sending older logs: peer="{Voter server-aecede68-0038-d0d2-a0bf-dfc29611e5d7 aecede68-0038-d0d2-a0bf-dfc29611e5d7}" next=1 +testLoggerAdapter verbose: server-745c502e-38cb-1b1e-a943-bc158cafcce8: 2024-06-12T04:32:30.743+1400 [INFO] server-745c502e-38cb-1b1e-a943-bc158cafcce8: pipelining replication: peer="{Voter server-aecede68-0038-d0d2-a0bf-dfc29611e5d7 aecede68-0038-d0d2-a0bf-dfc29611e5d7}" + testing.go:149: server-745c502e-38cb-1b1e-a943-bc158cafcce8: 2024-06-12T04:32:30.743+1400 [INFO] server-745c502e-38cb-1b1e-a943-bc158cafcce8: pipelining replication: peer="{Voter server-aecede68-0038-d0d2-a0bf-dfc29611e5d7 aecede68-0038-d0d2-a0bf-dfc29611e5d7}" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:30.849+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:30.743417534 +1400 +14 m=+7.650936113 (1 nodes), 2.44µs from start of poll, 158.749634ms from cluster start. Timeout at 2024-06-12 04:32:30.849172066 +1400 +14 m=+7.756690685, 105.754572ms after stability + testing.go:149: cluster: 2024-06-12T04:32:30.849+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:30.743417534 +1400 +14 m=+7.650936113 (1 nodes), 2.44µs from start of poll, 158.749634ms from cluster start. Timeout at 2024-06-12 04:32:30.849172066 +1400 +14 m=+7.756690685, 105.754572ms after stability +testLoggerAdapter verbose: server-745c502e-38cb-1b1e-a943-bc158cafcce8: 2024-06-12T04:32:30.849+1400 [INFO] server-745c502e-38cb-1b1e-a943-bc158cafcce8: updating configuration: command=AddStaging server-id=server-f3da6f6c-5d28-9436-09ca-c821425fade9 server-addr=f3da6f6c-5d28-9436-09ca-c821425fade9 servers="[{Suffrage:Voter ID:server-745c502e-38cb-1b1e-a943-bc158cafcce8 Address:745c502e-38cb-1b1e-a943-bc158cafcce8} {Suffrage:Voter ID:server-aecede68-0038-d0d2-a0bf-dfc29611e5d7 Address:aecede68-0038-d0d2-a0bf-dfc29611e5d7} {Suffrage:Voter ID:server-f3da6f6c-5d28-9436-09ca-c821425fade9 Address:f3da6f6c-5d28-9436-09ca-c821425fade9}]" + testing.go:149: server-745c502e-38cb-1b1e-a943-bc158cafcce8: 2024-06-12T04:32:30.849+1400 [INFO] server-745c502e-38cb-1b1e-a943-bc158cafcce8: updating configuration: command=AddStaging server-id=server-f3da6f6c-5d28-9436-09ca-c821425fade9 server-addr=f3da6f6c-5d28-9436-09ca-c821425fade9 servers="[{Suffrage:Voter ID:server-745c502e-38cb-1b1e-a943-bc158cafcce8 Address:745c502e-38cb-1b1e-a943-bc158cafcce8} {Suffrage:Voter ID:server-aecede68-0038-d0d2-a0bf-dfc29611e5d7 Address:aecede68-0038-d0d2-a0bf-dfc29611e5d7} {Suffrage:Voter ID:server-f3da6f6c-5d28-9436-09ca-c821425fade9 Address:f3da6f6c-5d28-9436-09ca-c821425fade9}]" +testLoggerAdapter verbose: server-745c502e-38cb-1b1e-a943-bc158cafcce8: 2024-06-12T04:32:30.849+1400 [INFO] server-745c502e-38cb-1b1e-a943-bc158cafcce8: added peer, starting replication: peer=server-f3da6f6c-5d28-9436-09ca-c821425fade9 + testing.go:149: server-745c502e-38cb-1b1e-a943-bc158cafcce8: 2024-06-12T04:32:30.849+1400 [INFO] server-745c502e-38cb-1b1e-a943-bc158cafcce8: added peer, starting replication: peer=server-f3da6f6c-5d28-9436-09ca-c821425fade9 +testLoggerAdapter verbose: server-f3da6f6c-5d28-9436-09ca-c821425fade9: 2024-06-12T04:32:30.849+1400 [WARN] server-f3da6f6c-5d28-9436-09ca-c821425fade9: failed to get previous log: previous-index=4 last-index=0 error="log not found" + testing.go:149: server-f3da6f6c-5d28-9436-09ca-c821425fade9: 2024-06-12T04:32:30.849+1400 [WARN] server-f3da6f6c-5d28-9436-09ca-c821425fade9: failed to get previous log: previous-index=4 last-index=0 error="log not found" +testLoggerAdapter verbose: server-745c502e-38cb-1b1e-a943-bc158cafcce8: 2024-06-12T04:32:30.849+1400 [WARN] server-745c502e-38cb-1b1e-a943-bc158cafcce8: appendEntries rejected, sending older logs: peer="{Voter server-f3da6f6c-5d28-9436-09ca-c821425fade9 f3da6f6c-5d28-9436-09ca-c821425fade9}" next=1 + testing.go:149: server-745c502e-38cb-1b1e-a943-bc158cafcce8: 2024-06-12T04:32:30.849+1400 [WARN] server-745c502e-38cb-1b1e-a943-bc158cafcce8: appendEntries rejected, sending older logs: peer="{Voter server-f3da6f6c-5d28-9436-09ca-c821425fade9 f3da6f6c-5d28-9436-09ca-c821425fade9}" next=1 +testLoggerAdapter verbose: server-745c502e-38cb-1b1e-a943-bc158cafcce8: 2024-06-12T04:32:30.849+1400 [INFO] server-745c502e-38cb-1b1e-a943-bc158cafcce8: pipelining replication: peer="{Voter server-f3da6f6c-5d28-9436-09ca-c821425fade9 f3da6f6c-5d28-9436-09ca-c821425fade9}" + testing.go:149: server-745c502e-38cb-1b1e-a943-bc158cafcce8: 2024-06-12T04:32:30.849+1400 [INFO] server-745c502e-38cb-1b1e-a943-bc158cafcce8: pipelining replication: peer="{Voter server-f3da6f6c-5d28-9436-09ca-c821425fade9 f3da6f6c-5d28-9436-09ca-c821425fade9}" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:30.850+1400 [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-06-12T04:32:30.850+1400 [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:30.955+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:30.850102673 +1400 +14 m=+7.757621252 (1 nodes), 1.36µs from start of poll, 265.434773ms from cluster start. Timeout at 2024-06-12 04:32:30.955440459 +1400 +14 m=+7.862959038, 105.337786ms after stability + testing.go:149: cluster: 2024-06-12T04:32:30.955+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:30.850102673 +1400 +14 m=+7.757621252 (1 nodes), 1.36µs from start of poll, 265.434773ms from cluster start. Timeout at 2024-06-12 04:32:30.955440459 +1400 +14 m=+7.862959038, 105.337786ms after stability +testLoggerAdapter verbose: server-745c502e-38cb-1b1e-a943-bc158cafcce8: 2024-06-12T04:32:30.955+1400 [INFO] server-745c502e-38cb-1b1e-a943-bc158cafcce8: aborting pipeline replication: peer="{Voter server-f3da6f6c-5d28-9436-09ca-c821425fade9 f3da6f6c-5d28-9436-09ca-c821425fade9}" + testing.go:149: server-745c502e-38cb-1b1e-a943-bc158cafcce8: 2024-06-12T04:32:30.955+1400 [INFO] server-745c502e-38cb-1b1e-a943-bc158cafcce8: aborting pipeline replication: peer="{Voter server-f3da6f6c-5d28-9436-09ca-c821425fade9 f3da6f6c-5d28-9436-09ca-c821425fade9}" +testLoggerAdapter verbose: server-745c502e-38cb-1b1e-a943-bc158cafcce8: 2024-06-12T04:32:30.956+1400 [INFO] server-745c502e-38cb-1b1e-a943-bc158cafcce8: aborting pipeline replication: peer="{Voter server-aecede68-0038-d0d2-a0bf-dfc29611e5d7 aecede68-0038-d0d2-a0bf-dfc29611e5d7}" + testing.go:149: server-745c502e-38cb-1b1e-a943-bc158cafcce8: 2024-06-12T04:32:30.956+1400 [INFO] server-745c502e-38cb-1b1e-a943-bc158cafcce8: aborting pipeline replication: peer="{Voter server-aecede68-0038-d0d2-a0bf-dfc29611e5d7 aecede68-0038-d0d2-a0bf-dfc29611e5d7}" +--- PASS: TestRaft_JoinNode_ConfigStore (0.37s) === RUN TestRaft_RemoveFollower -testLoggerAdapter verbose: server-89c4c182-b7ab-adba-9a77-64a4a21eaedc: 2024-06-11T02:29:17.581-1200 [INFO] server-89c4c182-b7ab-adba-9a77-64a4a21eaedc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-89c4c182-b7ab-adba-9a77-64a4a21eaedc Address:89c4c182-b7ab-adba-9a77-64a4a21eaedc} {Suffrage:Voter ID:server-cf196aac-930b-eaa9-17a0-e8901971ea89 Address:cf196aac-930b-eaa9-17a0-e8901971ea89} {Suffrage:Voter ID:server-3142ee68-106d-d227-ee46-8a730782c212 Address:3142ee68-106d-d227-ee46-8a730782c212}]" - testing.go:149: server-89c4c182-b7ab-adba-9a77-64a4a21eaedc: 2024-06-11T02:29:17.581-1200 [INFO] server-89c4c182-b7ab-adba-9a77-64a4a21eaedc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-89c4c182-b7ab-adba-9a77-64a4a21eaedc Address:89c4c182-b7ab-adba-9a77-64a4a21eaedc} {Suffrage:Voter ID:server-cf196aac-930b-eaa9-17a0-e8901971ea89 Address:cf196aac-930b-eaa9-17a0-e8901971ea89} {Suffrage:Voter ID:server-3142ee68-106d-d227-ee46-8a730782c212 Address:3142ee68-106d-d227-ee46-8a730782c212}]" -testLoggerAdapter verbose: server-cf196aac-930b-eaa9-17a0-e8901971ea89: 2024-06-11T02:29:17.581-1200 [INFO] server-cf196aac-930b-eaa9-17a0-e8901971ea89: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-89c4c182-b7ab-adba-9a77-64a4a21eaedc Address:89c4c182-b7ab-adba-9a77-64a4a21eaedc} {Suffrage:Voter ID:server-cf196aac-930b-eaa9-17a0-e8901971ea89 Address:cf196aac-930b-eaa9-17a0-e8901971ea89} {Suffrage:Voter ID:server-3142ee68-106d-d227-ee46-8a730782c212 Address:3142ee68-106d-d227-ee46-8a730782c212}]" - testing.go:149: server-cf196aac-930b-eaa9-17a0-e8901971ea89: 2024-06-11T02:29:17.581-1200 [INFO] server-cf196aac-930b-eaa9-17a0-e8901971ea89: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-89c4c182-b7ab-adba-9a77-64a4a21eaedc Address:89c4c182-b7ab-adba-9a77-64a4a21eaedc} {Suffrage:Voter ID:server-cf196aac-930b-eaa9-17a0-e8901971ea89 Address:cf196aac-930b-eaa9-17a0-e8901971ea89} {Suffrage:Voter ID:server-3142ee68-106d-d227-ee46-8a730782c212 Address:3142ee68-106d-d227-ee46-8a730782c212}]" -testLoggerAdapter verbose: server-3142ee68-106d-d227-ee46-8a730782c212: 2024-06-11T02:29:17.582-1200 [INFO] server-3142ee68-106d-d227-ee46-8a730782c212: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-89c4c182-b7ab-adba-9a77-64a4a21eaedc Address:89c4c182-b7ab-adba-9a77-64a4a21eaedc} {Suffrage:Voter ID:server-cf196aac-930b-eaa9-17a0-e8901971ea89 Address:cf196aac-930b-eaa9-17a0-e8901971ea89} {Suffrage:Voter ID:server-3142ee68-106d-d227-ee46-8a730782c212 Address:3142ee68-106d-d227-ee46-8a730782c212}]" - testing.go:149: server-3142ee68-106d-d227-ee46-8a730782c212: 2024-06-11T02:29:17.582-1200 [INFO] server-3142ee68-106d-d227-ee46-8a730782c212: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-89c4c182-b7ab-adba-9a77-64a4a21eaedc Address:89c4c182-b7ab-adba-9a77-64a4a21eaedc} {Suffrage:Voter ID:server-cf196aac-930b-eaa9-17a0-e8901971ea89 Address:cf196aac-930b-eaa9-17a0-e8901971ea89} {Suffrage:Voter ID:server-3142ee68-106d-d227-ee46-8a730782c212 Address:3142ee68-106d-d227-ee46-8a730782c212}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:17.582-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:17.582-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-89c4c182-b7ab-adba-9a77-64a4a21eaedc: 2024-06-11T02:29:17.582-1200 [INFO] server-89c4c182-b7ab-adba-9a77-64a4a21eaedc: entering follower state: follower="Node at 89c4c182-b7ab-adba-9a77-64a4a21eaedc [Follower]" leader= - testing.go:149: server-89c4c182-b7ab-adba-9a77-64a4a21eaedc: 2024-06-11T02:29:17.582-1200 [INFO] server-89c4c182-b7ab-adba-9a77-64a4a21eaedc: entering follower state: follower="Node at 89c4c182-b7ab-adba-9a77-64a4a21eaedc [Follower]" leader= -testLoggerAdapter verbose: server-cf196aac-930b-eaa9-17a0-e8901971ea89: 2024-06-11T02:29:17.582-1200 [INFO] server-cf196aac-930b-eaa9-17a0-e8901971ea89: entering follower state: follower="Node at cf196aac-930b-eaa9-17a0-e8901971ea89 [Follower]" leader= - testing.go:149: server-cf196aac-930b-eaa9-17a0-e8901971ea89: 2024-06-11T02:29:17.582-1200 [INFO] server-cf196aac-930b-eaa9-17a0-e8901971ea89: entering follower state: follower="Node at cf196aac-930b-eaa9-17a0-e8901971ea89 [Follower]" leader= -testLoggerAdapter verbose: server-3142ee68-106d-d227-ee46-8a730782c212: 2024-06-11T02:29:17.582-1200 [INFO] server-3142ee68-106d-d227-ee46-8a730782c212: entering follower state: follower="Node at 3142ee68-106d-d227-ee46-8a730782c212 [Follower]" leader= - testing.go:149: server-3142ee68-106d-d227-ee46-8a730782c212: 2024-06-11T02:29:17.582-1200 [INFO] server-3142ee68-106d-d227-ee46-8a730782c212: entering follower state: follower="Node at 3142ee68-106d-d227-ee46-8a730782c212 [Follower]" leader= -testLoggerAdapter verbose: server-3142ee68-106d-d227-ee46-8a730782c212: 2024-06-11T02:29:17.651-1200 [WARN] server-3142ee68-106d-d227-ee46-8a730782c212: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-3142ee68-106d-d227-ee46-8a730782c212: 2024-06-11T02:29:17.651-1200 [WARN] server-3142ee68-106d-d227-ee46-8a730782c212: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-3142ee68-106d-d227-ee46-8a730782c212: 2024-06-11T02:29:17.651-1200 [INFO] server-3142ee68-106d-d227-ee46-8a730782c212: entering candidate state: node="Node at 3142ee68-106d-d227-ee46-8a730782c212 [Candidate]" term=2 - testing.go:149: server-3142ee68-106d-d227-ee46-8a730782c212: 2024-06-11T02:29:17.651-1200 [INFO] server-3142ee68-106d-d227-ee46-8a730782c212: entering candidate state: node="Node at 3142ee68-106d-d227-ee46-8a730782c212 [Candidate]" term=2 -testLoggerAdapter verbose: server-3142ee68-106d-d227-ee46-8a730782c212: 2024-06-11T02:29:17.652-1200 [INFO] server-3142ee68-106d-d227-ee46-8a730782c212: election won: tally=2 - testing.go:149: server-3142ee68-106d-d227-ee46-8a730782c212: 2024-06-11T02:29:17.652-1200 [INFO] server-3142ee68-106d-d227-ee46-8a730782c212: election won: tally=2 -testLoggerAdapter verbose: server-3142ee68-106d-d227-ee46-8a730782c212: 2024-06-11T02:29:17.652-1200 [INFO] server-3142ee68-106d-d227-ee46-8a730782c212: entering leader state: leader="Node at 3142ee68-106d-d227-ee46-8a730782c212 [Leader]" - testing.go:149: server-3142ee68-106d-d227-ee46-8a730782c212: 2024-06-11T02:29:17.652-1200 [INFO] server-3142ee68-106d-d227-ee46-8a730782c212: entering leader state: leader="Node at 3142ee68-106d-d227-ee46-8a730782c212 [Leader]" -testLoggerAdapter verbose: server-3142ee68-106d-d227-ee46-8a730782c212: 2024-06-11T02:29:17.652-1200 [INFO] server-3142ee68-106d-d227-ee46-8a730782c212: added peer, starting replication: peer=server-89c4c182-b7ab-adba-9a77-64a4a21eaedc - testing.go:149: server-3142ee68-106d-d227-ee46-8a730782c212: 2024-06-11T02:29:17.652-1200 [INFO] server-3142ee68-106d-d227-ee46-8a730782c212: added peer, starting replication: peer=server-89c4c182-b7ab-adba-9a77-64a4a21eaedc -testLoggerAdapter verbose: server-3142ee68-106d-d227-ee46-8a730782c212: 2024-06-11T02:29:17.652-1200 [INFO] server-3142ee68-106d-d227-ee46-8a730782c212: added peer, starting replication: peer=server-cf196aac-930b-eaa9-17a0-e8901971ea89 - testing.go:149: server-3142ee68-106d-d227-ee46-8a730782c212: 2024-06-11T02:29:17.652-1200 [INFO] server-3142ee68-106d-d227-ee46-8a730782c212: added peer, starting replication: peer=server-cf196aac-930b-eaa9-17a0-e8901971ea89 -testLoggerAdapter verbose: server-3142ee68-106d-d227-ee46-8a730782c212: 2024-06-11T02:29:17.652-1200 [INFO] server-3142ee68-106d-d227-ee46-8a730782c212: pipelining replication: peer="{Voter server-cf196aac-930b-eaa9-17a0-e8901971ea89 cf196aac-930b-eaa9-17a0-e8901971ea89}" - testing.go:149: server-3142ee68-106d-d227-ee46-8a730782c212: 2024-06-11T02:29:17.652-1200 [INFO] server-3142ee68-106d-d227-ee46-8a730782c212: pipelining replication: peer="{Voter server-cf196aac-930b-eaa9-17a0-e8901971ea89 cf196aac-930b-eaa9-17a0-e8901971ea89}" -testLoggerAdapter verbose: server-3142ee68-106d-d227-ee46-8a730782c212: 2024-06-11T02:29:17.653-1200 [INFO] server-3142ee68-106d-d227-ee46-8a730782c212: pipelining replication: peer="{Voter server-89c4c182-b7ab-adba-9a77-64a4a21eaedc 89c4c182-b7ab-adba-9a77-64a4a21eaedc}" - testing.go:149: server-3142ee68-106d-d227-ee46-8a730782c212: 2024-06-11T02:29:17.653-1200 [INFO] server-3142ee68-106d-d227-ee46-8a730782c212: pipelining replication: peer="{Voter server-89c4c182-b7ab-adba-9a77-64a4a21eaedc 89c4c182-b7ab-adba-9a77-64a4a21eaedc}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:17.758-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:17.653441338 -1200 -12 m=+8.648813880 (1 nodes), 71.022706ms from start of poll, 72.263141ms from cluster start. Timeout at 2024-06-11 02:29:17.758873646 -1200 -12 m=+8.754246188, 105.432308ms after stability - testing.go:149: cluster: 2024-06-11T02:29:17.758-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:17.653441338 -1200 -12 m=+8.648813880 (1 nodes), 71.022706ms from start of poll, 72.263141ms from cluster start. Timeout at 2024-06-11 02:29:17.758873646 -1200 -12 m=+8.754246188, 105.432308ms after stability -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:17.764-1200 [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2024-06-11T02:29:17.764-1200 [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:17.870-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:17.764837534 -1200 -12 m=+8.760210076 (2 nodes), 3.5µs from start of poll, 183.659337ms from cluster start. Timeout at 2024-06-11 02:29:17.869993639 -1200 -12 m=+8.865366181, 105.156105ms after stability - testing.go:149: cluster: 2024-06-11T02:29:17.870-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:17.764837534 -1200 -12 m=+8.760210076 (2 nodes), 3.5µs from start of poll, 183.659337ms from cluster start. Timeout at 2024-06-11 02:29:17.869993639 -1200 -12 m=+8.865366181, 105.156105ms after stability -testLoggerAdapter verbose: server-3142ee68-106d-d227-ee46-8a730782c212: 2024-06-11T02:29:17.870-1200 [INFO] server-3142ee68-106d-d227-ee46-8a730782c212: updating configuration: command=RemoveServer server-id=server-89c4c182-b7ab-adba-9a77-64a4a21eaedc server-addr= servers="[{Suffrage:Voter ID:server-cf196aac-930b-eaa9-17a0-e8901971ea89 Address:cf196aac-930b-eaa9-17a0-e8901971ea89} {Suffrage:Voter ID:server-3142ee68-106d-d227-ee46-8a730782c212 Address:3142ee68-106d-d227-ee46-8a730782c212}]" - testing.go:149: server-3142ee68-106d-d227-ee46-8a730782c212: 2024-06-11T02:29:17.870-1200 [INFO] server-3142ee68-106d-d227-ee46-8a730782c212: updating configuration: command=RemoveServer server-id=server-89c4c182-b7ab-adba-9a77-64a4a21eaedc server-addr= servers="[{Suffrage:Voter ID:server-cf196aac-930b-eaa9-17a0-e8901971ea89 Address:cf196aac-930b-eaa9-17a0-e8901971ea89} {Suffrage:Voter ID:server-3142ee68-106d-d227-ee46-8a730782c212 Address:3142ee68-106d-d227-ee46-8a730782c212}]" -testLoggerAdapter verbose: server-3142ee68-106d-d227-ee46-8a730782c212: 2024-06-11T02:29:17.870-1200 [INFO] server-3142ee68-106d-d227-ee46-8a730782c212: removed peer, stopping replication: peer=server-89c4c182-b7ab-adba-9a77-64a4a21eaedc last-index=3 - testing.go:149: server-3142ee68-106d-d227-ee46-8a730782c212: 2024-06-11T02:29:17.870-1200 [INFO] server-3142ee68-106d-d227-ee46-8a730782c212: removed peer, stopping replication: peer=server-89c4c182-b7ab-adba-9a77-64a4a21eaedc last-index=3 -testLoggerAdapter verbose: server-3142ee68-106d-d227-ee46-8a730782c212: 2024-06-11T02:29:17.870-1200 [INFO] server-3142ee68-106d-d227-ee46-8a730782c212: aborting pipeline replication: peer="{Voter server-89c4c182-b7ab-adba-9a77-64a4a21eaedc 89c4c182-b7ab-adba-9a77-64a4a21eaedc}" - testing.go:149: server-3142ee68-106d-d227-ee46-8a730782c212: 2024-06-11T02:29:17.870-1200 [INFO] server-3142ee68-106d-d227-ee46-8a730782c212: aborting pipeline replication: peer="{Voter server-89c4c182-b7ab-adba-9a77-64a4a21eaedc 89c4c182-b7ab-adba-9a77-64a4a21eaedc}" -testLoggerAdapter verbose: server-3142ee68-106d-d227-ee46-8a730782c212: 2024-06-11T02:29:17.976-1200 [INFO] server-3142ee68-106d-d227-ee46-8a730782c212: aborting pipeline replication: peer="{Voter server-cf196aac-930b-eaa9-17a0-e8901971ea89 cf196aac-930b-eaa9-17a0-e8901971ea89}" - testing.go:149: server-3142ee68-106d-d227-ee46-8a730782c212: 2024-06-11T02:29:17.976-1200 [INFO] server-3142ee68-106d-d227-ee46-8a730782c212: aborting pipeline replication: peer="{Voter server-cf196aac-930b-eaa9-17a0-e8901971ea89 cf196aac-930b-eaa9-17a0-e8901971ea89}" ---- PASS: TestRaft_RemoveFollower (0.40s) +testLoggerAdapter verbose: server-26b9b025-eced-2e04-4877-6a2e92ef64a1: 2024-06-12T04:32:30.958+1400 [INFO] server-26b9b025-eced-2e04-4877-6a2e92ef64a1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-26b9b025-eced-2e04-4877-6a2e92ef64a1 Address:26b9b025-eced-2e04-4877-6a2e92ef64a1} {Suffrage:Voter ID:server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328 Address:a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328} {Suffrage:Voter ID:server-3e3b0020-764a-ead7-384c-623cc6f4ae61 Address:3e3b0020-764a-ead7-384c-623cc6f4ae61}]" + testing.go:149: server-26b9b025-eced-2e04-4877-6a2e92ef64a1: 2024-06-12T04:32:30.958+1400 [INFO] server-26b9b025-eced-2e04-4877-6a2e92ef64a1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-26b9b025-eced-2e04-4877-6a2e92ef64a1 Address:26b9b025-eced-2e04-4877-6a2e92ef64a1} {Suffrage:Voter ID:server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328 Address:a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328} {Suffrage:Voter ID:server-3e3b0020-764a-ead7-384c-623cc6f4ae61 Address:3e3b0020-764a-ead7-384c-623cc6f4ae61}]" +testLoggerAdapter verbose: server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: 2024-06-12T04:32:30.958+1400 [INFO] server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-26b9b025-eced-2e04-4877-6a2e92ef64a1 Address:26b9b025-eced-2e04-4877-6a2e92ef64a1} {Suffrage:Voter ID:server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328 Address:a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328} {Suffrage:Voter ID:server-3e3b0020-764a-ead7-384c-623cc6f4ae61 Address:3e3b0020-764a-ead7-384c-623cc6f4ae61}]" + testing.go:149: server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: 2024-06-12T04:32:30.958+1400 [INFO] server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-26b9b025-eced-2e04-4877-6a2e92ef64a1 Address:26b9b025-eced-2e04-4877-6a2e92ef64a1} {Suffrage:Voter ID:server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328 Address:a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328} {Suffrage:Voter ID:server-3e3b0020-764a-ead7-384c-623cc6f4ae61 Address:3e3b0020-764a-ead7-384c-623cc6f4ae61}]" +testLoggerAdapter verbose: server-3e3b0020-764a-ead7-384c-623cc6f4ae61: 2024-06-12T04:32:30.958+1400 [INFO] server-3e3b0020-764a-ead7-384c-623cc6f4ae61: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-26b9b025-eced-2e04-4877-6a2e92ef64a1 Address:26b9b025-eced-2e04-4877-6a2e92ef64a1} {Suffrage:Voter ID:server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328 Address:a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328} {Suffrage:Voter ID:server-3e3b0020-764a-ead7-384c-623cc6f4ae61 Address:3e3b0020-764a-ead7-384c-623cc6f4ae61}]" + testing.go:149: server-3e3b0020-764a-ead7-384c-623cc6f4ae61: 2024-06-12T04:32:30.958+1400 [INFO] server-3e3b0020-764a-ead7-384c-623cc6f4ae61: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-26b9b025-eced-2e04-4877-6a2e92ef64a1 Address:26b9b025-eced-2e04-4877-6a2e92ef64a1} {Suffrage:Voter ID:server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328 Address:a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328} {Suffrage:Voter ID:server-3e3b0020-764a-ead7-384c-623cc6f4ae61 Address:3e3b0020-764a-ead7-384c-623cc6f4ae61}]" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:30.958+1400 [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-06-12T04:32:30.958+1400 [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-26b9b025-eced-2e04-4877-6a2e92ef64a1: 2024-06-12T04:32:30.958+1400 [INFO] server-26b9b025-eced-2e04-4877-6a2e92ef64a1: entering follower state: follower="Node at 26b9b025-eced-2e04-4877-6a2e92ef64a1 [Follower]" leader= + testing.go:149: server-26b9b025-eced-2e04-4877-6a2e92ef64a1: 2024-06-12T04:32:30.958+1400 [INFO] server-26b9b025-eced-2e04-4877-6a2e92ef64a1: entering follower state: follower="Node at 26b9b025-eced-2e04-4877-6a2e92ef64a1 [Follower]" leader= +testLoggerAdapter verbose: server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: 2024-06-12T04:32:30.959+1400 [INFO] server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: entering follower state: follower="Node at a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328 [Follower]" leader= + testing.go:149: server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: 2024-06-12T04:32:30.959+1400 [INFO] server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: entering follower state: follower="Node at a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328 [Follower]" leader= +testLoggerAdapter verbose: server-3e3b0020-764a-ead7-384c-623cc6f4ae61: 2024-06-12T04:32:30.959+1400 [INFO] server-3e3b0020-764a-ead7-384c-623cc6f4ae61: entering follower state: follower="Node at 3e3b0020-764a-ead7-384c-623cc6f4ae61 [Follower]" leader= + testing.go:149: server-3e3b0020-764a-ead7-384c-623cc6f4ae61: 2024-06-12T04:32:30.959+1400 [INFO] server-3e3b0020-764a-ead7-384c-623cc6f4ae61: entering follower state: follower="Node at 3e3b0020-764a-ead7-384c-623cc6f4ae61 [Follower]" leader= +testLoggerAdapter verbose: server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: 2024-06-12T04:32:31.015+1400 [WARN] server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: 2024-06-12T04:32:31.015+1400 [WARN] server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: 2024-06-12T04:32:31.015+1400 [INFO] server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: entering candidate state: node="Node at a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328 [Candidate]" term=2 + testing.go:149: server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: 2024-06-12T04:32:31.015+1400 [INFO] server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: entering candidate state: node="Node at a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328 [Candidate]" term=2 +testLoggerAdapter verbose: server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: 2024-06-12T04:32:31.015+1400 [INFO] server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: election won: tally=2 + testing.go:149: server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: 2024-06-12T04:32:31.015+1400 [INFO] server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: election won: tally=2 +testLoggerAdapter verbose: server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: 2024-06-12T04:32:31.015+1400 [INFO] server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: entering leader state: leader="Node at a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328 [Leader]" + testing.go:149: server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: 2024-06-12T04:32:31.015+1400 [INFO] server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: entering leader state: leader="Node at a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328 [Leader]" +testLoggerAdapter verbose: server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: 2024-06-12T04:32:31.015+1400 [INFO] server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: added peer, starting replication: peer=server-26b9b025-eced-2e04-4877-6a2e92ef64a1 + testing.go:149: server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: 2024-06-12T04:32:31.015+1400 [INFO] server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: added peer, starting replication: peer=server-26b9b025-eced-2e04-4877-6a2e92ef64a1 +testLoggerAdapter verbose: server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: 2024-06-12T04:32:31.015+1400 [INFO] server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: added peer, starting replication: peer=server-3e3b0020-764a-ead7-384c-623cc6f4ae61 + testing.go:149: server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: 2024-06-12T04:32:31.015+1400 [INFO] server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: added peer, starting replication: peer=server-3e3b0020-764a-ead7-384c-623cc6f4ae61 +testLoggerAdapter verbose: server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: 2024-06-12T04:32:31.015+1400 [INFO] server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: pipelining replication: peer="{Voter server-3e3b0020-764a-ead7-384c-623cc6f4ae61 3e3b0020-764a-ead7-384c-623cc6f4ae61}" +testLoggerAdapter verbose: server-26b9b025-eced-2e04-4877-6a2e92ef64a1: 2024-06-12T04:32:31.015+1400 [WARN] server-26b9b025-eced-2e04-4877-6a2e92ef64a1: rejecting vote request since our last term is greater: candidate=a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328 last-term=2 last-candidate-term=1 + testing.go:149: server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: 2024-06-12T04:32:31.015+1400 [INFO] server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: pipelining replication: peer="{Voter server-3e3b0020-764a-ead7-384c-623cc6f4ae61 3e3b0020-764a-ead7-384c-623cc6f4ae61}" + testing.go:149: server-26b9b025-eced-2e04-4877-6a2e92ef64a1: 2024-06-12T04:32:31.015+1400 [WARN] server-26b9b025-eced-2e04-4877-6a2e92ef64a1: rejecting vote request since our last term is greater: candidate=a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328 last-term=2 last-candidate-term=1 +testLoggerAdapter verbose: server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: 2024-06-12T04:32:31.015+1400 [INFO] server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: pipelining replication: peer="{Voter server-26b9b025-eced-2e04-4877-6a2e92ef64a1 26b9b025-eced-2e04-4877-6a2e92ef64a1}" + testing.go:149: server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: 2024-06-12T04:32:31.015+1400 [INFO] server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: pipelining replication: peer="{Voter server-26b9b025-eced-2e04-4877-6a2e92ef64a1 26b9b025-eced-2e04-4877-6a2e92ef64a1}" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:31.120+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:31.015898454 +1400 +14 m=+7.923417033 (1 nodes), 56.934721ms from start of poll, 57.752372ms from cluster start. Timeout at 2024-06-12 04:32:31.120914012 +1400 +14 m=+8.028432591, 105.015558ms after stability + testing.go:149: cluster: 2024-06-12T04:32:31.120+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:31.015898454 +1400 +14 m=+7.923417033 (1 nodes), 56.934721ms from start of poll, 57.752372ms from cluster start. Timeout at 2024-06-12 04:32:31.120914012 +1400 +14 m=+8.028432591, 105.015558ms after stability +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:31.127+1400 [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-06-12T04:32:31.127+1400 [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:31.232+1400 [INFO] stable state for Follower reached at 2024-06-12 04:32:31.127170708 +1400 +14 m=+8.034689287 (2 nodes), 1.96µs from start of poll, 169.024626ms from cluster start. Timeout at 2024-06-12 04:32:31.232233305 +1400 +14 m=+8.139751884, 105.062597ms after stability + testing.go:149: cluster: 2024-06-12T04:32:31.232+1400 [INFO] stable state for Follower reached at 2024-06-12 04:32:31.127170708 +1400 +14 m=+8.034689287 (2 nodes), 1.96µs from start of poll, 169.024626ms from cluster start. Timeout at 2024-06-12 04:32:31.232233305 +1400 +14 m=+8.139751884, 105.062597ms after stability +testLoggerAdapter verbose: server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: 2024-06-12T04:32:31.232+1400 [INFO] server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: updating configuration: command=RemoveServer server-id=server-26b9b025-eced-2e04-4877-6a2e92ef64a1 server-addr= servers="[{Suffrage:Voter ID:server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328 Address:a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328} {Suffrage:Voter ID:server-3e3b0020-764a-ead7-384c-623cc6f4ae61 Address:3e3b0020-764a-ead7-384c-623cc6f4ae61}]" + testing.go:149: server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: 2024-06-12T04:32:31.232+1400 [INFO] server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: updating configuration: command=RemoveServer server-id=server-26b9b025-eced-2e04-4877-6a2e92ef64a1 server-addr= servers="[{Suffrage:Voter ID:server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328 Address:a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328} {Suffrage:Voter ID:server-3e3b0020-764a-ead7-384c-623cc6f4ae61 Address:3e3b0020-764a-ead7-384c-623cc6f4ae61}]" +testLoggerAdapter verbose: server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: 2024-06-12T04:32:31.232+1400 [INFO] server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: removed peer, stopping replication: peer=server-26b9b025-eced-2e04-4877-6a2e92ef64a1 last-index=3 + testing.go:149: server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: 2024-06-12T04:32:31.232+1400 [INFO] server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: removed peer, stopping replication: peer=server-26b9b025-eced-2e04-4877-6a2e92ef64a1 last-index=3 +testLoggerAdapter verbose: server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: 2024-06-12T04:32:31.232+1400 [INFO] server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: aborting pipeline replication: peer="{Voter server-26b9b025-eced-2e04-4877-6a2e92ef64a1 26b9b025-eced-2e04-4877-6a2e92ef64a1}" + testing.go:149: server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: 2024-06-12T04:32:31.232+1400 [INFO] server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: aborting pipeline replication: peer="{Voter server-26b9b025-eced-2e04-4877-6a2e92ef64a1 26b9b025-eced-2e04-4877-6a2e92ef64a1}" +testLoggerAdapter verbose: server-26b9b025-eced-2e04-4877-6a2e92ef64a1: 2024-06-12T04:32:31.328+1400 [WARN] server-26b9b025-eced-2e04-4877-6a2e92ef64a1: heartbeat timeout reached, starting election: last-leader=a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328 + testing.go:149: server-26b9b025-eced-2e04-4877-6a2e92ef64a1: 2024-06-12T04:32:31.328+1400 [WARN] server-26b9b025-eced-2e04-4877-6a2e92ef64a1: heartbeat timeout reached, starting election: last-leader=a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328 +testLoggerAdapter verbose: server-26b9b025-eced-2e04-4877-6a2e92ef64a1: 2024-06-12T04:32:31.328+1400 [INFO] server-26b9b025-eced-2e04-4877-6a2e92ef64a1: entering candidate state: node="Node at 26b9b025-eced-2e04-4877-6a2e92ef64a1 [Candidate]" term=3 + testing.go:149: server-26b9b025-eced-2e04-4877-6a2e92ef64a1: 2024-06-12T04:32:31.328+1400 [INFO] server-26b9b025-eced-2e04-4877-6a2e92ef64a1: entering candidate state: node="Node at 26b9b025-eced-2e04-4877-6a2e92ef64a1 [Candidate]" term=3 +testLoggerAdapter verbose: server-3e3b0020-764a-ead7-384c-623cc6f4ae61: 2024-06-12T04:32:31.328+1400 [WARN] server-3e3b0020-764a-ead7-384c-623cc6f4ae61: rejecting vote request since we have a leader: from=26b9b025-eced-2e04-4877-6a2e92ef64a1 leader=a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328 + testing.go:149: server-3e3b0020-764a-ead7-384c-623cc6f4ae61: 2024-06-12T04:32:31.328+1400 [WARN] server-3e3b0020-764a-ead7-384c-623cc6f4ae61: rejecting vote request since we have a leader: from=26b9b025-eced-2e04-4877-6a2e92ef64a1 leader=a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328 +testLoggerAdapter verbose: server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: 2024-06-12T04:32:31.328+1400 [WARN] server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: rejecting vote request since we have a leader: from=26b9b025-eced-2e04-4877-6a2e92ef64a1 leader=a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328 + testing.go:149: server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: 2024-06-12T04:32:31.328+1400 [WARN] server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: rejecting vote request since we have a leader: from=26b9b025-eced-2e04-4877-6a2e92ef64a1 leader=a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328 +testLoggerAdapter verbose: server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: 2024-06-12T04:32:31.338+1400 [INFO] server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: aborting pipeline replication: peer="{Voter server-3e3b0020-764a-ead7-384c-623cc6f4ae61 3e3b0020-764a-ead7-384c-623cc6f4ae61}" + testing.go:149: server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: 2024-06-12T04:32:31.338+1400 [INFO] server-a12bf9da-cfb7-9a5b-bb07-29b5d5fe7328: aborting pipeline replication: peer="{Voter server-3e3b0020-764a-ead7-384c-623cc6f4ae61 3e3b0020-764a-ead7-384c-623cc6f4ae61}" +--- PASS: TestRaft_RemoveFollower (0.38s) === RUN TestRaft_RemoveLeader -testLoggerAdapter verbose: server-0b9658b2-60dc-4ce8-f441-865e608f1983: 2024-06-11T02:29:17.981-1200 [INFO] server-0b9658b2-60dc-4ce8-f441-865e608f1983: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0b9658b2-60dc-4ce8-f441-865e608f1983 Address:0b9658b2-60dc-4ce8-f441-865e608f1983} {Suffrage:Voter ID:server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60 Address:95dc7e3b-f5fd-6dd7-0c51-2a36579eed60} {Suffrage:Voter ID:server-4293ed08-f49d-224a-fd9a-bfa7cbfae180 Address:4293ed08-f49d-224a-fd9a-bfa7cbfae180}]" - testing.go:149: server-0b9658b2-60dc-4ce8-f441-865e608f1983: 2024-06-11T02:29:17.981-1200 [INFO] server-0b9658b2-60dc-4ce8-f441-865e608f1983: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0b9658b2-60dc-4ce8-f441-865e608f1983 Address:0b9658b2-60dc-4ce8-f441-865e608f1983} {Suffrage:Voter ID:server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60 Address:95dc7e3b-f5fd-6dd7-0c51-2a36579eed60} {Suffrage:Voter ID:server-4293ed08-f49d-224a-fd9a-bfa7cbfae180 Address:4293ed08-f49d-224a-fd9a-bfa7cbfae180}]" -testLoggerAdapter verbose: server-0b9658b2-60dc-4ce8-f441-865e608f1983: 2024-06-11T02:29:17.981-1200 [INFO] server-0b9658b2-60dc-4ce8-f441-865e608f1983: entering follower state: follower="Node at 0b9658b2-60dc-4ce8-f441-865e608f1983 [Follower]" leader= - testing.go:149: server-0b9658b2-60dc-4ce8-f441-865e608f1983: 2024-06-11T02:29:17.981-1200 [INFO] server-0b9658b2-60dc-4ce8-f441-865e608f1983: entering follower state: follower="Node at 0b9658b2-60dc-4ce8-f441-865e608f1983 [Follower]" leader= -testLoggerAdapter verbose: server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: 2024-06-11T02:29:17.981-1200 [INFO] server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0b9658b2-60dc-4ce8-f441-865e608f1983 Address:0b9658b2-60dc-4ce8-f441-865e608f1983} {Suffrage:Voter ID:server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60 Address:95dc7e3b-f5fd-6dd7-0c51-2a36579eed60} {Suffrage:Voter ID:server-4293ed08-f49d-224a-fd9a-bfa7cbfae180 Address:4293ed08-f49d-224a-fd9a-bfa7cbfae180}]" - testing.go:149: server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: 2024-06-11T02:29:17.981-1200 [INFO] server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0b9658b2-60dc-4ce8-f441-865e608f1983 Address:0b9658b2-60dc-4ce8-f441-865e608f1983} {Suffrage:Voter ID:server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60 Address:95dc7e3b-f5fd-6dd7-0c51-2a36579eed60} {Suffrage:Voter ID:server-4293ed08-f49d-224a-fd9a-bfa7cbfae180 Address:4293ed08-f49d-224a-fd9a-bfa7cbfae180}]" -testLoggerAdapter verbose: server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: 2024-06-11T02:29:17.982-1200 [INFO] server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: entering follower state: follower="Node at 95dc7e3b-f5fd-6dd7-0c51-2a36579eed60 [Follower]" leader= - testing.go:149: server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: 2024-06-11T02:29:17.982-1200 [INFO] server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: entering follower state: follower="Node at 95dc7e3b-f5fd-6dd7-0c51-2a36579eed60 [Follower]" leader= -testLoggerAdapter verbose: server-4293ed08-f49d-224a-fd9a-bfa7cbfae180: 2024-06-11T02:29:17.982-1200 [INFO] server-4293ed08-f49d-224a-fd9a-bfa7cbfae180: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0b9658b2-60dc-4ce8-f441-865e608f1983 Address:0b9658b2-60dc-4ce8-f441-865e608f1983} {Suffrage:Voter ID:server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60 Address:95dc7e3b-f5fd-6dd7-0c51-2a36579eed60} {Suffrage:Voter ID:server-4293ed08-f49d-224a-fd9a-bfa7cbfae180 Address:4293ed08-f49d-224a-fd9a-bfa7cbfae180}]" - testing.go:149: server-4293ed08-f49d-224a-fd9a-bfa7cbfae180: 2024-06-11T02:29:17.982-1200 [INFO] server-4293ed08-f49d-224a-fd9a-bfa7cbfae180: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0b9658b2-60dc-4ce8-f441-865e608f1983 Address:0b9658b2-60dc-4ce8-f441-865e608f1983} {Suffrage:Voter ID:server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60 Address:95dc7e3b-f5fd-6dd7-0c51-2a36579eed60} {Suffrage:Voter ID:server-4293ed08-f49d-224a-fd9a-bfa7cbfae180 Address:4293ed08-f49d-224a-fd9a-bfa7cbfae180}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:17.982-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:17.982-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-4293ed08-f49d-224a-fd9a-bfa7cbfae180: 2024-06-11T02:29:17.982-1200 [INFO] server-4293ed08-f49d-224a-fd9a-bfa7cbfae180: entering follower state: follower="Node at 4293ed08-f49d-224a-fd9a-bfa7cbfae180 [Follower]" leader= - testing.go:149: server-4293ed08-f49d-224a-fd9a-bfa7cbfae180: 2024-06-11T02:29:17.982-1200 [INFO] server-4293ed08-f49d-224a-fd9a-bfa7cbfae180: entering follower state: follower="Node at 4293ed08-f49d-224a-fd9a-bfa7cbfae180 [Follower]" leader= -testLoggerAdapter verbose: server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: 2024-06-11T02:29:18.039-1200 [WARN] server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: 2024-06-11T02:29:18.039-1200 [WARN] server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: 2024-06-11T02:29:18.039-1200 [INFO] server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: entering candidate state: node="Node at 95dc7e3b-f5fd-6dd7-0c51-2a36579eed60 [Candidate]" term=2 - testing.go:149: server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: 2024-06-11T02:29:18.039-1200 [INFO] server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: entering candidate state: node="Node at 95dc7e3b-f5fd-6dd7-0c51-2a36579eed60 [Candidate]" term=2 -testLoggerAdapter verbose: server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: 2024-06-11T02:29:18.039-1200 [INFO] server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: election won: tally=2 - testing.go:149: server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: 2024-06-11T02:29:18.039-1200 [INFO] server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: election won: tally=2 -testLoggerAdapter verbose: server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: 2024-06-11T02:29:18.040-1200 [INFO] server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: entering leader state: leader="Node at 95dc7e3b-f5fd-6dd7-0c51-2a36579eed60 [Leader]" - testing.go:149: server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: 2024-06-11T02:29:18.040-1200 [INFO] server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: entering leader state: leader="Node at 95dc7e3b-f5fd-6dd7-0c51-2a36579eed60 [Leader]" -testLoggerAdapter verbose: server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: 2024-06-11T02:29:18.040-1200 [INFO] server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: added peer, starting replication: peer=server-0b9658b2-60dc-4ce8-f441-865e608f1983 - testing.go:149: server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: 2024-06-11T02:29:18.040-1200 [INFO] server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: added peer, starting replication: peer=server-0b9658b2-60dc-4ce8-f441-865e608f1983 -testLoggerAdapter verbose: server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: 2024-06-11T02:29:18.040-1200 [INFO] server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: added peer, starting replication: peer=server-4293ed08-f49d-224a-fd9a-bfa7cbfae180 - testing.go:149: server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: 2024-06-11T02:29:18.040-1200 [INFO] server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: added peer, starting replication: peer=server-4293ed08-f49d-224a-fd9a-bfa7cbfae180 -testLoggerAdapter verbose: server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: 2024-06-11T02:29:18.040-1200 [INFO] server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: pipelining replication: peer="{Voter server-0b9658b2-60dc-4ce8-f441-865e608f1983 0b9658b2-60dc-4ce8-f441-865e608f1983}" - testing.go:149: server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: 2024-06-11T02:29:18.040-1200 [INFO] server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: pipelining replication: peer="{Voter server-0b9658b2-60dc-4ce8-f441-865e608f1983 0b9658b2-60dc-4ce8-f441-865e608f1983}" -testLoggerAdapter verbose: server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: 2024-06-11T02:29:18.040-1200 [INFO] server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: pipelining replication: peer="{Voter server-4293ed08-f49d-224a-fd9a-bfa7cbfae180 4293ed08-f49d-224a-fd9a-bfa7cbfae180}" - testing.go:149: server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: 2024-06-11T02:29:18.040-1200 [INFO] server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: pipelining replication: peer="{Voter server-4293ed08-f49d-224a-fd9a-bfa7cbfae180 4293ed08-f49d-224a-fd9a-bfa7cbfae180}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:18.146-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:18.040129522 -1200 -12 m=+9.035502355 (1 nodes), 57.452877ms from start of poll, 59.030181ms from cluster start. Timeout at 2024-06-11 02:29:18.146073112 -1200 -12 m=+9.141445946, 105.943591ms after stability - testing.go:149: cluster: 2024-06-11T02:29:18.146-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:18.040129522 -1200 -12 m=+9.035502355 (1 nodes), 57.452877ms from start of poll, 59.030181ms from cluster start. Timeout at 2024-06-11 02:29:18.146073112 -1200 -12 m=+9.141445946, 105.943591ms after stability -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:18.152-1200 [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2024-06-11T02:29:18.152-1200 [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:18.258-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:18.152346744 -1200 -12 m=+9.147719286 (2 nodes), 3.792µs from start of poll, 171.247112ms from cluster start. Timeout at 2024-06-11 02:29:18.257976216 -1200 -12 m=+9.253349049, 105.629763ms after stability - testing.go:149: cluster: 2024-06-11T02:29:18.258-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:18.152346744 -1200 -12 m=+9.147719286 (2 nodes), 3.792µs from start of poll, 171.247112ms from cluster start. Timeout at 2024-06-11 02:29:18.257976216 -1200 -12 m=+9.253349049, 105.629763ms after stability -testLoggerAdapter verbose: server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: 2024-06-11T02:29:18.258-1200 [INFO] server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: updating configuration: command=RemoveServer server-id=server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60 server-addr= servers="[{Suffrage:Voter ID:server-0b9658b2-60dc-4ce8-f441-865e608f1983 Address:0b9658b2-60dc-4ce8-f441-865e608f1983} {Suffrage:Voter ID:server-4293ed08-f49d-224a-fd9a-bfa7cbfae180 Address:4293ed08-f49d-224a-fd9a-bfa7cbfae180}]" - testing.go:149: server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: 2024-06-11T02:29:18.258-1200 [INFO] server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: updating configuration: command=RemoveServer server-id=server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60 server-addr= servers="[{Suffrage:Voter ID:server-0b9658b2-60dc-4ce8-f441-865e608f1983 Address:0b9658b2-60dc-4ce8-f441-865e608f1983} {Suffrage:Voter ID:server-4293ed08-f49d-224a-fd9a-bfa7cbfae180 Address:4293ed08-f49d-224a-fd9a-bfa7cbfae180}]" -testLoggerAdapter verbose: server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: 2024-06-11T02:29:18.258-1200 [INFO] server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: removed ourself, shutting down - testing.go:149: server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: 2024-06-11T02:29:18.258-1200 [INFO] server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: removed ourself, shutting down -testLoggerAdapter verbose: server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: 2024-06-11T02:29:18.258-1200 [INFO] server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: aborting pipeline replication: peer="{Voter server-4293ed08-f49d-224a-fd9a-bfa7cbfae180 4293ed08-f49d-224a-fd9a-bfa7cbfae180}" - testing.go:149: server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: 2024-06-11T02:29:18.258-1200 [INFO] server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: aborting pipeline replication: peer="{Voter server-4293ed08-f49d-224a-fd9a-bfa7cbfae180 4293ed08-f49d-224a-fd9a-bfa7cbfae180}" -testLoggerAdapter verbose: server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: 2024-06-11T02:29:18.259-1200 [INFO] server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: aborting pipeline replication: peer="{Voter server-0b9658b2-60dc-4ce8-f441-865e608f1983 0b9658b2-60dc-4ce8-f441-865e608f1983}" - testing.go:149: server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: 2024-06-11T02:29:18.259-1200 [INFO] server-95dc7e3b-f5fd-6dd7-0c51-2a36579eed60: aborting pipeline replication: peer="{Voter server-0b9658b2-60dc-4ce8-f441-865e608f1983 0b9658b2-60dc-4ce8-f441-865e608f1983}" -testLoggerAdapter verbose: server-0b9658b2-60dc-4ce8-f441-865e608f1983: 2024-06-11T02:29:18.320-1200 [WARN] server-0b9658b2-60dc-4ce8-f441-865e608f1983: heartbeat timeout reached, starting election: last-leader=95dc7e3b-f5fd-6dd7-0c51-2a36579eed60 - testing.go:149: server-0b9658b2-60dc-4ce8-f441-865e608f1983: 2024-06-11T02:29:18.320-1200 [WARN] server-0b9658b2-60dc-4ce8-f441-865e608f1983: heartbeat timeout reached, starting election: last-leader=95dc7e3b-f5fd-6dd7-0c51-2a36579eed60 -testLoggerAdapter verbose: server-0b9658b2-60dc-4ce8-f441-865e608f1983: 2024-06-11T02:29:18.320-1200 [INFO] server-0b9658b2-60dc-4ce8-f441-865e608f1983: entering candidate state: node="Node at 0b9658b2-60dc-4ce8-f441-865e608f1983 [Candidate]" term=3 - testing.go:149: server-0b9658b2-60dc-4ce8-f441-865e608f1983: 2024-06-11T02:29:18.320-1200 [INFO] server-0b9658b2-60dc-4ce8-f441-865e608f1983: entering candidate state: node="Node at 0b9658b2-60dc-4ce8-f441-865e608f1983 [Candidate]" term=3 -testLoggerAdapter verbose: server-4293ed08-f49d-224a-fd9a-bfa7cbfae180: 2024-06-11T02:29:18.320-1200 [WARN] server-4293ed08-f49d-224a-fd9a-bfa7cbfae180: rejecting vote request since we have a leader: from=0b9658b2-60dc-4ce8-f441-865e608f1983 leader=95dc7e3b-f5fd-6dd7-0c51-2a36579eed60 - testing.go:149: server-4293ed08-f49d-224a-fd9a-bfa7cbfae180: 2024-06-11T02:29:18.320-1200 [WARN] server-4293ed08-f49d-224a-fd9a-bfa7cbfae180: rejecting vote request since we have a leader: from=0b9658b2-60dc-4ce8-f441-865e608f1983 leader=95dc7e3b-f5fd-6dd7-0c51-2a36579eed60 -testLoggerAdapter verbose: server-4293ed08-f49d-224a-fd9a-bfa7cbfae180: 2024-06-11T02:29:18.352-1200 [WARN] server-4293ed08-f49d-224a-fd9a-bfa7cbfae180: heartbeat timeout reached, starting election: last-leader=95dc7e3b-f5fd-6dd7-0c51-2a36579eed60 - testing.go:149: server-4293ed08-f49d-224a-fd9a-bfa7cbfae180: 2024-06-11T02:29:18.352-1200 [WARN] server-4293ed08-f49d-224a-fd9a-bfa7cbfae180: heartbeat timeout reached, starting election: last-leader=95dc7e3b-f5fd-6dd7-0c51-2a36579eed60 -testLoggerAdapter verbose: server-4293ed08-f49d-224a-fd9a-bfa7cbfae180: 2024-06-11T02:29:18.352-1200 [INFO] server-4293ed08-f49d-224a-fd9a-bfa7cbfae180: entering candidate state: node="Node at 4293ed08-f49d-224a-fd9a-bfa7cbfae180 [Candidate]" term=3 - testing.go:149: server-4293ed08-f49d-224a-fd9a-bfa7cbfae180: 2024-06-11T02:29:18.352-1200 [INFO] server-4293ed08-f49d-224a-fd9a-bfa7cbfae180: entering candidate state: node="Node at 4293ed08-f49d-224a-fd9a-bfa7cbfae180 [Candidate]" term=3 -testLoggerAdapter verbose: server-0b9658b2-60dc-4ce8-f441-865e608f1983: 2024-06-11T02:29:18.352-1200 [INFO] server-0b9658b2-60dc-4ce8-f441-865e608f1983: duplicate requestVote for same term: term=3 - testing.go:149: server-0b9658b2-60dc-4ce8-f441-865e608f1983: 2024-06-11T02:29:18.352-1200 [INFO] server-0b9658b2-60dc-4ce8-f441-865e608f1983: duplicate requestVote for same term: term=3 -testLoggerAdapter verbose: server-0b9658b2-60dc-4ce8-f441-865e608f1983: 2024-06-11T02:29:18.375-1200 [WARN] server-0b9658b2-60dc-4ce8-f441-865e608f1983: Election timeout reached, restarting election - testing.go:149: server-0b9658b2-60dc-4ce8-f441-865e608f1983: 2024-06-11T02:29:18.375-1200 [WARN] server-0b9658b2-60dc-4ce8-f441-865e608f1983: Election timeout reached, restarting election -testLoggerAdapter verbose: server-0b9658b2-60dc-4ce8-f441-865e608f1983: 2024-06-11T02:29:18.375-1200 [INFO] server-0b9658b2-60dc-4ce8-f441-865e608f1983: entering candidate state: node="Node at 0b9658b2-60dc-4ce8-f441-865e608f1983 [Candidate]" term=4 - testing.go:149: server-0b9658b2-60dc-4ce8-f441-865e608f1983: 2024-06-11T02:29:18.375-1200 [INFO] server-0b9658b2-60dc-4ce8-f441-865e608f1983: entering candidate state: node="Node at 0b9658b2-60dc-4ce8-f441-865e608f1983 [Candidate]" term=4 -testLoggerAdapter verbose: server-4293ed08-f49d-224a-fd9a-bfa7cbfae180: 2024-06-11T02:29:18.375-1200 [INFO] server-4293ed08-f49d-224a-fd9a-bfa7cbfae180: entering follower state: follower="Node at 4293ed08-f49d-224a-fd9a-bfa7cbfae180 [Follower]" leader= - testing.go:149: server-4293ed08-f49d-224a-fd9a-bfa7cbfae180: 2024-06-11T02:29:18.375-1200 [INFO] server-4293ed08-f49d-224a-fd9a-bfa7cbfae180: entering follower state: follower="Node at 4293ed08-f49d-224a-fd9a-bfa7cbfae180 [Follower]" leader= -testLoggerAdapter verbose: server-0b9658b2-60dc-4ce8-f441-865e608f1983: 2024-06-11T02:29:18.375-1200 [INFO] server-0b9658b2-60dc-4ce8-f441-865e608f1983: election won: tally=2 - testing.go:149: server-0b9658b2-60dc-4ce8-f441-865e608f1983: 2024-06-11T02:29:18.375-1200 [INFO] server-0b9658b2-60dc-4ce8-f441-865e608f1983: election won: tally=2 -testLoggerAdapter verbose: server-0b9658b2-60dc-4ce8-f441-865e608f1983: 2024-06-11T02:29:18.375-1200 [INFO] server-0b9658b2-60dc-4ce8-f441-865e608f1983: entering leader state: leader="Node at 0b9658b2-60dc-4ce8-f441-865e608f1983 [Leader]" - testing.go:149: server-0b9658b2-60dc-4ce8-f441-865e608f1983: 2024-06-11T02:29:18.375-1200 [INFO] server-0b9658b2-60dc-4ce8-f441-865e608f1983: entering leader state: leader="Node at 0b9658b2-60dc-4ce8-f441-865e608f1983 [Leader]" -testLoggerAdapter verbose: server-0b9658b2-60dc-4ce8-f441-865e608f1983: 2024-06-11T02:29:18.375-1200 [INFO] server-0b9658b2-60dc-4ce8-f441-865e608f1983: added peer, starting replication: peer=server-4293ed08-f49d-224a-fd9a-bfa7cbfae180 - testing.go:149: server-0b9658b2-60dc-4ce8-f441-865e608f1983: 2024-06-11T02:29:18.375-1200 [INFO] server-0b9658b2-60dc-4ce8-f441-865e608f1983: added peer, starting replication: peer=server-4293ed08-f49d-224a-fd9a-bfa7cbfae180 -testLoggerAdapter verbose: server-0b9658b2-60dc-4ce8-f441-865e608f1983: 2024-06-11T02:29:18.376-1200 [INFO] server-0b9658b2-60dc-4ce8-f441-865e608f1983: pipelining replication: peer="{Voter server-4293ed08-f49d-224a-fd9a-bfa7cbfae180 4293ed08-f49d-224a-fd9a-bfa7cbfae180}" - testing.go:149: server-0b9658b2-60dc-4ce8-f441-865e608f1983: 2024-06-11T02:29:18.376-1200 [INFO] server-0b9658b2-60dc-4ce8-f441-865e608f1983: pipelining replication: peer="{Voter server-4293ed08-f49d-224a-fd9a-bfa7cbfae180 4293ed08-f49d-224a-fd9a-bfa7cbfae180}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:18.470-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:18.470-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:18.576-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:18.470464218 -1200 -12 m=+9.465836760 (1 nodes), 106.748µs from start of poll, 489.364586ms from cluster start. Timeout at 2024-06-11 02:29:18.576311852 -1200 -12 m=+9.571684394, 105.847634ms after stability - testing.go:149: cluster: 2024-06-11T02:29:18.576-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:18.470464218 -1200 -12 m=+9.465836760 (1 nodes), 106.748µs from start of poll, 489.364586ms from cluster start. Timeout at 2024-06-11 02:29:18.576311852 -1200 -12 m=+9.571684394, 105.847634ms after stability -testLoggerAdapter verbose: server-0b9658b2-60dc-4ce8-f441-865e608f1983: 2024-06-11T02:29:18.576-1200 [INFO] server-0b9658b2-60dc-4ce8-f441-865e608f1983: aborting pipeline replication: peer="{Voter server-4293ed08-f49d-224a-fd9a-bfa7cbfae180 4293ed08-f49d-224a-fd9a-bfa7cbfae180}" - testing.go:149: server-0b9658b2-60dc-4ce8-f441-865e608f1983: 2024-06-11T02:29:18.576-1200 [INFO] server-0b9658b2-60dc-4ce8-f441-865e608f1983: aborting pipeline replication: peer="{Voter server-4293ed08-f49d-224a-fd9a-bfa7cbfae180 4293ed08-f49d-224a-fd9a-bfa7cbfae180}" +testLoggerAdapter verbose: server-22894afa-22a3-fb46-6be6-70213ddabb61: 2024-06-12T04:32:31.341+1400 [INFO] server-22894afa-22a3-fb46-6be6-70213ddabb61: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-22894afa-22a3-fb46-6be6-70213ddabb61 Address:22894afa-22a3-fb46-6be6-70213ddabb61} {Suffrage:Voter ID:server-063c306d-3b25-9ea2-cd1d-b631d74ac673 Address:063c306d-3b25-9ea2-cd1d-b631d74ac673} {Suffrage:Voter ID:server-35cc5789-32c0-509d-3472-243f4999d8a5 Address:35cc5789-32c0-509d-3472-243f4999d8a5}]" + testing.go:149: server-22894afa-22a3-fb46-6be6-70213ddabb61: 2024-06-12T04:32:31.341+1400 [INFO] server-22894afa-22a3-fb46-6be6-70213ddabb61: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-22894afa-22a3-fb46-6be6-70213ddabb61 Address:22894afa-22a3-fb46-6be6-70213ddabb61} {Suffrage:Voter ID:server-063c306d-3b25-9ea2-cd1d-b631d74ac673 Address:063c306d-3b25-9ea2-cd1d-b631d74ac673} {Suffrage:Voter ID:server-35cc5789-32c0-509d-3472-243f4999d8a5 Address:35cc5789-32c0-509d-3472-243f4999d8a5}]" +testLoggerAdapter verbose: server-063c306d-3b25-9ea2-cd1d-b631d74ac673: 2024-06-12T04:32:31.341+1400 [INFO] server-063c306d-3b25-9ea2-cd1d-b631d74ac673: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-22894afa-22a3-fb46-6be6-70213ddabb61 Address:22894afa-22a3-fb46-6be6-70213ddabb61} {Suffrage:Voter ID:server-063c306d-3b25-9ea2-cd1d-b631d74ac673 Address:063c306d-3b25-9ea2-cd1d-b631d74ac673} {Suffrage:Voter ID:server-35cc5789-32c0-509d-3472-243f4999d8a5 Address:35cc5789-32c0-509d-3472-243f4999d8a5}]" + testing.go:149: server-063c306d-3b25-9ea2-cd1d-b631d74ac673: 2024-06-12T04:32:31.341+1400 [INFO] server-063c306d-3b25-9ea2-cd1d-b631d74ac673: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-22894afa-22a3-fb46-6be6-70213ddabb61 Address:22894afa-22a3-fb46-6be6-70213ddabb61} {Suffrage:Voter ID:server-063c306d-3b25-9ea2-cd1d-b631d74ac673 Address:063c306d-3b25-9ea2-cd1d-b631d74ac673} {Suffrage:Voter ID:server-35cc5789-32c0-509d-3472-243f4999d8a5 Address:35cc5789-32c0-509d-3472-243f4999d8a5}]" +testLoggerAdapter verbose: server-35cc5789-32c0-509d-3472-243f4999d8a5: 2024-06-12T04:32:31.341+1400 [INFO] server-35cc5789-32c0-509d-3472-243f4999d8a5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-22894afa-22a3-fb46-6be6-70213ddabb61 Address:22894afa-22a3-fb46-6be6-70213ddabb61} {Suffrage:Voter ID:server-063c306d-3b25-9ea2-cd1d-b631d74ac673 Address:063c306d-3b25-9ea2-cd1d-b631d74ac673} {Suffrage:Voter ID:server-35cc5789-32c0-509d-3472-243f4999d8a5 Address:35cc5789-32c0-509d-3472-243f4999d8a5}]" + testing.go:149: server-35cc5789-32c0-509d-3472-243f4999d8a5: 2024-06-12T04:32:31.341+1400 [INFO] server-35cc5789-32c0-509d-3472-243f4999d8a5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-22894afa-22a3-fb46-6be6-70213ddabb61 Address:22894afa-22a3-fb46-6be6-70213ddabb61} {Suffrage:Voter ID:server-063c306d-3b25-9ea2-cd1d-b631d74ac673 Address:063c306d-3b25-9ea2-cd1d-b631d74ac673} {Suffrage:Voter ID:server-35cc5789-32c0-509d-3472-243f4999d8a5 Address:35cc5789-32c0-509d-3472-243f4999d8a5}]" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:31.341+1400 [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-06-12T04:32:31.341+1400 [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-22894afa-22a3-fb46-6be6-70213ddabb61: 2024-06-12T04:32:31.341+1400 [INFO] server-22894afa-22a3-fb46-6be6-70213ddabb61: entering follower state: follower="Node at 22894afa-22a3-fb46-6be6-70213ddabb61 [Follower]" leader= + testing.go:149: server-22894afa-22a3-fb46-6be6-70213ddabb61: 2024-06-12T04:32:31.341+1400 [INFO] server-22894afa-22a3-fb46-6be6-70213ddabb61: entering follower state: follower="Node at 22894afa-22a3-fb46-6be6-70213ddabb61 [Follower]" leader= +testLoggerAdapter verbose: server-063c306d-3b25-9ea2-cd1d-b631d74ac673: 2024-06-12T04:32:31.341+1400 [INFO] server-063c306d-3b25-9ea2-cd1d-b631d74ac673: entering follower state: follower="Node at 063c306d-3b25-9ea2-cd1d-b631d74ac673 [Follower]" leader= + testing.go:149: server-063c306d-3b25-9ea2-cd1d-b631d74ac673: 2024-06-12T04:32:31.341+1400 [INFO] server-063c306d-3b25-9ea2-cd1d-b631d74ac673: entering follower state: follower="Node at 063c306d-3b25-9ea2-cd1d-b631d74ac673 [Follower]" leader= +testLoggerAdapter verbose: server-35cc5789-32c0-509d-3472-243f4999d8a5: 2024-06-12T04:32:31.341+1400 [INFO] server-35cc5789-32c0-509d-3472-243f4999d8a5: entering follower state: follower="Node at 35cc5789-32c0-509d-3472-243f4999d8a5 [Follower]" leader= + testing.go:149: server-35cc5789-32c0-509d-3472-243f4999d8a5: 2024-06-12T04:32:31.341+1400 [INFO] server-35cc5789-32c0-509d-3472-243f4999d8a5: entering follower state: follower="Node at 35cc5789-32c0-509d-3472-243f4999d8a5 [Follower]" leader= +testLoggerAdapter verbose: server-063c306d-3b25-9ea2-cd1d-b631d74ac673: 2024-06-12T04:32:31.399+1400 [WARN] server-063c306d-3b25-9ea2-cd1d-b631d74ac673: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-063c306d-3b25-9ea2-cd1d-b631d74ac673: 2024-06-12T04:32:31.399+1400 [WARN] server-063c306d-3b25-9ea2-cd1d-b631d74ac673: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-063c306d-3b25-9ea2-cd1d-b631d74ac673: 2024-06-12T04:32:31.400+1400 [INFO] server-063c306d-3b25-9ea2-cd1d-b631d74ac673: entering candidate state: node="Node at 063c306d-3b25-9ea2-cd1d-b631d74ac673 [Candidate]" term=2 + testing.go:149: server-063c306d-3b25-9ea2-cd1d-b631d74ac673: 2024-06-12T04:32:31.400+1400 [INFO] server-063c306d-3b25-9ea2-cd1d-b631d74ac673: entering candidate state: node="Node at 063c306d-3b25-9ea2-cd1d-b631d74ac673 [Candidate]" term=2 +testLoggerAdapter verbose: server-063c306d-3b25-9ea2-cd1d-b631d74ac673: 2024-06-12T04:32:31.400+1400 [INFO] server-063c306d-3b25-9ea2-cd1d-b631d74ac673: election won: tally=2 + testing.go:149: server-063c306d-3b25-9ea2-cd1d-b631d74ac673: 2024-06-12T04:32:31.400+1400 [INFO] server-063c306d-3b25-9ea2-cd1d-b631d74ac673: election won: tally=2 +testLoggerAdapter verbose: server-063c306d-3b25-9ea2-cd1d-b631d74ac673: 2024-06-12T04:32:31.400+1400 [INFO] server-063c306d-3b25-9ea2-cd1d-b631d74ac673: entering leader state: leader="Node at 063c306d-3b25-9ea2-cd1d-b631d74ac673 [Leader]" + testing.go:149: server-063c306d-3b25-9ea2-cd1d-b631d74ac673: 2024-06-12T04:32:31.400+1400 [INFO] server-063c306d-3b25-9ea2-cd1d-b631d74ac673: entering leader state: leader="Node at 063c306d-3b25-9ea2-cd1d-b631d74ac673 [Leader]" +testLoggerAdapter verbose: server-063c306d-3b25-9ea2-cd1d-b631d74ac673: 2024-06-12T04:32:31.400+1400 [INFO] server-063c306d-3b25-9ea2-cd1d-b631d74ac673: added peer, starting replication: peer=server-22894afa-22a3-fb46-6be6-70213ddabb61 + testing.go:149: server-063c306d-3b25-9ea2-cd1d-b631d74ac673: 2024-06-12T04:32:31.400+1400 [INFO] server-063c306d-3b25-9ea2-cd1d-b631d74ac673: added peer, starting replication: peer=server-22894afa-22a3-fb46-6be6-70213ddabb61 +testLoggerAdapter verbose: server-063c306d-3b25-9ea2-cd1d-b631d74ac673: 2024-06-12T04:32:31.400+1400 [INFO] server-063c306d-3b25-9ea2-cd1d-b631d74ac673: added peer, starting replication: peer=server-35cc5789-32c0-509d-3472-243f4999d8a5 + testing.go:149: server-063c306d-3b25-9ea2-cd1d-b631d74ac673: 2024-06-12T04:32:31.400+1400 [INFO] server-063c306d-3b25-9ea2-cd1d-b631d74ac673: added peer, starting replication: peer=server-35cc5789-32c0-509d-3472-243f4999d8a5 +testLoggerAdapter verbose: server-063c306d-3b25-9ea2-cd1d-b631d74ac673: 2024-06-12T04:32:31.400+1400 [INFO] server-063c306d-3b25-9ea2-cd1d-b631d74ac673: pipelining replication: peer="{Voter server-35cc5789-32c0-509d-3472-243f4999d8a5 35cc5789-32c0-509d-3472-243f4999d8a5}" + testing.go:149: server-063c306d-3b25-9ea2-cd1d-b631d74ac673: 2024-06-12T04:32:31.400+1400 [INFO] server-063c306d-3b25-9ea2-cd1d-b631d74ac673: pipelining replication: peer="{Voter server-35cc5789-32c0-509d-3472-243f4999d8a5 35cc5789-32c0-509d-3472-243f4999d8a5}" +testLoggerAdapter verbose: server-063c306d-3b25-9ea2-cd1d-b631d74ac673: 2024-06-12T04:32:31.400+1400 [INFO] server-063c306d-3b25-9ea2-cd1d-b631d74ac673: pipelining replication: peer="{Voter server-22894afa-22a3-fb46-6be6-70213ddabb61 22894afa-22a3-fb46-6be6-70213ddabb61}" + testing.go:149: server-063c306d-3b25-9ea2-cd1d-b631d74ac673: 2024-06-12T04:32:31.400+1400 [INFO] server-063c306d-3b25-9ea2-cd1d-b631d74ac673: pipelining replication: peer="{Voter server-22894afa-22a3-fb46-6be6-70213ddabb61 22894afa-22a3-fb46-6be6-70213ddabb61}" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:31.505+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:31.400649392 +1400 +14 m=+8.308168011 (1 nodes), 59.048965ms from start of poll, 59.72258ms from cluster start. Timeout at 2024-06-12 04:32:31.505760947 +1400 +14 m=+8.413279526, 105.111515ms after stability + testing.go:149: cluster: 2024-06-12T04:32:31.505+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:31.400649392 +1400 +14 m=+8.308168011 (1 nodes), 59.048965ms from start of poll, 59.72258ms from cluster start. Timeout at 2024-06-12 04:32:31.505760947 +1400 +14 m=+8.413279526, 105.111515ms after stability +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:31.511+1400 [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-06-12T04:32:31.511+1400 [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:31.616+1400 [INFO] stable state for Follower reached at 2024-06-12 04:32:31.51155074 +1400 +14 m=+8.419069319 (2 nodes), 1.92µs from start of poll, 170.623888ms from cluster start. Timeout at 2024-06-12 04:32:31.616901086 +1400 +14 m=+8.524419705, 105.350386ms after stability + testing.go:149: cluster: 2024-06-12T04:32:31.616+1400 [INFO] stable state for Follower reached at 2024-06-12 04:32:31.51155074 +1400 +14 m=+8.419069319 (2 nodes), 1.92µs from start of poll, 170.623888ms from cluster start. Timeout at 2024-06-12 04:32:31.616901086 +1400 +14 m=+8.524419705, 105.350386ms after stability +testLoggerAdapter verbose: server-063c306d-3b25-9ea2-cd1d-b631d74ac673: 2024-06-12T04:32:31.617+1400 [INFO] server-063c306d-3b25-9ea2-cd1d-b631d74ac673: updating configuration: command=RemoveServer server-id=server-063c306d-3b25-9ea2-cd1d-b631d74ac673 server-addr= servers="[{Suffrage:Voter ID:server-22894afa-22a3-fb46-6be6-70213ddabb61 Address:22894afa-22a3-fb46-6be6-70213ddabb61} {Suffrage:Voter ID:server-35cc5789-32c0-509d-3472-243f4999d8a5 Address:35cc5789-32c0-509d-3472-243f4999d8a5}]" + testing.go:149: server-063c306d-3b25-9ea2-cd1d-b631d74ac673: 2024-06-12T04:32:31.617+1400 [INFO] server-063c306d-3b25-9ea2-cd1d-b631d74ac673: updating configuration: command=RemoveServer server-id=server-063c306d-3b25-9ea2-cd1d-b631d74ac673 server-addr= servers="[{Suffrage:Voter ID:server-22894afa-22a3-fb46-6be6-70213ddabb61 Address:22894afa-22a3-fb46-6be6-70213ddabb61} {Suffrage:Voter ID:server-35cc5789-32c0-509d-3472-243f4999d8a5 Address:35cc5789-32c0-509d-3472-243f4999d8a5}]" +testLoggerAdapter verbose: server-063c306d-3b25-9ea2-cd1d-b631d74ac673: 2024-06-12T04:32:31.617+1400 [INFO] server-063c306d-3b25-9ea2-cd1d-b631d74ac673: removed ourself, shutting down + testing.go:149: server-063c306d-3b25-9ea2-cd1d-b631d74ac673: 2024-06-12T04:32:31.617+1400 [INFO] server-063c306d-3b25-9ea2-cd1d-b631d74ac673: removed ourself, shutting down +testLoggerAdapter verbose: server-063c306d-3b25-9ea2-cd1d-b631d74ac673: 2024-06-12T04:32:31.617+1400 [INFO] server-063c306d-3b25-9ea2-cd1d-b631d74ac673: aborting pipeline replication: peer="{Voter server-35cc5789-32c0-509d-3472-243f4999d8a5 35cc5789-32c0-509d-3472-243f4999d8a5}" + testing.go:149: server-063c306d-3b25-9ea2-cd1d-b631d74ac673: 2024-06-12T04:32:31.617+1400 [INFO] server-063c306d-3b25-9ea2-cd1d-b631d74ac673: aborting pipeline replication: peer="{Voter server-35cc5789-32c0-509d-3472-243f4999d8a5 35cc5789-32c0-509d-3472-243f4999d8a5}" +testLoggerAdapter verbose: server-063c306d-3b25-9ea2-cd1d-b631d74ac673: 2024-06-12T04:32:31.617+1400 [INFO] server-063c306d-3b25-9ea2-cd1d-b631d74ac673: aborting pipeline replication: peer="{Voter server-22894afa-22a3-fb46-6be6-70213ddabb61 22894afa-22a3-fb46-6be6-70213ddabb61}" + testing.go:149: server-063c306d-3b25-9ea2-cd1d-b631d74ac673: 2024-06-12T04:32:31.617+1400 [INFO] server-063c306d-3b25-9ea2-cd1d-b631d74ac673: aborting pipeline replication: peer="{Voter server-22894afa-22a3-fb46-6be6-70213ddabb61 22894afa-22a3-fb46-6be6-70213ddabb61}" +testLoggerAdapter verbose: server-22894afa-22a3-fb46-6be6-70213ddabb61: 2024-06-12T04:32:31.669+1400 [WARN] server-22894afa-22a3-fb46-6be6-70213ddabb61: heartbeat timeout reached, starting election: last-leader=063c306d-3b25-9ea2-cd1d-b631d74ac673 + testing.go:149: server-22894afa-22a3-fb46-6be6-70213ddabb61: 2024-06-12T04:32:31.669+1400 [WARN] server-22894afa-22a3-fb46-6be6-70213ddabb61: heartbeat timeout reached, starting election: last-leader=063c306d-3b25-9ea2-cd1d-b631d74ac673 +testLoggerAdapter verbose: server-22894afa-22a3-fb46-6be6-70213ddabb61: 2024-06-12T04:32:31.669+1400 [INFO] server-22894afa-22a3-fb46-6be6-70213ddabb61: entering candidate state: node="Node at 22894afa-22a3-fb46-6be6-70213ddabb61 [Candidate]" term=3 + testing.go:149: server-22894afa-22a3-fb46-6be6-70213ddabb61: 2024-06-12T04:32:31.669+1400 [INFO] server-22894afa-22a3-fb46-6be6-70213ddabb61: entering candidate state: node="Node at 22894afa-22a3-fb46-6be6-70213ddabb61 [Candidate]" term=3 +testLoggerAdapter verbose: server-35cc5789-32c0-509d-3472-243f4999d8a5: 2024-06-12T04:32:31.669+1400 [WARN] server-35cc5789-32c0-509d-3472-243f4999d8a5: rejecting vote request since we have a leader: from=22894afa-22a3-fb46-6be6-70213ddabb61 leader=063c306d-3b25-9ea2-cd1d-b631d74ac673 + testing.go:149: server-35cc5789-32c0-509d-3472-243f4999d8a5: 2024-06-12T04:32:31.669+1400 [WARN] server-35cc5789-32c0-509d-3472-243f4999d8a5: rejecting vote request since we have a leader: from=22894afa-22a3-fb46-6be6-70213ddabb61 leader=063c306d-3b25-9ea2-cd1d-b631d74ac673 +testLoggerAdapter verbose: server-35cc5789-32c0-509d-3472-243f4999d8a5: 2024-06-12T04:32:31.692+1400 [WARN] server-35cc5789-32c0-509d-3472-243f4999d8a5: heartbeat timeout reached, starting election: last-leader=063c306d-3b25-9ea2-cd1d-b631d74ac673 + testing.go:149: server-35cc5789-32c0-509d-3472-243f4999d8a5: 2024-06-12T04:32:31.692+1400 [WARN] server-35cc5789-32c0-509d-3472-243f4999d8a5: heartbeat timeout reached, starting election: last-leader=063c306d-3b25-9ea2-cd1d-b631d74ac673 +testLoggerAdapter verbose: server-35cc5789-32c0-509d-3472-243f4999d8a5: 2024-06-12T04:32:31.692+1400 [INFO] server-35cc5789-32c0-509d-3472-243f4999d8a5: entering candidate state: node="Node at 35cc5789-32c0-509d-3472-243f4999d8a5 [Candidate]" term=3 + testing.go:149: server-35cc5789-32c0-509d-3472-243f4999d8a5: 2024-06-12T04:32:31.692+1400 [INFO] server-35cc5789-32c0-509d-3472-243f4999d8a5: entering candidate state: node="Node at 35cc5789-32c0-509d-3472-243f4999d8a5 [Candidate]" term=3 +testLoggerAdapter verbose: server-22894afa-22a3-fb46-6be6-70213ddabb61: 2024-06-12T04:32:31.692+1400 [INFO] server-22894afa-22a3-fb46-6be6-70213ddabb61: duplicate requestVote for same term: term=3 + testing.go:149: server-22894afa-22a3-fb46-6be6-70213ddabb61: 2024-06-12T04:32:31.692+1400 [INFO] server-22894afa-22a3-fb46-6be6-70213ddabb61: duplicate requestVote for same term: term=3 +testLoggerAdapter verbose: server-22894afa-22a3-fb46-6be6-70213ddabb61: 2024-06-12T04:32:31.753+1400 [WARN] server-22894afa-22a3-fb46-6be6-70213ddabb61: Election timeout reached, restarting election + testing.go:149: server-22894afa-22a3-fb46-6be6-70213ddabb61: 2024-06-12T04:32:31.753+1400 [WARN] server-22894afa-22a3-fb46-6be6-70213ddabb61: Election timeout reached, restarting election +testLoggerAdapter verbose: server-22894afa-22a3-fb46-6be6-70213ddabb61: 2024-06-12T04:32:31.753+1400 [INFO] server-22894afa-22a3-fb46-6be6-70213ddabb61: entering candidate state: node="Node at 22894afa-22a3-fb46-6be6-70213ddabb61 [Candidate]" term=4 + testing.go:149: server-22894afa-22a3-fb46-6be6-70213ddabb61: 2024-06-12T04:32:31.753+1400 [INFO] server-22894afa-22a3-fb46-6be6-70213ddabb61: entering candidate state: node="Node at 22894afa-22a3-fb46-6be6-70213ddabb61 [Candidate]" term=4 +testLoggerAdapter verbose: server-35cc5789-32c0-509d-3472-243f4999d8a5: 2024-06-12T04:32:31.753+1400 [INFO] server-35cc5789-32c0-509d-3472-243f4999d8a5: entering follower state: follower="Node at 35cc5789-32c0-509d-3472-243f4999d8a5 [Follower]" leader= + testing.go:149: server-35cc5789-32c0-509d-3472-243f4999d8a5: 2024-06-12T04:32:31.753+1400 [INFO] server-35cc5789-32c0-509d-3472-243f4999d8a5: entering follower state: follower="Node at 35cc5789-32c0-509d-3472-243f4999d8a5 [Follower]" leader= +testLoggerAdapter verbose: server-22894afa-22a3-fb46-6be6-70213ddabb61: 2024-06-12T04:32:31.753+1400 [INFO] server-22894afa-22a3-fb46-6be6-70213ddabb61: election won: tally=2 + testing.go:149: server-22894afa-22a3-fb46-6be6-70213ddabb61: 2024-06-12T04:32:31.753+1400 [INFO] server-22894afa-22a3-fb46-6be6-70213ddabb61: election won: tally=2 +testLoggerAdapter verbose: server-22894afa-22a3-fb46-6be6-70213ddabb61: 2024-06-12T04:32:31.753+1400 [INFO] server-22894afa-22a3-fb46-6be6-70213ddabb61: entering leader state: leader="Node at 22894afa-22a3-fb46-6be6-70213ddabb61 [Leader]" + testing.go:149: server-22894afa-22a3-fb46-6be6-70213ddabb61: 2024-06-12T04:32:31.753+1400 [INFO] server-22894afa-22a3-fb46-6be6-70213ddabb61: entering leader state: leader="Node at 22894afa-22a3-fb46-6be6-70213ddabb61 [Leader]" +testLoggerAdapter verbose: server-22894afa-22a3-fb46-6be6-70213ddabb61: 2024-06-12T04:32:31.753+1400 [INFO] server-22894afa-22a3-fb46-6be6-70213ddabb61: added peer, starting replication: peer=server-35cc5789-32c0-509d-3472-243f4999d8a5 + testing.go:149: server-22894afa-22a3-fb46-6be6-70213ddabb61: 2024-06-12T04:32:31.753+1400 [INFO] server-22894afa-22a3-fb46-6be6-70213ddabb61: added peer, starting replication: peer=server-35cc5789-32c0-509d-3472-243f4999d8a5 +testLoggerAdapter verbose: server-22894afa-22a3-fb46-6be6-70213ddabb61: 2024-06-12T04:32:31.753+1400 [INFO] server-22894afa-22a3-fb46-6be6-70213ddabb61: pipelining replication: peer="{Voter server-35cc5789-32c0-509d-3472-243f4999d8a5 35cc5789-32c0-509d-3472-243f4999d8a5}" + testing.go:149: server-22894afa-22a3-fb46-6be6-70213ddabb61: 2024-06-12T04:32:31.753+1400 [INFO] server-22894afa-22a3-fb46-6be6-70213ddabb61: pipelining replication: peer="{Voter server-35cc5789-32c0-509d-3472-243f4999d8a5 35cc5789-32c0-509d-3472-243f4999d8a5}" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:31.828+1400 [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-06-12T04:32:31.828+1400 [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:31.933+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:31.82869906 +1400 +14 m=+8.736217639 (1 nodes), 60.678µs from start of poll, 487.772208ms from cluster start. Timeout at 2024-06-12 04:32:31.933912771 +1400 +14 m=+8.841431390, 105.213751ms after stability + testing.go:149: cluster: 2024-06-12T04:32:31.933+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:31.82869906 +1400 +14 m=+8.736217639 (1 nodes), 60.678µs from start of poll, 487.772208ms from cluster start. Timeout at 2024-06-12 04:32:31.933912771 +1400 +14 m=+8.841431390, 105.213751ms after stability +testLoggerAdapter verbose: server-22894afa-22a3-fb46-6be6-70213ddabb61: 2024-06-12T04:32:31.934+1400 [INFO] server-22894afa-22a3-fb46-6be6-70213ddabb61: aborting pipeline replication: peer="{Voter server-35cc5789-32c0-509d-3472-243f4999d8a5 35cc5789-32c0-509d-3472-243f4999d8a5}" + testing.go:149: server-22894afa-22a3-fb46-6be6-70213ddabb61: 2024-06-12T04:32:31.934+1400 [INFO] server-22894afa-22a3-fb46-6be6-70213ddabb61: aborting pipeline replication: peer="{Voter server-35cc5789-32c0-509d-3472-243f4999d8a5 35cc5789-32c0-509d-3472-243f4999d8a5}" --- PASS: TestRaft_RemoveLeader (0.60s) === RUN TestRaft_RemoveLeader_NoShutdown -testLoggerAdapter verbose: server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: 2024-06-11T02:29:18.580-1200 [INFO] server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66 Address:29addc99-1b26-3a13-bfd5-c0e03e0a2c66} {Suffrage:Voter ID:server-642329a2-4fcd-3792-9843-9b991e6897dd Address:642329a2-4fcd-3792-9843-9b991e6897dd} {Suffrage:Voter ID:server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66 Address:96bf3f21-a9c2-731e-fe03-72eaf04e5b66}]" - testing.go:149: server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: 2024-06-11T02:29:18.580-1200 [INFO] server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66 Address:29addc99-1b26-3a13-bfd5-c0e03e0a2c66} {Suffrage:Voter ID:server-642329a2-4fcd-3792-9843-9b991e6897dd Address:642329a2-4fcd-3792-9843-9b991e6897dd} {Suffrage:Voter ID:server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66 Address:96bf3f21-a9c2-731e-fe03-72eaf04e5b66}]" -testLoggerAdapter verbose: server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: 2024-06-11T02:29:18.581-1200 [INFO] server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: entering follower state: follower="Node at 29addc99-1b26-3a13-bfd5-c0e03e0a2c66 [Follower]" leader= - testing.go:149: server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: 2024-06-11T02:29:18.581-1200 [INFO] server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: entering follower state: follower="Node at 29addc99-1b26-3a13-bfd5-c0e03e0a2c66 [Follower]" leader= -testLoggerAdapter verbose: server-642329a2-4fcd-3792-9843-9b991e6897dd: 2024-06-11T02:29:18.581-1200 [INFO] server-642329a2-4fcd-3792-9843-9b991e6897dd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66 Address:29addc99-1b26-3a13-bfd5-c0e03e0a2c66} {Suffrage:Voter ID:server-642329a2-4fcd-3792-9843-9b991e6897dd Address:642329a2-4fcd-3792-9843-9b991e6897dd} {Suffrage:Voter ID:server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66 Address:96bf3f21-a9c2-731e-fe03-72eaf04e5b66}]" - testing.go:149: server-642329a2-4fcd-3792-9843-9b991e6897dd: 2024-06-11T02:29:18.581-1200 [INFO] server-642329a2-4fcd-3792-9843-9b991e6897dd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66 Address:29addc99-1b26-3a13-bfd5-c0e03e0a2c66} {Suffrage:Voter ID:server-642329a2-4fcd-3792-9843-9b991e6897dd Address:642329a2-4fcd-3792-9843-9b991e6897dd} {Suffrage:Voter ID:server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66 Address:96bf3f21-a9c2-731e-fe03-72eaf04e5b66}]" -testLoggerAdapter verbose: server-642329a2-4fcd-3792-9843-9b991e6897dd: 2024-06-11T02:29:18.581-1200 [INFO] server-642329a2-4fcd-3792-9843-9b991e6897dd: entering follower state: follower="Node at 642329a2-4fcd-3792-9843-9b991e6897dd [Follower]" leader= - testing.go:149: server-642329a2-4fcd-3792-9843-9b991e6897dd: 2024-06-11T02:29:18.581-1200 [INFO] server-642329a2-4fcd-3792-9843-9b991e6897dd: entering follower state: follower="Node at 642329a2-4fcd-3792-9843-9b991e6897dd [Follower]" leader= -testLoggerAdapter verbose: server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66: 2024-06-11T02:29:18.581-1200 [INFO] server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66 Address:29addc99-1b26-3a13-bfd5-c0e03e0a2c66} {Suffrage:Voter ID:server-642329a2-4fcd-3792-9843-9b991e6897dd Address:642329a2-4fcd-3792-9843-9b991e6897dd} {Suffrage:Voter ID:server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66 Address:96bf3f21-a9c2-731e-fe03-72eaf04e5b66}]" - testing.go:149: server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66: 2024-06-11T02:29:18.581-1200 [INFO] server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66 Address:29addc99-1b26-3a13-bfd5-c0e03e0a2c66} {Suffrage:Voter ID:server-642329a2-4fcd-3792-9843-9b991e6897dd Address:642329a2-4fcd-3792-9843-9b991e6897dd} {Suffrage:Voter ID:server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66 Address:96bf3f21-a9c2-731e-fe03-72eaf04e5b66}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:18.582-1200 [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2024-06-11T02:29:18.582-1200 [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66: 2024-06-11T02:29:18.582-1200 [INFO] server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66: entering follower state: follower="Node at 96bf3f21-a9c2-731e-fe03-72eaf04e5b66 [Follower]" leader= - testing.go:149: server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66: 2024-06-11T02:29:18.582-1200 [INFO] server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66: entering follower state: follower="Node at 96bf3f21-a9c2-731e-fe03-72eaf04e5b66 [Follower]" leader= -testLoggerAdapter verbose: server-642329a2-4fcd-3792-9843-9b991e6897dd: 2024-06-11T02:29:18.634-1200 [WARN] server-642329a2-4fcd-3792-9843-9b991e6897dd: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-642329a2-4fcd-3792-9843-9b991e6897dd: 2024-06-11T02:29:18.634-1200 [WARN] server-642329a2-4fcd-3792-9843-9b991e6897dd: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-642329a2-4fcd-3792-9843-9b991e6897dd: 2024-06-11T02:29:18.634-1200 [INFO] server-642329a2-4fcd-3792-9843-9b991e6897dd: entering candidate state: node="Node at 642329a2-4fcd-3792-9843-9b991e6897dd [Candidate]" term=2 - testing.go:149: server-642329a2-4fcd-3792-9843-9b991e6897dd: 2024-06-11T02:29:18.634-1200 [INFO] server-642329a2-4fcd-3792-9843-9b991e6897dd: entering candidate state: node="Node at 642329a2-4fcd-3792-9843-9b991e6897dd [Candidate]" term=2 -testLoggerAdapter verbose: server-642329a2-4fcd-3792-9843-9b991e6897dd: 2024-06-11T02:29:18.634-1200 [INFO] server-642329a2-4fcd-3792-9843-9b991e6897dd: election won: tally=2 - testing.go:149: server-642329a2-4fcd-3792-9843-9b991e6897dd: 2024-06-11T02:29:18.634-1200 [INFO] server-642329a2-4fcd-3792-9843-9b991e6897dd: election won: tally=2 -testLoggerAdapter verbose: server-642329a2-4fcd-3792-9843-9b991e6897dd: 2024-06-11T02:29:18.634-1200 [INFO] server-642329a2-4fcd-3792-9843-9b991e6897dd: entering leader state: leader="Node at 642329a2-4fcd-3792-9843-9b991e6897dd [Leader]" - testing.go:149: server-642329a2-4fcd-3792-9843-9b991e6897dd: 2024-06-11T02:29:18.634-1200 [INFO] server-642329a2-4fcd-3792-9843-9b991e6897dd: entering leader state: leader="Node at 642329a2-4fcd-3792-9843-9b991e6897dd [Leader]" -testLoggerAdapter verbose: server-642329a2-4fcd-3792-9843-9b991e6897dd: 2024-06-11T02:29:18.634-1200 [INFO] server-642329a2-4fcd-3792-9843-9b991e6897dd: added peer, starting replication: peer=server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66 - testing.go:149: server-642329a2-4fcd-3792-9843-9b991e6897dd: 2024-06-11T02:29:18.634-1200 [INFO] server-642329a2-4fcd-3792-9843-9b991e6897dd: added peer, starting replication: peer=server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66 -testLoggerAdapter verbose: server-642329a2-4fcd-3792-9843-9b991e6897dd: 2024-06-11T02:29:18.634-1200 [INFO] server-642329a2-4fcd-3792-9843-9b991e6897dd: added peer, starting replication: peer=server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66 - testing.go:149: server-642329a2-4fcd-3792-9843-9b991e6897dd: 2024-06-11T02:29:18.634-1200 [INFO] server-642329a2-4fcd-3792-9843-9b991e6897dd: added peer, starting replication: peer=server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66 -testLoggerAdapter verbose: server-642329a2-4fcd-3792-9843-9b991e6897dd: 2024-06-11T02:29:18.635-1200 [INFO] server-642329a2-4fcd-3792-9843-9b991e6897dd: pipelining replication: peer="{Voter server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66 96bf3f21-a9c2-731e-fe03-72eaf04e5b66}" - testing.go:149: server-642329a2-4fcd-3792-9843-9b991e6897dd: 2024-06-11T02:29:18.635-1200 [INFO] server-642329a2-4fcd-3792-9843-9b991e6897dd: pipelining replication: peer="{Voter server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66 96bf3f21-a9c2-731e-fe03-72eaf04e5b66}" -testLoggerAdapter verbose: server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: 2024-06-11T02:29:18.635-1200 [WARN] server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: rejecting vote request since our last term is greater: candidate=642329a2-4fcd-3792-9843-9b991e6897dd last-term=2 last-candidate-term=1 - testing.go:149: server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: 2024-06-11T02:29:18.635-1200 [WARN] server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: rejecting vote request since our last term is greater: candidate=642329a2-4fcd-3792-9843-9b991e6897dd last-term=2 last-candidate-term=1 -testLoggerAdapter verbose: server-642329a2-4fcd-3792-9843-9b991e6897dd: 2024-06-11T02:29:18.635-1200 [INFO] server-642329a2-4fcd-3792-9843-9b991e6897dd: pipelining replication: peer="{Voter server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66 29addc99-1b26-3a13-bfd5-c0e03e0a2c66}" - testing.go:149: server-642329a2-4fcd-3792-9843-9b991e6897dd: 2024-06-11T02:29:18.635-1200 [INFO] server-642329a2-4fcd-3792-9843-9b991e6897dd: pipelining replication: peer="{Voter server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66 29addc99-1b26-3a13-bfd5-c0e03e0a2c66}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:18.741-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:18.635396282 -1200 -12 m=+9.630768824 (2 nodes), 53.209207ms from start of poll, 54.67393ms from cluster start. Timeout at 2024-06-11 02:29:18.741484537 -1200 -12 m=+9.736857078, 106.088254ms after stability - testing.go:149: cluster: 2024-06-11T02:29:18.741-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:18.635396282 -1200 -12 m=+9.630768824 (2 nodes), 53.209207ms from start of poll, 54.67393ms from cluster start. Timeout at 2024-06-11 02:29:18.741484537 -1200 -12 m=+9.736857078, 106.088254ms after stability -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:18.741-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:18.741-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:18.847-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:18.741747907 -1200 -12 m=+9.737120448 (1 nodes), 2.916µs from start of poll, 161.025554ms from cluster start. Timeout at 2024-06-11 02:29:18.846995593 -1200 -12 m=+9.842368135, 105.247687ms after stability - testing.go:149: cluster: 2024-06-11T02:29:18.847-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:18.741747907 -1200 -12 m=+9.737120448 (1 nodes), 2.916µs from start of poll, 161.025554ms from cluster start. Timeout at 2024-06-11 02:29:18.846995593 -1200 -12 m=+9.842368135, 105.247687ms after stability -testLoggerAdapter verbose: server-642329a2-4fcd-3792-9843-9b991e6897dd: 2024-06-11T02:29:18.851-1200 [INFO] server-642329a2-4fcd-3792-9843-9b991e6897dd: updating configuration: command=RemoveServer server-id=server-642329a2-4fcd-3792-9843-9b991e6897dd server-addr= servers="[{Suffrage:Voter ID:server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66 Address:29addc99-1b26-3a13-bfd5-c0e03e0a2c66} {Suffrage:Voter ID:server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66 Address:96bf3f21-a9c2-731e-fe03-72eaf04e5b66}]" - testing.go:149: server-642329a2-4fcd-3792-9843-9b991e6897dd: 2024-06-11T02:29:18.851-1200 [INFO] server-642329a2-4fcd-3792-9843-9b991e6897dd: updating configuration: command=RemoveServer server-id=server-642329a2-4fcd-3792-9843-9b991e6897dd server-addr= servers="[{Suffrage:Voter ID:server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66 Address:29addc99-1b26-3a13-bfd5-c0e03e0a2c66} {Suffrage:Voter ID:server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66 Address:96bf3f21-a9c2-731e-fe03-72eaf04e5b66}]" -testLoggerAdapter verbose: server-642329a2-4fcd-3792-9843-9b991e6897dd: 2024-06-11T02:29:18.852-1200 [INFO] server-642329a2-4fcd-3792-9843-9b991e6897dd: removed ourself, transitioning to follower - testing.go:149: server-642329a2-4fcd-3792-9843-9b991e6897dd: 2024-06-11T02:29:18.852-1200 [INFO] server-642329a2-4fcd-3792-9843-9b991e6897dd: removed ourself, transitioning to follower -testLoggerAdapter verbose: server-642329a2-4fcd-3792-9843-9b991e6897dd: 2024-06-11T02:29:18.852-1200 [INFO] server-642329a2-4fcd-3792-9843-9b991e6897dd: entering follower state: follower="Node at 642329a2-4fcd-3792-9843-9b991e6897dd [Follower]" leader= - testing.go:149: server-642329a2-4fcd-3792-9843-9b991e6897dd: 2024-06-11T02:29:18.852-1200 [INFO] server-642329a2-4fcd-3792-9843-9b991e6897dd: entering follower state: follower="Node at 642329a2-4fcd-3792-9843-9b991e6897dd [Follower]" leader= -testLoggerAdapter verbose: server-642329a2-4fcd-3792-9843-9b991e6897dd: 2024-06-11T02:29:18.852-1200 [INFO] server-642329a2-4fcd-3792-9843-9b991e6897dd: aborting pipeline replication: peer="{Voter server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66 96bf3f21-a9c2-731e-fe03-72eaf04e5b66}" - testing.go:149: server-642329a2-4fcd-3792-9843-9b991e6897dd: 2024-06-11T02:29:18.852-1200 [INFO] server-642329a2-4fcd-3792-9843-9b991e6897dd: aborting pipeline replication: peer="{Voter server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66 96bf3f21-a9c2-731e-fe03-72eaf04e5b66}" -testLoggerAdapter verbose: server-642329a2-4fcd-3792-9843-9b991e6897dd: 2024-06-11T02:29:18.852-1200 [INFO] server-642329a2-4fcd-3792-9843-9b991e6897dd: aborting pipeline replication: peer="{Voter server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66 29addc99-1b26-3a13-bfd5-c0e03e0a2c66}" - testing.go:149: server-642329a2-4fcd-3792-9843-9b991e6897dd: 2024-06-11T02:29:18.852-1200 [INFO] server-642329a2-4fcd-3792-9843-9b991e6897dd: aborting pipeline replication: peer="{Voter server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66 29addc99-1b26-3a13-bfd5-c0e03e0a2c66}" -testLoggerAdapter verbose: server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: 2024-06-11T02:29:18.920-1200 [WARN] server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: heartbeat timeout reached, starting election: last-leader=642329a2-4fcd-3792-9843-9b991e6897dd - testing.go:149: server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: 2024-06-11T02:29:18.920-1200 [WARN] server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: heartbeat timeout reached, starting election: last-leader=642329a2-4fcd-3792-9843-9b991e6897dd -testLoggerAdapter verbose: server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: 2024-06-11T02:29:18.920-1200 [INFO] server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: entering candidate state: node="Node at 29addc99-1b26-3a13-bfd5-c0e03e0a2c66 [Candidate]" term=3 - testing.go:149: server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: 2024-06-11T02:29:18.920-1200 [INFO] server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: entering candidate state: node="Node at 29addc99-1b26-3a13-bfd5-c0e03e0a2c66 [Candidate]" term=3 -testLoggerAdapter verbose: server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66: 2024-06-11T02:29:18.921-1200 [WARN] server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66: rejecting vote request since we have a leader: from=29addc99-1b26-3a13-bfd5-c0e03e0a2c66 leader=642329a2-4fcd-3792-9843-9b991e6897dd - testing.go:149: server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66: 2024-06-11T02:29:18.921-1200 [WARN] server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66: rejecting vote request since we have a leader: from=29addc99-1b26-3a13-bfd5-c0e03e0a2c66 leader=642329a2-4fcd-3792-9843-9b991e6897dd -testLoggerAdapter verbose: server-642329a2-4fcd-3792-9843-9b991e6897dd: 2024-06-11T02:29:18.946-1200 [WARN] server-642329a2-4fcd-3792-9843-9b991e6897dd: not part of stable configuration, aborting election - testing.go:149: server-642329a2-4fcd-3792-9843-9b991e6897dd: 2024-06-11T02:29:18.946-1200 [WARN] server-642329a2-4fcd-3792-9843-9b991e6897dd: not part of stable configuration, aborting election -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:18.958-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:18.958-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66: 2024-06-11T02:29:18.959-1200 [WARN] server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66: heartbeat timeout reached, starting election: last-leader=642329a2-4fcd-3792-9843-9b991e6897dd - testing.go:149: server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66: 2024-06-11T02:29:18.959-1200 [WARN] server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66: heartbeat timeout reached, starting election: last-leader=642329a2-4fcd-3792-9843-9b991e6897dd -testLoggerAdapter verbose: server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66: 2024-06-11T02:29:18.960-1200 [INFO] server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66: entering candidate state: node="Node at 96bf3f21-a9c2-731e-fe03-72eaf04e5b66 [Candidate]" term=3 - testing.go:149: server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66: 2024-06-11T02:29:18.960-1200 [INFO] server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66: entering candidate state: node="Node at 96bf3f21-a9c2-731e-fe03-72eaf04e5b66 [Candidate]" term=3 -testLoggerAdapter verbose: server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: 2024-06-11T02:29:18.960-1200 [INFO] server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: duplicate requestVote for same term: term=3 - testing.go:149: server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: 2024-06-11T02:29:18.960-1200 [INFO] server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: duplicate requestVote for same term: term=3 -testLoggerAdapter verbose: server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: 2024-06-11T02:29:18.974-1200 [WARN] server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: Election timeout reached, restarting election - testing.go:149: server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: 2024-06-11T02:29:18.974-1200 [WARN] server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: Election timeout reached, restarting election -testLoggerAdapter verbose: server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: 2024-06-11T02:29:18.974-1200 [INFO] server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: entering candidate state: node="Node at 29addc99-1b26-3a13-bfd5-c0e03e0a2c66 [Candidate]" term=4 - testing.go:149: server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: 2024-06-11T02:29:18.974-1200 [INFO] server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: entering candidate state: node="Node at 29addc99-1b26-3a13-bfd5-c0e03e0a2c66 [Candidate]" term=4 -testLoggerAdapter verbose: server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66: 2024-06-11T02:29:18.974-1200 [INFO] server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66: entering follower state: follower="Node at 96bf3f21-a9c2-731e-fe03-72eaf04e5b66 [Follower]" leader= - testing.go:149: server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66: 2024-06-11T02:29:18.974-1200 [INFO] server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66: entering follower state: follower="Node at 96bf3f21-a9c2-731e-fe03-72eaf04e5b66 [Follower]" leader= -testLoggerAdapter verbose: server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: 2024-06-11T02:29:18.974-1200 [INFO] server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: election won: tally=2 - testing.go:149: server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: 2024-06-11T02:29:18.974-1200 [INFO] server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: election won: tally=2 -testLoggerAdapter verbose: server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: 2024-06-11T02:29:18.975-1200 [INFO] server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: entering leader state: leader="Node at 29addc99-1b26-3a13-bfd5-c0e03e0a2c66 [Leader]" - testing.go:149: server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: 2024-06-11T02:29:18.975-1200 [INFO] server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: entering leader state: leader="Node at 29addc99-1b26-3a13-bfd5-c0e03e0a2c66 [Leader]" -testLoggerAdapter verbose: server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: 2024-06-11T02:29:18.975-1200 [INFO] server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: added peer, starting replication: peer=server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66 - testing.go:149: server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: 2024-06-11T02:29:18.975-1200 [INFO] server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: added peer, starting replication: peer=server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66 -testLoggerAdapter verbose: server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: 2024-06-11T02:29:18.975-1200 [INFO] server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: pipelining replication: peer="{Voter server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66 96bf3f21-a9c2-731e-fe03-72eaf04e5b66}" - testing.go:149: server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: 2024-06-11T02:29:18.975-1200 [INFO] server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: pipelining replication: peer="{Voter server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66 96bf3f21-a9c2-731e-fe03-72eaf04e5b66}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:19.080-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:18.975649381 -1200 -12 m=+9.971021923 (1 nodes), 17.170385ms from start of poll, 394.927029ms from cluster start. Timeout at 2024-06-11 02:29:19.08072703 -1200 -12 m=+10.076099863, 105.07794ms after stability - testing.go:149: cluster: 2024-06-11T02:29:19.080-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:18.975649381 -1200 -12 m=+9.971021923 (1 nodes), 17.170385ms from start of poll, 394.927029ms from cluster start. Timeout at 2024-06-11 02:29:19.08072703 -1200 -12 m=+10.076099863, 105.07794ms after stability -testLoggerAdapter verbose: server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: 2024-06-11T02:29:19.186-1200 [INFO] server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: aborting pipeline replication: peer="{Voter server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66 96bf3f21-a9c2-731e-fe03-72eaf04e5b66}" - testing.go:149: server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: 2024-06-11T02:29:19.186-1200 [INFO] server-29addc99-1b26-3a13-bfd5-c0e03e0a2c66: aborting pipeline replication: peer="{Voter server-96bf3f21-a9c2-731e-fe03-72eaf04e5b66 96bf3f21-a9c2-731e-fe03-72eaf04e5b66}" ---- PASS: TestRaft_RemoveLeader_NoShutdown (0.61s) +testLoggerAdapter verbose: server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: 2024-06-12T04:32:31.936+1400 [INFO] server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0 Address:3a835b66-4fb4-cfb0-2ed8-f7974de544d0} {Suffrage:Voter ID:server-ff61209f-da10-2114-5db9-ef83dfc32b83 Address:ff61209f-da10-2114-5db9-ef83dfc32b83} {Suffrage:Voter ID:server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0 Address:2b0eb6ae-173a-b016-69ac-0125a01f9cd0}]" + testing.go:149: server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: 2024-06-12T04:32:31.936+1400 [INFO] server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0 Address:3a835b66-4fb4-cfb0-2ed8-f7974de544d0} {Suffrage:Voter ID:server-ff61209f-da10-2114-5db9-ef83dfc32b83 Address:ff61209f-da10-2114-5db9-ef83dfc32b83} {Suffrage:Voter ID:server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0 Address:2b0eb6ae-173a-b016-69ac-0125a01f9cd0}]" +testLoggerAdapter verbose: server-ff61209f-da10-2114-5db9-ef83dfc32b83: 2024-06-12T04:32:31.936+1400 [INFO] server-ff61209f-da10-2114-5db9-ef83dfc32b83: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0 Address:3a835b66-4fb4-cfb0-2ed8-f7974de544d0} {Suffrage:Voter ID:server-ff61209f-da10-2114-5db9-ef83dfc32b83 Address:ff61209f-da10-2114-5db9-ef83dfc32b83} {Suffrage:Voter ID:server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0 Address:2b0eb6ae-173a-b016-69ac-0125a01f9cd0}]" + testing.go:149: server-ff61209f-da10-2114-5db9-ef83dfc32b83: 2024-06-12T04:32:31.936+1400 [INFO] server-ff61209f-da10-2114-5db9-ef83dfc32b83: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0 Address:3a835b66-4fb4-cfb0-2ed8-f7974de544d0} {Suffrage:Voter ID:server-ff61209f-da10-2114-5db9-ef83dfc32b83 Address:ff61209f-da10-2114-5db9-ef83dfc32b83} {Suffrage:Voter ID:server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0 Address:2b0eb6ae-173a-b016-69ac-0125a01f9cd0}]" +testLoggerAdapter verbose: server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: 2024-06-12T04:32:31.937+1400 [INFO] server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0 Address:3a835b66-4fb4-cfb0-2ed8-f7974de544d0} {Suffrage:Voter ID:server-ff61209f-da10-2114-5db9-ef83dfc32b83 Address:ff61209f-da10-2114-5db9-ef83dfc32b83} {Suffrage:Voter ID:server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0 Address:2b0eb6ae-173a-b016-69ac-0125a01f9cd0}]" + testing.go:149: server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: 2024-06-12T04:32:31.937+1400 [INFO] server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0 Address:3a835b66-4fb4-cfb0-2ed8-f7974de544d0} {Suffrage:Voter ID:server-ff61209f-da10-2114-5db9-ef83dfc32b83 Address:ff61209f-da10-2114-5db9-ef83dfc32b83} {Suffrage:Voter ID:server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0 Address:2b0eb6ae-173a-b016-69ac-0125a01f9cd0}]" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:31.937+1400 [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-06-12T04:32:31.937+1400 [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: 2024-06-12T04:32:31.937+1400 [INFO] server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: entering follower state: follower="Node at 3a835b66-4fb4-cfb0-2ed8-f7974de544d0 [Follower]" leader= + testing.go:149: server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: 2024-06-12T04:32:31.937+1400 [INFO] server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: entering follower state: follower="Node at 3a835b66-4fb4-cfb0-2ed8-f7974de544d0 [Follower]" leader= +testLoggerAdapter verbose: server-ff61209f-da10-2114-5db9-ef83dfc32b83: 2024-06-12T04:32:31.937+1400 [INFO] server-ff61209f-da10-2114-5db9-ef83dfc32b83: entering follower state: follower="Node at ff61209f-da10-2114-5db9-ef83dfc32b83 [Follower]" leader= + testing.go:149: server-ff61209f-da10-2114-5db9-ef83dfc32b83: 2024-06-12T04:32:31.937+1400 [INFO] server-ff61209f-da10-2114-5db9-ef83dfc32b83: entering follower state: follower="Node at ff61209f-da10-2114-5db9-ef83dfc32b83 [Follower]" leader= +testLoggerAdapter verbose: server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: 2024-06-12T04:32:31.937+1400 [INFO] server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: entering follower state: follower="Node at 2b0eb6ae-173a-b016-69ac-0125a01f9cd0 [Follower]" leader= + testing.go:149: server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: 2024-06-12T04:32:31.937+1400 [INFO] server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: entering follower state: follower="Node at 2b0eb6ae-173a-b016-69ac-0125a01f9cd0 [Follower]" leader= +testLoggerAdapter verbose: server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: 2024-06-12T04:32:32.009+1400 [WARN] server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: 2024-06-12T04:32:32.009+1400 [WARN] server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: 2024-06-12T04:32:32.009+1400 [INFO] server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: entering candidate state: node="Node at 3a835b66-4fb4-cfb0-2ed8-f7974de544d0 [Candidate]" term=2 + testing.go:149: server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: 2024-06-12T04:32:32.009+1400 [INFO] server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: entering candidate state: node="Node at 3a835b66-4fb4-cfb0-2ed8-f7974de544d0 [Candidate]" term=2 +testLoggerAdapter verbose: server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: 2024-06-12T04:32:32.010+1400 [INFO] server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: election won: tally=2 + testing.go:149: server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: 2024-06-12T04:32:32.010+1400 [INFO] server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: election won: tally=2 +testLoggerAdapter verbose: server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: 2024-06-12T04:32:32.010+1400 [INFO] server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: entering leader state: leader="Node at 3a835b66-4fb4-cfb0-2ed8-f7974de544d0 [Leader]" + testing.go:149: server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: 2024-06-12T04:32:32.010+1400 [INFO] server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: entering leader state: leader="Node at 3a835b66-4fb4-cfb0-2ed8-f7974de544d0 [Leader]" +testLoggerAdapter verbose: server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: 2024-06-12T04:32:32.010+1400 [INFO] server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: added peer, starting replication: peer=server-ff61209f-da10-2114-5db9-ef83dfc32b83 + testing.go:149: server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: 2024-06-12T04:32:32.010+1400 [INFO] server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: added peer, starting replication: peer=server-ff61209f-da10-2114-5db9-ef83dfc32b83 +testLoggerAdapter verbose: server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: 2024-06-12T04:32:32.010+1400 [INFO] server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: added peer, starting replication: peer=server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0 + testing.go:149: server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: 2024-06-12T04:32:32.010+1400 [INFO] server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: added peer, starting replication: peer=server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0 +testLoggerAdapter verbose: server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: 2024-06-12T04:32:32.010+1400 [INFO] server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: pipelining replication: peer="{Voter server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0 2b0eb6ae-173a-b016-69ac-0125a01f9cd0}" + testing.go:149: server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: 2024-06-12T04:32:32.010+1400 [INFO] server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: pipelining replication: peer="{Voter server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0 2b0eb6ae-173a-b016-69ac-0125a01f9cd0}" +testLoggerAdapter verbose: server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: 2024-06-12T04:32:32.010+1400 [INFO] server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: pipelining replication: peer="{Voter server-ff61209f-da10-2114-5db9-ef83dfc32b83 ff61209f-da10-2114-5db9-ef83dfc32b83}" + testing.go:149: server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: 2024-06-12T04:32:32.010+1400 [INFO] server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: pipelining replication: peer="{Voter server-ff61209f-da10-2114-5db9-ef83dfc32b83 ff61209f-da10-2114-5db9-ef83dfc32b83}" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:32.116+1400 [INFO] stable state for Follower reached at 2024-06-12 04:32:32.010570386 +1400 +14 m=+8.918088965 (2 nodes), 73.250497ms from start of poll, 74.071067ms from cluster start. Timeout at 2024-06-12 04:32:32.116673905 +1400 +14 m=+9.024192484, 106.103519ms after stability + testing.go:149: cluster: 2024-06-12T04:32:32.116+1400 [INFO] stable state for Follower reached at 2024-06-12 04:32:32.010570386 +1400 +14 m=+8.918088965 (2 nodes), 73.250497ms from start of poll, 74.071067ms from cluster start. Timeout at 2024-06-12 04:32:32.116673905 +1400 +14 m=+9.024192484, 106.103519ms after stability +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:32.116+1400 [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-06-12T04:32:32.116+1400 [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:32.222+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:32.116895737 +1400 +14 m=+9.024414356 (1 nodes), 1.84µs from start of poll, 180.396458ms from cluster start. Timeout at 2024-06-12 04:32:32.222444916 +1400 +14 m=+9.129963495, 105.549139ms after stability + testing.go:149: cluster: 2024-06-12T04:32:32.222+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:32.116895737 +1400 +14 m=+9.024414356 (1 nodes), 1.84µs from start of poll, 180.396458ms from cluster start. Timeout at 2024-06-12 04:32:32.222444916 +1400 +14 m=+9.129963495, 105.549139ms after stability +testLoggerAdapter verbose: server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: 2024-06-12T04:32:32.226+1400 [INFO] server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: updating configuration: command=RemoveServer server-id=server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0 server-addr= servers="[{Suffrage:Voter ID:server-ff61209f-da10-2114-5db9-ef83dfc32b83 Address:ff61209f-da10-2114-5db9-ef83dfc32b83} {Suffrage:Voter ID:server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0 Address:2b0eb6ae-173a-b016-69ac-0125a01f9cd0}]" + testing.go:149: server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: 2024-06-12T04:32:32.226+1400 [INFO] server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: updating configuration: command=RemoveServer server-id=server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0 server-addr= servers="[{Suffrage:Voter ID:server-ff61209f-da10-2114-5db9-ef83dfc32b83 Address:ff61209f-da10-2114-5db9-ef83dfc32b83} {Suffrage:Voter ID:server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0 Address:2b0eb6ae-173a-b016-69ac-0125a01f9cd0}]" +testLoggerAdapter verbose: server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: 2024-06-12T04:32:32.226+1400 [INFO] server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: removed ourself, transitioning to follower + testing.go:149: server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: 2024-06-12T04:32:32.226+1400 [INFO] server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: removed ourself, transitioning to follower +testLoggerAdapter verbose: server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: 2024-06-12T04:32:32.227+1400 [INFO] server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: entering follower state: follower="Node at 3a835b66-4fb4-cfb0-2ed8-f7974de544d0 [Follower]" leader= + testing.go:149: server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: 2024-06-12T04:32:32.227+1400 [INFO] server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: entering follower state: follower="Node at 3a835b66-4fb4-cfb0-2ed8-f7974de544d0 [Follower]" leader= +testLoggerAdapter verbose: server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: 2024-06-12T04:32:32.227+1400 [INFO] server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: aborting pipeline replication: peer="{Voter server-ff61209f-da10-2114-5db9-ef83dfc32b83 ff61209f-da10-2114-5db9-ef83dfc32b83}" + testing.go:149: server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: 2024-06-12T04:32:32.227+1400 [INFO] server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: aborting pipeline replication: peer="{Voter server-ff61209f-da10-2114-5db9-ef83dfc32b83 ff61209f-da10-2114-5db9-ef83dfc32b83}" +testLoggerAdapter verbose: server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: 2024-06-12T04:32:32.227+1400 [INFO] server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: aborting pipeline replication: peer="{Voter server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0 2b0eb6ae-173a-b016-69ac-0125a01f9cd0}" + testing.go:149: server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: 2024-06-12T04:32:32.227+1400 [INFO] server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: aborting pipeline replication: peer="{Voter server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0 2b0eb6ae-173a-b016-69ac-0125a01f9cd0}" +testLoggerAdapter verbose: server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: 2024-06-12T04:32:32.306+1400 [WARN] server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: heartbeat timeout reached, starting election: last-leader=3a835b66-4fb4-cfb0-2ed8-f7974de544d0 + testing.go:149: server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: 2024-06-12T04:32:32.306+1400 [WARN] server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: heartbeat timeout reached, starting election: last-leader=3a835b66-4fb4-cfb0-2ed8-f7974de544d0 +testLoggerAdapter verbose: server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: 2024-06-12T04:32:32.306+1400 [INFO] server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: entering candidate state: node="Node at 2b0eb6ae-173a-b016-69ac-0125a01f9cd0 [Candidate]" term=3 + testing.go:149: server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: 2024-06-12T04:32:32.306+1400 [INFO] server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: entering candidate state: node="Node at 2b0eb6ae-173a-b016-69ac-0125a01f9cd0 [Candidate]" term=3 +testLoggerAdapter verbose: server-ff61209f-da10-2114-5db9-ef83dfc32b83: 2024-06-12T04:32:32.306+1400 [WARN] server-ff61209f-da10-2114-5db9-ef83dfc32b83: rejecting vote request since we have a leader: from=2b0eb6ae-173a-b016-69ac-0125a01f9cd0 leader=3a835b66-4fb4-cfb0-2ed8-f7974de544d0 + testing.go:149: server-ff61209f-da10-2114-5db9-ef83dfc32b83: 2024-06-12T04:32:32.306+1400 [WARN] server-ff61209f-da10-2114-5db9-ef83dfc32b83: rejecting vote request since we have a leader: from=2b0eb6ae-173a-b016-69ac-0125a01f9cd0 leader=3a835b66-4fb4-cfb0-2ed8-f7974de544d0 +testLoggerAdapter verbose: server-ff61209f-da10-2114-5db9-ef83dfc32b83: 2024-06-12T04:32:32.307+1400 [WARN] server-ff61209f-da10-2114-5db9-ef83dfc32b83: heartbeat timeout reached, starting election: last-leader=3a835b66-4fb4-cfb0-2ed8-f7974de544d0 + testing.go:149: server-ff61209f-da10-2114-5db9-ef83dfc32b83: 2024-06-12T04:32:32.307+1400 [WARN] server-ff61209f-da10-2114-5db9-ef83dfc32b83: heartbeat timeout reached, starting election: last-leader=3a835b66-4fb4-cfb0-2ed8-f7974de544d0 +testLoggerAdapter verbose: server-ff61209f-da10-2114-5db9-ef83dfc32b83: 2024-06-12T04:32:32.307+1400 [INFO] server-ff61209f-da10-2114-5db9-ef83dfc32b83: entering candidate state: node="Node at ff61209f-da10-2114-5db9-ef83dfc32b83 [Candidate]" term=3 + testing.go:149: server-ff61209f-da10-2114-5db9-ef83dfc32b83: 2024-06-12T04:32:32.307+1400 [INFO] server-ff61209f-da10-2114-5db9-ef83dfc32b83: entering candidate state: node="Node at ff61209f-da10-2114-5db9-ef83dfc32b83 [Candidate]" term=3 +testLoggerAdapter verbose: server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: 2024-06-12T04:32:32.307+1400 [INFO] server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: duplicate requestVote for same term: term=3 + testing.go:149: server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: 2024-06-12T04:32:32.307+1400 [INFO] server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: duplicate requestVote for same term: term=3 +testLoggerAdapter verbose: server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: 2024-06-12T04:32:32.318+1400 [WARN] server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: not part of stable configuration, aborting election + testing.go:149: server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: 2024-06-12T04:32:32.318+1400 [WARN] server-3a835b66-4fb4-cfb0-2ed8-f7974de544d0: not part of stable configuration, aborting election +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:32.333+1400 [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-06-12T04:32:32.333+1400 [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: 2024-06-12T04:32:32.365+1400 [WARN] server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: Election timeout reached, restarting election + testing.go:149: server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: 2024-06-12T04:32:32.365+1400 [WARN] server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: Election timeout reached, restarting election +testLoggerAdapter verbose: server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: 2024-06-12T04:32:32.365+1400 [INFO] server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: entering candidate state: node="Node at 2b0eb6ae-173a-b016-69ac-0125a01f9cd0 [Candidate]" term=4 + testing.go:149: server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: 2024-06-12T04:32:32.365+1400 [INFO] server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: entering candidate state: node="Node at 2b0eb6ae-173a-b016-69ac-0125a01f9cd0 [Candidate]" term=4 +testLoggerAdapter verbose: server-ff61209f-da10-2114-5db9-ef83dfc32b83: 2024-06-12T04:32:32.365+1400 [INFO] server-ff61209f-da10-2114-5db9-ef83dfc32b83: entering follower state: follower="Node at ff61209f-da10-2114-5db9-ef83dfc32b83 [Follower]" leader= + testing.go:149: server-ff61209f-da10-2114-5db9-ef83dfc32b83: 2024-06-12T04:32:32.365+1400 [INFO] server-ff61209f-da10-2114-5db9-ef83dfc32b83: entering follower state: follower="Node at ff61209f-da10-2114-5db9-ef83dfc32b83 [Follower]" leader= +testLoggerAdapter verbose: server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: 2024-06-12T04:32:32.365+1400 [INFO] server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: election won: tally=2 + testing.go:149: server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: 2024-06-12T04:32:32.365+1400 [INFO] server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: election won: tally=2 +testLoggerAdapter verbose: server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: 2024-06-12T04:32:32.365+1400 [INFO] server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: entering leader state: leader="Node at 2b0eb6ae-173a-b016-69ac-0125a01f9cd0 [Leader]" + testing.go:149: server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: 2024-06-12T04:32:32.365+1400 [INFO] server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: entering leader state: leader="Node at 2b0eb6ae-173a-b016-69ac-0125a01f9cd0 [Leader]" +testLoggerAdapter verbose: server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: 2024-06-12T04:32:32.365+1400 [INFO] server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: added peer, starting replication: peer=server-ff61209f-da10-2114-5db9-ef83dfc32b83 + testing.go:149: server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: 2024-06-12T04:32:32.365+1400 [INFO] server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: added peer, starting replication: peer=server-ff61209f-da10-2114-5db9-ef83dfc32b83 +testLoggerAdapter verbose: server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: 2024-06-12T04:32:32.365+1400 [INFO] server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: pipelining replication: peer="{Voter server-ff61209f-da10-2114-5db9-ef83dfc32b83 ff61209f-da10-2114-5db9-ef83dfc32b83}" + testing.go:149: server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: 2024-06-12T04:32:32.365+1400 [INFO] server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: pipelining replication: peer="{Voter server-ff61209f-da10-2114-5db9-ef83dfc32b83 ff61209f-da10-2114-5db9-ef83dfc32b83}" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:32.471+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:32.365769183 +1400 +14 m=+9.273287762 (1 nodes), 32.547275ms from start of poll, 429.269864ms from cluster start. Timeout at 2024-06-12 04:32:32.471807824 +1400 +14 m=+9.379326403, 106.038641ms after stability + testing.go:149: cluster: 2024-06-12T04:32:32.471+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:32.365769183 +1400 +14 m=+9.273287762 (1 nodes), 32.547275ms from start of poll, 429.269864ms from cluster start. Timeout at 2024-06-12 04:32:32.471807824 +1400 +14 m=+9.379326403, 106.038641ms after stability +testLoggerAdapter verbose: server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: 2024-06-12T04:32:32.577+1400 [INFO] server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: aborting pipeline replication: peer="{Voter server-ff61209f-da10-2114-5db9-ef83dfc32b83 ff61209f-da10-2114-5db9-ef83dfc32b83}" + testing.go:149: server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: 2024-06-12T04:32:32.577+1400 [INFO] server-2b0eb6ae-173a-b016-69ac-0125a01f9cd0: aborting pipeline replication: peer="{Voter server-ff61209f-da10-2114-5db9-ef83dfc32b83 ff61209f-da10-2114-5db9-ef83dfc32b83}" +--- PASS: TestRaft_RemoveLeader_NoShutdown (0.64s) === RUN TestRaft_RemoveFollower_SplitCluster -testLoggerAdapter verbose: server-cf43a5f5-167b-29a8-0184-d6d99ec41d1b: 2024-06-11T02:29:19.191-1200 [INFO] server-cf43a5f5-167b-29a8-0184-d6d99ec41d1b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cf43a5f5-167b-29a8-0184-d6d99ec41d1b Address:cf43a5f5-167b-29a8-0184-d6d99ec41d1b} {Suffrage:Voter ID:server-baac88c8-0da7-eed7-36f1-c3172d433a09 Address:baac88c8-0da7-eed7-36f1-c3172d433a09} {Suffrage:Voter ID:server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd Address:fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd} {Suffrage:Voter ID:server-363ccd9f-d2bd-2d64-b8d9-06082d00b985 Address:363ccd9f-d2bd-2d64-b8d9-06082d00b985}]" - testing.go:149: server-cf43a5f5-167b-29a8-0184-d6d99ec41d1b: 2024-06-11T02:29:19.191-1200 [INFO] server-cf43a5f5-167b-29a8-0184-d6d99ec41d1b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cf43a5f5-167b-29a8-0184-d6d99ec41d1b Address:cf43a5f5-167b-29a8-0184-d6d99ec41d1b} {Suffrage:Voter ID:server-baac88c8-0da7-eed7-36f1-c3172d433a09 Address:baac88c8-0da7-eed7-36f1-c3172d433a09} {Suffrage:Voter ID:server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd Address:fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd} {Suffrage:Voter ID:server-363ccd9f-d2bd-2d64-b8d9-06082d00b985 Address:363ccd9f-d2bd-2d64-b8d9-06082d00b985}]" -testLoggerAdapter verbose: server-cf43a5f5-167b-29a8-0184-d6d99ec41d1b: 2024-06-11T02:29:19.191-1200 [INFO] server-cf43a5f5-167b-29a8-0184-d6d99ec41d1b: entering follower state: follower="Node at cf43a5f5-167b-29a8-0184-d6d99ec41d1b [Follower]" leader= - testing.go:149: server-cf43a5f5-167b-29a8-0184-d6d99ec41d1b: 2024-06-11T02:29:19.191-1200 [INFO] server-cf43a5f5-167b-29a8-0184-d6d99ec41d1b: entering follower state: follower="Node at cf43a5f5-167b-29a8-0184-d6d99ec41d1b [Follower]" leader= -testLoggerAdapter verbose: server-baac88c8-0da7-eed7-36f1-c3172d433a09: 2024-06-11T02:29:19.191-1200 [INFO] server-baac88c8-0da7-eed7-36f1-c3172d433a09: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cf43a5f5-167b-29a8-0184-d6d99ec41d1b Address:cf43a5f5-167b-29a8-0184-d6d99ec41d1b} {Suffrage:Voter ID:server-baac88c8-0da7-eed7-36f1-c3172d433a09 Address:baac88c8-0da7-eed7-36f1-c3172d433a09} {Suffrage:Voter ID:server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd Address:fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd} {Suffrage:Voter ID:server-363ccd9f-d2bd-2d64-b8d9-06082d00b985 Address:363ccd9f-d2bd-2d64-b8d9-06082d00b985}]" - testing.go:149: server-baac88c8-0da7-eed7-36f1-c3172d433a09: 2024-06-11T02:29:19.191-1200 [INFO] server-baac88c8-0da7-eed7-36f1-c3172d433a09: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cf43a5f5-167b-29a8-0184-d6d99ec41d1b Address:cf43a5f5-167b-29a8-0184-d6d99ec41d1b} {Suffrage:Voter ID:server-baac88c8-0da7-eed7-36f1-c3172d433a09 Address:baac88c8-0da7-eed7-36f1-c3172d433a09} {Suffrage:Voter ID:server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd Address:fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd} {Suffrage:Voter ID:server-363ccd9f-d2bd-2d64-b8d9-06082d00b985 Address:363ccd9f-d2bd-2d64-b8d9-06082d00b985}]" -testLoggerAdapter verbose: server-baac88c8-0da7-eed7-36f1-c3172d433a09: 2024-06-11T02:29:19.192-1200 [INFO] server-baac88c8-0da7-eed7-36f1-c3172d433a09: entering follower state: follower="Node at baac88c8-0da7-eed7-36f1-c3172d433a09 [Follower]" leader= - testing.go:149: server-baac88c8-0da7-eed7-36f1-c3172d433a09: 2024-06-11T02:29:19.192-1200 [INFO] server-baac88c8-0da7-eed7-36f1-c3172d433a09: entering follower state: follower="Node at baac88c8-0da7-eed7-36f1-c3172d433a09 [Follower]" leader= -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.192-1200 [INFO] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cf43a5f5-167b-29a8-0184-d6d99ec41d1b Address:cf43a5f5-167b-29a8-0184-d6d99ec41d1b} {Suffrage:Voter ID:server-baac88c8-0da7-eed7-36f1-c3172d433a09 Address:baac88c8-0da7-eed7-36f1-c3172d433a09} {Suffrage:Voter ID:server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd Address:fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd} {Suffrage:Voter ID:server-363ccd9f-d2bd-2d64-b8d9-06082d00b985 Address:363ccd9f-d2bd-2d64-b8d9-06082d00b985}]" - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.192-1200 [INFO] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cf43a5f5-167b-29a8-0184-d6d99ec41d1b Address:cf43a5f5-167b-29a8-0184-d6d99ec41d1b} {Suffrage:Voter ID:server-baac88c8-0da7-eed7-36f1-c3172d433a09 Address:baac88c8-0da7-eed7-36f1-c3172d433a09} {Suffrage:Voter ID:server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd Address:fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd} {Suffrage:Voter ID:server-363ccd9f-d2bd-2d64-b8d9-06082d00b985 Address:363ccd9f-d2bd-2d64-b8d9-06082d00b985}]" -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.192-1200 [INFO] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: entering follower state: follower="Node at fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd [Follower]" leader= - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.192-1200 [INFO] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: entering follower state: follower="Node at fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd [Follower]" leader= -testLoggerAdapter verbose: server-363ccd9f-d2bd-2d64-b8d9-06082d00b985: 2024-06-11T02:29:19.192-1200 [INFO] server-363ccd9f-d2bd-2d64-b8d9-06082d00b985: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cf43a5f5-167b-29a8-0184-d6d99ec41d1b Address:cf43a5f5-167b-29a8-0184-d6d99ec41d1b} {Suffrage:Voter ID:server-baac88c8-0da7-eed7-36f1-c3172d433a09 Address:baac88c8-0da7-eed7-36f1-c3172d433a09} {Suffrage:Voter ID:server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd Address:fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd} {Suffrage:Voter ID:server-363ccd9f-d2bd-2d64-b8d9-06082d00b985 Address:363ccd9f-d2bd-2d64-b8d9-06082d00b985}]" - testing.go:149: server-363ccd9f-d2bd-2d64-b8d9-06082d00b985: 2024-06-11T02:29:19.192-1200 [INFO] server-363ccd9f-d2bd-2d64-b8d9-06082d00b985: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cf43a5f5-167b-29a8-0184-d6d99ec41d1b Address:cf43a5f5-167b-29a8-0184-d6d99ec41d1b} {Suffrage:Voter ID:server-baac88c8-0da7-eed7-36f1-c3172d433a09 Address:baac88c8-0da7-eed7-36f1-c3172d433a09} {Suffrage:Voter ID:server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd Address:fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd} {Suffrage:Voter ID:server-363ccd9f-d2bd-2d64-b8d9-06082d00b985 Address:363ccd9f-d2bd-2d64-b8d9-06082d00b985}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:19.192-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:19.192-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-363ccd9f-d2bd-2d64-b8d9-06082d00b985: 2024-06-11T02:29:19.192-1200 [INFO] server-363ccd9f-d2bd-2d64-b8d9-06082d00b985: entering follower state: follower="Node at 363ccd9f-d2bd-2d64-b8d9-06082d00b985 [Follower]" leader= - testing.go:149: server-363ccd9f-d2bd-2d64-b8d9-06082d00b985: 2024-06-11T02:29:19.192-1200 [INFO] server-363ccd9f-d2bd-2d64-b8d9-06082d00b985: entering follower state: follower="Node at 363ccd9f-d2bd-2d64-b8d9-06082d00b985 [Follower]" leader= -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.245-1200 [WARN] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.245-1200 [WARN] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.245-1200 [INFO] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: entering candidate state: node="Node at fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd [Candidate]" term=2 - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.245-1200 [INFO] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: entering candidate state: node="Node at fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd [Candidate]" term=2 -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.245-1200 [INFO] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: election won: tally=3 - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.245-1200 [INFO] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: election won: tally=3 -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.246-1200 [INFO] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: entering leader state: leader="Node at fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd [Leader]" - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.246-1200 [INFO] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: entering leader state: leader="Node at fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd [Leader]" -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.246-1200 [INFO] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: added peer, starting replication: peer=server-cf43a5f5-167b-29a8-0184-d6d99ec41d1b - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.246-1200 [INFO] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: added peer, starting replication: peer=server-cf43a5f5-167b-29a8-0184-d6d99ec41d1b -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.246-1200 [INFO] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: added peer, starting replication: peer=server-baac88c8-0da7-eed7-36f1-c3172d433a09 - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.246-1200 [INFO] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: added peer, starting replication: peer=server-baac88c8-0da7-eed7-36f1-c3172d433a09 -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.246-1200 [INFO] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: added peer, starting replication: peer=server-363ccd9f-d2bd-2d64-b8d9-06082d00b985 - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.246-1200 [INFO] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: added peer, starting replication: peer=server-363ccd9f-d2bd-2d64-b8d9-06082d00b985 -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.246-1200 [INFO] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: pipelining replication: peer="{Voter server-baac88c8-0da7-eed7-36f1-c3172d433a09 baac88c8-0da7-eed7-36f1-c3172d433a09}" - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.246-1200 [INFO] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: pipelining replication: peer="{Voter server-baac88c8-0da7-eed7-36f1-c3172d433a09 baac88c8-0da7-eed7-36f1-c3172d433a09}" -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.246-1200 [INFO] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: pipelining replication: peer="{Voter server-363ccd9f-d2bd-2d64-b8d9-06082d00b985 363ccd9f-d2bd-2d64-b8d9-06082d00b985}" - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.246-1200 [INFO] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: pipelining replication: peer="{Voter server-363ccd9f-d2bd-2d64-b8d9-06082d00b985 363ccd9f-d2bd-2d64-b8d9-06082d00b985}" -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.246-1200 [INFO] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: pipelining replication: peer="{Voter server-cf43a5f5-167b-29a8-0184-d6d99ec41d1b cf43a5f5-167b-29a8-0184-d6d99ec41d1b}" - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.246-1200 [INFO] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: pipelining replication: peer="{Voter server-cf43a5f5-167b-29a8-0184-d6d99ec41d1b cf43a5f5-167b-29a8-0184-d6d99ec41d1b}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:19.351-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:19.246137126 -1200 -12 m=+10.241509668 (1 nodes), 53.283289ms from start of poll, 55.216419ms from cluster start. Timeout at 2024-06-11 02:29:19.351171317 -1200 -12 m=+10.346543859, 105.034191ms after stability - testing.go:149: cluster: 2024-06-11T02:29:19.351-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:19.246137126 -1200 -12 m=+10.241509668 (1 nodes), 53.283289ms from start of poll, 55.216419ms from cluster start. Timeout at 2024-06-11 02:29:19.351171317 -1200 -12 m=+10.346543859, 105.034191ms after stability -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:19.457-1200 [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2024-06-11T02:29:19.457-1200 [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:19.563-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:19.457263655 -1200 -12 m=+10.452636196 (3 nodes), 4.666µs from start of poll, 266.342947ms from cluster start. Timeout at 2024-06-11 02:29:19.56328541 -1200 -12 m=+10.558657952, 106.021756ms after stability - testing.go:149: cluster: 2024-06-11T02:29:19.563-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:19.457263655 -1200 -12 m=+10.452636196 (3 nodes), 4.666µs from start of poll, 266.342947ms from cluster start. Timeout at 2024-06-11 02:29:19.56328541 -1200 -12 m=+10.558657952, 106.021756ms after stability -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.563-1200 [INFO] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: updating configuration: command=RemoveServer server-id=server-363ccd9f-d2bd-2d64-b8d9-06082d00b985 server-addr= servers="[{Suffrage:Voter ID:server-cf43a5f5-167b-29a8-0184-d6d99ec41d1b Address:cf43a5f5-167b-29a8-0184-d6d99ec41d1b} {Suffrage:Voter ID:server-baac88c8-0da7-eed7-36f1-c3172d433a09 Address:baac88c8-0da7-eed7-36f1-c3172d433a09} {Suffrage:Voter ID:server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd Address:fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd}]" - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.563-1200 [INFO] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: updating configuration: command=RemoveServer server-id=server-363ccd9f-d2bd-2d64-b8d9-06082d00b985 server-addr= servers="[{Suffrage:Voter ID:server-cf43a5f5-167b-29a8-0184-d6d99ec41d1b Address:cf43a5f5-167b-29a8-0184-d6d99ec41d1b} {Suffrage:Voter ID:server-baac88c8-0da7-eed7-36f1-c3172d433a09 Address:baac88c8-0da7-eed7-36f1-c3172d433a09} {Suffrage:Voter ID:server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd Address:fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd}]" -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.563-1200 [INFO] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: removed peer, stopping replication: peer=server-363ccd9f-d2bd-2d64-b8d9-06082d00b985 last-index=3 - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.563-1200 [INFO] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: removed peer, stopping replication: peer=server-363ccd9f-d2bd-2d64-b8d9-06082d00b985 last-index=3 -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.563-1200 [ERROR] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: failed to pipeline appendEntries: peer="{Voter server-cf43a5f5-167b-29a8-0184-d6d99ec41d1b cf43a5f5-167b-29a8-0184-d6d99ec41d1b}" error="append pipeline closed" - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.563-1200 [ERROR] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: failed to pipeline appendEntries: peer="{Voter server-cf43a5f5-167b-29a8-0184-d6d99ec41d1b cf43a5f5-167b-29a8-0184-d6d99ec41d1b}" error="append pipeline closed" -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.563-1200 [INFO] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: aborting pipeline replication: peer="{Voter server-cf43a5f5-167b-29a8-0184-d6d99ec41d1b cf43a5f5-167b-29a8-0184-d6d99ec41d1b}" - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.563-1200 [INFO] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: aborting pipeline replication: peer="{Voter server-cf43a5f5-167b-29a8-0184-d6d99ec41d1b cf43a5f5-167b-29a8-0184-d6d99ec41d1b}" -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.564-1200 [ERROR] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: failed to pipeline appendEntries: peer="{Voter server-baac88c8-0da7-eed7-36f1-c3172d433a09 baac88c8-0da7-eed7-36f1-c3172d433a09}" error="append pipeline closed" - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.564-1200 [ERROR] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: failed to pipeline appendEntries: peer="{Voter server-baac88c8-0da7-eed7-36f1-c3172d433a09 baac88c8-0da7-eed7-36f1-c3172d433a09}" error="append pipeline closed" -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.564-1200 [INFO] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: aborting pipeline replication: peer="{Voter server-baac88c8-0da7-eed7-36f1-c3172d433a09 baac88c8-0da7-eed7-36f1-c3172d433a09}" - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.564-1200 [INFO] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: aborting pipeline replication: peer="{Voter server-baac88c8-0da7-eed7-36f1-c3172d433a09 baac88c8-0da7-eed7-36f1-c3172d433a09}" -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.564-1200 [INFO] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: aborting pipeline replication: peer="{Voter server-363ccd9f-d2bd-2d64-b8d9-06082d00b985 363ccd9f-d2bd-2d64-b8d9-06082d00b985}" - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.564-1200 [INFO] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: aborting pipeline replication: peer="{Voter server-363ccd9f-d2bd-2d64-b8d9-06082d00b985 363ccd9f-d2bd-2d64-b8d9-06082d00b985}" -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.568-1200 [ERROR] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: failed to heartbeat to: peer=cf43a5f5-167b-29a8-0184-d6d99ec41d1b error="failed to connect to peer: cf43a5f5-167b-29a8-0184-d6d99ec41d1b" - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.568-1200 [ERROR] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: failed to heartbeat to: peer=cf43a5f5-167b-29a8-0184-d6d99ec41d1b error="failed to connect to peer: cf43a5f5-167b-29a8-0184-d6d99ec41d1b" -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.569-1200 [ERROR] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: failed to heartbeat to: peer=baac88c8-0da7-eed7-36f1-c3172d433a09 error="failed to connect to peer: baac88c8-0da7-eed7-36f1-c3172d433a09" - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.569-1200 [ERROR] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: failed to heartbeat to: peer=baac88c8-0da7-eed7-36f1-c3172d433a09 error="failed to connect to peer: baac88c8-0da7-eed7-36f1-c3172d433a09" -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.569-1200 [ERROR] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: failed to appendEntries to: peer="{Voter server-baac88c8-0da7-eed7-36f1-c3172d433a09 baac88c8-0da7-eed7-36f1-c3172d433a09}" error="failed to connect to peer: baac88c8-0da7-eed7-36f1-c3172d433a09" - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.569-1200 [ERROR] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: failed to appendEntries to: peer="{Voter server-baac88c8-0da7-eed7-36f1-c3172d433a09 baac88c8-0da7-eed7-36f1-c3172d433a09}" error="failed to connect to peer: baac88c8-0da7-eed7-36f1-c3172d433a09" -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.572-1200 [ERROR] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: failed to appendEntries to: peer="{Voter server-cf43a5f5-167b-29a8-0184-d6d99ec41d1b cf43a5f5-167b-29a8-0184-d6d99ec41d1b}" error="failed to connect to peer: cf43a5f5-167b-29a8-0184-d6d99ec41d1b" - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.572-1200 [ERROR] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: failed to appendEntries to: peer="{Voter server-cf43a5f5-167b-29a8-0184-d6d99ec41d1b cf43a5f5-167b-29a8-0184-d6d99ec41d1b}" error="failed to connect to peer: cf43a5f5-167b-29a8-0184-d6d99ec41d1b" -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.588-1200 [ERROR] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: failed to heartbeat to: peer=cf43a5f5-167b-29a8-0184-d6d99ec41d1b error="failed to connect to peer: cf43a5f5-167b-29a8-0184-d6d99ec41d1b" - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.588-1200 [ERROR] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: failed to heartbeat to: peer=cf43a5f5-167b-29a8-0184-d6d99ec41d1b error="failed to connect to peer: cf43a5f5-167b-29a8-0184-d6d99ec41d1b" -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.589-1200 [ERROR] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: failed to appendEntries to: peer="{Voter server-baac88c8-0da7-eed7-36f1-c3172d433a09 baac88c8-0da7-eed7-36f1-c3172d433a09}" error="failed to connect to peer: baac88c8-0da7-eed7-36f1-c3172d433a09" - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.589-1200 [ERROR] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: failed to appendEntries to: peer="{Voter server-baac88c8-0da7-eed7-36f1-c3172d433a09 baac88c8-0da7-eed7-36f1-c3172d433a09}" error="failed to connect to peer: baac88c8-0da7-eed7-36f1-c3172d433a09" -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.589-1200 [ERROR] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: failed to heartbeat to: peer=baac88c8-0da7-eed7-36f1-c3172d433a09 error="failed to connect to peer: baac88c8-0da7-eed7-36f1-c3172d433a09" - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.589-1200 [ERROR] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: failed to heartbeat to: peer=baac88c8-0da7-eed7-36f1-c3172d433a09 error="failed to connect to peer: baac88c8-0da7-eed7-36f1-c3172d433a09" -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.589-1200 [ERROR] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: failed to appendEntries to: peer="{Voter server-cf43a5f5-167b-29a8-0184-d6d99ec41d1b cf43a5f5-167b-29a8-0184-d6d99ec41d1b}" error="failed to connect to peer: cf43a5f5-167b-29a8-0184-d6d99ec41d1b" - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.589-1200 [ERROR] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: failed to appendEntries to: peer="{Voter server-cf43a5f5-167b-29a8-0184-d6d99ec41d1b cf43a5f5-167b-29a8-0184-d6d99ec41d1b}" error="failed to connect to peer: cf43a5f5-167b-29a8-0184-d6d99ec41d1b" -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.607-1200 [ERROR] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: failed to appendEntries to: peer="{Voter server-baac88c8-0da7-eed7-36f1-c3172d433a09 baac88c8-0da7-eed7-36f1-c3172d433a09}" error="failed to connect to peer: baac88c8-0da7-eed7-36f1-c3172d433a09" - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.607-1200 [ERROR] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: failed to appendEntries to: peer="{Voter server-baac88c8-0da7-eed7-36f1-c3172d433a09 baac88c8-0da7-eed7-36f1-c3172d433a09}" error="failed to connect to peer: baac88c8-0da7-eed7-36f1-c3172d433a09" -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.607-1200 [ERROR] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: failed to heartbeat to: peer=cf43a5f5-167b-29a8-0184-d6d99ec41d1b error="failed to connect to peer: cf43a5f5-167b-29a8-0184-d6d99ec41d1b" - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.607-1200 [ERROR] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: failed to heartbeat to: peer=cf43a5f5-167b-29a8-0184-d6d99ec41d1b error="failed to connect to peer: cf43a5f5-167b-29a8-0184-d6d99ec41d1b" -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.609-1200 [ERROR] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: failed to appendEntries to: peer="{Voter server-cf43a5f5-167b-29a8-0184-d6d99ec41d1b cf43a5f5-167b-29a8-0184-d6d99ec41d1b}" error="failed to connect to peer: cf43a5f5-167b-29a8-0184-d6d99ec41d1b" - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.609-1200 [ERROR] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: failed to appendEntries to: peer="{Voter server-cf43a5f5-167b-29a8-0184-d6d99ec41d1b cf43a5f5-167b-29a8-0184-d6d99ec41d1b}" error="failed to connect to peer: cf43a5f5-167b-29a8-0184-d6d99ec41d1b" -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.609-1200 [ERROR] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: failed to heartbeat to: peer=baac88c8-0da7-eed7-36f1-c3172d433a09 error="failed to connect to peer: baac88c8-0da7-eed7-36f1-c3172d433a09" - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.609-1200 [ERROR] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: failed to heartbeat to: peer=baac88c8-0da7-eed7-36f1-c3172d433a09 error="failed to connect to peer: baac88c8-0da7-eed7-36f1-c3172d433a09" -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.612-1200 [WARN] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: failed to contact: server-id=server-cf43a5f5-167b-29a8-0184-d6d99ec41d1b time=50.972166ms - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.612-1200 [WARN] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: failed to contact: server-id=server-cf43a5f5-167b-29a8-0184-d6d99ec41d1b time=50.972166ms -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.613-1200 [WARN] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: failed to contact: server-id=server-baac88c8-0da7-eed7-36f1-c3172d433a09 time=51.057039ms - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.613-1200 [WARN] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: failed to contact: server-id=server-baac88c8-0da7-eed7-36f1-c3172d433a09 time=51.057039ms -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.613-1200 [WARN] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: failed to contact quorum of nodes, stepping down - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.613-1200 [WARN] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: failed to contact quorum of nodes, stepping down -testLoggerAdapter verbose: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.613-1200 [INFO] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: entering follower state: follower="Node at fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd [Follower]" leader= - testing.go:149: server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: 2024-06-11T02:29:19.613-1200 [INFO] server-fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd: entering follower state: follower="Node at fe86bf2f-c97f-8d6e-9e75-3f4e2d1c4cdd [Follower]" leader= ---- PASS: TestRaft_RemoveFollower_SplitCluster (0.43s) +testLoggerAdapter verbose: server-3752d97e-4476-231f-036e-426ffa2b8cc4: 2024-06-12T04:32:32.580+1400 [INFO] server-3752d97e-4476-231f-036e-426ffa2b8cc4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3752d97e-4476-231f-036e-426ffa2b8cc4 Address:3752d97e-4476-231f-036e-426ffa2b8cc4} {Suffrage:Voter ID:server-35b87fe5-0871-752e-2f20-244b7e1cc588 Address:35b87fe5-0871-752e-2f20-244b7e1cc588} {Suffrage:Voter ID:server-b715d6b2-772a-4683-e9c2-327ca0a06b02 Address:b715d6b2-772a-4683-e9c2-327ca0a06b02} {Suffrage:Voter ID:server-bc730afa-d518-0d9f-9227-85bdb50cda7a Address:bc730afa-d518-0d9f-9227-85bdb50cda7a}]" + testing.go:149: server-3752d97e-4476-231f-036e-426ffa2b8cc4: 2024-06-12T04:32:32.580+1400 [INFO] server-3752d97e-4476-231f-036e-426ffa2b8cc4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3752d97e-4476-231f-036e-426ffa2b8cc4 Address:3752d97e-4476-231f-036e-426ffa2b8cc4} {Suffrage:Voter ID:server-35b87fe5-0871-752e-2f20-244b7e1cc588 Address:35b87fe5-0871-752e-2f20-244b7e1cc588} {Suffrage:Voter ID:server-b715d6b2-772a-4683-e9c2-327ca0a06b02 Address:b715d6b2-772a-4683-e9c2-327ca0a06b02} {Suffrage:Voter ID:server-bc730afa-d518-0d9f-9227-85bdb50cda7a Address:bc730afa-d518-0d9f-9227-85bdb50cda7a}]" +testLoggerAdapter verbose: server-35b87fe5-0871-752e-2f20-244b7e1cc588: 2024-06-12T04:32:32.580+1400 [INFO] server-35b87fe5-0871-752e-2f20-244b7e1cc588: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3752d97e-4476-231f-036e-426ffa2b8cc4 Address:3752d97e-4476-231f-036e-426ffa2b8cc4} {Suffrage:Voter ID:server-35b87fe5-0871-752e-2f20-244b7e1cc588 Address:35b87fe5-0871-752e-2f20-244b7e1cc588} {Suffrage:Voter ID:server-b715d6b2-772a-4683-e9c2-327ca0a06b02 Address:b715d6b2-772a-4683-e9c2-327ca0a06b02} {Suffrage:Voter ID:server-bc730afa-d518-0d9f-9227-85bdb50cda7a Address:bc730afa-d518-0d9f-9227-85bdb50cda7a}]" + testing.go:149: server-35b87fe5-0871-752e-2f20-244b7e1cc588: 2024-06-12T04:32:32.580+1400 [INFO] server-35b87fe5-0871-752e-2f20-244b7e1cc588: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3752d97e-4476-231f-036e-426ffa2b8cc4 Address:3752d97e-4476-231f-036e-426ffa2b8cc4} {Suffrage:Voter ID:server-35b87fe5-0871-752e-2f20-244b7e1cc588 Address:35b87fe5-0871-752e-2f20-244b7e1cc588} {Suffrage:Voter ID:server-b715d6b2-772a-4683-e9c2-327ca0a06b02 Address:b715d6b2-772a-4683-e9c2-327ca0a06b02} {Suffrage:Voter ID:server-bc730afa-d518-0d9f-9227-85bdb50cda7a Address:bc730afa-d518-0d9f-9227-85bdb50cda7a}]" +testLoggerAdapter verbose: server-b715d6b2-772a-4683-e9c2-327ca0a06b02: 2024-06-12T04:32:32.580+1400 [INFO] server-b715d6b2-772a-4683-e9c2-327ca0a06b02: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3752d97e-4476-231f-036e-426ffa2b8cc4 Address:3752d97e-4476-231f-036e-426ffa2b8cc4} {Suffrage:Voter ID:server-35b87fe5-0871-752e-2f20-244b7e1cc588 Address:35b87fe5-0871-752e-2f20-244b7e1cc588} {Suffrage:Voter ID:server-b715d6b2-772a-4683-e9c2-327ca0a06b02 Address:b715d6b2-772a-4683-e9c2-327ca0a06b02} {Suffrage:Voter ID:server-bc730afa-d518-0d9f-9227-85bdb50cda7a Address:bc730afa-d518-0d9f-9227-85bdb50cda7a}]" + testing.go:149: server-b715d6b2-772a-4683-e9c2-327ca0a06b02: 2024-06-12T04:32:32.580+1400 [INFO] server-b715d6b2-772a-4683-e9c2-327ca0a06b02: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3752d97e-4476-231f-036e-426ffa2b8cc4 Address:3752d97e-4476-231f-036e-426ffa2b8cc4} {Suffrage:Voter ID:server-35b87fe5-0871-752e-2f20-244b7e1cc588 Address:35b87fe5-0871-752e-2f20-244b7e1cc588} {Suffrage:Voter ID:server-b715d6b2-772a-4683-e9c2-327ca0a06b02 Address:b715d6b2-772a-4683-e9c2-327ca0a06b02} {Suffrage:Voter ID:server-bc730afa-d518-0d9f-9227-85bdb50cda7a Address:bc730afa-d518-0d9f-9227-85bdb50cda7a}]" +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.581+1400 [INFO] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3752d97e-4476-231f-036e-426ffa2b8cc4 Address:3752d97e-4476-231f-036e-426ffa2b8cc4} {Suffrage:Voter ID:server-35b87fe5-0871-752e-2f20-244b7e1cc588 Address:35b87fe5-0871-752e-2f20-244b7e1cc588} {Suffrage:Voter ID:server-b715d6b2-772a-4683-e9c2-327ca0a06b02 Address:b715d6b2-772a-4683-e9c2-327ca0a06b02} {Suffrage:Voter ID:server-bc730afa-d518-0d9f-9227-85bdb50cda7a Address:bc730afa-d518-0d9f-9227-85bdb50cda7a}]" + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.581+1400 [INFO] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3752d97e-4476-231f-036e-426ffa2b8cc4 Address:3752d97e-4476-231f-036e-426ffa2b8cc4} {Suffrage:Voter ID:server-35b87fe5-0871-752e-2f20-244b7e1cc588 Address:35b87fe5-0871-752e-2f20-244b7e1cc588} {Suffrage:Voter ID:server-b715d6b2-772a-4683-e9c2-327ca0a06b02 Address:b715d6b2-772a-4683-e9c2-327ca0a06b02} {Suffrage:Voter ID:server-bc730afa-d518-0d9f-9227-85bdb50cda7a Address:bc730afa-d518-0d9f-9227-85bdb50cda7a}]" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:32.581+1400 [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-06-12T04:32:32.581+1400 [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-3752d97e-4476-231f-036e-426ffa2b8cc4: 2024-06-12T04:32:32.581+1400 [INFO] server-3752d97e-4476-231f-036e-426ffa2b8cc4: entering follower state: follower="Node at 3752d97e-4476-231f-036e-426ffa2b8cc4 [Follower]" leader= + testing.go:149: server-3752d97e-4476-231f-036e-426ffa2b8cc4: 2024-06-12T04:32:32.581+1400 [INFO] server-3752d97e-4476-231f-036e-426ffa2b8cc4: entering follower state: follower="Node at 3752d97e-4476-231f-036e-426ffa2b8cc4 [Follower]" leader= +testLoggerAdapter verbose: server-35b87fe5-0871-752e-2f20-244b7e1cc588: 2024-06-12T04:32:32.581+1400 [INFO] server-35b87fe5-0871-752e-2f20-244b7e1cc588: entering follower state: follower="Node at 35b87fe5-0871-752e-2f20-244b7e1cc588 [Follower]" leader= + testing.go:149: server-35b87fe5-0871-752e-2f20-244b7e1cc588: 2024-06-12T04:32:32.581+1400 [INFO] server-35b87fe5-0871-752e-2f20-244b7e1cc588: entering follower state: follower="Node at 35b87fe5-0871-752e-2f20-244b7e1cc588 [Follower]" leader= +testLoggerAdapter verbose: server-b715d6b2-772a-4683-e9c2-327ca0a06b02: 2024-06-12T04:32:32.581+1400 [INFO] server-b715d6b2-772a-4683-e9c2-327ca0a06b02: entering follower state: follower="Node at b715d6b2-772a-4683-e9c2-327ca0a06b02 [Follower]" leader= + testing.go:149: server-b715d6b2-772a-4683-e9c2-327ca0a06b02: 2024-06-12T04:32:32.581+1400 [INFO] server-b715d6b2-772a-4683-e9c2-327ca0a06b02: entering follower state: follower="Node at b715d6b2-772a-4683-e9c2-327ca0a06b02 [Follower]" leader= +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.581+1400 [INFO] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: entering follower state: follower="Node at bc730afa-d518-0d9f-9227-85bdb50cda7a [Follower]" leader= + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.581+1400 [INFO] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: entering follower state: follower="Node at bc730afa-d518-0d9f-9227-85bdb50cda7a [Follower]" leader= +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.636+1400 [WARN] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.636+1400 [WARN] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.636+1400 [INFO] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: entering candidate state: node="Node at bc730afa-d518-0d9f-9227-85bdb50cda7a [Candidate]" term=2 + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.636+1400 [INFO] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: entering candidate state: node="Node at bc730afa-d518-0d9f-9227-85bdb50cda7a [Candidate]" term=2 +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.636+1400 [INFO] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: election won: tally=3 + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.636+1400 [INFO] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: election won: tally=3 +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.636+1400 [INFO] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: entering leader state: leader="Node at bc730afa-d518-0d9f-9227-85bdb50cda7a [Leader]" + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.636+1400 [INFO] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: entering leader state: leader="Node at bc730afa-d518-0d9f-9227-85bdb50cda7a [Leader]" +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.636+1400 [INFO] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: added peer, starting replication: peer=server-3752d97e-4476-231f-036e-426ffa2b8cc4 + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.636+1400 [INFO] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: added peer, starting replication: peer=server-3752d97e-4476-231f-036e-426ffa2b8cc4 +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.636+1400 [INFO] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: added peer, starting replication: peer=server-35b87fe5-0871-752e-2f20-244b7e1cc588 + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.636+1400 [INFO] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: added peer, starting replication: peer=server-35b87fe5-0871-752e-2f20-244b7e1cc588 +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.636+1400 [INFO] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: added peer, starting replication: peer=server-b715d6b2-772a-4683-e9c2-327ca0a06b02 + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.636+1400 [INFO] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: added peer, starting replication: peer=server-b715d6b2-772a-4683-e9c2-327ca0a06b02 +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.636+1400 [INFO] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: pipelining replication: peer="{Voter server-b715d6b2-772a-4683-e9c2-327ca0a06b02 b715d6b2-772a-4683-e9c2-327ca0a06b02}" + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.636+1400 [INFO] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: pipelining replication: peer="{Voter server-b715d6b2-772a-4683-e9c2-327ca0a06b02 b715d6b2-772a-4683-e9c2-327ca0a06b02}" +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.636+1400 [INFO] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: pipelining replication: peer="{Voter server-3752d97e-4476-231f-036e-426ffa2b8cc4 3752d97e-4476-231f-036e-426ffa2b8cc4}" + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.636+1400 [INFO] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: pipelining replication: peer="{Voter server-3752d97e-4476-231f-036e-426ffa2b8cc4 3752d97e-4476-231f-036e-426ffa2b8cc4}" +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.637+1400 [INFO] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: pipelining replication: peer="{Voter server-35b87fe5-0871-752e-2f20-244b7e1cc588 35b87fe5-0871-752e-2f20-244b7e1cc588}" + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.637+1400 [INFO] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: pipelining replication: peer="{Voter server-35b87fe5-0871-752e-2f20-244b7e1cc588 35b87fe5-0871-752e-2f20-244b7e1cc588}" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:32.742+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:32.636950669 +1400 +14 m=+9.544469248 (1 nodes), 55.670046ms from start of poll, 56.596133ms from cluster start. Timeout at 2024-06-12 04:32:32.742258257 +1400 +14 m=+9.649776876, 105.307628ms after stability + testing.go:149: cluster: 2024-06-12T04:32:32.742+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:32.636950669 +1400 +14 m=+9.544469248 (1 nodes), 55.670046ms from start of poll, 56.596133ms from cluster start. Timeout at 2024-06-12 04:32:32.742258257 +1400 +14 m=+9.649776876, 105.307628ms after stability +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:32.847+1400 [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-06-12T04:32:32.847+1400 [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:32.952+1400 [INFO] stable state for Follower reached at 2024-06-12 04:32:32.847783117 +1400 +14 m=+9.755301736 (3 nodes), 2.96µs from start of poll, 267.428621ms from cluster start. Timeout at 2024-06-12 04:32:32.952837514 +1400 +14 m=+9.860356133, 105.054397ms after stability + testing.go:149: cluster: 2024-06-12T04:32:32.952+1400 [INFO] stable state for Follower reached at 2024-06-12 04:32:32.847783117 +1400 +14 m=+9.755301736 (3 nodes), 2.96µs from start of poll, 267.428621ms from cluster start. Timeout at 2024-06-12 04:32:32.952837514 +1400 +14 m=+9.860356133, 105.054397ms after stability +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.953+1400 [INFO] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: updating configuration: command=RemoveServer server-id=server-b715d6b2-772a-4683-e9c2-327ca0a06b02 server-addr= servers="[{Suffrage:Voter ID:server-3752d97e-4476-231f-036e-426ffa2b8cc4 Address:3752d97e-4476-231f-036e-426ffa2b8cc4} {Suffrage:Voter ID:server-35b87fe5-0871-752e-2f20-244b7e1cc588 Address:35b87fe5-0871-752e-2f20-244b7e1cc588} {Suffrage:Voter ID:server-bc730afa-d518-0d9f-9227-85bdb50cda7a Address:bc730afa-d518-0d9f-9227-85bdb50cda7a}]" + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.953+1400 [INFO] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: updating configuration: command=RemoveServer server-id=server-b715d6b2-772a-4683-e9c2-327ca0a06b02 server-addr= servers="[{Suffrage:Voter ID:server-3752d97e-4476-231f-036e-426ffa2b8cc4 Address:3752d97e-4476-231f-036e-426ffa2b8cc4} {Suffrage:Voter ID:server-35b87fe5-0871-752e-2f20-244b7e1cc588 Address:35b87fe5-0871-752e-2f20-244b7e1cc588} {Suffrage:Voter ID:server-bc730afa-d518-0d9f-9227-85bdb50cda7a Address:bc730afa-d518-0d9f-9227-85bdb50cda7a}]" +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.953+1400 [INFO] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: removed peer, stopping replication: peer=server-b715d6b2-772a-4683-e9c2-327ca0a06b02 last-index=3 + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.953+1400 [INFO] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: removed peer, stopping replication: peer=server-b715d6b2-772a-4683-e9c2-327ca0a06b02 last-index=3 +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.953+1400 [ERROR] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to pipeline appendEntries: peer="{Voter server-3752d97e-4476-231f-036e-426ffa2b8cc4 3752d97e-4476-231f-036e-426ffa2b8cc4}" error="append pipeline closed" + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.953+1400 [ERROR] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to pipeline appendEntries: peer="{Voter server-3752d97e-4476-231f-036e-426ffa2b8cc4 3752d97e-4476-231f-036e-426ffa2b8cc4}" error="append pipeline closed" +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.953+1400 [INFO] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: aborting pipeline replication: peer="{Voter server-3752d97e-4476-231f-036e-426ffa2b8cc4 3752d97e-4476-231f-036e-426ffa2b8cc4}" + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.953+1400 [INFO] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: aborting pipeline replication: peer="{Voter server-3752d97e-4476-231f-036e-426ffa2b8cc4 3752d97e-4476-231f-036e-426ffa2b8cc4}" +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.953+1400 [ERROR] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to pipeline appendEntries: peer="{Voter server-35b87fe5-0871-752e-2f20-244b7e1cc588 35b87fe5-0871-752e-2f20-244b7e1cc588}" error="append pipeline closed" + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.953+1400 [ERROR] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to pipeline appendEntries: peer="{Voter server-35b87fe5-0871-752e-2f20-244b7e1cc588 35b87fe5-0871-752e-2f20-244b7e1cc588}" error="append pipeline closed" +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.953+1400 [INFO] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: aborting pipeline replication: peer="{Voter server-35b87fe5-0871-752e-2f20-244b7e1cc588 35b87fe5-0871-752e-2f20-244b7e1cc588}" + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.953+1400 [INFO] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: aborting pipeline replication: peer="{Voter server-35b87fe5-0871-752e-2f20-244b7e1cc588 35b87fe5-0871-752e-2f20-244b7e1cc588}" +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.953+1400 [INFO] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: aborting pipeline replication: peer="{Voter server-b715d6b2-772a-4683-e9c2-327ca0a06b02 b715d6b2-772a-4683-e9c2-327ca0a06b02}" + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.953+1400 [INFO] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: aborting pipeline replication: peer="{Voter server-b715d6b2-772a-4683-e9c2-327ca0a06b02 b715d6b2-772a-4683-e9c2-327ca0a06b02}" +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.958+1400 [ERROR] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to heartbeat to: peer=35b87fe5-0871-752e-2f20-244b7e1cc588 error="failed to connect to peer: 35b87fe5-0871-752e-2f20-244b7e1cc588" + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.958+1400 [ERROR] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to heartbeat to: peer=35b87fe5-0871-752e-2f20-244b7e1cc588 error="failed to connect to peer: 35b87fe5-0871-752e-2f20-244b7e1cc588" +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.960+1400 [ERROR] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to appendEntries to: peer="{Voter server-3752d97e-4476-231f-036e-426ffa2b8cc4 3752d97e-4476-231f-036e-426ffa2b8cc4}" error="failed to connect to peer: 3752d97e-4476-231f-036e-426ffa2b8cc4" + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.960+1400 [ERROR] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to appendEntries to: peer="{Voter server-3752d97e-4476-231f-036e-426ffa2b8cc4 3752d97e-4476-231f-036e-426ffa2b8cc4}" error="failed to connect to peer: 3752d97e-4476-231f-036e-426ffa2b8cc4" +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.960+1400 [ERROR] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to heartbeat to: peer=3752d97e-4476-231f-036e-426ffa2b8cc4 error="failed to connect to peer: 3752d97e-4476-231f-036e-426ffa2b8cc4" + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.960+1400 [ERROR] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to heartbeat to: peer=3752d97e-4476-231f-036e-426ffa2b8cc4 error="failed to connect to peer: 3752d97e-4476-231f-036e-426ffa2b8cc4" +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.962+1400 [ERROR] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to appendEntries to: peer="{Voter server-35b87fe5-0871-752e-2f20-244b7e1cc588 35b87fe5-0871-752e-2f20-244b7e1cc588}" error="failed to connect to peer: 35b87fe5-0871-752e-2f20-244b7e1cc588" + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.962+1400 [ERROR] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to appendEntries to: peer="{Voter server-35b87fe5-0871-752e-2f20-244b7e1cc588 35b87fe5-0871-752e-2f20-244b7e1cc588}" error="failed to connect to peer: 35b87fe5-0871-752e-2f20-244b7e1cc588" +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.978+1400 [ERROR] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to heartbeat to: peer=35b87fe5-0871-752e-2f20-244b7e1cc588 error="failed to connect to peer: 35b87fe5-0871-752e-2f20-244b7e1cc588" + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.978+1400 [ERROR] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to heartbeat to: peer=35b87fe5-0871-752e-2f20-244b7e1cc588 error="failed to connect to peer: 35b87fe5-0871-752e-2f20-244b7e1cc588" +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.980+1400 [ERROR] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to heartbeat to: peer=3752d97e-4476-231f-036e-426ffa2b8cc4 error="failed to connect to peer: 3752d97e-4476-231f-036e-426ffa2b8cc4" + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.980+1400 [ERROR] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to heartbeat to: peer=3752d97e-4476-231f-036e-426ffa2b8cc4 error="failed to connect to peer: 3752d97e-4476-231f-036e-426ffa2b8cc4" +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.981+1400 [ERROR] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to appendEntries to: peer="{Voter server-3752d97e-4476-231f-036e-426ffa2b8cc4 3752d97e-4476-231f-036e-426ffa2b8cc4}" error="failed to connect to peer: 3752d97e-4476-231f-036e-426ffa2b8cc4" + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.981+1400 [ERROR] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to appendEntries to: peer="{Voter server-3752d97e-4476-231f-036e-426ffa2b8cc4 3752d97e-4476-231f-036e-426ffa2b8cc4}" error="failed to connect to peer: 3752d97e-4476-231f-036e-426ffa2b8cc4" +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.982+1400 [ERROR] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to appendEntries to: peer="{Voter server-35b87fe5-0871-752e-2f20-244b7e1cc588 35b87fe5-0871-752e-2f20-244b7e1cc588}" error="failed to connect to peer: 35b87fe5-0871-752e-2f20-244b7e1cc588" + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.982+1400 [ERROR] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to appendEntries to: peer="{Voter server-35b87fe5-0871-752e-2f20-244b7e1cc588 35b87fe5-0871-752e-2f20-244b7e1cc588}" error="failed to connect to peer: 35b87fe5-0871-752e-2f20-244b7e1cc588" +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.997+1400 [ERROR] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to heartbeat to: peer=3752d97e-4476-231f-036e-426ffa2b8cc4 error="failed to connect to peer: 3752d97e-4476-231f-036e-426ffa2b8cc4" + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.997+1400 [ERROR] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to heartbeat to: peer=3752d97e-4476-231f-036e-426ffa2b8cc4 error="failed to connect to peer: 3752d97e-4476-231f-036e-426ffa2b8cc4" +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.997+1400 [ERROR] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to heartbeat to: peer=35b87fe5-0871-752e-2f20-244b7e1cc588 error="failed to connect to peer: 35b87fe5-0871-752e-2f20-244b7e1cc588" + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.997+1400 [ERROR] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to heartbeat to: peer=35b87fe5-0871-752e-2f20-244b7e1cc588 error="failed to connect to peer: 35b87fe5-0871-752e-2f20-244b7e1cc588" +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.998+1400 [ERROR] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to appendEntries to: peer="{Voter server-3752d97e-4476-231f-036e-426ffa2b8cc4 3752d97e-4476-231f-036e-426ffa2b8cc4}" error="failed to connect to peer: 3752d97e-4476-231f-036e-426ffa2b8cc4" + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:32.998+1400 [ERROR] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to appendEntries to: peer="{Voter server-3752d97e-4476-231f-036e-426ffa2b8cc4 3752d97e-4476-231f-036e-426ffa2b8cc4}" error="failed to connect to peer: 3752d97e-4476-231f-036e-426ffa2b8cc4" +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:33.001+1400 [ERROR] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to appendEntries to: peer="{Voter server-35b87fe5-0871-752e-2f20-244b7e1cc588 35b87fe5-0871-752e-2f20-244b7e1cc588}" error="failed to connect to peer: 35b87fe5-0871-752e-2f20-244b7e1cc588" + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:33.001+1400 [ERROR] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to appendEntries to: peer="{Voter server-35b87fe5-0871-752e-2f20-244b7e1cc588 35b87fe5-0871-752e-2f20-244b7e1cc588}" error="failed to connect to peer: 35b87fe5-0871-752e-2f20-244b7e1cc588" +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:33.002+1400 [WARN] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to contact: server-id=server-35b87fe5-0871-752e-2f20-244b7e1cc588 time=50.724183ms + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:33.002+1400 [WARN] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to contact: server-id=server-35b87fe5-0871-752e-2f20-244b7e1cc588 time=50.724183ms +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:33.013+1400 [WARN] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to contact: server-id=server-3752d97e-4476-231f-036e-426ffa2b8cc4 time=60.430155ms + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:33.013+1400 [WARN] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to contact: server-id=server-3752d97e-4476-231f-036e-426ffa2b8cc4 time=60.430155ms +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:33.013+1400 [WARN] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to contact: server-id=server-35b87fe5-0871-752e-2f20-244b7e1cc588 time=61.624472ms + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:33.013+1400 [WARN] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to contact: server-id=server-35b87fe5-0871-752e-2f20-244b7e1cc588 time=61.624472ms +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:33.013+1400 [WARN] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to contact quorum of nodes, stepping down + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:33.013+1400 [WARN] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to contact quorum of nodes, stepping down +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:33.013+1400 [INFO] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: entering follower state: follower="Node at bc730afa-d518-0d9f-9227-85bdb50cda7a [Follower]" leader= + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:33.013+1400 [INFO] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: entering follower state: follower="Node at bc730afa-d518-0d9f-9227-85bdb50cda7a [Follower]" leader= +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:33.013+1400 [ERROR] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to appendEntries to: peer="{Voter server-35b87fe5-0871-752e-2f20-244b7e1cc588 35b87fe5-0871-752e-2f20-244b7e1cc588}" error="failed to connect to peer: 35b87fe5-0871-752e-2f20-244b7e1cc588" + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:33.013+1400 [ERROR] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to appendEntries to: peer="{Voter server-35b87fe5-0871-752e-2f20-244b7e1cc588 35b87fe5-0871-752e-2f20-244b7e1cc588}" error="failed to connect to peer: 35b87fe5-0871-752e-2f20-244b7e1cc588" +testLoggerAdapter verbose: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:33.013+1400 [ERROR] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to appendEntries to: peer="{Voter server-3752d97e-4476-231f-036e-426ffa2b8cc4 3752d97e-4476-231f-036e-426ffa2b8cc4}" error="failed to connect to peer: 3752d97e-4476-231f-036e-426ffa2b8cc4" + testing.go:149: server-bc730afa-d518-0d9f-9227-85bdb50cda7a: 2024-06-12T04:32:33.013+1400 [ERROR] server-bc730afa-d518-0d9f-9227-85bdb50cda7a: failed to appendEntries to: peer="{Voter server-3752d97e-4476-231f-036e-426ffa2b8cc4 3752d97e-4476-231f-036e-426ffa2b8cc4}" error="failed to connect to peer: 3752d97e-4476-231f-036e-426ffa2b8cc4" +--- PASS: TestRaft_RemoveFollower_SplitCluster (0.44s) === RUN TestRaft_AddKnownPeer -testLoggerAdapter verbose: server-3c2ff6b4-48e5-0498-6d36-baa17293b179: 2024-06-11T02:29:19.618-1200 [INFO] server-3c2ff6b4-48e5-0498-6d36-baa17293b179: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3c2ff6b4-48e5-0498-6d36-baa17293b179 Address:3c2ff6b4-48e5-0498-6d36-baa17293b179} {Suffrage:Voter ID:server-523b7898-244b-852f-f72e-01c1db48a4f7 Address:523b7898-244b-852f-f72e-01c1db48a4f7} {Suffrage:Voter ID:server-17145aaf-3f74-1480-dfe4-52396c6b260c Address:17145aaf-3f74-1480-dfe4-52396c6b260c}]" - testing.go:149: server-3c2ff6b4-48e5-0498-6d36-baa17293b179: 2024-06-11T02:29:19.618-1200 [INFO] server-3c2ff6b4-48e5-0498-6d36-baa17293b179: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3c2ff6b4-48e5-0498-6d36-baa17293b179 Address:3c2ff6b4-48e5-0498-6d36-baa17293b179} {Suffrage:Voter ID:server-523b7898-244b-852f-f72e-01c1db48a4f7 Address:523b7898-244b-852f-f72e-01c1db48a4f7} {Suffrage:Voter ID:server-17145aaf-3f74-1480-dfe4-52396c6b260c Address:17145aaf-3f74-1480-dfe4-52396c6b260c}]" -testLoggerAdapter verbose: server-523b7898-244b-852f-f72e-01c1db48a4f7: 2024-06-11T02:29:19.618-1200 [INFO] server-523b7898-244b-852f-f72e-01c1db48a4f7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3c2ff6b4-48e5-0498-6d36-baa17293b179 Address:3c2ff6b4-48e5-0498-6d36-baa17293b179} {Suffrage:Voter ID:server-523b7898-244b-852f-f72e-01c1db48a4f7 Address:523b7898-244b-852f-f72e-01c1db48a4f7} {Suffrage:Voter ID:server-17145aaf-3f74-1480-dfe4-52396c6b260c Address:17145aaf-3f74-1480-dfe4-52396c6b260c}]" - testing.go:149: server-523b7898-244b-852f-f72e-01c1db48a4f7: 2024-06-11T02:29:19.618-1200 [INFO] server-523b7898-244b-852f-f72e-01c1db48a4f7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3c2ff6b4-48e5-0498-6d36-baa17293b179 Address:3c2ff6b4-48e5-0498-6d36-baa17293b179} {Suffrage:Voter ID:server-523b7898-244b-852f-f72e-01c1db48a4f7 Address:523b7898-244b-852f-f72e-01c1db48a4f7} {Suffrage:Voter ID:server-17145aaf-3f74-1480-dfe4-52396c6b260c Address:17145aaf-3f74-1480-dfe4-52396c6b260c}]" -testLoggerAdapter verbose: server-17145aaf-3f74-1480-dfe4-52396c6b260c: 2024-06-11T02:29:19.618-1200 [INFO] server-17145aaf-3f74-1480-dfe4-52396c6b260c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3c2ff6b4-48e5-0498-6d36-baa17293b179 Address:3c2ff6b4-48e5-0498-6d36-baa17293b179} {Suffrage:Voter ID:server-523b7898-244b-852f-f72e-01c1db48a4f7 Address:523b7898-244b-852f-f72e-01c1db48a4f7} {Suffrage:Voter ID:server-17145aaf-3f74-1480-dfe4-52396c6b260c Address:17145aaf-3f74-1480-dfe4-52396c6b260c}]" - testing.go:149: server-17145aaf-3f74-1480-dfe4-52396c6b260c: 2024-06-11T02:29:19.618-1200 [INFO] server-17145aaf-3f74-1480-dfe4-52396c6b260c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3c2ff6b4-48e5-0498-6d36-baa17293b179 Address:3c2ff6b4-48e5-0498-6d36-baa17293b179} {Suffrage:Voter ID:server-523b7898-244b-852f-f72e-01c1db48a4f7 Address:523b7898-244b-852f-f72e-01c1db48a4f7} {Suffrage:Voter ID:server-17145aaf-3f74-1480-dfe4-52396c6b260c Address:17145aaf-3f74-1480-dfe4-52396c6b260c}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:19.618-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:19.618-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-3c2ff6b4-48e5-0498-6d36-baa17293b179: 2024-06-11T02:29:19.619-1200 [INFO] server-3c2ff6b4-48e5-0498-6d36-baa17293b179: entering follower state: follower="Node at 3c2ff6b4-48e5-0498-6d36-baa17293b179 [Follower]" leader= -testLoggerAdapter verbose: server-523b7898-244b-852f-f72e-01c1db48a4f7: 2024-06-11T02:29:19.620-1200 [INFO] server-523b7898-244b-852f-f72e-01c1db48a4f7: entering follower state: follower="Node at 523b7898-244b-852f-f72e-01c1db48a4f7 [Follower]" leader= -testLoggerAdapter verbose: server-17145aaf-3f74-1480-dfe4-52396c6b260c: 2024-06-11T02:29:19.620-1200 [INFO] server-17145aaf-3f74-1480-dfe4-52396c6b260c: entering follower state: follower="Node at 17145aaf-3f74-1480-dfe4-52396c6b260c [Follower]" leader= - testing.go:149: server-3c2ff6b4-48e5-0498-6d36-baa17293b179: 2024-06-11T02:29:19.619-1200 [INFO] server-3c2ff6b4-48e5-0498-6d36-baa17293b179: entering follower state: follower="Node at 3c2ff6b4-48e5-0498-6d36-baa17293b179 [Follower]" leader= - testing.go:149: server-523b7898-244b-852f-f72e-01c1db48a4f7: 2024-06-11T02:29:19.620-1200 [INFO] server-523b7898-244b-852f-f72e-01c1db48a4f7: entering follower state: follower="Node at 523b7898-244b-852f-f72e-01c1db48a4f7 [Follower]" leader= - testing.go:149: server-17145aaf-3f74-1480-dfe4-52396c6b260c: 2024-06-11T02:29:19.620-1200 [INFO] server-17145aaf-3f74-1480-dfe4-52396c6b260c: entering follower state: follower="Node at 17145aaf-3f74-1480-dfe4-52396c6b260c [Follower]" leader= -testLoggerAdapter verbose: server-523b7898-244b-852f-f72e-01c1db48a4f7: 2024-06-11T02:29:19.676-1200 [WARN] server-523b7898-244b-852f-f72e-01c1db48a4f7: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-523b7898-244b-852f-f72e-01c1db48a4f7: 2024-06-11T02:29:19.676-1200 [WARN] server-523b7898-244b-852f-f72e-01c1db48a4f7: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-523b7898-244b-852f-f72e-01c1db48a4f7: 2024-06-11T02:29:19.676-1200 [INFO] server-523b7898-244b-852f-f72e-01c1db48a4f7: entering candidate state: node="Node at 523b7898-244b-852f-f72e-01c1db48a4f7 [Candidate]" term=2 - testing.go:149: server-523b7898-244b-852f-f72e-01c1db48a4f7: 2024-06-11T02:29:19.676-1200 [INFO] server-523b7898-244b-852f-f72e-01c1db48a4f7: entering candidate state: node="Node at 523b7898-244b-852f-f72e-01c1db48a4f7 [Candidate]" term=2 -testLoggerAdapter verbose: server-523b7898-244b-852f-f72e-01c1db48a4f7: 2024-06-11T02:29:19.676-1200 [INFO] server-523b7898-244b-852f-f72e-01c1db48a4f7: election won: tally=2 - testing.go:149: server-523b7898-244b-852f-f72e-01c1db48a4f7: 2024-06-11T02:29:19.676-1200 [INFO] server-523b7898-244b-852f-f72e-01c1db48a4f7: election won: tally=2 -testLoggerAdapter verbose: server-523b7898-244b-852f-f72e-01c1db48a4f7: 2024-06-11T02:29:19.677-1200 [INFO] server-523b7898-244b-852f-f72e-01c1db48a4f7: entering leader state: leader="Node at 523b7898-244b-852f-f72e-01c1db48a4f7 [Leader]" - testing.go:149: server-523b7898-244b-852f-f72e-01c1db48a4f7: 2024-06-11T02:29:19.677-1200 [INFO] server-523b7898-244b-852f-f72e-01c1db48a4f7: entering leader state: leader="Node at 523b7898-244b-852f-f72e-01c1db48a4f7 [Leader]" -testLoggerAdapter verbose: server-523b7898-244b-852f-f72e-01c1db48a4f7: 2024-06-11T02:29:19.677-1200 [INFO] server-523b7898-244b-852f-f72e-01c1db48a4f7: added peer, starting replication: peer=server-3c2ff6b4-48e5-0498-6d36-baa17293b179 - testing.go:149: server-523b7898-244b-852f-f72e-01c1db48a4f7: 2024-06-11T02:29:19.677-1200 [INFO] server-523b7898-244b-852f-f72e-01c1db48a4f7: added peer, starting replication: peer=server-3c2ff6b4-48e5-0498-6d36-baa17293b179 -testLoggerAdapter verbose: server-523b7898-244b-852f-f72e-01c1db48a4f7: 2024-06-11T02:29:19.677-1200 [INFO] server-523b7898-244b-852f-f72e-01c1db48a4f7: added peer, starting replication: peer=server-17145aaf-3f74-1480-dfe4-52396c6b260c - testing.go:149: server-523b7898-244b-852f-f72e-01c1db48a4f7: 2024-06-11T02:29:19.677-1200 [INFO] server-523b7898-244b-852f-f72e-01c1db48a4f7: added peer, starting replication: peer=server-17145aaf-3f74-1480-dfe4-52396c6b260c -testLoggerAdapter verbose: server-523b7898-244b-852f-f72e-01c1db48a4f7: 2024-06-11T02:29:19.677-1200 [INFO] server-523b7898-244b-852f-f72e-01c1db48a4f7: pipelining replication: peer="{Voter server-17145aaf-3f74-1480-dfe4-52396c6b260c 17145aaf-3f74-1480-dfe4-52396c6b260c}" - testing.go:149: server-523b7898-244b-852f-f72e-01c1db48a4f7: 2024-06-11T02:29:19.677-1200 [INFO] server-523b7898-244b-852f-f72e-01c1db48a4f7: pipelining replication: peer="{Voter server-17145aaf-3f74-1480-dfe4-52396c6b260c 17145aaf-3f74-1480-dfe4-52396c6b260c}" -testLoggerAdapter verbose: server-523b7898-244b-852f-f72e-01c1db48a4f7: 2024-06-11T02:29:19.678-1200 [INFO] server-523b7898-244b-852f-f72e-01c1db48a4f7: pipelining replication: peer="{Voter server-3c2ff6b4-48e5-0498-6d36-baa17293b179 3c2ff6b4-48e5-0498-6d36-baa17293b179}" - testing.go:149: server-523b7898-244b-852f-f72e-01c1db48a4f7: 2024-06-11T02:29:19.678-1200 [INFO] server-523b7898-244b-852f-f72e-01c1db48a4f7: pipelining replication: peer="{Voter server-3c2ff6b4-48e5-0498-6d36-baa17293b179 3c2ff6b4-48e5-0498-6d36-baa17293b179}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:19.784-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:19.678287122 -1200 -12 m=+10.673659955 (1 nodes), 59.265551ms from start of poll, 60.500737ms from cluster start. Timeout at 2024-06-11 02:29:19.783757638 -1200 -12 m=+10.779130180, 105.470225ms after stability - testing.go:149: cluster: 2024-06-11T02:29:19.784-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:19.678287122 -1200 -12 m=+10.673659955 (1 nodes), 59.265551ms from start of poll, 60.500737ms from cluster start. Timeout at 2024-06-11 02:29:19.783757638 -1200 -12 m=+10.779130180, 105.470225ms after stability -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:19.784-1200 [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2024-06-11T02:29:19.784-1200 [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:19.889-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:19.784405709 -1200 -12 m=+10.779778251 (2 nodes), 4.667µs from start of poll, 166.619033ms from cluster start. Timeout at 2024-06-11 02:29:19.889576981 -1200 -12 m=+10.884949522, 105.171271ms after stability - testing.go:149: cluster: 2024-06-11T02:29:19.889-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:19.784405709 -1200 -12 m=+10.779778251 (2 nodes), 4.667µs from start of poll, 166.619033ms from cluster start. Timeout at 2024-06-11 02:29:19.889576981 -1200 -12 m=+10.884949522, 105.171271ms after stability -testLoggerAdapter verbose: server-523b7898-244b-852f-f72e-01c1db48a4f7: 2024-06-11T02:29:19.889-1200 [INFO] server-523b7898-244b-852f-f72e-01c1db48a4f7: updating configuration: command=AddStaging server-id=server-3c2ff6b4-48e5-0498-6d36-baa17293b179 server-addr=3c2ff6b4-48e5-0498-6d36-baa17293b179 servers="[{Suffrage:Voter ID:server-3c2ff6b4-48e5-0498-6d36-baa17293b179 Address:3c2ff6b4-48e5-0498-6d36-baa17293b179} {Suffrage:Voter ID:server-523b7898-244b-852f-f72e-01c1db48a4f7 Address:523b7898-244b-852f-f72e-01c1db48a4f7} {Suffrage:Voter ID:server-17145aaf-3f74-1480-dfe4-52396c6b260c Address:17145aaf-3f74-1480-dfe4-52396c6b260c}]" - testing.go:149: server-523b7898-244b-852f-f72e-01c1db48a4f7: 2024-06-11T02:29:19.889-1200 [INFO] server-523b7898-244b-852f-f72e-01c1db48a4f7: updating configuration: command=AddStaging server-id=server-3c2ff6b4-48e5-0498-6d36-baa17293b179 server-addr=3c2ff6b4-48e5-0498-6d36-baa17293b179 servers="[{Suffrage:Voter ID:server-3c2ff6b4-48e5-0498-6d36-baa17293b179 Address:3c2ff6b4-48e5-0498-6d36-baa17293b179} {Suffrage:Voter ID:server-523b7898-244b-852f-f72e-01c1db48a4f7 Address:523b7898-244b-852f-f72e-01c1db48a4f7} {Suffrage:Voter ID:server-17145aaf-3f74-1480-dfe4-52396c6b260c Address:17145aaf-3f74-1480-dfe4-52396c6b260c}]" -testLoggerAdapter verbose: server-523b7898-244b-852f-f72e-01c1db48a4f7: 2024-06-11T02:29:19.890-1200 [INFO] server-523b7898-244b-852f-f72e-01c1db48a4f7: aborting pipeline replication: peer="{Voter server-3c2ff6b4-48e5-0498-6d36-baa17293b179 3c2ff6b4-48e5-0498-6d36-baa17293b179}" - testing.go:149: server-523b7898-244b-852f-f72e-01c1db48a4f7: 2024-06-11T02:29:19.890-1200 [INFO] server-523b7898-244b-852f-f72e-01c1db48a4f7: aborting pipeline replication: peer="{Voter server-3c2ff6b4-48e5-0498-6d36-baa17293b179 3c2ff6b4-48e5-0498-6d36-baa17293b179}" -testLoggerAdapter verbose: server-523b7898-244b-852f-f72e-01c1db48a4f7: 2024-06-11T02:29:19.890-1200 [INFO] server-523b7898-244b-852f-f72e-01c1db48a4f7: aborting pipeline replication: peer="{Voter server-17145aaf-3f74-1480-dfe4-52396c6b260c 17145aaf-3f74-1480-dfe4-52396c6b260c}" - testing.go:149: server-523b7898-244b-852f-f72e-01c1db48a4f7: 2024-06-11T02:29:19.890-1200 [INFO] server-523b7898-244b-852f-f72e-01c1db48a4f7: aborting pipeline replication: peer="{Voter server-17145aaf-3f74-1480-dfe4-52396c6b260c 17145aaf-3f74-1480-dfe4-52396c6b260c}" -testLoggerAdapter verbose: server-523b7898-244b-852f-f72e-01c1db48a4f7: 2024-06-11T02:29:19.941-1200 [ERROR] server-523b7898-244b-852f-f72e-01c1db48a4f7: failed to heartbeat to: peer=17145aaf-3f74-1480-dfe4-52396c6b260c error="command timed out" - testing.go:149: server-523b7898-244b-852f-f72e-01c1db48a4f7: 2024-06-11T02:29:19.941-1200 [ERROR] server-523b7898-244b-852f-f72e-01c1db48a4f7: failed to heartbeat to: peer=17145aaf-3f74-1480-dfe4-52396c6b260c error="command timed out" ---- PASS: TestRaft_AddKnownPeer (0.33s) +testLoggerAdapter verbose: server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: 2024-06-12T04:32:33.016+1400 [INFO] server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-aeee72ce-d122-70b7-0c20-6d2de3753a9b Address:aeee72ce-d122-70b7-0c20-6d2de3753a9b} {Suffrage:Voter ID:server-fc7844a6-f4d7-529a-910b-35bc8543d450 Address:fc7844a6-f4d7-529a-910b-35bc8543d450} {Suffrage:Voter ID:server-00b2c93b-52c4-0149-9b88-876f27241473 Address:00b2c93b-52c4-0149-9b88-876f27241473}]" + testing.go:149: server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: 2024-06-12T04:32:33.016+1400 [INFO] server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-aeee72ce-d122-70b7-0c20-6d2de3753a9b Address:aeee72ce-d122-70b7-0c20-6d2de3753a9b} {Suffrage:Voter ID:server-fc7844a6-f4d7-529a-910b-35bc8543d450 Address:fc7844a6-f4d7-529a-910b-35bc8543d450} {Suffrage:Voter ID:server-00b2c93b-52c4-0149-9b88-876f27241473 Address:00b2c93b-52c4-0149-9b88-876f27241473}]" +testLoggerAdapter verbose: server-fc7844a6-f4d7-529a-910b-35bc8543d450: 2024-06-12T04:32:33.017+1400 [INFO] server-fc7844a6-f4d7-529a-910b-35bc8543d450: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-aeee72ce-d122-70b7-0c20-6d2de3753a9b Address:aeee72ce-d122-70b7-0c20-6d2de3753a9b} {Suffrage:Voter ID:server-fc7844a6-f4d7-529a-910b-35bc8543d450 Address:fc7844a6-f4d7-529a-910b-35bc8543d450} {Suffrage:Voter ID:server-00b2c93b-52c4-0149-9b88-876f27241473 Address:00b2c93b-52c4-0149-9b88-876f27241473}]" + testing.go:149: server-fc7844a6-f4d7-529a-910b-35bc8543d450: 2024-06-12T04:32:33.017+1400 [INFO] server-fc7844a6-f4d7-529a-910b-35bc8543d450: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-aeee72ce-d122-70b7-0c20-6d2de3753a9b Address:aeee72ce-d122-70b7-0c20-6d2de3753a9b} {Suffrage:Voter ID:server-fc7844a6-f4d7-529a-910b-35bc8543d450 Address:fc7844a6-f4d7-529a-910b-35bc8543d450} {Suffrage:Voter ID:server-00b2c93b-52c4-0149-9b88-876f27241473 Address:00b2c93b-52c4-0149-9b88-876f27241473}]" +testLoggerAdapter verbose: server-00b2c93b-52c4-0149-9b88-876f27241473: 2024-06-12T04:32:33.017+1400 [INFO] server-00b2c93b-52c4-0149-9b88-876f27241473: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-aeee72ce-d122-70b7-0c20-6d2de3753a9b Address:aeee72ce-d122-70b7-0c20-6d2de3753a9b} {Suffrage:Voter ID:server-fc7844a6-f4d7-529a-910b-35bc8543d450 Address:fc7844a6-f4d7-529a-910b-35bc8543d450} {Suffrage:Voter ID:server-00b2c93b-52c4-0149-9b88-876f27241473 Address:00b2c93b-52c4-0149-9b88-876f27241473}]" + testing.go:149: server-00b2c93b-52c4-0149-9b88-876f27241473: 2024-06-12T04:32:33.017+1400 [INFO] server-00b2c93b-52c4-0149-9b88-876f27241473: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-aeee72ce-d122-70b7-0c20-6d2de3753a9b Address:aeee72ce-d122-70b7-0c20-6d2de3753a9b} {Suffrage:Voter ID:server-fc7844a6-f4d7-529a-910b-35bc8543d450 Address:fc7844a6-f4d7-529a-910b-35bc8543d450} {Suffrage:Voter ID:server-00b2c93b-52c4-0149-9b88-876f27241473 Address:00b2c93b-52c4-0149-9b88-876f27241473}]" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:33.017+1400 [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-06-12T04:32:33.017+1400 [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: 2024-06-12T04:32:33.017+1400 [INFO] server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: entering follower state: follower="Node at aeee72ce-d122-70b7-0c20-6d2de3753a9b [Follower]" leader= + testing.go:149: server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: 2024-06-12T04:32:33.017+1400 [INFO] server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: entering follower state: follower="Node at aeee72ce-d122-70b7-0c20-6d2de3753a9b [Follower]" leader= +testLoggerAdapter verbose: server-fc7844a6-f4d7-529a-910b-35bc8543d450: 2024-06-12T04:32:33.017+1400 [INFO] server-fc7844a6-f4d7-529a-910b-35bc8543d450: entering follower state: follower="Node at fc7844a6-f4d7-529a-910b-35bc8543d450 [Follower]" leader= + testing.go:149: server-fc7844a6-f4d7-529a-910b-35bc8543d450: 2024-06-12T04:32:33.017+1400 [INFO] server-fc7844a6-f4d7-529a-910b-35bc8543d450: entering follower state: follower="Node at fc7844a6-f4d7-529a-910b-35bc8543d450 [Follower]" leader= +testLoggerAdapter verbose: server-00b2c93b-52c4-0149-9b88-876f27241473: 2024-06-12T04:32:33.017+1400 [INFO] server-00b2c93b-52c4-0149-9b88-876f27241473: entering follower state: follower="Node at 00b2c93b-52c4-0149-9b88-876f27241473 [Follower]" leader= + testing.go:149: server-00b2c93b-52c4-0149-9b88-876f27241473: 2024-06-12T04:32:33.017+1400 [INFO] server-00b2c93b-52c4-0149-9b88-876f27241473: entering follower state: follower="Node at 00b2c93b-52c4-0149-9b88-876f27241473 [Follower]" leader= +testLoggerAdapter verbose: server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: 2024-06-12T04:32:33.069+1400 [WARN] server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: 2024-06-12T04:32:33.069+1400 [WARN] server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: 2024-06-12T04:32:33.069+1400 [INFO] server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: entering candidate state: node="Node at aeee72ce-d122-70b7-0c20-6d2de3753a9b [Candidate]" term=2 + testing.go:149: server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: 2024-06-12T04:32:33.069+1400 [INFO] server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: entering candidate state: node="Node at aeee72ce-d122-70b7-0c20-6d2de3753a9b [Candidate]" term=2 +testLoggerAdapter verbose: server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: 2024-06-12T04:32:33.069+1400 [INFO] server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: election won: tally=2 + testing.go:149: server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: 2024-06-12T04:32:33.069+1400 [INFO] server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: election won: tally=2 +testLoggerAdapter verbose: server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: 2024-06-12T04:32:33.069+1400 [INFO] server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: entering leader state: leader="Node at aeee72ce-d122-70b7-0c20-6d2de3753a9b [Leader]" + testing.go:149: server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: 2024-06-12T04:32:33.069+1400 [INFO] server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: entering leader state: leader="Node at aeee72ce-d122-70b7-0c20-6d2de3753a9b [Leader]" +testLoggerAdapter verbose: server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: 2024-06-12T04:32:33.069+1400 [INFO] server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: added peer, starting replication: peer=server-fc7844a6-f4d7-529a-910b-35bc8543d450 + testing.go:149: server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: 2024-06-12T04:32:33.069+1400 [INFO] server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: added peer, starting replication: peer=server-fc7844a6-f4d7-529a-910b-35bc8543d450 +testLoggerAdapter verbose: server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: 2024-06-12T04:32:33.069+1400 [INFO] server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: added peer, starting replication: peer=server-00b2c93b-52c4-0149-9b88-876f27241473 + testing.go:149: server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: 2024-06-12T04:32:33.069+1400 [INFO] server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: added peer, starting replication: peer=server-00b2c93b-52c4-0149-9b88-876f27241473 +testLoggerAdapter verbose: server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: 2024-06-12T04:32:33.069+1400 [INFO] server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: pipelining replication: peer="{Voter server-00b2c93b-52c4-0149-9b88-876f27241473 00b2c93b-52c4-0149-9b88-876f27241473}" + testing.go:149: server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: 2024-06-12T04:32:33.069+1400 [INFO] server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: pipelining replication: peer="{Voter server-00b2c93b-52c4-0149-9b88-876f27241473 00b2c93b-52c4-0149-9b88-876f27241473}" +testLoggerAdapter verbose: server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: 2024-06-12T04:32:33.069+1400 [INFO] server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: pipelining replication: peer="{Voter server-fc7844a6-f4d7-529a-910b-35bc8543d450 fc7844a6-f4d7-529a-910b-35bc8543d450}" + testing.go:149: server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: 2024-06-12T04:32:33.069+1400 [INFO] server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: pipelining replication: peer="{Voter server-fc7844a6-f4d7-529a-910b-35bc8543d450 fc7844a6-f4d7-529a-910b-35bc8543d450}" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:33.175+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:33.069825524 +1400 +14 m=+9.977344103 (1 nodes), 52.332686ms from start of poll, 53.090659ms from cluster start. Timeout at 2024-06-12 04:32:33.175791968 +1400 +14 m=+10.083310547, 105.966444ms after stability + testing.go:149: cluster: 2024-06-12T04:32:33.175+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:33.069825524 +1400 +14 m=+9.977344103 (1 nodes), 52.332686ms from start of poll, 53.090659ms from cluster start. Timeout at 2024-06-12 04:32:33.175791968 +1400 +14 m=+10.083310547, 105.966444ms after stability +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:33.175+1400 [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-06-12T04:32:33.175+1400 [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:33.281+1400 [INFO] stable state for Follower reached at 2024-06-12 04:32:33.17601228 +1400 +14 m=+10.083530859 (2 nodes), 2.08µs from start of poll, 159.277415ms from cluster start. Timeout at 2024-06-12 04:32:33.281394106 +1400 +14 m=+10.188912685, 105.381826ms after stability + testing.go:149: cluster: 2024-06-12T04:32:33.281+1400 [INFO] stable state for Follower reached at 2024-06-12 04:32:33.17601228 +1400 +14 m=+10.083530859 (2 nodes), 2.08µs from start of poll, 159.277415ms from cluster start. Timeout at 2024-06-12 04:32:33.281394106 +1400 +14 m=+10.188912685, 105.381826ms after stability +testLoggerAdapter verbose: server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: 2024-06-12T04:32:33.281+1400 [INFO] server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: updating configuration: command=AddStaging server-id=server-fc7844a6-f4d7-529a-910b-35bc8543d450 server-addr=fc7844a6-f4d7-529a-910b-35bc8543d450 servers="[{Suffrage:Voter ID:server-aeee72ce-d122-70b7-0c20-6d2de3753a9b Address:aeee72ce-d122-70b7-0c20-6d2de3753a9b} {Suffrage:Voter ID:server-fc7844a6-f4d7-529a-910b-35bc8543d450 Address:fc7844a6-f4d7-529a-910b-35bc8543d450} {Suffrage:Voter ID:server-00b2c93b-52c4-0149-9b88-876f27241473 Address:00b2c93b-52c4-0149-9b88-876f27241473}]" + testing.go:149: server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: 2024-06-12T04:32:33.281+1400 [INFO] server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: updating configuration: command=AddStaging server-id=server-fc7844a6-f4d7-529a-910b-35bc8543d450 server-addr=fc7844a6-f4d7-529a-910b-35bc8543d450 servers="[{Suffrage:Voter ID:server-aeee72ce-d122-70b7-0c20-6d2de3753a9b Address:aeee72ce-d122-70b7-0c20-6d2de3753a9b} {Suffrage:Voter ID:server-fc7844a6-f4d7-529a-910b-35bc8543d450 Address:fc7844a6-f4d7-529a-910b-35bc8543d450} {Suffrage:Voter ID:server-00b2c93b-52c4-0149-9b88-876f27241473 Address:00b2c93b-52c4-0149-9b88-876f27241473}]" +testLoggerAdapter verbose: server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: 2024-06-12T04:32:33.281+1400 [INFO] server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: aborting pipeline replication: peer="{Voter server-00b2c93b-52c4-0149-9b88-876f27241473 00b2c93b-52c4-0149-9b88-876f27241473}" + testing.go:149: server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: 2024-06-12T04:32:33.281+1400 [INFO] server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: aborting pipeline replication: peer="{Voter server-00b2c93b-52c4-0149-9b88-876f27241473 00b2c93b-52c4-0149-9b88-876f27241473}" +testLoggerAdapter verbose: server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: 2024-06-12T04:32:33.282+1400 [INFO] server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: aborting pipeline replication: peer="{Voter server-fc7844a6-f4d7-529a-910b-35bc8543d450 fc7844a6-f4d7-529a-910b-35bc8543d450}" + testing.go:149: server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: 2024-06-12T04:32:33.282+1400 [INFO] server-aeee72ce-d122-70b7-0c20-6d2de3753a9b: aborting pipeline replication: peer="{Voter server-fc7844a6-f4d7-529a-910b-35bc8543d450 fc7844a6-f4d7-529a-910b-35bc8543d450}" +--- PASS: TestRaft_AddKnownPeer (0.27s) === RUN TestRaft_RemoveUnknownPeer -testLoggerAdapter verbose: server-e9adca04-3f17-f15c-c035-a29de6c12ada: 2024-06-11T02:29:19.950-1200 [INFO] server-e9adca04-3f17-f15c-c035-a29de6c12ada: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e9adca04-3f17-f15c-c035-a29de6c12ada Address:e9adca04-3f17-f15c-c035-a29de6c12ada} {Suffrage:Voter ID:server-a7f68db0-71a6-e748-9125-63b11b9b7cbb Address:a7f68db0-71a6-e748-9125-63b11b9b7cbb} {Suffrage:Voter ID:server-7365ee01-49ff-f56a-c1ae-92ff143298d7 Address:7365ee01-49ff-f56a-c1ae-92ff143298d7}]" - testing.go:149: server-e9adca04-3f17-f15c-c035-a29de6c12ada: 2024-06-11T02:29:19.950-1200 [INFO] server-e9adca04-3f17-f15c-c035-a29de6c12ada: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e9adca04-3f17-f15c-c035-a29de6c12ada Address:e9adca04-3f17-f15c-c035-a29de6c12ada} {Suffrage:Voter ID:server-a7f68db0-71a6-e748-9125-63b11b9b7cbb Address:a7f68db0-71a6-e748-9125-63b11b9b7cbb} {Suffrage:Voter ID:server-7365ee01-49ff-f56a-c1ae-92ff143298d7 Address:7365ee01-49ff-f56a-c1ae-92ff143298d7}]" -testLoggerAdapter verbose: server-e9adca04-3f17-f15c-c035-a29de6c12ada: 2024-06-11T02:29:19.951-1200 [INFO] server-e9adca04-3f17-f15c-c035-a29de6c12ada: entering follower state: follower="Node at e9adca04-3f17-f15c-c035-a29de6c12ada [Follower]" leader= - testing.go:149: server-e9adca04-3f17-f15c-c035-a29de6c12ada: 2024-06-11T02:29:19.951-1200 [INFO] server-e9adca04-3f17-f15c-c035-a29de6c12ada: entering follower state: follower="Node at e9adca04-3f17-f15c-c035-a29de6c12ada [Follower]" leader= -testLoggerAdapter verbose: server-a7f68db0-71a6-e748-9125-63b11b9b7cbb: 2024-06-11T02:29:19.952-1200 [INFO] server-a7f68db0-71a6-e748-9125-63b11b9b7cbb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e9adca04-3f17-f15c-c035-a29de6c12ada Address:e9adca04-3f17-f15c-c035-a29de6c12ada} {Suffrage:Voter ID:server-a7f68db0-71a6-e748-9125-63b11b9b7cbb Address:a7f68db0-71a6-e748-9125-63b11b9b7cbb} {Suffrage:Voter ID:server-7365ee01-49ff-f56a-c1ae-92ff143298d7 Address:7365ee01-49ff-f56a-c1ae-92ff143298d7}]" - testing.go:149: server-a7f68db0-71a6-e748-9125-63b11b9b7cbb: 2024-06-11T02:29:19.952-1200 [INFO] server-a7f68db0-71a6-e748-9125-63b11b9b7cbb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e9adca04-3f17-f15c-c035-a29de6c12ada Address:e9adca04-3f17-f15c-c035-a29de6c12ada} {Suffrage:Voter ID:server-a7f68db0-71a6-e748-9125-63b11b9b7cbb Address:a7f68db0-71a6-e748-9125-63b11b9b7cbb} {Suffrage:Voter ID:server-7365ee01-49ff-f56a-c1ae-92ff143298d7 Address:7365ee01-49ff-f56a-c1ae-92ff143298d7}]" -testLoggerAdapter verbose: server-7365ee01-49ff-f56a-c1ae-92ff143298d7: 2024-06-11T02:29:19.952-1200 [INFO] server-7365ee01-49ff-f56a-c1ae-92ff143298d7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e9adca04-3f17-f15c-c035-a29de6c12ada Address:e9adca04-3f17-f15c-c035-a29de6c12ada} {Suffrage:Voter ID:server-a7f68db0-71a6-e748-9125-63b11b9b7cbb Address:a7f68db0-71a6-e748-9125-63b11b9b7cbb} {Suffrage:Voter ID:server-7365ee01-49ff-f56a-c1ae-92ff143298d7 Address:7365ee01-49ff-f56a-c1ae-92ff143298d7}]" - testing.go:149: server-7365ee01-49ff-f56a-c1ae-92ff143298d7: 2024-06-11T02:29:19.952-1200 [INFO] server-7365ee01-49ff-f56a-c1ae-92ff143298d7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e9adca04-3f17-f15c-c035-a29de6c12ada Address:e9adca04-3f17-f15c-c035-a29de6c12ada} {Suffrage:Voter ID:server-a7f68db0-71a6-e748-9125-63b11b9b7cbb Address:a7f68db0-71a6-e748-9125-63b11b9b7cbb} {Suffrage:Voter ID:server-7365ee01-49ff-f56a-c1ae-92ff143298d7 Address:7365ee01-49ff-f56a-c1ae-92ff143298d7}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:19.952-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:19.952-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-a7f68db0-71a6-e748-9125-63b11b9b7cbb: 2024-06-11T02:29:19.953-1200 [INFO] server-a7f68db0-71a6-e748-9125-63b11b9b7cbb: entering follower state: follower="Node at a7f68db0-71a6-e748-9125-63b11b9b7cbb [Follower]" leader= - testing.go:149: server-a7f68db0-71a6-e748-9125-63b11b9b7cbb: 2024-06-11T02:29:19.953-1200 [INFO] server-a7f68db0-71a6-e748-9125-63b11b9b7cbb: entering follower state: follower="Node at a7f68db0-71a6-e748-9125-63b11b9b7cbb [Follower]" leader= -testLoggerAdapter verbose: server-7365ee01-49ff-f56a-c1ae-92ff143298d7: 2024-06-11T02:29:19.953-1200 [INFO] server-7365ee01-49ff-f56a-c1ae-92ff143298d7: entering follower state: follower="Node at 7365ee01-49ff-f56a-c1ae-92ff143298d7 [Follower]" leader= - testing.go:149: server-7365ee01-49ff-f56a-c1ae-92ff143298d7: 2024-06-11T02:29:19.953-1200 [INFO] server-7365ee01-49ff-f56a-c1ae-92ff143298d7: entering follower state: follower="Node at 7365ee01-49ff-f56a-c1ae-92ff143298d7 [Follower]" leader= -testLoggerAdapter verbose: server-7365ee01-49ff-f56a-c1ae-92ff143298d7: 2024-06-11T02:29:20.018-1200 [WARN] server-7365ee01-49ff-f56a-c1ae-92ff143298d7: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-7365ee01-49ff-f56a-c1ae-92ff143298d7: 2024-06-11T02:29:20.018-1200 [WARN] server-7365ee01-49ff-f56a-c1ae-92ff143298d7: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-7365ee01-49ff-f56a-c1ae-92ff143298d7: 2024-06-11T02:29:20.018-1200 [INFO] server-7365ee01-49ff-f56a-c1ae-92ff143298d7: entering candidate state: node="Node at 7365ee01-49ff-f56a-c1ae-92ff143298d7 [Candidate]" term=2 - testing.go:149: server-7365ee01-49ff-f56a-c1ae-92ff143298d7: 2024-06-11T02:29:20.018-1200 [INFO] server-7365ee01-49ff-f56a-c1ae-92ff143298d7: entering candidate state: node="Node at 7365ee01-49ff-f56a-c1ae-92ff143298d7 [Candidate]" term=2 -testLoggerAdapter verbose: server-7365ee01-49ff-f56a-c1ae-92ff143298d7: 2024-06-11T02:29:20.018-1200 [INFO] server-7365ee01-49ff-f56a-c1ae-92ff143298d7: election won: tally=2 - testing.go:149: server-7365ee01-49ff-f56a-c1ae-92ff143298d7: 2024-06-11T02:29:20.018-1200 [INFO] server-7365ee01-49ff-f56a-c1ae-92ff143298d7: election won: tally=2 -testLoggerAdapter verbose: server-7365ee01-49ff-f56a-c1ae-92ff143298d7: 2024-06-11T02:29:20.018-1200 [INFO] server-7365ee01-49ff-f56a-c1ae-92ff143298d7: entering leader state: leader="Node at 7365ee01-49ff-f56a-c1ae-92ff143298d7 [Leader]" - testing.go:149: server-7365ee01-49ff-f56a-c1ae-92ff143298d7: 2024-06-11T02:29:20.018-1200 [INFO] server-7365ee01-49ff-f56a-c1ae-92ff143298d7: entering leader state: leader="Node at 7365ee01-49ff-f56a-c1ae-92ff143298d7 [Leader]" -testLoggerAdapter verbose: server-7365ee01-49ff-f56a-c1ae-92ff143298d7: 2024-06-11T02:29:20.018-1200 [INFO] server-7365ee01-49ff-f56a-c1ae-92ff143298d7: added peer, starting replication: peer=server-e9adca04-3f17-f15c-c035-a29de6c12ada - testing.go:149: server-7365ee01-49ff-f56a-c1ae-92ff143298d7: 2024-06-11T02:29:20.018-1200 [INFO] server-7365ee01-49ff-f56a-c1ae-92ff143298d7: added peer, starting replication: peer=server-e9adca04-3f17-f15c-c035-a29de6c12ada -testLoggerAdapter verbose: server-7365ee01-49ff-f56a-c1ae-92ff143298d7: 2024-06-11T02:29:20.018-1200 [INFO] server-7365ee01-49ff-f56a-c1ae-92ff143298d7: added peer, starting replication: peer=server-a7f68db0-71a6-e748-9125-63b11b9b7cbb - testing.go:149: server-7365ee01-49ff-f56a-c1ae-92ff143298d7: 2024-06-11T02:29:20.018-1200 [INFO] server-7365ee01-49ff-f56a-c1ae-92ff143298d7: added peer, starting replication: peer=server-a7f68db0-71a6-e748-9125-63b11b9b7cbb -testLoggerAdapter verbose: server-7365ee01-49ff-f56a-c1ae-92ff143298d7: 2024-06-11T02:29:20.019-1200 [INFO] server-7365ee01-49ff-f56a-c1ae-92ff143298d7: pipelining replication: peer="{Voter server-a7f68db0-71a6-e748-9125-63b11b9b7cbb a7f68db0-71a6-e748-9125-63b11b9b7cbb}" - testing.go:149: server-7365ee01-49ff-f56a-c1ae-92ff143298d7: 2024-06-11T02:29:20.019-1200 [INFO] server-7365ee01-49ff-f56a-c1ae-92ff143298d7: pipelining replication: peer="{Voter server-a7f68db0-71a6-e748-9125-63b11b9b7cbb a7f68db0-71a6-e748-9125-63b11b9b7cbb}" -testLoggerAdapter verbose: server-7365ee01-49ff-f56a-c1ae-92ff143298d7: 2024-06-11T02:29:20.019-1200 [INFO] server-7365ee01-49ff-f56a-c1ae-92ff143298d7: pipelining replication: peer="{Voter server-e9adca04-3f17-f15c-c035-a29de6c12ada e9adca04-3f17-f15c-c035-a29de6c12ada}" - testing.go:149: server-7365ee01-49ff-f56a-c1ae-92ff143298d7: 2024-06-11T02:29:20.019-1200 [INFO] server-7365ee01-49ff-f56a-c1ae-92ff143298d7: pipelining replication: peer="{Voter server-e9adca04-3f17-f15c-c035-a29de6c12ada e9adca04-3f17-f15c-c035-a29de6c12ada}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:20.125-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:20.01968295 -1200 -12 m=+11.015055491 (1 nodes), 66.693287ms from start of poll, 69.236864ms from cluster start. Timeout at 2024-06-11 02:29:20.124882221 -1200 -12 m=+11.120254762, 105.199271ms after stability - testing.go:149: cluster: 2024-06-11T02:29:20.125-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:20.01968295 -1200 -12 m=+11.015055491 (1 nodes), 66.693287ms from start of poll, 69.236864ms from cluster start. Timeout at 2024-06-11 02:29:20.124882221 -1200 -12 m=+11.120254762, 105.199271ms after stability -testLoggerAdapter verbose: server-7365ee01-49ff-f56a-c1ae-92ff143298d7: 2024-06-11T02:29:20.125-1200 [INFO] server-7365ee01-49ff-f56a-c1ae-92ff143298d7: updating configuration: command=RemoveServer server-id=8263fc30-b3bc-d962-461d-ba545e8d9da4 server-addr= servers="[{Suffrage:Voter ID:server-e9adca04-3f17-f15c-c035-a29de6c12ada Address:e9adca04-3f17-f15c-c035-a29de6c12ada} {Suffrage:Voter ID:server-a7f68db0-71a6-e748-9125-63b11b9b7cbb Address:a7f68db0-71a6-e748-9125-63b11b9b7cbb} {Suffrage:Voter ID:server-7365ee01-49ff-f56a-c1ae-92ff143298d7 Address:7365ee01-49ff-f56a-c1ae-92ff143298d7}]" - testing.go:149: server-7365ee01-49ff-f56a-c1ae-92ff143298d7: 2024-06-11T02:29:20.125-1200 [INFO] server-7365ee01-49ff-f56a-c1ae-92ff143298d7: updating configuration: command=RemoveServer server-id=8263fc30-b3bc-d962-461d-ba545e8d9da4 server-addr= servers="[{Suffrage:Voter ID:server-e9adca04-3f17-f15c-c035-a29de6c12ada Address:e9adca04-3f17-f15c-c035-a29de6c12ada} {Suffrage:Voter ID:server-a7f68db0-71a6-e748-9125-63b11b9b7cbb Address:a7f68db0-71a6-e748-9125-63b11b9b7cbb} {Suffrage:Voter ID:server-7365ee01-49ff-f56a-c1ae-92ff143298d7 Address:7365ee01-49ff-f56a-c1ae-92ff143298d7}]" -testLoggerAdapter verbose: server-7365ee01-49ff-f56a-c1ae-92ff143298d7: 2024-06-11T02:29:20.126-1200 [INFO] server-7365ee01-49ff-f56a-c1ae-92ff143298d7: aborting pipeline replication: peer="{Voter server-e9adca04-3f17-f15c-c035-a29de6c12ada e9adca04-3f17-f15c-c035-a29de6c12ada}" - testing.go:149: server-7365ee01-49ff-f56a-c1ae-92ff143298d7: 2024-06-11T02:29:20.126-1200 [INFO] server-7365ee01-49ff-f56a-c1ae-92ff143298d7: aborting pipeline replication: peer="{Voter server-e9adca04-3f17-f15c-c035-a29de6c12ada e9adca04-3f17-f15c-c035-a29de6c12ada}" -testLoggerAdapter verbose: server-7365ee01-49ff-f56a-c1ae-92ff143298d7: 2024-06-11T02:29:20.126-1200 [INFO] server-7365ee01-49ff-f56a-c1ae-92ff143298d7: aborting pipeline replication: peer="{Voter server-a7f68db0-71a6-e748-9125-63b11b9b7cbb a7f68db0-71a6-e748-9125-63b11b9b7cbb}" - testing.go:149: server-7365ee01-49ff-f56a-c1ae-92ff143298d7: 2024-06-11T02:29:20.126-1200 [INFO] server-7365ee01-49ff-f56a-c1ae-92ff143298d7: aborting pipeline replication: peer="{Voter server-a7f68db0-71a6-e748-9125-63b11b9b7cbb a7f68db0-71a6-e748-9125-63b11b9b7cbb}" ---- PASS: TestRaft_RemoveUnknownPeer (0.18s) +testLoggerAdapter verbose: server-a26699da-244b-4810-d306-09f303bf1791: 2024-06-12T04:32:33.284+1400 [INFO] server-a26699da-244b-4810-d306-09f303bf1791: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a26699da-244b-4810-d306-09f303bf1791 Address:a26699da-244b-4810-d306-09f303bf1791} {Suffrage:Voter ID:server-104b0210-66ce-5126-f4a9-3e646128c308 Address:104b0210-66ce-5126-f4a9-3e646128c308} {Suffrage:Voter ID:server-1dcf86cc-12f0-c8c7-4730-9f107a299f98 Address:1dcf86cc-12f0-c8c7-4730-9f107a299f98}]" + testing.go:149: server-a26699da-244b-4810-d306-09f303bf1791: 2024-06-12T04:32:33.284+1400 [INFO] server-a26699da-244b-4810-d306-09f303bf1791: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a26699da-244b-4810-d306-09f303bf1791 Address:a26699da-244b-4810-d306-09f303bf1791} {Suffrage:Voter ID:server-104b0210-66ce-5126-f4a9-3e646128c308 Address:104b0210-66ce-5126-f4a9-3e646128c308} {Suffrage:Voter ID:server-1dcf86cc-12f0-c8c7-4730-9f107a299f98 Address:1dcf86cc-12f0-c8c7-4730-9f107a299f98}]" +testLoggerAdapter verbose: server-104b0210-66ce-5126-f4a9-3e646128c308: 2024-06-12T04:32:33.284+1400 [INFO] server-104b0210-66ce-5126-f4a9-3e646128c308: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a26699da-244b-4810-d306-09f303bf1791 Address:a26699da-244b-4810-d306-09f303bf1791} {Suffrage:Voter ID:server-104b0210-66ce-5126-f4a9-3e646128c308 Address:104b0210-66ce-5126-f4a9-3e646128c308} {Suffrage:Voter ID:server-1dcf86cc-12f0-c8c7-4730-9f107a299f98 Address:1dcf86cc-12f0-c8c7-4730-9f107a299f98}]" + testing.go:149: server-104b0210-66ce-5126-f4a9-3e646128c308: 2024-06-12T04:32:33.284+1400 [INFO] server-104b0210-66ce-5126-f4a9-3e646128c308: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a26699da-244b-4810-d306-09f303bf1791 Address:a26699da-244b-4810-d306-09f303bf1791} {Suffrage:Voter ID:server-104b0210-66ce-5126-f4a9-3e646128c308 Address:104b0210-66ce-5126-f4a9-3e646128c308} {Suffrage:Voter ID:server-1dcf86cc-12f0-c8c7-4730-9f107a299f98 Address:1dcf86cc-12f0-c8c7-4730-9f107a299f98}]" +testLoggerAdapter verbose: server-1dcf86cc-12f0-c8c7-4730-9f107a299f98: 2024-06-12T04:32:33.285+1400 [INFO] server-1dcf86cc-12f0-c8c7-4730-9f107a299f98: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a26699da-244b-4810-d306-09f303bf1791 Address:a26699da-244b-4810-d306-09f303bf1791} {Suffrage:Voter ID:server-104b0210-66ce-5126-f4a9-3e646128c308 Address:104b0210-66ce-5126-f4a9-3e646128c308} {Suffrage:Voter ID:server-1dcf86cc-12f0-c8c7-4730-9f107a299f98 Address:1dcf86cc-12f0-c8c7-4730-9f107a299f98}]" + testing.go:149: server-1dcf86cc-12f0-c8c7-4730-9f107a299f98: 2024-06-12T04:32:33.285+1400 [INFO] server-1dcf86cc-12f0-c8c7-4730-9f107a299f98: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a26699da-244b-4810-d306-09f303bf1791 Address:a26699da-244b-4810-d306-09f303bf1791} {Suffrage:Voter ID:server-104b0210-66ce-5126-f4a9-3e646128c308 Address:104b0210-66ce-5126-f4a9-3e646128c308} {Suffrage:Voter ID:server-1dcf86cc-12f0-c8c7-4730-9f107a299f98 Address:1dcf86cc-12f0-c8c7-4730-9f107a299f98}]" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:33.285+1400 [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-06-12T04:32:33.285+1400 [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-a26699da-244b-4810-d306-09f303bf1791: 2024-06-12T04:32:33.285+1400 [INFO] server-a26699da-244b-4810-d306-09f303bf1791: entering follower state: follower="Node at a26699da-244b-4810-d306-09f303bf1791 [Follower]" leader= + testing.go:149: server-a26699da-244b-4810-d306-09f303bf1791: 2024-06-12T04:32:33.285+1400 [INFO] server-a26699da-244b-4810-d306-09f303bf1791: entering follower state: follower="Node at a26699da-244b-4810-d306-09f303bf1791 [Follower]" leader= +testLoggerAdapter verbose: server-104b0210-66ce-5126-f4a9-3e646128c308: 2024-06-12T04:32:33.285+1400 [INFO] server-104b0210-66ce-5126-f4a9-3e646128c308: entering follower state: follower="Node at 104b0210-66ce-5126-f4a9-3e646128c308 [Follower]" leader= + testing.go:149: server-104b0210-66ce-5126-f4a9-3e646128c308: 2024-06-12T04:32:33.285+1400 [INFO] server-104b0210-66ce-5126-f4a9-3e646128c308: entering follower state: follower="Node at 104b0210-66ce-5126-f4a9-3e646128c308 [Follower]" leader= +testLoggerAdapter verbose: server-1dcf86cc-12f0-c8c7-4730-9f107a299f98: 2024-06-12T04:32:33.285+1400 [INFO] server-1dcf86cc-12f0-c8c7-4730-9f107a299f98: entering follower state: follower="Node at 1dcf86cc-12f0-c8c7-4730-9f107a299f98 [Follower]" leader= + testing.go:149: server-1dcf86cc-12f0-c8c7-4730-9f107a299f98: 2024-06-12T04:32:33.285+1400 [INFO] server-1dcf86cc-12f0-c8c7-4730-9f107a299f98: entering follower state: follower="Node at 1dcf86cc-12f0-c8c7-4730-9f107a299f98 [Follower]" leader= +testLoggerAdapter verbose: server-104b0210-66ce-5126-f4a9-3e646128c308: 2024-06-12T04:32:33.343+1400 [WARN] server-104b0210-66ce-5126-f4a9-3e646128c308: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-104b0210-66ce-5126-f4a9-3e646128c308: 2024-06-12T04:32:33.343+1400 [WARN] server-104b0210-66ce-5126-f4a9-3e646128c308: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-104b0210-66ce-5126-f4a9-3e646128c308: 2024-06-12T04:32:33.343+1400 [INFO] server-104b0210-66ce-5126-f4a9-3e646128c308: entering candidate state: node="Node at 104b0210-66ce-5126-f4a9-3e646128c308 [Candidate]" term=2 + testing.go:149: server-104b0210-66ce-5126-f4a9-3e646128c308: 2024-06-12T04:32:33.343+1400 [INFO] server-104b0210-66ce-5126-f4a9-3e646128c308: entering candidate state: node="Node at 104b0210-66ce-5126-f4a9-3e646128c308 [Candidate]" term=2 +testLoggerAdapter verbose: server-104b0210-66ce-5126-f4a9-3e646128c308: 2024-06-12T04:32:33.343+1400 [INFO] server-104b0210-66ce-5126-f4a9-3e646128c308: election won: tally=2 + testing.go:149: server-104b0210-66ce-5126-f4a9-3e646128c308: 2024-06-12T04:32:33.343+1400 [INFO] server-104b0210-66ce-5126-f4a9-3e646128c308: election won: tally=2 +testLoggerAdapter verbose: server-104b0210-66ce-5126-f4a9-3e646128c308: 2024-06-12T04:32:33.343+1400 [INFO] server-104b0210-66ce-5126-f4a9-3e646128c308: entering leader state: leader="Node at 104b0210-66ce-5126-f4a9-3e646128c308 [Leader]" + testing.go:149: server-104b0210-66ce-5126-f4a9-3e646128c308: 2024-06-12T04:32:33.343+1400 [INFO] server-104b0210-66ce-5126-f4a9-3e646128c308: entering leader state: leader="Node at 104b0210-66ce-5126-f4a9-3e646128c308 [Leader]" +testLoggerAdapter verbose: server-104b0210-66ce-5126-f4a9-3e646128c308: 2024-06-12T04:32:33.343+1400 [INFO] server-104b0210-66ce-5126-f4a9-3e646128c308: added peer, starting replication: peer=server-a26699da-244b-4810-d306-09f303bf1791 + testing.go:149: server-104b0210-66ce-5126-f4a9-3e646128c308: 2024-06-12T04:32:33.343+1400 [INFO] server-104b0210-66ce-5126-f4a9-3e646128c308: added peer, starting replication: peer=server-a26699da-244b-4810-d306-09f303bf1791 +testLoggerAdapter verbose: server-104b0210-66ce-5126-f4a9-3e646128c308: 2024-06-12T04:32:33.343+1400 [INFO] server-104b0210-66ce-5126-f4a9-3e646128c308: added peer, starting replication: peer=server-1dcf86cc-12f0-c8c7-4730-9f107a299f98 + testing.go:149: server-104b0210-66ce-5126-f4a9-3e646128c308: 2024-06-12T04:32:33.343+1400 [INFO] server-104b0210-66ce-5126-f4a9-3e646128c308: added peer, starting replication: peer=server-1dcf86cc-12f0-c8c7-4730-9f107a299f98 +testLoggerAdapter verbose: server-104b0210-66ce-5126-f4a9-3e646128c308: 2024-06-12T04:32:33.344+1400 [INFO] server-104b0210-66ce-5126-f4a9-3e646128c308: pipelining replication: peer="{Voter server-a26699da-244b-4810-d306-09f303bf1791 a26699da-244b-4810-d306-09f303bf1791}" + testing.go:149: server-104b0210-66ce-5126-f4a9-3e646128c308: 2024-06-12T04:32:33.344+1400 [INFO] server-104b0210-66ce-5126-f4a9-3e646128c308: pipelining replication: peer="{Voter server-a26699da-244b-4810-d306-09f303bf1791 a26699da-244b-4810-d306-09f303bf1791}" +testLoggerAdapter verbose: server-104b0210-66ce-5126-f4a9-3e646128c308: 2024-06-12T04:32:33.344+1400 [INFO] server-104b0210-66ce-5126-f4a9-3e646128c308: pipelining replication: peer="{Voter server-1dcf86cc-12f0-c8c7-4730-9f107a299f98 1dcf86cc-12f0-c8c7-4730-9f107a299f98}" + testing.go:149: server-104b0210-66ce-5126-f4a9-3e646128c308: 2024-06-12T04:32:33.344+1400 [INFO] server-104b0210-66ce-5126-f4a9-3e646128c308: pipelining replication: peer="{Voter server-1dcf86cc-12f0-c8c7-4730-9f107a299f98 1dcf86cc-12f0-c8c7-4730-9f107a299f98}" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:33.449+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:33.343860068 +1400 +14 m=+10.251378687 (1 nodes), 58.666539ms from start of poll, 59.437071ms from cluster start. Timeout at 2024-06-12 04:32:33.449532963 +1400 +14 m=+10.357051582, 105.672895ms after stability + testing.go:149: cluster: 2024-06-12T04:32:33.449+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:33.343860068 +1400 +14 m=+10.251378687 (1 nodes), 58.666539ms from start of poll, 59.437071ms from cluster start. Timeout at 2024-06-12 04:32:33.449532963 +1400 +14 m=+10.357051582, 105.672895ms after stability +testLoggerAdapter verbose: server-104b0210-66ce-5126-f4a9-3e646128c308: 2024-06-12T04:32:33.449+1400 [INFO] server-104b0210-66ce-5126-f4a9-3e646128c308: updating configuration: command=RemoveServer server-id=785bb06e-d5af-b38c-cd05-ae5106d67551 server-addr= servers="[{Suffrage:Voter ID:server-a26699da-244b-4810-d306-09f303bf1791 Address:a26699da-244b-4810-d306-09f303bf1791} {Suffrage:Voter ID:server-104b0210-66ce-5126-f4a9-3e646128c308 Address:104b0210-66ce-5126-f4a9-3e646128c308} {Suffrage:Voter ID:server-1dcf86cc-12f0-c8c7-4730-9f107a299f98 Address:1dcf86cc-12f0-c8c7-4730-9f107a299f98}]" + testing.go:149: server-104b0210-66ce-5126-f4a9-3e646128c308: 2024-06-12T04:32:33.449+1400 [INFO] server-104b0210-66ce-5126-f4a9-3e646128c308: updating configuration: command=RemoveServer server-id=785bb06e-d5af-b38c-cd05-ae5106d67551 server-addr= servers="[{Suffrage:Voter ID:server-a26699da-244b-4810-d306-09f303bf1791 Address:a26699da-244b-4810-d306-09f303bf1791} {Suffrage:Voter ID:server-104b0210-66ce-5126-f4a9-3e646128c308 Address:104b0210-66ce-5126-f4a9-3e646128c308} {Suffrage:Voter ID:server-1dcf86cc-12f0-c8c7-4730-9f107a299f98 Address:1dcf86cc-12f0-c8c7-4730-9f107a299f98}]" +testLoggerAdapter verbose: server-104b0210-66ce-5126-f4a9-3e646128c308: 2024-06-12T04:32:33.450+1400 [INFO] server-104b0210-66ce-5126-f4a9-3e646128c308: aborting pipeline replication: peer="{Voter server-a26699da-244b-4810-d306-09f303bf1791 a26699da-244b-4810-d306-09f303bf1791}" + testing.go:149: server-104b0210-66ce-5126-f4a9-3e646128c308: 2024-06-12T04:32:33.450+1400 [INFO] server-104b0210-66ce-5126-f4a9-3e646128c308: aborting pipeline replication: peer="{Voter server-a26699da-244b-4810-d306-09f303bf1791 a26699da-244b-4810-d306-09f303bf1791}" +testLoggerAdapter verbose: server-104b0210-66ce-5126-f4a9-3e646128c308: 2024-06-12T04:32:33.450+1400 [INFO] server-104b0210-66ce-5126-f4a9-3e646128c308: aborting pipeline replication: peer="{Voter server-1dcf86cc-12f0-c8c7-4730-9f107a299f98 1dcf86cc-12f0-c8c7-4730-9f107a299f98}" + testing.go:149: server-104b0210-66ce-5126-f4a9-3e646128c308: 2024-06-12T04:32:33.450+1400 [INFO] server-104b0210-66ce-5126-f4a9-3e646128c308: aborting pipeline replication: peer="{Voter server-1dcf86cc-12f0-c8c7-4730-9f107a299f98 1dcf86cc-12f0-c8c7-4730-9f107a299f98}" +--- PASS: TestRaft_RemoveUnknownPeer (0.17s) === RUN TestRaft_SnapshotRestore -testLoggerAdapter verbose: server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: 2024-06-11T02:29:20.129-1200 [INFO] server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0 Address:14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0}]" - testing.go:149: server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: 2024-06-11T02:29:20.129-1200 [INFO] server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0 Address:14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:20.130-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:20.130-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: 2024-06-11T02:29:20.130-1200 [INFO] server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: entering follower state: follower="Node at 14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0 [Follower]" leader= - testing.go:149: server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: 2024-06-11T02:29:20.130-1200 [INFO] server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: entering follower state: follower="Node at 14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0 [Follower]" leader= -testLoggerAdapter verbose: server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: 2024-06-11T02:29:20.207-1200 [WARN] server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: 2024-06-11T02:29:20.207-1200 [WARN] server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: 2024-06-11T02:29:20.207-1200 [INFO] server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: entering candidate state: node="Node at 14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0 [Candidate]" term=2 - testing.go:149: server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: 2024-06-11T02:29:20.207-1200 [INFO] server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: entering candidate state: node="Node at 14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0 [Candidate]" term=2 -testLoggerAdapter verbose: server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: 2024-06-11T02:29:20.207-1200 [INFO] server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: election won: tally=1 - testing.go:149: server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: 2024-06-11T02:29:20.207-1200 [INFO] server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: election won: tally=1 -testLoggerAdapter verbose: server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: 2024-06-11T02:29:20.207-1200 [INFO] server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: entering leader state: leader="Node at 14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0 [Leader]" - testing.go:149: server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: 2024-06-11T02:29:20.207-1200 [INFO] server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: entering leader state: leader="Node at 14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0 [Leader]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:20.313-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:20.207863326 -1200 -12 m=+11.203235868 (1 nodes), 77.700872ms from start of poll, 78.408733ms from cluster start. Timeout at 2024-06-11 02:29:20.313404133 -1200 -12 m=+11.308776966, 105.541098ms after stability - testing.go:149: cluster: 2024-06-11T02:29:20.313-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:20.207863326 -1200 -12 m=+11.203235868 (1 nodes), 77.700872ms from start of poll, 78.408733ms from cluster start. Timeout at 2024-06-11 02:29:20.313404133 -1200 -12 m=+11.308776966, 105.541098ms after stability -testLoggerAdapter verbose: server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: 2024-06-11T02:29:20.316-1200 [INFO] server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: starting snapshot up to: index=102 - testing.go:149: server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: 2024-06-11T02:29:20.316-1200 [INFO] server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: starting snapshot up to: index=102 - testing.go:153: 2024-06-11T02:29:20.316-1200 [INFO] creating new snapshot: path=/tmp/raft2087944659/snapshots/2-102-1718116160316.tmp -testLoggerAdapter verbose: server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: 2024-06-11T02:29:20.339-1200 [INFO] server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: compacting logs: from=1 to=92 - testing.go:149: server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: 2024-06-11T02:29:20.339-1200 [INFO] server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: compacting logs: from=1 to=92 -testLoggerAdapter verbose: server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: 2024-06-11T02:29:20.339-1200 [INFO] server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: snapshot complete up to: index=102 - testing.go:149: server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: 2024-06-11T02:29:20.339-1200 [INFO] server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: snapshot complete up to: index=102 -testLoggerAdapter verbose: server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: 2024-06-11T02:29:20.340-1200 [INFO] server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: restored from snapshot: id=2-102-1718116160316 - testing.go:149: server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: 2024-06-11T02:29:20.340-1200 [INFO] server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: restored from snapshot: id=2-102-1718116160316 -testLoggerAdapter verbose: server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: 2024-06-11T02:29:20.340-1200 [INFO] server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0 Address:14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0}]" - testing.go:149: server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: 2024-06-11T02:29:20.340-1200 [INFO] server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0 Address:14c7b7ac-305e-98bf-6bf9-aa3d650f5fc0}]" ---- PASS: TestRaft_SnapshotRestore (0.21s) +testLoggerAdapter verbose: server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: 2024-06-12T04:32:33.451+1400 [INFO] server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fa3b7c40-9756-1c66-e936-5c0087df4c6e Address:fa3b7c40-9756-1c66-e936-5c0087df4c6e}]" + testing.go:149: server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: 2024-06-12T04:32:33.451+1400 [INFO] server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fa3b7c40-9756-1c66-e936-5c0087df4c6e Address:fa3b7c40-9756-1c66-e936-5c0087df4c6e}]" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:33.451+1400 [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-06-12T04:32:33.451+1400 [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: 2024-06-12T04:32:33.451+1400 [INFO] server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: entering follower state: follower="Node at fa3b7c40-9756-1c66-e936-5c0087df4c6e [Follower]" leader= + testing.go:149: server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: 2024-06-12T04:32:33.451+1400 [INFO] server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: entering follower state: follower="Node at fa3b7c40-9756-1c66-e936-5c0087df4c6e [Follower]" leader= +testLoggerAdapter verbose: server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: 2024-06-12T04:32:33.513+1400 [WARN] server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: 2024-06-12T04:32:33.513+1400 [WARN] server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: 2024-06-12T04:32:33.513+1400 [INFO] server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: entering candidate state: node="Node at fa3b7c40-9756-1c66-e936-5c0087df4c6e [Candidate]" term=2 + testing.go:149: server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: 2024-06-12T04:32:33.513+1400 [INFO] server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: entering candidate state: node="Node at fa3b7c40-9756-1c66-e936-5c0087df4c6e [Candidate]" term=2 +testLoggerAdapter verbose: server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: 2024-06-12T04:32:33.513+1400 [INFO] server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: election won: tally=1 + testing.go:149: server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: 2024-06-12T04:32:33.513+1400 [INFO] server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: election won: tally=1 +testLoggerAdapter verbose: server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: 2024-06-12T04:32:33.513+1400 [INFO] server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: entering leader state: leader="Node at fa3b7c40-9756-1c66-e936-5c0087df4c6e [Leader]" + testing.go:149: server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: 2024-06-12T04:32:33.513+1400 [INFO] server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: entering leader state: leader="Node at fa3b7c40-9756-1c66-e936-5c0087df4c6e [Leader]" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:33.619+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:33.51383066 +1400 +14 m=+10.421349279 (1 nodes), 61.955661ms from start of poll, 62.25981ms from cluster start. Timeout at 2024-06-12 04:32:33.61991862 +1400 +14 m=+10.527437239, 106.08796ms after stability + testing.go:149: cluster: 2024-06-12T04:32:33.619+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:33.51383066 +1400 +14 m=+10.421349279 (1 nodes), 61.955661ms from start of poll, 62.25981ms from cluster start. Timeout at 2024-06-12 04:32:33.61991862 +1400 +14 m=+10.527437239, 106.08796ms after stability +testLoggerAdapter verbose: server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: 2024-06-12T04:32:33.623+1400 [INFO] server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: starting snapshot up to: index=102 + testing.go:149: server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: 2024-06-12T04:32:33.623+1400 [INFO] server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: starting snapshot up to: index=102 + testing.go:153: 2024-06-12T04:32:33.624+1400 [INFO] creating new snapshot: path=/tmp/raft321927674/snapshots/2-102-1718116353624.tmp +testLoggerAdapter verbose: server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: 2024-06-12T04:32:33.640+1400 [INFO] server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: compacting logs: from=1 to=92 + testing.go:149: server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: 2024-06-12T04:32:33.640+1400 [INFO] server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: compacting logs: from=1 to=92 +testLoggerAdapter verbose: server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: 2024-06-12T04:32:33.640+1400 [INFO] server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: snapshot complete up to: index=102 + testing.go:149: server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: 2024-06-12T04:32:33.640+1400 [INFO] server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: snapshot complete up to: index=102 +testLoggerAdapter verbose: server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: 2024-06-12T04:32:33.641+1400 [INFO] server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: restored from snapshot: id=2-102-1718116353624 + testing.go:149: server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: 2024-06-12T04:32:33.641+1400 [INFO] server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: restored from snapshot: id=2-102-1718116353624 +testLoggerAdapter verbose: server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: 2024-06-12T04:32:33.641+1400 [INFO] server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fa3b7c40-9756-1c66-e936-5c0087df4c6e Address:fa3b7c40-9756-1c66-e936-5c0087df4c6e}]" + testing.go:149: server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: 2024-06-12T04:32:33.641+1400 [INFO] server-fa3b7c40-9756-1c66-e936-5c0087df4c6e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fa3b7c40-9756-1c66-e936-5c0087df4c6e Address:fa3b7c40-9756-1c66-e936-5c0087df4c6e}]" +--- PASS: TestRaft_SnapshotRestore (0.19s) === RUN TestRaft_NoRestoreOnStart -testLoggerAdapter verbose: server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: 2024-06-11T02:29:20.343-1200 [INFO] server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1 Address:56a25a78-d621-6b0a-01db-4cf0ae71b1e1}]" - testing.go:149: server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: 2024-06-11T02:29:20.343-1200 [INFO] server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1 Address:56a25a78-d621-6b0a-01db-4cf0ae71b1e1}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:20.343-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:20.343-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: 2024-06-11T02:29:20.343-1200 [INFO] server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: entering follower state: follower="Node at 56a25a78-d621-6b0a-01db-4cf0ae71b1e1 [Follower]" leader= - testing.go:149: server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: 2024-06-11T02:29:20.343-1200 [INFO] server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: entering follower state: follower="Node at 56a25a78-d621-6b0a-01db-4cf0ae71b1e1 [Follower]" leader= -testLoggerAdapter verbose: server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: 2024-06-11T02:29:20.434-1200 [WARN] server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: 2024-06-11T02:29:20.434-1200 [WARN] server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: 2024-06-11T02:29:20.434-1200 [INFO] server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: entering candidate state: node="Node at 56a25a78-d621-6b0a-01db-4cf0ae71b1e1 [Candidate]" term=2 - testing.go:149: server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: 2024-06-11T02:29:20.434-1200 [INFO] server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: entering candidate state: node="Node at 56a25a78-d621-6b0a-01db-4cf0ae71b1e1 [Candidate]" term=2 -testLoggerAdapter verbose: server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: 2024-06-11T02:29:20.435-1200 [INFO] server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: election won: tally=1 - testing.go:149: server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: 2024-06-11T02:29:20.435-1200 [INFO] server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: election won: tally=1 -testLoggerAdapter verbose: server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: 2024-06-11T02:29:20.435-1200 [INFO] server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: entering leader state: leader="Node at 56a25a78-d621-6b0a-01db-4cf0ae71b1e1 [Leader]" - testing.go:149: server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: 2024-06-11T02:29:20.435-1200 [INFO] server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: entering leader state: leader="Node at 56a25a78-d621-6b0a-01db-4cf0ae71b1e1 [Leader]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:20.541-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:20.435350005 -1200 -12 m=+11.430722547 (1 nodes), 91.914688ms from start of poll, 92.474677ms from cluster start. Timeout at 2024-06-11 02:29:20.540920269 -1200 -12 m=+11.536293103, 105.570556ms after stability - testing.go:149: cluster: 2024-06-11T02:29:20.541-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:20.435350005 -1200 -12 m=+11.430722547 (1 nodes), 91.914688ms from start of poll, 92.474677ms from cluster start. Timeout at 2024-06-11 02:29:20.540920269 -1200 -12 m=+11.536293103, 105.570556ms after stability -testLoggerAdapter verbose: server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: 2024-06-11T02:29:20.545-1200 [INFO] server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: starting snapshot up to: index=102 - testing.go:149: server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: 2024-06-11T02:29:20.545-1200 [INFO] server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: starting snapshot up to: index=102 - testing.go:153: 2024-06-11T02:29:20.545-1200 [INFO] creating new snapshot: path=/tmp/raft2082192713/snapshots/2-102-1718116160545.tmp -testLoggerAdapter verbose: server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: 2024-06-11T02:29:20.567-1200 [INFO] server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: compacting logs: from=1 to=92 - testing.go:149: server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: 2024-06-11T02:29:20.567-1200 [INFO] server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: compacting logs: from=1 to=92 -testLoggerAdapter verbose: server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: 2024-06-11T02:29:20.567-1200 [INFO] server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: snapshot complete up to: index=102 - testing.go:149: server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: 2024-06-11T02:29:20.567-1200 [INFO] server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: snapshot complete up to: index=102 -testLoggerAdapter verbose: server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: 2024-06-11T02:29:20.568-1200 [INFO] server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1 Address:56a25a78-d621-6b0a-01db-4cf0ae71b1e1}]" - testing.go:149: server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: 2024-06-11T02:29:20.568-1200 [INFO] server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1 Address:56a25a78-d621-6b0a-01db-4cf0ae71b1e1}]" ---- PASS: TestRaft_NoRestoreOnStart (0.23s) +testLoggerAdapter verbose: server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: 2024-06-12T04:32:33.642+1400 [INFO] server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b3285295-113b-81b1-255e-ae5ef1b3d8ea Address:b3285295-113b-81b1-255e-ae5ef1b3d8ea}]" + testing.go:149: server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: 2024-06-12T04:32:33.642+1400 [INFO] server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b3285295-113b-81b1-255e-ae5ef1b3d8ea Address:b3285295-113b-81b1-255e-ae5ef1b3d8ea}]" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:33.642+1400 [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-06-12T04:32:33.642+1400 [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: 2024-06-12T04:32:33.642+1400 [INFO] server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: entering follower state: follower="Node at b3285295-113b-81b1-255e-ae5ef1b3d8ea [Follower]" leader= + testing.go:149: server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: 2024-06-12T04:32:33.642+1400 [INFO] server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: entering follower state: follower="Node at b3285295-113b-81b1-255e-ae5ef1b3d8ea [Follower]" leader= +testLoggerAdapter verbose: server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: 2024-06-12T04:32:33.726+1400 [WARN] server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: 2024-06-12T04:32:33.726+1400 [WARN] server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: 2024-06-12T04:32:33.726+1400 [INFO] server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: entering candidate state: node="Node at b3285295-113b-81b1-255e-ae5ef1b3d8ea [Candidate]" term=2 + testing.go:149: server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: 2024-06-12T04:32:33.726+1400 [INFO] server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: entering candidate state: node="Node at b3285295-113b-81b1-255e-ae5ef1b3d8ea [Candidate]" term=2 +testLoggerAdapter verbose: server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: 2024-06-12T04:32:33.726+1400 [INFO] server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: election won: tally=1 + testing.go:149: server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: 2024-06-12T04:32:33.726+1400 [INFO] server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: election won: tally=1 +testLoggerAdapter verbose: server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: 2024-06-12T04:32:33.726+1400 [INFO] server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: entering leader state: leader="Node at b3285295-113b-81b1-255e-ae5ef1b3d8ea [Leader]" + testing.go:149: server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: 2024-06-12T04:32:33.726+1400 [INFO] server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: entering leader state: leader="Node at b3285295-113b-81b1-255e-ae5ef1b3d8ea [Leader]" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:33.832+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:33.726804951 +1400 +14 m=+10.634323530 (1 nodes), 84.196664ms from start of poll, 84.532292ms from cluster start. Timeout at 2024-06-12 04:32:33.832781635 +1400 +14 m=+10.740300214, 105.976684ms after stability + testing.go:149: cluster: 2024-06-12T04:32:33.832+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:33.726804951 +1400 +14 m=+10.634323530 (1 nodes), 84.196664ms from start of poll, 84.532292ms from cluster start. Timeout at 2024-06-12 04:32:33.832781635 +1400 +14 m=+10.740300214, 105.976684ms after stability +testLoggerAdapter verbose: server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: 2024-06-12T04:32:33.834+1400 [INFO] server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: starting snapshot up to: index=102 + testing.go:149: server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: 2024-06-12T04:32:33.834+1400 [INFO] server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: starting snapshot up to: index=102 + testing.go:153: 2024-06-12T04:32:33.834+1400 [INFO] creating new snapshot: path=/tmp/raft3375294114/snapshots/2-102-1718116353834.tmp +testLoggerAdapter verbose: server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: 2024-06-12T04:32:33.848+1400 [INFO] server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: compacting logs: from=1 to=92 + testing.go:149: server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: 2024-06-12T04:32:33.848+1400 [INFO] server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: compacting logs: from=1 to=92 +testLoggerAdapter verbose: server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: 2024-06-12T04:32:33.848+1400 [INFO] server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: snapshot complete up to: index=102 + testing.go:149: server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: 2024-06-12T04:32:33.848+1400 [INFO] server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: snapshot complete up to: index=102 +testLoggerAdapter verbose: server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: 2024-06-12T04:32:33.848+1400 [INFO] server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b3285295-113b-81b1-255e-ae5ef1b3d8ea Address:b3285295-113b-81b1-255e-ae5ef1b3d8ea}]" + testing.go:149: server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: 2024-06-12T04:32:33.848+1400 [INFO] server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b3285295-113b-81b1-255e-ae5ef1b3d8ea Address:b3285295-113b-81b1-255e-ae5ef1b3d8ea}]" +--- PASS: TestRaft_NoRestoreOnStart (0.21s) === RUN TestRaft_SnapshotRestore_PeerChange raft_test.go:1026: Disabled unreliable test --- SKIP: TestRaft_SnapshotRestore_PeerChange (0.00s) -testLoggerAdapter verbose: server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: 2024-06-11T02:29:20.568-1200 [INFO] server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: entering follower state: follower="Node at 56a25a78-d621-6b0a-01db-4cf0ae71b1e1 [Follower]" leader= === RUN TestRaft_AutoSnapshot raft_test.go:1132: Disabled unreliable test --- SKIP: TestRaft_AutoSnapshot (0.00s) === RUN TestRaft_UserSnapshot -testLoggerAdapter verbose: server-8d238717-e4aa-012c-828d-5f412e067807: 2024-06-11T02:29:20.569-1200 [INFO] server-8d238717-e4aa-012c-828d-5f412e067807: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8d238717-e4aa-012c-828d-5f412e067807 Address:8d238717-e4aa-012c-828d-5f412e067807}]" - testing.go:149: server-8d238717-e4aa-012c-828d-5f412e067807: 2024-06-11T02:29:20.569-1200 [INFO] server-8d238717-e4aa-012c-828d-5f412e067807: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8d238717-e4aa-012c-828d-5f412e067807 Address:8d238717-e4aa-012c-828d-5f412e067807}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:20.569-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:20.569-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-8d238717-e4aa-012c-828d-5f412e067807: 2024-06-11T02:29:20.569-1200 [INFO] server-8d238717-e4aa-012c-828d-5f412e067807: entering follower state: follower="Node at 8d238717-e4aa-012c-828d-5f412e067807 [Follower]" leader= - testing.go:149: server-8d238717-e4aa-012c-828d-5f412e067807: 2024-06-11T02:29:20.569-1200 [INFO] server-8d238717-e4aa-012c-828d-5f412e067807: entering follower state: follower="Node at 8d238717-e4aa-012c-828d-5f412e067807 [Follower]" leader= -testLoggerAdapter verbose: server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: 2024-06-11T02:29:20.646-1200 [WARN] server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: 2024-06-11T02:29:20.646-1200 [INFO] server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: entering candidate state: node="Node at 56a25a78-d621-6b0a-01db-4cf0ae71b1e1 [Candidate]" term=3 -testLoggerAdapter verbose: server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: 2024-06-11T02:29:20.646-1200 [INFO] server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: election won: tally=1 -testLoggerAdapter verbose: server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: 2024-06-11T02:29:20.646-1200 [INFO] server-56a25a78-d621-6b0a-01db-4cf0ae71b1e1: entering leader state: leader="Node at 56a25a78-d621-6b0a-01db-4cf0ae71b1e1 [Leader]" -testLoggerAdapter verbose: server-8d238717-e4aa-012c-828d-5f412e067807: 2024-06-11T02:29:20.667-1200 [WARN] server-8d238717-e4aa-012c-828d-5f412e067807: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-8d238717-e4aa-012c-828d-5f412e067807: 2024-06-11T02:29:20.667-1200 [WARN] server-8d238717-e4aa-012c-828d-5f412e067807: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-8d238717-e4aa-012c-828d-5f412e067807: 2024-06-11T02:29:20.668-1200 [INFO] server-8d238717-e4aa-012c-828d-5f412e067807: entering candidate state: node="Node at 8d238717-e4aa-012c-828d-5f412e067807 [Candidate]" term=2 - testing.go:149: server-8d238717-e4aa-012c-828d-5f412e067807: 2024-06-11T02:29:20.668-1200 [INFO] server-8d238717-e4aa-012c-828d-5f412e067807: entering candidate state: node="Node at 8d238717-e4aa-012c-828d-5f412e067807 [Candidate]" term=2 -testLoggerAdapter verbose: server-8d238717-e4aa-012c-828d-5f412e067807: 2024-06-11T02:29:20.668-1200 [INFO] server-8d238717-e4aa-012c-828d-5f412e067807: election won: tally=1 - testing.go:149: server-8d238717-e4aa-012c-828d-5f412e067807: 2024-06-11T02:29:20.668-1200 [INFO] server-8d238717-e4aa-012c-828d-5f412e067807: election won: tally=1 -testLoggerAdapter verbose: server-8d238717-e4aa-012c-828d-5f412e067807: 2024-06-11T02:29:20.668-1200 [INFO] server-8d238717-e4aa-012c-828d-5f412e067807: entering leader state: leader="Node at 8d238717-e4aa-012c-828d-5f412e067807 [Leader]" - testing.go:149: server-8d238717-e4aa-012c-828d-5f412e067807: 2024-06-11T02:29:20.668-1200 [INFO] server-8d238717-e4aa-012c-828d-5f412e067807: entering leader state: leader="Node at 8d238717-e4aa-012c-828d-5f412e067807 [Leader]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:20.774-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:20.66872999 -1200 -12 m=+11.664102824 (1 nodes), 99.00118ms from start of poll, 99.442171ms from cluster start. Timeout at 2024-06-11 02:29:20.774318046 -1200 -12 m=+11.769691171, 105.588347ms after stability - testing.go:149: cluster: 2024-06-11T02:29:20.774-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:20.66872999 -1200 -12 m=+11.664102824 (1 nodes), 99.00118ms from start of poll, 99.442171ms from cluster start. Timeout at 2024-06-11 02:29:20.774318046 -1200 -12 m=+11.769691171, 105.588347ms after stability -testLoggerAdapter verbose: server-8d238717-e4aa-012c-828d-5f412e067807: 2024-06-11T02:29:20.774-1200 [ERROR] server-8d238717-e4aa-012c-828d-5f412e067807: failed to take snapshot: error="nothing new to snapshot" - testing.go:149: server-8d238717-e4aa-012c-828d-5f412e067807: 2024-06-11T02:29:20.774-1200 [ERROR] server-8d238717-e4aa-012c-828d-5f412e067807: failed to take snapshot: error="nothing new to snapshot" -testLoggerAdapter verbose: server-8d238717-e4aa-012c-828d-5f412e067807: 2024-06-11T02:29:20.775-1200 [INFO] server-8d238717-e4aa-012c-828d-5f412e067807: starting snapshot up to: index=12 - testing.go:149: server-8d238717-e4aa-012c-828d-5f412e067807: 2024-06-11T02:29:20.775-1200 [INFO] server-8d238717-e4aa-012c-828d-5f412e067807: starting snapshot up to: index=12 - testing.go:153: 2024-06-11T02:29:20.775-1200 [INFO] creating new snapshot: path=/tmp/raft3558854973/snapshots/2-12-1718116160775.tmp -testLoggerAdapter verbose: server-8d238717-e4aa-012c-828d-5f412e067807: 2024-06-11T02:29:20.803-1200 [INFO] server-8d238717-e4aa-012c-828d-5f412e067807: compacting logs: from=1 to=2 - testing.go:149: server-8d238717-e4aa-012c-828d-5f412e067807: 2024-06-11T02:29:20.803-1200 [INFO] server-8d238717-e4aa-012c-828d-5f412e067807: compacting logs: from=1 to=2 -testLoggerAdapter verbose: server-8d238717-e4aa-012c-828d-5f412e067807: 2024-06-11T02:29:20.803-1200 [INFO] server-8d238717-e4aa-012c-828d-5f412e067807: snapshot complete up to: index=12 - testing.go:149: server-8d238717-e4aa-012c-828d-5f412e067807: 2024-06-11T02:29:20.803-1200 [INFO] server-8d238717-e4aa-012c-828d-5f412e067807: snapshot complete up to: index=12 ---- PASS: TestRaft_UserSnapshot (0.24s) +testLoggerAdapter verbose: server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: 2024-06-12T04:32:33.849+1400 [INFO] server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b3fd43eb-bd2c-988a-969a-f6131bc82b82 Address:b3fd43eb-bd2c-988a-969a-f6131bc82b82}]" + testing.go:149: server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: 2024-06-12T04:32:33.849+1400 [INFO] server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b3fd43eb-bd2c-988a-969a-f6131bc82b82 Address:b3fd43eb-bd2c-988a-969a-f6131bc82b82}]" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:33.849+1400 [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-06-12T04:32:33.849+1400 [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: 2024-06-12T04:32:33.849+1400 [INFO] server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: entering follower state: follower="Node at b3285295-113b-81b1-255e-ae5ef1b3d8ea [Follower]" leader= +testLoggerAdapter verbose: server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: 2024-06-12T04:32:33.849+1400 [INFO] server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: entering follower state: follower="Node at b3fd43eb-bd2c-988a-969a-f6131bc82b82 [Follower]" leader= + testing.go:149: server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: 2024-06-12T04:32:33.849+1400 [INFO] server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: entering follower state: follower="Node at b3fd43eb-bd2c-988a-969a-f6131bc82b82 [Follower]" leader= +testLoggerAdapter verbose: server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: 2024-06-12T04:32:33.910+1400 [WARN] server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: 2024-06-12T04:32:33.910+1400 [INFO] server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: entering candidate state: node="Node at b3285295-113b-81b1-255e-ae5ef1b3d8ea [Candidate]" term=3 +testLoggerAdapter verbose: server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: 2024-06-12T04:32:33.910+1400 [INFO] server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: election won: tally=1 +testLoggerAdapter verbose: server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: 2024-06-12T04:32:33.910+1400 [INFO] server-b3285295-113b-81b1-255e-ae5ef1b3d8ea: entering leader state: leader="Node at b3285295-113b-81b1-255e-ae5ef1b3d8ea [Leader]" +testLoggerAdapter verbose: server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: 2024-06-12T04:32:33.918+1400 [WARN] server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: 2024-06-12T04:32:33.918+1400 [WARN] server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: 2024-06-12T04:32:33.918+1400 [INFO] server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: entering candidate state: node="Node at b3fd43eb-bd2c-988a-969a-f6131bc82b82 [Candidate]" term=2 + testing.go:149: server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: 2024-06-12T04:32:33.918+1400 [INFO] server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: entering candidate state: node="Node at b3fd43eb-bd2c-988a-969a-f6131bc82b82 [Candidate]" term=2 +testLoggerAdapter verbose: server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: 2024-06-12T04:32:33.919+1400 [INFO] server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: election won: tally=1 + testing.go:149: server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: 2024-06-12T04:32:33.919+1400 [INFO] server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: election won: tally=1 +testLoggerAdapter verbose: server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: 2024-06-12T04:32:33.919+1400 [INFO] server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: entering leader state: leader="Node at b3fd43eb-bd2c-988a-969a-f6131bc82b82 [Leader]" + testing.go:149: server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: 2024-06-12T04:32:33.919+1400 [INFO] server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: entering leader state: leader="Node at b3fd43eb-bd2c-988a-969a-f6131bc82b82 [Leader]" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:34.025+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:33.919175741 +1400 +14 m=+10.826694360 (1 nodes), 69.286919ms from start of poll, 69.538549ms from cluster start. Timeout at 2024-06-12 04:32:34.02500335 +1400 +14 m=+10.932521969, 105.827609ms after stability + testing.go:149: cluster: 2024-06-12T04:32:34.025+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:33.919175741 +1400 +14 m=+10.826694360 (1 nodes), 69.286919ms from start of poll, 69.538549ms from cluster start. Timeout at 2024-06-12 04:32:34.02500335 +1400 +14 m=+10.932521969, 105.827609ms after stability +testLoggerAdapter verbose: server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: 2024-06-12T04:32:34.025+1400 [ERROR] server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: failed to take snapshot: error="nothing new to snapshot" + testing.go:149: server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: 2024-06-12T04:32:34.025+1400 [ERROR] server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: failed to take snapshot: error="nothing new to snapshot" +testLoggerAdapter verbose: server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: 2024-06-12T04:32:34.025+1400 [INFO] server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: starting snapshot up to: index=12 + testing.go:149: server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: 2024-06-12T04:32:34.025+1400 [INFO] server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: starting snapshot up to: index=12 + testing.go:153: 2024-06-12T04:32:34.025+1400 [INFO] creating new snapshot: path=/tmp/raft163758661/snapshots/2-12-1718116354025.tmp +testLoggerAdapter verbose: server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: 2024-06-12T04:32:34.040+1400 [INFO] server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: compacting logs: from=1 to=2 + testing.go:149: server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: 2024-06-12T04:32:34.040+1400 [INFO] server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: compacting logs: from=1 to=2 +testLoggerAdapter verbose: server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: 2024-06-12T04:32:34.040+1400 [INFO] server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: snapshot complete up to: index=12 + testing.go:149: server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: 2024-06-12T04:32:34.040+1400 [INFO] server-b3fd43eb-bd2c-988a-969a-f6131bc82b82: snapshot complete up to: index=12 +--- PASS: TestRaft_UserSnapshot (0.19s) === RUN TestRaft_UserRestore -testLoggerAdapter verbose: server-67989d74-46df-1808-fa36-c1d29acb8c7b: 2024-06-11T02:29:20.809-1200 [INFO] server-67989d74-46df-1808-fa36-c1d29acb8c7b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-67989d74-46df-1808-fa36-c1d29acb8c7b Address:67989d74-46df-1808-fa36-c1d29acb8c7b} {Suffrage:Voter ID:server-b09f7cab-00cd-8e17-fce4-301e04838d84 Address:b09f7cab-00cd-8e17-fce4-301e04838d84} {Suffrage:Voter ID:server-d95e9130-8571-59f2-debd-c4d83c3bd6e8 Address:d95e9130-8571-59f2-debd-c4d83c3bd6e8}]" - testing.go:149: server-67989d74-46df-1808-fa36-c1d29acb8c7b: 2024-06-11T02:29:20.809-1200 [INFO] server-67989d74-46df-1808-fa36-c1d29acb8c7b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-67989d74-46df-1808-fa36-c1d29acb8c7b Address:67989d74-46df-1808-fa36-c1d29acb8c7b} {Suffrage:Voter ID:server-b09f7cab-00cd-8e17-fce4-301e04838d84 Address:b09f7cab-00cd-8e17-fce4-301e04838d84} {Suffrage:Voter ID:server-d95e9130-8571-59f2-debd-c4d83c3bd6e8 Address:d95e9130-8571-59f2-debd-c4d83c3bd6e8}]" -testLoggerAdapter verbose: server-67989d74-46df-1808-fa36-c1d29acb8c7b: 2024-06-11T02:29:20.810-1200 [INFO] server-67989d74-46df-1808-fa36-c1d29acb8c7b: entering follower state: follower="Node at 67989d74-46df-1808-fa36-c1d29acb8c7b [Follower]" leader= - testing.go:149: server-67989d74-46df-1808-fa36-c1d29acb8c7b: 2024-06-11T02:29:20.810-1200 [INFO] server-67989d74-46df-1808-fa36-c1d29acb8c7b: entering follower state: follower="Node at 67989d74-46df-1808-fa36-c1d29acb8c7b [Follower]" leader= -testLoggerAdapter verbose: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:20.810-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-67989d74-46df-1808-fa36-c1d29acb8c7b Address:67989d74-46df-1808-fa36-c1d29acb8c7b} {Suffrage:Voter ID:server-b09f7cab-00cd-8e17-fce4-301e04838d84 Address:b09f7cab-00cd-8e17-fce4-301e04838d84} {Suffrage:Voter ID:server-d95e9130-8571-59f2-debd-c4d83c3bd6e8 Address:d95e9130-8571-59f2-debd-c4d83c3bd6e8}]" - testing.go:149: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:20.810-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-67989d74-46df-1808-fa36-c1d29acb8c7b Address:67989d74-46df-1808-fa36-c1d29acb8c7b} {Suffrage:Voter ID:server-b09f7cab-00cd-8e17-fce4-301e04838d84 Address:b09f7cab-00cd-8e17-fce4-301e04838d84} {Suffrage:Voter ID:server-d95e9130-8571-59f2-debd-c4d83c3bd6e8 Address:d95e9130-8571-59f2-debd-c4d83c3bd6e8}]" -testLoggerAdapter verbose: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:20.810-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: entering follower state: follower="Node at b09f7cab-00cd-8e17-fce4-301e04838d84 [Follower]" leader= - testing.go:149: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:20.810-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: entering follower state: follower="Node at b09f7cab-00cd-8e17-fce4-301e04838d84 [Follower]" leader= -testLoggerAdapter verbose: server-d95e9130-8571-59f2-debd-c4d83c3bd6e8: 2024-06-11T02:29:20.811-1200 [INFO] server-d95e9130-8571-59f2-debd-c4d83c3bd6e8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-67989d74-46df-1808-fa36-c1d29acb8c7b Address:67989d74-46df-1808-fa36-c1d29acb8c7b} {Suffrage:Voter ID:server-b09f7cab-00cd-8e17-fce4-301e04838d84 Address:b09f7cab-00cd-8e17-fce4-301e04838d84} {Suffrage:Voter ID:server-d95e9130-8571-59f2-debd-c4d83c3bd6e8 Address:d95e9130-8571-59f2-debd-c4d83c3bd6e8}]" - testing.go:149: server-d95e9130-8571-59f2-debd-c4d83c3bd6e8: 2024-06-11T02:29:20.811-1200 [INFO] server-d95e9130-8571-59f2-debd-c4d83c3bd6e8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-67989d74-46df-1808-fa36-c1d29acb8c7b Address:67989d74-46df-1808-fa36-c1d29acb8c7b} {Suffrage:Voter ID:server-b09f7cab-00cd-8e17-fce4-301e04838d84 Address:b09f7cab-00cd-8e17-fce4-301e04838d84} {Suffrage:Voter ID:server-d95e9130-8571-59f2-debd-c4d83c3bd6e8 Address:d95e9130-8571-59f2-debd-c4d83c3bd6e8}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:20.811-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-d95e9130-8571-59f2-debd-c4d83c3bd6e8: 2024-06-11T02:29:20.811-1200 [INFO] server-d95e9130-8571-59f2-debd-c4d83c3bd6e8: entering follower state: follower="Node at d95e9130-8571-59f2-debd-c4d83c3bd6e8 [Follower]" leader= - testing.go:149: cluster: 2024-06-11T02:29:20.811-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: server-d95e9130-8571-59f2-debd-c4d83c3bd6e8: 2024-06-11T02:29:20.811-1200 [INFO] server-d95e9130-8571-59f2-debd-c4d83c3bd6e8: entering follower state: follower="Node at d95e9130-8571-59f2-debd-c4d83c3bd6e8 [Follower]" leader= -testLoggerAdapter verbose: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:20.878-1200 [WARN] server-b09f7cab-00cd-8e17-fce4-301e04838d84: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:20.878-1200 [WARN] server-b09f7cab-00cd-8e17-fce4-301e04838d84: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:20.878-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: entering candidate state: node="Node at b09f7cab-00cd-8e17-fce4-301e04838d84 [Candidate]" term=2 - testing.go:149: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:20.878-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: entering candidate state: node="Node at b09f7cab-00cd-8e17-fce4-301e04838d84 [Candidate]" term=2 -testLoggerAdapter verbose: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:20.878-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: election won: tally=2 - testing.go:149: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:20.878-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: election won: tally=2 -testLoggerAdapter verbose: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:20.878-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: entering leader state: leader="Node at b09f7cab-00cd-8e17-fce4-301e04838d84 [Leader]" - testing.go:149: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:20.878-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: entering leader state: leader="Node at b09f7cab-00cd-8e17-fce4-301e04838d84 [Leader]" -testLoggerAdapter verbose: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:20.878-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: added peer, starting replication: peer=server-67989d74-46df-1808-fa36-c1d29acb8c7b - testing.go:149: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:20.878-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: added peer, starting replication: peer=server-67989d74-46df-1808-fa36-c1d29acb8c7b -testLoggerAdapter verbose: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:20.878-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: added peer, starting replication: peer=server-d95e9130-8571-59f2-debd-c4d83c3bd6e8 - testing.go:149: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:20.878-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: added peer, starting replication: peer=server-d95e9130-8571-59f2-debd-c4d83c3bd6e8 -testLoggerAdapter verbose: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:20.878-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: pipelining replication: peer="{Voter server-67989d74-46df-1808-fa36-c1d29acb8c7b 67989d74-46df-1808-fa36-c1d29acb8c7b}" - testing.go:149: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:20.878-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: pipelining replication: peer="{Voter server-67989d74-46df-1808-fa36-c1d29acb8c7b 67989d74-46df-1808-fa36-c1d29acb8c7b}" -testLoggerAdapter verbose: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:20.879-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: pipelining replication: peer="{Voter server-d95e9130-8571-59f2-debd-c4d83c3bd6e8 d95e9130-8571-59f2-debd-c4d83c3bd6e8}" - testing.go:149: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:20.879-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: pipelining replication: peer="{Voter server-d95e9130-8571-59f2-debd-c4d83c3bd6e8 d95e9130-8571-59f2-debd-c4d83c3bd6e8}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:20.983-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:20.878789622 -1200 -12 m=+11.874162164 (1 nodes), 67.157612ms from start of poll, 69.342446ms from cluster start. Timeout at 2024-06-11 02:29:20.983843354 -1200 -12 m=+11.979215896, 105.053732ms after stability - testing.go:149: cluster: 2024-06-11T02:29:20.983-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:20.878789622 -1200 -12 m=+11.874162164 (1 nodes), 67.157612ms from start of poll, 69.342446ms from cluster start. Timeout at 2024-06-11 02:29:20.983843354 -1200 -12 m=+11.979215896, 105.053732ms after stability -testLoggerAdapter verbose: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:20.985-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: starting snapshot up to: index=12 - testing.go:149: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:20.985-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: starting snapshot up to: index=12 - testing.go:153: 2024-06-11T02:29:20.985-1200 [INFO] creating new snapshot: path=/tmp/raft3409042987/snapshots/2-12-1718116160985.tmp -testLoggerAdapter verbose: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:21.009-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: snapshot complete up to: index=12 - testing.go:149: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:21.009-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: snapshot complete up to: index=12 - testing.go:153: 2024-06-11T02:29:21.010-1200 [INFO] creating new snapshot: path=/tmp/raft3409042987/snapshots/2-23-1718116161010.tmp -testLoggerAdapter verbose: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:21.031-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: copied to local snapshot: bytes=71 - testing.go:149: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:21.031-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:21.032-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: restored user snapshot: index=1 - testing.go:149: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:21.032-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: restored user snapshot: index=1 -testLoggerAdapter verbose: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:21.032-1200 [ERROR] server-b09f7cab-00cd-8e17-fce4-301e04838d84: failed to get log: index=23 error="log not found" - testing.go:149: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:21.032-1200 [ERROR] server-b09f7cab-00cd-8e17-fce4-301e04838d84: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:21.032-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: aborting pipeline replication: peer="{Voter server-d95e9130-8571-59f2-debd-c4d83c3bd6e8 d95e9130-8571-59f2-debd-c4d83c3bd6e8}" - testing.go:149: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:21.032-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: aborting pipeline replication: peer="{Voter server-d95e9130-8571-59f2-debd-c4d83c3bd6e8 d95e9130-8571-59f2-debd-c4d83c3bd6e8}" -testLoggerAdapter verbose: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:21.032-1200 [ERROR] server-b09f7cab-00cd-8e17-fce4-301e04838d84: failed to get log: index=23 error="log not found" - testing.go:149: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:21.032-1200 [ERROR] server-b09f7cab-00cd-8e17-fce4-301e04838d84: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:21.032-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: aborting pipeline replication: peer="{Voter server-67989d74-46df-1808-fa36-c1d29acb8c7b 67989d74-46df-1808-fa36-c1d29acb8c7b}" - testing.go:149: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:21.032-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: aborting pipeline replication: peer="{Voter server-67989d74-46df-1808-fa36-c1d29acb8c7b 67989d74-46df-1808-fa36-c1d29acb8c7b}" -testLoggerAdapter verbose: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:21.039-1200 [ERROR] server-b09f7cab-00cd-8e17-fce4-301e04838d84: failed to get log: index=23 error="log not found" - testing.go:149: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:21.039-1200 [ERROR] server-b09f7cab-00cd-8e17-fce4-301e04838d84: failed to get log: index=23 error="log not found" - testing.go:153: 2024-06-11T02:29:21.040-1200 [INFO] creating new snapshot: path=/tmp/raft710091764/snapshots/2-23-1718116161040.tmp -testLoggerAdapter verbose: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:21.041-1200 [ERROR] server-b09f7cab-00cd-8e17-fce4-301e04838d84: failed to get log: index=23 error="log not found" - testing.go:149: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:21.041-1200 [ERROR] server-b09f7cab-00cd-8e17-fce4-301e04838d84: failed to get log: index=23 error="log not found" - testing.go:153: 2024-06-11T02:29:21.042-1200 [INFO] creating new snapshot: path=/tmp/raft4159630181/snapshots/2-23-1718116161042.tmp -testLoggerAdapter verbose: server-67989d74-46df-1808-fa36-c1d29acb8c7b: 2024-06-11T02:29:21.078-1200 [INFO] server-67989d74-46df-1808-fa36-c1d29acb8c7b: copied to local snapshot: bytes=71 - testing.go:149: server-67989d74-46df-1808-fa36-c1d29acb8c7b: 2024-06-11T02:29:21.078-1200 [INFO] server-67989d74-46df-1808-fa36-c1d29acb8c7b: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-67989d74-46df-1808-fa36-c1d29acb8c7b: 2024-06-11T02:29:21.078-1200 [INFO] server-67989d74-46df-1808-fa36-c1d29acb8c7b: Installed remote snapshot - testing.go:149: server-67989d74-46df-1808-fa36-c1d29acb8c7b: 2024-06-11T02:29:21.078-1200 [INFO] server-67989d74-46df-1808-fa36-c1d29acb8c7b: Installed remote snapshot -testLoggerAdapter verbose: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:21.079-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: pipelining replication: peer="{Voter server-67989d74-46df-1808-fa36-c1d29acb8c7b 67989d74-46df-1808-fa36-c1d29acb8c7b}" - testing.go:149: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:21.079-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: pipelining replication: peer="{Voter server-67989d74-46df-1808-fa36-c1d29acb8c7b 67989d74-46df-1808-fa36-c1d29acb8c7b}" -testLoggerAdapter verbose: server-d95e9130-8571-59f2-debd-c4d83c3bd6e8: 2024-06-11T02:29:21.081-1200 [INFO] server-d95e9130-8571-59f2-debd-c4d83c3bd6e8: copied to local snapshot: bytes=71 - testing.go:149: server-d95e9130-8571-59f2-debd-c4d83c3bd6e8: 2024-06-11T02:29:21.081-1200 [INFO] server-d95e9130-8571-59f2-debd-c4d83c3bd6e8: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-d95e9130-8571-59f2-debd-c4d83c3bd6e8: 2024-06-11T02:29:21.082-1200 [INFO] server-d95e9130-8571-59f2-debd-c4d83c3bd6e8: Installed remote snapshot - testing.go:149: server-d95e9130-8571-59f2-debd-c4d83c3bd6e8: 2024-06-11T02:29:21.082-1200 [INFO] server-d95e9130-8571-59f2-debd-c4d83c3bd6e8: Installed remote snapshot -testLoggerAdapter verbose: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:21.083-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: pipelining replication: peer="{Voter server-d95e9130-8571-59f2-debd-c4d83c3bd6e8 d95e9130-8571-59f2-debd-c4d83c3bd6e8}" - testing.go:149: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:21.083-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: pipelining replication: peer="{Voter server-d95e9130-8571-59f2-debd-c4d83c3bd6e8 d95e9130-8571-59f2-debd-c4d83c3bd6e8}" -testLoggerAdapter verbose: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:21.096-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: aborting pipeline replication: peer="{Voter server-67989d74-46df-1808-fa36-c1d29acb8c7b 67989d74-46df-1808-fa36-c1d29acb8c7b}" - testing.go:149: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:21.096-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: aborting pipeline replication: peer="{Voter server-67989d74-46df-1808-fa36-c1d29acb8c7b 67989d74-46df-1808-fa36-c1d29acb8c7b}" -testLoggerAdapter verbose: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:21.096-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: aborting pipeline replication: peer="{Voter server-d95e9130-8571-59f2-debd-c4d83c3bd6e8 d95e9130-8571-59f2-debd-c4d83c3bd6e8}" - testing.go:149: server-b09f7cab-00cd-8e17-fce4-301e04838d84: 2024-06-11T02:29:21.096-1200 [INFO] server-b09f7cab-00cd-8e17-fce4-301e04838d84: aborting pipeline replication: peer="{Voter server-d95e9130-8571-59f2-debd-c4d83c3bd6e8 d95e9130-8571-59f2-debd-c4d83c3bd6e8}" -testLoggerAdapter verbose: server-ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9: 2024-06-11T02:29:21.104-1200 [INFO] server-ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9 Address:ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9} {Suffrage:Voter ID:server-53985535-a4aa-61e2-6392-2f63235897c2 Address:53985535-a4aa-61e2-6392-2f63235897c2} {Suffrage:Voter ID:server-42524719-0b61-fdb8-6ac6-83307c077a67 Address:42524719-0b61-fdb8-6ac6-83307c077a67}]" - testing.go:149: server-ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9: 2024-06-11T02:29:21.104-1200 [INFO] server-ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9 Address:ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9} {Suffrage:Voter ID:server-53985535-a4aa-61e2-6392-2f63235897c2 Address:53985535-a4aa-61e2-6392-2f63235897c2} {Suffrage:Voter ID:server-42524719-0b61-fdb8-6ac6-83307c077a67 Address:42524719-0b61-fdb8-6ac6-83307c077a67}]" -testLoggerAdapter verbose: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.104-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9 Address:ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9} {Suffrage:Voter ID:server-53985535-a4aa-61e2-6392-2f63235897c2 Address:53985535-a4aa-61e2-6392-2f63235897c2} {Suffrage:Voter ID:server-42524719-0b61-fdb8-6ac6-83307c077a67 Address:42524719-0b61-fdb8-6ac6-83307c077a67}]" - testing.go:149: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.104-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9 Address:ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9} {Suffrage:Voter ID:server-53985535-a4aa-61e2-6392-2f63235897c2 Address:53985535-a4aa-61e2-6392-2f63235897c2} {Suffrage:Voter ID:server-42524719-0b61-fdb8-6ac6-83307c077a67 Address:42524719-0b61-fdb8-6ac6-83307c077a67}]" -testLoggerAdapter verbose: server-42524719-0b61-fdb8-6ac6-83307c077a67: 2024-06-11T02:29:21.105-1200 [INFO] server-42524719-0b61-fdb8-6ac6-83307c077a67: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9 Address:ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9} {Suffrage:Voter ID:server-53985535-a4aa-61e2-6392-2f63235897c2 Address:53985535-a4aa-61e2-6392-2f63235897c2} {Suffrage:Voter ID:server-42524719-0b61-fdb8-6ac6-83307c077a67 Address:42524719-0b61-fdb8-6ac6-83307c077a67}]" - testing.go:149: server-42524719-0b61-fdb8-6ac6-83307c077a67: 2024-06-11T02:29:21.105-1200 [INFO] server-42524719-0b61-fdb8-6ac6-83307c077a67: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9 Address:ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9} {Suffrage:Voter ID:server-53985535-a4aa-61e2-6392-2f63235897c2 Address:53985535-a4aa-61e2-6392-2f63235897c2} {Suffrage:Voter ID:server-42524719-0b61-fdb8-6ac6-83307c077a67 Address:42524719-0b61-fdb8-6ac6-83307c077a67}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:21.105-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:21.105-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9: 2024-06-11T02:29:21.105-1200 [INFO] server-ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9: entering follower state: follower="Node at ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9 [Follower]" leader= - testing.go:149: server-ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9: 2024-06-11T02:29:21.105-1200 [INFO] server-ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9: entering follower state: follower="Node at ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9 [Follower]" leader= -testLoggerAdapter verbose: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.105-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: entering follower state: follower="Node at 53985535-a4aa-61e2-6392-2f63235897c2 [Follower]" leader= - testing.go:149: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.105-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: entering follower state: follower="Node at 53985535-a4aa-61e2-6392-2f63235897c2 [Follower]" leader= -testLoggerAdapter verbose: server-42524719-0b61-fdb8-6ac6-83307c077a67: 2024-06-11T02:29:21.105-1200 [INFO] server-42524719-0b61-fdb8-6ac6-83307c077a67: entering follower state: follower="Node at 42524719-0b61-fdb8-6ac6-83307c077a67 [Follower]" leader= - testing.go:149: server-42524719-0b61-fdb8-6ac6-83307c077a67: 2024-06-11T02:29:21.105-1200 [INFO] server-42524719-0b61-fdb8-6ac6-83307c077a67: entering follower state: follower="Node at 42524719-0b61-fdb8-6ac6-83307c077a67 [Follower]" leader= -testLoggerAdapter verbose: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.174-1200 [WARN] server-53985535-a4aa-61e2-6392-2f63235897c2: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.174-1200 [WARN] server-53985535-a4aa-61e2-6392-2f63235897c2: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.174-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: entering candidate state: node="Node at 53985535-a4aa-61e2-6392-2f63235897c2 [Candidate]" term=2 - testing.go:149: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.174-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: entering candidate state: node="Node at 53985535-a4aa-61e2-6392-2f63235897c2 [Candidate]" term=2 -testLoggerAdapter verbose: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.174-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: election won: tally=2 - testing.go:149: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.174-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: election won: tally=2 -testLoggerAdapter verbose: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.174-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: entering leader state: leader="Node at 53985535-a4aa-61e2-6392-2f63235897c2 [Leader]" - testing.go:149: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.174-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: entering leader state: leader="Node at 53985535-a4aa-61e2-6392-2f63235897c2 [Leader]" -testLoggerAdapter verbose: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.174-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: added peer, starting replication: peer=server-ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9 - testing.go:149: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.174-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: added peer, starting replication: peer=server-ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9 -testLoggerAdapter verbose: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.175-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: added peer, starting replication: peer=server-42524719-0b61-fdb8-6ac6-83307c077a67 - testing.go:149: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.175-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: added peer, starting replication: peer=server-42524719-0b61-fdb8-6ac6-83307c077a67 -testLoggerAdapter verbose: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.175-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: pipelining replication: peer="{Voter server-42524719-0b61-fdb8-6ac6-83307c077a67 42524719-0b61-fdb8-6ac6-83307c077a67}" - testing.go:149: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.175-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: pipelining replication: peer="{Voter server-42524719-0b61-fdb8-6ac6-83307c077a67 42524719-0b61-fdb8-6ac6-83307c077a67}" -testLoggerAdapter verbose: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.175-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: pipelining replication: peer="{Voter server-ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9 ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9}" - testing.go:149: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.175-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: pipelining replication: peer="{Voter server-ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9 ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:21.281-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:21.175661912 -1200 -12 m=+12.171034454 (1 nodes), 70.161722ms from start of poll, 71.698485ms from cluster start. Timeout at 2024-06-11 02:29:21.281662085 -1200 -12 m=+12.277034918, 106.000464ms after stability - testing.go:149: cluster: 2024-06-11T02:29:21.281-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:21.175661912 -1200 -12 m=+12.171034454 (1 nodes), 70.161722ms from start of poll, 71.698485ms from cluster start. Timeout at 2024-06-11 02:29:21.281662085 -1200 -12 m=+12.277034918, 106.000464ms after stability -testLoggerAdapter verbose: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.283-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: starting snapshot up to: index=12 - testing.go:149: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.283-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: starting snapshot up to: index=12 - testing.go:153: 2024-06-11T02:29:21.283-1200 [INFO] creating new snapshot: path=/tmp/raft2211211413/snapshots/2-12-1718116161283.tmp -testLoggerAdapter verbose: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.308-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: snapshot complete up to: index=12 - testing.go:149: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.308-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: snapshot complete up to: index=12 - testing.go:153: 2024-06-11T02:29:21.309-1200 [INFO] creating new snapshot: path=/tmp/raft2211211413/snapshots/2-23-1718116161309.tmp -testLoggerAdapter verbose: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.332-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: copied to local snapshot: bytes=71 - testing.go:149: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.332-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.332-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: restored user snapshot: index=1 - testing.go:149: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.332-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: restored user snapshot: index=1 -testLoggerAdapter verbose: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.333-1200 [ERROR] server-53985535-a4aa-61e2-6392-2f63235897c2: failed to get log: index=23 error="log not found" - testing.go:149: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.333-1200 [ERROR] server-53985535-a4aa-61e2-6392-2f63235897c2: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.333-1200 [ERROR] server-53985535-a4aa-61e2-6392-2f63235897c2: failed to get log: index=23 error="log not found" - testing.go:149: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.333-1200 [ERROR] server-53985535-a4aa-61e2-6392-2f63235897c2: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.333-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: aborting pipeline replication: peer="{Voter server-ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9 ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9}" - testing.go:149: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.333-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: aborting pipeline replication: peer="{Voter server-ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9 ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9}" -testLoggerAdapter verbose: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.333-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: aborting pipeline replication: peer="{Voter server-42524719-0b61-fdb8-6ac6-83307c077a67 42524719-0b61-fdb8-6ac6-83307c077a67}" - testing.go:149: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.333-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: aborting pipeline replication: peer="{Voter server-42524719-0b61-fdb8-6ac6-83307c077a67 42524719-0b61-fdb8-6ac6-83307c077a67}" -testLoggerAdapter verbose: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.342-1200 [ERROR] server-53985535-a4aa-61e2-6392-2f63235897c2: failed to get log: index=23 error="log not found" - testing.go:149: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.342-1200 [ERROR] server-53985535-a4aa-61e2-6392-2f63235897c2: failed to get log: index=23 error="log not found" - testing.go:153: 2024-06-11T02:29:21.343-1200 [INFO] creating new snapshot: path=/tmp/raft3884469150/snapshots/2-23-1718116161343.tmp -testLoggerAdapter verbose: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.344-1200 [ERROR] server-53985535-a4aa-61e2-6392-2f63235897c2: failed to get log: index=23 error="log not found" - testing.go:149: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.344-1200 [ERROR] server-53985535-a4aa-61e2-6392-2f63235897c2: failed to get log: index=23 error="log not found" - testing.go:153: 2024-06-11T02:29:21.347-1200 [INFO] creating new snapshot: path=/tmp/raft3384910023/snapshots/2-23-1718116161347.tmp -testLoggerAdapter verbose: server-ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9: 2024-06-11T02:29:21.385-1200 [INFO] server-ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9: copied to local snapshot: bytes=71 - testing.go:149: server-ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9: 2024-06-11T02:29:21.385-1200 [INFO] server-ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9: 2024-06-11T02:29:21.386-1200 [INFO] server-ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9: Installed remote snapshot - testing.go:149: server-ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9: 2024-06-11T02:29:21.386-1200 [INFO] server-ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9: Installed remote snapshot -testLoggerAdapter verbose: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.386-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: pipelining replication: peer="{Voter server-ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9 ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9}" - testing.go:149: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.386-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: pipelining replication: peer="{Voter server-ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9 ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9}" -testLoggerAdapter verbose: server-42524719-0b61-fdb8-6ac6-83307c077a67: 2024-06-11T02:29:21.389-1200 [INFO] server-42524719-0b61-fdb8-6ac6-83307c077a67: copied to local snapshot: bytes=71 - testing.go:149: server-42524719-0b61-fdb8-6ac6-83307c077a67: 2024-06-11T02:29:21.389-1200 [INFO] server-42524719-0b61-fdb8-6ac6-83307c077a67: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-42524719-0b61-fdb8-6ac6-83307c077a67: 2024-06-11T02:29:21.389-1200 [INFO] server-42524719-0b61-fdb8-6ac6-83307c077a67: Installed remote snapshot - testing.go:149: server-42524719-0b61-fdb8-6ac6-83307c077a67: 2024-06-11T02:29:21.389-1200 [INFO] server-42524719-0b61-fdb8-6ac6-83307c077a67: Installed remote snapshot -testLoggerAdapter verbose: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.390-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: pipelining replication: peer="{Voter server-42524719-0b61-fdb8-6ac6-83307c077a67 42524719-0b61-fdb8-6ac6-83307c077a67}" - testing.go:149: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.390-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: pipelining replication: peer="{Voter server-42524719-0b61-fdb8-6ac6-83307c077a67 42524719-0b61-fdb8-6ac6-83307c077a67}" -testLoggerAdapter verbose: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.404-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: aborting pipeline replication: peer="{Voter server-ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9 ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9}" - testing.go:149: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.404-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: aborting pipeline replication: peer="{Voter server-ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9 ff3bc21d-2cc1-cd33-9a2e-023ac279bdd9}" -testLoggerAdapter verbose: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.404-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: aborting pipeline replication: peer="{Voter server-42524719-0b61-fdb8-6ac6-83307c077a67 42524719-0b61-fdb8-6ac6-83307c077a67}" - testing.go:149: server-53985535-a4aa-61e2-6392-2f63235897c2: 2024-06-11T02:29:21.404-1200 [INFO] server-53985535-a4aa-61e2-6392-2f63235897c2: aborting pipeline replication: peer="{Voter server-42524719-0b61-fdb8-6ac6-83307c077a67 42524719-0b61-fdb8-6ac6-83307c077a67}" -testLoggerAdapter verbose: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.411-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-25818660-1d99-14dd-9658-4244e5c31494 Address:25818660-1d99-14dd-9658-4244e5c31494} {Suffrage:Voter ID:server-e0a19e50-f247-1ab4-95a8-4b86e9c1ac36 Address:e0a19e50-f247-1ab4-95a8-4b86e9c1ac36} {Suffrage:Voter ID:server-7436e20f-6810-a9fc-0356-5574e8ef453b Address:7436e20f-6810-a9fc-0356-5574e8ef453b}]" - testing.go:149: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.411-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-25818660-1d99-14dd-9658-4244e5c31494 Address:25818660-1d99-14dd-9658-4244e5c31494} {Suffrage:Voter ID:server-e0a19e50-f247-1ab4-95a8-4b86e9c1ac36 Address:e0a19e50-f247-1ab4-95a8-4b86e9c1ac36} {Suffrage:Voter ID:server-7436e20f-6810-a9fc-0356-5574e8ef453b Address:7436e20f-6810-a9fc-0356-5574e8ef453b}]" -testLoggerAdapter verbose: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.411-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: entering follower state: follower="Node at 25818660-1d99-14dd-9658-4244e5c31494 [Follower]" leader= - testing.go:149: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.411-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: entering follower state: follower="Node at 25818660-1d99-14dd-9658-4244e5c31494 [Follower]" leader= -testLoggerAdapter verbose: server-e0a19e50-f247-1ab4-95a8-4b86e9c1ac36: 2024-06-11T02:29:21.411-1200 [INFO] server-e0a19e50-f247-1ab4-95a8-4b86e9c1ac36: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-25818660-1d99-14dd-9658-4244e5c31494 Address:25818660-1d99-14dd-9658-4244e5c31494} {Suffrage:Voter ID:server-e0a19e50-f247-1ab4-95a8-4b86e9c1ac36 Address:e0a19e50-f247-1ab4-95a8-4b86e9c1ac36} {Suffrage:Voter ID:server-7436e20f-6810-a9fc-0356-5574e8ef453b Address:7436e20f-6810-a9fc-0356-5574e8ef453b}]" - testing.go:149: server-e0a19e50-f247-1ab4-95a8-4b86e9c1ac36: 2024-06-11T02:29:21.411-1200 [INFO] server-e0a19e50-f247-1ab4-95a8-4b86e9c1ac36: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-25818660-1d99-14dd-9658-4244e5c31494 Address:25818660-1d99-14dd-9658-4244e5c31494} {Suffrage:Voter ID:server-e0a19e50-f247-1ab4-95a8-4b86e9c1ac36 Address:e0a19e50-f247-1ab4-95a8-4b86e9c1ac36} {Suffrage:Voter ID:server-7436e20f-6810-a9fc-0356-5574e8ef453b Address:7436e20f-6810-a9fc-0356-5574e8ef453b}]" -testLoggerAdapter verbose: server-e0a19e50-f247-1ab4-95a8-4b86e9c1ac36: 2024-06-11T02:29:21.412-1200 [INFO] server-e0a19e50-f247-1ab4-95a8-4b86e9c1ac36: entering follower state: follower="Node at e0a19e50-f247-1ab4-95a8-4b86e9c1ac36 [Follower]" leader= - testing.go:149: server-e0a19e50-f247-1ab4-95a8-4b86e9c1ac36: 2024-06-11T02:29:21.412-1200 [INFO] server-e0a19e50-f247-1ab4-95a8-4b86e9c1ac36: entering follower state: follower="Node at e0a19e50-f247-1ab4-95a8-4b86e9c1ac36 [Follower]" leader= -testLoggerAdapter verbose: server-7436e20f-6810-a9fc-0356-5574e8ef453b: 2024-06-11T02:29:21.412-1200 [INFO] server-7436e20f-6810-a9fc-0356-5574e8ef453b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-25818660-1d99-14dd-9658-4244e5c31494 Address:25818660-1d99-14dd-9658-4244e5c31494} {Suffrage:Voter ID:server-e0a19e50-f247-1ab4-95a8-4b86e9c1ac36 Address:e0a19e50-f247-1ab4-95a8-4b86e9c1ac36} {Suffrage:Voter ID:server-7436e20f-6810-a9fc-0356-5574e8ef453b Address:7436e20f-6810-a9fc-0356-5574e8ef453b}]" - testing.go:149: server-7436e20f-6810-a9fc-0356-5574e8ef453b: 2024-06-11T02:29:21.412-1200 [INFO] server-7436e20f-6810-a9fc-0356-5574e8ef453b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-25818660-1d99-14dd-9658-4244e5c31494 Address:25818660-1d99-14dd-9658-4244e5c31494} {Suffrage:Voter ID:server-e0a19e50-f247-1ab4-95a8-4b86e9c1ac36 Address:e0a19e50-f247-1ab4-95a8-4b86e9c1ac36} {Suffrage:Voter ID:server-7436e20f-6810-a9fc-0356-5574e8ef453b Address:7436e20f-6810-a9fc-0356-5574e8ef453b}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:21.412-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:21.412-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-7436e20f-6810-a9fc-0356-5574e8ef453b: 2024-06-11T02:29:21.412-1200 [INFO] server-7436e20f-6810-a9fc-0356-5574e8ef453b: entering follower state: follower="Node at 7436e20f-6810-a9fc-0356-5574e8ef453b [Follower]" leader= - testing.go:149: server-7436e20f-6810-a9fc-0356-5574e8ef453b: 2024-06-11T02:29:21.412-1200 [INFO] server-7436e20f-6810-a9fc-0356-5574e8ef453b: entering follower state: follower="Node at 7436e20f-6810-a9fc-0356-5574e8ef453b [Follower]" leader= -testLoggerAdapter verbose: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.464-1200 [WARN] server-25818660-1d99-14dd-9658-4244e5c31494: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.464-1200 [WARN] server-25818660-1d99-14dd-9658-4244e5c31494: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.464-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: entering candidate state: node="Node at 25818660-1d99-14dd-9658-4244e5c31494 [Candidate]" term=2 - testing.go:149: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.464-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: entering candidate state: node="Node at 25818660-1d99-14dd-9658-4244e5c31494 [Candidate]" term=2 -testLoggerAdapter verbose: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.464-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: election won: tally=2 - testing.go:149: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.464-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: election won: tally=2 -testLoggerAdapter verbose: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.464-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: entering leader state: leader="Node at 25818660-1d99-14dd-9658-4244e5c31494 [Leader]" - testing.go:149: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.464-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: entering leader state: leader="Node at 25818660-1d99-14dd-9658-4244e5c31494 [Leader]" -testLoggerAdapter verbose: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.464-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: added peer, starting replication: peer=server-e0a19e50-f247-1ab4-95a8-4b86e9c1ac36 - testing.go:149: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.464-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: added peer, starting replication: peer=server-e0a19e50-f247-1ab4-95a8-4b86e9c1ac36 -testLoggerAdapter verbose: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.464-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: added peer, starting replication: peer=server-7436e20f-6810-a9fc-0356-5574e8ef453b - testing.go:149: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.464-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: added peer, starting replication: peer=server-7436e20f-6810-a9fc-0356-5574e8ef453b -testLoggerAdapter verbose: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.464-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: pipelining replication: peer="{Voter server-7436e20f-6810-a9fc-0356-5574e8ef453b 7436e20f-6810-a9fc-0356-5574e8ef453b}" - testing.go:149: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.464-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: pipelining replication: peer="{Voter server-7436e20f-6810-a9fc-0356-5574e8ef453b 7436e20f-6810-a9fc-0356-5574e8ef453b}" -testLoggerAdapter verbose: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.465-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: pipelining replication: peer="{Voter server-e0a19e50-f247-1ab4-95a8-4b86e9c1ac36 e0a19e50-f247-1ab4-95a8-4b86e9c1ac36}" - testing.go:149: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.465-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: pipelining replication: peer="{Voter server-e0a19e50-f247-1ab4-95a8-4b86e9c1ac36 e0a19e50-f247-1ab4-95a8-4b86e9c1ac36}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:21.570-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:21.465343296 -1200 -12 m=+12.460715838 (1 nodes), 52.748383ms from start of poll, 54.439726ms from cluster start. Timeout at 2024-06-11 02:29:21.570548692 -1200 -12 m=+12.565921234, 105.205396ms after stability - testing.go:149: cluster: 2024-06-11T02:29:21.570-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:21.465343296 -1200 -12 m=+12.460715838 (1 nodes), 52.748383ms from start of poll, 54.439726ms from cluster start. Timeout at 2024-06-11 02:29:21.570548692 -1200 -12 m=+12.565921234, 105.205396ms after stability -testLoggerAdapter verbose: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.571-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: starting snapshot up to: index=12 - testing.go:149: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.571-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: starting snapshot up to: index=12 - testing.go:153: 2024-06-11T02:29:21.571-1200 [INFO] creating new snapshot: path=/tmp/raft1397225268/snapshots/2-12-1718116161571.tmp -testLoggerAdapter verbose: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.596-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: snapshot complete up to: index=12 - testing.go:149: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.596-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: snapshot complete up to: index=12 - testing.go:153: 2024-06-11T02:29:21.597-1200 [INFO] creating new snapshot: path=/tmp/raft1397225268/snapshots/2-23-1718116161597.tmp -testLoggerAdapter verbose: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.619-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: copied to local snapshot: bytes=71 - testing.go:149: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.619-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.620-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: restored user snapshot: index=1 - testing.go:149: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.620-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: restored user snapshot: index=1 -testLoggerAdapter verbose: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.620-1200 [ERROR] server-25818660-1d99-14dd-9658-4244e5c31494: failed to get log: index=23 error="log not found" - testing.go:149: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.620-1200 [ERROR] server-25818660-1d99-14dd-9658-4244e5c31494: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.620-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: aborting pipeline replication: peer="{Voter server-7436e20f-6810-a9fc-0356-5574e8ef453b 7436e20f-6810-a9fc-0356-5574e8ef453b}" - testing.go:149: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.620-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: aborting pipeline replication: peer="{Voter server-7436e20f-6810-a9fc-0356-5574e8ef453b 7436e20f-6810-a9fc-0356-5574e8ef453b}" -testLoggerAdapter verbose: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.620-1200 [ERROR] server-25818660-1d99-14dd-9658-4244e5c31494: failed to get log: index=23 error="log not found" - testing.go:149: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.620-1200 [ERROR] server-25818660-1d99-14dd-9658-4244e5c31494: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.620-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: aborting pipeline replication: peer="{Voter server-e0a19e50-f247-1ab4-95a8-4b86e9c1ac36 e0a19e50-f247-1ab4-95a8-4b86e9c1ac36}" - testing.go:149: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.620-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: aborting pipeline replication: peer="{Voter server-e0a19e50-f247-1ab4-95a8-4b86e9c1ac36 e0a19e50-f247-1ab4-95a8-4b86e9c1ac36}" -testLoggerAdapter verbose: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.628-1200 [ERROR] server-25818660-1d99-14dd-9658-4244e5c31494: failed to get log: index=23 error="log not found" - testing.go:149: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.628-1200 [ERROR] server-25818660-1d99-14dd-9658-4244e5c31494: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.629-1200 [ERROR] server-25818660-1d99-14dd-9658-4244e5c31494: failed to get log: index=23 error="log not found" - testing.go:149: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.629-1200 [ERROR] server-25818660-1d99-14dd-9658-4244e5c31494: failed to get log: index=23 error="log not found" - testing.go:153: 2024-06-11T02:29:21.629-1200 [INFO] creating new snapshot: path=/tmp/raft1242728075/snapshots/2-23-1718116161629.tmp - testing.go:153: 2024-06-11T02:29:21.630-1200 [INFO] creating new snapshot: path=/tmp/raft3166768854/snapshots/2-23-1718116161630.tmp -testLoggerAdapter verbose: server-7436e20f-6810-a9fc-0356-5574e8ef453b: 2024-06-11T02:29:21.668-1200 [INFO] server-7436e20f-6810-a9fc-0356-5574e8ef453b: copied to local snapshot: bytes=71 - testing.go:149: server-7436e20f-6810-a9fc-0356-5574e8ef453b: 2024-06-11T02:29:21.668-1200 [INFO] server-7436e20f-6810-a9fc-0356-5574e8ef453b: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-7436e20f-6810-a9fc-0356-5574e8ef453b: 2024-06-11T02:29:21.669-1200 [INFO] server-7436e20f-6810-a9fc-0356-5574e8ef453b: Installed remote snapshot - testing.go:149: server-7436e20f-6810-a9fc-0356-5574e8ef453b: 2024-06-11T02:29:21.669-1200 [INFO] server-7436e20f-6810-a9fc-0356-5574e8ef453b: Installed remote snapshot -testLoggerAdapter verbose: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.669-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: pipelining replication: peer="{Voter server-7436e20f-6810-a9fc-0356-5574e8ef453b 7436e20f-6810-a9fc-0356-5574e8ef453b}" - testing.go:149: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.669-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: pipelining replication: peer="{Voter server-7436e20f-6810-a9fc-0356-5574e8ef453b 7436e20f-6810-a9fc-0356-5574e8ef453b}" -testLoggerAdapter verbose: server-e0a19e50-f247-1ab4-95a8-4b86e9c1ac36: 2024-06-11T02:29:21.672-1200 [INFO] server-e0a19e50-f247-1ab4-95a8-4b86e9c1ac36: copied to local snapshot: bytes=71 - testing.go:149: server-e0a19e50-f247-1ab4-95a8-4b86e9c1ac36: 2024-06-11T02:29:21.672-1200 [INFO] server-e0a19e50-f247-1ab4-95a8-4b86e9c1ac36: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-e0a19e50-f247-1ab4-95a8-4b86e9c1ac36: 2024-06-11T02:29:21.672-1200 [INFO] server-e0a19e50-f247-1ab4-95a8-4b86e9c1ac36: Installed remote snapshot - testing.go:149: server-e0a19e50-f247-1ab4-95a8-4b86e9c1ac36: 2024-06-11T02:29:21.672-1200 [INFO] server-e0a19e50-f247-1ab4-95a8-4b86e9c1ac36: Installed remote snapshot -testLoggerAdapter verbose: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.673-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: pipelining replication: peer="{Voter server-e0a19e50-f247-1ab4-95a8-4b86e9c1ac36 e0a19e50-f247-1ab4-95a8-4b86e9c1ac36}" - testing.go:149: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.673-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: pipelining replication: peer="{Voter server-e0a19e50-f247-1ab4-95a8-4b86e9c1ac36 e0a19e50-f247-1ab4-95a8-4b86e9c1ac36}" -testLoggerAdapter verbose: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.688-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: aborting pipeline replication: peer="{Voter server-7436e20f-6810-a9fc-0356-5574e8ef453b 7436e20f-6810-a9fc-0356-5574e8ef453b}" - testing.go:149: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.688-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: aborting pipeline replication: peer="{Voter server-7436e20f-6810-a9fc-0356-5574e8ef453b 7436e20f-6810-a9fc-0356-5574e8ef453b}" -testLoggerAdapter verbose: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.688-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: aborting pipeline replication: peer="{Voter server-e0a19e50-f247-1ab4-95a8-4b86e9c1ac36 e0a19e50-f247-1ab4-95a8-4b86e9c1ac36}" - testing.go:149: server-25818660-1d99-14dd-9658-4244e5c31494: 2024-06-11T02:29:21.688-1200 [INFO] server-25818660-1d99-14dd-9658-4244e5c31494: aborting pipeline replication: peer="{Voter server-e0a19e50-f247-1ab4-95a8-4b86e9c1ac36 e0a19e50-f247-1ab4-95a8-4b86e9c1ac36}" -testLoggerAdapter verbose: server-ba7d1440-bc18-af30-a710-ac9ff72be94d: 2024-06-11T02:29:21.697-1200 [INFO] server-ba7d1440-bc18-af30-a710-ac9ff72be94d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ba7d1440-bc18-af30-a710-ac9ff72be94d Address:ba7d1440-bc18-af30-a710-ac9ff72be94d} {Suffrage:Voter ID:server-bb296b30-fe94-6473-a436-01bda40f8495 Address:bb296b30-fe94-6473-a436-01bda40f8495} {Suffrage:Voter ID:server-b41584b9-6a39-417f-b36d-fb83a584adbc Address:b41584b9-6a39-417f-b36d-fb83a584adbc}]" - testing.go:149: server-ba7d1440-bc18-af30-a710-ac9ff72be94d: 2024-06-11T02:29:21.697-1200 [INFO] server-ba7d1440-bc18-af30-a710-ac9ff72be94d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ba7d1440-bc18-af30-a710-ac9ff72be94d Address:ba7d1440-bc18-af30-a710-ac9ff72be94d} {Suffrage:Voter ID:server-bb296b30-fe94-6473-a436-01bda40f8495 Address:bb296b30-fe94-6473-a436-01bda40f8495} {Suffrage:Voter ID:server-b41584b9-6a39-417f-b36d-fb83a584adbc Address:b41584b9-6a39-417f-b36d-fb83a584adbc}]" -testLoggerAdapter verbose: server-bb296b30-fe94-6473-a436-01bda40f8495: 2024-06-11T02:29:21.697-1200 [INFO] server-bb296b30-fe94-6473-a436-01bda40f8495: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ba7d1440-bc18-af30-a710-ac9ff72be94d Address:ba7d1440-bc18-af30-a710-ac9ff72be94d} {Suffrage:Voter ID:server-bb296b30-fe94-6473-a436-01bda40f8495 Address:bb296b30-fe94-6473-a436-01bda40f8495} {Suffrage:Voter ID:server-b41584b9-6a39-417f-b36d-fb83a584adbc Address:b41584b9-6a39-417f-b36d-fb83a584adbc}]" - testing.go:149: server-bb296b30-fe94-6473-a436-01bda40f8495: 2024-06-11T02:29:21.697-1200 [INFO] server-bb296b30-fe94-6473-a436-01bda40f8495: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ba7d1440-bc18-af30-a710-ac9ff72be94d Address:ba7d1440-bc18-af30-a710-ac9ff72be94d} {Suffrage:Voter ID:server-bb296b30-fe94-6473-a436-01bda40f8495 Address:bb296b30-fe94-6473-a436-01bda40f8495} {Suffrage:Voter ID:server-b41584b9-6a39-417f-b36d-fb83a584adbc Address:b41584b9-6a39-417f-b36d-fb83a584adbc}]" -testLoggerAdapter verbose: server-ba7d1440-bc18-af30-a710-ac9ff72be94d: 2024-06-11T02:29:21.698-1200 [INFO] server-ba7d1440-bc18-af30-a710-ac9ff72be94d: entering follower state: follower="Node at ba7d1440-bc18-af30-a710-ac9ff72be94d [Follower]" leader= -testLoggerAdapter verbose: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.698-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ba7d1440-bc18-af30-a710-ac9ff72be94d Address:ba7d1440-bc18-af30-a710-ac9ff72be94d} {Suffrage:Voter ID:server-bb296b30-fe94-6473-a436-01bda40f8495 Address:bb296b30-fe94-6473-a436-01bda40f8495} {Suffrage:Voter ID:server-b41584b9-6a39-417f-b36d-fb83a584adbc Address:b41584b9-6a39-417f-b36d-fb83a584adbc}]" - testing.go:149: server-ba7d1440-bc18-af30-a710-ac9ff72be94d: 2024-06-11T02:29:21.698-1200 [INFO] server-ba7d1440-bc18-af30-a710-ac9ff72be94d: entering follower state: follower="Node at ba7d1440-bc18-af30-a710-ac9ff72be94d [Follower]" leader= -testLoggerAdapter verbose: server-bb296b30-fe94-6473-a436-01bda40f8495: 2024-06-11T02:29:21.698-1200 [INFO] server-bb296b30-fe94-6473-a436-01bda40f8495: entering follower state: follower="Node at bb296b30-fe94-6473-a436-01bda40f8495 [Follower]" leader= - testing.go:149: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.698-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ba7d1440-bc18-af30-a710-ac9ff72be94d Address:ba7d1440-bc18-af30-a710-ac9ff72be94d} {Suffrage:Voter ID:server-bb296b30-fe94-6473-a436-01bda40f8495 Address:bb296b30-fe94-6473-a436-01bda40f8495} {Suffrage:Voter ID:server-b41584b9-6a39-417f-b36d-fb83a584adbc Address:b41584b9-6a39-417f-b36d-fb83a584adbc}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:21.698-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:21.698-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: server-bb296b30-fe94-6473-a436-01bda40f8495: 2024-06-11T02:29:21.698-1200 [INFO] server-bb296b30-fe94-6473-a436-01bda40f8495: entering follower state: follower="Node at bb296b30-fe94-6473-a436-01bda40f8495 [Follower]" leader= -testLoggerAdapter verbose: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.698-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: entering follower state: follower="Node at b41584b9-6a39-417f-b36d-fb83a584adbc [Follower]" leader= - testing.go:149: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.698-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: entering follower state: follower="Node at b41584b9-6a39-417f-b36d-fb83a584adbc [Follower]" leader= -testLoggerAdapter verbose: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.756-1200 [WARN] server-b41584b9-6a39-417f-b36d-fb83a584adbc: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.756-1200 [WARN] server-b41584b9-6a39-417f-b36d-fb83a584adbc: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.756-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: entering candidate state: node="Node at b41584b9-6a39-417f-b36d-fb83a584adbc [Candidate]" term=2 - testing.go:149: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.756-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: entering candidate state: node="Node at b41584b9-6a39-417f-b36d-fb83a584adbc [Candidate]" term=2 -testLoggerAdapter verbose: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.757-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: election won: tally=2 - testing.go:149: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.757-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: election won: tally=2 -testLoggerAdapter verbose: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.757-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: entering leader state: leader="Node at b41584b9-6a39-417f-b36d-fb83a584adbc [Leader]" - testing.go:149: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.757-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: entering leader state: leader="Node at b41584b9-6a39-417f-b36d-fb83a584adbc [Leader]" -testLoggerAdapter verbose: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.757-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: added peer, starting replication: peer=server-ba7d1440-bc18-af30-a710-ac9ff72be94d - testing.go:149: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.757-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: added peer, starting replication: peer=server-ba7d1440-bc18-af30-a710-ac9ff72be94d -testLoggerAdapter verbose: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.757-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: added peer, starting replication: peer=server-bb296b30-fe94-6473-a436-01bda40f8495 - testing.go:149: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.757-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: added peer, starting replication: peer=server-bb296b30-fe94-6473-a436-01bda40f8495 -testLoggerAdapter verbose: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.757-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: pipelining replication: peer="{Voter server-ba7d1440-bc18-af30-a710-ac9ff72be94d ba7d1440-bc18-af30-a710-ac9ff72be94d}" - testing.go:149: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.757-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: pipelining replication: peer="{Voter server-ba7d1440-bc18-af30-a710-ac9ff72be94d ba7d1440-bc18-af30-a710-ac9ff72be94d}" -testLoggerAdapter verbose: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.757-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: pipelining replication: peer="{Voter server-bb296b30-fe94-6473-a436-01bda40f8495 bb296b30-fe94-6473-a436-01bda40f8495}" - testing.go:149: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.757-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: pipelining replication: peer="{Voter server-bb296b30-fe94-6473-a436-01bda40f8495 bb296b30-fe94-6473-a436-01bda40f8495}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:21.862-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:21.757421552 -1200 -12 m=+12.752794094 (1 nodes), 58.527066ms from start of poll, 60.478279ms from cluster start. Timeout at 2024-06-11 02:29:21.862445535 -1200 -12 m=+12.857818076, 105.023982ms after stability - testing.go:149: cluster: 2024-06-11T02:29:21.862-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:21.757421552 -1200 -12 m=+12.752794094 (1 nodes), 58.527066ms from start of poll, 60.478279ms from cluster start. Timeout at 2024-06-11 02:29:21.862445535 -1200 -12 m=+12.857818076, 105.023982ms after stability -testLoggerAdapter verbose: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.863-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: starting snapshot up to: index=12 - testing.go:149: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.863-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: starting snapshot up to: index=12 - testing.go:153: 2024-06-11T02:29:21.863-1200 [INFO] creating new snapshot: path=/tmp/raft3005702365/snapshots/2-12-1718116161863.tmp -testLoggerAdapter verbose: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.887-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: snapshot complete up to: index=12 - testing.go:149: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.887-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: snapshot complete up to: index=12 - testing.go:153: 2024-06-11T02:29:21.889-1200 [INFO] creating new snapshot: path=/tmp/raft3005702365/snapshots/2-113-1718116161889.tmp -testLoggerAdapter verbose: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.912-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: copied to local snapshot: bytes=71 - testing.go:149: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.912-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.913-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: restored user snapshot: index=1 - testing.go:149: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.913-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: restored user snapshot: index=1 -testLoggerAdapter verbose: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.913-1200 [ERROR] server-b41584b9-6a39-417f-b36d-fb83a584adbc: failed to get log: index=23 error="log not found" - testing.go:149: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.913-1200 [ERROR] server-b41584b9-6a39-417f-b36d-fb83a584adbc: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.913-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: aborting pipeline replication: peer="{Voter server-ba7d1440-bc18-af30-a710-ac9ff72be94d ba7d1440-bc18-af30-a710-ac9ff72be94d}" - testing.go:149: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.913-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: aborting pipeline replication: peer="{Voter server-ba7d1440-bc18-af30-a710-ac9ff72be94d ba7d1440-bc18-af30-a710-ac9ff72be94d}" -testLoggerAdapter verbose: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.913-1200 [ERROR] server-b41584b9-6a39-417f-b36d-fb83a584adbc: failed to get log: index=23 error="log not found" - testing.go:149: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.913-1200 [ERROR] server-b41584b9-6a39-417f-b36d-fb83a584adbc: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.914-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: aborting pipeline replication: peer="{Voter server-bb296b30-fe94-6473-a436-01bda40f8495 bb296b30-fe94-6473-a436-01bda40f8495}" - testing.go:149: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.914-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: aborting pipeline replication: peer="{Voter server-bb296b30-fe94-6473-a436-01bda40f8495 bb296b30-fe94-6473-a436-01bda40f8495}" -testLoggerAdapter verbose: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.922-1200 [ERROR] server-b41584b9-6a39-417f-b36d-fb83a584adbc: failed to get log: index=23 error="log not found" - testing.go:149: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.922-1200 [ERROR] server-b41584b9-6a39-417f-b36d-fb83a584adbc: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.923-1200 [ERROR] server-b41584b9-6a39-417f-b36d-fb83a584adbc: failed to get log: index=23 error="log not found" - testing.go:149: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.923-1200 [ERROR] server-b41584b9-6a39-417f-b36d-fb83a584adbc: failed to get log: index=23 error="log not found" - testing.go:153: 2024-06-11T02:29:21.925-1200 [INFO] creating new snapshot: path=/tmp/raft732812855/snapshots/2-113-1718116161925.tmp - testing.go:153: 2024-06-11T02:29:21.927-1200 [INFO] creating new snapshot: path=/tmp/raft2273779743/snapshots/2-113-1718116161927.tmp -testLoggerAdapter verbose: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.968-1200 [WARN] server-b41584b9-6a39-417f-b36d-fb83a584adbc: failed to contact: server-id=server-bb296b30-fe94-6473-a436-01bda40f8495 time=53.770655ms - testing.go:149: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.968-1200 [WARN] server-b41584b9-6a39-417f-b36d-fb83a584adbc: failed to contact: server-id=server-bb296b30-fe94-6473-a436-01bda40f8495 time=53.770655ms -testLoggerAdapter verbose: server-bb296b30-fe94-6473-a436-01bda40f8495: 2024-06-11T02:29:21.968-1200 [INFO] server-bb296b30-fe94-6473-a436-01bda40f8495: copied to local snapshot: bytes=71 - testing.go:149: server-bb296b30-fe94-6473-a436-01bda40f8495: 2024-06-11T02:29:21.968-1200 [INFO] server-bb296b30-fe94-6473-a436-01bda40f8495: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-bb296b30-fe94-6473-a436-01bda40f8495: 2024-06-11T02:29:21.969-1200 [INFO] server-bb296b30-fe94-6473-a436-01bda40f8495: Installed remote snapshot - testing.go:149: server-bb296b30-fe94-6473-a436-01bda40f8495: 2024-06-11T02:29:21.969-1200 [INFO] server-bb296b30-fe94-6473-a436-01bda40f8495: Installed remote snapshot -testLoggerAdapter verbose: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.969-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: pipelining replication: peer="{Voter server-bb296b30-fe94-6473-a436-01bda40f8495 bb296b30-fe94-6473-a436-01bda40f8495}" - testing.go:149: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.969-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: pipelining replication: peer="{Voter server-bb296b30-fe94-6473-a436-01bda40f8495 bb296b30-fe94-6473-a436-01bda40f8495}" -testLoggerAdapter verbose: server-ba7d1440-bc18-af30-a710-ac9ff72be94d: 2024-06-11T02:29:21.972-1200 [INFO] server-ba7d1440-bc18-af30-a710-ac9ff72be94d: copied to local snapshot: bytes=71 - testing.go:149: server-ba7d1440-bc18-af30-a710-ac9ff72be94d: 2024-06-11T02:29:21.972-1200 [INFO] server-ba7d1440-bc18-af30-a710-ac9ff72be94d: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-ba7d1440-bc18-af30-a710-ac9ff72be94d: 2024-06-11T02:29:21.973-1200 [INFO] server-ba7d1440-bc18-af30-a710-ac9ff72be94d: Installed remote snapshot - testing.go:149: server-ba7d1440-bc18-af30-a710-ac9ff72be94d: 2024-06-11T02:29:21.973-1200 [INFO] server-ba7d1440-bc18-af30-a710-ac9ff72be94d: Installed remote snapshot -testLoggerAdapter verbose: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.973-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: pipelining replication: peer="{Voter server-ba7d1440-bc18-af30-a710-ac9ff72be94d ba7d1440-bc18-af30-a710-ac9ff72be94d}" - testing.go:149: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.973-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: pipelining replication: peer="{Voter server-ba7d1440-bc18-af30-a710-ac9ff72be94d ba7d1440-bc18-af30-a710-ac9ff72be94d}" -testLoggerAdapter verbose: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.987-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: aborting pipeline replication: peer="{Voter server-bb296b30-fe94-6473-a436-01bda40f8495 bb296b30-fe94-6473-a436-01bda40f8495}" - testing.go:149: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.987-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: aborting pipeline replication: peer="{Voter server-bb296b30-fe94-6473-a436-01bda40f8495 bb296b30-fe94-6473-a436-01bda40f8495}" -testLoggerAdapter verbose: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.988-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: aborting pipeline replication: peer="{Voter server-ba7d1440-bc18-af30-a710-ac9ff72be94d ba7d1440-bc18-af30-a710-ac9ff72be94d}" - testing.go:149: server-b41584b9-6a39-417f-b36d-fb83a584adbc: 2024-06-11T02:29:21.988-1200 [INFO] server-b41584b9-6a39-417f-b36d-fb83a584adbc: aborting pipeline replication: peer="{Voter server-ba7d1440-bc18-af30-a710-ac9ff72be94d ba7d1440-bc18-af30-a710-ac9ff72be94d}" -testLoggerAdapter verbose: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:21.993-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6517289a-87eb-9fbf-30f5-e120555304bb Address:6517289a-87eb-9fbf-30f5-e120555304bb} {Suffrage:Voter ID:server-4b3eae16-e242-cadc-6cc3-69eeb043ed7a Address:4b3eae16-e242-cadc-6cc3-69eeb043ed7a} {Suffrage:Voter ID:server-e3803da5-e977-cff2-4fcb-6708c88caadf Address:e3803da5-e977-cff2-4fcb-6708c88caadf}]" - testing.go:149: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:21.993-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6517289a-87eb-9fbf-30f5-e120555304bb Address:6517289a-87eb-9fbf-30f5-e120555304bb} {Suffrage:Voter ID:server-4b3eae16-e242-cadc-6cc3-69eeb043ed7a Address:4b3eae16-e242-cadc-6cc3-69eeb043ed7a} {Suffrage:Voter ID:server-e3803da5-e977-cff2-4fcb-6708c88caadf Address:e3803da5-e977-cff2-4fcb-6708c88caadf}]" -testLoggerAdapter verbose: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:21.993-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: entering follower state: follower="Node at 6517289a-87eb-9fbf-30f5-e120555304bb [Follower]" leader= - testing.go:149: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:21.993-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: entering follower state: follower="Node at 6517289a-87eb-9fbf-30f5-e120555304bb [Follower]" leader= -testLoggerAdapter verbose: server-4b3eae16-e242-cadc-6cc3-69eeb043ed7a: 2024-06-11T02:29:21.993-1200 [INFO] server-4b3eae16-e242-cadc-6cc3-69eeb043ed7a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6517289a-87eb-9fbf-30f5-e120555304bb Address:6517289a-87eb-9fbf-30f5-e120555304bb} {Suffrage:Voter ID:server-4b3eae16-e242-cadc-6cc3-69eeb043ed7a Address:4b3eae16-e242-cadc-6cc3-69eeb043ed7a} {Suffrage:Voter ID:server-e3803da5-e977-cff2-4fcb-6708c88caadf Address:e3803da5-e977-cff2-4fcb-6708c88caadf}]" - testing.go:149: server-4b3eae16-e242-cadc-6cc3-69eeb043ed7a: 2024-06-11T02:29:21.993-1200 [INFO] server-4b3eae16-e242-cadc-6cc3-69eeb043ed7a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6517289a-87eb-9fbf-30f5-e120555304bb Address:6517289a-87eb-9fbf-30f5-e120555304bb} {Suffrage:Voter ID:server-4b3eae16-e242-cadc-6cc3-69eeb043ed7a Address:4b3eae16-e242-cadc-6cc3-69eeb043ed7a} {Suffrage:Voter ID:server-e3803da5-e977-cff2-4fcb-6708c88caadf Address:e3803da5-e977-cff2-4fcb-6708c88caadf}]" -testLoggerAdapter verbose: server-e3803da5-e977-cff2-4fcb-6708c88caadf: 2024-06-11T02:29:21.994-1200 [INFO] server-e3803da5-e977-cff2-4fcb-6708c88caadf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6517289a-87eb-9fbf-30f5-e120555304bb Address:6517289a-87eb-9fbf-30f5-e120555304bb} {Suffrage:Voter ID:server-4b3eae16-e242-cadc-6cc3-69eeb043ed7a Address:4b3eae16-e242-cadc-6cc3-69eeb043ed7a} {Suffrage:Voter ID:server-e3803da5-e977-cff2-4fcb-6708c88caadf Address:e3803da5-e977-cff2-4fcb-6708c88caadf}]" - testing.go:149: server-e3803da5-e977-cff2-4fcb-6708c88caadf: 2024-06-11T02:29:21.994-1200 [INFO] server-e3803da5-e977-cff2-4fcb-6708c88caadf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6517289a-87eb-9fbf-30f5-e120555304bb Address:6517289a-87eb-9fbf-30f5-e120555304bb} {Suffrage:Voter ID:server-4b3eae16-e242-cadc-6cc3-69eeb043ed7a Address:4b3eae16-e242-cadc-6cc3-69eeb043ed7a} {Suffrage:Voter ID:server-e3803da5-e977-cff2-4fcb-6708c88caadf Address:e3803da5-e977-cff2-4fcb-6708c88caadf}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:21.994-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:21.994-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-4b3eae16-e242-cadc-6cc3-69eeb043ed7a: 2024-06-11T02:29:21.994-1200 [INFO] server-4b3eae16-e242-cadc-6cc3-69eeb043ed7a: entering follower state: follower="Node at 4b3eae16-e242-cadc-6cc3-69eeb043ed7a [Follower]" leader= - testing.go:149: server-4b3eae16-e242-cadc-6cc3-69eeb043ed7a: 2024-06-11T02:29:21.994-1200 [INFO] server-4b3eae16-e242-cadc-6cc3-69eeb043ed7a: entering follower state: follower="Node at 4b3eae16-e242-cadc-6cc3-69eeb043ed7a [Follower]" leader= -testLoggerAdapter verbose: server-e3803da5-e977-cff2-4fcb-6708c88caadf: 2024-06-11T02:29:21.994-1200 [INFO] server-e3803da5-e977-cff2-4fcb-6708c88caadf: entering follower state: follower="Node at e3803da5-e977-cff2-4fcb-6708c88caadf [Follower]" leader= - testing.go:149: server-e3803da5-e977-cff2-4fcb-6708c88caadf: 2024-06-11T02:29:21.994-1200 [INFO] server-e3803da5-e977-cff2-4fcb-6708c88caadf: entering follower state: follower="Node at e3803da5-e977-cff2-4fcb-6708c88caadf [Follower]" leader= -testLoggerAdapter verbose: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.046-1200 [WARN] server-6517289a-87eb-9fbf-30f5-e120555304bb: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.046-1200 [WARN] server-6517289a-87eb-9fbf-30f5-e120555304bb: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.047-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: entering candidate state: node="Node at 6517289a-87eb-9fbf-30f5-e120555304bb [Candidate]" term=2 - testing.go:149: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.047-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: entering candidate state: node="Node at 6517289a-87eb-9fbf-30f5-e120555304bb [Candidate]" term=2 -testLoggerAdapter verbose: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.047-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: election won: tally=2 - testing.go:149: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.047-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: election won: tally=2 -testLoggerAdapter verbose: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.047-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: entering leader state: leader="Node at 6517289a-87eb-9fbf-30f5-e120555304bb [Leader]" - testing.go:149: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.047-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: entering leader state: leader="Node at 6517289a-87eb-9fbf-30f5-e120555304bb [Leader]" -testLoggerAdapter verbose: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.047-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: added peer, starting replication: peer=server-4b3eae16-e242-cadc-6cc3-69eeb043ed7a - testing.go:149: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.047-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: added peer, starting replication: peer=server-4b3eae16-e242-cadc-6cc3-69eeb043ed7a -testLoggerAdapter verbose: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.047-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: added peer, starting replication: peer=server-e3803da5-e977-cff2-4fcb-6708c88caadf - testing.go:149: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.047-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: added peer, starting replication: peer=server-e3803da5-e977-cff2-4fcb-6708c88caadf -testLoggerAdapter verbose: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.048-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: pipelining replication: peer="{Voter server-e3803da5-e977-cff2-4fcb-6708c88caadf e3803da5-e977-cff2-4fcb-6708c88caadf}" - testing.go:149: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.048-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: pipelining replication: peer="{Voter server-e3803da5-e977-cff2-4fcb-6708c88caadf e3803da5-e977-cff2-4fcb-6708c88caadf}" -testLoggerAdapter verbose: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.048-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: pipelining replication: peer="{Voter server-4b3eae16-e242-cadc-6cc3-69eeb043ed7a 4b3eae16-e242-cadc-6cc3-69eeb043ed7a}" - testing.go:149: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.048-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: pipelining replication: peer="{Voter server-4b3eae16-e242-cadc-6cc3-69eeb043ed7a 4b3eae16-e242-cadc-6cc3-69eeb043ed7a}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:22.154-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:22.048830153 -1200 -12 m=+13.044202695 (1 nodes), 54.405018ms from start of poll, 55.750743ms from cluster start. Timeout at 2024-06-11 02:29:22.154564039 -1200 -12 m=+13.149936873, 105.734178ms after stability - testing.go:149: cluster: 2024-06-11T02:29:22.154-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:22.048830153 -1200 -12 m=+13.044202695 (1 nodes), 54.405018ms from start of poll, 55.750743ms from cluster start. Timeout at 2024-06-11 02:29:22.154564039 -1200 -12 m=+13.149936873, 105.734178ms after stability -testLoggerAdapter verbose: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.155-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: starting snapshot up to: index=12 - testing.go:149: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.155-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: starting snapshot up to: index=12 - testing.go:153: 2024-06-11T02:29:22.155-1200 [INFO] creating new snapshot: path=/tmp/raft2477218683/snapshots/2-12-1718116162155.tmp -testLoggerAdapter verbose: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.183-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: snapshot complete up to: index=12 - testing.go:149: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.183-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: snapshot complete up to: index=12 - testing.go:153: 2024-06-11T02:29:22.184-1200 [INFO] creating new snapshot: path=/tmp/raft2477218683/snapshots/2-1013-1718116162184.tmp -testLoggerAdapter verbose: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.212-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: copied to local snapshot: bytes=71 - testing.go:149: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.212-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.213-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: restored user snapshot: index=1 - testing.go:149: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.213-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: restored user snapshot: index=1 -testLoggerAdapter verbose: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.214-1200 [ERROR] server-6517289a-87eb-9fbf-30f5-e120555304bb: failed to get log: index=23 error="log not found" - testing.go:149: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.214-1200 [ERROR] server-6517289a-87eb-9fbf-30f5-e120555304bb: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.214-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: aborting pipeline replication: peer="{Voter server-e3803da5-e977-cff2-4fcb-6708c88caadf e3803da5-e977-cff2-4fcb-6708c88caadf}" - testing.go:149: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.214-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: aborting pipeline replication: peer="{Voter server-e3803da5-e977-cff2-4fcb-6708c88caadf e3803da5-e977-cff2-4fcb-6708c88caadf}" -testLoggerAdapter verbose: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.214-1200 [ERROR] server-6517289a-87eb-9fbf-30f5-e120555304bb: failed to get log: index=23 error="log not found" - testing.go:149: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.214-1200 [ERROR] server-6517289a-87eb-9fbf-30f5-e120555304bb: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.214-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: aborting pipeline replication: peer="{Voter server-4b3eae16-e242-cadc-6cc3-69eeb043ed7a 4b3eae16-e242-cadc-6cc3-69eeb043ed7a}" - testing.go:149: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.214-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: aborting pipeline replication: peer="{Voter server-4b3eae16-e242-cadc-6cc3-69eeb043ed7a 4b3eae16-e242-cadc-6cc3-69eeb043ed7a}" -testLoggerAdapter verbose: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.222-1200 [ERROR] server-6517289a-87eb-9fbf-30f5-e120555304bb: failed to get log: index=23 error="log not found" - testing.go:149: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.222-1200 [ERROR] server-6517289a-87eb-9fbf-30f5-e120555304bb: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.223-1200 [ERROR] server-6517289a-87eb-9fbf-30f5-e120555304bb: failed to get log: index=23 error="log not found" - testing.go:149: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.223-1200 [ERROR] server-6517289a-87eb-9fbf-30f5-e120555304bb: failed to get log: index=23 error="log not found" - testing.go:153: 2024-06-11T02:29:22.224-1200 [INFO] creating new snapshot: path=/tmp/raft3225299268/snapshots/2-1013-1718116162224.tmp - testing.go:153: 2024-06-11T02:29:22.225-1200 [INFO] creating new snapshot: path=/tmp/raft261460529/snapshots/2-1013-1718116162225.tmp -testLoggerAdapter verbose: server-e3803da5-e977-cff2-4fcb-6708c88caadf: 2024-06-11T02:29:22.260-1200 [INFO] server-e3803da5-e977-cff2-4fcb-6708c88caadf: copied to local snapshot: bytes=71 - testing.go:149: server-e3803da5-e977-cff2-4fcb-6708c88caadf: 2024-06-11T02:29:22.260-1200 [INFO] server-e3803da5-e977-cff2-4fcb-6708c88caadf: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-e3803da5-e977-cff2-4fcb-6708c88caadf: 2024-06-11T02:29:22.261-1200 [INFO] server-e3803da5-e977-cff2-4fcb-6708c88caadf: Installed remote snapshot - testing.go:149: server-e3803da5-e977-cff2-4fcb-6708c88caadf: 2024-06-11T02:29:22.261-1200 [INFO] server-e3803da5-e977-cff2-4fcb-6708c88caadf: Installed remote snapshot -testLoggerAdapter verbose: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.261-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: pipelining replication: peer="{Voter server-e3803da5-e977-cff2-4fcb-6708c88caadf e3803da5-e977-cff2-4fcb-6708c88caadf}" - testing.go:149: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.261-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: pipelining replication: peer="{Voter server-e3803da5-e977-cff2-4fcb-6708c88caadf e3803da5-e977-cff2-4fcb-6708c88caadf}" -testLoggerAdapter verbose: server-4b3eae16-e242-cadc-6cc3-69eeb043ed7a: 2024-06-11T02:29:22.264-1200 [INFO] server-4b3eae16-e242-cadc-6cc3-69eeb043ed7a: copied to local snapshot: bytes=71 - testing.go:149: server-4b3eae16-e242-cadc-6cc3-69eeb043ed7a: 2024-06-11T02:29:22.264-1200 [INFO] server-4b3eae16-e242-cadc-6cc3-69eeb043ed7a: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-4b3eae16-e242-cadc-6cc3-69eeb043ed7a: 2024-06-11T02:29:22.264-1200 [INFO] server-4b3eae16-e242-cadc-6cc3-69eeb043ed7a: Installed remote snapshot - testing.go:149: server-4b3eae16-e242-cadc-6cc3-69eeb043ed7a: 2024-06-11T02:29:22.264-1200 [INFO] server-4b3eae16-e242-cadc-6cc3-69eeb043ed7a: Installed remote snapshot -testLoggerAdapter verbose: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.265-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: pipelining replication: peer="{Voter server-4b3eae16-e242-cadc-6cc3-69eeb043ed7a 4b3eae16-e242-cadc-6cc3-69eeb043ed7a}" - testing.go:149: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.265-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: pipelining replication: peer="{Voter server-4b3eae16-e242-cadc-6cc3-69eeb043ed7a 4b3eae16-e242-cadc-6cc3-69eeb043ed7a}" -testLoggerAdapter verbose: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.278-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: aborting pipeline replication: peer="{Voter server-4b3eae16-e242-cadc-6cc3-69eeb043ed7a 4b3eae16-e242-cadc-6cc3-69eeb043ed7a}" - testing.go:149: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.278-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: aborting pipeline replication: peer="{Voter server-4b3eae16-e242-cadc-6cc3-69eeb043ed7a 4b3eae16-e242-cadc-6cc3-69eeb043ed7a}" -testLoggerAdapter verbose: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.278-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: aborting pipeline replication: peer="{Voter server-e3803da5-e977-cff2-4fcb-6708c88caadf e3803da5-e977-cff2-4fcb-6708c88caadf}" - testing.go:149: server-6517289a-87eb-9fbf-30f5-e120555304bb: 2024-06-11T02:29:22.278-1200 [INFO] server-6517289a-87eb-9fbf-30f5-e120555304bb: aborting pipeline replication: peer="{Voter server-e3803da5-e977-cff2-4fcb-6708c88caadf e3803da5-e977-cff2-4fcb-6708c88caadf}" -testLoggerAdapter verbose: server-7a82776e-fb5b-1fc9-a547-fcdc66295480: 2024-06-11T02:29:22.283-1200 [INFO] server-7a82776e-fb5b-1fc9-a547-fcdc66295480: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7a82776e-fb5b-1fc9-a547-fcdc66295480 Address:7a82776e-fb5b-1fc9-a547-fcdc66295480} {Suffrage:Voter ID:server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25 Address:d4c17559-c7cf-f866-cbfa-b4773ab8ac25} {Suffrage:Voter ID:server-918b0d2f-2eec-60a7-839e-61c69d9699db Address:918b0d2f-2eec-60a7-839e-61c69d9699db}]" - testing.go:149: server-7a82776e-fb5b-1fc9-a547-fcdc66295480: 2024-06-11T02:29:22.283-1200 [INFO] server-7a82776e-fb5b-1fc9-a547-fcdc66295480: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7a82776e-fb5b-1fc9-a547-fcdc66295480 Address:7a82776e-fb5b-1fc9-a547-fcdc66295480} {Suffrage:Voter ID:server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25 Address:d4c17559-c7cf-f866-cbfa-b4773ab8ac25} {Suffrage:Voter ID:server-918b0d2f-2eec-60a7-839e-61c69d9699db Address:918b0d2f-2eec-60a7-839e-61c69d9699db}]" -testLoggerAdapter verbose: server-7a82776e-fb5b-1fc9-a547-fcdc66295480: 2024-06-11T02:29:22.284-1200 [INFO] server-7a82776e-fb5b-1fc9-a547-fcdc66295480: entering follower state: follower="Node at 7a82776e-fb5b-1fc9-a547-fcdc66295480 [Follower]" leader= - testing.go:149: server-7a82776e-fb5b-1fc9-a547-fcdc66295480: 2024-06-11T02:29:22.284-1200 [INFO] server-7a82776e-fb5b-1fc9-a547-fcdc66295480: entering follower state: follower="Node at 7a82776e-fb5b-1fc9-a547-fcdc66295480 [Follower]" leader= -testLoggerAdapter verbose: server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25: 2024-06-11T02:29:22.284-1200 [INFO] server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7a82776e-fb5b-1fc9-a547-fcdc66295480 Address:7a82776e-fb5b-1fc9-a547-fcdc66295480} {Suffrage:Voter ID:server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25 Address:d4c17559-c7cf-f866-cbfa-b4773ab8ac25} {Suffrage:Voter ID:server-918b0d2f-2eec-60a7-839e-61c69d9699db Address:918b0d2f-2eec-60a7-839e-61c69d9699db}]" - testing.go:149: server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25: 2024-06-11T02:29:22.284-1200 [INFO] server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7a82776e-fb5b-1fc9-a547-fcdc66295480 Address:7a82776e-fb5b-1fc9-a547-fcdc66295480} {Suffrage:Voter ID:server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25 Address:d4c17559-c7cf-f866-cbfa-b4773ab8ac25} {Suffrage:Voter ID:server-918b0d2f-2eec-60a7-839e-61c69d9699db Address:918b0d2f-2eec-60a7-839e-61c69d9699db}]" -testLoggerAdapter verbose: server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25: 2024-06-11T02:29:22.284-1200 [INFO] server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25: entering follower state: follower="Node at d4c17559-c7cf-f866-cbfa-b4773ab8ac25 [Follower]" leader= - testing.go:149: server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25: 2024-06-11T02:29:22.284-1200 [INFO] server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25: entering follower state: follower="Node at d4c17559-c7cf-f866-cbfa-b4773ab8ac25 [Follower]" leader= -testLoggerAdapter verbose: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.284-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7a82776e-fb5b-1fc9-a547-fcdc66295480 Address:7a82776e-fb5b-1fc9-a547-fcdc66295480} {Suffrage:Voter ID:server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25 Address:d4c17559-c7cf-f866-cbfa-b4773ab8ac25} {Suffrage:Voter ID:server-918b0d2f-2eec-60a7-839e-61c69d9699db Address:918b0d2f-2eec-60a7-839e-61c69d9699db}]" - testing.go:149: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.284-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7a82776e-fb5b-1fc9-a547-fcdc66295480 Address:7a82776e-fb5b-1fc9-a547-fcdc66295480} {Suffrage:Voter ID:server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25 Address:d4c17559-c7cf-f866-cbfa-b4773ab8ac25} {Suffrage:Voter ID:server-918b0d2f-2eec-60a7-839e-61c69d9699db Address:918b0d2f-2eec-60a7-839e-61c69d9699db}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:22.284-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:22.284-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.284-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: entering follower state: follower="Node at 918b0d2f-2eec-60a7-839e-61c69d9699db [Follower]" leader= - testing.go:149: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.284-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: entering follower state: follower="Node at 918b0d2f-2eec-60a7-839e-61c69d9699db [Follower]" leader= -testLoggerAdapter verbose: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.358-1200 [WARN] server-918b0d2f-2eec-60a7-839e-61c69d9699db: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.358-1200 [WARN] server-918b0d2f-2eec-60a7-839e-61c69d9699db: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.358-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: entering candidate state: node="Node at 918b0d2f-2eec-60a7-839e-61c69d9699db [Candidate]" term=2 - testing.go:149: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.358-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: entering candidate state: node="Node at 918b0d2f-2eec-60a7-839e-61c69d9699db [Candidate]" term=2 -testLoggerAdapter verbose: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.359-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: election won: tally=2 - testing.go:149: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.359-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: election won: tally=2 -testLoggerAdapter verbose: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.359-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: entering leader state: leader="Node at 918b0d2f-2eec-60a7-839e-61c69d9699db [Leader]" - testing.go:149: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.359-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: entering leader state: leader="Node at 918b0d2f-2eec-60a7-839e-61c69d9699db [Leader]" -testLoggerAdapter verbose: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.359-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: added peer, starting replication: peer=server-7a82776e-fb5b-1fc9-a547-fcdc66295480 - testing.go:149: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.359-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: added peer, starting replication: peer=server-7a82776e-fb5b-1fc9-a547-fcdc66295480 -testLoggerAdapter verbose: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.359-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: added peer, starting replication: peer=server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25 - testing.go:149: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.359-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: added peer, starting replication: peer=server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25 -testLoggerAdapter verbose: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.359-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: pipelining replication: peer="{Voter server-7a82776e-fb5b-1fc9-a547-fcdc66295480 7a82776e-fb5b-1fc9-a547-fcdc66295480}" - testing.go:149: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.359-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: pipelining replication: peer="{Voter server-7a82776e-fb5b-1fc9-a547-fcdc66295480 7a82776e-fb5b-1fc9-a547-fcdc66295480}" -testLoggerAdapter verbose: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.359-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: pipelining replication: peer="{Voter server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25 d4c17559-c7cf-f866-cbfa-b4773ab8ac25}" - testing.go:149: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.359-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: pipelining replication: peer="{Voter server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25 d4c17559-c7cf-f866-cbfa-b4773ab8ac25}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:22.464-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:22.35936152 -1200 -12 m=+13.354734062 (1 nodes), 74.404225ms from start of poll, 75.65166ms from cluster start. Timeout at 2024-06-11 02:29:22.464595499 -1200 -12 m=+13.459968041, 105.233979ms after stability - testing.go:149: cluster: 2024-06-11T02:29:22.464-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:22.35936152 -1200 -12 m=+13.354734062 (1 nodes), 74.404225ms from start of poll, 75.65166ms from cluster start. Timeout at 2024-06-11 02:29:22.464595499 -1200 -12 m=+13.459968041, 105.233979ms after stability -testLoggerAdapter verbose: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.466-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: starting snapshot up to: index=12 - testing.go:149: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.466-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: starting snapshot up to: index=12 - testing.go:153: 2024-06-11T02:29:22.466-1200 [INFO] creating new snapshot: path=/tmp/raft2033994446/snapshots/2-12-1718116162466.tmp -testLoggerAdapter verbose: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.491-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: snapshot complete up to: index=12 - testing.go:149: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.491-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: snapshot complete up to: index=12 - testing.go:153: 2024-06-11T02:29:22.493-1200 [INFO] creating new snapshot: path=/tmp/raft2033994446/snapshots/2-10013-1718116162493.tmp -testLoggerAdapter verbose: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.513-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: copied to local snapshot: bytes=71 - testing.go:149: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.513-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.514-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: restored user snapshot: index=1 - testing.go:149: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.514-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: restored user snapshot: index=1 -testLoggerAdapter verbose: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.514-1200 [ERROR] server-918b0d2f-2eec-60a7-839e-61c69d9699db: failed to get log: index=23 error="log not found" - testing.go:149: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.514-1200 [ERROR] server-918b0d2f-2eec-60a7-839e-61c69d9699db: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.514-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: aborting pipeline replication: peer="{Voter server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25 d4c17559-c7cf-f866-cbfa-b4773ab8ac25}" - testing.go:149: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.514-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: aborting pipeline replication: peer="{Voter server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25 d4c17559-c7cf-f866-cbfa-b4773ab8ac25}" -testLoggerAdapter verbose: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.514-1200 [ERROR] server-918b0d2f-2eec-60a7-839e-61c69d9699db: failed to get log: index=23 error="log not found" - testing.go:149: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.514-1200 [ERROR] server-918b0d2f-2eec-60a7-839e-61c69d9699db: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.514-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: aborting pipeline replication: peer="{Voter server-7a82776e-fb5b-1fc9-a547-fcdc66295480 7a82776e-fb5b-1fc9-a547-fcdc66295480}" - testing.go:149: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.514-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: aborting pipeline replication: peer="{Voter server-7a82776e-fb5b-1fc9-a547-fcdc66295480 7a82776e-fb5b-1fc9-a547-fcdc66295480}" -testLoggerAdapter verbose: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.521-1200 [ERROR] server-918b0d2f-2eec-60a7-839e-61c69d9699db: failed to get log: index=23 error="log not found" - testing.go:149: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.521-1200 [ERROR] server-918b0d2f-2eec-60a7-839e-61c69d9699db: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.522-1200 [ERROR] server-918b0d2f-2eec-60a7-839e-61c69d9699db: failed to get log: index=23 error="log not found" - testing.go:153: 2024-06-11T02:29:22.522-1200 [INFO] creating new snapshot: path=/tmp/raft1639574106/snapshots/2-10013-1718116162522.tmp - testing.go:149: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.522-1200 [ERROR] server-918b0d2f-2eec-60a7-839e-61c69d9699db: failed to get log: index=23 error="log not found" - testing.go:153: 2024-06-11T02:29:22.523-1200 [INFO] creating new snapshot: path=/tmp/raft2702895071/snapshots/2-10013-1718116162523.tmp -testLoggerAdapter verbose: server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25: 2024-06-11T02:29:22.557-1200 [INFO] server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25: copied to local snapshot: bytes=71 - testing.go:149: server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25: 2024-06-11T02:29:22.557-1200 [INFO] server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25: 2024-06-11T02:29:22.558-1200 [INFO] server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25: Installed remote snapshot - testing.go:149: server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25: 2024-06-11T02:29:22.558-1200 [INFO] server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25: Installed remote snapshot -testLoggerAdapter verbose: server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25: 2024-06-11T02:29:22.559-1200 [WARN] server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25: skipping application of old log: index=10013 - testing.go:149: server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25: 2024-06-11T02:29:22.559-1200 [WARN] server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25: skipping application of old log: index=10013 -testLoggerAdapter verbose: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.559-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: pipelining replication: peer="{Voter server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25 d4c17559-c7cf-f866-cbfa-b4773ab8ac25}" - testing.go:149: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.559-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: pipelining replication: peer="{Voter server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25 d4c17559-c7cf-f866-cbfa-b4773ab8ac25}" -testLoggerAdapter verbose: server-7a82776e-fb5b-1fc9-a547-fcdc66295480: 2024-06-11T02:29:22.564-1200 [INFO] server-7a82776e-fb5b-1fc9-a547-fcdc66295480: copied to local snapshot: bytes=71 - testing.go:149: server-7a82776e-fb5b-1fc9-a547-fcdc66295480: 2024-06-11T02:29:22.564-1200 [INFO] server-7a82776e-fb5b-1fc9-a547-fcdc66295480: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-7a82776e-fb5b-1fc9-a547-fcdc66295480: 2024-06-11T02:29:22.565-1200 [INFO] server-7a82776e-fb5b-1fc9-a547-fcdc66295480: Installed remote snapshot - testing.go:149: server-7a82776e-fb5b-1fc9-a547-fcdc66295480: 2024-06-11T02:29:22.565-1200 [INFO] server-7a82776e-fb5b-1fc9-a547-fcdc66295480: Installed remote snapshot -testLoggerAdapter verbose: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.566-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: pipelining replication: peer="{Voter server-7a82776e-fb5b-1fc9-a547-fcdc66295480 7a82776e-fb5b-1fc9-a547-fcdc66295480}" - testing.go:149: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.566-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: pipelining replication: peer="{Voter server-7a82776e-fb5b-1fc9-a547-fcdc66295480 7a82776e-fb5b-1fc9-a547-fcdc66295480}" -testLoggerAdapter verbose: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.583-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: aborting pipeline replication: peer="{Voter server-7a82776e-fb5b-1fc9-a547-fcdc66295480 7a82776e-fb5b-1fc9-a547-fcdc66295480}" - testing.go:149: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.583-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: aborting pipeline replication: peer="{Voter server-7a82776e-fb5b-1fc9-a547-fcdc66295480 7a82776e-fb5b-1fc9-a547-fcdc66295480}" -testLoggerAdapter verbose: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.583-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: aborting pipeline replication: peer="{Voter server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25 d4c17559-c7cf-f866-cbfa-b4773ab8ac25}" - testing.go:149: server-918b0d2f-2eec-60a7-839e-61c69d9699db: 2024-06-11T02:29:22.583-1200 [INFO] server-918b0d2f-2eec-60a7-839e-61c69d9699db: aborting pipeline replication: peer="{Voter server-d4c17559-c7cf-f866-cbfa-b4773ab8ac25 d4c17559-c7cf-f866-cbfa-b4773ab8ac25}" ---- PASS: TestRaft_UserRestore (1.78s) -=== RUN TestRaft_SendSnapshotFollower -testLoggerAdapter verbose: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.593-1200 [INFO] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cb0ac0bd-3251-e119-192a-edc96d61eb5d Address:cb0ac0bd-3251-e119-192a-edc96d61eb5d} {Suffrage:Voter ID:server-130cacbb-9be7-2e54-d564-541e8faec7ad Address:130cacbb-9be7-2e54-d564-541e8faec7ad} {Suffrage:Voter ID:server-978e9054-d9ef-3e7f-cc56-db2b9416dd05 Address:978e9054-d9ef-3e7f-cc56-db2b9416dd05}]" - testing.go:149: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.593-1200 [INFO] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cb0ac0bd-3251-e119-192a-edc96d61eb5d Address:cb0ac0bd-3251-e119-192a-edc96d61eb5d} {Suffrage:Voter ID:server-130cacbb-9be7-2e54-d564-541e8faec7ad Address:130cacbb-9be7-2e54-d564-541e8faec7ad} {Suffrage:Voter ID:server-978e9054-d9ef-3e7f-cc56-db2b9416dd05 Address:978e9054-d9ef-3e7f-cc56-db2b9416dd05}]" -testLoggerAdapter verbose: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.593-1200 [INFO] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: entering follower state: follower="Node at cb0ac0bd-3251-e119-192a-edc96d61eb5d [Follower]" leader= - testing.go:149: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.593-1200 [INFO] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: entering follower state: follower="Node at cb0ac0bd-3251-e119-192a-edc96d61eb5d [Follower]" leader= -testLoggerAdapter verbose: server-130cacbb-9be7-2e54-d564-541e8faec7ad: 2024-06-11T02:29:22.593-1200 [INFO] server-130cacbb-9be7-2e54-d564-541e8faec7ad: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cb0ac0bd-3251-e119-192a-edc96d61eb5d Address:cb0ac0bd-3251-e119-192a-edc96d61eb5d} {Suffrage:Voter ID:server-130cacbb-9be7-2e54-d564-541e8faec7ad Address:130cacbb-9be7-2e54-d564-541e8faec7ad} {Suffrage:Voter ID:server-978e9054-d9ef-3e7f-cc56-db2b9416dd05 Address:978e9054-d9ef-3e7f-cc56-db2b9416dd05}]" - testing.go:149: server-130cacbb-9be7-2e54-d564-541e8faec7ad: 2024-06-11T02:29:22.593-1200 [INFO] server-130cacbb-9be7-2e54-d564-541e8faec7ad: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cb0ac0bd-3251-e119-192a-edc96d61eb5d Address:cb0ac0bd-3251-e119-192a-edc96d61eb5d} {Suffrage:Voter ID:server-130cacbb-9be7-2e54-d564-541e8faec7ad Address:130cacbb-9be7-2e54-d564-541e8faec7ad} {Suffrage:Voter ID:server-978e9054-d9ef-3e7f-cc56-db2b9416dd05 Address:978e9054-d9ef-3e7f-cc56-db2b9416dd05}]" -testLoggerAdapter verbose: server-130cacbb-9be7-2e54-d564-541e8faec7ad: 2024-06-11T02:29:22.594-1200 [INFO] server-130cacbb-9be7-2e54-d564-541e8faec7ad: entering follower state: follower="Node at 130cacbb-9be7-2e54-d564-541e8faec7ad [Follower]" leader= - testing.go:149: server-130cacbb-9be7-2e54-d564-541e8faec7ad: 2024-06-11T02:29:22.594-1200 [INFO] server-130cacbb-9be7-2e54-d564-541e8faec7ad: entering follower state: follower="Node at 130cacbb-9be7-2e54-d564-541e8faec7ad [Follower]" leader= -testLoggerAdapter verbose: server-978e9054-d9ef-3e7f-cc56-db2b9416dd05: 2024-06-11T02:29:22.594-1200 [INFO] server-978e9054-d9ef-3e7f-cc56-db2b9416dd05: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cb0ac0bd-3251-e119-192a-edc96d61eb5d Address:cb0ac0bd-3251-e119-192a-edc96d61eb5d} {Suffrage:Voter ID:server-130cacbb-9be7-2e54-d564-541e8faec7ad Address:130cacbb-9be7-2e54-d564-541e8faec7ad} {Suffrage:Voter ID:server-978e9054-d9ef-3e7f-cc56-db2b9416dd05 Address:978e9054-d9ef-3e7f-cc56-db2b9416dd05}]" - testing.go:149: server-978e9054-d9ef-3e7f-cc56-db2b9416dd05: 2024-06-11T02:29:22.594-1200 [INFO] server-978e9054-d9ef-3e7f-cc56-db2b9416dd05: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cb0ac0bd-3251-e119-192a-edc96d61eb5d Address:cb0ac0bd-3251-e119-192a-edc96d61eb5d} {Suffrage:Voter ID:server-130cacbb-9be7-2e54-d564-541e8faec7ad Address:130cacbb-9be7-2e54-d564-541e8faec7ad} {Suffrage:Voter ID:server-978e9054-d9ef-3e7f-cc56-db2b9416dd05 Address:978e9054-d9ef-3e7f-cc56-db2b9416dd05}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:22.594-1200 [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: server-978e9054-d9ef-3e7f-cc56-db2b9416dd05: 2024-06-11T02:29:22.594-1200 [INFO] server-978e9054-d9ef-3e7f-cc56-db2b9416dd05: entering follower state: follower="Node at 978e9054-d9ef-3e7f-cc56-db2b9416dd05 [Follower]" leader= - testing.go:149: cluster: 2024-06-11T02:29:22.594-1200 [INFO] starting stability test: raft-state=Follower - testing.go:149: server-978e9054-d9ef-3e7f-cc56-db2b9416dd05: 2024-06-11T02:29:22.594-1200 [INFO] server-978e9054-d9ef-3e7f-cc56-db2b9416dd05: entering follower state: follower="Node at 978e9054-d9ef-3e7f-cc56-db2b9416dd05 [Follower]" leader= -testLoggerAdapter verbose: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.644-1200 [WARN] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.644-1200 [WARN] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.644-1200 [INFO] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: entering candidate state: node="Node at cb0ac0bd-3251-e119-192a-edc96d61eb5d [Candidate]" term=2 - testing.go:149: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.644-1200 [INFO] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: entering candidate state: node="Node at cb0ac0bd-3251-e119-192a-edc96d61eb5d [Candidate]" term=2 -testLoggerAdapter verbose: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.645-1200 [INFO] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: election won: tally=2 - testing.go:149: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.645-1200 [INFO] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: election won: tally=2 -testLoggerAdapter verbose: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.645-1200 [INFO] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: entering leader state: leader="Node at cb0ac0bd-3251-e119-192a-edc96d61eb5d [Leader]" - testing.go:149: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.645-1200 [INFO] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: entering leader state: leader="Node at cb0ac0bd-3251-e119-192a-edc96d61eb5d [Leader]" -testLoggerAdapter verbose: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.645-1200 [INFO] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: added peer, starting replication: peer=server-130cacbb-9be7-2e54-d564-541e8faec7ad - testing.go:149: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.645-1200 [INFO] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: added peer, starting replication: peer=server-130cacbb-9be7-2e54-d564-541e8faec7ad -testLoggerAdapter verbose: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.645-1200 [INFO] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: added peer, starting replication: peer=server-978e9054-d9ef-3e7f-cc56-db2b9416dd05 - testing.go:149: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.645-1200 [INFO] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: added peer, starting replication: peer=server-978e9054-d9ef-3e7f-cc56-db2b9416dd05 -testLoggerAdapter verbose: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.645-1200 [INFO] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: pipelining replication: peer="{Voter server-978e9054-d9ef-3e7f-cc56-db2b9416dd05 978e9054-d9ef-3e7f-cc56-db2b9416dd05}" - testing.go:149: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.645-1200 [INFO] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: pipelining replication: peer="{Voter server-978e9054-d9ef-3e7f-cc56-db2b9416dd05 978e9054-d9ef-3e7f-cc56-db2b9416dd05}" -testLoggerAdapter verbose: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.645-1200 [INFO] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: pipelining replication: peer="{Voter server-130cacbb-9be7-2e54-d564-541e8faec7ad 130cacbb-9be7-2e54-d564-541e8faec7ad}" - testing.go:149: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.645-1200 [INFO] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: pipelining replication: peer="{Voter server-130cacbb-9be7-2e54-d564-541e8faec7ad 130cacbb-9be7-2e54-d564-541e8faec7ad}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:22.751-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:22.646047836 -1200 -12 m=+13.641420377 (2 nodes), 51.17487ms from start of poll, 53.24304ms from cluster start. Timeout at 2024-06-11 02:29:22.751757222 -1200 -12 m=+13.747130055, 105.709678ms after stability - testing.go:149: cluster: 2024-06-11T02:29:22.751-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:22.646047836 -1200 -12 m=+13.641420377 (2 nodes), 51.17487ms from start of poll, 53.24304ms from cluster start. Timeout at 2024-06-11 02:29:22.751757222 -1200 -12 m=+13.747130055, 105.709678ms after stability -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:22.751-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:22.751-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:22.857-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:22.752084174 -1200 -12 m=+13.747456716 (1 nodes), 3.208µs from start of poll, 159.279379ms from cluster start. Timeout at 2024-06-11 02:29:22.857818644 -1200 -12 m=+13.853191477, 105.734761ms after stability - testing.go:149: cluster: 2024-06-11T02:29:22.857-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:22.752084174 -1200 -12 m=+13.747456716 (1 nodes), 3.208µs from start of poll, 159.279379ms from cluster start. Timeout at 2024-06-11 02:29:22.857818644 -1200 -12 m=+13.853191477, 105.734761ms after stability -testLoggerAdapter verbose: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.858-1200 [ERROR] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: failed to pipeline appendEntries: peer="{Voter server-130cacbb-9be7-2e54-d564-541e8faec7ad 130cacbb-9be7-2e54-d564-541e8faec7ad}" error="append pipeline closed" - testing.go:149: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.858-1200 [ERROR] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: failed to pipeline appendEntries: peer="{Voter server-130cacbb-9be7-2e54-d564-541e8faec7ad 130cacbb-9be7-2e54-d564-541e8faec7ad}" error="append pipeline closed" -testLoggerAdapter verbose: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.858-1200 [INFO] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: aborting pipeline replication: peer="{Voter server-130cacbb-9be7-2e54-d564-541e8faec7ad 130cacbb-9be7-2e54-d564-541e8faec7ad}" - testing.go:149: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.858-1200 [INFO] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: aborting pipeline replication: peer="{Voter server-130cacbb-9be7-2e54-d564-541e8faec7ad 130cacbb-9be7-2e54-d564-541e8faec7ad}" -testLoggerAdapter verbose: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.858-1200 [ERROR] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: failed to appendEntries to: peer="{Voter server-130cacbb-9be7-2e54-d564-541e8faec7ad 130cacbb-9be7-2e54-d564-541e8faec7ad}" error="failed to connect to peer: 130cacbb-9be7-2e54-d564-541e8faec7ad" - testing.go:149: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.858-1200 [ERROR] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: failed to appendEntries to: peer="{Voter server-130cacbb-9be7-2e54-d564-541e8faec7ad 130cacbb-9be7-2e54-d564-541e8faec7ad}" error="failed to connect to peer: 130cacbb-9be7-2e54-d564-541e8faec7ad" - raft_test.go:1320: [INFO] Finished apply without behind follower -testLoggerAdapter verbose: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.861-1200 [INFO] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: starting snapshot up to: index=102 - testing.go:149: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.861-1200 [INFO] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: starting snapshot up to: index=102 - testing.go:153: 2024-06-11T02:29:22.862-1200 [INFO] creating new snapshot: path=/tmp/raft1172882057/snapshots/2-102-1718116162862.tmp -testLoggerAdapter verbose: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.862-1200 [ERROR] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: failed to heartbeat to: peer=130cacbb-9be7-2e54-d564-541e8faec7ad error="failed to connect to peer: 130cacbb-9be7-2e54-d564-541e8faec7ad" - testing.go:149: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.862-1200 [ERROR] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: failed to heartbeat to: peer=130cacbb-9be7-2e54-d564-541e8faec7ad error="failed to connect to peer: 130cacbb-9be7-2e54-d564-541e8faec7ad" -testLoggerAdapter verbose: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.870-1200 [ERROR] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: failed to appendEntries to: peer="{Voter server-130cacbb-9be7-2e54-d564-541e8faec7ad 130cacbb-9be7-2e54-d564-541e8faec7ad}" error="failed to connect to peer: 130cacbb-9be7-2e54-d564-541e8faec7ad" - testing.go:149: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.870-1200 [ERROR] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: failed to appendEntries to: peer="{Voter server-130cacbb-9be7-2e54-d564-541e8faec7ad 130cacbb-9be7-2e54-d564-541e8faec7ad}" error="failed to connect to peer: 130cacbb-9be7-2e54-d564-541e8faec7ad" -testLoggerAdapter verbose: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.880-1200 [ERROR] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: failed to appendEntries to: peer="{Voter server-130cacbb-9be7-2e54-d564-541e8faec7ad 130cacbb-9be7-2e54-d564-541e8faec7ad}" error="failed to connect to peer: 130cacbb-9be7-2e54-d564-541e8faec7ad" - testing.go:149: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.880-1200 [ERROR] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: failed to appendEntries to: peer="{Voter server-130cacbb-9be7-2e54-d564-541e8faec7ad 130cacbb-9be7-2e54-d564-541e8faec7ad}" error="failed to connect to peer: 130cacbb-9be7-2e54-d564-541e8faec7ad" -testLoggerAdapter verbose: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.882-1200 [ERROR] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: failed to heartbeat to: peer=130cacbb-9be7-2e54-d564-541e8faec7ad error="failed to connect to peer: 130cacbb-9be7-2e54-d564-541e8faec7ad" - testing.go:149: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.882-1200 [ERROR] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: failed to heartbeat to: peer=130cacbb-9be7-2e54-d564-541e8faec7ad error="failed to connect to peer: 130cacbb-9be7-2e54-d564-541e8faec7ad" -testLoggerAdapter verbose: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.888-1200 [INFO] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: compacting logs: from=1 to=92 - testing.go:149: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.888-1200 [INFO] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: compacting logs: from=1 to=92 -testLoggerAdapter verbose: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.888-1200 [INFO] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: snapshot complete up to: index=102 - testing.go:149: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.888-1200 [INFO] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: snapshot complete up to: index=102 -testLoggerAdapter verbose: server-130cacbb-9be7-2e54-d564-541e8faec7ad: 2024-06-11T02:29:22.888-1200 [ERROR] server-130cacbb-9be7-2e54-d564-541e8faec7ad: failed to take snapshot: error="nothing new to snapshot" - testing.go:149: server-130cacbb-9be7-2e54-d564-541e8faec7ad: 2024-06-11T02:29:22.888-1200 [ERROR] server-130cacbb-9be7-2e54-d564-541e8faec7ad: failed to take snapshot: error="nothing new to snapshot" -testLoggerAdapter verbose: server-978e9054-d9ef-3e7f-cc56-db2b9416dd05: 2024-06-11T02:29:22.889-1200 [INFO] server-978e9054-d9ef-3e7f-cc56-db2b9416dd05: starting snapshot up to: index=102 - testing.go:149: server-978e9054-d9ef-3e7f-cc56-db2b9416dd05: 2024-06-11T02:29:22.889-1200 [INFO] server-978e9054-d9ef-3e7f-cc56-db2b9416dd05: starting snapshot up to: index=102 - testing.go:153: 2024-06-11T02:29:22.889-1200 [INFO] creating new snapshot: path=/tmp/raft1705576839/snapshots/2-102-1718116162889.tmp -testLoggerAdapter verbose: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.902-1200 [ERROR] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: failed to heartbeat to: peer=130cacbb-9be7-2e54-d564-541e8faec7ad error="failed to connect to peer: 130cacbb-9be7-2e54-d564-541e8faec7ad" - testing.go:149: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.902-1200 [ERROR] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: failed to heartbeat to: peer=130cacbb-9be7-2e54-d564-541e8faec7ad error="failed to connect to peer: 130cacbb-9be7-2e54-d564-541e8faec7ad" -testLoggerAdapter verbose: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.906-1200 [WARN] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: failed to contact: server-id=server-130cacbb-9be7-2e54-d564-541e8faec7ad time=50.664172ms - testing.go:149: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.906-1200 [WARN] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: failed to contact: server-id=server-130cacbb-9be7-2e54-d564-541e8faec7ad time=50.664172ms -testLoggerAdapter verbose: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.908-1200 [ERROR] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: failed to get log: index=2 error="log not found" - testing.go:149: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.908-1200 [ERROR] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: failed to get log: index=2 error="log not found" -testLoggerAdapter verbose: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.909-1200 [ERROR] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: failed to install snapshot: id=2-102-1718116162862 error="failed to connect to peer: 130cacbb-9be7-2e54-d564-541e8faec7ad" - testing.go:149: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.909-1200 [ERROR] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: failed to install snapshot: id=2-102-1718116162862 error="failed to connect to peer: 130cacbb-9be7-2e54-d564-541e8faec7ad" -testLoggerAdapter verbose: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.909-1200 [ERROR] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: failed to send snapshot to: peer="{Voter server-130cacbb-9be7-2e54-d564-541e8faec7ad 130cacbb-9be7-2e54-d564-541e8faec7ad}" error="failed to connect to peer: 130cacbb-9be7-2e54-d564-541e8faec7ad" - testing.go:149: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.909-1200 [ERROR] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: failed to send snapshot to: peer="{Voter server-130cacbb-9be7-2e54-d564-541e8faec7ad 130cacbb-9be7-2e54-d564-541e8faec7ad}" error="failed to connect to peer: 130cacbb-9be7-2e54-d564-541e8faec7ad" -testLoggerAdapter verbose: server-978e9054-d9ef-3e7f-cc56-db2b9416dd05: 2024-06-11T02:29:22.911-1200 [INFO] server-978e9054-d9ef-3e7f-cc56-db2b9416dd05: compacting logs: from=1 to=92 - testing.go:149: server-978e9054-d9ef-3e7f-cc56-db2b9416dd05: 2024-06-11T02:29:22.911-1200 [INFO] server-978e9054-d9ef-3e7f-cc56-db2b9416dd05: compacting logs: from=1 to=92 -testLoggerAdapter verbose: server-978e9054-d9ef-3e7f-cc56-db2b9416dd05: 2024-06-11T02:29:22.911-1200 [INFO] server-978e9054-d9ef-3e7f-cc56-db2b9416dd05: snapshot complete up to: index=102 - testing.go:149: server-978e9054-d9ef-3e7f-cc56-db2b9416dd05: 2024-06-11T02:29:22.911-1200 [INFO] server-978e9054-d9ef-3e7f-cc56-db2b9416dd05: snapshot complete up to: index=102 -testLoggerAdapter verbose: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.957-1200 [ERROR] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: failed to get log: index=2 error="log not found" - testing.go:149: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.957-1200 [ERROR] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: failed to get log: index=2 error="log not found" - testing.go:153: 2024-06-11T02:29:22.958-1200 [INFO] creating new snapshot: path=/tmp/raft2276185088/snapshots/2-102-1718116162958.tmp -testLoggerAdapter verbose: server-130cacbb-9be7-2e54-d564-541e8faec7ad: 2024-06-11T02:29:22.979-1200 [INFO] server-130cacbb-9be7-2e54-d564-541e8faec7ad: copied to local snapshot: bytes=693 - testing.go:149: server-130cacbb-9be7-2e54-d564-541e8faec7ad: 2024-06-11T02:29:22.979-1200 [INFO] server-130cacbb-9be7-2e54-d564-541e8faec7ad: copied to local snapshot: bytes=693 -testLoggerAdapter verbose: server-130cacbb-9be7-2e54-d564-541e8faec7ad: 2024-06-11T02:29:22.979-1200 [INFO] server-130cacbb-9be7-2e54-d564-541e8faec7ad: Installed remote snapshot - testing.go:149: server-130cacbb-9be7-2e54-d564-541e8faec7ad: 2024-06-11T02:29:22.979-1200 [INFO] server-130cacbb-9be7-2e54-d564-541e8faec7ad: Installed remote snapshot -testLoggerAdapter verbose: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.983-1200 [INFO] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: aborting pipeline replication: peer="{Voter server-978e9054-d9ef-3e7f-cc56-db2b9416dd05 978e9054-d9ef-3e7f-cc56-db2b9416dd05}" - testing.go:149: server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: 2024-06-11T02:29:22.983-1200 [INFO] server-cb0ac0bd-3251-e119-192a-edc96d61eb5d: aborting pipeline replication: peer="{Voter server-978e9054-d9ef-3e7f-cc56-db2b9416dd05 978e9054-d9ef-3e7f-cc56-db2b9416dd05}" ---- PASS: TestRaft_SendSnapshotFollower (0.40s) -=== RUN TestRaft_SendSnapshotAndLogsFollower -testLoggerAdapter verbose: server-7b4a130c-b831-b05d-909c-1d73a3120086: 2024-06-11T02:29:22.988-1200 [INFO] server-7b4a130c-b831-b05d-909c-1d73a3120086: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b4a130c-b831-b05d-909c-1d73a3120086 Address:7b4a130c-b831-b05d-909c-1d73a3120086} {Suffrage:Voter ID:server-d40855d8-da48-083f-7e6d-66819e47bbee Address:d40855d8-da48-083f-7e6d-66819e47bbee} {Suffrage:Voter ID:server-62cf5662-f189-f90a-c21a-a03b035d9a1e Address:62cf5662-f189-f90a-c21a-a03b035d9a1e}]" - testing.go:149: server-7b4a130c-b831-b05d-909c-1d73a3120086: 2024-06-11T02:29:22.988-1200 [INFO] server-7b4a130c-b831-b05d-909c-1d73a3120086: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b4a130c-b831-b05d-909c-1d73a3120086 Address:7b4a130c-b831-b05d-909c-1d73a3120086} {Suffrage:Voter ID:server-d40855d8-da48-083f-7e6d-66819e47bbee Address:d40855d8-da48-083f-7e6d-66819e47bbee} {Suffrage:Voter ID:server-62cf5662-f189-f90a-c21a-a03b035d9a1e Address:62cf5662-f189-f90a-c21a-a03b035d9a1e}]" -testLoggerAdapter verbose: server-d40855d8-da48-083f-7e6d-66819e47bbee: 2024-06-11T02:29:22.989-1200 [INFO] server-d40855d8-da48-083f-7e6d-66819e47bbee: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b4a130c-b831-b05d-909c-1d73a3120086 Address:7b4a130c-b831-b05d-909c-1d73a3120086} {Suffrage:Voter ID:server-d40855d8-da48-083f-7e6d-66819e47bbee Address:d40855d8-da48-083f-7e6d-66819e47bbee} {Suffrage:Voter ID:server-62cf5662-f189-f90a-c21a-a03b035d9a1e Address:62cf5662-f189-f90a-c21a-a03b035d9a1e}]" - testing.go:149: server-d40855d8-da48-083f-7e6d-66819e47bbee: 2024-06-11T02:29:22.989-1200 [INFO] server-d40855d8-da48-083f-7e6d-66819e47bbee: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b4a130c-b831-b05d-909c-1d73a3120086 Address:7b4a130c-b831-b05d-909c-1d73a3120086} {Suffrage:Voter ID:server-d40855d8-da48-083f-7e6d-66819e47bbee Address:d40855d8-da48-083f-7e6d-66819e47bbee} {Suffrage:Voter ID:server-62cf5662-f189-f90a-c21a-a03b035d9a1e Address:62cf5662-f189-f90a-c21a-a03b035d9a1e}]" -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:22.989-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b4a130c-b831-b05d-909c-1d73a3120086 Address:7b4a130c-b831-b05d-909c-1d73a3120086} {Suffrage:Voter ID:server-d40855d8-da48-083f-7e6d-66819e47bbee Address:d40855d8-da48-083f-7e6d-66819e47bbee} {Suffrage:Voter ID:server-62cf5662-f189-f90a-c21a-a03b035d9a1e Address:62cf5662-f189-f90a-c21a-a03b035d9a1e}]" - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:22.989-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b4a130c-b831-b05d-909c-1d73a3120086 Address:7b4a130c-b831-b05d-909c-1d73a3120086} {Suffrage:Voter ID:server-d40855d8-da48-083f-7e6d-66819e47bbee Address:d40855d8-da48-083f-7e6d-66819e47bbee} {Suffrage:Voter ID:server-62cf5662-f189-f90a-c21a-a03b035d9a1e Address:62cf5662-f189-f90a-c21a-a03b035d9a1e}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:22.989-1200 [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2024-06-11T02:29:22.989-1200 [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: server-7b4a130c-b831-b05d-909c-1d73a3120086: 2024-06-11T02:29:22.989-1200 [INFO] server-7b4a130c-b831-b05d-909c-1d73a3120086: entering follower state: follower="Node at 7b4a130c-b831-b05d-909c-1d73a3120086 [Follower]" leader= - testing.go:149: server-7b4a130c-b831-b05d-909c-1d73a3120086: 2024-06-11T02:29:22.989-1200 [INFO] server-7b4a130c-b831-b05d-909c-1d73a3120086: entering follower state: follower="Node at 7b4a130c-b831-b05d-909c-1d73a3120086 [Follower]" leader= -testLoggerAdapter verbose: server-d40855d8-da48-083f-7e6d-66819e47bbee: 2024-06-11T02:29:22.989-1200 [INFO] server-d40855d8-da48-083f-7e6d-66819e47bbee: entering follower state: follower="Node at d40855d8-da48-083f-7e6d-66819e47bbee [Follower]" leader= - testing.go:149: server-d40855d8-da48-083f-7e6d-66819e47bbee: 2024-06-11T02:29:22.989-1200 [INFO] server-d40855d8-da48-083f-7e6d-66819e47bbee: entering follower state: follower="Node at d40855d8-da48-083f-7e6d-66819e47bbee [Follower]" leader= -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:22.989-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: entering follower state: follower="Node at 62cf5662-f189-f90a-c21a-a03b035d9a1e [Follower]" leader= - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:22.989-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: entering follower state: follower="Node at 62cf5662-f189-f90a-c21a-a03b035d9a1e [Follower]" leader= -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.044-1200 [WARN] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.044-1200 [WARN] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.044-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: entering candidate state: node="Node at 62cf5662-f189-f90a-c21a-a03b035d9a1e [Candidate]" term=2 - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.044-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: entering candidate state: node="Node at 62cf5662-f189-f90a-c21a-a03b035d9a1e [Candidate]" term=2 -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.045-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: election won: tally=2 - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.045-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: election won: tally=2 -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.045-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: entering leader state: leader="Node at 62cf5662-f189-f90a-c21a-a03b035d9a1e [Leader]" - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.045-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: entering leader state: leader="Node at 62cf5662-f189-f90a-c21a-a03b035d9a1e [Leader]" -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.045-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: added peer, starting replication: peer=server-7b4a130c-b831-b05d-909c-1d73a3120086 - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.045-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: added peer, starting replication: peer=server-7b4a130c-b831-b05d-909c-1d73a3120086 -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.045-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: added peer, starting replication: peer=server-d40855d8-da48-083f-7e6d-66819e47bbee - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.045-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: added peer, starting replication: peer=server-d40855d8-da48-083f-7e6d-66819e47bbee -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.045-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: pipelining replication: peer="{Voter server-d40855d8-da48-083f-7e6d-66819e47bbee d40855d8-da48-083f-7e6d-66819e47bbee}" - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.045-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: pipelining replication: peer="{Voter server-d40855d8-da48-083f-7e6d-66819e47bbee d40855d8-da48-083f-7e6d-66819e47bbee}" -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.046-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: pipelining replication: peer="{Voter server-7b4a130c-b831-b05d-909c-1d73a3120086 7b4a130c-b831-b05d-909c-1d73a3120086}" - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.046-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: pipelining replication: peer="{Voter server-7b4a130c-b831-b05d-909c-1d73a3120086 7b4a130c-b831-b05d-909c-1d73a3120086}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:23.152-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:23.046335306 -1200 -12 m=+14.041707847 (2 nodes), 56.65985ms from start of poll, 57.90991ms from cluster start. Timeout at 2024-06-11 02:29:23.152387977 -1200 -12 m=+14.147760811, 106.052964ms after stability - testing.go:149: cluster: 2024-06-11T02:29:23.152-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:23.046335306 -1200 -12 m=+14.041707847 (2 nodes), 56.65985ms from start of poll, 57.90991ms from cluster start. Timeout at 2024-06-11 02:29:23.152387977 -1200 -12 m=+14.147760811, 106.052964ms after stability -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:23.152-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:23.152-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:23.258-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:23.152773553 -1200 -12 m=+14.148146095 (1 nodes), 4.666µs from start of poll, 164.348158ms from cluster start. Timeout at 2024-06-11 02:29:23.258746602 -1200 -12 m=+14.254119435, 105.97334ms after stability - testing.go:149: cluster: 2024-06-11T02:29:23.258-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:23.152773553 -1200 -12 m=+14.148146095 (1 nodes), 4.666µs from start of poll, 164.348158ms from cluster start. Timeout at 2024-06-11 02:29:23.258746602 -1200 -12 m=+14.254119435, 105.97334ms after stability -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.259-1200 [ERROR] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: failed to pipeline appendEntries: peer="{Voter server-7b4a130c-b831-b05d-909c-1d73a3120086 7b4a130c-b831-b05d-909c-1d73a3120086}" error="append pipeline closed" - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.259-1200 [ERROR] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: failed to pipeline appendEntries: peer="{Voter server-7b4a130c-b831-b05d-909c-1d73a3120086 7b4a130c-b831-b05d-909c-1d73a3120086}" error="append pipeline closed" -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.259-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: aborting pipeline replication: peer="{Voter server-7b4a130c-b831-b05d-909c-1d73a3120086 7b4a130c-b831-b05d-909c-1d73a3120086}" - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.259-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: aborting pipeline replication: peer="{Voter server-7b4a130c-b831-b05d-909c-1d73a3120086 7b4a130c-b831-b05d-909c-1d73a3120086}" -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.259-1200 [ERROR] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: failed to appendEntries to: peer="{Voter server-7b4a130c-b831-b05d-909c-1d73a3120086 7b4a130c-b831-b05d-909c-1d73a3120086}" error="failed to connect to peer: 7b4a130c-b831-b05d-909c-1d73a3120086" - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.259-1200 [ERROR] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: failed to appendEntries to: peer="{Voter server-7b4a130c-b831-b05d-909c-1d73a3120086 7b4a130c-b831-b05d-909c-1d73a3120086}" error="failed to connect to peer: 7b4a130c-b831-b05d-909c-1d73a3120086" -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.263-1200 [ERROR] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: failed to heartbeat to: peer=7b4a130c-b831-b05d-909c-1d73a3120086 error="failed to connect to peer: 7b4a130c-b831-b05d-909c-1d73a3120086" - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.263-1200 [ERROR] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: failed to heartbeat to: peer=7b4a130c-b831-b05d-909c-1d73a3120086 error="failed to connect to peer: 7b4a130c-b831-b05d-909c-1d73a3120086" - raft_test.go:1362: [INFO] Finished apply without behind follower -testLoggerAdapter verbose: server-7b4a130c-b831-b05d-909c-1d73a3120086: 2024-06-11T02:29:23.268-1200 [ERROR] server-7b4a130c-b831-b05d-909c-1d73a3120086: failed to take snapshot: error="nothing new to snapshot" - testing.go:149: server-7b4a130c-b831-b05d-909c-1d73a3120086: 2024-06-11T02:29:23.268-1200 [ERROR] server-7b4a130c-b831-b05d-909c-1d73a3120086: failed to take snapshot: error="nothing new to snapshot" -testLoggerAdapter verbose: server-d40855d8-da48-083f-7e6d-66819e47bbee: 2024-06-11T02:29:23.268-1200 [INFO] server-d40855d8-da48-083f-7e6d-66819e47bbee: starting snapshot up to: index=70 - testing.go:149: server-d40855d8-da48-083f-7e6d-66819e47bbee: 2024-06-11T02:29:23.268-1200 [INFO] server-d40855d8-da48-083f-7e6d-66819e47bbee: starting snapshot up to: index=70 - testing.go:153: 2024-06-11T02:29:23.268-1200 [INFO] creating new snapshot: path=/tmp/raft253017687/snapshots/2-70-1718116163268.tmp -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.270-1200 [ERROR] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: failed to appendEntries to: peer="{Voter server-7b4a130c-b831-b05d-909c-1d73a3120086 7b4a130c-b831-b05d-909c-1d73a3120086}" error="failed to connect to peer: 7b4a130c-b831-b05d-909c-1d73a3120086" - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.270-1200 [ERROR] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: failed to appendEntries to: peer="{Voter server-7b4a130c-b831-b05d-909c-1d73a3120086 7b4a130c-b831-b05d-909c-1d73a3120086}" error="failed to connect to peer: 7b4a130c-b831-b05d-909c-1d73a3120086" -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.281-1200 [ERROR] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: failed to heartbeat to: peer=7b4a130c-b831-b05d-909c-1d73a3120086 error="failed to connect to peer: 7b4a130c-b831-b05d-909c-1d73a3120086" - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.281-1200 [ERROR] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: failed to heartbeat to: peer=7b4a130c-b831-b05d-909c-1d73a3120086 error="failed to connect to peer: 7b4a130c-b831-b05d-909c-1d73a3120086" -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.281-1200 [ERROR] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: failed to appendEntries to: peer="{Voter server-7b4a130c-b831-b05d-909c-1d73a3120086 7b4a130c-b831-b05d-909c-1d73a3120086}" error="failed to connect to peer: 7b4a130c-b831-b05d-909c-1d73a3120086" - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.281-1200 [ERROR] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: failed to appendEntries to: peer="{Voter server-7b4a130c-b831-b05d-909c-1d73a3120086 7b4a130c-b831-b05d-909c-1d73a3120086}" error="failed to connect to peer: 7b4a130c-b831-b05d-909c-1d73a3120086" -testLoggerAdapter verbose: server-d40855d8-da48-083f-7e6d-66819e47bbee: 2024-06-11T02:29:23.292-1200 [INFO] server-d40855d8-da48-083f-7e6d-66819e47bbee: compacting logs: from=1 to=70 - testing.go:149: server-d40855d8-da48-083f-7e6d-66819e47bbee: 2024-06-11T02:29:23.292-1200 [INFO] server-d40855d8-da48-083f-7e6d-66819e47bbee: compacting logs: from=1 to=70 -testLoggerAdapter verbose: server-d40855d8-da48-083f-7e6d-66819e47bbee: 2024-06-11T02:29:23.292-1200 [INFO] server-d40855d8-da48-083f-7e6d-66819e47bbee: snapshot complete up to: index=70 - testing.go:149: server-d40855d8-da48-083f-7e6d-66819e47bbee: 2024-06-11T02:29:23.292-1200 [INFO] server-d40855d8-da48-083f-7e6d-66819e47bbee: snapshot complete up to: index=70 -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.292-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: starting snapshot up to: index=102 - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.292-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: starting snapshot up to: index=102 - testing.go:153: 2024-06-11T02:29:23.292-1200 [INFO] creating new snapshot: path=/tmp/raft4082943477/snapshots/2-102-1718116163292.tmp -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.301-1200 [ERROR] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: failed to heartbeat to: peer=7b4a130c-b831-b05d-909c-1d73a3120086 error="failed to connect to peer: 7b4a130c-b831-b05d-909c-1d73a3120086" - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.301-1200 [ERROR] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: failed to heartbeat to: peer=7b4a130c-b831-b05d-909c-1d73a3120086 error="failed to connect to peer: 7b4a130c-b831-b05d-909c-1d73a3120086" -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.306-1200 [WARN] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: failed to contact: server-id=server-7b4a130c-b831-b05d-909c-1d73a3120086 time=50.287638ms - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.306-1200 [WARN] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: failed to contact: server-id=server-7b4a130c-b831-b05d-909c-1d73a3120086 time=50.287638ms -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.310-1200 [ERROR] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: failed to appendEntries to: peer="{Voter server-7b4a130c-b831-b05d-909c-1d73a3120086 7b4a130c-b831-b05d-909c-1d73a3120086}" error="failed to connect to peer: 7b4a130c-b831-b05d-909c-1d73a3120086" - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.310-1200 [ERROR] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: failed to appendEntries to: peer="{Voter server-7b4a130c-b831-b05d-909c-1d73a3120086 7b4a130c-b831-b05d-909c-1d73a3120086}" error="failed to connect to peer: 7b4a130c-b831-b05d-909c-1d73a3120086" -testLoggerAdapter verbose: server-7b4a130c-b831-b05d-909c-1d73a3120086: 2024-06-11T02:29:23.310-1200 [WARN] server-7b4a130c-b831-b05d-909c-1d73a3120086: heartbeat timeout reached, starting election: last-leader=62cf5662-f189-f90a-c21a-a03b035d9a1e - testing.go:149: server-7b4a130c-b831-b05d-909c-1d73a3120086: 2024-06-11T02:29:23.310-1200 [WARN] server-7b4a130c-b831-b05d-909c-1d73a3120086: heartbeat timeout reached, starting election: last-leader=62cf5662-f189-f90a-c21a-a03b035d9a1e -testLoggerAdapter verbose: server-7b4a130c-b831-b05d-909c-1d73a3120086: 2024-06-11T02:29:23.311-1200 [INFO] server-7b4a130c-b831-b05d-909c-1d73a3120086: entering candidate state: node="Node at 7b4a130c-b831-b05d-909c-1d73a3120086 [Candidate]" term=3 - testing.go:149: server-7b4a130c-b831-b05d-909c-1d73a3120086: 2024-06-11T02:29:23.311-1200 [INFO] server-7b4a130c-b831-b05d-909c-1d73a3120086: entering candidate state: node="Node at 7b4a130c-b831-b05d-909c-1d73a3120086 [Candidate]" term=3 -testLoggerAdapter verbose: server-7b4a130c-b831-b05d-909c-1d73a3120086: 2024-06-11T02:29:23.311-1200 [ERROR] server-7b4a130c-b831-b05d-909c-1d73a3120086: failed to make requestVote RPC: target="{Voter server-62cf5662-f189-f90a-c21a-a03b035d9a1e 62cf5662-f189-f90a-c21a-a03b035d9a1e}" error="failed to connect to peer: 62cf5662-f189-f90a-c21a-a03b035d9a1e" - testing.go:149: server-7b4a130c-b831-b05d-909c-1d73a3120086: 2024-06-11T02:29:23.311-1200 [ERROR] server-7b4a130c-b831-b05d-909c-1d73a3120086: failed to make requestVote RPC: target="{Voter server-62cf5662-f189-f90a-c21a-a03b035d9a1e 62cf5662-f189-f90a-c21a-a03b035d9a1e}" error="failed to connect to peer: 62cf5662-f189-f90a-c21a-a03b035d9a1e" -testLoggerAdapter verbose: server-7b4a130c-b831-b05d-909c-1d73a3120086: 2024-06-11T02:29:23.311-1200 [ERROR] server-7b4a130c-b831-b05d-909c-1d73a3120086: failed to make requestVote RPC: target="{Voter server-d40855d8-da48-083f-7e6d-66819e47bbee d40855d8-da48-083f-7e6d-66819e47bbee}" error="failed to connect to peer: d40855d8-da48-083f-7e6d-66819e47bbee" - testing.go:149: server-7b4a130c-b831-b05d-909c-1d73a3120086: 2024-06-11T02:29:23.311-1200 [ERROR] server-7b4a130c-b831-b05d-909c-1d73a3120086: failed to make requestVote RPC: target="{Voter server-d40855d8-da48-083f-7e6d-66819e47bbee d40855d8-da48-083f-7e6d-66819e47bbee}" error="failed to connect to peer: d40855d8-da48-083f-7e6d-66819e47bbee" -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.317-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: compacting logs: from=1 to=92 - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.317-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: compacting logs: from=1 to=92 -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.317-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: snapshot complete up to: index=102 - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.317-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: snapshot complete up to: index=102 - raft_test.go:1383: [INFO] Finished apply without behind follower -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.356-1200 [ERROR] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: failed to get log: index=2 error="log not found" - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.356-1200 [ERROR] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: failed to get log: index=2 error="log not found" -testLoggerAdapter verbose: server-7b4a130c-b831-b05d-909c-1d73a3120086: 2024-06-11T02:29:23.357-1200 [INFO] server-7b4a130c-b831-b05d-909c-1d73a3120086: ignoring installSnapshot request with older term than current term: request-term=2 current-term=3 - testing.go:149: server-7b4a130c-b831-b05d-909c-1d73a3120086: 2024-06-11T02:29:23.357-1200 [INFO] server-7b4a130c-b831-b05d-909c-1d73a3120086: ignoring installSnapshot request with older term than current term: request-term=2 current-term=3 -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.357-1200 [ERROR] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: peer has newer term, stopping replication: peer="{Voter server-7b4a130c-b831-b05d-909c-1d73a3120086 7b4a130c-b831-b05d-909c-1d73a3120086}" - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.357-1200 [ERROR] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: peer has newer term, stopping replication: peer="{Voter server-7b4a130c-b831-b05d-909c-1d73a3120086 7b4a130c-b831-b05d-909c-1d73a3120086}" -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.358-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: entering follower state: follower="Node at 62cf5662-f189-f90a-c21a-a03b035d9a1e [Follower]" leader= - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.358-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: entering follower state: follower="Node at 62cf5662-f189-f90a-c21a-a03b035d9a1e [Follower]" leader= -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.358-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: aborting pipeline replication: peer="{Voter server-d40855d8-da48-083f-7e6d-66819e47bbee d40855d8-da48-083f-7e6d-66819e47bbee}" - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.358-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: aborting pipeline replication: peer="{Voter server-d40855d8-da48-083f-7e6d-66819e47bbee d40855d8-da48-083f-7e6d-66819e47bbee}" -testLoggerAdapter verbose: server-7b4a130c-b831-b05d-909c-1d73a3120086: 2024-06-11T02:29:23.405-1200 [WARN] server-7b4a130c-b831-b05d-909c-1d73a3120086: Election timeout reached, restarting election - testing.go:149: server-7b4a130c-b831-b05d-909c-1d73a3120086: 2024-06-11T02:29:23.405-1200 [WARN] server-7b4a130c-b831-b05d-909c-1d73a3120086: Election timeout reached, restarting election -testLoggerAdapter verbose: server-7b4a130c-b831-b05d-909c-1d73a3120086: 2024-06-11T02:29:23.405-1200 [INFO] server-7b4a130c-b831-b05d-909c-1d73a3120086: entering candidate state: node="Node at 7b4a130c-b831-b05d-909c-1d73a3120086 [Candidate]" term=4 - testing.go:149: server-7b4a130c-b831-b05d-909c-1d73a3120086: 2024-06-11T02:29:23.405-1200 [INFO] server-7b4a130c-b831-b05d-909c-1d73a3120086: entering candidate state: node="Node at 7b4a130c-b831-b05d-909c-1d73a3120086 [Candidate]" term=4 -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.405-1200 [WARN] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: rejecting vote request since our last index is greater: candidate=7b4a130c-b831-b05d-909c-1d73a3120086 last-index=202 last-candidate-index=2 - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.405-1200 [WARN] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: rejecting vote request since our last index is greater: candidate=7b4a130c-b831-b05d-909c-1d73a3120086 last-index=202 last-candidate-index=2 -testLoggerAdapter verbose: server-d40855d8-da48-083f-7e6d-66819e47bbee: 2024-06-11T02:29:23.405-1200 [WARN] server-d40855d8-da48-083f-7e6d-66819e47bbee: rejecting vote request since we have a leader: from=7b4a130c-b831-b05d-909c-1d73a3120086 leader=62cf5662-f189-f90a-c21a-a03b035d9a1e - testing.go:149: server-d40855d8-da48-083f-7e6d-66819e47bbee: 2024-06-11T02:29:23.405-1200 [WARN] server-d40855d8-da48-083f-7e6d-66819e47bbee: rejecting vote request since we have a leader: from=7b4a130c-b831-b05d-909c-1d73a3120086 leader=62cf5662-f189-f90a-c21a-a03b035d9a1e -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.444-1200 [WARN] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.444-1200 [WARN] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.444-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: entering candidate state: node="Node at 62cf5662-f189-f90a-c21a-a03b035d9a1e [Candidate]" term=5 - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.444-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: entering candidate state: node="Node at 62cf5662-f189-f90a-c21a-a03b035d9a1e [Candidate]" term=5 -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.444-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: election won: tally=2 - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.444-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: election won: tally=2 -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.444-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: entering leader state: leader="Node at 62cf5662-f189-f90a-c21a-a03b035d9a1e [Leader]" - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.444-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: entering leader state: leader="Node at 62cf5662-f189-f90a-c21a-a03b035d9a1e [Leader]" -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.444-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: added peer, starting replication: peer=server-7b4a130c-b831-b05d-909c-1d73a3120086 - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.444-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: added peer, starting replication: peer=server-7b4a130c-b831-b05d-909c-1d73a3120086 -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.445-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: added peer, starting replication: peer=server-d40855d8-da48-083f-7e6d-66819e47bbee - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.445-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: added peer, starting replication: peer=server-d40855d8-da48-083f-7e6d-66819e47bbee -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.445-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: pipelining replication: peer="{Voter server-d40855d8-da48-083f-7e6d-66819e47bbee d40855d8-da48-083f-7e6d-66819e47bbee}" - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.445-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: pipelining replication: peer="{Voter server-d40855d8-da48-083f-7e6d-66819e47bbee d40855d8-da48-083f-7e6d-66819e47bbee}" -testLoggerAdapter verbose: server-7b4a130c-b831-b05d-909c-1d73a3120086: 2024-06-11T02:29:23.445-1200 [INFO] server-7b4a130c-b831-b05d-909c-1d73a3120086: entering follower state: follower="Node at 7b4a130c-b831-b05d-909c-1d73a3120086 [Follower]" leader= - testing.go:149: server-7b4a130c-b831-b05d-909c-1d73a3120086: 2024-06-11T02:29:23.445-1200 [INFO] server-7b4a130c-b831-b05d-909c-1d73a3120086: entering follower state: follower="Node at 7b4a130c-b831-b05d-909c-1d73a3120086 [Follower]" leader= -testLoggerAdapter verbose: server-7b4a130c-b831-b05d-909c-1d73a3120086: 2024-06-11T02:29:23.445-1200 [WARN] server-7b4a130c-b831-b05d-909c-1d73a3120086: failed to get previous log: previous-index=202 last-index=2 error="log not found" - testing.go:149: server-7b4a130c-b831-b05d-909c-1d73a3120086: 2024-06-11T02:29:23.445-1200 [WARN] server-7b4a130c-b831-b05d-909c-1d73a3120086: failed to get previous log: previous-index=202 last-index=2 error="log not found" -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.445-1200 [WARN] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: appendEntries rejected, sending older logs: peer="{Voter server-7b4a130c-b831-b05d-909c-1d73a3120086 7b4a130c-b831-b05d-909c-1d73a3120086}" next=3 - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.445-1200 [WARN] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: appendEntries rejected, sending older logs: peer="{Voter server-7b4a130c-b831-b05d-909c-1d73a3120086 7b4a130c-b831-b05d-909c-1d73a3120086}" next=3 -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.445-1200 [ERROR] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: failed to get log: index=2 error="log not found" - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.445-1200 [ERROR] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: failed to get log: index=2 error="log not found" - testing.go:153: 2024-06-11T02:29:23.446-1200 [INFO] creating new snapshot: path=/tmp/raft268053024/snapshots/2-102-1718116163446.tmp -testLoggerAdapter verbose: server-7b4a130c-b831-b05d-909c-1d73a3120086: 2024-06-11T02:29:23.471-1200 [INFO] server-7b4a130c-b831-b05d-909c-1d73a3120086: copied to local snapshot: bytes=693 - testing.go:149: server-7b4a130c-b831-b05d-909c-1d73a3120086: 2024-06-11T02:29:23.471-1200 [INFO] server-7b4a130c-b831-b05d-909c-1d73a3120086: copied to local snapshot: bytes=693 -testLoggerAdapter verbose: server-7b4a130c-b831-b05d-909c-1d73a3120086: 2024-06-11T02:29:23.472-1200 [INFO] server-7b4a130c-b831-b05d-909c-1d73a3120086: Installed remote snapshot - testing.go:149: server-7b4a130c-b831-b05d-909c-1d73a3120086: 2024-06-11T02:29:23.472-1200 [INFO] server-7b4a130c-b831-b05d-909c-1d73a3120086: Installed remote snapshot -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.473-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: pipelining replication: peer="{Voter server-7b4a130c-b831-b05d-909c-1d73a3120086 7b4a130c-b831-b05d-909c-1d73a3120086}" - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.473-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: pipelining replication: peer="{Voter server-7b4a130c-b831-b05d-909c-1d73a3120086 7b4a130c-b831-b05d-909c-1d73a3120086}" -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.476-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: aborting pipeline replication: peer="{Voter server-d40855d8-da48-083f-7e6d-66819e47bbee d40855d8-da48-083f-7e6d-66819e47bbee}" - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.476-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: aborting pipeline replication: peer="{Voter server-d40855d8-da48-083f-7e6d-66819e47bbee d40855d8-da48-083f-7e6d-66819e47bbee}" -testLoggerAdapter verbose: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.476-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: aborting pipeline replication: peer="{Voter server-7b4a130c-b831-b05d-909c-1d73a3120086 7b4a130c-b831-b05d-909c-1d73a3120086}" - testing.go:149: server-62cf5662-f189-f90a-c21a-a03b035d9a1e: 2024-06-11T02:29:23.476-1200 [INFO] server-62cf5662-f189-f90a-c21a-a03b035d9a1e: aborting pipeline replication: peer="{Voter server-7b4a130c-b831-b05d-909c-1d73a3120086 7b4a130c-b831-b05d-909c-1d73a3120086}" ---- PASS: TestRaft_SendSnapshotAndLogsFollower (0.49s) -=== RUN TestRaft_ReJoinFollower -testLoggerAdapter verbose: server-20f0f0ab-e470-101d-9908-346e4378b6f6: 2024-06-11T02:29:23.483-1200 [INFO] server-20f0f0ab-e470-101d-9908-346e4378b6f6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-20f0f0ab-e470-101d-9908-346e4378b6f6 Address:20f0f0ab-e470-101d-9908-346e4378b6f6} {Suffrage:Voter ID:server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62 Address:252f1fc1-cdcb-ed13-0b66-9543b2f01f62} {Suffrage:Voter ID:server-06130b71-dcf9-fede-550d-9f2c6588bf5f Address:06130b71-dcf9-fede-550d-9f2c6588bf5f}]" - testing.go:149: server-20f0f0ab-e470-101d-9908-346e4378b6f6: 2024-06-11T02:29:23.483-1200 [INFO] server-20f0f0ab-e470-101d-9908-346e4378b6f6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-20f0f0ab-e470-101d-9908-346e4378b6f6 Address:20f0f0ab-e470-101d-9908-346e4378b6f6} {Suffrage:Voter ID:server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62 Address:252f1fc1-cdcb-ed13-0b66-9543b2f01f62} {Suffrage:Voter ID:server-06130b71-dcf9-fede-550d-9f2c6588bf5f Address:06130b71-dcf9-fede-550d-9f2c6588bf5f}]" -testLoggerAdapter verbose: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.484-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-20f0f0ab-e470-101d-9908-346e4378b6f6 Address:20f0f0ab-e470-101d-9908-346e4378b6f6} {Suffrage:Voter ID:server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62 Address:252f1fc1-cdcb-ed13-0b66-9543b2f01f62} {Suffrage:Voter ID:server-06130b71-dcf9-fede-550d-9f2c6588bf5f Address:06130b71-dcf9-fede-550d-9f2c6588bf5f}]" - testing.go:149: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.484-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-20f0f0ab-e470-101d-9908-346e4378b6f6 Address:20f0f0ab-e470-101d-9908-346e4378b6f6} {Suffrage:Voter ID:server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62 Address:252f1fc1-cdcb-ed13-0b66-9543b2f01f62} {Suffrage:Voter ID:server-06130b71-dcf9-fede-550d-9f2c6588bf5f Address:06130b71-dcf9-fede-550d-9f2c6588bf5f}]" -testLoggerAdapter verbose: server-06130b71-dcf9-fede-550d-9f2c6588bf5f: 2024-06-11T02:29:23.485-1200 [INFO] server-06130b71-dcf9-fede-550d-9f2c6588bf5f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-20f0f0ab-e470-101d-9908-346e4378b6f6 Address:20f0f0ab-e470-101d-9908-346e4378b6f6} {Suffrage:Voter ID:server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62 Address:252f1fc1-cdcb-ed13-0b66-9543b2f01f62} {Suffrage:Voter ID:server-06130b71-dcf9-fede-550d-9f2c6588bf5f Address:06130b71-dcf9-fede-550d-9f2c6588bf5f}]" - testing.go:149: server-06130b71-dcf9-fede-550d-9f2c6588bf5f: 2024-06-11T02:29:23.485-1200 [INFO] server-06130b71-dcf9-fede-550d-9f2c6588bf5f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-20f0f0ab-e470-101d-9908-346e4378b6f6 Address:20f0f0ab-e470-101d-9908-346e4378b6f6} {Suffrage:Voter ID:server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62 Address:252f1fc1-cdcb-ed13-0b66-9543b2f01f62} {Suffrage:Voter ID:server-06130b71-dcf9-fede-550d-9f2c6588bf5f Address:06130b71-dcf9-fede-550d-9f2c6588bf5f}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:23.485-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:23.485-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-20f0f0ab-e470-101d-9908-346e4378b6f6: 2024-06-11T02:29:23.485-1200 [INFO] server-20f0f0ab-e470-101d-9908-346e4378b6f6: entering follower state: follower="Node at 20f0f0ab-e470-101d-9908-346e4378b6f6 [Follower]" leader= - testing.go:149: server-20f0f0ab-e470-101d-9908-346e4378b6f6: 2024-06-11T02:29:23.485-1200 [INFO] server-20f0f0ab-e470-101d-9908-346e4378b6f6: entering follower state: follower="Node at 20f0f0ab-e470-101d-9908-346e4378b6f6 [Follower]" leader= -testLoggerAdapter verbose: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.485-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: entering follower state: follower="Node at 252f1fc1-cdcb-ed13-0b66-9543b2f01f62 [Follower]" leader= - testing.go:149: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.485-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: entering follower state: follower="Node at 252f1fc1-cdcb-ed13-0b66-9543b2f01f62 [Follower]" leader= -testLoggerAdapter verbose: server-06130b71-dcf9-fede-550d-9f2c6588bf5f: 2024-06-11T02:29:23.485-1200 [INFO] server-06130b71-dcf9-fede-550d-9f2c6588bf5f: entering follower state: follower="Node at 06130b71-dcf9-fede-550d-9f2c6588bf5f [Follower]" leader= - testing.go:149: server-06130b71-dcf9-fede-550d-9f2c6588bf5f: 2024-06-11T02:29:23.485-1200 [INFO] server-06130b71-dcf9-fede-550d-9f2c6588bf5f: entering follower state: follower="Node at 06130b71-dcf9-fede-550d-9f2c6588bf5f [Follower]" leader= -testLoggerAdapter verbose: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.549-1200 [WARN] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.549-1200 [WARN] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.549-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: entering candidate state: node="Node at 252f1fc1-cdcb-ed13-0b66-9543b2f01f62 [Candidate]" term=2 - testing.go:149: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.549-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: entering candidate state: node="Node at 252f1fc1-cdcb-ed13-0b66-9543b2f01f62 [Candidate]" term=2 -testLoggerAdapter verbose: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.549-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: election won: tally=2 - testing.go:149: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.549-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: election won: tally=2 -testLoggerAdapter verbose: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.549-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: entering leader state: leader="Node at 252f1fc1-cdcb-ed13-0b66-9543b2f01f62 [Leader]" - testing.go:149: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.549-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: entering leader state: leader="Node at 252f1fc1-cdcb-ed13-0b66-9543b2f01f62 [Leader]" -testLoggerAdapter verbose: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.549-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: added peer, starting replication: peer=server-20f0f0ab-e470-101d-9908-346e4378b6f6 - testing.go:149: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.549-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: added peer, starting replication: peer=server-20f0f0ab-e470-101d-9908-346e4378b6f6 -testLoggerAdapter verbose: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.549-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: added peer, starting replication: peer=server-06130b71-dcf9-fede-550d-9f2c6588bf5f - testing.go:149: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.549-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: added peer, starting replication: peer=server-06130b71-dcf9-fede-550d-9f2c6588bf5f -testLoggerAdapter verbose: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.549-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: pipelining replication: peer="{Voter server-06130b71-dcf9-fede-550d-9f2c6588bf5f 06130b71-dcf9-fede-550d-9f2c6588bf5f}" - testing.go:149: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.549-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: pipelining replication: peer="{Voter server-06130b71-dcf9-fede-550d-9f2c6588bf5f 06130b71-dcf9-fede-550d-9f2c6588bf5f}" -testLoggerAdapter verbose: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.550-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: pipelining replication: peer="{Voter server-20f0f0ab-e470-101d-9908-346e4378b6f6 20f0f0ab-e470-101d-9908-346e4378b6f6}" - testing.go:149: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.550-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: pipelining replication: peer="{Voter server-20f0f0ab-e470-101d-9908-346e4378b6f6 20f0f0ab-e470-101d-9908-346e4378b6f6}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:23.655-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:23.549981373 -1200 -12 m=+14.545353915 (1 nodes), 64.608493ms from start of poll, 66.368377ms from cluster start. Timeout at 2024-06-11 02:29:23.655367891 -1200 -12 m=+14.650740432, 105.386517ms after stability - testing.go:149: cluster: 2024-06-11T02:29:23.655-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:23.549981373 -1200 -12 m=+14.545353915 (1 nodes), 64.608493ms from start of poll, 66.368377ms from cluster start. Timeout at 2024-06-11 02:29:23.655367891 -1200 -12 m=+14.650740432, 105.386517ms after stability -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:23.661-1200 [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2024-06-11T02:29:23.661-1200 [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:23.766-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:23.661506192 -1200 -12 m=+14.656879025 (2 nodes), 4.375µs from start of poll, 177.893487ms from cluster start. Timeout at 2024-06-11 02:29:23.766555258 -1200 -12 m=+14.761928091, 105.049066ms after stability - testing.go:149: cluster: 2024-06-11T02:29:23.766-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:23.661506192 -1200 -12 m=+14.656879025 (2 nodes), 4.375µs from start of poll, 177.893487ms from cluster start. Timeout at 2024-06-11 02:29:23.766555258 -1200 -12 m=+14.761928091, 105.049066ms after stability -testLoggerAdapter verbose: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.766-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: updating configuration: command=RemoveServer server-id=server-20f0f0ab-e470-101d-9908-346e4378b6f6 server-addr= servers="[{Suffrage:Voter ID:server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62 Address:252f1fc1-cdcb-ed13-0b66-9543b2f01f62} {Suffrage:Voter ID:server-06130b71-dcf9-fede-550d-9f2c6588bf5f Address:06130b71-dcf9-fede-550d-9f2c6588bf5f}]" - testing.go:149: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.766-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: updating configuration: command=RemoveServer server-id=server-20f0f0ab-e470-101d-9908-346e4378b6f6 server-addr= servers="[{Suffrage:Voter ID:server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62 Address:252f1fc1-cdcb-ed13-0b66-9543b2f01f62} {Suffrage:Voter ID:server-06130b71-dcf9-fede-550d-9f2c6588bf5f Address:06130b71-dcf9-fede-550d-9f2c6588bf5f}]" -testLoggerAdapter verbose: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.767-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: removed peer, stopping replication: peer=server-20f0f0ab-e470-101d-9908-346e4378b6f6 last-index=3 - testing.go:149: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.767-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: removed peer, stopping replication: peer=server-20f0f0ab-e470-101d-9908-346e4378b6f6 last-index=3 -testLoggerAdapter verbose: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.767-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: aborting pipeline replication: peer="{Voter server-20f0f0ab-e470-101d-9908-346e4378b6f6 20f0f0ab-e470-101d-9908-346e4378b6f6}" - testing.go:149: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.767-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: aborting pipeline replication: peer="{Voter server-20f0f0ab-e470-101d-9908-346e4378b6f6 20f0f0ab-e470-101d-9908-346e4378b6f6}" -testLoggerAdapter verbose: server-20f0f0ab-e470-101d-9908-346e4378b6f6: 2024-06-11T02:29:23.830-1200 [WARN] server-20f0f0ab-e470-101d-9908-346e4378b6f6: heartbeat timeout reached, starting election: last-leader=252f1fc1-cdcb-ed13-0b66-9543b2f01f62 - testing.go:149: server-20f0f0ab-e470-101d-9908-346e4378b6f6: 2024-06-11T02:29:23.830-1200 [WARN] server-20f0f0ab-e470-101d-9908-346e4378b6f6: heartbeat timeout reached, starting election: last-leader=252f1fc1-cdcb-ed13-0b66-9543b2f01f62 -testLoggerAdapter verbose: server-20f0f0ab-e470-101d-9908-346e4378b6f6: 2024-06-11T02:29:23.830-1200 [INFO] server-20f0f0ab-e470-101d-9908-346e4378b6f6: entering candidate state: node="Node at 20f0f0ab-e470-101d-9908-346e4378b6f6 [Candidate]" term=3 - testing.go:149: server-20f0f0ab-e470-101d-9908-346e4378b6f6: 2024-06-11T02:29:23.830-1200 [INFO] server-20f0f0ab-e470-101d-9908-346e4378b6f6: entering candidate state: node="Node at 20f0f0ab-e470-101d-9908-346e4378b6f6 [Candidate]" term=3 -testLoggerAdapter verbose: server-06130b71-dcf9-fede-550d-9f2c6588bf5f: 2024-06-11T02:29:23.830-1200 [WARN] server-06130b71-dcf9-fede-550d-9f2c6588bf5f: rejecting vote request since we have a leader: from=20f0f0ab-e470-101d-9908-346e4378b6f6 leader=252f1fc1-cdcb-ed13-0b66-9543b2f01f62 - testing.go:149: server-06130b71-dcf9-fede-550d-9f2c6588bf5f: 2024-06-11T02:29:23.830-1200 [WARN] server-06130b71-dcf9-fede-550d-9f2c6588bf5f: rejecting vote request since we have a leader: from=20f0f0ab-e470-101d-9908-346e4378b6f6 leader=252f1fc1-cdcb-ed13-0b66-9543b2f01f62 -testLoggerAdapter verbose: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.830-1200 [WARN] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: rejecting vote request since we have a leader: from=20f0f0ab-e470-101d-9908-346e4378b6f6 leader=252f1fc1-cdcb-ed13-0b66-9543b2f01f62 - testing.go:149: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.830-1200 [WARN] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: rejecting vote request since we have a leader: from=20f0f0ab-e470-101d-9908-346e4378b6f6 leader=252f1fc1-cdcb-ed13-0b66-9543b2f01f62 -testLoggerAdapter verbose: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.873-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: updating configuration: command=AddStaging server-id=server-20f0f0ab-e470-101d-9908-346e4378b6f6 server-addr=20f0f0ab-e470-101d-9908-346e4378b6f6 servers="[{Suffrage:Voter ID:server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62 Address:252f1fc1-cdcb-ed13-0b66-9543b2f01f62} {Suffrage:Voter ID:server-06130b71-dcf9-fede-550d-9f2c6588bf5f Address:06130b71-dcf9-fede-550d-9f2c6588bf5f} {Suffrage:Voter ID:server-20f0f0ab-e470-101d-9908-346e4378b6f6 Address:20f0f0ab-e470-101d-9908-346e4378b6f6}]" - testing.go:149: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.873-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: updating configuration: command=AddStaging server-id=server-20f0f0ab-e470-101d-9908-346e4378b6f6 server-addr=20f0f0ab-e470-101d-9908-346e4378b6f6 servers="[{Suffrage:Voter ID:server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62 Address:252f1fc1-cdcb-ed13-0b66-9543b2f01f62} {Suffrage:Voter ID:server-06130b71-dcf9-fede-550d-9f2c6588bf5f Address:06130b71-dcf9-fede-550d-9f2c6588bf5f} {Suffrage:Voter ID:server-20f0f0ab-e470-101d-9908-346e4378b6f6 Address:20f0f0ab-e470-101d-9908-346e4378b6f6}]" -testLoggerAdapter verbose: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.873-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: added peer, starting replication: peer=server-20f0f0ab-e470-101d-9908-346e4378b6f6 - testing.go:149: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.873-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: added peer, starting replication: peer=server-20f0f0ab-e470-101d-9908-346e4378b6f6 -testLoggerAdapter verbose: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.873-1200 [ERROR] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: peer has newer term, stopping replication: peer="{Voter server-20f0f0ab-e470-101d-9908-346e4378b6f6 20f0f0ab-e470-101d-9908-346e4378b6f6}" - testing.go:149: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.873-1200 [ERROR] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: peer has newer term, stopping replication: peer="{Voter server-20f0f0ab-e470-101d-9908-346e4378b6f6 20f0f0ab-e470-101d-9908-346e4378b6f6}" -testLoggerAdapter verbose: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.873-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: entering follower state: follower="Node at 252f1fc1-cdcb-ed13-0b66-9543b2f01f62 [Follower]" leader= - testing.go:149: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.873-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: entering follower state: follower="Node at 252f1fc1-cdcb-ed13-0b66-9543b2f01f62 [Follower]" leader= -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:23.874-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:23.874-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.874-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: aborting pipeline replication: peer="{Voter server-06130b71-dcf9-fede-550d-9f2c6588bf5f 06130b71-dcf9-fede-550d-9f2c6588bf5f}" - testing.go:149: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.874-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: aborting pipeline replication: peer="{Voter server-06130b71-dcf9-fede-550d-9f2c6588bf5f 06130b71-dcf9-fede-550d-9f2c6588bf5f}" -testLoggerAdapter verbose: server-20f0f0ab-e470-101d-9908-346e4378b6f6: 2024-06-11T02:29:23.923-1200 [WARN] server-20f0f0ab-e470-101d-9908-346e4378b6f6: Election timeout reached, restarting election - testing.go:149: server-20f0f0ab-e470-101d-9908-346e4378b6f6: 2024-06-11T02:29:23.923-1200 [WARN] server-20f0f0ab-e470-101d-9908-346e4378b6f6: Election timeout reached, restarting election -testLoggerAdapter verbose: server-20f0f0ab-e470-101d-9908-346e4378b6f6: 2024-06-11T02:29:23.923-1200 [INFO] server-20f0f0ab-e470-101d-9908-346e4378b6f6: entering candidate state: node="Node at 20f0f0ab-e470-101d-9908-346e4378b6f6 [Candidate]" term=4 - testing.go:149: server-20f0f0ab-e470-101d-9908-346e4378b6f6: 2024-06-11T02:29:23.923-1200 [INFO] server-20f0f0ab-e470-101d-9908-346e4378b6f6: entering candidate state: node="Node at 20f0f0ab-e470-101d-9908-346e4378b6f6 [Candidate]" term=4 -testLoggerAdapter verbose: server-06130b71-dcf9-fede-550d-9f2c6588bf5f: 2024-06-11T02:29:23.923-1200 [WARN] server-06130b71-dcf9-fede-550d-9f2c6588bf5f: rejecting vote request since we have a leader: from=20f0f0ab-e470-101d-9908-346e4378b6f6 leader=252f1fc1-cdcb-ed13-0b66-9543b2f01f62 - testing.go:149: server-06130b71-dcf9-fede-550d-9f2c6588bf5f: 2024-06-11T02:29:23.923-1200 [WARN] server-06130b71-dcf9-fede-550d-9f2c6588bf5f: rejecting vote request since we have a leader: from=20f0f0ab-e470-101d-9908-346e4378b6f6 leader=252f1fc1-cdcb-ed13-0b66-9543b2f01f62 -testLoggerAdapter verbose: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.923-1200 [WARN] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: rejecting vote request since our last index is greater: candidate=20f0f0ab-e470-101d-9908-346e4378b6f6 last-index=4 last-candidate-index=3 - testing.go:149: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.923-1200 [WARN] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: rejecting vote request since our last index is greater: candidate=20f0f0ab-e470-101d-9908-346e4378b6f6 last-index=4 last-candidate-index=3 -testLoggerAdapter verbose: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.948-1200 [WARN] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.948-1200 [WARN] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.948-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: entering candidate state: node="Node at 252f1fc1-cdcb-ed13-0b66-9543b2f01f62 [Candidate]" term=5 - testing.go:149: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.948-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: entering candidate state: node="Node at 252f1fc1-cdcb-ed13-0b66-9543b2f01f62 [Candidate]" term=5 -testLoggerAdapter verbose: server-20f0f0ab-e470-101d-9908-346e4378b6f6: 2024-06-11T02:29:23.948-1200 [INFO] server-20f0f0ab-e470-101d-9908-346e4378b6f6: entering follower state: follower="Node at 20f0f0ab-e470-101d-9908-346e4378b6f6 [Follower]" leader= -testLoggerAdapter verbose: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.948-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: election won: tally=2 - testing.go:149: server-20f0f0ab-e470-101d-9908-346e4378b6f6: 2024-06-11T02:29:23.948-1200 [INFO] server-20f0f0ab-e470-101d-9908-346e4378b6f6: entering follower state: follower="Node at 20f0f0ab-e470-101d-9908-346e4378b6f6 [Follower]" leader= - testing.go:149: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.948-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: election won: tally=2 -testLoggerAdapter verbose: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.948-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: entering leader state: leader="Node at 252f1fc1-cdcb-ed13-0b66-9543b2f01f62 [Leader]" - testing.go:149: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.948-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: entering leader state: leader="Node at 252f1fc1-cdcb-ed13-0b66-9543b2f01f62 [Leader]" -testLoggerAdapter verbose: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.948-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: added peer, starting replication: peer=server-06130b71-dcf9-fede-550d-9f2c6588bf5f - testing.go:149: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.948-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: added peer, starting replication: peer=server-06130b71-dcf9-fede-550d-9f2c6588bf5f -testLoggerAdapter verbose: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.948-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: added peer, starting replication: peer=server-20f0f0ab-e470-101d-9908-346e4378b6f6 - testing.go:149: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.948-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: added peer, starting replication: peer=server-20f0f0ab-e470-101d-9908-346e4378b6f6 -testLoggerAdapter verbose: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.949-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: pipelining replication: peer="{Voter server-06130b71-dcf9-fede-550d-9f2c6588bf5f 06130b71-dcf9-fede-550d-9f2c6588bf5f}" - testing.go:149: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.949-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: pipelining replication: peer="{Voter server-06130b71-dcf9-fede-550d-9f2c6588bf5f 06130b71-dcf9-fede-550d-9f2c6588bf5f}" -testLoggerAdapter verbose: server-20f0f0ab-e470-101d-9908-346e4378b6f6: 2024-06-11T02:29:23.949-1200 [WARN] server-20f0f0ab-e470-101d-9908-346e4378b6f6: failed to get previous log: previous-index=4 last-index=3 error="log not found" - testing.go:149: server-20f0f0ab-e470-101d-9908-346e4378b6f6: 2024-06-11T02:29:23.949-1200 [WARN] server-20f0f0ab-e470-101d-9908-346e4378b6f6: failed to get previous log: previous-index=4 last-index=3 error="log not found" -testLoggerAdapter verbose: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.949-1200 [WARN] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: appendEntries rejected, sending older logs: peer="{Voter server-20f0f0ab-e470-101d-9908-346e4378b6f6 20f0f0ab-e470-101d-9908-346e4378b6f6}" next=4 - testing.go:149: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.949-1200 [WARN] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: appendEntries rejected, sending older logs: peer="{Voter server-20f0f0ab-e470-101d-9908-346e4378b6f6 20f0f0ab-e470-101d-9908-346e4378b6f6}" next=4 -testLoggerAdapter verbose: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.949-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: pipelining replication: peer="{Voter server-20f0f0ab-e470-101d-9908-346e4378b6f6 20f0f0ab-e470-101d-9908-346e4378b6f6}" - testing.go:149: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:23.949-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: pipelining replication: peer="{Voter server-20f0f0ab-e470-101d-9908-346e4378b6f6 20f0f0ab-e470-101d-9908-346e4378b6f6}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:24.054-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:23.948864786 -1200 -12 m=+14.944237328 (1 nodes), 74.750427ms from start of poll, 465.25179ms from cluster start. Timeout at 2024-06-11 02:29:24.053934268 -1200 -12 m=+15.049307102, 105.069774ms after stability - testing.go:149: cluster: 2024-06-11T02:29:24.054-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:23.948864786 -1200 -12 m=+14.944237328 (1 nodes), 74.750427ms from start of poll, 465.25179ms from cluster start. Timeout at 2024-06-11 02:29:24.053934268 -1200 -12 m=+15.049307102, 105.069774ms after stability -testLoggerAdapter verbose: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:24.054-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: aborting pipeline replication: peer="{Voter server-20f0f0ab-e470-101d-9908-346e4378b6f6 20f0f0ab-e470-101d-9908-346e4378b6f6}" - testing.go:149: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:24.054-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: aborting pipeline replication: peer="{Voter server-20f0f0ab-e470-101d-9908-346e4378b6f6 20f0f0ab-e470-101d-9908-346e4378b6f6}" -testLoggerAdapter verbose: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:24.054-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: aborting pipeline replication: peer="{Voter server-06130b71-dcf9-fede-550d-9f2c6588bf5f 06130b71-dcf9-fede-550d-9f2c6588bf5f}" - testing.go:149: server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: 2024-06-11T02:29:24.054-1200 [INFO] server-252f1fc1-cdcb-ed13-0b66-9543b2f01f62: aborting pipeline replication: peer="{Voter server-06130b71-dcf9-fede-550d-9f2c6588bf5f 06130b71-dcf9-fede-550d-9f2c6588bf5f}" ---- PASS: TestRaft_ReJoinFollower (0.58s) -=== RUN TestRaft_LeaderLeaseExpire -testLoggerAdapter verbose: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.059-1200 [INFO] server-92718062-0f69-6906-79c5-6e0d37f8e841: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-92718062-0f69-6906-79c5-6e0d37f8e841 Address:92718062-0f69-6906-79c5-6e0d37f8e841} {Suffrage:Voter ID:server-0b778711-3bde-3bd0-63c1-ec868742ed86 Address:0b778711-3bde-3bd0-63c1-ec868742ed86}]" - testing.go:149: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.059-1200 [INFO] server-92718062-0f69-6906-79c5-6e0d37f8e841: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-92718062-0f69-6906-79c5-6e0d37f8e841 Address:92718062-0f69-6906-79c5-6e0d37f8e841} {Suffrage:Voter ID:server-0b778711-3bde-3bd0-63c1-ec868742ed86 Address:0b778711-3bde-3bd0-63c1-ec868742ed86}]" -testLoggerAdapter verbose: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.059-1200 [INFO] server-92718062-0f69-6906-79c5-6e0d37f8e841: entering follower state: follower="Node at 92718062-0f69-6906-79c5-6e0d37f8e841 [Follower]" leader= - testing.go:149: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.059-1200 [INFO] server-92718062-0f69-6906-79c5-6e0d37f8e841: entering follower state: follower="Node at 92718062-0f69-6906-79c5-6e0d37f8e841 [Follower]" leader= -testLoggerAdapter verbose: server-0b778711-3bde-3bd0-63c1-ec868742ed86: 2024-06-11T02:29:24.059-1200 [INFO] server-0b778711-3bde-3bd0-63c1-ec868742ed86: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-92718062-0f69-6906-79c5-6e0d37f8e841 Address:92718062-0f69-6906-79c5-6e0d37f8e841} {Suffrage:Voter ID:server-0b778711-3bde-3bd0-63c1-ec868742ed86 Address:0b778711-3bde-3bd0-63c1-ec868742ed86}]" - testing.go:149: server-0b778711-3bde-3bd0-63c1-ec868742ed86: 2024-06-11T02:29:24.059-1200 [INFO] server-0b778711-3bde-3bd0-63c1-ec868742ed86: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-92718062-0f69-6906-79c5-6e0d37f8e841 Address:92718062-0f69-6906-79c5-6e0d37f8e841} {Suffrage:Voter ID:server-0b778711-3bde-3bd0-63c1-ec868742ed86 Address:0b778711-3bde-3bd0-63c1-ec868742ed86}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:24.060-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:24.060-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-0b778711-3bde-3bd0-63c1-ec868742ed86: 2024-06-11T02:29:24.060-1200 [INFO] server-0b778711-3bde-3bd0-63c1-ec868742ed86: entering follower state: follower="Node at 0b778711-3bde-3bd0-63c1-ec868742ed86 [Follower]" leader= - testing.go:149: server-0b778711-3bde-3bd0-63c1-ec868742ed86: 2024-06-11T02:29:24.060-1200 [INFO] server-0b778711-3bde-3bd0-63c1-ec868742ed86: entering follower state: follower="Node at 0b778711-3bde-3bd0-63c1-ec868742ed86 [Follower]" leader= -testLoggerAdapter verbose: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.128-1200 [WARN] server-92718062-0f69-6906-79c5-6e0d37f8e841: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.128-1200 [WARN] server-92718062-0f69-6906-79c5-6e0d37f8e841: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.128-1200 [INFO] server-92718062-0f69-6906-79c5-6e0d37f8e841: entering candidate state: node="Node at 92718062-0f69-6906-79c5-6e0d37f8e841 [Candidate]" term=2 - testing.go:149: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.128-1200 [INFO] server-92718062-0f69-6906-79c5-6e0d37f8e841: entering candidate state: node="Node at 92718062-0f69-6906-79c5-6e0d37f8e841 [Candidate]" term=2 -testLoggerAdapter verbose: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.129-1200 [INFO] server-92718062-0f69-6906-79c5-6e0d37f8e841: election won: tally=2 - testing.go:149: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.129-1200 [INFO] server-92718062-0f69-6906-79c5-6e0d37f8e841: election won: tally=2 -testLoggerAdapter verbose: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.129-1200 [INFO] server-92718062-0f69-6906-79c5-6e0d37f8e841: entering leader state: leader="Node at 92718062-0f69-6906-79c5-6e0d37f8e841 [Leader]" - testing.go:149: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.129-1200 [INFO] server-92718062-0f69-6906-79c5-6e0d37f8e841: entering leader state: leader="Node at 92718062-0f69-6906-79c5-6e0d37f8e841 [Leader]" -testLoggerAdapter verbose: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.129-1200 [INFO] server-92718062-0f69-6906-79c5-6e0d37f8e841: added peer, starting replication: peer=server-0b778711-3bde-3bd0-63c1-ec868742ed86 - testing.go:149: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.129-1200 [INFO] server-92718062-0f69-6906-79c5-6e0d37f8e841: added peer, starting replication: peer=server-0b778711-3bde-3bd0-63c1-ec868742ed86 -testLoggerAdapter verbose: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.129-1200 [INFO] server-92718062-0f69-6906-79c5-6e0d37f8e841: pipelining replication: peer="{Voter server-0b778711-3bde-3bd0-63c1-ec868742ed86 0b778711-3bde-3bd0-63c1-ec868742ed86}" - testing.go:149: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.129-1200 [INFO] server-92718062-0f69-6906-79c5-6e0d37f8e841: pipelining replication: peer="{Voter server-0b778711-3bde-3bd0-63c1-ec868742ed86 0b778711-3bde-3bd0-63c1-ec868742ed86}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:24.235-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:24.129252268 -1200 -12 m=+15.124625101 (1 nodes), 68.941994ms from start of poll, 70.213054ms from cluster start. Timeout at 2024-06-11 02:29:24.235288315 -1200 -12 m=+15.230661148, 106.036047ms after stability - testing.go:149: cluster: 2024-06-11T02:29:24.235-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:24.129252268 -1200 -12 m=+15.124625101 (1 nodes), 68.941994ms from start of poll, 70.213054ms from cluster start. Timeout at 2024-06-11 02:29:24.235288315 -1200 -12 m=+15.230661148, 106.036047ms after stability -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:24.241-1200 [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2024-06-11T02:29:24.241-1200 [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:24.347-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:24.241395409 -1200 -12 m=+15.236767950 (1 nodes), 2.625µs from start of poll, 182.355903ms from cluster start. Timeout at 2024-06-11 02:29:24.347155836 -1200 -12 m=+15.342528669, 105.760719ms after stability - testing.go:149: cluster: 2024-06-11T02:29:24.347-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:24.241395409 -1200 -12 m=+15.236767950 (1 nodes), 2.625µs from start of poll, 182.355903ms from cluster start. Timeout at 2024-06-11 02:29:24.347155836 -1200 -12 m=+15.342528669, 105.760719ms after stability - raft_test.go:1492: [INFO] Disconnecting Node at 0b778711-3bde-3bd0-63c1-ec868742ed86 [Follower] -testLoggerAdapter verbose: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.350-1200 [ERROR] server-92718062-0f69-6906-79c5-6e0d37f8e841: failed to pipeline appendEntries: peer="{Voter server-0b778711-3bde-3bd0-63c1-ec868742ed86 0b778711-3bde-3bd0-63c1-ec868742ed86}" error="append pipeline closed" - testing.go:149: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.350-1200 [ERROR] server-92718062-0f69-6906-79c5-6e0d37f8e841: failed to pipeline appendEntries: peer="{Voter server-0b778711-3bde-3bd0-63c1-ec868742ed86 0b778711-3bde-3bd0-63c1-ec868742ed86}" error="append pipeline closed" -testLoggerAdapter verbose: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.350-1200 [INFO] server-92718062-0f69-6906-79c5-6e0d37f8e841: aborting pipeline replication: peer="{Voter server-0b778711-3bde-3bd0-63c1-ec868742ed86 0b778711-3bde-3bd0-63c1-ec868742ed86}" - testing.go:149: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.350-1200 [INFO] server-92718062-0f69-6906-79c5-6e0d37f8e841: aborting pipeline replication: peer="{Voter server-0b778711-3bde-3bd0-63c1-ec868742ed86 0b778711-3bde-3bd0-63c1-ec868742ed86}" -testLoggerAdapter verbose: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.350-1200 [ERROR] server-92718062-0f69-6906-79c5-6e0d37f8e841: failed to heartbeat to: peer=0b778711-3bde-3bd0-63c1-ec868742ed86 error="failed to connect to peer: 0b778711-3bde-3bd0-63c1-ec868742ed86" - testing.go:149: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.350-1200 [ERROR] server-92718062-0f69-6906-79c5-6e0d37f8e841: failed to heartbeat to: peer=0b778711-3bde-3bd0-63c1-ec868742ed86 error="failed to connect to peer: 0b778711-3bde-3bd0-63c1-ec868742ed86" -testLoggerAdapter verbose: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.358-1200 [ERROR] server-92718062-0f69-6906-79c5-6e0d37f8e841: failed to appendEntries to: peer="{Voter server-0b778711-3bde-3bd0-63c1-ec868742ed86 0b778711-3bde-3bd0-63c1-ec868742ed86}" error="failed to connect to peer: 0b778711-3bde-3bd0-63c1-ec868742ed86" - testing.go:149: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.358-1200 [ERROR] server-92718062-0f69-6906-79c5-6e0d37f8e841: failed to appendEntries to: peer="{Voter server-0b778711-3bde-3bd0-63c1-ec868742ed86 0b778711-3bde-3bd0-63c1-ec868742ed86}" error="failed to connect to peer: 0b778711-3bde-3bd0-63c1-ec868742ed86" -testLoggerAdapter verbose: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.369-1200 [ERROR] server-92718062-0f69-6906-79c5-6e0d37f8e841: failed to heartbeat to: peer=0b778711-3bde-3bd0-63c1-ec868742ed86 error="failed to connect to peer: 0b778711-3bde-3bd0-63c1-ec868742ed86" - testing.go:149: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.369-1200 [ERROR] server-92718062-0f69-6906-79c5-6e0d37f8e841: failed to heartbeat to: peer=0b778711-3bde-3bd0-63c1-ec868742ed86 error="failed to connect to peer: 0b778711-3bde-3bd0-63c1-ec868742ed86" -testLoggerAdapter verbose: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.377-1200 [ERROR] server-92718062-0f69-6906-79c5-6e0d37f8e841: failed to appendEntries to: peer="{Voter server-0b778711-3bde-3bd0-63c1-ec868742ed86 0b778711-3bde-3bd0-63c1-ec868742ed86}" error="failed to connect to peer: 0b778711-3bde-3bd0-63c1-ec868742ed86" - testing.go:149: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.377-1200 [ERROR] server-92718062-0f69-6906-79c5-6e0d37f8e841: failed to appendEntries to: peer="{Voter server-0b778711-3bde-3bd0-63c1-ec868742ed86 0b778711-3bde-3bd0-63c1-ec868742ed86}" error="failed to connect to peer: 0b778711-3bde-3bd0-63c1-ec868742ed86" -testLoggerAdapter verbose: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.388-1200 [ERROR] server-92718062-0f69-6906-79c5-6e0d37f8e841: failed to heartbeat to: peer=0b778711-3bde-3bd0-63c1-ec868742ed86 error="failed to connect to peer: 0b778711-3bde-3bd0-63c1-ec868742ed86" - testing.go:149: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.388-1200 [ERROR] server-92718062-0f69-6906-79c5-6e0d37f8e841: failed to heartbeat to: peer=0b778711-3bde-3bd0-63c1-ec868742ed86 error="failed to connect to peer: 0b778711-3bde-3bd0-63c1-ec868742ed86" -testLoggerAdapter verbose: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.392-1200 [WARN] server-92718062-0f69-6906-79c5-6e0d37f8e841: failed to contact: server-id=server-0b778711-3bde-3bd0-63c1-ec868742ed86 time=50.630048ms - testing.go:149: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.392-1200 [WARN] server-92718062-0f69-6906-79c5-6e0d37f8e841: failed to contact: server-id=server-0b778711-3bde-3bd0-63c1-ec868742ed86 time=50.630048ms -testLoggerAdapter verbose: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.392-1200 [WARN] server-92718062-0f69-6906-79c5-6e0d37f8e841: failed to contact quorum of nodes, stepping down - testing.go:149: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.392-1200 [WARN] server-92718062-0f69-6906-79c5-6e0d37f8e841: failed to contact quorum of nodes, stepping down -testLoggerAdapter verbose: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.392-1200 [INFO] server-92718062-0f69-6906-79c5-6e0d37f8e841: entering follower state: follower="Node at 92718062-0f69-6906-79c5-6e0d37f8e841 [Follower]" leader= - testing.go:149: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.392-1200 [INFO] server-92718062-0f69-6906-79c5-6e0d37f8e841: entering follower state: follower="Node at 92718062-0f69-6906-79c5-6e0d37f8e841 [Follower]" leader= -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:24.392-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:24.392-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.394-1200 [ERROR] server-92718062-0f69-6906-79c5-6e0d37f8e841: failed to appendEntries to: peer="{Voter server-0b778711-3bde-3bd0-63c1-ec868742ed86 0b778711-3bde-3bd0-63c1-ec868742ed86}" error="failed to connect to peer: 0b778711-3bde-3bd0-63c1-ec868742ed86" - testing.go:149: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.394-1200 [ERROR] server-92718062-0f69-6906-79c5-6e0d37f8e841: failed to appendEntries to: peer="{Voter server-0b778711-3bde-3bd0-63c1-ec868742ed86 0b778711-3bde-3bd0-63c1-ec868742ed86}" error="failed to connect to peer: 0b778711-3bde-3bd0-63c1-ec868742ed86" -testLoggerAdapter verbose: server-0b778711-3bde-3bd0-63c1-ec868742ed86: 2024-06-11T02:29:24.463-1200 [WARN] server-0b778711-3bde-3bd0-63c1-ec868742ed86: heartbeat timeout reached, starting election: last-leader=92718062-0f69-6906-79c5-6e0d37f8e841 - testing.go:149: server-0b778711-3bde-3bd0-63c1-ec868742ed86: 2024-06-11T02:29:24.463-1200 [WARN] server-0b778711-3bde-3bd0-63c1-ec868742ed86: heartbeat timeout reached, starting election: last-leader=92718062-0f69-6906-79c5-6e0d37f8e841 -testLoggerAdapter verbose: server-0b778711-3bde-3bd0-63c1-ec868742ed86: 2024-06-11T02:29:24.463-1200 [INFO] server-0b778711-3bde-3bd0-63c1-ec868742ed86: entering candidate state: node="Node at 0b778711-3bde-3bd0-63c1-ec868742ed86 [Candidate]" term=3 - testing.go:149: server-0b778711-3bde-3bd0-63c1-ec868742ed86: 2024-06-11T02:29:24.463-1200 [INFO] server-0b778711-3bde-3bd0-63c1-ec868742ed86: entering candidate state: node="Node at 0b778711-3bde-3bd0-63c1-ec868742ed86 [Candidate]" term=3 -testLoggerAdapter verbose: server-0b778711-3bde-3bd0-63c1-ec868742ed86: 2024-06-11T02:29:24.463-1200 [ERROR] server-0b778711-3bde-3bd0-63c1-ec868742ed86: failed to make requestVote RPC: target="{Voter server-92718062-0f69-6906-79c5-6e0d37f8e841 92718062-0f69-6906-79c5-6e0d37f8e841}" error="failed to connect to peer: 92718062-0f69-6906-79c5-6e0d37f8e841" - testing.go:149: server-0b778711-3bde-3bd0-63c1-ec868742ed86: 2024-06-11T02:29:24.463-1200 [ERROR] server-0b778711-3bde-3bd0-63c1-ec868742ed86: failed to make requestVote RPC: target="{Voter server-92718062-0f69-6906-79c5-6e0d37f8e841 92718062-0f69-6906-79c5-6e0d37f8e841}" error="failed to connect to peer: 92718062-0f69-6906-79c5-6e0d37f8e841" -testLoggerAdapter verbose: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.474-1200 [WARN] server-92718062-0f69-6906-79c5-6e0d37f8e841: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.474-1200 [WARN] server-92718062-0f69-6906-79c5-6e0d37f8e841: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.475-1200 [INFO] server-92718062-0f69-6906-79c5-6e0d37f8e841: entering candidate state: node="Node at 92718062-0f69-6906-79c5-6e0d37f8e841 [Candidate]" term=3 - testing.go:149: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.475-1200 [INFO] server-92718062-0f69-6906-79c5-6e0d37f8e841: entering candidate state: node="Node at 92718062-0f69-6906-79c5-6e0d37f8e841 [Candidate]" term=3 -testLoggerAdapter verbose: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.475-1200 [ERROR] server-92718062-0f69-6906-79c5-6e0d37f8e841: failed to make requestVote RPC: target="{Voter server-0b778711-3bde-3bd0-63c1-ec868742ed86 0b778711-3bde-3bd0-63c1-ec868742ed86}" error="failed to connect to peer: 0b778711-3bde-3bd0-63c1-ec868742ed86" - testing.go:149: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.475-1200 [ERROR] server-92718062-0f69-6906-79c5-6e0d37f8e841: failed to make requestVote RPC: target="{Voter server-0b778711-3bde-3bd0-63c1-ec868742ed86 0b778711-3bde-3bd0-63c1-ec868742ed86}" error="failed to connect to peer: 0b778711-3bde-3bd0-63c1-ec868742ed86" -testLoggerAdapter verbose: server-0b778711-3bde-3bd0-63c1-ec868742ed86: 2024-06-11T02:29:24.546-1200 [WARN] server-0b778711-3bde-3bd0-63c1-ec868742ed86: Election timeout reached, restarting election - testing.go:149: server-0b778711-3bde-3bd0-63c1-ec868742ed86: 2024-06-11T02:29:24.546-1200 [WARN] server-0b778711-3bde-3bd0-63c1-ec868742ed86: Election timeout reached, restarting election -testLoggerAdapter verbose: server-0b778711-3bde-3bd0-63c1-ec868742ed86: 2024-06-11T02:29:24.547-1200 [INFO] server-0b778711-3bde-3bd0-63c1-ec868742ed86: entering candidate state: node="Node at 0b778711-3bde-3bd0-63c1-ec868742ed86 [Candidate]" term=4 - testing.go:149: server-0b778711-3bde-3bd0-63c1-ec868742ed86: 2024-06-11T02:29:24.547-1200 [INFO] server-0b778711-3bde-3bd0-63c1-ec868742ed86: entering candidate state: node="Node at 0b778711-3bde-3bd0-63c1-ec868742ed86 [Candidate]" term=4 -testLoggerAdapter verbose: server-0b778711-3bde-3bd0-63c1-ec868742ed86: 2024-06-11T02:29:24.547-1200 [ERROR] server-0b778711-3bde-3bd0-63c1-ec868742ed86: failed to make requestVote RPC: target="{Voter server-92718062-0f69-6906-79c5-6e0d37f8e841 92718062-0f69-6906-79c5-6e0d37f8e841}" error="failed to connect to peer: 92718062-0f69-6906-79c5-6e0d37f8e841" - testing.go:149: server-0b778711-3bde-3bd0-63c1-ec868742ed86: 2024-06-11T02:29:24.547-1200 [ERROR] server-0b778711-3bde-3bd0-63c1-ec868742ed86: failed to make requestVote RPC: target="{Voter server-92718062-0f69-6906-79c5-6e0d37f8e841 92718062-0f69-6906-79c5-6e0d37f8e841}" error="failed to connect to peer: 92718062-0f69-6906-79c5-6e0d37f8e841" -testLoggerAdapter verbose: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.550-1200 [WARN] server-92718062-0f69-6906-79c5-6e0d37f8e841: Election timeout reached, restarting election - testing.go:149: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.550-1200 [WARN] server-92718062-0f69-6906-79c5-6e0d37f8e841: Election timeout reached, restarting election -testLoggerAdapter verbose: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.550-1200 [INFO] server-92718062-0f69-6906-79c5-6e0d37f8e841: entering candidate state: node="Node at 92718062-0f69-6906-79c5-6e0d37f8e841 [Candidate]" term=4 - testing.go:149: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.550-1200 [INFO] server-92718062-0f69-6906-79c5-6e0d37f8e841: entering candidate state: node="Node at 92718062-0f69-6906-79c5-6e0d37f8e841 [Candidate]" term=4 -testLoggerAdapter verbose: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.550-1200 [ERROR] server-92718062-0f69-6906-79c5-6e0d37f8e841: failed to make requestVote RPC: target="{Voter server-0b778711-3bde-3bd0-63c1-ec868742ed86 0b778711-3bde-3bd0-63c1-ec868742ed86}" error="failed to connect to peer: 0b778711-3bde-3bd0-63c1-ec868742ed86" - testing.go:149: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.550-1200 [ERROR] server-92718062-0f69-6906-79c5-6e0d37f8e841: failed to make requestVote RPC: target="{Voter server-0b778711-3bde-3bd0-63c1-ec868742ed86 0b778711-3bde-3bd0-63c1-ec868742ed86}" error="failed to connect to peer: 0b778711-3bde-3bd0-63c1-ec868742ed86" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:24.569-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:24.463584062 -1200 -12 m=+15.458956896 (0 nodes), 70.54059ms from start of poll, 404.544849ms from cluster start. Timeout at 2024-06-11 02:29:24.569625942 -1200 -12 m=+15.564998776, 106.04188ms after stability - testing.go:149: cluster: 2024-06-11T02:29:24.569-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:24.463584062 -1200 -12 m=+15.458956896 (0 nodes), 70.54059ms from start of poll, 404.544849ms from cluster start. Timeout at 2024-06-11 02:29:24.569625942 -1200 -12 m=+15.564998776, 106.04188ms after stability -testLoggerAdapter verbose: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.618-1200 [WARN] server-92718062-0f69-6906-79c5-6e0d37f8e841: Election timeout reached, restarting election - testing.go:149: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.618-1200 [WARN] server-92718062-0f69-6906-79c5-6e0d37f8e841: Election timeout reached, restarting election -testLoggerAdapter verbose: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.619-1200 [INFO] server-92718062-0f69-6906-79c5-6e0d37f8e841: entering candidate state: node="Node at 92718062-0f69-6906-79c5-6e0d37f8e841 [Candidate]" term=5 - testing.go:149: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.619-1200 [INFO] server-92718062-0f69-6906-79c5-6e0d37f8e841: entering candidate state: node="Node at 92718062-0f69-6906-79c5-6e0d37f8e841 [Candidate]" term=5 -testLoggerAdapter verbose: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.619-1200 [ERROR] server-92718062-0f69-6906-79c5-6e0d37f8e841: failed to make requestVote RPC: target="{Voter server-0b778711-3bde-3bd0-63c1-ec868742ed86 0b778711-3bde-3bd0-63c1-ec868742ed86}" error="failed to connect to peer: 0b778711-3bde-3bd0-63c1-ec868742ed86" - testing.go:149: server-92718062-0f69-6906-79c5-6e0d37f8e841: 2024-06-11T02:29:24.619-1200 [ERROR] server-92718062-0f69-6906-79c5-6e0d37f8e841: failed to make requestVote RPC: target="{Voter server-0b778711-3bde-3bd0-63c1-ec868742ed86 0b778711-3bde-3bd0-63c1-ec868742ed86}" error="failed to connect to peer: 0b778711-3bde-3bd0-63c1-ec868742ed86" -testLoggerAdapter verbose: server-0b778711-3bde-3bd0-63c1-ec868742ed86: 2024-06-11T02:29:24.638-1200 [WARN] server-0b778711-3bde-3bd0-63c1-ec868742ed86: Election timeout reached, restarting election - testing.go:149: server-0b778711-3bde-3bd0-63c1-ec868742ed86: 2024-06-11T02:29:24.638-1200 [WARN] server-0b778711-3bde-3bd0-63c1-ec868742ed86: Election timeout reached, restarting election -testLoggerAdapter verbose: server-0b778711-3bde-3bd0-63c1-ec868742ed86: 2024-06-11T02:29:24.638-1200 [INFO] server-0b778711-3bde-3bd0-63c1-ec868742ed86: entering candidate state: node="Node at 0b778711-3bde-3bd0-63c1-ec868742ed86 [Candidate]" term=5 - testing.go:149: server-0b778711-3bde-3bd0-63c1-ec868742ed86: 2024-06-11T02:29:24.638-1200 [INFO] server-0b778711-3bde-3bd0-63c1-ec868742ed86: entering candidate state: node="Node at 0b778711-3bde-3bd0-63c1-ec868742ed86 [Candidate]" term=5 -testLoggerAdapter verbose: server-0b778711-3bde-3bd0-63c1-ec868742ed86: 2024-06-11T02:29:24.638-1200 [ERROR] server-0b778711-3bde-3bd0-63c1-ec868742ed86: failed to make requestVote RPC: target="{Voter server-92718062-0f69-6906-79c5-6e0d37f8e841 92718062-0f69-6906-79c5-6e0d37f8e841}" error="failed to connect to peer: 92718062-0f69-6906-79c5-6e0d37f8e841" - testing.go:149: server-0b778711-3bde-3bd0-63c1-ec868742ed86: 2024-06-11T02:29:24.638-1200 [ERROR] server-0b778711-3bde-3bd0-63c1-ec868742ed86: failed to make requestVote RPC: target="{Voter server-92718062-0f69-6906-79c5-6e0d37f8e841 92718062-0f69-6906-79c5-6e0d37f8e841}" error="failed to connect to peer: 92718062-0f69-6906-79c5-6e0d37f8e841" ---- PASS: TestRaft_LeaderLeaseExpire (0.62s) -=== RUN TestRaft_Barrier -testLoggerAdapter verbose: server-21c42580-549f-0bf2-8ce9-3624111729fe: 2024-06-11T02:29:24.678-1200 [INFO] server-21c42580-549f-0bf2-8ce9-3624111729fe: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-21c42580-549f-0bf2-8ce9-3624111729fe Address:21c42580-549f-0bf2-8ce9-3624111729fe} {Suffrage:Voter ID:server-0da9c2c8-02b1-6cac-352a-e9934ffd9889 Address:0da9c2c8-02b1-6cac-352a-e9934ffd9889} {Suffrage:Voter ID:server-05409270-6769-658a-2658-cf8d3605c1ba Address:05409270-6769-658a-2658-cf8d3605c1ba}]" - testing.go:149: server-21c42580-549f-0bf2-8ce9-3624111729fe: 2024-06-11T02:29:24.678-1200 [INFO] server-21c42580-549f-0bf2-8ce9-3624111729fe: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-21c42580-549f-0bf2-8ce9-3624111729fe Address:21c42580-549f-0bf2-8ce9-3624111729fe} {Suffrage:Voter ID:server-0da9c2c8-02b1-6cac-352a-e9934ffd9889 Address:0da9c2c8-02b1-6cac-352a-e9934ffd9889} {Suffrage:Voter ID:server-05409270-6769-658a-2658-cf8d3605c1ba Address:05409270-6769-658a-2658-cf8d3605c1ba}]" -testLoggerAdapter verbose: server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: 2024-06-11T02:29:24.678-1200 [INFO] server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-21c42580-549f-0bf2-8ce9-3624111729fe Address:21c42580-549f-0bf2-8ce9-3624111729fe} {Suffrage:Voter ID:server-0da9c2c8-02b1-6cac-352a-e9934ffd9889 Address:0da9c2c8-02b1-6cac-352a-e9934ffd9889} {Suffrage:Voter ID:server-05409270-6769-658a-2658-cf8d3605c1ba Address:05409270-6769-658a-2658-cf8d3605c1ba}]" - testing.go:149: server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: 2024-06-11T02:29:24.678-1200 [INFO] server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-21c42580-549f-0bf2-8ce9-3624111729fe Address:21c42580-549f-0bf2-8ce9-3624111729fe} {Suffrage:Voter ID:server-0da9c2c8-02b1-6cac-352a-e9934ffd9889 Address:0da9c2c8-02b1-6cac-352a-e9934ffd9889} {Suffrage:Voter ID:server-05409270-6769-658a-2658-cf8d3605c1ba Address:05409270-6769-658a-2658-cf8d3605c1ba}]" -testLoggerAdapter verbose: server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: 2024-06-11T02:29:24.679-1200 [INFO] server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: entering follower state: follower="Node at 0da9c2c8-02b1-6cac-352a-e9934ffd9889 [Follower]" leader= - testing.go:149: server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: 2024-06-11T02:29:24.679-1200 [INFO] server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: entering follower state: follower="Node at 0da9c2c8-02b1-6cac-352a-e9934ffd9889 [Follower]" leader= -testLoggerAdapter verbose: server-21c42580-549f-0bf2-8ce9-3624111729fe: 2024-06-11T02:29:24.679-1200 [INFO] server-21c42580-549f-0bf2-8ce9-3624111729fe: entering follower state: follower="Node at 21c42580-549f-0bf2-8ce9-3624111729fe [Follower]" leader= - testing.go:149: server-21c42580-549f-0bf2-8ce9-3624111729fe: 2024-06-11T02:29:24.679-1200 [INFO] server-21c42580-549f-0bf2-8ce9-3624111729fe: entering follower state: follower="Node at 21c42580-549f-0bf2-8ce9-3624111729fe [Follower]" leader= -testLoggerAdapter verbose: server-05409270-6769-658a-2658-cf8d3605c1ba: 2024-06-11T02:29:24.679-1200 [INFO] server-05409270-6769-658a-2658-cf8d3605c1ba: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-21c42580-549f-0bf2-8ce9-3624111729fe Address:21c42580-549f-0bf2-8ce9-3624111729fe} {Suffrage:Voter ID:server-0da9c2c8-02b1-6cac-352a-e9934ffd9889 Address:0da9c2c8-02b1-6cac-352a-e9934ffd9889} {Suffrage:Voter ID:server-05409270-6769-658a-2658-cf8d3605c1ba Address:05409270-6769-658a-2658-cf8d3605c1ba}]" - testing.go:149: server-05409270-6769-658a-2658-cf8d3605c1ba: 2024-06-11T02:29:24.679-1200 [INFO] server-05409270-6769-658a-2658-cf8d3605c1ba: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-21c42580-549f-0bf2-8ce9-3624111729fe Address:21c42580-549f-0bf2-8ce9-3624111729fe} {Suffrage:Voter ID:server-0da9c2c8-02b1-6cac-352a-e9934ffd9889 Address:0da9c2c8-02b1-6cac-352a-e9934ffd9889} {Suffrage:Voter ID:server-05409270-6769-658a-2658-cf8d3605c1ba Address:05409270-6769-658a-2658-cf8d3605c1ba}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:24.679-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:24.679-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-05409270-6769-658a-2658-cf8d3605c1ba: 2024-06-11T02:29:24.679-1200 [INFO] server-05409270-6769-658a-2658-cf8d3605c1ba: entering follower state: follower="Node at 05409270-6769-658a-2658-cf8d3605c1ba [Follower]" leader= - testing.go:149: server-05409270-6769-658a-2658-cf8d3605c1ba: 2024-06-11T02:29:24.679-1200 [INFO] server-05409270-6769-658a-2658-cf8d3605c1ba: entering follower state: follower="Node at 05409270-6769-658a-2658-cf8d3605c1ba [Follower]" leader= -testLoggerAdapter verbose: server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: 2024-06-11T02:29:24.750-1200 [WARN] server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: 2024-06-11T02:29:24.750-1200 [WARN] server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: 2024-06-11T02:29:24.750-1200 [INFO] server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: entering candidate state: node="Node at 0da9c2c8-02b1-6cac-352a-e9934ffd9889 [Candidate]" term=2 - testing.go:149: server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: 2024-06-11T02:29:24.750-1200 [INFO] server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: entering candidate state: node="Node at 0da9c2c8-02b1-6cac-352a-e9934ffd9889 [Candidate]" term=2 -testLoggerAdapter verbose: server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: 2024-06-11T02:29:24.751-1200 [INFO] server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: election won: tally=2 - testing.go:149: server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: 2024-06-11T02:29:24.751-1200 [INFO] server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: election won: tally=2 -testLoggerAdapter verbose: server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: 2024-06-11T02:29:24.751-1200 [INFO] server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: entering leader state: leader="Node at 0da9c2c8-02b1-6cac-352a-e9934ffd9889 [Leader]" - testing.go:149: server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: 2024-06-11T02:29:24.751-1200 [INFO] server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: entering leader state: leader="Node at 0da9c2c8-02b1-6cac-352a-e9934ffd9889 [Leader]" -testLoggerAdapter verbose: server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: 2024-06-11T02:29:24.751-1200 [INFO] server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: added peer, starting replication: peer=server-21c42580-549f-0bf2-8ce9-3624111729fe - testing.go:149: server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: 2024-06-11T02:29:24.751-1200 [INFO] server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: added peer, starting replication: peer=server-21c42580-549f-0bf2-8ce9-3624111729fe -testLoggerAdapter verbose: server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: 2024-06-11T02:29:24.751-1200 [INFO] server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: added peer, starting replication: peer=server-05409270-6769-658a-2658-cf8d3605c1ba - testing.go:149: server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: 2024-06-11T02:29:24.751-1200 [INFO] server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: added peer, starting replication: peer=server-05409270-6769-658a-2658-cf8d3605c1ba -testLoggerAdapter verbose: server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: 2024-06-11T02:29:24.751-1200 [INFO] server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: pipelining replication: peer="{Voter server-05409270-6769-658a-2658-cf8d3605c1ba 05409270-6769-658a-2658-cf8d3605c1ba}" - testing.go:149: server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: 2024-06-11T02:29:24.751-1200 [INFO] server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: pipelining replication: peer="{Voter server-05409270-6769-658a-2658-cf8d3605c1ba 05409270-6769-658a-2658-cf8d3605c1ba}" -testLoggerAdapter verbose: server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: 2024-06-11T02:29:24.751-1200 [INFO] server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: pipelining replication: peer="{Voter server-21c42580-549f-0bf2-8ce9-3624111729fe 21c42580-549f-0bf2-8ce9-3624111729fe}" - testing.go:149: server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: 2024-06-11T02:29:24.751-1200 [INFO] server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: pipelining replication: peer="{Voter server-21c42580-549f-0bf2-8ce9-3624111729fe 21c42580-549f-0bf2-8ce9-3624111729fe}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:24.857-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:24.751966387 -1200 -12 m=+15.747338929 (1 nodes), 72.077644ms from start of poll, 73.659906ms from cluster start. Timeout at 2024-06-11 02:29:24.857478611 -1200 -12 m=+15.852852028, 105.513099ms after stability - testing.go:149: cluster: 2024-06-11T02:29:24.857-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:24.751966387 -1200 -12 m=+15.747338929 (1 nodes), 72.077644ms from start of poll, 73.659906ms from cluster start. Timeout at 2024-06-11 02:29:24.857478611 -1200 -12 m=+15.852852028, 105.513099ms after stability -testLoggerAdapter verbose: server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: 2024-06-11T02:29:24.877-1200 [INFO] server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: aborting pipeline replication: peer="{Voter server-05409270-6769-658a-2658-cf8d3605c1ba 05409270-6769-658a-2658-cf8d3605c1ba}" - testing.go:149: server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: 2024-06-11T02:29:24.877-1200 [INFO] server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: aborting pipeline replication: peer="{Voter server-05409270-6769-658a-2658-cf8d3605c1ba 05409270-6769-658a-2658-cf8d3605c1ba}" -testLoggerAdapter verbose: server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: 2024-06-11T02:29:24.877-1200 [INFO] server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: aborting pipeline replication: peer="{Voter server-21c42580-549f-0bf2-8ce9-3624111729fe 21c42580-549f-0bf2-8ce9-3624111729fe}" - testing.go:149: server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: 2024-06-11T02:29:24.877-1200 [INFO] server-0da9c2c8-02b1-6cac-352a-e9934ffd9889: aborting pipeline replication: peer="{Voter server-21c42580-549f-0bf2-8ce9-3624111729fe 21c42580-549f-0bf2-8ce9-3624111729fe}" ---- PASS: TestRaft_Barrier (0.20s) -=== RUN TestRaft_VerifyLeader -testLoggerAdapter verbose: server-d3ab7084-3ba6-315e-904a-35fe9c9bb531: 2024-06-11T02:29:24.881-1200 [INFO] server-d3ab7084-3ba6-315e-904a-35fe9c9bb531: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d3ab7084-3ba6-315e-904a-35fe9c9bb531 Address:d3ab7084-3ba6-315e-904a-35fe9c9bb531} {Suffrage:Voter ID:server-3492aad5-75d6-5de7-b678-44875ee40253 Address:3492aad5-75d6-5de7-b678-44875ee40253} {Suffrage:Voter ID:server-64f83319-19a0-f56e-c20c-a978578f0fd3 Address:64f83319-19a0-f56e-c20c-a978578f0fd3}]" - testing.go:149: server-d3ab7084-3ba6-315e-904a-35fe9c9bb531: 2024-06-11T02:29:24.881-1200 [INFO] server-d3ab7084-3ba6-315e-904a-35fe9c9bb531: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d3ab7084-3ba6-315e-904a-35fe9c9bb531 Address:d3ab7084-3ba6-315e-904a-35fe9c9bb531} {Suffrage:Voter ID:server-3492aad5-75d6-5de7-b678-44875ee40253 Address:3492aad5-75d6-5de7-b678-44875ee40253} {Suffrage:Voter ID:server-64f83319-19a0-f56e-c20c-a978578f0fd3 Address:64f83319-19a0-f56e-c20c-a978578f0fd3}]" -testLoggerAdapter verbose: server-3492aad5-75d6-5de7-b678-44875ee40253: 2024-06-11T02:29:24.881-1200 [INFO] server-3492aad5-75d6-5de7-b678-44875ee40253: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d3ab7084-3ba6-315e-904a-35fe9c9bb531 Address:d3ab7084-3ba6-315e-904a-35fe9c9bb531} {Suffrage:Voter ID:server-3492aad5-75d6-5de7-b678-44875ee40253 Address:3492aad5-75d6-5de7-b678-44875ee40253} {Suffrage:Voter ID:server-64f83319-19a0-f56e-c20c-a978578f0fd3 Address:64f83319-19a0-f56e-c20c-a978578f0fd3}]" - testing.go:149: server-3492aad5-75d6-5de7-b678-44875ee40253: 2024-06-11T02:29:24.881-1200 [INFO] server-3492aad5-75d6-5de7-b678-44875ee40253: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d3ab7084-3ba6-315e-904a-35fe9c9bb531 Address:d3ab7084-3ba6-315e-904a-35fe9c9bb531} {Suffrage:Voter ID:server-3492aad5-75d6-5de7-b678-44875ee40253 Address:3492aad5-75d6-5de7-b678-44875ee40253} {Suffrage:Voter ID:server-64f83319-19a0-f56e-c20c-a978578f0fd3 Address:64f83319-19a0-f56e-c20c-a978578f0fd3}]" -testLoggerAdapter verbose: server-64f83319-19a0-f56e-c20c-a978578f0fd3: 2024-06-11T02:29:24.882-1200 [INFO] server-64f83319-19a0-f56e-c20c-a978578f0fd3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d3ab7084-3ba6-315e-904a-35fe9c9bb531 Address:d3ab7084-3ba6-315e-904a-35fe9c9bb531} {Suffrage:Voter ID:server-3492aad5-75d6-5de7-b678-44875ee40253 Address:3492aad5-75d6-5de7-b678-44875ee40253} {Suffrage:Voter ID:server-64f83319-19a0-f56e-c20c-a978578f0fd3 Address:64f83319-19a0-f56e-c20c-a978578f0fd3}]" - testing.go:149: server-64f83319-19a0-f56e-c20c-a978578f0fd3: 2024-06-11T02:29:24.882-1200 [INFO] server-64f83319-19a0-f56e-c20c-a978578f0fd3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d3ab7084-3ba6-315e-904a-35fe9c9bb531 Address:d3ab7084-3ba6-315e-904a-35fe9c9bb531} {Suffrage:Voter ID:server-3492aad5-75d6-5de7-b678-44875ee40253 Address:3492aad5-75d6-5de7-b678-44875ee40253} {Suffrage:Voter ID:server-64f83319-19a0-f56e-c20c-a978578f0fd3 Address:64f83319-19a0-f56e-c20c-a978578f0fd3}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:24.882-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:24.882-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-d3ab7084-3ba6-315e-904a-35fe9c9bb531: 2024-06-11T02:29:24.882-1200 [INFO] server-d3ab7084-3ba6-315e-904a-35fe9c9bb531: entering follower state: follower="Node at d3ab7084-3ba6-315e-904a-35fe9c9bb531 [Follower]" leader= - testing.go:149: server-d3ab7084-3ba6-315e-904a-35fe9c9bb531: 2024-06-11T02:29:24.882-1200 [INFO] server-d3ab7084-3ba6-315e-904a-35fe9c9bb531: entering follower state: follower="Node at d3ab7084-3ba6-315e-904a-35fe9c9bb531 [Follower]" leader= -testLoggerAdapter verbose: server-3492aad5-75d6-5de7-b678-44875ee40253: 2024-06-11T02:29:24.882-1200 [INFO] server-3492aad5-75d6-5de7-b678-44875ee40253: entering follower state: follower="Node at 3492aad5-75d6-5de7-b678-44875ee40253 [Follower]" leader= - testing.go:149: server-3492aad5-75d6-5de7-b678-44875ee40253: 2024-06-11T02:29:24.882-1200 [INFO] server-3492aad5-75d6-5de7-b678-44875ee40253: entering follower state: follower="Node at 3492aad5-75d6-5de7-b678-44875ee40253 [Follower]" leader= -testLoggerAdapter verbose: server-64f83319-19a0-f56e-c20c-a978578f0fd3: 2024-06-11T02:29:24.882-1200 [INFO] server-64f83319-19a0-f56e-c20c-a978578f0fd3: entering follower state: follower="Node at 64f83319-19a0-f56e-c20c-a978578f0fd3 [Follower]" leader= - testing.go:149: server-64f83319-19a0-f56e-c20c-a978578f0fd3: 2024-06-11T02:29:24.882-1200 [INFO] server-64f83319-19a0-f56e-c20c-a978578f0fd3: entering follower state: follower="Node at 64f83319-19a0-f56e-c20c-a978578f0fd3 [Follower]" leader= -testLoggerAdapter verbose: server-3492aad5-75d6-5de7-b678-44875ee40253: 2024-06-11T02:29:24.937-1200 [WARN] server-3492aad5-75d6-5de7-b678-44875ee40253: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-3492aad5-75d6-5de7-b678-44875ee40253: 2024-06-11T02:29:24.937-1200 [WARN] server-3492aad5-75d6-5de7-b678-44875ee40253: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-3492aad5-75d6-5de7-b678-44875ee40253: 2024-06-11T02:29:24.937-1200 [INFO] server-3492aad5-75d6-5de7-b678-44875ee40253: entering candidate state: node="Node at 3492aad5-75d6-5de7-b678-44875ee40253 [Candidate]" term=2 - testing.go:149: server-3492aad5-75d6-5de7-b678-44875ee40253: 2024-06-11T02:29:24.937-1200 [INFO] server-3492aad5-75d6-5de7-b678-44875ee40253: entering candidate state: node="Node at 3492aad5-75d6-5de7-b678-44875ee40253 [Candidate]" term=2 -testLoggerAdapter verbose: server-3492aad5-75d6-5de7-b678-44875ee40253: 2024-06-11T02:29:24.938-1200 [INFO] server-3492aad5-75d6-5de7-b678-44875ee40253: election won: tally=2 - testing.go:149: server-3492aad5-75d6-5de7-b678-44875ee40253: 2024-06-11T02:29:24.938-1200 [INFO] server-3492aad5-75d6-5de7-b678-44875ee40253: election won: tally=2 -testLoggerAdapter verbose: server-3492aad5-75d6-5de7-b678-44875ee40253: 2024-06-11T02:29:24.938-1200 [INFO] server-3492aad5-75d6-5de7-b678-44875ee40253: entering leader state: leader="Node at 3492aad5-75d6-5de7-b678-44875ee40253 [Leader]" - testing.go:149: server-3492aad5-75d6-5de7-b678-44875ee40253: 2024-06-11T02:29:24.938-1200 [INFO] server-3492aad5-75d6-5de7-b678-44875ee40253: entering leader state: leader="Node at 3492aad5-75d6-5de7-b678-44875ee40253 [Leader]" -testLoggerAdapter verbose: server-3492aad5-75d6-5de7-b678-44875ee40253: 2024-06-11T02:29:24.938-1200 [INFO] server-3492aad5-75d6-5de7-b678-44875ee40253: added peer, starting replication: peer=server-d3ab7084-3ba6-315e-904a-35fe9c9bb531 - testing.go:149: server-3492aad5-75d6-5de7-b678-44875ee40253: 2024-06-11T02:29:24.938-1200 [INFO] server-3492aad5-75d6-5de7-b678-44875ee40253: added peer, starting replication: peer=server-d3ab7084-3ba6-315e-904a-35fe9c9bb531 -testLoggerAdapter verbose: server-3492aad5-75d6-5de7-b678-44875ee40253: 2024-06-11T02:29:24.938-1200 [INFO] server-3492aad5-75d6-5de7-b678-44875ee40253: added peer, starting replication: peer=server-64f83319-19a0-f56e-c20c-a978578f0fd3 - testing.go:149: server-3492aad5-75d6-5de7-b678-44875ee40253: 2024-06-11T02:29:24.938-1200 [INFO] server-3492aad5-75d6-5de7-b678-44875ee40253: added peer, starting replication: peer=server-64f83319-19a0-f56e-c20c-a978578f0fd3 -testLoggerAdapter verbose: server-3492aad5-75d6-5de7-b678-44875ee40253: 2024-06-11T02:29:24.938-1200 [INFO] server-3492aad5-75d6-5de7-b678-44875ee40253: pipelining replication: peer="{Voter server-64f83319-19a0-f56e-c20c-a978578f0fd3 64f83319-19a0-f56e-c20c-a978578f0fd3}" - testing.go:149: server-3492aad5-75d6-5de7-b678-44875ee40253: 2024-06-11T02:29:24.938-1200 [INFO] server-3492aad5-75d6-5de7-b678-44875ee40253: pipelining replication: peer="{Voter server-64f83319-19a0-f56e-c20c-a978578f0fd3 64f83319-19a0-f56e-c20c-a978578f0fd3}" -testLoggerAdapter verbose: server-3492aad5-75d6-5de7-b678-44875ee40253: 2024-06-11T02:29:24.938-1200 [INFO] server-3492aad5-75d6-5de7-b678-44875ee40253: pipelining replication: peer="{Voter server-d3ab7084-3ba6-315e-904a-35fe9c9bb531 d3ab7084-3ba6-315e-904a-35fe9c9bb531}" - testing.go:149: server-3492aad5-75d6-5de7-b678-44875ee40253: 2024-06-11T02:29:24.938-1200 [INFO] server-3492aad5-75d6-5de7-b678-44875ee40253: pipelining replication: peer="{Voter server-d3ab7084-3ba6-315e-904a-35fe9c9bb531 d3ab7084-3ba6-315e-904a-35fe9c9bb531}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:25.044-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:24.938973411 -1200 -12 m=+15.934345953 (1 nodes), 56.750557ms from start of poll, 57.931785ms from cluster start. Timeout at 2024-06-11 02:29:25.044506051 -1200 -12 m=+16.039878884, 105.532931ms after stability - testing.go:149: cluster: 2024-06-11T02:29:25.044-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:24.938973411 -1200 -12 m=+15.934345953 (1 nodes), 56.750557ms from start of poll, 57.931785ms from cluster start. Timeout at 2024-06-11 02:29:25.044506051 -1200 -12 m=+16.039878884, 105.532931ms after stability -testLoggerAdapter verbose: server-3492aad5-75d6-5de7-b678-44875ee40253: 2024-06-11T02:29:25.044-1200 [INFO] server-3492aad5-75d6-5de7-b678-44875ee40253: aborting pipeline replication: peer="{Voter server-64f83319-19a0-f56e-c20c-a978578f0fd3 64f83319-19a0-f56e-c20c-a978578f0fd3}" - testing.go:149: server-3492aad5-75d6-5de7-b678-44875ee40253: 2024-06-11T02:29:25.044-1200 [INFO] server-3492aad5-75d6-5de7-b678-44875ee40253: aborting pipeline replication: peer="{Voter server-64f83319-19a0-f56e-c20c-a978578f0fd3 64f83319-19a0-f56e-c20c-a978578f0fd3}" -testLoggerAdapter verbose: server-3492aad5-75d6-5de7-b678-44875ee40253: 2024-06-11T02:29:25.045-1200 [INFO] server-3492aad5-75d6-5de7-b678-44875ee40253: aborting pipeline replication: peer="{Voter server-d3ab7084-3ba6-315e-904a-35fe9c9bb531 d3ab7084-3ba6-315e-904a-35fe9c9bb531}" - testing.go:149: server-3492aad5-75d6-5de7-b678-44875ee40253: 2024-06-11T02:29:25.045-1200 [INFO] server-3492aad5-75d6-5de7-b678-44875ee40253: aborting pipeline replication: peer="{Voter server-d3ab7084-3ba6-315e-904a-35fe9c9bb531 d3ab7084-3ba6-315e-904a-35fe9c9bb531}" -testLoggerAdapter verbose: server-3492aad5-75d6-5de7-b678-44875ee40253: 2024-06-11T02:29:25.095-1200 [ERROR] server-3492aad5-75d6-5de7-b678-44875ee40253: failed to heartbeat to: peer=d3ab7084-3ba6-315e-904a-35fe9c9bb531 error="command timed out" - testing.go:149: server-3492aad5-75d6-5de7-b678-44875ee40253: 2024-06-11T02:29:25.095-1200 [ERROR] server-3492aad5-75d6-5de7-b678-44875ee40253: failed to heartbeat to: peer=d3ab7084-3ba6-315e-904a-35fe9c9bb531 error="command timed out" ---- PASS: TestRaft_VerifyLeader (0.22s) -=== RUN TestRaft_VerifyLeader_Single -testLoggerAdapter verbose: server-26e8ec73-4d7a-d64b-6131-51bf035d8ff6: 2024-06-11T02:29:25.098-1200 [INFO] server-26e8ec73-4d7a-d64b-6131-51bf035d8ff6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-26e8ec73-4d7a-d64b-6131-51bf035d8ff6 Address:26e8ec73-4d7a-d64b-6131-51bf035d8ff6}]" - testing.go:149: server-26e8ec73-4d7a-d64b-6131-51bf035d8ff6: 2024-06-11T02:29:25.098-1200 [INFO] server-26e8ec73-4d7a-d64b-6131-51bf035d8ff6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-26e8ec73-4d7a-d64b-6131-51bf035d8ff6 Address:26e8ec73-4d7a-d64b-6131-51bf035d8ff6}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:25.099-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:25.099-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-26e8ec73-4d7a-d64b-6131-51bf035d8ff6: 2024-06-11T02:29:25.099-1200 [INFO] server-26e8ec73-4d7a-d64b-6131-51bf035d8ff6: entering follower state: follower="Node at 26e8ec73-4d7a-d64b-6131-51bf035d8ff6 [Follower]" leader= - testing.go:149: server-26e8ec73-4d7a-d64b-6131-51bf035d8ff6: 2024-06-11T02:29:25.099-1200 [INFO] server-26e8ec73-4d7a-d64b-6131-51bf035d8ff6: entering follower state: follower="Node at 26e8ec73-4d7a-d64b-6131-51bf035d8ff6 [Follower]" leader= -testLoggerAdapter verbose: server-26e8ec73-4d7a-d64b-6131-51bf035d8ff6: 2024-06-11T02:29:25.187-1200 [WARN] server-26e8ec73-4d7a-d64b-6131-51bf035d8ff6: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-26e8ec73-4d7a-d64b-6131-51bf035d8ff6: 2024-06-11T02:29:25.187-1200 [WARN] server-26e8ec73-4d7a-d64b-6131-51bf035d8ff6: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-26e8ec73-4d7a-d64b-6131-51bf035d8ff6: 2024-06-11T02:29:25.187-1200 [INFO] server-26e8ec73-4d7a-d64b-6131-51bf035d8ff6: entering candidate state: node="Node at 26e8ec73-4d7a-d64b-6131-51bf035d8ff6 [Candidate]" term=2 - testing.go:149: server-26e8ec73-4d7a-d64b-6131-51bf035d8ff6: 2024-06-11T02:29:25.187-1200 [INFO] server-26e8ec73-4d7a-d64b-6131-51bf035d8ff6: entering candidate state: node="Node at 26e8ec73-4d7a-d64b-6131-51bf035d8ff6 [Candidate]" term=2 -testLoggerAdapter verbose: server-26e8ec73-4d7a-d64b-6131-51bf035d8ff6: 2024-06-11T02:29:25.187-1200 [INFO] server-26e8ec73-4d7a-d64b-6131-51bf035d8ff6: election won: tally=1 - testing.go:149: server-26e8ec73-4d7a-d64b-6131-51bf035d8ff6: 2024-06-11T02:29:25.187-1200 [INFO] server-26e8ec73-4d7a-d64b-6131-51bf035d8ff6: election won: tally=1 -testLoggerAdapter verbose: server-26e8ec73-4d7a-d64b-6131-51bf035d8ff6: 2024-06-11T02:29:25.187-1200 [INFO] server-26e8ec73-4d7a-d64b-6131-51bf035d8ff6: entering leader state: leader="Node at 26e8ec73-4d7a-d64b-6131-51bf035d8ff6 [Leader]" - testing.go:149: server-26e8ec73-4d7a-d64b-6131-51bf035d8ff6: 2024-06-11T02:29:25.187-1200 [INFO] server-26e8ec73-4d7a-d64b-6131-51bf035d8ff6: entering leader state: leader="Node at 26e8ec73-4d7a-d64b-6131-51bf035d8ff6 [Leader]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:25.292-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:25.187672608 -1200 -12 m=+16.183045150 (1 nodes), 88.543085ms from start of poll, 89.080033ms from cluster start. Timeout at 2024-06-11 02:29:25.292698924 -1200 -12 m=+16.288071466, 105.026316ms after stability - testing.go:149: cluster: 2024-06-11T02:29:25.292-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:25.187672608 -1200 -12 m=+16.183045150 (1 nodes), 88.543085ms from start of poll, 89.080033ms from cluster start. Timeout at 2024-06-11 02:29:25.292698924 -1200 -12 m=+16.288071466, 105.026316ms after stability ---- PASS: TestRaft_VerifyLeader_Single (0.20s) -=== RUN TestRaft_VerifyLeader_Fail -testLoggerAdapter verbose: server-9fb74d7a-67a2-5c1b-9a2d-4bb3eda8054c: 2024-06-11T02:29:25.295-1200 [INFO] server-9fb74d7a-67a2-5c1b-9a2d-4bb3eda8054c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9fb74d7a-67a2-5c1b-9a2d-4bb3eda8054c Address:9fb74d7a-67a2-5c1b-9a2d-4bb3eda8054c} {Suffrage:Voter ID:server-ad72fb59-2c84-6abc-408c-f57e9a3016d3 Address:ad72fb59-2c84-6abc-408c-f57e9a3016d3}]" - testing.go:149: server-9fb74d7a-67a2-5c1b-9a2d-4bb3eda8054c: 2024-06-11T02:29:25.295-1200 [INFO] server-9fb74d7a-67a2-5c1b-9a2d-4bb3eda8054c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9fb74d7a-67a2-5c1b-9a2d-4bb3eda8054c Address:9fb74d7a-67a2-5c1b-9a2d-4bb3eda8054c} {Suffrage:Voter ID:server-ad72fb59-2c84-6abc-408c-f57e9a3016d3 Address:ad72fb59-2c84-6abc-408c-f57e9a3016d3}]" -testLoggerAdapter verbose: server-9fb74d7a-67a2-5c1b-9a2d-4bb3eda8054c: 2024-06-11T02:29:25.296-1200 [INFO] server-9fb74d7a-67a2-5c1b-9a2d-4bb3eda8054c: entering follower state: follower="Node at 9fb74d7a-67a2-5c1b-9a2d-4bb3eda8054c [Follower]" leader= - testing.go:149: server-9fb74d7a-67a2-5c1b-9a2d-4bb3eda8054c: 2024-06-11T02:29:25.296-1200 [INFO] server-9fb74d7a-67a2-5c1b-9a2d-4bb3eda8054c: entering follower state: follower="Node at 9fb74d7a-67a2-5c1b-9a2d-4bb3eda8054c [Follower]" leader= -testLoggerAdapter verbose: server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: 2024-06-11T02:29:25.296-1200 [INFO] server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9fb74d7a-67a2-5c1b-9a2d-4bb3eda8054c Address:9fb74d7a-67a2-5c1b-9a2d-4bb3eda8054c} {Suffrage:Voter ID:server-ad72fb59-2c84-6abc-408c-f57e9a3016d3 Address:ad72fb59-2c84-6abc-408c-f57e9a3016d3}]" - testing.go:149: server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: 2024-06-11T02:29:25.296-1200 [INFO] server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9fb74d7a-67a2-5c1b-9a2d-4bb3eda8054c Address:9fb74d7a-67a2-5c1b-9a2d-4bb3eda8054c} {Suffrage:Voter ID:server-ad72fb59-2c84-6abc-408c-f57e9a3016d3 Address:ad72fb59-2c84-6abc-408c-f57e9a3016d3}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:25.296-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:25.296-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: 2024-06-11T02:29:25.296-1200 [INFO] server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: entering follower state: follower="Node at ad72fb59-2c84-6abc-408c-f57e9a3016d3 [Follower]" leader= - testing.go:149: server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: 2024-06-11T02:29:25.296-1200 [INFO] server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: entering follower state: follower="Node at ad72fb59-2c84-6abc-408c-f57e9a3016d3 [Follower]" leader= -testLoggerAdapter verbose: server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: 2024-06-11T02:29:25.378-1200 [WARN] server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: 2024-06-11T02:29:25.378-1200 [WARN] server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: 2024-06-11T02:29:25.378-1200 [INFO] server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: entering candidate state: node="Node at ad72fb59-2c84-6abc-408c-f57e9a3016d3 [Candidate]" term=2 - testing.go:149: server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: 2024-06-11T02:29:25.378-1200 [INFO] server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: entering candidate state: node="Node at ad72fb59-2c84-6abc-408c-f57e9a3016d3 [Candidate]" term=2 -testLoggerAdapter verbose: server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: 2024-06-11T02:29:25.378-1200 [INFO] server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: election won: tally=2 - testing.go:149: server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: 2024-06-11T02:29:25.378-1200 [INFO] server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: election won: tally=2 -testLoggerAdapter verbose: server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: 2024-06-11T02:29:25.378-1200 [INFO] server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: entering leader state: leader="Node at ad72fb59-2c84-6abc-408c-f57e9a3016d3 [Leader]" - testing.go:149: server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: 2024-06-11T02:29:25.378-1200 [INFO] server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: entering leader state: leader="Node at ad72fb59-2c84-6abc-408c-f57e9a3016d3 [Leader]" -testLoggerAdapter verbose: server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: 2024-06-11T02:29:25.378-1200 [INFO] server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: added peer, starting replication: peer=server-9fb74d7a-67a2-5c1b-9a2d-4bb3eda8054c - testing.go:149: server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: 2024-06-11T02:29:25.378-1200 [INFO] server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: added peer, starting replication: peer=server-9fb74d7a-67a2-5c1b-9a2d-4bb3eda8054c -testLoggerAdapter verbose: server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: 2024-06-11T02:29:25.378-1200 [INFO] server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: pipelining replication: peer="{Voter server-9fb74d7a-67a2-5c1b-9a2d-4bb3eda8054c 9fb74d7a-67a2-5c1b-9a2d-4bb3eda8054c}" - testing.go:149: server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: 2024-06-11T02:29:25.378-1200 [INFO] server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: pipelining replication: peer="{Voter server-9fb74d7a-67a2-5c1b-9a2d-4bb3eda8054c 9fb74d7a-67a2-5c1b-9a2d-4bb3eda8054c}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:25.485-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:25.379192214 -1200 -12 m=+16.374564755 (1 nodes), 82.699069ms from start of poll, 83.583094ms from cluster start. Timeout at 2024-06-11 02:29:25.485242552 -1200 -12 m=+16.480615094, 106.050339ms after stability - testing.go:149: cluster: 2024-06-11T02:29:25.485-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:25.379192214 -1200 -12 m=+16.374564755 (1 nodes), 82.699069ms from start of poll, 83.583094ms from cluster start. Timeout at 2024-06-11 02:29:25.485242552 -1200 -12 m=+16.480615094, 106.050339ms after stability -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:25.485-1200 [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2024-06-11T02:29:25.485-1200 [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:25.591-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:25.485500089 -1200 -12 m=+16.480872631 (1 nodes), 2.334µs from start of poll, 189.89097ms from cluster start. Timeout at 2024-06-11 02:29:25.591034479 -1200 -12 m=+16.586407020, 105.534389ms after stability - testing.go:149: cluster: 2024-06-11T02:29:25.591-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:25.485500089 -1200 -12 m=+16.480872631 (1 nodes), 2.334µs from start of poll, 189.89097ms from cluster start. Timeout at 2024-06-11 02:29:25.591034479 -1200 -12 m=+16.586407020, 105.534389ms after stability -testLoggerAdapter verbose: server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: 2024-06-11T02:29:25.591-1200 [WARN] server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: new leader elected, stepping down - testing.go:149: server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: 2024-06-11T02:29:25.591-1200 [WARN] server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: new leader elected, stepping down -testLoggerAdapter verbose: server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: 2024-06-11T02:29:25.591-1200 [INFO] server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: entering follower state: follower="Node at ad72fb59-2c84-6abc-408c-f57e9a3016d3 [Follower]" leader= - testing.go:149: server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: 2024-06-11T02:29:25.591-1200 [INFO] server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: entering follower state: follower="Node at ad72fb59-2c84-6abc-408c-f57e9a3016d3 [Follower]" leader= -testLoggerAdapter verbose: server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: 2024-06-11T02:29:25.591-1200 [INFO] server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: aborting pipeline replication: peer="{Voter server-9fb74d7a-67a2-5c1b-9a2d-4bb3eda8054c 9fb74d7a-67a2-5c1b-9a2d-4bb3eda8054c}" - testing.go:149: server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: 2024-06-11T02:29:25.591-1200 [INFO] server-ad72fb59-2c84-6abc-408c-f57e9a3016d3: aborting pipeline replication: peer="{Voter server-9fb74d7a-67a2-5c1b-9a2d-4bb3eda8054c 9fb74d7a-67a2-5c1b-9a2d-4bb3eda8054c}" ---- PASS: TestRaft_VerifyLeader_Fail (0.30s) -=== RUN TestRaft_VerifyLeader_PartialConnect -testLoggerAdapter verbose: server-7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5: 2024-06-11T02:29:25.595-1200 [INFO] server-7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5 Address:7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5} {Suffrage:Voter ID:server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b Address:fd0e0067-04d6-ebd6-24fe-a098b5c30a2b} {Suffrage:Voter ID:server-2892e42a-17de-0908-9259-389048b128ec Address:2892e42a-17de-0908-9259-389048b128ec}]" - testing.go:149: server-7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5: 2024-06-11T02:29:25.595-1200 [INFO] server-7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5 Address:7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5} {Suffrage:Voter ID:server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b Address:fd0e0067-04d6-ebd6-24fe-a098b5c30a2b} {Suffrage:Voter ID:server-2892e42a-17de-0908-9259-389048b128ec Address:2892e42a-17de-0908-9259-389048b128ec}]" -testLoggerAdapter verbose: server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: 2024-06-11T02:29:25.596-1200 [INFO] server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5 Address:7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5} {Suffrage:Voter ID:server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b Address:fd0e0067-04d6-ebd6-24fe-a098b5c30a2b} {Suffrage:Voter ID:server-2892e42a-17de-0908-9259-389048b128ec Address:2892e42a-17de-0908-9259-389048b128ec}]" - testing.go:149: server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: 2024-06-11T02:29:25.596-1200 [INFO] server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5 Address:7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5} {Suffrage:Voter ID:server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b Address:fd0e0067-04d6-ebd6-24fe-a098b5c30a2b} {Suffrage:Voter ID:server-2892e42a-17de-0908-9259-389048b128ec Address:2892e42a-17de-0908-9259-389048b128ec}]" -testLoggerAdapter verbose: server-2892e42a-17de-0908-9259-389048b128ec: 2024-06-11T02:29:25.596-1200 [INFO] server-2892e42a-17de-0908-9259-389048b128ec: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5 Address:7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5} {Suffrage:Voter ID:server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b Address:fd0e0067-04d6-ebd6-24fe-a098b5c30a2b} {Suffrage:Voter ID:server-2892e42a-17de-0908-9259-389048b128ec Address:2892e42a-17de-0908-9259-389048b128ec}]" - testing.go:149: server-2892e42a-17de-0908-9259-389048b128ec: 2024-06-11T02:29:25.596-1200 [INFO] server-2892e42a-17de-0908-9259-389048b128ec: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5 Address:7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5} {Suffrage:Voter ID:server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b Address:fd0e0067-04d6-ebd6-24fe-a098b5c30a2b} {Suffrage:Voter ID:server-2892e42a-17de-0908-9259-389048b128ec Address:2892e42a-17de-0908-9259-389048b128ec}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:25.596-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:25.596-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5: 2024-06-11T02:29:25.596-1200 [INFO] server-7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5: entering follower state: follower="Node at 7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5 [Follower]" leader= - testing.go:149: server-7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5: 2024-06-11T02:29:25.596-1200 [INFO] server-7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5: entering follower state: follower="Node at 7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5 [Follower]" leader= -testLoggerAdapter verbose: server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: 2024-06-11T02:29:25.596-1200 [INFO] server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: entering follower state: follower="Node at fd0e0067-04d6-ebd6-24fe-a098b5c30a2b [Follower]" leader= - testing.go:149: server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: 2024-06-11T02:29:25.596-1200 [INFO] server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: entering follower state: follower="Node at fd0e0067-04d6-ebd6-24fe-a098b5c30a2b [Follower]" leader= -testLoggerAdapter verbose: server-2892e42a-17de-0908-9259-389048b128ec: 2024-06-11T02:29:25.596-1200 [INFO] server-2892e42a-17de-0908-9259-389048b128ec: entering follower state: follower="Node at 2892e42a-17de-0908-9259-389048b128ec [Follower]" leader= - testing.go:149: server-2892e42a-17de-0908-9259-389048b128ec: 2024-06-11T02:29:25.596-1200 [INFO] server-2892e42a-17de-0908-9259-389048b128ec: entering follower state: follower="Node at 2892e42a-17de-0908-9259-389048b128ec [Follower]" leader= -testLoggerAdapter verbose: server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: 2024-06-11T02:29:25.652-1200 [WARN] server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: 2024-06-11T02:29:25.652-1200 [WARN] server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: 2024-06-11T02:29:25.652-1200 [INFO] server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: entering candidate state: node="Node at fd0e0067-04d6-ebd6-24fe-a098b5c30a2b [Candidate]" term=2 - testing.go:149: server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: 2024-06-11T02:29:25.652-1200 [INFO] server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: entering candidate state: node="Node at fd0e0067-04d6-ebd6-24fe-a098b5c30a2b [Candidate]" term=2 -testLoggerAdapter verbose: server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: 2024-06-11T02:29:25.652-1200 [INFO] server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: election won: tally=2 - testing.go:149: server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: 2024-06-11T02:29:25.652-1200 [INFO] server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: election won: tally=2 -testLoggerAdapter verbose: server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: 2024-06-11T02:29:25.652-1200 [INFO] server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: entering leader state: leader="Node at fd0e0067-04d6-ebd6-24fe-a098b5c30a2b [Leader]" - testing.go:149: server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: 2024-06-11T02:29:25.652-1200 [INFO] server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: entering leader state: leader="Node at fd0e0067-04d6-ebd6-24fe-a098b5c30a2b [Leader]" -testLoggerAdapter verbose: server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: 2024-06-11T02:29:25.652-1200 [INFO] server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: added peer, starting replication: peer=server-7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5 - testing.go:149: server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: 2024-06-11T02:29:25.652-1200 [INFO] server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: added peer, starting replication: peer=server-7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5 -testLoggerAdapter verbose: server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: 2024-06-11T02:29:25.653-1200 [INFO] server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: added peer, starting replication: peer=server-2892e42a-17de-0908-9259-389048b128ec - testing.go:149: server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: 2024-06-11T02:29:25.653-1200 [INFO] server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: added peer, starting replication: peer=server-2892e42a-17de-0908-9259-389048b128ec -testLoggerAdapter verbose: server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: 2024-06-11T02:29:25.653-1200 [INFO] server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: pipelining replication: peer="{Voter server-2892e42a-17de-0908-9259-389048b128ec 2892e42a-17de-0908-9259-389048b128ec}" - testing.go:149: server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: 2024-06-11T02:29:25.653-1200 [INFO] server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: pipelining replication: peer="{Voter server-2892e42a-17de-0908-9259-389048b128ec 2892e42a-17de-0908-9259-389048b128ec}" -testLoggerAdapter verbose: server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: 2024-06-11T02:29:25.653-1200 [INFO] server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: pipelining replication: peer="{Voter server-7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5 7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5}" - testing.go:149: server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: 2024-06-11T02:29:25.653-1200 [INFO] server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: pipelining replication: peer="{Voter server-7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5 7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:25.759-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:25.654289081 -1200 -12 m=+16.649661622 (1 nodes), 57.646249ms from start of poll, 59.006557ms from cluster start. Timeout at 2024-06-11 02:29:25.759301688 -1200 -12 m=+16.754674230, 105.012608ms after stability - testing.go:149: cluster: 2024-06-11T02:29:25.759-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:25.654289081 -1200 -12 m=+16.649661622 (1 nodes), 57.646249ms from start of poll, 59.006557ms from cluster start. Timeout at 2024-06-11 02:29:25.759301688 -1200 -12 m=+16.754674230, 105.012608ms after stability -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:25.765-1200 [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2024-06-11T02:29:25.765-1200 [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:25.870-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:25.765168745 -1200 -12 m=+16.760541286 (2 nodes), 3.5µs from start of poll, 169.886221ms from cluster start. Timeout at 2024-06-11 02:29:25.870875506 -1200 -12 m=+16.866248048, 105.706762ms after stability - testing.go:149: cluster: 2024-06-11T02:29:25.870-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:25.765168745 -1200 -12 m=+16.760541286 (2 nodes), 3.5µs from start of poll, 169.886221ms from cluster start. Timeout at 2024-06-11 02:29:25.870875506 -1200 -12 m=+16.866248048, 105.706762ms after stability - raft_test.go:1647: [INFO] Disconnecting Node at 7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5 [Follower] -testLoggerAdapter verbose: server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: 2024-06-11T02:29:25.871-1200 [ERROR] server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: failed to heartbeat to: peer=7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5 error="failed to connect to peer: 7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5" - testing.go:149: server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: 2024-06-11T02:29:25.871-1200 [ERROR] server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: failed to heartbeat to: peer=7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5 error="failed to connect to peer: 7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5" -testLoggerAdapter verbose: server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: 2024-06-11T02:29:25.871-1200 [INFO] server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: aborting pipeline replication: peer="{Voter server-7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5 7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5}" - testing.go:149: server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: 2024-06-11T02:29:25.871-1200 [INFO] server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: aborting pipeline replication: peer="{Voter server-7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5 7b7f43d9-d9f9-6f99-6fe1-768dae3a9ea5}" -testLoggerAdapter verbose: server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: 2024-06-11T02:29:25.871-1200 [INFO] server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: aborting pipeline replication: peer="{Voter server-2892e42a-17de-0908-9259-389048b128ec 2892e42a-17de-0908-9259-389048b128ec}" - testing.go:149: server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: 2024-06-11T02:29:25.871-1200 [INFO] server-fd0e0067-04d6-ebd6-24fe-a098b5c30a2b: aborting pipeline replication: peer="{Voter server-2892e42a-17de-0908-9259-389048b128ec 2892e42a-17de-0908-9259-389048b128ec}" ---- PASS: TestRaft_VerifyLeader_PartialConnect (0.28s) -=== RUN TestRaft_StartAsLeader -testLoggerAdapter verbose: server-21fa9e1e-2f3d-623e-81b5-8ef4986041ea: 2024-06-11T02:29:25.874-1200 [INFO] server-21fa9e1e-2f3d-623e-81b5-8ef4986041ea: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-21fa9e1e-2f3d-623e-81b5-8ef4986041ea Address:21fa9e1e-2f3d-623e-81b5-8ef4986041ea}]" - testing.go:149: server-21fa9e1e-2f3d-623e-81b5-8ef4986041ea: 2024-06-11T02:29:25.874-1200 [INFO] server-21fa9e1e-2f3d-623e-81b5-8ef4986041ea: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-21fa9e1e-2f3d-623e-81b5-8ef4986041ea Address:21fa9e1e-2f3d-623e-81b5-8ef4986041ea}]" -testLoggerAdapter verbose: server-21fa9e1e-2f3d-623e-81b5-8ef4986041ea: 2024-06-11T02:29:25.874-1200 [INFO] server-21fa9e1e-2f3d-623e-81b5-8ef4986041ea: entering leader state: leader="Node at 21fa9e1e-2f3d-623e-81b5-8ef4986041ea [Leader]" - testing.go:149: server-21fa9e1e-2f3d-623e-81b5-8ef4986041ea: 2024-06-11T02:29:25.874-1200 [INFO] server-21fa9e1e-2f3d-623e-81b5-8ef4986041ea: entering leader state: leader="Node at 21fa9e1e-2f3d-623e-81b5-8ef4986041ea [Leader]" ---- PASS: TestRaft_StartAsLeader (0.00s) -=== RUN TestRaft_NotifyCh -testLoggerAdapter verbose: server-e018141d-cad0-f5b3-882d-7cd0b7824d7f: 2024-06-11T02:29:25.876-1200 [INFO] server-e018141d-cad0-f5b3-882d-7cd0b7824d7f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e018141d-cad0-f5b3-882d-7cd0b7824d7f Address:e018141d-cad0-f5b3-882d-7cd0b7824d7f}]" - testing.go:149: server-e018141d-cad0-f5b3-882d-7cd0b7824d7f: 2024-06-11T02:29:25.876-1200 [INFO] server-e018141d-cad0-f5b3-882d-7cd0b7824d7f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e018141d-cad0-f5b3-882d-7cd0b7824d7f Address:e018141d-cad0-f5b3-882d-7cd0b7824d7f}]" -testLoggerAdapter verbose: server-e018141d-cad0-f5b3-882d-7cd0b7824d7f: 2024-06-11T02:29:25.876-1200 [INFO] server-e018141d-cad0-f5b3-882d-7cd0b7824d7f: entering follower state: follower="Node at e018141d-cad0-f5b3-882d-7cd0b7824d7f [Follower]" leader= - testing.go:149: server-e018141d-cad0-f5b3-882d-7cd0b7824d7f: 2024-06-11T02:29:25.876-1200 [INFO] server-e018141d-cad0-f5b3-882d-7cd0b7824d7f: entering follower state: follower="Node at e018141d-cad0-f5b3-882d-7cd0b7824d7f [Follower]" leader= -testLoggerAdapter verbose: server-e018141d-cad0-f5b3-882d-7cd0b7824d7f: 2024-06-11T02:29:25.961-1200 [WARN] server-e018141d-cad0-f5b3-882d-7cd0b7824d7f: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-e018141d-cad0-f5b3-882d-7cd0b7824d7f: 2024-06-11T02:29:25.961-1200 [WARN] server-e018141d-cad0-f5b3-882d-7cd0b7824d7f: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-e018141d-cad0-f5b3-882d-7cd0b7824d7f: 2024-06-11T02:29:25.961-1200 [INFO] server-e018141d-cad0-f5b3-882d-7cd0b7824d7f: entering candidate state: node="Node at e018141d-cad0-f5b3-882d-7cd0b7824d7f [Candidate]" term=2 - testing.go:149: server-e018141d-cad0-f5b3-882d-7cd0b7824d7f: 2024-06-11T02:29:25.961-1200 [INFO] server-e018141d-cad0-f5b3-882d-7cd0b7824d7f: entering candidate state: node="Node at e018141d-cad0-f5b3-882d-7cd0b7824d7f [Candidate]" term=2 -testLoggerAdapter verbose: server-e018141d-cad0-f5b3-882d-7cd0b7824d7f: 2024-06-11T02:29:25.961-1200 [INFO] server-e018141d-cad0-f5b3-882d-7cd0b7824d7f: election won: tally=1 - testing.go:149: server-e018141d-cad0-f5b3-882d-7cd0b7824d7f: 2024-06-11T02:29:25.961-1200 [INFO] server-e018141d-cad0-f5b3-882d-7cd0b7824d7f: election won: tally=1 -testLoggerAdapter verbose: server-e018141d-cad0-f5b3-882d-7cd0b7824d7f: 2024-06-11T02:29:25.961-1200 [INFO] server-e018141d-cad0-f5b3-882d-7cd0b7824d7f: entering leader state: leader="Node at e018141d-cad0-f5b3-882d-7cd0b7824d7f [Leader]" - testing.go:149: server-e018141d-cad0-f5b3-882d-7cd0b7824d7f: 2024-06-11T02:29:25.961-1200 [INFO] server-e018141d-cad0-f5b3-882d-7cd0b7824d7f: entering leader state: leader="Node at e018141d-cad0-f5b3-882d-7cd0b7824d7f [Leader]" ---- PASS: TestRaft_NotifyCh (0.09s) -=== RUN TestRaft_Voting -testLoggerAdapter verbose: server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: 2024-06-11T02:29:25.965-1200 [INFO] server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78 Address:8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78} {Suffrage:Voter ID:server-f5f4b281-1a4c-c67a-2c69-010813cdfa70 Address:f5f4b281-1a4c-c67a-2c69-010813cdfa70} {Suffrage:Voter ID:server-df4592b3-6179-0097-2b74-a44d36f44fc2 Address:df4592b3-6179-0097-2b74-a44d36f44fc2}]" - testing.go:149: server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: 2024-06-11T02:29:25.965-1200 [INFO] server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78 Address:8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78} {Suffrage:Voter ID:server-f5f4b281-1a4c-c67a-2c69-010813cdfa70 Address:f5f4b281-1a4c-c67a-2c69-010813cdfa70} {Suffrage:Voter ID:server-df4592b3-6179-0097-2b74-a44d36f44fc2 Address:df4592b3-6179-0097-2b74-a44d36f44fc2}]" -testLoggerAdapter verbose: server-f5f4b281-1a4c-c67a-2c69-010813cdfa70: 2024-06-11T02:29:25.966-1200 [INFO] server-f5f4b281-1a4c-c67a-2c69-010813cdfa70: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78 Address:8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78} {Suffrage:Voter ID:server-f5f4b281-1a4c-c67a-2c69-010813cdfa70 Address:f5f4b281-1a4c-c67a-2c69-010813cdfa70} {Suffrage:Voter ID:server-df4592b3-6179-0097-2b74-a44d36f44fc2 Address:df4592b3-6179-0097-2b74-a44d36f44fc2}]" - testing.go:149: server-f5f4b281-1a4c-c67a-2c69-010813cdfa70: 2024-06-11T02:29:25.966-1200 [INFO] server-f5f4b281-1a4c-c67a-2c69-010813cdfa70: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78 Address:8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78} {Suffrage:Voter ID:server-f5f4b281-1a4c-c67a-2c69-010813cdfa70 Address:f5f4b281-1a4c-c67a-2c69-010813cdfa70} {Suffrage:Voter ID:server-df4592b3-6179-0097-2b74-a44d36f44fc2 Address:df4592b3-6179-0097-2b74-a44d36f44fc2}]" -testLoggerAdapter verbose: server-df4592b3-6179-0097-2b74-a44d36f44fc2: 2024-06-11T02:29:25.966-1200 [INFO] server-df4592b3-6179-0097-2b74-a44d36f44fc2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78 Address:8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78} {Suffrage:Voter ID:server-f5f4b281-1a4c-c67a-2c69-010813cdfa70 Address:f5f4b281-1a4c-c67a-2c69-010813cdfa70} {Suffrage:Voter ID:server-df4592b3-6179-0097-2b74-a44d36f44fc2 Address:df4592b3-6179-0097-2b74-a44d36f44fc2}]" - testing.go:149: server-df4592b3-6179-0097-2b74-a44d36f44fc2: 2024-06-11T02:29:25.966-1200 [INFO] server-df4592b3-6179-0097-2b74-a44d36f44fc2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78 Address:8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78} {Suffrage:Voter ID:server-f5f4b281-1a4c-c67a-2c69-010813cdfa70 Address:f5f4b281-1a4c-c67a-2c69-010813cdfa70} {Suffrage:Voter ID:server-df4592b3-6179-0097-2b74-a44d36f44fc2 Address:df4592b3-6179-0097-2b74-a44d36f44fc2}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:25.966-1200 [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2024-06-11T02:29:25.966-1200 [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: 2024-06-11T02:29:25.966-1200 [INFO] server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: entering follower state: follower="Node at 8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78 [Follower]" leader= - testing.go:149: server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: 2024-06-11T02:29:25.966-1200 [INFO] server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: entering follower state: follower="Node at 8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78 [Follower]" leader= -testLoggerAdapter verbose: server-f5f4b281-1a4c-c67a-2c69-010813cdfa70: 2024-06-11T02:29:25.966-1200 [INFO] server-f5f4b281-1a4c-c67a-2c69-010813cdfa70: entering follower state: follower="Node at f5f4b281-1a4c-c67a-2c69-010813cdfa70 [Follower]" leader= - testing.go:149: server-f5f4b281-1a4c-c67a-2c69-010813cdfa70: 2024-06-11T02:29:25.966-1200 [INFO] server-f5f4b281-1a4c-c67a-2c69-010813cdfa70: entering follower state: follower="Node at f5f4b281-1a4c-c67a-2c69-010813cdfa70 [Follower]" leader= -testLoggerAdapter verbose: server-df4592b3-6179-0097-2b74-a44d36f44fc2: 2024-06-11T02:29:25.967-1200 [INFO] server-df4592b3-6179-0097-2b74-a44d36f44fc2: entering follower state: follower="Node at df4592b3-6179-0097-2b74-a44d36f44fc2 [Follower]" leader= - testing.go:149: server-df4592b3-6179-0097-2b74-a44d36f44fc2: 2024-06-11T02:29:25.967-1200 [INFO] server-df4592b3-6179-0097-2b74-a44d36f44fc2: entering follower state: follower="Node at df4592b3-6179-0097-2b74-a44d36f44fc2 [Follower]" leader= -testLoggerAdapter verbose: server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: 2024-06-11T02:29:26.026-1200 [WARN] server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: 2024-06-11T02:29:26.026-1200 [WARN] server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: 2024-06-11T02:29:26.026-1200 [INFO] server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: entering candidate state: node="Node at 8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78 [Candidate]" term=2 - testing.go:149: server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: 2024-06-11T02:29:26.026-1200 [INFO] server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: entering candidate state: node="Node at 8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78 [Candidate]" term=2 -testLoggerAdapter verbose: server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: 2024-06-11T02:29:26.026-1200 [INFO] server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: election won: tally=2 - testing.go:149: server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: 2024-06-11T02:29:26.026-1200 [INFO] server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: election won: tally=2 -testLoggerAdapter verbose: server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: 2024-06-11T02:29:26.026-1200 [INFO] server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: entering leader state: leader="Node at 8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78 [Leader]" - testing.go:149: server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: 2024-06-11T02:29:26.026-1200 [INFO] server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: entering leader state: leader="Node at 8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78 [Leader]" -testLoggerAdapter verbose: server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: 2024-06-11T02:29:26.026-1200 [INFO] server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: added peer, starting replication: peer=server-f5f4b281-1a4c-c67a-2c69-010813cdfa70 - testing.go:149: server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: 2024-06-11T02:29:26.026-1200 [INFO] server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: added peer, starting replication: peer=server-f5f4b281-1a4c-c67a-2c69-010813cdfa70 -testLoggerAdapter verbose: server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: 2024-06-11T02:29:26.026-1200 [INFO] server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: added peer, starting replication: peer=server-df4592b3-6179-0097-2b74-a44d36f44fc2 - testing.go:149: server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: 2024-06-11T02:29:26.026-1200 [INFO] server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: added peer, starting replication: peer=server-df4592b3-6179-0097-2b74-a44d36f44fc2 -testLoggerAdapter verbose: server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: 2024-06-11T02:29:26.027-1200 [INFO] server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: pipelining replication: peer="{Voter server-df4592b3-6179-0097-2b74-a44d36f44fc2 df4592b3-6179-0097-2b74-a44d36f44fc2}" - testing.go:149: server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: 2024-06-11T02:29:26.027-1200 [INFO] server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: pipelining replication: peer="{Voter server-df4592b3-6179-0097-2b74-a44d36f44fc2 df4592b3-6179-0097-2b74-a44d36f44fc2}" -testLoggerAdapter verbose: server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: 2024-06-11T02:29:26.027-1200 [INFO] server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: pipelining replication: peer="{Voter server-f5f4b281-1a4c-c67a-2c69-010813cdfa70 f5f4b281-1a4c-c67a-2c69-010813cdfa70}" - testing.go:149: server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: 2024-06-11T02:29:26.027-1200 [INFO] server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: pipelining replication: peer="{Voter server-f5f4b281-1a4c-c67a-2c69-010813cdfa70 f5f4b281-1a4c-c67a-2c69-010813cdfa70}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:26.132-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:26.027706681 -1200 -12 m=+17.023079223 (2 nodes), 60.930937ms from start of poll, 62.325369ms from cluster start. Timeout at 2024-06-11 02:29:26.132875911 -1200 -12 m=+17.128248453, 105.16923ms after stability - testing.go:149: cluster: 2024-06-11T02:29:26.132-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:26.027706681 -1200 -12 m=+17.023079223 (2 nodes), 60.930937ms from start of poll, 62.325369ms from cluster start. Timeout at 2024-06-11 02:29:26.132875911 -1200 -12 m=+17.128248453, 105.16923ms after stability -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:26.133-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:26.133-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:26.238-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:26.133152698 -1200 -12 m=+17.128525239 (1 nodes), 2.625µs from start of poll, 167.771385ms from cluster start. Timeout at 2024-06-11 02:29:26.238819793 -1200 -12 m=+17.234192335, 105.667096ms after stability - testing.go:149: cluster: 2024-06-11T02:29:26.238-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:26.133152698 -1200 -12 m=+17.128525239 (1 nodes), 2.625µs from start of poll, 167.771385ms from cluster start. Timeout at 2024-06-11 02:29:26.238819793 -1200 -12 m=+17.234192335, 105.667096ms after stability -testLoggerAdapter verbose: server-df4592b3-6179-0097-2b74-a44d36f44fc2: 2024-06-11T02:29:26.239-1200 [WARN] server-df4592b3-6179-0097-2b74-a44d36f44fc2: rejecting vote request since we have a leader: from=f5f4b281-1a4c-c67a-2c69-010813cdfa70 leader=8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78 - testing.go:149: server-df4592b3-6179-0097-2b74-a44d36f44fc2: 2024-06-11T02:29:26.239-1200 [WARN] server-df4592b3-6179-0097-2b74-a44d36f44fc2: rejecting vote request since we have a leader: from=f5f4b281-1a4c-c67a-2c69-010813cdfa70 leader=8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78 -testLoggerAdapter verbose: server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: 2024-06-11T02:29:26.239-1200 [INFO] server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: aborting pipeline replication: peer="{Voter server-df4592b3-6179-0097-2b74-a44d36f44fc2 df4592b3-6179-0097-2b74-a44d36f44fc2}" - testing.go:149: server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: 2024-06-11T02:29:26.239-1200 [INFO] server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: aborting pipeline replication: peer="{Voter server-df4592b3-6179-0097-2b74-a44d36f44fc2 df4592b3-6179-0097-2b74-a44d36f44fc2}" -testLoggerAdapter verbose: server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: 2024-06-11T02:29:26.239-1200 [INFO] server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: aborting pipeline replication: peer="{Voter server-f5f4b281-1a4c-c67a-2c69-010813cdfa70 f5f4b281-1a4c-c67a-2c69-010813cdfa70}" - testing.go:149: server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: 2024-06-11T02:29:26.239-1200 [INFO] server-8dc8bf7b-26c1-2e65-dcf4-dd7ec6d1cc78: aborting pipeline replication: peer="{Voter server-f5f4b281-1a4c-c67a-2c69-010813cdfa70 f5f4b281-1a4c-c67a-2c69-010813cdfa70}" ---- PASS: TestRaft_Voting (0.28s) -=== RUN TestRaft_ProtocolVersion_RejectRPC -testLoggerAdapter verbose: server-af35ef29-d90a-2aa4-2159-aa051615ac77: 2024-06-11T02:29:26.243-1200 [INFO] server-af35ef29-d90a-2aa4-2159-aa051615ac77: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-af35ef29-d90a-2aa4-2159-aa051615ac77 Address:af35ef29-d90a-2aa4-2159-aa051615ac77} {Suffrage:Voter ID:server-8d7babda-f34d-599a-cb36-b2c98cd82e68 Address:8d7babda-f34d-599a-cb36-b2c98cd82e68} {Suffrage:Voter ID:server-06263338-4471-2afa-63d6-883dfd9de605 Address:06263338-4471-2afa-63d6-883dfd9de605}]" - testing.go:149: server-af35ef29-d90a-2aa4-2159-aa051615ac77: 2024-06-11T02:29:26.243-1200 [INFO] server-af35ef29-d90a-2aa4-2159-aa051615ac77: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-af35ef29-d90a-2aa4-2159-aa051615ac77 Address:af35ef29-d90a-2aa4-2159-aa051615ac77} {Suffrage:Voter ID:server-8d7babda-f34d-599a-cb36-b2c98cd82e68 Address:8d7babda-f34d-599a-cb36-b2c98cd82e68} {Suffrage:Voter ID:server-06263338-4471-2afa-63d6-883dfd9de605 Address:06263338-4471-2afa-63d6-883dfd9de605}]" -testLoggerAdapter verbose: server-af35ef29-d90a-2aa4-2159-aa051615ac77: 2024-06-11T02:29:26.243-1200 [INFO] server-af35ef29-d90a-2aa4-2159-aa051615ac77: entering follower state: follower="Node at af35ef29-d90a-2aa4-2159-aa051615ac77 [Follower]" leader= - testing.go:149: server-af35ef29-d90a-2aa4-2159-aa051615ac77: 2024-06-11T02:29:26.243-1200 [INFO] server-af35ef29-d90a-2aa4-2159-aa051615ac77: entering follower state: follower="Node at af35ef29-d90a-2aa4-2159-aa051615ac77 [Follower]" leader= -testLoggerAdapter verbose: server-8d7babda-f34d-599a-cb36-b2c98cd82e68: 2024-06-11T02:29:26.243-1200 [INFO] server-8d7babda-f34d-599a-cb36-b2c98cd82e68: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-af35ef29-d90a-2aa4-2159-aa051615ac77 Address:af35ef29-d90a-2aa4-2159-aa051615ac77} {Suffrage:Voter ID:server-8d7babda-f34d-599a-cb36-b2c98cd82e68 Address:8d7babda-f34d-599a-cb36-b2c98cd82e68} {Suffrage:Voter ID:server-06263338-4471-2afa-63d6-883dfd9de605 Address:06263338-4471-2afa-63d6-883dfd9de605}]" - testing.go:149: server-8d7babda-f34d-599a-cb36-b2c98cd82e68: 2024-06-11T02:29:26.243-1200 [INFO] server-8d7babda-f34d-599a-cb36-b2c98cd82e68: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-af35ef29-d90a-2aa4-2159-aa051615ac77 Address:af35ef29-d90a-2aa4-2159-aa051615ac77} {Suffrage:Voter ID:server-8d7babda-f34d-599a-cb36-b2c98cd82e68 Address:8d7babda-f34d-599a-cb36-b2c98cd82e68} {Suffrage:Voter ID:server-06263338-4471-2afa-63d6-883dfd9de605 Address:06263338-4471-2afa-63d6-883dfd9de605}]" -testLoggerAdapter verbose: server-8d7babda-f34d-599a-cb36-b2c98cd82e68: 2024-06-11T02:29:26.243-1200 [INFO] server-8d7babda-f34d-599a-cb36-b2c98cd82e68: entering follower state: follower="Node at 8d7babda-f34d-599a-cb36-b2c98cd82e68 [Follower]" leader= - testing.go:149: server-8d7babda-f34d-599a-cb36-b2c98cd82e68: 2024-06-11T02:29:26.243-1200 [INFO] server-8d7babda-f34d-599a-cb36-b2c98cd82e68: entering follower state: follower="Node at 8d7babda-f34d-599a-cb36-b2c98cd82e68 [Follower]" leader= -testLoggerAdapter verbose: server-06263338-4471-2afa-63d6-883dfd9de605: 2024-06-11T02:29:26.244-1200 [INFO] server-06263338-4471-2afa-63d6-883dfd9de605: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-af35ef29-d90a-2aa4-2159-aa051615ac77 Address:af35ef29-d90a-2aa4-2159-aa051615ac77} {Suffrage:Voter ID:server-8d7babda-f34d-599a-cb36-b2c98cd82e68 Address:8d7babda-f34d-599a-cb36-b2c98cd82e68} {Suffrage:Voter ID:server-06263338-4471-2afa-63d6-883dfd9de605 Address:06263338-4471-2afa-63d6-883dfd9de605}]" - testing.go:149: server-06263338-4471-2afa-63d6-883dfd9de605: 2024-06-11T02:29:26.244-1200 [INFO] server-06263338-4471-2afa-63d6-883dfd9de605: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-af35ef29-d90a-2aa4-2159-aa051615ac77 Address:af35ef29-d90a-2aa4-2159-aa051615ac77} {Suffrage:Voter ID:server-8d7babda-f34d-599a-cb36-b2c98cd82e68 Address:8d7babda-f34d-599a-cb36-b2c98cd82e68} {Suffrage:Voter ID:server-06263338-4471-2afa-63d6-883dfd9de605 Address:06263338-4471-2afa-63d6-883dfd9de605}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:26.244-1200 [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2024-06-11T02:29:26.244-1200 [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: server-06263338-4471-2afa-63d6-883dfd9de605: 2024-06-11T02:29:26.244-1200 [INFO] server-06263338-4471-2afa-63d6-883dfd9de605: entering follower state: follower="Node at 06263338-4471-2afa-63d6-883dfd9de605 [Follower]" leader= - testing.go:149: server-06263338-4471-2afa-63d6-883dfd9de605: 2024-06-11T02:29:26.244-1200 [INFO] server-06263338-4471-2afa-63d6-883dfd9de605: entering follower state: follower="Node at 06263338-4471-2afa-63d6-883dfd9de605 [Follower]" leader= -testLoggerAdapter verbose: server-af35ef29-d90a-2aa4-2159-aa051615ac77: 2024-06-11T02:29:26.312-1200 [WARN] server-af35ef29-d90a-2aa4-2159-aa051615ac77: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-af35ef29-d90a-2aa4-2159-aa051615ac77: 2024-06-11T02:29:26.312-1200 [WARN] server-af35ef29-d90a-2aa4-2159-aa051615ac77: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-af35ef29-d90a-2aa4-2159-aa051615ac77: 2024-06-11T02:29:26.312-1200 [INFO] server-af35ef29-d90a-2aa4-2159-aa051615ac77: entering candidate state: node="Node at af35ef29-d90a-2aa4-2159-aa051615ac77 [Candidate]" term=2 - testing.go:149: server-af35ef29-d90a-2aa4-2159-aa051615ac77: 2024-06-11T02:29:26.312-1200 [INFO] server-af35ef29-d90a-2aa4-2159-aa051615ac77: entering candidate state: node="Node at af35ef29-d90a-2aa4-2159-aa051615ac77 [Candidate]" term=2 -testLoggerAdapter verbose: server-af35ef29-d90a-2aa4-2159-aa051615ac77: 2024-06-11T02:29:26.312-1200 [INFO] server-af35ef29-d90a-2aa4-2159-aa051615ac77: election won: tally=2 - testing.go:149: server-af35ef29-d90a-2aa4-2159-aa051615ac77: 2024-06-11T02:29:26.312-1200 [INFO] server-af35ef29-d90a-2aa4-2159-aa051615ac77: election won: tally=2 -testLoggerAdapter verbose: server-af35ef29-d90a-2aa4-2159-aa051615ac77: 2024-06-11T02:29:26.312-1200 [INFO] server-af35ef29-d90a-2aa4-2159-aa051615ac77: entering leader state: leader="Node at af35ef29-d90a-2aa4-2159-aa051615ac77 [Leader]" - testing.go:149: server-af35ef29-d90a-2aa4-2159-aa051615ac77: 2024-06-11T02:29:26.312-1200 [INFO] server-af35ef29-d90a-2aa4-2159-aa051615ac77: entering leader state: leader="Node at af35ef29-d90a-2aa4-2159-aa051615ac77 [Leader]" -testLoggerAdapter verbose: server-af35ef29-d90a-2aa4-2159-aa051615ac77: 2024-06-11T02:29:26.313-1200 [INFO] server-af35ef29-d90a-2aa4-2159-aa051615ac77: added peer, starting replication: peer=server-8d7babda-f34d-599a-cb36-b2c98cd82e68 - testing.go:149: server-af35ef29-d90a-2aa4-2159-aa051615ac77: 2024-06-11T02:29:26.313-1200 [INFO] server-af35ef29-d90a-2aa4-2159-aa051615ac77: added peer, starting replication: peer=server-8d7babda-f34d-599a-cb36-b2c98cd82e68 -testLoggerAdapter verbose: server-af35ef29-d90a-2aa4-2159-aa051615ac77: 2024-06-11T02:29:26.313-1200 [INFO] server-af35ef29-d90a-2aa4-2159-aa051615ac77: added peer, starting replication: peer=server-06263338-4471-2afa-63d6-883dfd9de605 - testing.go:149: server-af35ef29-d90a-2aa4-2159-aa051615ac77: 2024-06-11T02:29:26.313-1200 [INFO] server-af35ef29-d90a-2aa4-2159-aa051615ac77: added peer, starting replication: peer=server-06263338-4471-2afa-63d6-883dfd9de605 -testLoggerAdapter verbose: server-af35ef29-d90a-2aa4-2159-aa051615ac77: 2024-06-11T02:29:26.313-1200 [INFO] server-af35ef29-d90a-2aa4-2159-aa051615ac77: pipelining replication: peer="{Voter server-06263338-4471-2afa-63d6-883dfd9de605 06263338-4471-2afa-63d6-883dfd9de605}" - testing.go:149: server-af35ef29-d90a-2aa4-2159-aa051615ac77: 2024-06-11T02:29:26.313-1200 [INFO] server-af35ef29-d90a-2aa4-2159-aa051615ac77: pipelining replication: peer="{Voter server-06263338-4471-2afa-63d6-883dfd9de605 06263338-4471-2afa-63d6-883dfd9de605}" -testLoggerAdapter verbose: server-af35ef29-d90a-2aa4-2159-aa051615ac77: 2024-06-11T02:29:26.314-1200 [INFO] server-af35ef29-d90a-2aa4-2159-aa051615ac77: pipelining replication: peer="{Voter server-8d7babda-f34d-599a-cb36-b2c98cd82e68 8d7babda-f34d-599a-cb36-b2c98cd82e68}" - testing.go:149: server-af35ef29-d90a-2aa4-2159-aa051615ac77: 2024-06-11T02:29:26.314-1200 [INFO] server-af35ef29-d90a-2aa4-2159-aa051615ac77: pipelining replication: peer="{Voter server-8d7babda-f34d-599a-cb36-b2c98cd82e68 8d7babda-f34d-599a-cb36-b2c98cd82e68}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:26.419-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:26.314410496 -1200 -12 m=+17.309783621 (2 nodes), 70.111847ms from start of poll, 71.524654ms from cluster start. Timeout at 2024-06-11 02:29:26.419522561 -1200 -12 m=+17.414895394, 105.111773ms after stability - testing.go:149: cluster: 2024-06-11T02:29:26.419-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:26.314410496 -1200 -12 m=+17.309783621 (2 nodes), 70.111847ms from start of poll, 71.524654ms from cluster start. Timeout at 2024-06-11 02:29:26.419522561 -1200 -12 m=+17.414895394, 105.111773ms after stability -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:26.419-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:26.419-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:26.525-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:26.419839888 -1200 -12 m=+17.415212430 (1 nodes), 2.917µs from start of poll, 176.953463ms from cluster start. Timeout at 2024-06-11 02:29:26.525650481 -1200 -12 m=+17.521023314, 105.810884ms after stability - testing.go:149: cluster: 2024-06-11T02:29:26.525-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:26.419839888 -1200 -12 m=+17.415212430 (1 nodes), 2.917µs from start of poll, 176.953463ms from cluster start. Timeout at 2024-06-11 02:29:26.525650481 -1200 -12 m=+17.521023314, 105.810884ms after stability -testLoggerAdapter verbose: server-af35ef29-d90a-2aa4-2159-aa051615ac77: 2024-06-11T02:29:26.525-1200 [INFO] server-af35ef29-d90a-2aa4-2159-aa051615ac77: aborting pipeline replication: peer="{Voter server-06263338-4471-2afa-63d6-883dfd9de605 06263338-4471-2afa-63d6-883dfd9de605}" - testing.go:149: server-af35ef29-d90a-2aa4-2159-aa051615ac77: 2024-06-11T02:29:26.525-1200 [INFO] server-af35ef29-d90a-2aa4-2159-aa051615ac77: aborting pipeline replication: peer="{Voter server-06263338-4471-2afa-63d6-883dfd9de605 06263338-4471-2afa-63d6-883dfd9de605}" -testLoggerAdapter verbose: server-af35ef29-d90a-2aa4-2159-aa051615ac77: 2024-06-11T02:29:26.526-1200 [INFO] server-af35ef29-d90a-2aa4-2159-aa051615ac77: aborting pipeline replication: peer="{Voter server-8d7babda-f34d-599a-cb36-b2c98cd82e68 8d7babda-f34d-599a-cb36-b2c98cd82e68}" - testing.go:149: server-af35ef29-d90a-2aa4-2159-aa051615ac77: 2024-06-11T02:29:26.526-1200 [INFO] server-af35ef29-d90a-2aa4-2159-aa051615ac77: aborting pipeline replication: peer="{Voter server-8d7babda-f34d-599a-cb36-b2c98cd82e68 8d7babda-f34d-599a-cb36-b2c98cd82e68}" ---- PASS: TestRaft_ProtocolVersion_RejectRPC (0.29s) -=== RUN TestRaft_ProtocolVersion_Upgrade_1_2 -testLoggerAdapter verbose: d6f27740-f635-9355-d564-c931bd94029d: 2024-06-11T02:29:26.529-1200 [INFO] d6f27740-f635-9355-d564-c931bd94029d: initial configuration: index=1 servers="[{Suffrage:Voter ID:d6f27740-f635-9355-d564-c931bd94029d Address:d6f27740-f635-9355-d564-c931bd94029d} {Suffrage:Voter ID:8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27 Address:8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27}]" - testing.go:149: d6f27740-f635-9355-d564-c931bd94029d: 2024-06-11T02:29:26.529-1200 [INFO] d6f27740-f635-9355-d564-c931bd94029d: initial configuration: index=1 servers="[{Suffrage:Voter ID:d6f27740-f635-9355-d564-c931bd94029d Address:d6f27740-f635-9355-d564-c931bd94029d} {Suffrage:Voter ID:8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27 Address:8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27}]" -testLoggerAdapter verbose: d6f27740-f635-9355-d564-c931bd94029d: 2024-06-11T02:29:26.529-1200 [INFO] d6f27740-f635-9355-d564-c931bd94029d: entering follower state: follower="Node at d6f27740-f635-9355-d564-c931bd94029d [Follower]" leader= - testing.go:149: d6f27740-f635-9355-d564-c931bd94029d: 2024-06-11T02:29:26.529-1200 [INFO] d6f27740-f635-9355-d564-c931bd94029d: entering follower state: follower="Node at d6f27740-f635-9355-d564-c931bd94029d [Follower]" leader= -testLoggerAdapter verbose: 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: 2024-06-11T02:29:26.529-1200 [INFO] 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: initial configuration: index=1 servers="[{Suffrage:Voter ID:d6f27740-f635-9355-d564-c931bd94029d Address:d6f27740-f635-9355-d564-c931bd94029d} {Suffrage:Voter ID:8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27 Address:8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27}]" - testing.go:149: 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: 2024-06-11T02:29:26.529-1200 [INFO] 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: initial configuration: index=1 servers="[{Suffrage:Voter ID:d6f27740-f635-9355-d564-c931bd94029d Address:d6f27740-f635-9355-d564-c931bd94029d} {Suffrage:Voter ID:8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27 Address:8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27}]" -testLoggerAdapter verbose: 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: 2024-06-11T02:29:26.529-1200 [INFO] 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: entering follower state: follower="Node at 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27 [Follower]" leader= - testing.go:149: 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: 2024-06-11T02:29:26.529-1200 [INFO] 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: entering follower state: follower="Node at 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27 [Follower]" leader= -testLoggerAdapter verbose: 43c64f58-f96f-f4d6-4598-b5773a9c3294: 2024-06-11T02:29:26.531-1200 [INFO] 43c64f58-f96f-f4d6-4598-b5773a9c3294: initial configuration: index=0 servers=[] - testing.go:149: 43c64f58-f96f-f4d6-4598-b5773a9c3294: 2024-06-11T02:29:26.531-1200 [INFO] 43c64f58-f96f-f4d6-4598-b5773a9c3294: initial configuration: index=0 servers=[] -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:26.531-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:26.531-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: 43c64f58-f96f-f4d6-4598-b5773a9c3294: 2024-06-11T02:29:26.532-1200 [INFO] 43c64f58-f96f-f4d6-4598-b5773a9c3294: entering follower state: follower="Node at 43c64f58-f96f-f4d6-4598-b5773a9c3294 [Follower]" leader= - testing.go:149: 43c64f58-f96f-f4d6-4598-b5773a9c3294: 2024-06-11T02:29:26.532-1200 [INFO] 43c64f58-f96f-f4d6-4598-b5773a9c3294: entering follower state: follower="Node at 43c64f58-f96f-f4d6-4598-b5773a9c3294 [Follower]" leader= -testLoggerAdapter verbose: 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: 2024-06-11T02:29:26.595-1200 [WARN] 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: heartbeat timeout reached, starting election: last-leader= - testing.go:149: 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: 2024-06-11T02:29:26.595-1200 [WARN] 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: 2024-06-11T02:29:26.595-1200 [INFO] 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: entering candidate state: node="Node at 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27 [Candidate]" term=2 - testing.go:149: 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: 2024-06-11T02:29:26.595-1200 [INFO] 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: entering candidate state: node="Node at 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27 [Candidate]" term=2 -testLoggerAdapter verbose: 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: 2024-06-11T02:29:26.595-1200 [INFO] 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: election won: tally=2 - testing.go:149: 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: 2024-06-11T02:29:26.595-1200 [INFO] 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: election won: tally=2 -testLoggerAdapter verbose: 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: 2024-06-11T02:29:26.595-1200 [INFO] 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: entering leader state: leader="Node at 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27 [Leader]" - testing.go:149: 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: 2024-06-11T02:29:26.595-1200 [INFO] 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: entering leader state: leader="Node at 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27 [Leader]" -testLoggerAdapter verbose: 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: 2024-06-11T02:29:26.595-1200 [INFO] 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: added peer, starting replication: peer=d6f27740-f635-9355-d564-c931bd94029d - testing.go:149: 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: 2024-06-11T02:29:26.595-1200 [INFO] 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: added peer, starting replication: peer=d6f27740-f635-9355-d564-c931bd94029d -testLoggerAdapter verbose: 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: 2024-06-11T02:29:26.596-1200 [INFO] 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: pipelining replication: peer="{Voter d6f27740-f635-9355-d564-c931bd94029d d6f27740-f635-9355-d564-c931bd94029d}" - testing.go:149: 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: 2024-06-11T02:29:26.596-1200 [INFO] 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: pipelining replication: peer="{Voter d6f27740-f635-9355-d564-c931bd94029d d6f27740-f635-9355-d564-c931bd94029d}" -testLoggerAdapter verbose: 43c64f58-f96f-f4d6-4598-b5773a9c3294: 2024-06-11T02:29:26.622-1200 [WARN] 43c64f58-f96f-f4d6-4598-b5773a9c3294: no known peers, aborting election - testing.go:149: 43c64f58-f96f-f4d6-4598-b5773a9c3294: 2024-06-11T02:29:26.622-1200 [WARN] 43c64f58-f96f-f4d6-4598-b5773a9c3294: no known peers, aborting election -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:26.702-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:26.596441316 -1200 -12 m=+17.591813858 (1 nodes), 64.374581ms from start of poll, 67.414565ms from cluster start. Timeout at 2024-06-11 02:29:26.702188327 -1200 -12 m=+17.697561452, 105.747594ms after stability - testing.go:149: cluster: 2024-06-11T02:29:26.702-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:26.596441316 -1200 -12 m=+17.591813858 (1 nodes), 64.374581ms from start of poll, 67.414565ms from cluster start. Timeout at 2024-06-11 02:29:26.702188327 -1200 -12 m=+17.697561452, 105.747594ms after stability -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:26.702-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:26.702-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:26.807-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:26.702590236 -1200 -12 m=+17.697962778 (1 nodes), 2.917µs from start of poll, 173.563485ms from cluster start. Timeout at 2024-06-11 02:29:26.807860381 -1200 -12 m=+17.803232922, 105.270144ms after stability - testing.go:149: cluster: 2024-06-11T02:29:26.807-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:26.702590236 -1200 -12 m=+17.697962778 (1 nodes), 2.917µs from start of poll, 173.563485ms from cluster start. Timeout at 2024-06-11 02:29:26.807860381 -1200 -12 m=+17.803232922, 105.270144ms after stability -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:26.808-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:26.808-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:26.914-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:26.808141542 -1200 -12 m=+17.803514084 (1 nodes), 2.333µs from start of poll, 279.114791ms from cluster start. Timeout at 2024-06-11 02:29:26.914181964 -1200 -12 m=+17.909554797, 106.040713ms after stability - testing.go:149: cluster: 2024-06-11T02:29:26.914-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:26.808141542 -1200 -12 m=+17.803514084 (1 nodes), 2.333µs from start of poll, 279.114791ms from cluster start. Timeout at 2024-06-11 02:29:26.914181964 -1200 -12 m=+17.909554797, 106.040713ms after stability -testLoggerAdapter verbose: 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: 2024-06-11T02:29:26.914-1200 [INFO] 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: updating configuration: command=AddStaging server-id=43c64f58-f96f-f4d6-4598-b5773a9c3294 server-addr=43c64f58-f96f-f4d6-4598-b5773a9c3294 servers="[{Suffrage:Voter ID:d6f27740-f635-9355-d564-c931bd94029d Address:d6f27740-f635-9355-d564-c931bd94029d} {Suffrage:Voter ID:8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27 Address:8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27} {Suffrage:Voter ID:43c64f58-f96f-f4d6-4598-b5773a9c3294 Address:43c64f58-f96f-f4d6-4598-b5773a9c3294}]" - testing.go:149: 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: 2024-06-11T02:29:26.914-1200 [INFO] 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: updating configuration: command=AddStaging server-id=43c64f58-f96f-f4d6-4598-b5773a9c3294 server-addr=43c64f58-f96f-f4d6-4598-b5773a9c3294 servers="[{Suffrage:Voter ID:d6f27740-f635-9355-d564-c931bd94029d Address:d6f27740-f635-9355-d564-c931bd94029d} {Suffrage:Voter ID:8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27 Address:8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27} {Suffrage:Voter ID:43c64f58-f96f-f4d6-4598-b5773a9c3294 Address:43c64f58-f96f-f4d6-4598-b5773a9c3294}]" -testLoggerAdapter verbose: 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: 2024-06-11T02:29:26.914-1200 [INFO] 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: added peer, starting replication: peer=43c64f58-f96f-f4d6-4598-b5773a9c3294 - testing.go:149: 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: 2024-06-11T02:29:26.914-1200 [INFO] 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: added peer, starting replication: peer=43c64f58-f96f-f4d6-4598-b5773a9c3294 -testLoggerAdapter verbose: 43c64f58-f96f-f4d6-4598-b5773a9c3294: 2024-06-11T02:29:26.915-1200 [WARN] 43c64f58-f96f-f4d6-4598-b5773a9c3294: failed to get previous log: previous-index=3 last-index=0 error="log not found" - testing.go:149: 43c64f58-f96f-f4d6-4598-b5773a9c3294: 2024-06-11T02:29:26.915-1200 [WARN] 43c64f58-f96f-f4d6-4598-b5773a9c3294: failed to get previous log: previous-index=3 last-index=0 error="log not found" -testLoggerAdapter verbose: 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: 2024-06-11T02:29:26.915-1200 [WARN] 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: appendEntries rejected, sending older logs: peer="{Voter 43c64f58-f96f-f4d6-4598-b5773a9c3294 43c64f58-f96f-f4d6-4598-b5773a9c3294}" next=1 - testing.go:149: 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: 2024-06-11T02:29:26.915-1200 [WARN] 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: appendEntries rejected, sending older logs: peer="{Voter 43c64f58-f96f-f4d6-4598-b5773a9c3294 43c64f58-f96f-f4d6-4598-b5773a9c3294}" next=1 -testLoggerAdapter verbose: 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: 2024-06-11T02:29:26.915-1200 [INFO] 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: pipelining replication: peer="{Voter 43c64f58-f96f-f4d6-4598-b5773a9c3294 43c64f58-f96f-f4d6-4598-b5773a9c3294}" - testing.go:149: 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: 2024-06-11T02:29:26.915-1200 [INFO] 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: pipelining replication: peer="{Voter 43c64f58-f96f-f4d6-4598-b5773a9c3294 43c64f58-f96f-f4d6-4598-b5773a9c3294}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:26.915-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:26.915-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:27.021-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:26.916053846 -1200 -12 m=+17.911426387 (1 nodes), 3.5µs from start of poll, 387.027094ms from cluster start. Timeout at 2024-06-11 02:29:27.02152757 -1200 -12 m=+18.016900403, 105.474016ms after stability - testing.go:149: cluster: 2024-06-11T02:29:27.021-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:26.916053846 -1200 -12 m=+17.911426387 (1 nodes), 3.5µs from start of poll, 387.027094ms from cluster start. Timeout at 2024-06-11 02:29:27.02152757 -1200 -12 m=+18.016900403, 105.474016ms after stability -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:27.021-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:27.021-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:27.127-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:27.021857147 -1200 -12 m=+18.017229689 (1 nodes), 3.209µs from start of poll, 492.830396ms from cluster start. Timeout at 2024-06-11 02:29:27.127103084 -1200 -12 m=+18.122475626, 105.245937ms after stability - testing.go:149: cluster: 2024-06-11T02:29:27.127-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:27.021857147 -1200 -12 m=+18.017229689 (1 nodes), 3.209µs from start of poll, 492.830396ms from cluster start. Timeout at 2024-06-11 02:29:27.127103084 -1200 -12 m=+18.122475626, 105.245937ms after stability -testLoggerAdapter verbose: 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: 2024-06-11T02:29:27.127-1200 [INFO] 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: updating configuration: command=RemoveServer server-id=43c64f58-f96f-f4d6-4598-b5773a9c3294 server-addr= servers="[{Suffrage:Voter ID:d6f27740-f635-9355-d564-c931bd94029d Address:d6f27740-f635-9355-d564-c931bd94029d} {Suffrage:Voter ID:8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27 Address:8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27}]" - testing.go:149: 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: 2024-06-11T02:29:27.127-1200 [INFO] 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: updating configuration: command=RemoveServer server-id=43c64f58-f96f-f4d6-4598-b5773a9c3294 server-addr= servers="[{Suffrage:Voter ID:d6f27740-f635-9355-d564-c931bd94029d Address:d6f27740-f635-9355-d564-c931bd94029d} {Suffrage:Voter ID:8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27 Address:8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27}]" -testLoggerAdapter verbose: 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: 2024-06-11T02:29:27.127-1200 [INFO] 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: removed peer, stopping replication: peer=43c64f58-f96f-f4d6-4598-b5773a9c3294 last-index=4 - testing.go:149: 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: 2024-06-11T02:29:27.127-1200 [INFO] 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: removed peer, stopping replication: peer=43c64f58-f96f-f4d6-4598-b5773a9c3294 last-index=4 -testLoggerAdapter verbose: 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: 2024-06-11T02:29:27.127-1200 [INFO] 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: aborting pipeline replication: peer="{Voter 43c64f58-f96f-f4d6-4598-b5773a9c3294 43c64f58-f96f-f4d6-4598-b5773a9c3294}" - testing.go:149: 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: 2024-06-11T02:29:27.127-1200 [INFO] 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: aborting pipeline replication: peer="{Voter 43c64f58-f96f-f4d6-4598-b5773a9c3294 43c64f58-f96f-f4d6-4598-b5773a9c3294}" -testLoggerAdapter verbose: 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: 2024-06-11T02:29:27.128-1200 [INFO] 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: aborting pipeline replication: peer="{Voter d6f27740-f635-9355-d564-c931bd94029d d6f27740-f635-9355-d564-c931bd94029d}" - testing.go:149: 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: 2024-06-11T02:29:27.128-1200 [INFO] 8e2995cc-08f9-45e9-9cc3-a2d7c6cfaa27: aborting pipeline replication: peer="{Voter d6f27740-f635-9355-d564-c931bd94029d d6f27740-f635-9355-d564-c931bd94029d}" ---- PASS: TestRaft_ProtocolVersion_Upgrade_1_2 (0.60s) -=== RUN TestRaft_ProtocolVersion_Upgrade_2_3 -testLoggerAdapter verbose: fbc90435-4ea6-70ff-59e8-022f84fa9cac: 2024-06-11T02:29:27.131-1200 [INFO] fbc90435-4ea6-70ff-59e8-022f84fa9cac: initial configuration: index=1 servers="[{Suffrage:Voter ID:fbc90435-4ea6-70ff-59e8-022f84fa9cac Address:fbc90435-4ea6-70ff-59e8-022f84fa9cac} {Suffrage:Voter ID:2c0be127-77bc-c9a9-fcc3-ea0c86932ffb Address:2c0be127-77bc-c9a9-fcc3-ea0c86932ffb}]" - testing.go:149: fbc90435-4ea6-70ff-59e8-022f84fa9cac: 2024-06-11T02:29:27.131-1200 [INFO] fbc90435-4ea6-70ff-59e8-022f84fa9cac: initial configuration: index=1 servers="[{Suffrage:Voter ID:fbc90435-4ea6-70ff-59e8-022f84fa9cac Address:fbc90435-4ea6-70ff-59e8-022f84fa9cac} {Suffrage:Voter ID:2c0be127-77bc-c9a9-fcc3-ea0c86932ffb Address:2c0be127-77bc-c9a9-fcc3-ea0c86932ffb}]" -testLoggerAdapter verbose: 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: 2024-06-11T02:29:27.131-1200 [INFO] 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: initial configuration: index=1 servers="[{Suffrage:Voter ID:fbc90435-4ea6-70ff-59e8-022f84fa9cac Address:fbc90435-4ea6-70ff-59e8-022f84fa9cac} {Suffrage:Voter ID:2c0be127-77bc-c9a9-fcc3-ea0c86932ffb Address:2c0be127-77bc-c9a9-fcc3-ea0c86932ffb}]" - testing.go:149: 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: 2024-06-11T02:29:27.131-1200 [INFO] 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: initial configuration: index=1 servers="[{Suffrage:Voter ID:fbc90435-4ea6-70ff-59e8-022f84fa9cac Address:fbc90435-4ea6-70ff-59e8-022f84fa9cac} {Suffrage:Voter ID:2c0be127-77bc-c9a9-fcc3-ea0c86932ffb Address:2c0be127-77bc-c9a9-fcc3-ea0c86932ffb}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:27.131-1200 [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2024-06-11T02:29:27.131-1200 [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: fbc90435-4ea6-70ff-59e8-022f84fa9cac: 2024-06-11T02:29:27.131-1200 [INFO] fbc90435-4ea6-70ff-59e8-022f84fa9cac: entering follower state: follower="Node at fbc90435-4ea6-70ff-59e8-022f84fa9cac [Follower]" leader= - testing.go:149: fbc90435-4ea6-70ff-59e8-022f84fa9cac: 2024-06-11T02:29:27.131-1200 [INFO] fbc90435-4ea6-70ff-59e8-022f84fa9cac: entering follower state: follower="Node at fbc90435-4ea6-70ff-59e8-022f84fa9cac [Follower]" leader= -testLoggerAdapter verbose: 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: 2024-06-11T02:29:27.132-1200 [INFO] 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: entering follower state: follower="Node at 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb [Follower]" leader= - testing.go:149: 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: 2024-06-11T02:29:27.132-1200 [INFO] 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: entering follower state: follower="Node at 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb [Follower]" leader= -testLoggerAdapter verbose: 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: 2024-06-11T02:29:27.185-1200 [WARN] 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: heartbeat timeout reached, starting election: last-leader= - testing.go:149: 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: 2024-06-11T02:29:27.185-1200 [WARN] 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: 2024-06-11T02:29:27.185-1200 [INFO] 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: entering candidate state: node="Node at 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb [Candidate]" term=2 - testing.go:149: 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: 2024-06-11T02:29:27.185-1200 [INFO] 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: entering candidate state: node="Node at 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb [Candidate]" term=2 -testLoggerAdapter verbose: 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: 2024-06-11T02:29:27.185-1200 [INFO] 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: election won: tally=2 - testing.go:149: 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: 2024-06-11T02:29:27.185-1200 [INFO] 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: election won: tally=2 -testLoggerAdapter verbose: 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: 2024-06-11T02:29:27.185-1200 [INFO] 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: entering leader state: leader="Node at 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb [Leader]" - testing.go:149: 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: 2024-06-11T02:29:27.185-1200 [INFO] 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: entering leader state: leader="Node at 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb [Leader]" -testLoggerAdapter verbose: 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: 2024-06-11T02:29:27.185-1200 [INFO] 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: added peer, starting replication: peer=fbc90435-4ea6-70ff-59e8-022f84fa9cac - testing.go:149: 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: 2024-06-11T02:29:27.185-1200 [INFO] 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: added peer, starting replication: peer=fbc90435-4ea6-70ff-59e8-022f84fa9cac -testLoggerAdapter verbose: 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: 2024-06-11T02:29:27.186-1200 [INFO] 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: pipelining replication: peer="{Voter fbc90435-4ea6-70ff-59e8-022f84fa9cac fbc90435-4ea6-70ff-59e8-022f84fa9cac}" - testing.go:149: 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: 2024-06-11T02:29:27.186-1200 [INFO] 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: pipelining replication: peer="{Voter fbc90435-4ea6-70ff-59e8-022f84fa9cac fbc90435-4ea6-70ff-59e8-022f84fa9cac}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:27.291-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:27.186051308 -1200 -12 m=+18.181423850 (1 nodes), 54.105191ms from start of poll, 55.025965ms from cluster start. Timeout at 2024-06-11 02:29:27.29169128 -1200 -12 m=+18.287063821, 105.639971ms after stability - testing.go:149: cluster: 2024-06-11T02:29:27.291-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:27.186051308 -1200 -12 m=+18.181423850 (1 nodes), 54.105191ms from start of poll, 55.025965ms from cluster start. Timeout at 2024-06-11 02:29:27.29169128 -1200 -12 m=+18.287063821, 105.639971ms after stability -testLoggerAdapter verbose: server-9d9a8061-cc50-b6b2-b2d8-4f8c7119acba: 2024-06-11T02:29:27.292-1200 [INFO] server-9d9a8061-cc50-b6b2-b2d8-4f8c7119acba: initial configuration: index=0 servers=[] - testing.go:149: server-9d9a8061-cc50-b6b2-b2d8-4f8c7119acba: 2024-06-11T02:29:27.292-1200 [INFO] server-9d9a8061-cc50-b6b2-b2d8-4f8c7119acba: initial configuration: index=0 servers=[] -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:27.292-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:27.292-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-9d9a8061-cc50-b6b2-b2d8-4f8c7119acba: 2024-06-11T02:29:27.292-1200 [INFO] server-9d9a8061-cc50-b6b2-b2d8-4f8c7119acba: entering follower state: follower="Node at 9d9a8061-cc50-b6b2-b2d8-4f8c7119acba [Follower]" leader= - testing.go:149: server-9d9a8061-cc50-b6b2-b2d8-4f8c7119acba: 2024-06-11T02:29:27.292-1200 [INFO] server-9d9a8061-cc50-b6b2-b2d8-4f8c7119acba: entering follower state: follower="Node at 9d9a8061-cc50-b6b2-b2d8-4f8c7119acba [Follower]" leader= -testLoggerAdapter verbose: server-9d9a8061-cc50-b6b2-b2d8-4f8c7119acba: 2024-06-11T02:29:27.372-1200 [WARN] server-9d9a8061-cc50-b6b2-b2d8-4f8c7119acba: no known peers, aborting election - testing.go:149: server-9d9a8061-cc50-b6b2-b2d8-4f8c7119acba: 2024-06-11T02:29:27.372-1200 [WARN] server-9d9a8061-cc50-b6b2-b2d8-4f8c7119acba: no known peers, aborting election -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:27.398-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:27.292885632 -1200 -12 m=+18.288258174 (1 nodes), 2.917µs from start of poll, 161.860289ms from cluster start. Timeout at 2024-06-11 02:29:27.398512478 -1200 -12 m=+18.393885312, 105.627138ms after stability - testing.go:149: cluster: 2024-06-11T02:29:27.398-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:27.292885632 -1200 -12 m=+18.288258174 (1 nodes), 2.917µs from start of poll, 161.860289ms from cluster start. Timeout at 2024-06-11 02:29:27.398512478 -1200 -12 m=+18.393885312, 105.627138ms after stability -testLoggerAdapter verbose: 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: 2024-06-11T02:29:27.398-1200 [INFO] 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: updating configuration: command=AddStaging server-id=server-9d9a8061-cc50-b6b2-b2d8-4f8c7119acba server-addr=9d9a8061-cc50-b6b2-b2d8-4f8c7119acba servers="[{Suffrage:Voter ID:fbc90435-4ea6-70ff-59e8-022f84fa9cac Address:fbc90435-4ea6-70ff-59e8-022f84fa9cac} {Suffrage:Voter ID:2c0be127-77bc-c9a9-fcc3-ea0c86932ffb Address:2c0be127-77bc-c9a9-fcc3-ea0c86932ffb} {Suffrage:Voter ID:server-9d9a8061-cc50-b6b2-b2d8-4f8c7119acba Address:9d9a8061-cc50-b6b2-b2d8-4f8c7119acba}]" - testing.go:149: 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: 2024-06-11T02:29:27.398-1200 [INFO] 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: updating configuration: command=AddStaging server-id=server-9d9a8061-cc50-b6b2-b2d8-4f8c7119acba server-addr=9d9a8061-cc50-b6b2-b2d8-4f8c7119acba servers="[{Suffrage:Voter ID:fbc90435-4ea6-70ff-59e8-022f84fa9cac Address:fbc90435-4ea6-70ff-59e8-022f84fa9cac} {Suffrage:Voter ID:2c0be127-77bc-c9a9-fcc3-ea0c86932ffb Address:2c0be127-77bc-c9a9-fcc3-ea0c86932ffb} {Suffrage:Voter ID:server-9d9a8061-cc50-b6b2-b2d8-4f8c7119acba Address:9d9a8061-cc50-b6b2-b2d8-4f8c7119acba}]" -testLoggerAdapter verbose: 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: 2024-06-11T02:29:27.398-1200 [INFO] 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: added peer, starting replication: peer=server-9d9a8061-cc50-b6b2-b2d8-4f8c7119acba - testing.go:149: 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: 2024-06-11T02:29:27.398-1200 [INFO] 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: added peer, starting replication: peer=server-9d9a8061-cc50-b6b2-b2d8-4f8c7119acba -testLoggerAdapter verbose: server-9d9a8061-cc50-b6b2-b2d8-4f8c7119acba: 2024-06-11T02:29:27.399-1200 [WARN] server-9d9a8061-cc50-b6b2-b2d8-4f8c7119acba: failed to get previous log: previous-index=3 last-index=0 error="log not found" - testing.go:149: server-9d9a8061-cc50-b6b2-b2d8-4f8c7119acba: 2024-06-11T02:29:27.399-1200 [WARN] server-9d9a8061-cc50-b6b2-b2d8-4f8c7119acba: failed to get previous log: previous-index=3 last-index=0 error="log not found" -testLoggerAdapter verbose: 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: 2024-06-11T02:29:27.399-1200 [WARN] 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: appendEntries rejected, sending older logs: peer="{Voter server-9d9a8061-cc50-b6b2-b2d8-4f8c7119acba 9d9a8061-cc50-b6b2-b2d8-4f8c7119acba}" next=1 - testing.go:149: 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: 2024-06-11T02:29:27.399-1200 [WARN] 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: appendEntries rejected, sending older logs: peer="{Voter server-9d9a8061-cc50-b6b2-b2d8-4f8c7119acba 9d9a8061-cc50-b6b2-b2d8-4f8c7119acba}" next=1 -testLoggerAdapter verbose: 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: 2024-06-11T02:29:27.399-1200 [INFO] 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: pipelining replication: peer="{Voter server-9d9a8061-cc50-b6b2-b2d8-4f8c7119acba 9d9a8061-cc50-b6b2-b2d8-4f8c7119acba}" - testing.go:149: 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: 2024-06-11T02:29:27.399-1200 [INFO] 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: pipelining replication: peer="{Voter server-9d9a8061-cc50-b6b2-b2d8-4f8c7119acba 9d9a8061-cc50-b6b2-b2d8-4f8c7119acba}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:27.405-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:27.405-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:27.511-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:27.40533035 -1200 -12 m=+18.400702892 (1 nodes), 3.208µs from start of poll, 274.305007ms from cluster start. Timeout at 2024-06-11 02:29:27.511195775 -1200 -12 m=+18.506568900, 105.866008ms after stability - testing.go:149: cluster: 2024-06-11T02:29:27.511-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:27.40533035 -1200 -12 m=+18.400702892 (1 nodes), 3.208µs from start of poll, 274.305007ms from cluster start. Timeout at 2024-06-11 02:29:27.511195775 -1200 -12 m=+18.506568900, 105.866008ms after stability -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:27.511-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:27.511-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:27.617-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:27.511506686 -1200 -12 m=+18.506879228 (1 nodes), 2.917µs from start of poll, 380.481343ms from cluster start. Timeout at 2024-06-11 02:29:27.617454652 -1200 -12 m=+18.612827485, 105.948257ms after stability - testing.go:149: cluster: 2024-06-11T02:29:27.617-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:27.511506686 -1200 -12 m=+18.506879228 (1 nodes), 2.917µs from start of poll, 380.481343ms from cluster start. Timeout at 2024-06-11 02:29:27.617454652 -1200 -12 m=+18.612827485, 105.948257ms after stability -testLoggerAdapter verbose: 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: 2024-06-11T02:29:27.617-1200 [INFO] 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: updating configuration: command=RemoveServer server-id=fbc90435-4ea6-70ff-59e8-022f84fa9cac server-addr= servers="[{Suffrage:Voter ID:2c0be127-77bc-c9a9-fcc3-ea0c86932ffb Address:2c0be127-77bc-c9a9-fcc3-ea0c86932ffb} {Suffrage:Voter ID:server-9d9a8061-cc50-b6b2-b2d8-4f8c7119acba Address:9d9a8061-cc50-b6b2-b2d8-4f8c7119acba}]" - testing.go:149: 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: 2024-06-11T02:29:27.617-1200 [INFO] 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: updating configuration: command=RemoveServer server-id=fbc90435-4ea6-70ff-59e8-022f84fa9cac server-addr= servers="[{Suffrage:Voter ID:2c0be127-77bc-c9a9-fcc3-ea0c86932ffb Address:2c0be127-77bc-c9a9-fcc3-ea0c86932ffb} {Suffrage:Voter ID:server-9d9a8061-cc50-b6b2-b2d8-4f8c7119acba Address:9d9a8061-cc50-b6b2-b2d8-4f8c7119acba}]" -testLoggerAdapter verbose: 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: 2024-06-11T02:29:27.618-1200 [INFO] 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: removed peer, stopping replication: peer=fbc90435-4ea6-70ff-59e8-022f84fa9cac last-index=4 - testing.go:149: 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: 2024-06-11T02:29:27.618-1200 [INFO] 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: removed peer, stopping replication: peer=fbc90435-4ea6-70ff-59e8-022f84fa9cac last-index=4 -testLoggerAdapter verbose: 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: 2024-06-11T02:29:27.618-1200 [INFO] 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: aborting pipeline replication: peer="{Voter fbc90435-4ea6-70ff-59e8-022f84fa9cac fbc90435-4ea6-70ff-59e8-022f84fa9cac}" - testing.go:149: 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: 2024-06-11T02:29:27.618-1200 [INFO] 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: aborting pipeline replication: peer="{Voter fbc90435-4ea6-70ff-59e8-022f84fa9cac fbc90435-4ea6-70ff-59e8-022f84fa9cac}" -testLoggerAdapter verbose: 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: 2024-06-11T02:29:27.618-1200 [INFO] 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: aborting pipeline replication: peer="{Voter server-9d9a8061-cc50-b6b2-b2d8-4f8c7119acba 9d9a8061-cc50-b6b2-b2d8-4f8c7119acba}" - testing.go:149: 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: 2024-06-11T02:29:27.618-1200 [INFO] 2c0be127-77bc-c9a9-fcc3-ea0c86932ffb: aborting pipeline replication: peer="{Voter server-9d9a8061-cc50-b6b2-b2d8-4f8c7119acba 9d9a8061-cc50-b6b2-b2d8-4f8c7119acba}" ---- PASS: TestRaft_ProtocolVersion_Upgrade_2_3 (0.49s) -=== RUN TestRaft_LeadershipTransferInProgress ---- PASS: TestRaft_LeadershipTransferInProgress (0.00s) -=== RUN TestRaft_LeadershipTransferPickServer ---- PASS: TestRaft_LeadershipTransferPickServer (0.00s) -=== RUN TestRaft_LeadershipTransfer -testLoggerAdapter verbose: server-bd560674-5316-7c77-7ecf-1ee4735382af: 2024-06-11T02:29:27.625-1200 [INFO] server-bd560674-5316-7c77-7ecf-1ee4735382af: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bd560674-5316-7c77-7ecf-1ee4735382af Address:bd560674-5316-7c77-7ecf-1ee4735382af} {Suffrage:Voter ID:server-56cb3e84-edaf-7027-dbb7-85e912733825 Address:56cb3e84-edaf-7027-dbb7-85e912733825} {Suffrage:Voter ID:server-30d05822-0955-9ec3-4054-fb4bdee64c11 Address:30d05822-0955-9ec3-4054-fb4bdee64c11}]" - testing.go:149: server-bd560674-5316-7c77-7ecf-1ee4735382af: 2024-06-11T02:29:27.625-1200 [INFO] server-bd560674-5316-7c77-7ecf-1ee4735382af: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bd560674-5316-7c77-7ecf-1ee4735382af Address:bd560674-5316-7c77-7ecf-1ee4735382af} {Suffrage:Voter ID:server-56cb3e84-edaf-7027-dbb7-85e912733825 Address:56cb3e84-edaf-7027-dbb7-85e912733825} {Suffrage:Voter ID:server-30d05822-0955-9ec3-4054-fb4bdee64c11 Address:30d05822-0955-9ec3-4054-fb4bdee64c11}]" -testLoggerAdapter verbose: server-bd560674-5316-7c77-7ecf-1ee4735382af: 2024-06-11T02:29:27.625-1200 [INFO] server-bd560674-5316-7c77-7ecf-1ee4735382af: entering follower state: follower="Node at bd560674-5316-7c77-7ecf-1ee4735382af [Follower]" leader= - testing.go:149: server-bd560674-5316-7c77-7ecf-1ee4735382af: 2024-06-11T02:29:27.625-1200 [INFO] server-bd560674-5316-7c77-7ecf-1ee4735382af: entering follower state: follower="Node at bd560674-5316-7c77-7ecf-1ee4735382af [Follower]" leader= -testLoggerAdapter verbose: server-56cb3e84-edaf-7027-dbb7-85e912733825: 2024-06-11T02:29:27.625-1200 [INFO] server-56cb3e84-edaf-7027-dbb7-85e912733825: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bd560674-5316-7c77-7ecf-1ee4735382af Address:bd560674-5316-7c77-7ecf-1ee4735382af} {Suffrage:Voter ID:server-56cb3e84-edaf-7027-dbb7-85e912733825 Address:56cb3e84-edaf-7027-dbb7-85e912733825} {Suffrage:Voter ID:server-30d05822-0955-9ec3-4054-fb4bdee64c11 Address:30d05822-0955-9ec3-4054-fb4bdee64c11}]" - testing.go:149: server-56cb3e84-edaf-7027-dbb7-85e912733825: 2024-06-11T02:29:27.625-1200 [INFO] server-56cb3e84-edaf-7027-dbb7-85e912733825: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bd560674-5316-7c77-7ecf-1ee4735382af Address:bd560674-5316-7c77-7ecf-1ee4735382af} {Suffrage:Voter ID:server-56cb3e84-edaf-7027-dbb7-85e912733825 Address:56cb3e84-edaf-7027-dbb7-85e912733825} {Suffrage:Voter ID:server-30d05822-0955-9ec3-4054-fb4bdee64c11 Address:30d05822-0955-9ec3-4054-fb4bdee64c11}]" -testLoggerAdapter verbose: server-56cb3e84-edaf-7027-dbb7-85e912733825: 2024-06-11T02:29:27.626-1200 [INFO] server-56cb3e84-edaf-7027-dbb7-85e912733825: entering follower state: follower="Node at 56cb3e84-edaf-7027-dbb7-85e912733825 [Follower]" leader= - testing.go:149: server-56cb3e84-edaf-7027-dbb7-85e912733825: 2024-06-11T02:29:27.626-1200 [INFO] server-56cb3e84-edaf-7027-dbb7-85e912733825: entering follower state: follower="Node at 56cb3e84-edaf-7027-dbb7-85e912733825 [Follower]" leader= -testLoggerAdapter verbose: server-30d05822-0955-9ec3-4054-fb4bdee64c11: 2024-06-11T02:29:27.626-1200 [INFO] server-30d05822-0955-9ec3-4054-fb4bdee64c11: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bd560674-5316-7c77-7ecf-1ee4735382af Address:bd560674-5316-7c77-7ecf-1ee4735382af} {Suffrage:Voter ID:server-56cb3e84-edaf-7027-dbb7-85e912733825 Address:56cb3e84-edaf-7027-dbb7-85e912733825} {Suffrage:Voter ID:server-30d05822-0955-9ec3-4054-fb4bdee64c11 Address:30d05822-0955-9ec3-4054-fb4bdee64c11}]" - testing.go:149: server-30d05822-0955-9ec3-4054-fb4bdee64c11: 2024-06-11T02:29:27.626-1200 [INFO] server-30d05822-0955-9ec3-4054-fb4bdee64c11: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bd560674-5316-7c77-7ecf-1ee4735382af Address:bd560674-5316-7c77-7ecf-1ee4735382af} {Suffrage:Voter ID:server-56cb3e84-edaf-7027-dbb7-85e912733825 Address:56cb3e84-edaf-7027-dbb7-85e912733825} {Suffrage:Voter ID:server-30d05822-0955-9ec3-4054-fb4bdee64c11 Address:30d05822-0955-9ec3-4054-fb4bdee64c11}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:27.626-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-30d05822-0955-9ec3-4054-fb4bdee64c11: 2024-06-11T02:29:27.626-1200 [INFO] server-30d05822-0955-9ec3-4054-fb4bdee64c11: entering follower state: follower="Node at 30d05822-0955-9ec3-4054-fb4bdee64c11 [Follower]" leader= - testing.go:149: cluster: 2024-06-11T02:29:27.626-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: server-30d05822-0955-9ec3-4054-fb4bdee64c11: 2024-06-11T02:29:27.626-1200 [INFO] server-30d05822-0955-9ec3-4054-fb4bdee64c11: entering follower state: follower="Node at 30d05822-0955-9ec3-4054-fb4bdee64c11 [Follower]" leader= -testLoggerAdapter verbose: server-30d05822-0955-9ec3-4054-fb4bdee64c11: 2024-06-11T02:29:27.678-1200 [WARN] server-30d05822-0955-9ec3-4054-fb4bdee64c11: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-30d05822-0955-9ec3-4054-fb4bdee64c11: 2024-06-11T02:29:27.678-1200 [WARN] server-30d05822-0955-9ec3-4054-fb4bdee64c11: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-30d05822-0955-9ec3-4054-fb4bdee64c11: 2024-06-11T02:29:27.678-1200 [INFO] server-30d05822-0955-9ec3-4054-fb4bdee64c11: entering candidate state: node="Node at 30d05822-0955-9ec3-4054-fb4bdee64c11 [Candidate]" term=2 - testing.go:149: server-30d05822-0955-9ec3-4054-fb4bdee64c11: 2024-06-11T02:29:27.678-1200 [INFO] server-30d05822-0955-9ec3-4054-fb4bdee64c11: entering candidate state: node="Node at 30d05822-0955-9ec3-4054-fb4bdee64c11 [Candidate]" term=2 -testLoggerAdapter verbose: server-30d05822-0955-9ec3-4054-fb4bdee64c11: 2024-06-11T02:29:27.678-1200 [INFO] server-30d05822-0955-9ec3-4054-fb4bdee64c11: election won: tally=2 - testing.go:149: server-30d05822-0955-9ec3-4054-fb4bdee64c11: 2024-06-11T02:29:27.678-1200 [INFO] server-30d05822-0955-9ec3-4054-fb4bdee64c11: election won: tally=2 -testLoggerAdapter verbose: server-30d05822-0955-9ec3-4054-fb4bdee64c11: 2024-06-11T02:29:27.679-1200 [INFO] server-30d05822-0955-9ec3-4054-fb4bdee64c11: entering leader state: leader="Node at 30d05822-0955-9ec3-4054-fb4bdee64c11 [Leader]" - testing.go:149: server-30d05822-0955-9ec3-4054-fb4bdee64c11: 2024-06-11T02:29:27.679-1200 [INFO] server-30d05822-0955-9ec3-4054-fb4bdee64c11: entering leader state: leader="Node at 30d05822-0955-9ec3-4054-fb4bdee64c11 [Leader]" -testLoggerAdapter verbose: server-30d05822-0955-9ec3-4054-fb4bdee64c11: 2024-06-11T02:29:27.679-1200 [INFO] server-30d05822-0955-9ec3-4054-fb4bdee64c11: added peer, starting replication: peer=server-bd560674-5316-7c77-7ecf-1ee4735382af - testing.go:149: server-30d05822-0955-9ec3-4054-fb4bdee64c11: 2024-06-11T02:29:27.679-1200 [INFO] server-30d05822-0955-9ec3-4054-fb4bdee64c11: added peer, starting replication: peer=server-bd560674-5316-7c77-7ecf-1ee4735382af -testLoggerAdapter verbose: server-30d05822-0955-9ec3-4054-fb4bdee64c11: 2024-06-11T02:29:27.679-1200 [INFO] server-30d05822-0955-9ec3-4054-fb4bdee64c11: added peer, starting replication: peer=server-56cb3e84-edaf-7027-dbb7-85e912733825 - testing.go:149: server-30d05822-0955-9ec3-4054-fb4bdee64c11: 2024-06-11T02:29:27.679-1200 [INFO] server-30d05822-0955-9ec3-4054-fb4bdee64c11: added peer, starting replication: peer=server-56cb3e84-edaf-7027-dbb7-85e912733825 -testLoggerAdapter verbose: server-30d05822-0955-9ec3-4054-fb4bdee64c11: 2024-06-11T02:29:27.679-1200 [INFO] server-30d05822-0955-9ec3-4054-fb4bdee64c11: pipelining replication: peer="{Voter server-bd560674-5316-7c77-7ecf-1ee4735382af bd560674-5316-7c77-7ecf-1ee4735382af}" - testing.go:149: server-30d05822-0955-9ec3-4054-fb4bdee64c11: 2024-06-11T02:29:27.679-1200 [INFO] server-30d05822-0955-9ec3-4054-fb4bdee64c11: pipelining replication: peer="{Voter server-bd560674-5316-7c77-7ecf-1ee4735382af bd560674-5316-7c77-7ecf-1ee4735382af}" -testLoggerAdapter verbose: server-30d05822-0955-9ec3-4054-fb4bdee64c11: 2024-06-11T02:29:27.679-1200 [INFO] server-30d05822-0955-9ec3-4054-fb4bdee64c11: pipelining replication: peer="{Voter server-56cb3e84-edaf-7027-dbb7-85e912733825 56cb3e84-edaf-7027-dbb7-85e912733825}" - testing.go:149: server-30d05822-0955-9ec3-4054-fb4bdee64c11: 2024-06-11T02:29:27.679-1200 [INFO] server-30d05822-0955-9ec3-4054-fb4bdee64c11: pipelining replication: peer="{Voter server-56cb3e84-edaf-7027-dbb7-85e912733825 56cb3e84-edaf-7027-dbb7-85e912733825}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:27.784-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:27.67923024 -1200 -12 m=+18.674602781 (1 nodes), 52.584177ms from start of poll, 54.309645ms from cluster start. Timeout at 2024-06-11 02:29:27.784331804 -1200 -12 m=+18.779704638, 105.101857ms after stability - testing.go:149: cluster: 2024-06-11T02:29:27.784-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:27.67923024 -1200 -12 m=+18.674602781 (1 nodes), 52.584177ms from start of poll, 54.309645ms from cluster start. Timeout at 2024-06-11 02:29:27.784331804 -1200 -12 m=+18.779704638, 105.101857ms after stability -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:27.784-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:27.784-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:27.890-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:27.784580883 -1200 -12 m=+18.779953425 (1 nodes), 2.625µs from start of poll, 159.660289ms from cluster start. Timeout at 2024-06-11 02:29:27.890100398 -1200 -12 m=+18.885473231, 105.519806ms after stability - testing.go:149: cluster: 2024-06-11T02:29:27.890-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:27.784580883 -1200 -12 m=+18.779953425 (1 nodes), 2.625µs from start of poll, 159.660289ms from cluster start. Timeout at 2024-06-11 02:29:27.890100398 -1200 -12 m=+18.885473231, 105.519806ms after stability -testLoggerAdapter verbose: server-bd560674-5316-7c77-7ecf-1ee4735382af: 2024-06-11T02:29:27.890-1200 [INFO] server-bd560674-5316-7c77-7ecf-1ee4735382af: entering candidate state: node="Node at bd560674-5316-7c77-7ecf-1ee4735382af [Candidate]" term=3 - testing.go:149: server-bd560674-5316-7c77-7ecf-1ee4735382af: 2024-06-11T02:29:27.890-1200 [INFO] server-bd560674-5316-7c77-7ecf-1ee4735382af: entering candidate state: node="Node at bd560674-5316-7c77-7ecf-1ee4735382af [Candidate]" term=3 -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:27.890-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-30d05822-0955-9ec3-4054-fb4bdee64c11: 2024-06-11T02:29:27.890-1200 [INFO] server-30d05822-0955-9ec3-4054-fb4bdee64c11: entering follower state: follower="Node at 30d05822-0955-9ec3-4054-fb4bdee64c11 [Follower]" leader= - testing.go:149: cluster: 2024-06-11T02:29:27.890-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-bd560674-5316-7c77-7ecf-1ee4735382af: 2024-06-11T02:29:27.890-1200 [INFO] server-bd560674-5316-7c77-7ecf-1ee4735382af: election won: tally=2 - testing.go:149: server-30d05822-0955-9ec3-4054-fb4bdee64c11: 2024-06-11T02:29:27.890-1200 [INFO] server-30d05822-0955-9ec3-4054-fb4bdee64c11: entering follower state: follower="Node at 30d05822-0955-9ec3-4054-fb4bdee64c11 [Follower]" leader= - testing.go:149: server-bd560674-5316-7c77-7ecf-1ee4735382af: 2024-06-11T02:29:27.890-1200 [INFO] server-bd560674-5316-7c77-7ecf-1ee4735382af: election won: tally=2 -testLoggerAdapter verbose: server-30d05822-0955-9ec3-4054-fb4bdee64c11: 2024-06-11T02:29:27.890-1200 [INFO] server-30d05822-0955-9ec3-4054-fb4bdee64c11: aborting pipeline replication: peer="{Voter server-56cb3e84-edaf-7027-dbb7-85e912733825 56cb3e84-edaf-7027-dbb7-85e912733825}" -testLoggerAdapter verbose: server-bd560674-5316-7c77-7ecf-1ee4735382af: 2024-06-11T02:29:27.891-1200 [INFO] server-bd560674-5316-7c77-7ecf-1ee4735382af: entering leader state: leader="Node at bd560674-5316-7c77-7ecf-1ee4735382af [Leader]" - testing.go:149: server-30d05822-0955-9ec3-4054-fb4bdee64c11: 2024-06-11T02:29:27.890-1200 [INFO] server-30d05822-0955-9ec3-4054-fb4bdee64c11: aborting pipeline replication: peer="{Voter server-56cb3e84-edaf-7027-dbb7-85e912733825 56cb3e84-edaf-7027-dbb7-85e912733825}" - testing.go:149: server-bd560674-5316-7c77-7ecf-1ee4735382af: 2024-06-11T02:29:27.891-1200 [INFO] server-bd560674-5316-7c77-7ecf-1ee4735382af: entering leader state: leader="Node at bd560674-5316-7c77-7ecf-1ee4735382af [Leader]" -testLoggerAdapter verbose: server-bd560674-5316-7c77-7ecf-1ee4735382af: 2024-06-11T02:29:27.891-1200 [INFO] server-bd560674-5316-7c77-7ecf-1ee4735382af: added peer, starting replication: peer=server-56cb3e84-edaf-7027-dbb7-85e912733825 - testing.go:149: server-bd560674-5316-7c77-7ecf-1ee4735382af: 2024-06-11T02:29:27.891-1200 [INFO] server-bd560674-5316-7c77-7ecf-1ee4735382af: added peer, starting replication: peer=server-56cb3e84-edaf-7027-dbb7-85e912733825 -testLoggerAdapter verbose: server-bd560674-5316-7c77-7ecf-1ee4735382af: 2024-06-11T02:29:27.891-1200 [INFO] server-bd560674-5316-7c77-7ecf-1ee4735382af: added peer, starting replication: peer=server-30d05822-0955-9ec3-4054-fb4bdee64c11 - testing.go:149: server-bd560674-5316-7c77-7ecf-1ee4735382af: 2024-06-11T02:29:27.891-1200 [INFO] server-bd560674-5316-7c77-7ecf-1ee4735382af: added peer, starting replication: peer=server-30d05822-0955-9ec3-4054-fb4bdee64c11 -testLoggerAdapter verbose: server-bd560674-5316-7c77-7ecf-1ee4735382af: 2024-06-11T02:29:27.891-1200 [INFO] server-bd560674-5316-7c77-7ecf-1ee4735382af: pipelining replication: peer="{Voter server-56cb3e84-edaf-7027-dbb7-85e912733825 56cb3e84-edaf-7027-dbb7-85e912733825}" -testLoggerAdapter verbose: server-30d05822-0955-9ec3-4054-fb4bdee64c11: 2024-06-11T02:29:27.890-1200 [INFO] server-30d05822-0955-9ec3-4054-fb4bdee64c11: aborting pipeline replication: peer="{Voter server-bd560674-5316-7c77-7ecf-1ee4735382af bd560674-5316-7c77-7ecf-1ee4735382af}" - testing.go:149: server-bd560674-5316-7c77-7ecf-1ee4735382af: 2024-06-11T02:29:27.891-1200 [INFO] server-bd560674-5316-7c77-7ecf-1ee4735382af: pipelining replication: peer="{Voter server-56cb3e84-edaf-7027-dbb7-85e912733825 56cb3e84-edaf-7027-dbb7-85e912733825}" - testing.go:149: server-30d05822-0955-9ec3-4054-fb4bdee64c11: 2024-06-11T02:29:27.890-1200 [INFO] server-30d05822-0955-9ec3-4054-fb4bdee64c11: aborting pipeline replication: peer="{Voter server-bd560674-5316-7c77-7ecf-1ee4735382af bd560674-5316-7c77-7ecf-1ee4735382af}" -testLoggerAdapter verbose: server-bd560674-5316-7c77-7ecf-1ee4735382af: 2024-06-11T02:29:27.891-1200 [INFO] server-bd560674-5316-7c77-7ecf-1ee4735382af: pipelining replication: peer="{Voter server-30d05822-0955-9ec3-4054-fb4bdee64c11 30d05822-0955-9ec3-4054-fb4bdee64c11}" - testing.go:149: server-bd560674-5316-7c77-7ecf-1ee4735382af: 2024-06-11T02:29:27.891-1200 [INFO] server-bd560674-5316-7c77-7ecf-1ee4735382af: pipelining replication: peer="{Voter server-30d05822-0955-9ec3-4054-fb4bdee64c11 30d05822-0955-9ec3-4054-fb4bdee64c11}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:27.996-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:27.891088838 -1200 -12 m=+18.886461379 (1 nodes), 235.079µs from start of poll, 266.168243ms from cluster start. Timeout at 2024-06-11 02:29:27.996130903 -1200 -12 m=+18.991503737, 105.042358ms after stability - testing.go:149: cluster: 2024-06-11T02:29:27.996-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:27.891088838 -1200 -12 m=+18.886461379 (1 nodes), 235.079µs from start of poll, 266.168243ms from cluster start. Timeout at 2024-06-11 02:29:27.996130903 -1200 -12 m=+18.991503737, 105.042358ms after stability -testLoggerAdapter verbose: server-bd560674-5316-7c77-7ecf-1ee4735382af: 2024-06-11T02:29:27.996-1200 [INFO] server-bd560674-5316-7c77-7ecf-1ee4735382af: aborting pipeline replication: peer="{Voter server-30d05822-0955-9ec3-4054-fb4bdee64c11 30d05822-0955-9ec3-4054-fb4bdee64c11}" - testing.go:149: server-bd560674-5316-7c77-7ecf-1ee4735382af: 2024-06-11T02:29:27.996-1200 [INFO] server-bd560674-5316-7c77-7ecf-1ee4735382af: aborting pipeline replication: peer="{Voter server-30d05822-0955-9ec3-4054-fb4bdee64c11 30d05822-0955-9ec3-4054-fb4bdee64c11}" -testLoggerAdapter verbose: server-bd560674-5316-7c77-7ecf-1ee4735382af: 2024-06-11T02:29:27.996-1200 [INFO] server-bd560674-5316-7c77-7ecf-1ee4735382af: aborting pipeline replication: peer="{Voter server-56cb3e84-edaf-7027-dbb7-85e912733825 56cb3e84-edaf-7027-dbb7-85e912733825}" - testing.go:149: server-bd560674-5316-7c77-7ecf-1ee4735382af: 2024-06-11T02:29:27.996-1200 [INFO] server-bd560674-5316-7c77-7ecf-1ee4735382af: aborting pipeline replication: peer="{Voter server-56cb3e84-edaf-7027-dbb7-85e912733825 56cb3e84-edaf-7027-dbb7-85e912733825}" ---- PASS: TestRaft_LeadershipTransfer (0.38s) -=== RUN TestRaft_LeadershipTransferWithOneNode -testLoggerAdapter verbose: server-40657be8-636f-3689-a739-005e7b01c5ae: 2024-06-11T02:29:28.000-1200 [INFO] server-40657be8-636f-3689-a739-005e7b01c5ae: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-40657be8-636f-3689-a739-005e7b01c5ae Address:40657be8-636f-3689-a739-005e7b01c5ae}]" - testing.go:149: server-40657be8-636f-3689-a739-005e7b01c5ae: 2024-06-11T02:29:28.000-1200 [INFO] server-40657be8-636f-3689-a739-005e7b01c5ae: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-40657be8-636f-3689-a739-005e7b01c5ae Address:40657be8-636f-3689-a739-005e7b01c5ae}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:28.000-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:28.000-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-40657be8-636f-3689-a739-005e7b01c5ae: 2024-06-11T02:29:28.000-1200 [INFO] server-40657be8-636f-3689-a739-005e7b01c5ae: entering follower state: follower="Node at 40657be8-636f-3689-a739-005e7b01c5ae [Follower]" leader= - testing.go:149: server-40657be8-636f-3689-a739-005e7b01c5ae: 2024-06-11T02:29:28.000-1200 [INFO] server-40657be8-636f-3689-a739-005e7b01c5ae: entering follower state: follower="Node at 40657be8-636f-3689-a739-005e7b01c5ae [Follower]" leader= -testLoggerAdapter verbose: server-40657be8-636f-3689-a739-005e7b01c5ae: 2024-06-11T02:29:28.070-1200 [WARN] server-40657be8-636f-3689-a739-005e7b01c5ae: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-40657be8-636f-3689-a739-005e7b01c5ae: 2024-06-11T02:29:28.070-1200 [WARN] server-40657be8-636f-3689-a739-005e7b01c5ae: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-40657be8-636f-3689-a739-005e7b01c5ae: 2024-06-11T02:29:28.070-1200 [INFO] server-40657be8-636f-3689-a739-005e7b01c5ae: entering candidate state: node="Node at 40657be8-636f-3689-a739-005e7b01c5ae [Candidate]" term=2 - testing.go:149: server-40657be8-636f-3689-a739-005e7b01c5ae: 2024-06-11T02:29:28.070-1200 [INFO] server-40657be8-636f-3689-a739-005e7b01c5ae: entering candidate state: node="Node at 40657be8-636f-3689-a739-005e7b01c5ae [Candidate]" term=2 -testLoggerAdapter verbose: server-40657be8-636f-3689-a739-005e7b01c5ae: 2024-06-11T02:29:28.070-1200 [INFO] server-40657be8-636f-3689-a739-005e7b01c5ae: election won: tally=1 - testing.go:149: server-40657be8-636f-3689-a739-005e7b01c5ae: 2024-06-11T02:29:28.070-1200 [INFO] server-40657be8-636f-3689-a739-005e7b01c5ae: election won: tally=1 -testLoggerAdapter verbose: server-40657be8-636f-3689-a739-005e7b01c5ae: 2024-06-11T02:29:28.070-1200 [INFO] server-40657be8-636f-3689-a739-005e7b01c5ae: entering leader state: leader="Node at 40657be8-636f-3689-a739-005e7b01c5ae [Leader]" - testing.go:149: server-40657be8-636f-3689-a739-005e7b01c5ae: 2024-06-11T02:29:28.070-1200 [INFO] server-40657be8-636f-3689-a739-005e7b01c5ae: entering leader state: leader="Node at 40657be8-636f-3689-a739-005e7b01c5ae [Leader]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:28.177-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:28.071173575 -1200 -12 m=+19.066546409 (1 nodes), 70.566256ms from start of poll, 71.168828ms from cluster start. Timeout at 2024-06-11 02:29:28.177208747 -1200 -12 m=+19.172581581, 106.035172ms after stability - testing.go:149: cluster: 2024-06-11T02:29:28.177-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:28.071173575 -1200 -12 m=+19.066546409 (1 nodes), 70.566256ms from start of poll, 71.168828ms from cluster start. Timeout at 2024-06-11 02:29:28.177208747 -1200 -12 m=+19.172581581, 106.035172ms after stability ---- PASS: TestRaft_LeadershipTransferWithOneNode (0.18s) -=== RUN TestRaft_LeadershipTransferWithSevenNodes -testLoggerAdapter verbose: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.183-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd Address:a5b3d033-4dcb-08fe-0c64-70dd120e59fd} {Suffrage:Voter ID:server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516 Address:e1f5d1c8-33ac-1e15-d91d-2f931bd06516} {Suffrage:Voter ID:server-23bcb432-249f-394e-3014-d9c1bd43d174 Address:23bcb432-249f-394e-3014-d9c1bd43d174} {Suffrage:Voter ID:server-8ff76d9d-f0c0-7567-6222-e54e650d330e Address:8ff76d9d-f0c0-7567-6222-e54e650d330e} {Suffrage:Voter ID:server-a6511aea-8f12-9faa-9390-4f7491bb82f0 Address:a6511aea-8f12-9faa-9390-4f7491bb82f0} {Suffrage:Voter ID:server-b75cd5ae-1669-5523-4c28-e4d23dd033ae Address:b75cd5ae-1669-5523-4c28-e4d23dd033ae} {Suffrage:Voter ID:server-7b2c9d34-cd81-f5ce-f708-afa2dae7713b Address:7b2c9d34-cd81-f5ce-f708-afa2dae7713b}]" - testing.go:149: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.183-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd Address:a5b3d033-4dcb-08fe-0c64-70dd120e59fd} {Suffrage:Voter ID:server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516 Address:e1f5d1c8-33ac-1e15-d91d-2f931bd06516} {Suffrage:Voter ID:server-23bcb432-249f-394e-3014-d9c1bd43d174 Address:23bcb432-249f-394e-3014-d9c1bd43d174} {Suffrage:Voter ID:server-8ff76d9d-f0c0-7567-6222-e54e650d330e Address:8ff76d9d-f0c0-7567-6222-e54e650d330e} {Suffrage:Voter ID:server-a6511aea-8f12-9faa-9390-4f7491bb82f0 Address:a6511aea-8f12-9faa-9390-4f7491bb82f0} {Suffrage:Voter ID:server-b75cd5ae-1669-5523-4c28-e4d23dd033ae Address:b75cd5ae-1669-5523-4c28-e4d23dd033ae} {Suffrage:Voter ID:server-7b2c9d34-cd81-f5ce-f708-afa2dae7713b Address:7b2c9d34-cd81-f5ce-f708-afa2dae7713b}]" -testLoggerAdapter verbose: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.183-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: entering follower state: follower="Node at a5b3d033-4dcb-08fe-0c64-70dd120e59fd [Follower]" leader= - testing.go:149: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.183-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: entering follower state: follower="Node at a5b3d033-4dcb-08fe-0c64-70dd120e59fd [Follower]" leader= -testLoggerAdapter verbose: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.184-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd Address:a5b3d033-4dcb-08fe-0c64-70dd120e59fd} {Suffrage:Voter ID:server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516 Address:e1f5d1c8-33ac-1e15-d91d-2f931bd06516} {Suffrage:Voter ID:server-23bcb432-249f-394e-3014-d9c1bd43d174 Address:23bcb432-249f-394e-3014-d9c1bd43d174} {Suffrage:Voter ID:server-8ff76d9d-f0c0-7567-6222-e54e650d330e Address:8ff76d9d-f0c0-7567-6222-e54e650d330e} {Suffrage:Voter ID:server-a6511aea-8f12-9faa-9390-4f7491bb82f0 Address:a6511aea-8f12-9faa-9390-4f7491bb82f0} {Suffrage:Voter ID:server-b75cd5ae-1669-5523-4c28-e4d23dd033ae Address:b75cd5ae-1669-5523-4c28-e4d23dd033ae} {Suffrage:Voter ID:server-7b2c9d34-cd81-f5ce-f708-afa2dae7713b Address:7b2c9d34-cd81-f5ce-f708-afa2dae7713b}]" - testing.go:149: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.184-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd Address:a5b3d033-4dcb-08fe-0c64-70dd120e59fd} {Suffrage:Voter ID:server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516 Address:e1f5d1c8-33ac-1e15-d91d-2f931bd06516} {Suffrage:Voter ID:server-23bcb432-249f-394e-3014-d9c1bd43d174 Address:23bcb432-249f-394e-3014-d9c1bd43d174} {Suffrage:Voter ID:server-8ff76d9d-f0c0-7567-6222-e54e650d330e Address:8ff76d9d-f0c0-7567-6222-e54e650d330e} {Suffrage:Voter ID:server-a6511aea-8f12-9faa-9390-4f7491bb82f0 Address:a6511aea-8f12-9faa-9390-4f7491bb82f0} {Suffrage:Voter ID:server-b75cd5ae-1669-5523-4c28-e4d23dd033ae Address:b75cd5ae-1669-5523-4c28-e4d23dd033ae} {Suffrage:Voter ID:server-7b2c9d34-cd81-f5ce-f708-afa2dae7713b Address:7b2c9d34-cd81-f5ce-f708-afa2dae7713b}]" -testLoggerAdapter verbose: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.184-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: entering follower state: follower="Node at e1f5d1c8-33ac-1e15-d91d-2f931bd06516 [Follower]" leader= - testing.go:149: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.184-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: entering follower state: follower="Node at e1f5d1c8-33ac-1e15-d91d-2f931bd06516 [Follower]" leader= -testLoggerAdapter verbose: server-23bcb432-249f-394e-3014-d9c1bd43d174: 2024-06-11T02:29:28.184-1200 [INFO] server-23bcb432-249f-394e-3014-d9c1bd43d174: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd Address:a5b3d033-4dcb-08fe-0c64-70dd120e59fd} {Suffrage:Voter ID:server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516 Address:e1f5d1c8-33ac-1e15-d91d-2f931bd06516} {Suffrage:Voter ID:server-23bcb432-249f-394e-3014-d9c1bd43d174 Address:23bcb432-249f-394e-3014-d9c1bd43d174} {Suffrage:Voter ID:server-8ff76d9d-f0c0-7567-6222-e54e650d330e Address:8ff76d9d-f0c0-7567-6222-e54e650d330e} {Suffrage:Voter ID:server-a6511aea-8f12-9faa-9390-4f7491bb82f0 Address:a6511aea-8f12-9faa-9390-4f7491bb82f0} {Suffrage:Voter ID:server-b75cd5ae-1669-5523-4c28-e4d23dd033ae Address:b75cd5ae-1669-5523-4c28-e4d23dd033ae} {Suffrage:Voter ID:server-7b2c9d34-cd81-f5ce-f708-afa2dae7713b Address:7b2c9d34-cd81-f5ce-f708-afa2dae7713b}]" - testing.go:149: server-23bcb432-249f-394e-3014-d9c1bd43d174: 2024-06-11T02:29:28.184-1200 [INFO] server-23bcb432-249f-394e-3014-d9c1bd43d174: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd Address:a5b3d033-4dcb-08fe-0c64-70dd120e59fd} {Suffrage:Voter ID:server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516 Address:e1f5d1c8-33ac-1e15-d91d-2f931bd06516} {Suffrage:Voter ID:server-23bcb432-249f-394e-3014-d9c1bd43d174 Address:23bcb432-249f-394e-3014-d9c1bd43d174} {Suffrage:Voter ID:server-8ff76d9d-f0c0-7567-6222-e54e650d330e Address:8ff76d9d-f0c0-7567-6222-e54e650d330e} {Suffrage:Voter ID:server-a6511aea-8f12-9faa-9390-4f7491bb82f0 Address:a6511aea-8f12-9faa-9390-4f7491bb82f0} {Suffrage:Voter ID:server-b75cd5ae-1669-5523-4c28-e4d23dd033ae Address:b75cd5ae-1669-5523-4c28-e4d23dd033ae} {Suffrage:Voter ID:server-7b2c9d34-cd81-f5ce-f708-afa2dae7713b Address:7b2c9d34-cd81-f5ce-f708-afa2dae7713b}]" -testLoggerAdapter verbose: server-23bcb432-249f-394e-3014-d9c1bd43d174: 2024-06-11T02:29:28.184-1200 [INFO] server-23bcb432-249f-394e-3014-d9c1bd43d174: entering follower state: follower="Node at 23bcb432-249f-394e-3014-d9c1bd43d174 [Follower]" leader= - testing.go:149: server-23bcb432-249f-394e-3014-d9c1bd43d174: 2024-06-11T02:29:28.184-1200 [INFO] server-23bcb432-249f-394e-3014-d9c1bd43d174: entering follower state: follower="Node at 23bcb432-249f-394e-3014-d9c1bd43d174 [Follower]" leader= -testLoggerAdapter verbose: server-8ff76d9d-f0c0-7567-6222-e54e650d330e: 2024-06-11T02:29:28.185-1200 [INFO] server-8ff76d9d-f0c0-7567-6222-e54e650d330e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd Address:a5b3d033-4dcb-08fe-0c64-70dd120e59fd} {Suffrage:Voter ID:server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516 Address:e1f5d1c8-33ac-1e15-d91d-2f931bd06516} {Suffrage:Voter ID:server-23bcb432-249f-394e-3014-d9c1bd43d174 Address:23bcb432-249f-394e-3014-d9c1bd43d174} {Suffrage:Voter ID:server-8ff76d9d-f0c0-7567-6222-e54e650d330e Address:8ff76d9d-f0c0-7567-6222-e54e650d330e} {Suffrage:Voter ID:server-a6511aea-8f12-9faa-9390-4f7491bb82f0 Address:a6511aea-8f12-9faa-9390-4f7491bb82f0} {Suffrage:Voter ID:server-b75cd5ae-1669-5523-4c28-e4d23dd033ae Address:b75cd5ae-1669-5523-4c28-e4d23dd033ae} {Suffrage:Voter ID:server-7b2c9d34-cd81-f5ce-f708-afa2dae7713b Address:7b2c9d34-cd81-f5ce-f708-afa2dae7713b}]" - testing.go:149: server-8ff76d9d-f0c0-7567-6222-e54e650d330e: 2024-06-11T02:29:28.185-1200 [INFO] server-8ff76d9d-f0c0-7567-6222-e54e650d330e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd Address:a5b3d033-4dcb-08fe-0c64-70dd120e59fd} {Suffrage:Voter ID:server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516 Address:e1f5d1c8-33ac-1e15-d91d-2f931bd06516} {Suffrage:Voter ID:server-23bcb432-249f-394e-3014-d9c1bd43d174 Address:23bcb432-249f-394e-3014-d9c1bd43d174} {Suffrage:Voter ID:server-8ff76d9d-f0c0-7567-6222-e54e650d330e Address:8ff76d9d-f0c0-7567-6222-e54e650d330e} {Suffrage:Voter ID:server-a6511aea-8f12-9faa-9390-4f7491bb82f0 Address:a6511aea-8f12-9faa-9390-4f7491bb82f0} {Suffrage:Voter ID:server-b75cd5ae-1669-5523-4c28-e4d23dd033ae Address:b75cd5ae-1669-5523-4c28-e4d23dd033ae} {Suffrage:Voter ID:server-7b2c9d34-cd81-f5ce-f708-afa2dae7713b Address:7b2c9d34-cd81-f5ce-f708-afa2dae7713b}]" -testLoggerAdapter verbose: server-8ff76d9d-f0c0-7567-6222-e54e650d330e: 2024-06-11T02:29:28.185-1200 [INFO] server-8ff76d9d-f0c0-7567-6222-e54e650d330e: entering follower state: follower="Node at 8ff76d9d-f0c0-7567-6222-e54e650d330e [Follower]" leader= - testing.go:149: server-8ff76d9d-f0c0-7567-6222-e54e650d330e: 2024-06-11T02:29:28.185-1200 [INFO] server-8ff76d9d-f0c0-7567-6222-e54e650d330e: entering follower state: follower="Node at 8ff76d9d-f0c0-7567-6222-e54e650d330e [Follower]" leader= -testLoggerAdapter verbose: server-a6511aea-8f12-9faa-9390-4f7491bb82f0: 2024-06-11T02:29:28.185-1200 [INFO] server-a6511aea-8f12-9faa-9390-4f7491bb82f0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd Address:a5b3d033-4dcb-08fe-0c64-70dd120e59fd} {Suffrage:Voter ID:server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516 Address:e1f5d1c8-33ac-1e15-d91d-2f931bd06516} {Suffrage:Voter ID:server-23bcb432-249f-394e-3014-d9c1bd43d174 Address:23bcb432-249f-394e-3014-d9c1bd43d174} {Suffrage:Voter ID:server-8ff76d9d-f0c0-7567-6222-e54e650d330e Address:8ff76d9d-f0c0-7567-6222-e54e650d330e} {Suffrage:Voter ID:server-a6511aea-8f12-9faa-9390-4f7491bb82f0 Address:a6511aea-8f12-9faa-9390-4f7491bb82f0} {Suffrage:Voter ID:server-b75cd5ae-1669-5523-4c28-e4d23dd033ae Address:b75cd5ae-1669-5523-4c28-e4d23dd033ae} {Suffrage:Voter ID:server-7b2c9d34-cd81-f5ce-f708-afa2dae7713b Address:7b2c9d34-cd81-f5ce-f708-afa2dae7713b}]" - testing.go:149: server-a6511aea-8f12-9faa-9390-4f7491bb82f0: 2024-06-11T02:29:28.185-1200 [INFO] server-a6511aea-8f12-9faa-9390-4f7491bb82f0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd Address:a5b3d033-4dcb-08fe-0c64-70dd120e59fd} {Suffrage:Voter ID:server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516 Address:e1f5d1c8-33ac-1e15-d91d-2f931bd06516} {Suffrage:Voter ID:server-23bcb432-249f-394e-3014-d9c1bd43d174 Address:23bcb432-249f-394e-3014-d9c1bd43d174} {Suffrage:Voter ID:server-8ff76d9d-f0c0-7567-6222-e54e650d330e Address:8ff76d9d-f0c0-7567-6222-e54e650d330e} {Suffrage:Voter ID:server-a6511aea-8f12-9faa-9390-4f7491bb82f0 Address:a6511aea-8f12-9faa-9390-4f7491bb82f0} {Suffrage:Voter ID:server-b75cd5ae-1669-5523-4c28-e4d23dd033ae Address:b75cd5ae-1669-5523-4c28-e4d23dd033ae} {Suffrage:Voter ID:server-7b2c9d34-cd81-f5ce-f708-afa2dae7713b Address:7b2c9d34-cd81-f5ce-f708-afa2dae7713b}]" -testLoggerAdapter verbose: server-a6511aea-8f12-9faa-9390-4f7491bb82f0: 2024-06-11T02:29:28.185-1200 [INFO] server-a6511aea-8f12-9faa-9390-4f7491bb82f0: entering follower state: follower="Node at a6511aea-8f12-9faa-9390-4f7491bb82f0 [Follower]" leader= - testing.go:149: server-a6511aea-8f12-9faa-9390-4f7491bb82f0: 2024-06-11T02:29:28.185-1200 [INFO] server-a6511aea-8f12-9faa-9390-4f7491bb82f0: entering follower state: follower="Node at a6511aea-8f12-9faa-9390-4f7491bb82f0 [Follower]" leader= -testLoggerAdapter verbose: server-b75cd5ae-1669-5523-4c28-e4d23dd033ae: 2024-06-11T02:29:28.186-1200 [INFO] server-b75cd5ae-1669-5523-4c28-e4d23dd033ae: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd Address:a5b3d033-4dcb-08fe-0c64-70dd120e59fd} {Suffrage:Voter ID:server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516 Address:e1f5d1c8-33ac-1e15-d91d-2f931bd06516} {Suffrage:Voter ID:server-23bcb432-249f-394e-3014-d9c1bd43d174 Address:23bcb432-249f-394e-3014-d9c1bd43d174} {Suffrage:Voter ID:server-8ff76d9d-f0c0-7567-6222-e54e650d330e Address:8ff76d9d-f0c0-7567-6222-e54e650d330e} {Suffrage:Voter ID:server-a6511aea-8f12-9faa-9390-4f7491bb82f0 Address:a6511aea-8f12-9faa-9390-4f7491bb82f0} {Suffrage:Voter ID:server-b75cd5ae-1669-5523-4c28-e4d23dd033ae Address:b75cd5ae-1669-5523-4c28-e4d23dd033ae} {Suffrage:Voter ID:server-7b2c9d34-cd81-f5ce-f708-afa2dae7713b Address:7b2c9d34-cd81-f5ce-f708-afa2dae7713b}]" - testing.go:149: server-b75cd5ae-1669-5523-4c28-e4d23dd033ae: 2024-06-11T02:29:28.186-1200 [INFO] server-b75cd5ae-1669-5523-4c28-e4d23dd033ae: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd Address:a5b3d033-4dcb-08fe-0c64-70dd120e59fd} {Suffrage:Voter ID:server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516 Address:e1f5d1c8-33ac-1e15-d91d-2f931bd06516} {Suffrage:Voter ID:server-23bcb432-249f-394e-3014-d9c1bd43d174 Address:23bcb432-249f-394e-3014-d9c1bd43d174} {Suffrage:Voter ID:server-8ff76d9d-f0c0-7567-6222-e54e650d330e Address:8ff76d9d-f0c0-7567-6222-e54e650d330e} {Suffrage:Voter ID:server-a6511aea-8f12-9faa-9390-4f7491bb82f0 Address:a6511aea-8f12-9faa-9390-4f7491bb82f0} {Suffrage:Voter ID:server-b75cd5ae-1669-5523-4c28-e4d23dd033ae Address:b75cd5ae-1669-5523-4c28-e4d23dd033ae} {Suffrage:Voter ID:server-7b2c9d34-cd81-f5ce-f708-afa2dae7713b Address:7b2c9d34-cd81-f5ce-f708-afa2dae7713b}]" -testLoggerAdapter verbose: server-b75cd5ae-1669-5523-4c28-e4d23dd033ae: 2024-06-11T02:29:28.186-1200 [INFO] server-b75cd5ae-1669-5523-4c28-e4d23dd033ae: entering follower state: follower="Node at b75cd5ae-1669-5523-4c28-e4d23dd033ae [Follower]" leader= - testing.go:149: server-b75cd5ae-1669-5523-4c28-e4d23dd033ae: 2024-06-11T02:29:28.186-1200 [INFO] server-b75cd5ae-1669-5523-4c28-e4d23dd033ae: entering follower state: follower="Node at b75cd5ae-1669-5523-4c28-e4d23dd033ae [Follower]" leader= -testLoggerAdapter verbose: server-7b2c9d34-cd81-f5ce-f708-afa2dae7713b: 2024-06-11T02:29:28.186-1200 [INFO] server-7b2c9d34-cd81-f5ce-f708-afa2dae7713b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd Address:a5b3d033-4dcb-08fe-0c64-70dd120e59fd} {Suffrage:Voter ID:server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516 Address:e1f5d1c8-33ac-1e15-d91d-2f931bd06516} {Suffrage:Voter ID:server-23bcb432-249f-394e-3014-d9c1bd43d174 Address:23bcb432-249f-394e-3014-d9c1bd43d174} {Suffrage:Voter ID:server-8ff76d9d-f0c0-7567-6222-e54e650d330e Address:8ff76d9d-f0c0-7567-6222-e54e650d330e} {Suffrage:Voter ID:server-a6511aea-8f12-9faa-9390-4f7491bb82f0 Address:a6511aea-8f12-9faa-9390-4f7491bb82f0} {Suffrage:Voter ID:server-b75cd5ae-1669-5523-4c28-e4d23dd033ae Address:b75cd5ae-1669-5523-4c28-e4d23dd033ae} {Suffrage:Voter ID:server-7b2c9d34-cd81-f5ce-f708-afa2dae7713b Address:7b2c9d34-cd81-f5ce-f708-afa2dae7713b}]" - testing.go:149: server-7b2c9d34-cd81-f5ce-f708-afa2dae7713b: 2024-06-11T02:29:28.186-1200 [INFO] server-7b2c9d34-cd81-f5ce-f708-afa2dae7713b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd Address:a5b3d033-4dcb-08fe-0c64-70dd120e59fd} {Suffrage:Voter ID:server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516 Address:e1f5d1c8-33ac-1e15-d91d-2f931bd06516} {Suffrage:Voter ID:server-23bcb432-249f-394e-3014-d9c1bd43d174 Address:23bcb432-249f-394e-3014-d9c1bd43d174} {Suffrage:Voter ID:server-8ff76d9d-f0c0-7567-6222-e54e650d330e Address:8ff76d9d-f0c0-7567-6222-e54e650d330e} {Suffrage:Voter ID:server-a6511aea-8f12-9faa-9390-4f7491bb82f0 Address:a6511aea-8f12-9faa-9390-4f7491bb82f0} {Suffrage:Voter ID:server-b75cd5ae-1669-5523-4c28-e4d23dd033ae Address:b75cd5ae-1669-5523-4c28-e4d23dd033ae} {Suffrage:Voter ID:server-7b2c9d34-cd81-f5ce-f708-afa2dae7713b Address:7b2c9d34-cd81-f5ce-f708-afa2dae7713b}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:28.186-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:28.186-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-7b2c9d34-cd81-f5ce-f708-afa2dae7713b: 2024-06-11T02:29:28.186-1200 [INFO] server-7b2c9d34-cd81-f5ce-f708-afa2dae7713b: entering follower state: follower="Node at 7b2c9d34-cd81-f5ce-f708-afa2dae7713b [Follower]" leader= - testing.go:149: server-7b2c9d34-cd81-f5ce-f708-afa2dae7713b: 2024-06-11T02:29:28.186-1200 [INFO] server-7b2c9d34-cd81-f5ce-f708-afa2dae7713b: entering follower state: follower="Node at 7b2c9d34-cd81-f5ce-f708-afa2dae7713b [Follower]" leader= -testLoggerAdapter verbose: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.245-1200 [WARN] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.245-1200 [WARN] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.245-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: entering candidate state: node="Node at e1f5d1c8-33ac-1e15-d91d-2f931bd06516 [Candidate]" term=2 - testing.go:149: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.245-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: entering candidate state: node="Node at e1f5d1c8-33ac-1e15-d91d-2f931bd06516 [Candidate]" term=2 -testLoggerAdapter verbose: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.246-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: election won: tally=4 - testing.go:149: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.246-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: election won: tally=4 -testLoggerAdapter verbose: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.246-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: entering leader state: leader="Node at e1f5d1c8-33ac-1e15-d91d-2f931bd06516 [Leader]" - testing.go:149: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.246-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: entering leader state: leader="Node at e1f5d1c8-33ac-1e15-d91d-2f931bd06516 [Leader]" -testLoggerAdapter verbose: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.246-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: added peer, starting replication: peer=server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd - testing.go:149: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.246-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: added peer, starting replication: peer=server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd -testLoggerAdapter verbose: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.246-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: added peer, starting replication: peer=server-23bcb432-249f-394e-3014-d9c1bd43d174 - testing.go:149: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.246-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: added peer, starting replication: peer=server-23bcb432-249f-394e-3014-d9c1bd43d174 -testLoggerAdapter verbose: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.246-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: added peer, starting replication: peer=server-8ff76d9d-f0c0-7567-6222-e54e650d330e - testing.go:149: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.246-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: added peer, starting replication: peer=server-8ff76d9d-f0c0-7567-6222-e54e650d330e -testLoggerAdapter verbose: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.246-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: added peer, starting replication: peer=server-a6511aea-8f12-9faa-9390-4f7491bb82f0 - testing.go:149: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.246-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: added peer, starting replication: peer=server-a6511aea-8f12-9faa-9390-4f7491bb82f0 -testLoggerAdapter verbose: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.246-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: added peer, starting replication: peer=server-b75cd5ae-1669-5523-4c28-e4d23dd033ae - testing.go:149: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.246-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: added peer, starting replication: peer=server-b75cd5ae-1669-5523-4c28-e4d23dd033ae -testLoggerAdapter verbose: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.246-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: added peer, starting replication: peer=server-7b2c9d34-cd81-f5ce-f708-afa2dae7713b - testing.go:149: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.246-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: added peer, starting replication: peer=server-7b2c9d34-cd81-f5ce-f708-afa2dae7713b -testLoggerAdapter verbose: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.246-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: pipelining replication: peer="{Voter server-7b2c9d34-cd81-f5ce-f708-afa2dae7713b 7b2c9d34-cd81-f5ce-f708-afa2dae7713b}" - testing.go:149: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.246-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: pipelining replication: peer="{Voter server-7b2c9d34-cd81-f5ce-f708-afa2dae7713b 7b2c9d34-cd81-f5ce-f708-afa2dae7713b}" -testLoggerAdapter verbose: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.247-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: pipelining replication: peer="{Voter server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd a5b3d033-4dcb-08fe-0c64-70dd120e59fd}" - testing.go:149: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.247-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: pipelining replication: peer="{Voter server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd a5b3d033-4dcb-08fe-0c64-70dd120e59fd}" -testLoggerAdapter verbose: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.247-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: pipelining replication: peer="{Voter server-b75cd5ae-1669-5523-4c28-e4d23dd033ae b75cd5ae-1669-5523-4c28-e4d23dd033ae}" - testing.go:149: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.247-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: pipelining replication: peer="{Voter server-b75cd5ae-1669-5523-4c28-e4d23dd033ae b75cd5ae-1669-5523-4c28-e4d23dd033ae}" -testLoggerAdapter verbose: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.247-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: pipelining replication: peer="{Voter server-23bcb432-249f-394e-3014-d9c1bd43d174 23bcb432-249f-394e-3014-d9c1bd43d174}" - testing.go:149: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.247-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: pipelining replication: peer="{Voter server-23bcb432-249f-394e-3014-d9c1bd43d174 23bcb432-249f-394e-3014-d9c1bd43d174}" -testLoggerAdapter verbose: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.247-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: pipelining replication: peer="{Voter server-8ff76d9d-f0c0-7567-6222-e54e650d330e 8ff76d9d-f0c0-7567-6222-e54e650d330e}" - testing.go:149: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.247-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: pipelining replication: peer="{Voter server-8ff76d9d-f0c0-7567-6222-e54e650d330e 8ff76d9d-f0c0-7567-6222-e54e650d330e}" -testLoggerAdapter verbose: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.247-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: pipelining replication: peer="{Voter server-a6511aea-8f12-9faa-9390-4f7491bb82f0 a6511aea-8f12-9faa-9390-4f7491bb82f0}" - testing.go:149: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.247-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: pipelining replication: peer="{Voter server-a6511aea-8f12-9faa-9390-4f7491bb82f0 a6511aea-8f12-9faa-9390-4f7491bb82f0}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:28.353-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:28.247044392 -1200 -12 m=+19.242416934 (1 nodes), 60.331574ms from start of poll, 63.944381ms from cluster start. Timeout at 2024-06-11 02:29:28.352902817 -1200 -12 m=+19.348275359, 105.858425ms after stability - testing.go:149: cluster: 2024-06-11T02:29:28.353-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:28.247044392 -1200 -12 m=+19.242416934 (1 nodes), 60.331574ms from start of poll, 63.944381ms from cluster start. Timeout at 2024-06-11 02:29:28.352902817 -1200 -12 m=+19.348275359, 105.858425ms after stability -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:28.353-1200 [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2024-06-11T02:29:28.353-1200 [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:28.459-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:28.353236478 -1200 -12 m=+19.348609019 (6 nodes), 6.416µs from start of poll, 170.136466ms from cluster start. Timeout at 2024-06-11 02:29:28.459182401 -1200 -12 m=+19.454555235, 105.946216ms after stability - testing.go:149: cluster: 2024-06-11T02:29:28.459-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:28.353236478 -1200 -12 m=+19.348609019 (6 nodes), 6.416µs from start of poll, 170.136466ms from cluster start. Timeout at 2024-06-11 02:29:28.459182401 -1200 -12 m=+19.454555235, 105.946216ms after stability -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:28.459-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:28.459-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:28.565-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:28.459514895 -1200 -12 m=+19.454887437 (1 nodes), 3.792µs from start of poll, 276.414884ms from cluster start. Timeout at 2024-06-11 02:29:28.565082243 -1200 -12 m=+19.560454784, 105.567347ms after stability - testing.go:149: cluster: 2024-06-11T02:29:28.565-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:28.459514895 -1200 -12 m=+19.454887437 (1 nodes), 3.792µs from start of poll, 276.414884ms from cluster start. Timeout at 2024-06-11 02:29:28.565082243 -1200 -12 m=+19.560454784, 105.567347ms after stability -testLoggerAdapter verbose: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.565-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: entering candidate state: node="Node at a5b3d033-4dcb-08fe-0c64-70dd120e59fd [Candidate]" term=3 - testing.go:149: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.565-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: entering candidate state: node="Node at a5b3d033-4dcb-08fe-0c64-70dd120e59fd [Candidate]" term=3 -testLoggerAdapter verbose: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.565-1200 [ERROR] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: peer has newer term, stopping replication: peer="{Voter server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd a5b3d033-4dcb-08fe-0c64-70dd120e59fd}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:28.565-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.565-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: election won: tally=4 - testing.go:149: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.565-1200 [ERROR] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: peer has newer term, stopping replication: peer="{Voter server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd a5b3d033-4dcb-08fe-0c64-70dd120e59fd}" -testLoggerAdapter verbose: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.566-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: aborting pipeline replication: peer="{Voter server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd a5b3d033-4dcb-08fe-0c64-70dd120e59fd}" - testing.go:149: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.566-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: aborting pipeline replication: peer="{Voter server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd a5b3d033-4dcb-08fe-0c64-70dd120e59fd}" - testing.go:149: cluster: 2024-06-11T02:29:28.565-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.566-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: entering follower state: follower="Node at e1f5d1c8-33ac-1e15-d91d-2f931bd06516 [Follower]" leader= - testing.go:149: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.566-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: entering follower state: follower="Node at e1f5d1c8-33ac-1e15-d91d-2f931bd06516 [Follower]" leader= -testLoggerAdapter verbose: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.566-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: aborting pipeline replication: peer="{Voter server-7b2c9d34-cd81-f5ce-f708-afa2dae7713b 7b2c9d34-cd81-f5ce-f708-afa2dae7713b}" - testing.go:149: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.565-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: election won: tally=4 -testLoggerAdapter verbose: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.566-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: entering leader state: leader="Node at a5b3d033-4dcb-08fe-0c64-70dd120e59fd [Leader]" - testing.go:149: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.566-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: entering leader state: leader="Node at a5b3d033-4dcb-08fe-0c64-70dd120e59fd [Leader]" -testLoggerAdapter verbose: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.566-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: added peer, starting replication: peer=server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516 - testing.go:149: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.566-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: added peer, starting replication: peer=server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516 - testing.go:149: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.566-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: aborting pipeline replication: peer="{Voter server-7b2c9d34-cd81-f5ce-f708-afa2dae7713b 7b2c9d34-cd81-f5ce-f708-afa2dae7713b}" -testLoggerAdapter verbose: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.566-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: added peer, starting replication: peer=server-23bcb432-249f-394e-3014-d9c1bd43d174 - testing.go:149: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.566-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: added peer, starting replication: peer=server-23bcb432-249f-394e-3014-d9c1bd43d174 -testLoggerAdapter verbose: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.566-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: pipelining replication: peer="{Voter server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516 e1f5d1c8-33ac-1e15-d91d-2f931bd06516}" - testing.go:149: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.566-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: pipelining replication: peer="{Voter server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516 e1f5d1c8-33ac-1e15-d91d-2f931bd06516}" -testLoggerAdapter verbose: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.566-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: aborting pipeline replication: peer="{Voter server-23bcb432-249f-394e-3014-d9c1bd43d174 23bcb432-249f-394e-3014-d9c1bd43d174}" - testing.go:149: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.566-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: aborting pipeline replication: peer="{Voter server-23bcb432-249f-394e-3014-d9c1bd43d174 23bcb432-249f-394e-3014-d9c1bd43d174}" -testLoggerAdapter verbose: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.566-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: added peer, starting replication: peer=server-8ff76d9d-f0c0-7567-6222-e54e650d330e - testing.go:149: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.566-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: added peer, starting replication: peer=server-8ff76d9d-f0c0-7567-6222-e54e650d330e -testLoggerAdapter verbose: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.566-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: aborting pipeline replication: peer="{Voter server-8ff76d9d-f0c0-7567-6222-e54e650d330e 8ff76d9d-f0c0-7567-6222-e54e650d330e}" - testing.go:149: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.566-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: aborting pipeline replication: peer="{Voter server-8ff76d9d-f0c0-7567-6222-e54e650d330e 8ff76d9d-f0c0-7567-6222-e54e650d330e}" -testLoggerAdapter verbose: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.567-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: added peer, starting replication: peer=server-a6511aea-8f12-9faa-9390-4f7491bb82f0 - testing.go:149: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.567-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: added peer, starting replication: peer=server-a6511aea-8f12-9faa-9390-4f7491bb82f0 -testLoggerAdapter verbose: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.566-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: aborting pipeline replication: peer="{Voter server-a6511aea-8f12-9faa-9390-4f7491bb82f0 a6511aea-8f12-9faa-9390-4f7491bb82f0}" -testLoggerAdapter verbose: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.567-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: added peer, starting replication: peer=server-b75cd5ae-1669-5523-4c28-e4d23dd033ae - testing.go:149: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.566-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: aborting pipeline replication: peer="{Voter server-a6511aea-8f12-9faa-9390-4f7491bb82f0 a6511aea-8f12-9faa-9390-4f7491bb82f0}" - testing.go:149: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.567-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: added peer, starting replication: peer=server-b75cd5ae-1669-5523-4c28-e4d23dd033ae -testLoggerAdapter verbose: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.567-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: pipelining replication: peer="{Voter server-a6511aea-8f12-9faa-9390-4f7491bb82f0 a6511aea-8f12-9faa-9390-4f7491bb82f0}" - testing.go:149: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.567-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: pipelining replication: peer="{Voter server-a6511aea-8f12-9faa-9390-4f7491bb82f0 a6511aea-8f12-9faa-9390-4f7491bb82f0}" -testLoggerAdapter verbose: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.566-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: aborting pipeline replication: peer="{Voter server-b75cd5ae-1669-5523-4c28-e4d23dd033ae b75cd5ae-1669-5523-4c28-e4d23dd033ae}" - testing.go:149: server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: 2024-06-11T02:29:28.566-1200 [INFO] server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516: aborting pipeline replication: peer="{Voter server-b75cd5ae-1669-5523-4c28-e4d23dd033ae b75cd5ae-1669-5523-4c28-e4d23dd033ae}" -testLoggerAdapter verbose: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.567-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: pipelining replication: peer="{Voter server-23bcb432-249f-394e-3014-d9c1bd43d174 23bcb432-249f-394e-3014-d9c1bd43d174}" - testing.go:149: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.567-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: pipelining replication: peer="{Voter server-23bcb432-249f-394e-3014-d9c1bd43d174 23bcb432-249f-394e-3014-d9c1bd43d174}" -testLoggerAdapter verbose: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.567-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: added peer, starting replication: peer=server-7b2c9d34-cd81-f5ce-f708-afa2dae7713b - testing.go:149: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.567-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: added peer, starting replication: peer=server-7b2c9d34-cd81-f5ce-f708-afa2dae7713b -testLoggerAdapter verbose: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.567-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: pipelining replication: peer="{Voter server-8ff76d9d-f0c0-7567-6222-e54e650d330e 8ff76d9d-f0c0-7567-6222-e54e650d330e}" - testing.go:149: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.567-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: pipelining replication: peer="{Voter server-8ff76d9d-f0c0-7567-6222-e54e650d330e 8ff76d9d-f0c0-7567-6222-e54e650d330e}" -testLoggerAdapter verbose: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.567-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: pipelining replication: peer="{Voter server-b75cd5ae-1669-5523-4c28-e4d23dd033ae b75cd5ae-1669-5523-4c28-e4d23dd033ae}" - testing.go:149: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.567-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: pipelining replication: peer="{Voter server-b75cd5ae-1669-5523-4c28-e4d23dd033ae b75cd5ae-1669-5523-4c28-e4d23dd033ae}" -testLoggerAdapter verbose: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.568-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: pipelining replication: peer="{Voter server-7b2c9d34-cd81-f5ce-f708-afa2dae7713b 7b2c9d34-cd81-f5ce-f708-afa2dae7713b}" - testing.go:149: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.568-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: pipelining replication: peer="{Voter server-7b2c9d34-cd81-f5ce-f708-afa2dae7713b 7b2c9d34-cd81-f5ce-f708-afa2dae7713b}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:28.671-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:28.566586631 -1200 -12 m=+19.561959756 (1 nodes), 418.825µs from start of poll, 383.487203ms from cluster start. Timeout at 2024-06-11 02:29:28.671790569 -1200 -12 m=+19.667163111, 105.203355ms after stability - testing.go:149: cluster: 2024-06-11T02:29:28.671-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:28.566586631 -1200 -12 m=+19.561959756 (1 nodes), 418.825µs from start of poll, 383.487203ms from cluster start. Timeout at 2024-06-11 02:29:28.671790569 -1200 -12 m=+19.667163111, 105.203355ms after stability -testLoggerAdapter verbose: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.672-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: aborting pipeline replication: peer="{Voter server-7b2c9d34-cd81-f5ce-f708-afa2dae7713b 7b2c9d34-cd81-f5ce-f708-afa2dae7713b}" - testing.go:149: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.672-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: aborting pipeline replication: peer="{Voter server-7b2c9d34-cd81-f5ce-f708-afa2dae7713b 7b2c9d34-cd81-f5ce-f708-afa2dae7713b}" -testLoggerAdapter verbose: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.672-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: aborting pipeline replication: peer="{Voter server-8ff76d9d-f0c0-7567-6222-e54e650d330e 8ff76d9d-f0c0-7567-6222-e54e650d330e}" - testing.go:149: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.672-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: aborting pipeline replication: peer="{Voter server-8ff76d9d-f0c0-7567-6222-e54e650d330e 8ff76d9d-f0c0-7567-6222-e54e650d330e}" -testLoggerAdapter verbose: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.672-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: aborting pipeline replication: peer="{Voter server-23bcb432-249f-394e-3014-d9c1bd43d174 23bcb432-249f-394e-3014-d9c1bd43d174}" - testing.go:149: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.672-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: aborting pipeline replication: peer="{Voter server-23bcb432-249f-394e-3014-d9c1bd43d174 23bcb432-249f-394e-3014-d9c1bd43d174}" -testLoggerAdapter verbose: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.672-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: aborting pipeline replication: peer="{Voter server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516 e1f5d1c8-33ac-1e15-d91d-2f931bd06516}" - testing.go:149: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.672-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: aborting pipeline replication: peer="{Voter server-e1f5d1c8-33ac-1e15-d91d-2f931bd06516 e1f5d1c8-33ac-1e15-d91d-2f931bd06516}" -testLoggerAdapter verbose: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.672-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: aborting pipeline replication: peer="{Voter server-b75cd5ae-1669-5523-4c28-e4d23dd033ae b75cd5ae-1669-5523-4c28-e4d23dd033ae}" - testing.go:149: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.672-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: aborting pipeline replication: peer="{Voter server-b75cd5ae-1669-5523-4c28-e4d23dd033ae b75cd5ae-1669-5523-4c28-e4d23dd033ae}" -testLoggerAdapter verbose: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.672-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: aborting pipeline replication: peer="{Voter server-a6511aea-8f12-9faa-9390-4f7491bb82f0 a6511aea-8f12-9faa-9390-4f7491bb82f0}" - testing.go:149: server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: 2024-06-11T02:29:28.672-1200 [INFO] server-a5b3d033-4dcb-08fe-0c64-70dd120e59fd: aborting pipeline replication: peer="{Voter server-a6511aea-8f12-9faa-9390-4f7491bb82f0 a6511aea-8f12-9faa-9390-4f7491bb82f0}" ---- PASS: TestRaft_LeadershipTransferWithSevenNodes (0.51s) -=== RUN TestRaft_LeadershipTransferToInvalidID -testLoggerAdapter verbose: server-75d6055d-6de8-28b2-a672-d76bbc3f8032: 2024-06-11T02:29:28.690-1200 [INFO] server-75d6055d-6de8-28b2-a672-d76bbc3f8032: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-75d6055d-6de8-28b2-a672-d76bbc3f8032 Address:75d6055d-6de8-28b2-a672-d76bbc3f8032} {Suffrage:Voter ID:server-e00db17a-60e5-2be5-5d7e-6834ba75f43a Address:e00db17a-60e5-2be5-5d7e-6834ba75f43a} {Suffrage:Voter ID:server-b0f3585d-24ae-2a03-9a46-b69003ebdda7 Address:b0f3585d-24ae-2a03-9a46-b69003ebdda7}]" - testing.go:149: server-75d6055d-6de8-28b2-a672-d76bbc3f8032: 2024-06-11T02:29:28.690-1200 [INFO] server-75d6055d-6de8-28b2-a672-d76bbc3f8032: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-75d6055d-6de8-28b2-a672-d76bbc3f8032 Address:75d6055d-6de8-28b2-a672-d76bbc3f8032} {Suffrage:Voter ID:server-e00db17a-60e5-2be5-5d7e-6834ba75f43a Address:e00db17a-60e5-2be5-5d7e-6834ba75f43a} {Suffrage:Voter ID:server-b0f3585d-24ae-2a03-9a46-b69003ebdda7 Address:b0f3585d-24ae-2a03-9a46-b69003ebdda7}]" -testLoggerAdapter verbose: server-75d6055d-6de8-28b2-a672-d76bbc3f8032: 2024-06-11T02:29:28.691-1200 [INFO] server-75d6055d-6de8-28b2-a672-d76bbc3f8032: entering follower state: follower="Node at 75d6055d-6de8-28b2-a672-d76bbc3f8032 [Follower]" leader= - testing.go:149: server-75d6055d-6de8-28b2-a672-d76bbc3f8032: 2024-06-11T02:29:28.691-1200 [INFO] server-75d6055d-6de8-28b2-a672-d76bbc3f8032: entering follower state: follower="Node at 75d6055d-6de8-28b2-a672-d76bbc3f8032 [Follower]" leader= -testLoggerAdapter verbose: server-e00db17a-60e5-2be5-5d7e-6834ba75f43a: 2024-06-11T02:29:28.691-1200 [INFO] server-e00db17a-60e5-2be5-5d7e-6834ba75f43a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-75d6055d-6de8-28b2-a672-d76bbc3f8032 Address:75d6055d-6de8-28b2-a672-d76bbc3f8032} {Suffrage:Voter ID:server-e00db17a-60e5-2be5-5d7e-6834ba75f43a Address:e00db17a-60e5-2be5-5d7e-6834ba75f43a} {Suffrage:Voter ID:server-b0f3585d-24ae-2a03-9a46-b69003ebdda7 Address:b0f3585d-24ae-2a03-9a46-b69003ebdda7}]" - testing.go:149: server-e00db17a-60e5-2be5-5d7e-6834ba75f43a: 2024-06-11T02:29:28.691-1200 [INFO] server-e00db17a-60e5-2be5-5d7e-6834ba75f43a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-75d6055d-6de8-28b2-a672-d76bbc3f8032 Address:75d6055d-6de8-28b2-a672-d76bbc3f8032} {Suffrage:Voter ID:server-e00db17a-60e5-2be5-5d7e-6834ba75f43a Address:e00db17a-60e5-2be5-5d7e-6834ba75f43a} {Suffrage:Voter ID:server-b0f3585d-24ae-2a03-9a46-b69003ebdda7 Address:b0f3585d-24ae-2a03-9a46-b69003ebdda7}]" -testLoggerAdapter verbose: server-e00db17a-60e5-2be5-5d7e-6834ba75f43a: 2024-06-11T02:29:28.691-1200 [INFO] server-e00db17a-60e5-2be5-5d7e-6834ba75f43a: entering follower state: follower="Node at e00db17a-60e5-2be5-5d7e-6834ba75f43a [Follower]" leader= - testing.go:149: server-e00db17a-60e5-2be5-5d7e-6834ba75f43a: 2024-06-11T02:29:28.691-1200 [INFO] server-e00db17a-60e5-2be5-5d7e-6834ba75f43a: entering follower state: follower="Node at e00db17a-60e5-2be5-5d7e-6834ba75f43a [Follower]" leader= -testLoggerAdapter verbose: server-b0f3585d-24ae-2a03-9a46-b69003ebdda7: 2024-06-11T02:29:28.692-1200 [INFO] server-b0f3585d-24ae-2a03-9a46-b69003ebdda7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-75d6055d-6de8-28b2-a672-d76bbc3f8032 Address:75d6055d-6de8-28b2-a672-d76bbc3f8032} {Suffrage:Voter ID:server-e00db17a-60e5-2be5-5d7e-6834ba75f43a Address:e00db17a-60e5-2be5-5d7e-6834ba75f43a} {Suffrage:Voter ID:server-b0f3585d-24ae-2a03-9a46-b69003ebdda7 Address:b0f3585d-24ae-2a03-9a46-b69003ebdda7}]" - testing.go:149: server-b0f3585d-24ae-2a03-9a46-b69003ebdda7: 2024-06-11T02:29:28.692-1200 [INFO] server-b0f3585d-24ae-2a03-9a46-b69003ebdda7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-75d6055d-6de8-28b2-a672-d76bbc3f8032 Address:75d6055d-6de8-28b2-a672-d76bbc3f8032} {Suffrage:Voter ID:server-e00db17a-60e5-2be5-5d7e-6834ba75f43a Address:e00db17a-60e5-2be5-5d7e-6834ba75f43a} {Suffrage:Voter ID:server-b0f3585d-24ae-2a03-9a46-b69003ebdda7 Address:b0f3585d-24ae-2a03-9a46-b69003ebdda7}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:28.692-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:28.692-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-b0f3585d-24ae-2a03-9a46-b69003ebdda7: 2024-06-11T02:29:28.692-1200 [INFO] server-b0f3585d-24ae-2a03-9a46-b69003ebdda7: entering follower state: follower="Node at b0f3585d-24ae-2a03-9a46-b69003ebdda7 [Follower]" leader= - testing.go:149: server-b0f3585d-24ae-2a03-9a46-b69003ebdda7: 2024-06-11T02:29:28.692-1200 [INFO] server-b0f3585d-24ae-2a03-9a46-b69003ebdda7: entering follower state: follower="Node at b0f3585d-24ae-2a03-9a46-b69003ebdda7 [Follower]" leader= -testLoggerAdapter verbose: server-75d6055d-6de8-28b2-a672-d76bbc3f8032: 2024-06-11T02:29:28.754-1200 [WARN] server-75d6055d-6de8-28b2-a672-d76bbc3f8032: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-75d6055d-6de8-28b2-a672-d76bbc3f8032: 2024-06-11T02:29:28.754-1200 [WARN] server-75d6055d-6de8-28b2-a672-d76bbc3f8032: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-75d6055d-6de8-28b2-a672-d76bbc3f8032: 2024-06-11T02:29:28.754-1200 [INFO] server-75d6055d-6de8-28b2-a672-d76bbc3f8032: entering candidate state: node="Node at 75d6055d-6de8-28b2-a672-d76bbc3f8032 [Candidate]" term=2 - testing.go:149: server-75d6055d-6de8-28b2-a672-d76bbc3f8032: 2024-06-11T02:29:28.754-1200 [INFO] server-75d6055d-6de8-28b2-a672-d76bbc3f8032: entering candidate state: node="Node at 75d6055d-6de8-28b2-a672-d76bbc3f8032 [Candidate]" term=2 -testLoggerAdapter verbose: server-75d6055d-6de8-28b2-a672-d76bbc3f8032: 2024-06-11T02:29:28.754-1200 [INFO] server-75d6055d-6de8-28b2-a672-d76bbc3f8032: election won: tally=2 - testing.go:149: server-75d6055d-6de8-28b2-a672-d76bbc3f8032: 2024-06-11T02:29:28.754-1200 [INFO] server-75d6055d-6de8-28b2-a672-d76bbc3f8032: election won: tally=2 -testLoggerAdapter verbose: server-75d6055d-6de8-28b2-a672-d76bbc3f8032: 2024-06-11T02:29:28.754-1200 [INFO] server-75d6055d-6de8-28b2-a672-d76bbc3f8032: entering leader state: leader="Node at 75d6055d-6de8-28b2-a672-d76bbc3f8032 [Leader]" - testing.go:149: server-75d6055d-6de8-28b2-a672-d76bbc3f8032: 2024-06-11T02:29:28.754-1200 [INFO] server-75d6055d-6de8-28b2-a672-d76bbc3f8032: entering leader state: leader="Node at 75d6055d-6de8-28b2-a672-d76bbc3f8032 [Leader]" -testLoggerAdapter verbose: server-75d6055d-6de8-28b2-a672-d76bbc3f8032: 2024-06-11T02:29:28.755-1200 [INFO] server-75d6055d-6de8-28b2-a672-d76bbc3f8032: added peer, starting replication: peer=server-e00db17a-60e5-2be5-5d7e-6834ba75f43a - testing.go:149: server-75d6055d-6de8-28b2-a672-d76bbc3f8032: 2024-06-11T02:29:28.755-1200 [INFO] server-75d6055d-6de8-28b2-a672-d76bbc3f8032: added peer, starting replication: peer=server-e00db17a-60e5-2be5-5d7e-6834ba75f43a -testLoggerAdapter verbose: server-75d6055d-6de8-28b2-a672-d76bbc3f8032: 2024-06-11T02:29:28.755-1200 [INFO] server-75d6055d-6de8-28b2-a672-d76bbc3f8032: added peer, starting replication: peer=server-b0f3585d-24ae-2a03-9a46-b69003ebdda7 - testing.go:149: server-75d6055d-6de8-28b2-a672-d76bbc3f8032: 2024-06-11T02:29:28.755-1200 [INFO] server-75d6055d-6de8-28b2-a672-d76bbc3f8032: added peer, starting replication: peer=server-b0f3585d-24ae-2a03-9a46-b69003ebdda7 -testLoggerAdapter verbose: server-75d6055d-6de8-28b2-a672-d76bbc3f8032: 2024-06-11T02:29:28.755-1200 [INFO] server-75d6055d-6de8-28b2-a672-d76bbc3f8032: pipelining replication: peer="{Voter server-e00db17a-60e5-2be5-5d7e-6834ba75f43a e00db17a-60e5-2be5-5d7e-6834ba75f43a}" - testing.go:149: server-75d6055d-6de8-28b2-a672-d76bbc3f8032: 2024-06-11T02:29:28.755-1200 [INFO] server-75d6055d-6de8-28b2-a672-d76bbc3f8032: pipelining replication: peer="{Voter server-e00db17a-60e5-2be5-5d7e-6834ba75f43a e00db17a-60e5-2be5-5d7e-6834ba75f43a}" -testLoggerAdapter verbose: server-75d6055d-6de8-28b2-a672-d76bbc3f8032: 2024-06-11T02:29:28.755-1200 [INFO] server-75d6055d-6de8-28b2-a672-d76bbc3f8032: pipelining replication: peer="{Voter server-b0f3585d-24ae-2a03-9a46-b69003ebdda7 b0f3585d-24ae-2a03-9a46-b69003ebdda7}" - testing.go:149: server-75d6055d-6de8-28b2-a672-d76bbc3f8032: 2024-06-11T02:29:28.755-1200 [INFO] server-75d6055d-6de8-28b2-a672-d76bbc3f8032: pipelining replication: peer="{Voter server-b0f3585d-24ae-2a03-9a46-b69003ebdda7 b0f3585d-24ae-2a03-9a46-b69003ebdda7}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:28.860-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:28.755082294 -1200 -12 m=+19.750454835 (1 nodes), 62.272578ms from start of poll, 64.780864ms from cluster start. Timeout at 2024-06-11 02:29:28.860896095 -1200 -12 m=+19.856268637, 105.813802ms after stability - testing.go:149: cluster: 2024-06-11T02:29:28.860-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:28.755082294 -1200 -12 m=+19.750454835 (1 nodes), 62.272578ms from start of poll, 64.780864ms from cluster start. Timeout at 2024-06-11 02:29:28.860896095 -1200 -12 m=+19.856268637, 105.813802ms after stability -testLoggerAdapter verbose: server-75d6055d-6de8-28b2-a672-d76bbc3f8032: 2024-06-11T02:29:28.861-1200 [INFO] server-75d6055d-6de8-28b2-a672-d76bbc3f8032: aborting pipeline replication: peer="{Voter server-b0f3585d-24ae-2a03-9a46-b69003ebdda7 b0f3585d-24ae-2a03-9a46-b69003ebdda7}" - testing.go:149: server-75d6055d-6de8-28b2-a672-d76bbc3f8032: 2024-06-11T02:29:28.861-1200 [INFO] server-75d6055d-6de8-28b2-a672-d76bbc3f8032: aborting pipeline replication: peer="{Voter server-b0f3585d-24ae-2a03-9a46-b69003ebdda7 b0f3585d-24ae-2a03-9a46-b69003ebdda7}" -testLoggerAdapter verbose: server-75d6055d-6de8-28b2-a672-d76bbc3f8032: 2024-06-11T02:29:28.861-1200 [INFO] server-75d6055d-6de8-28b2-a672-d76bbc3f8032: aborting pipeline replication: peer="{Voter server-e00db17a-60e5-2be5-5d7e-6834ba75f43a e00db17a-60e5-2be5-5d7e-6834ba75f43a}" - testing.go:149: server-75d6055d-6de8-28b2-a672-d76bbc3f8032: 2024-06-11T02:29:28.861-1200 [INFO] server-75d6055d-6de8-28b2-a672-d76bbc3f8032: aborting pipeline replication: peer="{Voter server-e00db17a-60e5-2be5-5d7e-6834ba75f43a e00db17a-60e5-2be5-5d7e-6834ba75f43a}" ---- PASS: TestRaft_LeadershipTransferToInvalidID (0.18s) -=== RUN TestRaft_LeadershipTransferToInvalidAddress -testLoggerAdapter verbose: server-4d2c6787-6e05-7f87-39b4-4e1d9320f812: 2024-06-11T02:29:28.865-1200 [INFO] server-4d2c6787-6e05-7f87-39b4-4e1d9320f812: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4d2c6787-6e05-7f87-39b4-4e1d9320f812 Address:4d2c6787-6e05-7f87-39b4-4e1d9320f812} {Suffrage:Voter ID:server-128047cc-ffe3-1291-0720-8a92589bf744 Address:128047cc-ffe3-1291-0720-8a92589bf744} {Suffrage:Voter ID:server-cbbc8cd2-edf7-750d-9d11-9f82f52c8715 Address:cbbc8cd2-edf7-750d-9d11-9f82f52c8715}]" - testing.go:149: server-4d2c6787-6e05-7f87-39b4-4e1d9320f812: 2024-06-11T02:29:28.865-1200 [INFO] server-4d2c6787-6e05-7f87-39b4-4e1d9320f812: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4d2c6787-6e05-7f87-39b4-4e1d9320f812 Address:4d2c6787-6e05-7f87-39b4-4e1d9320f812} {Suffrage:Voter ID:server-128047cc-ffe3-1291-0720-8a92589bf744 Address:128047cc-ffe3-1291-0720-8a92589bf744} {Suffrage:Voter ID:server-cbbc8cd2-edf7-750d-9d11-9f82f52c8715 Address:cbbc8cd2-edf7-750d-9d11-9f82f52c8715}]" -testLoggerAdapter verbose: server-128047cc-ffe3-1291-0720-8a92589bf744: 2024-06-11T02:29:28.865-1200 [INFO] server-128047cc-ffe3-1291-0720-8a92589bf744: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4d2c6787-6e05-7f87-39b4-4e1d9320f812 Address:4d2c6787-6e05-7f87-39b4-4e1d9320f812} {Suffrage:Voter ID:server-128047cc-ffe3-1291-0720-8a92589bf744 Address:128047cc-ffe3-1291-0720-8a92589bf744} {Suffrage:Voter ID:server-cbbc8cd2-edf7-750d-9d11-9f82f52c8715 Address:cbbc8cd2-edf7-750d-9d11-9f82f52c8715}]" - testing.go:149: server-128047cc-ffe3-1291-0720-8a92589bf744: 2024-06-11T02:29:28.865-1200 [INFO] server-128047cc-ffe3-1291-0720-8a92589bf744: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4d2c6787-6e05-7f87-39b4-4e1d9320f812 Address:4d2c6787-6e05-7f87-39b4-4e1d9320f812} {Suffrage:Voter ID:server-128047cc-ffe3-1291-0720-8a92589bf744 Address:128047cc-ffe3-1291-0720-8a92589bf744} {Suffrage:Voter ID:server-cbbc8cd2-edf7-750d-9d11-9f82f52c8715 Address:cbbc8cd2-edf7-750d-9d11-9f82f52c8715}]" -testLoggerAdapter verbose: server-cbbc8cd2-edf7-750d-9d11-9f82f52c8715: 2024-06-11T02:29:28.866-1200 [INFO] server-cbbc8cd2-edf7-750d-9d11-9f82f52c8715: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4d2c6787-6e05-7f87-39b4-4e1d9320f812 Address:4d2c6787-6e05-7f87-39b4-4e1d9320f812} {Suffrage:Voter ID:server-128047cc-ffe3-1291-0720-8a92589bf744 Address:128047cc-ffe3-1291-0720-8a92589bf744} {Suffrage:Voter ID:server-cbbc8cd2-edf7-750d-9d11-9f82f52c8715 Address:cbbc8cd2-edf7-750d-9d11-9f82f52c8715}]" - testing.go:149: server-cbbc8cd2-edf7-750d-9d11-9f82f52c8715: 2024-06-11T02:29:28.866-1200 [INFO] server-cbbc8cd2-edf7-750d-9d11-9f82f52c8715: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4d2c6787-6e05-7f87-39b4-4e1d9320f812 Address:4d2c6787-6e05-7f87-39b4-4e1d9320f812} {Suffrage:Voter ID:server-128047cc-ffe3-1291-0720-8a92589bf744 Address:128047cc-ffe3-1291-0720-8a92589bf744} {Suffrage:Voter ID:server-cbbc8cd2-edf7-750d-9d11-9f82f52c8715 Address:cbbc8cd2-edf7-750d-9d11-9f82f52c8715}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:28.866-1200 [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2024-06-11T02:29:28.866-1200 [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: server-4d2c6787-6e05-7f87-39b4-4e1d9320f812: 2024-06-11T02:29:28.866-1200 [INFO] server-4d2c6787-6e05-7f87-39b4-4e1d9320f812: entering follower state: follower="Node at 4d2c6787-6e05-7f87-39b4-4e1d9320f812 [Follower]" leader= - testing.go:149: server-4d2c6787-6e05-7f87-39b4-4e1d9320f812: 2024-06-11T02:29:28.866-1200 [INFO] server-4d2c6787-6e05-7f87-39b4-4e1d9320f812: entering follower state: follower="Node at 4d2c6787-6e05-7f87-39b4-4e1d9320f812 [Follower]" leader= -testLoggerAdapter verbose: server-128047cc-ffe3-1291-0720-8a92589bf744: 2024-06-11T02:29:28.866-1200 [INFO] server-128047cc-ffe3-1291-0720-8a92589bf744: entering follower state: follower="Node at 128047cc-ffe3-1291-0720-8a92589bf744 [Follower]" leader= - testing.go:149: server-128047cc-ffe3-1291-0720-8a92589bf744: 2024-06-11T02:29:28.866-1200 [INFO] server-128047cc-ffe3-1291-0720-8a92589bf744: entering follower state: follower="Node at 128047cc-ffe3-1291-0720-8a92589bf744 [Follower]" leader= -testLoggerAdapter verbose: server-cbbc8cd2-edf7-750d-9d11-9f82f52c8715: 2024-06-11T02:29:28.866-1200 [INFO] server-cbbc8cd2-edf7-750d-9d11-9f82f52c8715: entering follower state: follower="Node at cbbc8cd2-edf7-750d-9d11-9f82f52c8715 [Follower]" leader= - testing.go:149: server-cbbc8cd2-edf7-750d-9d11-9f82f52c8715: 2024-06-11T02:29:28.866-1200 [INFO] server-cbbc8cd2-edf7-750d-9d11-9f82f52c8715: entering follower state: follower="Node at cbbc8cd2-edf7-750d-9d11-9f82f52c8715 [Follower]" leader= -testLoggerAdapter verbose: server-128047cc-ffe3-1291-0720-8a92589bf744: 2024-06-11T02:29:28.931-1200 [WARN] server-128047cc-ffe3-1291-0720-8a92589bf744: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-128047cc-ffe3-1291-0720-8a92589bf744: 2024-06-11T02:29:28.931-1200 [WARN] server-128047cc-ffe3-1291-0720-8a92589bf744: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-128047cc-ffe3-1291-0720-8a92589bf744: 2024-06-11T02:29:28.931-1200 [INFO] server-128047cc-ffe3-1291-0720-8a92589bf744: entering candidate state: node="Node at 128047cc-ffe3-1291-0720-8a92589bf744 [Candidate]" term=2 - testing.go:149: server-128047cc-ffe3-1291-0720-8a92589bf744: 2024-06-11T02:29:28.931-1200 [INFO] server-128047cc-ffe3-1291-0720-8a92589bf744: entering candidate state: node="Node at 128047cc-ffe3-1291-0720-8a92589bf744 [Candidate]" term=2 -testLoggerAdapter verbose: server-128047cc-ffe3-1291-0720-8a92589bf744: 2024-06-11T02:29:28.931-1200 [INFO] server-128047cc-ffe3-1291-0720-8a92589bf744: election won: tally=2 - testing.go:149: server-128047cc-ffe3-1291-0720-8a92589bf744: 2024-06-11T02:29:28.931-1200 [INFO] server-128047cc-ffe3-1291-0720-8a92589bf744: election won: tally=2 -testLoggerAdapter verbose: server-128047cc-ffe3-1291-0720-8a92589bf744: 2024-06-11T02:29:28.931-1200 [INFO] server-128047cc-ffe3-1291-0720-8a92589bf744: entering leader state: leader="Node at 128047cc-ffe3-1291-0720-8a92589bf744 [Leader]" - testing.go:149: server-128047cc-ffe3-1291-0720-8a92589bf744: 2024-06-11T02:29:28.931-1200 [INFO] server-128047cc-ffe3-1291-0720-8a92589bf744: entering leader state: leader="Node at 128047cc-ffe3-1291-0720-8a92589bf744 [Leader]" -testLoggerAdapter verbose: server-128047cc-ffe3-1291-0720-8a92589bf744: 2024-06-11T02:29:28.932-1200 [INFO] server-128047cc-ffe3-1291-0720-8a92589bf744: added peer, starting replication: peer=server-4d2c6787-6e05-7f87-39b4-4e1d9320f812 - testing.go:149: server-128047cc-ffe3-1291-0720-8a92589bf744: 2024-06-11T02:29:28.932-1200 [INFO] server-128047cc-ffe3-1291-0720-8a92589bf744: added peer, starting replication: peer=server-4d2c6787-6e05-7f87-39b4-4e1d9320f812 -testLoggerAdapter verbose: server-128047cc-ffe3-1291-0720-8a92589bf744: 2024-06-11T02:29:28.932-1200 [INFO] server-128047cc-ffe3-1291-0720-8a92589bf744: added peer, starting replication: peer=server-cbbc8cd2-edf7-750d-9d11-9f82f52c8715 - testing.go:149: server-128047cc-ffe3-1291-0720-8a92589bf744: 2024-06-11T02:29:28.932-1200 [INFO] server-128047cc-ffe3-1291-0720-8a92589bf744: added peer, starting replication: peer=server-cbbc8cd2-edf7-750d-9d11-9f82f52c8715 -testLoggerAdapter verbose: server-128047cc-ffe3-1291-0720-8a92589bf744: 2024-06-11T02:29:28.932-1200 [INFO] server-128047cc-ffe3-1291-0720-8a92589bf744: pipelining replication: peer="{Voter server-cbbc8cd2-edf7-750d-9d11-9f82f52c8715 cbbc8cd2-edf7-750d-9d11-9f82f52c8715}" - testing.go:149: server-128047cc-ffe3-1291-0720-8a92589bf744: 2024-06-11T02:29:28.932-1200 [INFO] server-128047cc-ffe3-1291-0720-8a92589bf744: pipelining replication: peer="{Voter server-cbbc8cd2-edf7-750d-9d11-9f82f52c8715 cbbc8cd2-edf7-750d-9d11-9f82f52c8715}" -testLoggerAdapter verbose: server-128047cc-ffe3-1291-0720-8a92589bf744: 2024-06-11T02:29:28.932-1200 [INFO] server-128047cc-ffe3-1291-0720-8a92589bf744: pipelining replication: peer="{Voter server-4d2c6787-6e05-7f87-39b4-4e1d9320f812 4d2c6787-6e05-7f87-39b4-4e1d9320f812}" - testing.go:149: server-128047cc-ffe3-1291-0720-8a92589bf744: 2024-06-11T02:29:28.932-1200 [INFO] server-128047cc-ffe3-1291-0720-8a92589bf744: pipelining replication: peer="{Voter server-4d2c6787-6e05-7f87-39b4-4e1d9320f812 4d2c6787-6e05-7f87-39b4-4e1d9320f812}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:29.039-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:28.933210568 -1200 -12 m=+19.928583401 (2 nodes), 66.80966ms from start of poll, 68.044261ms from cluster start. Timeout at 2024-06-11 02:29:29.039256823 -1200 -12 m=+20.034629657, 106.046256ms after stability - testing.go:149: cluster: 2024-06-11T02:29:29.039-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:28.933210568 -1200 -12 m=+19.928583401 (2 nodes), 66.80966ms from start of poll, 68.044261ms from cluster start. Timeout at 2024-06-11 02:29:29.039256823 -1200 -12 m=+20.034629657, 106.046256ms after stability -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:29.039-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:29.039-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:29.145-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:29.039511735 -1200 -12 m=+20.034884277 (1 nodes), 2.917µs from start of poll, 174.345137ms from cluster start. Timeout at 2024-06-11 02:29:29.144901169 -1200 -12 m=+20.140274003, 105.389726ms after stability - testing.go:149: cluster: 2024-06-11T02:29:29.145-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:29.039511735 -1200 -12 m=+20.034884277 (1 nodes), 2.917µs from start of poll, 174.345137ms from cluster start. Timeout at 2024-06-11 02:29:29.144901169 -1200 -12 m=+20.140274003, 105.389726ms after stability -testLoggerAdapter verbose: server-128047cc-ffe3-1291-0720-8a92589bf744: 2024-06-11T02:29:29.145-1200 [INFO] server-128047cc-ffe3-1291-0720-8a92589bf744: aborting pipeline replication: peer="{Voter server-cbbc8cd2-edf7-750d-9d11-9f82f52c8715 cbbc8cd2-edf7-750d-9d11-9f82f52c8715}" - testing.go:149: server-128047cc-ffe3-1291-0720-8a92589bf744: 2024-06-11T02:29:29.145-1200 [INFO] server-128047cc-ffe3-1291-0720-8a92589bf744: aborting pipeline replication: peer="{Voter server-cbbc8cd2-edf7-750d-9d11-9f82f52c8715 cbbc8cd2-edf7-750d-9d11-9f82f52c8715}" -testLoggerAdapter verbose: server-128047cc-ffe3-1291-0720-8a92589bf744: 2024-06-11T02:29:29.145-1200 [INFO] server-128047cc-ffe3-1291-0720-8a92589bf744: aborting pipeline replication: peer="{Voter server-4d2c6787-6e05-7f87-39b4-4e1d9320f812 4d2c6787-6e05-7f87-39b4-4e1d9320f812}" - testing.go:149: server-128047cc-ffe3-1291-0720-8a92589bf744: 2024-06-11T02:29:29.145-1200 [INFO] server-128047cc-ffe3-1291-0720-8a92589bf744: aborting pipeline replication: peer="{Voter server-4d2c6787-6e05-7f87-39b4-4e1d9320f812 4d2c6787-6e05-7f87-39b4-4e1d9320f812}" ---- PASS: TestRaft_LeadershipTransferToInvalidAddress (0.29s) -=== RUN TestRaft_LeadershipTransferToBehindServer -testLoggerAdapter verbose: server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: 2024-06-11T02:29:29.151-1200 [INFO] server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-737bf72e-694e-ba03-2e4c-cb1c82c83c13 Address:737bf72e-694e-ba03-2e4c-cb1c82c83c13} {Suffrage:Voter ID:server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02 Address:1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02} {Suffrage:Voter ID:server-b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc Address:b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc}]" - testing.go:149: server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: 2024-06-11T02:29:29.151-1200 [INFO] server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-737bf72e-694e-ba03-2e4c-cb1c82c83c13 Address:737bf72e-694e-ba03-2e4c-cb1c82c83c13} {Suffrage:Voter ID:server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02 Address:1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02} {Suffrage:Voter ID:server-b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc Address:b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc}]" -testLoggerAdapter verbose: server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: 2024-06-11T02:29:29.152-1200 [INFO] server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-737bf72e-694e-ba03-2e4c-cb1c82c83c13 Address:737bf72e-694e-ba03-2e4c-cb1c82c83c13} {Suffrage:Voter ID:server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02 Address:1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02} {Suffrage:Voter ID:server-b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc Address:b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc}]" - testing.go:149: server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: 2024-06-11T02:29:29.152-1200 [INFO] server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-737bf72e-694e-ba03-2e4c-cb1c82c83c13 Address:737bf72e-694e-ba03-2e4c-cb1c82c83c13} {Suffrage:Voter ID:server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02 Address:1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02} {Suffrage:Voter ID:server-b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc Address:b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc}]" -testLoggerAdapter verbose: server-b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc: 2024-06-11T02:29:29.152-1200 [INFO] server-b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-737bf72e-694e-ba03-2e4c-cb1c82c83c13 Address:737bf72e-694e-ba03-2e4c-cb1c82c83c13} {Suffrage:Voter ID:server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02 Address:1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02} {Suffrage:Voter ID:server-b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc Address:b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc}]" - testing.go:149: server-b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc: 2024-06-11T02:29:29.152-1200 [INFO] server-b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-737bf72e-694e-ba03-2e4c-cb1c82c83c13 Address:737bf72e-694e-ba03-2e4c-cb1c82c83c13} {Suffrage:Voter ID:server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02 Address:1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02} {Suffrage:Voter ID:server-b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc Address:b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:29.152-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:29.152-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: 2024-06-11T02:29:29.152-1200 [INFO] server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: entering follower state: follower="Node at 737bf72e-694e-ba03-2e4c-cb1c82c83c13 [Follower]" leader= - testing.go:149: server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: 2024-06-11T02:29:29.152-1200 [INFO] server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: entering follower state: follower="Node at 737bf72e-694e-ba03-2e4c-cb1c82c83c13 [Follower]" leader= -testLoggerAdapter verbose: server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: 2024-06-11T02:29:29.153-1200 [INFO] server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: entering follower state: follower="Node at 1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02 [Follower]" leader= - testing.go:149: server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: 2024-06-11T02:29:29.153-1200 [INFO] server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: entering follower state: follower="Node at 1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02 [Follower]" leader= -testLoggerAdapter verbose: server-b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc: 2024-06-11T02:29:29.153-1200 [INFO] server-b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc: entering follower state: follower="Node at b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc [Follower]" leader= - testing.go:149: server-b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc: 2024-06-11T02:29:29.153-1200 [INFO] server-b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc: entering follower state: follower="Node at b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc [Follower]" leader= -testLoggerAdapter verbose: server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: 2024-06-11T02:29:29.205-1200 [WARN] server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: 2024-06-11T02:29:29.205-1200 [WARN] server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: 2024-06-11T02:29:29.206-1200 [INFO] server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: entering candidate state: node="Node at 737bf72e-694e-ba03-2e4c-cb1c82c83c13 [Candidate]" term=2 - testing.go:149: server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: 2024-06-11T02:29:29.206-1200 [INFO] server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: entering candidate state: node="Node at 737bf72e-694e-ba03-2e4c-cb1c82c83c13 [Candidate]" term=2 -testLoggerAdapter verbose: server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: 2024-06-11T02:29:29.206-1200 [INFO] server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: election won: tally=2 - testing.go:149: server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: 2024-06-11T02:29:29.206-1200 [INFO] server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: election won: tally=2 -testLoggerAdapter verbose: server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: 2024-06-11T02:29:29.206-1200 [INFO] server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: entering leader state: leader="Node at 737bf72e-694e-ba03-2e4c-cb1c82c83c13 [Leader]" - testing.go:149: server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: 2024-06-11T02:29:29.206-1200 [INFO] server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: entering leader state: leader="Node at 737bf72e-694e-ba03-2e4c-cb1c82c83c13 [Leader]" -testLoggerAdapter verbose: server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: 2024-06-11T02:29:29.206-1200 [INFO] server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: added peer, starting replication: peer=server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02 - testing.go:149: server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: 2024-06-11T02:29:29.206-1200 [INFO] server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: added peer, starting replication: peer=server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02 -testLoggerAdapter verbose: server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: 2024-06-11T02:29:29.206-1200 [INFO] server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: added peer, starting replication: peer=server-b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc - testing.go:149: server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: 2024-06-11T02:29:29.206-1200 [INFO] server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: added peer, starting replication: peer=server-b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc -testLoggerAdapter verbose: server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: 2024-06-11T02:29:29.206-1200 [INFO] server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: pipelining replication: peer="{Voter server-b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc}" - testing.go:149: server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: 2024-06-11T02:29:29.206-1200 [INFO] server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: pipelining replication: peer="{Voter server-b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc}" -testLoggerAdapter verbose: server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: 2024-06-11T02:29:29.206-1200 [INFO] server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: pipelining replication: peer="{Voter server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02 1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02}" - testing.go:149: server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: 2024-06-11T02:29:29.206-1200 [INFO] server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: pipelining replication: peer="{Voter server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02 1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:29.312-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:29.207078958 -1200 -12 m=+20.202451500 (1 nodes), 54.153315ms from start of poll, 56.11182ms from cluster start. Timeout at 2024-06-11 02:29:29.312373311 -1200 -12 m=+20.307746144, 105.294644ms after stability - testing.go:149: cluster: 2024-06-11T02:29:29.312-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:29.207078958 -1200 -12 m=+20.202451500 (1 nodes), 54.153315ms from start of poll, 56.11182ms from cluster start. Timeout at 2024-06-11 02:29:29.312373311 -1200 -12 m=+20.307746144, 105.294644ms after stability -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:29.312-1200 [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2024-06-11T02:29:29.312-1200 [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:29.418-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:29.312637556 -1200 -12 m=+20.308010097 (2 nodes), 3.499µs from start of poll, 161.670417ms from cluster start. Timeout at 2024-06-11 02:29:29.418625187 -1200 -12 m=+20.413997729, 105.987632ms after stability - testing.go:149: cluster: 2024-06-11T02:29:29.418-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:29.312637556 -1200 -12 m=+20.308010097 (2 nodes), 3.499µs from start of poll, 161.670417ms from cluster start. Timeout at 2024-06-11 02:29:29.418625187 -1200 -12 m=+20.413997729, 105.987632ms after stability -testLoggerAdapter verbose: server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: 2024-06-11T02:29:29.468-1200 [INFO] server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: entering candidate state: node="Node at 1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02 [Candidate]" term=3 -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:29.468-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: 2024-06-11T02:29:29.468-1200 [INFO] server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: entering candidate state: node="Node at 1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02 [Candidate]" term=3 - testing.go:149: cluster: 2024-06-11T02:29:29.468-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: 2024-06-11T02:29:29.468-1200 [INFO] server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: election won: tally=2 - testing.go:149: server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: 2024-06-11T02:29:29.468-1200 [INFO] server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: election won: tally=2 -testLoggerAdapter verbose: server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: 2024-06-11T02:29:29.469-1200 [INFO] server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: entering leader state: leader="Node at 1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02 [Leader]" -testLoggerAdapter verbose: server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: 2024-06-11T02:29:29.469-1200 [INFO] server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: entering follower state: follower="Node at 737bf72e-694e-ba03-2e4c-cb1c82c83c13 [Follower]" leader= - testing.go:149: server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: 2024-06-11T02:29:29.469-1200 [INFO] server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: entering leader state: leader="Node at 1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02 [Leader]" -testLoggerAdapter verbose: server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: 2024-06-11T02:29:29.469-1200 [INFO] server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: added peer, starting replication: peer=server-737bf72e-694e-ba03-2e4c-cb1c82c83c13 - testing.go:149: server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: 2024-06-11T02:29:29.469-1200 [INFO] server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: added peer, starting replication: peer=server-737bf72e-694e-ba03-2e4c-cb1c82c83c13 -testLoggerAdapter verbose: server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: 2024-06-11T02:29:29.469-1200 [INFO] server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: added peer, starting replication: peer=server-b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc - testing.go:149: server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: 2024-06-11T02:29:29.469-1200 [INFO] server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: entering follower state: follower="Node at 737bf72e-694e-ba03-2e4c-cb1c82c83c13 [Follower]" leader= - testing.go:149: server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: 2024-06-11T02:29:29.469-1200 [INFO] server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: added peer, starting replication: peer=server-b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc -testLoggerAdapter verbose: server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: 2024-06-11T02:29:29.469-1200 [INFO] server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: pipelining replication: peer="{Voter server-737bf72e-694e-ba03-2e4c-cb1c82c83c13 737bf72e-694e-ba03-2e4c-cb1c82c83c13}" - testing.go:149: server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: 2024-06-11T02:29:29.469-1200 [INFO] server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: pipelining replication: peer="{Voter server-737bf72e-694e-ba03-2e4c-cb1c82c83c13 737bf72e-694e-ba03-2e4c-cb1c82c83c13}" -testLoggerAdapter verbose: server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: 2024-06-11T02:29:29.469-1200 [INFO] server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: aborting pipeline replication: peer="{Voter server-b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc}" -testLoggerAdapter verbose: server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: 2024-06-11T02:29:29.469-1200 [INFO] server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: pipelining replication: peer="{Voter server-b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc}" - testing.go:149: server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: 2024-06-11T02:29:29.469-1200 [INFO] server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: aborting pipeline replication: peer="{Voter server-b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc}" - testing.go:149: server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: 2024-06-11T02:29:29.469-1200 [INFO] server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: pipelining replication: peer="{Voter server-b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc}" -testLoggerAdapter verbose: server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: 2024-06-11T02:29:29.469-1200 [INFO] server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: aborting pipeline replication: peer="{Voter server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02 1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02}" - testing.go:149: server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: 2024-06-11T02:29:29.469-1200 [INFO] server-737bf72e-694e-ba03-2e4c-cb1c82c83c13: aborting pipeline replication: peer="{Voter server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02 1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:29.574-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:29.469149362 -1200 -12 m=+20.464521903 (1 nodes), 195.121µs from start of poll, 318.182223ms from cluster start. Timeout at 2024-06-11 02:29:29.574658669 -1200 -12 m=+20.570031210, 105.509307ms after stability - testing.go:149: cluster: 2024-06-11T02:29:29.574-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:29.469149362 -1200 -12 m=+20.464521903 (1 nodes), 195.121µs from start of poll, 318.182223ms from cluster start. Timeout at 2024-06-11 02:29:29.574658669 -1200 -12 m=+20.570031210, 105.509307ms after stability -testLoggerAdapter verbose: server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: 2024-06-11T02:29:29.574-1200 [INFO] server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: aborting pipeline replication: peer="{Voter server-b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc}" - testing.go:149: server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: 2024-06-11T02:29:29.574-1200 [INFO] server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: aborting pipeline replication: peer="{Voter server-b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc b4a2a1f0-c0f2-60af-d7b9-55df08d7ecbc}" -testLoggerAdapter verbose: server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: 2024-06-11T02:29:29.575-1200 [INFO] server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: aborting pipeline replication: peer="{Voter server-737bf72e-694e-ba03-2e4c-cb1c82c83c13 737bf72e-694e-ba03-2e4c-cb1c82c83c13}" - testing.go:149: server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: 2024-06-11T02:29:29.575-1200 [INFO] server-1d3c7eea-08fc-eb94-1a3b-bc7dd401cc02: aborting pipeline replication: peer="{Voter server-737bf72e-694e-ba03-2e4c-cb1c82c83c13 737bf72e-694e-ba03-2e4c-cb1c82c83c13}" ---- PASS: TestRaft_LeadershipTransferToBehindServer (0.43s) -=== RUN TestRaft_LeadershipTransferToItself -testLoggerAdapter verbose: server-03b12ecf-f6a9-5c4c-3a0c-26940ff2b784: 2024-06-11T02:29:29.578-1200 [INFO] server-03b12ecf-f6a9-5c4c-3a0c-26940ff2b784: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-03b12ecf-f6a9-5c4c-3a0c-26940ff2b784 Address:03b12ecf-f6a9-5c4c-3a0c-26940ff2b784} {Suffrage:Voter ID:server-f26890e4-37e3-82ee-417b-f235502cbb35 Address:f26890e4-37e3-82ee-417b-f235502cbb35} {Suffrage:Voter ID:server-83b17fe0-a363-efac-0a67-8cd999391aef Address:83b17fe0-a363-efac-0a67-8cd999391aef}]" - testing.go:149: server-03b12ecf-f6a9-5c4c-3a0c-26940ff2b784: 2024-06-11T02:29:29.578-1200 [INFO] server-03b12ecf-f6a9-5c4c-3a0c-26940ff2b784: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-03b12ecf-f6a9-5c4c-3a0c-26940ff2b784 Address:03b12ecf-f6a9-5c4c-3a0c-26940ff2b784} {Suffrage:Voter ID:server-f26890e4-37e3-82ee-417b-f235502cbb35 Address:f26890e4-37e3-82ee-417b-f235502cbb35} {Suffrage:Voter ID:server-83b17fe0-a363-efac-0a67-8cd999391aef Address:83b17fe0-a363-efac-0a67-8cd999391aef}]" -testLoggerAdapter verbose: server-03b12ecf-f6a9-5c4c-3a0c-26940ff2b784: 2024-06-11T02:29:29.579-1200 [INFO] server-03b12ecf-f6a9-5c4c-3a0c-26940ff2b784: entering follower state: follower="Node at 03b12ecf-f6a9-5c4c-3a0c-26940ff2b784 [Follower]" leader= - testing.go:149: server-03b12ecf-f6a9-5c4c-3a0c-26940ff2b784: 2024-06-11T02:29:29.579-1200 [INFO] server-03b12ecf-f6a9-5c4c-3a0c-26940ff2b784: entering follower state: follower="Node at 03b12ecf-f6a9-5c4c-3a0c-26940ff2b784 [Follower]" leader= -testLoggerAdapter verbose: server-f26890e4-37e3-82ee-417b-f235502cbb35: 2024-06-11T02:29:29.579-1200 [INFO] server-f26890e4-37e3-82ee-417b-f235502cbb35: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-03b12ecf-f6a9-5c4c-3a0c-26940ff2b784 Address:03b12ecf-f6a9-5c4c-3a0c-26940ff2b784} {Suffrage:Voter ID:server-f26890e4-37e3-82ee-417b-f235502cbb35 Address:f26890e4-37e3-82ee-417b-f235502cbb35} {Suffrage:Voter ID:server-83b17fe0-a363-efac-0a67-8cd999391aef Address:83b17fe0-a363-efac-0a67-8cd999391aef}]" - testing.go:149: server-f26890e4-37e3-82ee-417b-f235502cbb35: 2024-06-11T02:29:29.579-1200 [INFO] server-f26890e4-37e3-82ee-417b-f235502cbb35: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-03b12ecf-f6a9-5c4c-3a0c-26940ff2b784 Address:03b12ecf-f6a9-5c4c-3a0c-26940ff2b784} {Suffrage:Voter ID:server-f26890e4-37e3-82ee-417b-f235502cbb35 Address:f26890e4-37e3-82ee-417b-f235502cbb35} {Suffrage:Voter ID:server-83b17fe0-a363-efac-0a67-8cd999391aef Address:83b17fe0-a363-efac-0a67-8cd999391aef}]" -testLoggerAdapter verbose: server-83b17fe0-a363-efac-0a67-8cd999391aef: 2024-06-11T02:29:29.580-1200 [INFO] server-83b17fe0-a363-efac-0a67-8cd999391aef: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-03b12ecf-f6a9-5c4c-3a0c-26940ff2b784 Address:03b12ecf-f6a9-5c4c-3a0c-26940ff2b784} {Suffrage:Voter ID:server-f26890e4-37e3-82ee-417b-f235502cbb35 Address:f26890e4-37e3-82ee-417b-f235502cbb35} {Suffrage:Voter ID:server-83b17fe0-a363-efac-0a67-8cd999391aef Address:83b17fe0-a363-efac-0a67-8cd999391aef}]" - testing.go:149: server-83b17fe0-a363-efac-0a67-8cd999391aef: 2024-06-11T02:29:29.580-1200 [INFO] server-83b17fe0-a363-efac-0a67-8cd999391aef: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-03b12ecf-f6a9-5c4c-3a0c-26940ff2b784 Address:03b12ecf-f6a9-5c4c-3a0c-26940ff2b784} {Suffrage:Voter ID:server-f26890e4-37e3-82ee-417b-f235502cbb35 Address:f26890e4-37e3-82ee-417b-f235502cbb35} {Suffrage:Voter ID:server-83b17fe0-a363-efac-0a67-8cd999391aef Address:83b17fe0-a363-efac-0a67-8cd999391aef}]" -testLoggerAdapter verbose: server-f26890e4-37e3-82ee-417b-f235502cbb35: 2024-06-11T02:29:29.580-1200 [INFO] server-f26890e4-37e3-82ee-417b-f235502cbb35: entering follower state: follower="Node at f26890e4-37e3-82ee-417b-f235502cbb35 [Follower]" leader= - testing.go:149: server-f26890e4-37e3-82ee-417b-f235502cbb35: 2024-06-11T02:29:29.580-1200 [INFO] server-f26890e4-37e3-82ee-417b-f235502cbb35: entering follower state: follower="Node at f26890e4-37e3-82ee-417b-f235502cbb35 [Follower]" leader= -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:29.580-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-83b17fe0-a363-efac-0a67-8cd999391aef: 2024-06-11T02:29:29.580-1200 [INFO] server-83b17fe0-a363-efac-0a67-8cd999391aef: entering follower state: follower="Node at 83b17fe0-a363-efac-0a67-8cd999391aef [Follower]" leader= - testing.go:149: cluster: 2024-06-11T02:29:29.580-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: server-83b17fe0-a363-efac-0a67-8cd999391aef: 2024-06-11T02:29:29.580-1200 [INFO] server-83b17fe0-a363-efac-0a67-8cd999391aef: entering follower state: follower="Node at 83b17fe0-a363-efac-0a67-8cd999391aef [Follower]" leader= -testLoggerAdapter verbose: server-f26890e4-37e3-82ee-417b-f235502cbb35: 2024-06-11T02:29:29.652-1200 [WARN] server-f26890e4-37e3-82ee-417b-f235502cbb35: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-f26890e4-37e3-82ee-417b-f235502cbb35: 2024-06-11T02:29:29.652-1200 [WARN] server-f26890e4-37e3-82ee-417b-f235502cbb35: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-f26890e4-37e3-82ee-417b-f235502cbb35: 2024-06-11T02:29:29.653-1200 [INFO] server-f26890e4-37e3-82ee-417b-f235502cbb35: entering candidate state: node="Node at f26890e4-37e3-82ee-417b-f235502cbb35 [Candidate]" term=2 - testing.go:149: server-f26890e4-37e3-82ee-417b-f235502cbb35: 2024-06-11T02:29:29.653-1200 [INFO] server-f26890e4-37e3-82ee-417b-f235502cbb35: entering candidate state: node="Node at f26890e4-37e3-82ee-417b-f235502cbb35 [Candidate]" term=2 -testLoggerAdapter verbose: server-f26890e4-37e3-82ee-417b-f235502cbb35: 2024-06-11T02:29:29.653-1200 [INFO] server-f26890e4-37e3-82ee-417b-f235502cbb35: election won: tally=2 - testing.go:149: server-f26890e4-37e3-82ee-417b-f235502cbb35: 2024-06-11T02:29:29.653-1200 [INFO] server-f26890e4-37e3-82ee-417b-f235502cbb35: election won: tally=2 -testLoggerAdapter verbose: server-f26890e4-37e3-82ee-417b-f235502cbb35: 2024-06-11T02:29:29.653-1200 [INFO] server-f26890e4-37e3-82ee-417b-f235502cbb35: entering leader state: leader="Node at f26890e4-37e3-82ee-417b-f235502cbb35 [Leader]" - testing.go:149: server-f26890e4-37e3-82ee-417b-f235502cbb35: 2024-06-11T02:29:29.653-1200 [INFO] server-f26890e4-37e3-82ee-417b-f235502cbb35: entering leader state: leader="Node at f26890e4-37e3-82ee-417b-f235502cbb35 [Leader]" -testLoggerAdapter verbose: server-f26890e4-37e3-82ee-417b-f235502cbb35: 2024-06-11T02:29:29.653-1200 [INFO] server-f26890e4-37e3-82ee-417b-f235502cbb35: added peer, starting replication: peer=server-03b12ecf-f6a9-5c4c-3a0c-26940ff2b784 - testing.go:149: server-f26890e4-37e3-82ee-417b-f235502cbb35: 2024-06-11T02:29:29.653-1200 [INFO] server-f26890e4-37e3-82ee-417b-f235502cbb35: added peer, starting replication: peer=server-03b12ecf-f6a9-5c4c-3a0c-26940ff2b784 -testLoggerAdapter verbose: server-f26890e4-37e3-82ee-417b-f235502cbb35: 2024-06-11T02:29:29.653-1200 [INFO] server-f26890e4-37e3-82ee-417b-f235502cbb35: added peer, starting replication: peer=server-83b17fe0-a363-efac-0a67-8cd999391aef - testing.go:149: server-f26890e4-37e3-82ee-417b-f235502cbb35: 2024-06-11T02:29:29.653-1200 [INFO] server-f26890e4-37e3-82ee-417b-f235502cbb35: added peer, starting replication: peer=server-83b17fe0-a363-efac-0a67-8cd999391aef -testLoggerAdapter verbose: server-f26890e4-37e3-82ee-417b-f235502cbb35: 2024-06-11T02:29:29.654-1200 [INFO] server-f26890e4-37e3-82ee-417b-f235502cbb35: pipelining replication: peer="{Voter server-83b17fe0-a363-efac-0a67-8cd999391aef 83b17fe0-a363-efac-0a67-8cd999391aef}" - testing.go:149: server-f26890e4-37e3-82ee-417b-f235502cbb35: 2024-06-11T02:29:29.654-1200 [INFO] server-f26890e4-37e3-82ee-417b-f235502cbb35: pipelining replication: peer="{Voter server-83b17fe0-a363-efac-0a67-8cd999391aef 83b17fe0-a363-efac-0a67-8cd999391aef}" -testLoggerAdapter verbose: server-f26890e4-37e3-82ee-417b-f235502cbb35: 2024-06-11T02:29:29.654-1200 [INFO] server-f26890e4-37e3-82ee-417b-f235502cbb35: pipelining replication: peer="{Voter server-03b12ecf-f6a9-5c4c-3a0c-26940ff2b784 03b12ecf-f6a9-5c4c-3a0c-26940ff2b784}" - testing.go:149: server-f26890e4-37e3-82ee-417b-f235502cbb35: 2024-06-11T02:29:29.654-1200 [INFO] server-f26890e4-37e3-82ee-417b-f235502cbb35: pipelining replication: peer="{Voter server-03b12ecf-f6a9-5c4c-3a0c-26940ff2b784 03b12ecf-f6a9-5c4c-3a0c-26940ff2b784}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:29.760-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:29.65496145 -1200 -12 m=+20.650333991 (1 nodes), 74.578638ms from start of poll, 76.279606ms from cluster start. Timeout at 2024-06-11 02:29:29.760779918 -1200 -12 m=+20.756152459, 105.818468ms after stability - testing.go:149: cluster: 2024-06-11T02:29:29.760-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:29.65496145 -1200 -12 m=+20.650333991 (1 nodes), 74.578638ms from start of poll, 76.279606ms from cluster start. Timeout at 2024-06-11 02:29:29.760779918 -1200 -12 m=+20.756152459, 105.818468ms after stability -testLoggerAdapter verbose: server-f26890e4-37e3-82ee-417b-f235502cbb35: 2024-06-11T02:29:29.761-1200 [INFO] server-f26890e4-37e3-82ee-417b-f235502cbb35: cannot transfer leadership to itself - testing.go:149: server-f26890e4-37e3-82ee-417b-f235502cbb35: 2024-06-11T02:29:29.761-1200 [INFO] server-f26890e4-37e3-82ee-417b-f235502cbb35: cannot transfer leadership to itself -testLoggerAdapter verbose: server-f26890e4-37e3-82ee-417b-f235502cbb35: 2024-06-11T02:29:29.761-1200 [INFO] server-f26890e4-37e3-82ee-417b-f235502cbb35: aborting pipeline replication: peer="{Voter server-83b17fe0-a363-efac-0a67-8cd999391aef 83b17fe0-a363-efac-0a67-8cd999391aef}" - testing.go:149: server-f26890e4-37e3-82ee-417b-f235502cbb35: 2024-06-11T02:29:29.761-1200 [INFO] server-f26890e4-37e3-82ee-417b-f235502cbb35: aborting pipeline replication: peer="{Voter server-83b17fe0-a363-efac-0a67-8cd999391aef 83b17fe0-a363-efac-0a67-8cd999391aef}" -testLoggerAdapter verbose: server-f26890e4-37e3-82ee-417b-f235502cbb35: 2024-06-11T02:29:29.761-1200 [INFO] server-f26890e4-37e3-82ee-417b-f235502cbb35: aborting pipeline replication: peer="{Voter server-03b12ecf-f6a9-5c4c-3a0c-26940ff2b784 03b12ecf-f6a9-5c4c-3a0c-26940ff2b784}" - testing.go:149: server-f26890e4-37e3-82ee-417b-f235502cbb35: 2024-06-11T02:29:29.761-1200 [INFO] server-f26890e4-37e3-82ee-417b-f235502cbb35: aborting pipeline replication: peer="{Voter server-03b12ecf-f6a9-5c4c-3a0c-26940ff2b784 03b12ecf-f6a9-5c4c-3a0c-26940ff2b784}" ---- PASS: TestRaft_LeadershipTransferToItself (0.19s) -=== RUN TestRaft_LeadershipTransferLeaderRejectsClientRequests - raft_test.go:2064: Disabled unreliable test ---- SKIP: TestRaft_LeadershipTransferLeaderRejectsClientRequests (0.00s) -=== RUN TestRaft_LeadershipTransferLeaderReplicationTimeout -testLoggerAdapter verbose: server-9d08fea9-3092-8248-9797-d738e3acf414: 2024-06-11T02:29:29.768-1200 [INFO] server-9d08fea9-3092-8248-9797-d738e3acf414: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9d08fea9-3092-8248-9797-d738e3acf414 Address:9d08fea9-3092-8248-9797-d738e3acf414} {Suffrage:Voter ID:server-248701f8-b374-e0df-118b-d785a03bf8a0 Address:248701f8-b374-e0df-118b-d785a03bf8a0} {Suffrage:Voter ID:server-789d45d6-5fc1-e7e6-5537-d96fdf431b05 Address:789d45d6-5fc1-e7e6-5537-d96fdf431b05}]" - testing.go:149: server-9d08fea9-3092-8248-9797-d738e3acf414: 2024-06-11T02:29:29.768-1200 [INFO] server-9d08fea9-3092-8248-9797-d738e3acf414: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9d08fea9-3092-8248-9797-d738e3acf414 Address:9d08fea9-3092-8248-9797-d738e3acf414} {Suffrage:Voter ID:server-248701f8-b374-e0df-118b-d785a03bf8a0 Address:248701f8-b374-e0df-118b-d785a03bf8a0} {Suffrage:Voter ID:server-789d45d6-5fc1-e7e6-5537-d96fdf431b05 Address:789d45d6-5fc1-e7e6-5537-d96fdf431b05}]" -testLoggerAdapter verbose: server-248701f8-b374-e0df-118b-d785a03bf8a0: 2024-06-11T02:29:29.769-1200 [INFO] server-248701f8-b374-e0df-118b-d785a03bf8a0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9d08fea9-3092-8248-9797-d738e3acf414 Address:9d08fea9-3092-8248-9797-d738e3acf414} {Suffrage:Voter ID:server-248701f8-b374-e0df-118b-d785a03bf8a0 Address:248701f8-b374-e0df-118b-d785a03bf8a0} {Suffrage:Voter ID:server-789d45d6-5fc1-e7e6-5537-d96fdf431b05 Address:789d45d6-5fc1-e7e6-5537-d96fdf431b05}]" - testing.go:149: server-248701f8-b374-e0df-118b-d785a03bf8a0: 2024-06-11T02:29:29.769-1200 [INFO] server-248701f8-b374-e0df-118b-d785a03bf8a0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9d08fea9-3092-8248-9797-d738e3acf414 Address:9d08fea9-3092-8248-9797-d738e3acf414} {Suffrage:Voter ID:server-248701f8-b374-e0df-118b-d785a03bf8a0 Address:248701f8-b374-e0df-118b-d785a03bf8a0} {Suffrage:Voter ID:server-789d45d6-5fc1-e7e6-5537-d96fdf431b05 Address:789d45d6-5fc1-e7e6-5537-d96fdf431b05}]" -testLoggerAdapter verbose: server-9d08fea9-3092-8248-9797-d738e3acf414: 2024-06-11T02:29:29.770-1200 [INFO] server-9d08fea9-3092-8248-9797-d738e3acf414: entering follower state: follower="Node at 9d08fea9-3092-8248-9797-d738e3acf414 [Follower]" leader= - testing.go:149: server-9d08fea9-3092-8248-9797-d738e3acf414: 2024-06-11T02:29:29.770-1200 [INFO] server-9d08fea9-3092-8248-9797-d738e3acf414: entering follower state: follower="Node at 9d08fea9-3092-8248-9797-d738e3acf414 [Follower]" leader= -testLoggerAdapter verbose: server-248701f8-b374-e0df-118b-d785a03bf8a0: 2024-06-11T02:29:29.770-1200 [INFO] server-248701f8-b374-e0df-118b-d785a03bf8a0: entering follower state: follower="Node at 248701f8-b374-e0df-118b-d785a03bf8a0 [Follower]" leader= - testing.go:149: server-248701f8-b374-e0df-118b-d785a03bf8a0: 2024-06-11T02:29:29.770-1200 [INFO] server-248701f8-b374-e0df-118b-d785a03bf8a0: entering follower state: follower="Node at 248701f8-b374-e0df-118b-d785a03bf8a0 [Follower]" leader= -testLoggerAdapter verbose: server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: 2024-06-11T02:29:29.771-1200 [INFO] server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9d08fea9-3092-8248-9797-d738e3acf414 Address:9d08fea9-3092-8248-9797-d738e3acf414} {Suffrage:Voter ID:server-248701f8-b374-e0df-118b-d785a03bf8a0 Address:248701f8-b374-e0df-118b-d785a03bf8a0} {Suffrage:Voter ID:server-789d45d6-5fc1-e7e6-5537-d96fdf431b05 Address:789d45d6-5fc1-e7e6-5537-d96fdf431b05}]" - testing.go:149: server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: 2024-06-11T02:29:29.771-1200 [INFO] server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9d08fea9-3092-8248-9797-d738e3acf414 Address:9d08fea9-3092-8248-9797-d738e3acf414} {Suffrage:Voter ID:server-248701f8-b374-e0df-118b-d785a03bf8a0 Address:248701f8-b374-e0df-118b-d785a03bf8a0} {Suffrage:Voter ID:server-789d45d6-5fc1-e7e6-5537-d96fdf431b05 Address:789d45d6-5fc1-e7e6-5537-d96fdf431b05}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:29.771-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:29.771-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: 2024-06-11T02:29:29.772-1200 [INFO] server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: entering follower state: follower="Node at 789d45d6-5fc1-e7e6-5537-d96fdf431b05 [Follower]" leader= - testing.go:149: server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: 2024-06-11T02:29:29.772-1200 [INFO] server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: entering follower state: follower="Node at 789d45d6-5fc1-e7e6-5537-d96fdf431b05 [Follower]" leader= -testLoggerAdapter verbose: server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: 2024-06-11T02:29:29.826-1200 [WARN] server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: 2024-06-11T02:29:29.826-1200 [WARN] server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: 2024-06-11T02:29:29.826-1200 [INFO] server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: entering candidate state: node="Node at 789d45d6-5fc1-e7e6-5537-d96fdf431b05 [Candidate]" term=2 - testing.go:149: server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: 2024-06-11T02:29:29.826-1200 [INFO] server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: entering candidate state: node="Node at 789d45d6-5fc1-e7e6-5537-d96fdf431b05 [Candidate]" term=2 -testLoggerAdapter verbose: server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: 2024-06-11T02:29:29.826-1200 [INFO] server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: election won: tally=2 - testing.go:149: server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: 2024-06-11T02:29:29.826-1200 [INFO] server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: election won: tally=2 -testLoggerAdapter verbose: server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: 2024-06-11T02:29:29.826-1200 [INFO] server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: entering leader state: leader="Node at 789d45d6-5fc1-e7e6-5537-d96fdf431b05 [Leader]" - testing.go:149: server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: 2024-06-11T02:29:29.826-1200 [INFO] server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: entering leader state: leader="Node at 789d45d6-5fc1-e7e6-5537-d96fdf431b05 [Leader]" -testLoggerAdapter verbose: server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: 2024-06-11T02:29:29.826-1200 [INFO] server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: added peer, starting replication: peer=server-9d08fea9-3092-8248-9797-d738e3acf414 - testing.go:149: server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: 2024-06-11T02:29:29.826-1200 [INFO] server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: added peer, starting replication: peer=server-9d08fea9-3092-8248-9797-d738e3acf414 -testLoggerAdapter verbose: server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: 2024-06-11T02:29:29.826-1200 [INFO] server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: added peer, starting replication: peer=server-248701f8-b374-e0df-118b-d785a03bf8a0 - testing.go:149: server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: 2024-06-11T02:29:29.826-1200 [INFO] server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: added peer, starting replication: peer=server-248701f8-b374-e0df-118b-d785a03bf8a0 -testLoggerAdapter verbose: server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: 2024-06-11T02:29:29.827-1200 [INFO] server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: pipelining replication: peer="{Voter server-9d08fea9-3092-8248-9797-d738e3acf414 9d08fea9-3092-8248-9797-d738e3acf414}" - testing.go:149: server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: 2024-06-11T02:29:29.827-1200 [INFO] server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: pipelining replication: peer="{Voter server-9d08fea9-3092-8248-9797-d738e3acf414 9d08fea9-3092-8248-9797-d738e3acf414}" -testLoggerAdapter verbose: server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: 2024-06-11T02:29:29.827-1200 [INFO] server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: pipelining replication: peer="{Voter server-248701f8-b374-e0df-118b-d785a03bf8a0 248701f8-b374-e0df-118b-d785a03bf8a0}" - testing.go:149: server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: 2024-06-11T02:29:29.827-1200 [INFO] server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: pipelining replication: peer="{Voter server-248701f8-b374-e0df-118b-d785a03bf8a0 248701f8-b374-e0df-118b-d785a03bf8a0}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:29.932-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:29.826867425 -1200 -12 m=+20.822240258 (1 nodes), 55.021007ms from start of poll, 58.936267ms from cluster start. Timeout at 2024-06-11 02:29:29.932325399 -1200 -12 m=+20.927697941, 105.457683ms after stability - testing.go:149: cluster: 2024-06-11T02:29:29.932-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:29.826867425 -1200 -12 m=+20.822240258 (1 nodes), 55.021007ms from start of poll, 58.936267ms from cluster start. Timeout at 2024-06-11 02:29:29.932325399 -1200 -12 m=+20.927697941, 105.457683ms after stability -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:29.932-1200 [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2024-06-11T02:29:29.932-1200 [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:30.038-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:29.932867889 -1200 -12 m=+20.928240431 (2 nodes), 3.792µs from start of poll, 164.93644ms from cluster start. Timeout at 2024-06-11 02:29:30.038100993 -1200 -12 m=+21.033473534, 105.233103ms after stability - testing.go:149: cluster: 2024-06-11T02:29:30.038-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:29.932867889 -1200 -12 m=+20.928240431 (2 nodes), 3.792µs from start of poll, 164.93644ms from cluster start. Timeout at 2024-06-11 02:29:30.038100993 -1200 -12 m=+21.033473534, 105.233103ms after stability -testLoggerAdapter verbose: server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: 2024-06-11T02:29:30.812-1200 [INFO] server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: aborting pipeline replication: peer="{Voter server-9d08fea9-3092-8248-9797-d738e3acf414 9d08fea9-3092-8248-9797-d738e3acf414}" - testing.go:149: server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: 2024-06-11T02:29:30.812-1200 [INFO] server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: aborting pipeline replication: peer="{Voter server-9d08fea9-3092-8248-9797-d738e3acf414 9d08fea9-3092-8248-9797-d738e3acf414}" -testLoggerAdapter verbose: server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: 2024-06-11T02:29:30.812-1200 [INFO] server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: aborting pipeline replication: peer="{Voter server-248701f8-b374-e0df-118b-d785a03bf8a0 248701f8-b374-e0df-118b-d785a03bf8a0}" - testing.go:149: server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: 2024-06-11T02:29:30.812-1200 [INFO] server-789d45d6-5fc1-e7e6-5537-d96fdf431b05: aborting pipeline replication: peer="{Voter server-248701f8-b374-e0df-118b-d785a03bf8a0 248701f8-b374-e0df-118b-d785a03bf8a0}" ---- PASS: TestRaft_LeadershipTransferLeaderReplicationTimeout (1.05s) -=== RUN TestRaft_LeadershipTransferStopRightAway ---- PASS: TestRaft_LeadershipTransferStopRightAway (0.00s) -=== RUN TestRaft_GetConfigurationNoBootstrap -testLoggerAdapter verbose: server-a5798458-aaed-5a8f-41cc-79d374b734d1: 2024-06-11T02:29:30.817-1200 [INFO] server-a5798458-aaed-5a8f-41cc-79d374b734d1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a5798458-aaed-5a8f-41cc-79d374b734d1 Address:a5798458-aaed-5a8f-41cc-79d374b734d1} {Suffrage:Voter ID:server-fb2d2c73-67ac-ede9-12d7-b25d82b963de Address:fb2d2c73-67ac-ede9-12d7-b25d82b963de}]" - testing.go:149: server-a5798458-aaed-5a8f-41cc-79d374b734d1: 2024-06-11T02:29:30.817-1200 [INFO] server-a5798458-aaed-5a8f-41cc-79d374b734d1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a5798458-aaed-5a8f-41cc-79d374b734d1 Address:a5798458-aaed-5a8f-41cc-79d374b734d1} {Suffrage:Voter ID:server-fb2d2c73-67ac-ede9-12d7-b25d82b963de Address:fb2d2c73-67ac-ede9-12d7-b25d82b963de}]" -testLoggerAdapter verbose: server-a5798458-aaed-5a8f-41cc-79d374b734d1: 2024-06-11T02:29:30.817-1200 [INFO] server-a5798458-aaed-5a8f-41cc-79d374b734d1: entering follower state: follower="Node at a5798458-aaed-5a8f-41cc-79d374b734d1 [Follower]" leader= - testing.go:149: server-a5798458-aaed-5a8f-41cc-79d374b734d1: 2024-06-11T02:29:30.817-1200 [INFO] server-a5798458-aaed-5a8f-41cc-79d374b734d1: entering follower state: follower="Node at a5798458-aaed-5a8f-41cc-79d374b734d1 [Follower]" leader= -testLoggerAdapter verbose: server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: 2024-06-11T02:29:30.817-1200 [INFO] server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a5798458-aaed-5a8f-41cc-79d374b734d1 Address:a5798458-aaed-5a8f-41cc-79d374b734d1} {Suffrage:Voter ID:server-fb2d2c73-67ac-ede9-12d7-b25d82b963de Address:fb2d2c73-67ac-ede9-12d7-b25d82b963de}]" - testing.go:149: server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: 2024-06-11T02:29:30.817-1200 [INFO] server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a5798458-aaed-5a8f-41cc-79d374b734d1 Address:a5798458-aaed-5a8f-41cc-79d374b734d1} {Suffrage:Voter ID:server-fb2d2c73-67ac-ede9-12d7-b25d82b963de Address:fb2d2c73-67ac-ede9-12d7-b25d82b963de}]" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:30.818-1200 [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2024-06-11T02:29:30.818-1200 [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: 2024-06-11T02:29:30.818-1200 [INFO] server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: entering follower state: follower="Node at fb2d2c73-67ac-ede9-12d7-b25d82b963de [Follower]" leader= - testing.go:149: server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: 2024-06-11T02:29:30.818-1200 [INFO] server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: entering follower state: follower="Node at fb2d2c73-67ac-ede9-12d7-b25d82b963de [Follower]" leader= -testLoggerAdapter verbose: server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: 2024-06-11T02:29:30.876-1200 [WARN] server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: 2024-06-11T02:29:30.876-1200 [WARN] server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: 2024-06-11T02:29:30.876-1200 [INFO] server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: entering candidate state: node="Node at fb2d2c73-67ac-ede9-12d7-b25d82b963de [Candidate]" term=2 - testing.go:149: server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: 2024-06-11T02:29:30.876-1200 [INFO] server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: entering candidate state: node="Node at fb2d2c73-67ac-ede9-12d7-b25d82b963de [Candidate]" term=2 -testLoggerAdapter verbose: server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: 2024-06-11T02:29:30.876-1200 [INFO] server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: election won: tally=2 - testing.go:149: server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: 2024-06-11T02:29:30.876-1200 [INFO] server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: election won: tally=2 -testLoggerAdapter verbose: server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: 2024-06-11T02:29:30.876-1200 [INFO] server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: entering leader state: leader="Node at fb2d2c73-67ac-ede9-12d7-b25d82b963de [Leader]" - testing.go:149: server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: 2024-06-11T02:29:30.876-1200 [INFO] server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: entering leader state: leader="Node at fb2d2c73-67ac-ede9-12d7-b25d82b963de [Leader]" -testLoggerAdapter verbose: server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: 2024-06-11T02:29:30.876-1200 [INFO] server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: added peer, starting replication: peer=server-a5798458-aaed-5a8f-41cc-79d374b734d1 - testing.go:149: server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: 2024-06-11T02:29:30.876-1200 [INFO] server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: added peer, starting replication: peer=server-a5798458-aaed-5a8f-41cc-79d374b734d1 -testLoggerAdapter verbose: server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: 2024-06-11T02:29:30.877-1200 [INFO] server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: pipelining replication: peer="{Voter server-a5798458-aaed-5a8f-41cc-79d374b734d1 a5798458-aaed-5a8f-41cc-79d374b734d1}" - testing.go:149: server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: 2024-06-11T02:29:30.877-1200 [INFO] server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: pipelining replication: peer="{Voter server-a5798458-aaed-5a8f-41cc-79d374b734d1 a5798458-aaed-5a8f-41cc-79d374b734d1}" -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:30.982-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:30.8772315 -1200 -12 m=+21.872604042 (1 nodes), 58.989349ms from start of poll, 60.160369ms from cluster start. Timeout at 2024-06-11 02:29:30.982377689 -1200 -12 m=+21.977750522, 105.14648ms after stability - testing.go:149: cluster: 2024-06-11T02:29:30.982-1200 [INFO] stable state for Follower reached at 2024-06-11 02:29:30.8772315 -1200 -12 m=+21.872604042 (1 nodes), 58.989349ms from start of poll, 60.160369ms from cluster start. Timeout at 2024-06-11 02:29:30.982377689 -1200 -12 m=+21.977750522, 105.14648ms after stability -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:30.982-1200 [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2024-06-11T02:29:30.982-1200 [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2024-06-11T02:29:31.088-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:30.982701724 -1200 -12 m=+21.978074266 (1 nodes), 2.625µs from start of poll, 165.630593ms from cluster start. Timeout at 2024-06-11 02:29:31.088328863 -1200 -12 m=+22.083701696, 105.62743ms after stability - testing.go:149: cluster: 2024-06-11T02:29:31.088-1200 [INFO] stable state for Leader reached at 2024-06-11 02:29:30.982701724 -1200 -12 m=+21.978074266 (1 nodes), 2.625µs from start of poll, 165.630593ms from cluster start. Timeout at 2024-06-11 02:29:31.088328863 -1200 -12 m=+22.083701696, 105.62743ms after stability -testLoggerAdapter verbose: server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: 2024-06-11T02:29:31.100-1200 [INFO] server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a5798458-aaed-5a8f-41cc-79d374b734d1 Address:a5798458-aaed-5a8f-41cc-79d374b734d1} {Suffrage:Voter ID:server-fb2d2c73-67ac-ede9-12d7-b25d82b963de Address:fb2d2c73-67ac-ede9-12d7-b25d82b963de}]" - testing.go:149: server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: 2024-06-11T02:29:31.100-1200 [INFO] server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a5798458-aaed-5a8f-41cc-79d374b734d1 Address:a5798458-aaed-5a8f-41cc-79d374b734d1} {Suffrage:Voter ID:server-fb2d2c73-67ac-ede9-12d7-b25d82b963de Address:fb2d2c73-67ac-ede9-12d7-b25d82b963de}]" -testLoggerAdapter verbose: server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: 2024-06-11T02:29:31.100-1200 [INFO] server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: aborting pipeline replication: peer="{Voter server-a5798458-aaed-5a8f-41cc-79d374b734d1 a5798458-aaed-5a8f-41cc-79d374b734d1}" - testing.go:149: server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: 2024-06-11T02:29:31.100-1200 [INFO] server-fb2d2c73-67ac-ede9-12d7-b25d82b963de: aborting pipeline replication: peer="{Voter server-a5798458-aaed-5a8f-41cc-79d374b734d1 a5798458-aaed-5a8f-41cc-79d374b734d1}" ---- PASS: TestRaft_GetConfigurationNoBootstrap (0.29s) -=== RUN TestTCPTransport_BadAddr ---- PASS: TestTCPTransport_BadAddr (0.00s) -=== RUN TestTCPTransport_WithAdvertise ---- PASS: TestTCPTransport_WithAdvertise (0.00s) -=== RUN TestTransport_StartStop ---- PASS: TestTransport_StartStop (0.00s) -=== RUN TestTransport_AppendEntries ---- PASS: TestTransport_AppendEntries (0.00s) -=== RUN TestTransport_AppendEntriesPipeline ---- PASS: TestTransport_AppendEntriesPipeline (0.00s) -=== RUN TestTransport_RequestVote ---- PASS: TestTransport_RequestVote (0.00s) -=== RUN TestTransport_InstallSnapshot ---- PASS: TestTransport_InstallSnapshot (0.00s) -=== RUN TestTransport_EncodeDecode ---- PASS: TestTransport_EncodeDecode (0.00s) -=== RUN TestRandomTimeout ---- PASS: TestRandomTimeout (0.00s) -=== RUN TestNewSeed ---- PASS: TestNewSeed (0.00s) -=== RUN TestRandomTimeout_NoTime ---- PASS: TestRandomTimeout_NoTime (0.00s) -=== RUN TestMin ---- PASS: TestMin (0.00s) -=== RUN TestMax ---- PASS: TestMax (0.00s) -=== RUN TestGenerateUUID ---- PASS: TestGenerateUUID (0.01s) -=== RUN TestBackoff ---- PASS: TestBackoff (0.00s) -PASS -ok github.com/hashicorp/raft 22.128s +testLoggerAdapter verbose: server-31dc35fc-435f-67cd-0a44-c91d963b34d6: 2024-06-12T04:32:34.042+1400 [INFO] server-31dc35fc-435f-67cd-0a44-c91d963b34d6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31dc35fc-435f-67cd-0a44-c91d963b34d6 Address:31dc35fc-435f-67cd-0a44-c91d963b34d6} {Suffrage:Voter ID:server-e3f27379-9cff-8617-88f3-80539ce8e42f Address:e3f27379-9cff-8617-88f3-80539ce8e42f} {Suffrage:Voter ID:server-b9be970b-d5d2-a979-7099-2aa5e66846e3 Address:b9be970b-d5d2-a979-7099-2aa5e66846e3}]" + testing.go:149: server-31dc35fc-435f-67cd-0a44-c91d963b34d6: 2024-06-12T04:32:34.042+1400 [INFO] server-31dc35fc-435f-67cd-0a44-c91d963b34d6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31dc35fc-435f-67cd-0a44-c91d963b34d6 Address:31dc35fc-435f-67cd-0a44-c91d963b34d6} {Suffrage:Voter ID:server-e3f27379-9cff-8617-88f3-80539ce8e42f Address:e3f27379-9cff-8617-88f3-80539ce8e42f} {Suffrage:Voter ID:server-b9be970b-d5d2-a979-7099-2aa5e66846e3 Address:b9be970b-d5d2-a979-7099-2aa5e66846e3}]" +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.042+1400 [INFO] server-e3f27379-9cff-8617-88f3-80539ce8e42f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31dc35fc-435f-67cd-0a44-c91d963b34d6 Address:31dc35fc-435f-67cd-0a44-c91d963b34d6} {Suffrage:Voter ID:server-e3f27379-9cff-8617-88f3-80539ce8e42f Address:e3f27379-9cff-8617-88f3-80539ce8e42f} {Suffrage:Voter ID:server-b9be970b-d5d2-a979-7099-2aa5e66846e3 Address:b9be970b-d5d2-a979-7099-2aa5e66846e3}]" + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.042+1400 [INFO] server-e3f27379-9cff-8617-88f3-80539ce8e42f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31dc35fc-435f-67cd-0a44-c91d963b34d6 Address:31dc35fc-435f-67cd-0a44-c91d963b34d6} {Suffrage:Voter ID:server-e3f27379-9cff-8617-88f3-80539ce8e42f Address:e3f27379-9cff-8617-88f3-80539ce8e42f} {Suffrage:Voter ID:server-b9be970b-d5d2-a979-7099-2aa5e66846e3 Address:b9be970b-d5d2-a979-7099-2aa5e66846e3}]" +testLoggerAdapter verbose: server-b9be970b-d5d2-a979-7099-2aa5e66846e3: 2024-06-12T04:32:34.042+1400 [INFO] server-b9be970b-d5d2-a979-7099-2aa5e66846e3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31dc35fc-435f-67cd-0a44-c91d963b34d6 Address:31dc35fc-435f-67cd-0a44-c91d963b34d6} {Suffrage:Voter ID:server-e3f27379-9cff-8617-88f3-80539ce8e42f Address:e3f27379-9cff-8617-88f3-80539ce8e42f} {Suffrage:Voter ID:server-b9be970b-d5d2-a979-7099-2aa5e66846e3 Address:b9be970b-d5d2-a979-7099-2aa5e66846e3}]" + testing.go:149: server-b9be970b-d5d2-a979-7099-2aa5e66846e3: 2024-06-12T04:32:34.042+1400 [INFO] server-b9be970b-d5d2-a979-7099-2aa5e66846e3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31dc35fc-435f-67cd-0a44-c91d963b34d6 Address:31dc35fc-435f-67cd-0a44-c91d963b34d6} {Suffrage:Voter ID:server-e3f27379-9cff-8617-88f3-80539ce8e42f Address:e3f27379-9cff-8617-88f3-80539ce8e42f} {Suffrage:Voter ID:server-b9be970b-d5d2-a979-7099-2aa5e66846e3 Address:b9be970b-d5d2-a979-7099-2aa5e66846e3}]" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:34.043+1400 [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-06-12T04:32:34.043+1400 [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-31dc35fc-435f-67cd-0a44-c91d963b34d6: 2024-06-12T04:32:34.043+1400 [INFO] server-31dc35fc-435f-67cd-0a44-c91d963b34d6: entering follower state: follower="Node at 31dc35fc-435f-67cd-0a44-c91d963b34d6 [Follower]" leader= + testing.go:149: server-31dc35fc-435f-67cd-0a44-c91d963b34d6: 2024-06-12T04:32:34.043+1400 [INFO] server-31dc35fc-435f-67cd-0a44-c91d963b34d6: entering follower state: follower="Node at 31dc35fc-435f-67cd-0a44-c91d963b34d6 [Follower]" leader= +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.043+1400 [INFO] server-e3f27379-9cff-8617-88f3-80539ce8e42f: entering follower state: follower="Node at e3f27379-9cff-8617-88f3-80539ce8e42f [Follower]" leader= + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.043+1400 [INFO] server-e3f27379-9cff-8617-88f3-80539ce8e42f: entering follower state: follower="Node at e3f27379-9cff-8617-88f3-80539ce8e42f [Follower]" leader= +testLoggerAdapter verbose: server-b9be970b-d5d2-a979-7099-2aa5e66846e3: 2024-06-12T04:32:34.043+1400 [INFO] server-b9be970b-d5d2-a979-7099-2aa5e66846e3: entering follower state: follower="Node at b9be970b-d5d2-a979-7099-2aa5e66846e3 [Follower]" leader= + testing.go:149: server-b9be970b-d5d2-a979-7099-2aa5e66846e3: 2024-06-12T04:32:34.043+1400 [INFO] server-b9be970b-d5d2-a979-7099-2aa5e66846e3: entering follower state: follower="Node at b9be970b-d5d2-a979-7099-2aa5e66846e3 [Follower]" leader= +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.096+1400 [WARN] server-e3f27379-9cff-8617-88f3-80539ce8e42f: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.096+1400 [WARN] server-e3f27379-9cff-8617-88f3-80539ce8e42f: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.096+1400 [INFO] server-e3f27379-9cff-8617-88f3-80539ce8e42f: entering candidate state: node="Node at e3f27379-9cff-8617-88f3-80539ce8e42f [Candidate]" term=2 + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.096+1400 [INFO] server-e3f27379-9cff-8617-88f3-80539ce8e42f: entering candidate state: node="Node at e3f27379-9cff-8617-88f3-80539ce8e42f [Candidate]" term=2 +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.096+1400 [INFO] server-e3f27379-9cff-8617-88f3-80539ce8e42f: election won: tally=2 + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.096+1400 [INFO] server-e3f27379-9cff-8617-88f3-80539ce8e42f: election won: tally=2 +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.096+1400 [INFO] server-e3f27379-9cff-8617-88f3-80539ce8e42f: entering leader state: leader="Node at e3f27379-9cff-8617-88f3-80539ce8e42f [Leader]" + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.096+1400 [INFO] server-e3f27379-9cff-8617-88f3-80539ce8e42f: entering leader state: leader="Node at e3f27379-9cff-8617-88f3-80539ce8e42f [Leader]" +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.096+1400 [INFO] server-e3f27379-9cff-8617-88f3-80539ce8e42f: added peer, starting replication: peer=server-31dc35fc-435f-67cd-0a44-c91d963b34d6 + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.096+1400 [INFO] server-e3f27379-9cff-8617-88f3-80539ce8e42f: added peer, starting replication: peer=server-31dc35fc-435f-67cd-0a44-c91d963b34d6 +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.096+1400 [INFO] server-e3f27379-9cff-8617-88f3-80539ce8e42f: added peer, starting replication: peer=server-b9be970b-d5d2-a979-7099-2aa5e66846e3 + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.096+1400 [INFO] server-e3f27379-9cff-8617-88f3-80539ce8e42f: added peer, starting replication: peer=server-b9be970b-d5d2-a979-7099-2aa5e66846e3 +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.096+1400 [INFO] server-e3f27379-9cff-8617-88f3-80539ce8e42f: pipelining replication: peer="{Voter server-b9be970b-d5d2-a979-7099-2aa5e66846e3 b9be970b-d5d2-a979-7099-2aa5e66846e3}" + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.096+1400 [INFO] server-e3f27379-9cff-8617-88f3-80539ce8e42f: pipelining replication: peer="{Voter server-b9be970b-d5d2-a979-7099-2aa5e66846e3 b9be970b-d5d2-a979-7099-2aa5e66846e3}" +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.096+1400 [INFO] server-e3f27379-9cff-8617-88f3-80539ce8e42f: pipelining replication: peer="{Voter server-31dc35fc-435f-67cd-0a44-c91d963b34d6 31dc35fc-435f-67cd-0a44-c91d963b34d6}" + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.096+1400 [INFO] server-e3f27379-9cff-8617-88f3-80539ce8e42f: pipelining replication: peer="{Voter server-31dc35fc-435f-67cd-0a44-c91d963b34d6 31dc35fc-435f-67cd-0a44-c91d963b34d6}" +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:34.202+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:34.096898455 +1400 +14 m=+11.004417034 (1 nodes), 53.827232ms from start of poll, 54.617204ms from cluster start. Timeout at 2024-06-12 04:32:34.202090967 +1400 +14 m=+11.109609546, 105.192512ms after stability + testing.go:149: cluster: 2024-06-12T04:32:34.202+1400 [INFO] stable state for Leader reached at 2024-06-12 04:32:34.096898455 +1400 +14 m=+11.004417034 (1 nodes), 53.827232ms from start of poll, 54.617204ms from cluster start. Timeout at 2024-06-12 04:32:34.202090967 +1400 +14 m=+11.109609546, 105.192512ms after stability +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.202+1400 [INFO] server-e3f27379-9cff-8617-88f3-80539ce8e42f: starting snapshot up to: index=12 + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.202+1400 [INFO] server-e3f27379-9cff-8617-88f3-80539ce8e42f: starting snapshot up to: index=12 + testing.go:153: 2024-06-12T04:32:34.202+1400 [INFO] creating new snapshot: path=/tmp/raft1004055718/snapshots/2-12-1718116354202.tmp +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.217+1400 [INFO] server-e3f27379-9cff-8617-88f3-80539ce8e42f: snapshot complete up to: index=12 + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.217+1400 [INFO] server-e3f27379-9cff-8617-88f3-80539ce8e42f: snapshot complete up to: index=12 + testing.go:153: 2024-06-12T04:32:34.218+1400 [INFO] creating new snapshot: path=/tmp/raft1004055718/snapshots/2-23-1718116354218.tmp +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.232+1400 [INFO] server-e3f27379-9cff-8617-88f3-80539ce8e42f: copied to local snapshot: bytes=71 + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.232+1400 [INFO] server-e3f27379-9cff-8617-88f3-80539ce8e42f: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.233+1400 [INFO] server-e3f27379-9cff-8617-88f3-80539ce8e42f: restored user snapshot: index=1 + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.233+1400 [INFO] server-e3f27379-9cff-8617-88f3-80539ce8e42f: restored user snapshot: index=1 +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.233+1400 [ERROR] server-e3f27379-9cff-8617-88f3-80539ce8e42f: failed to get log: index=23 error="log not found" + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.233+1400 [ERROR] server-e3f27379-9cff-8617-88f3-80539ce8e42f: failed to get log: index=23 error="log not found" +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.233+1400 [INFO] server-e3f27379-9cff-8617-88f3-80539ce8e42f: aborting pipeline replication: peer="{Voter server-b9be970b-d5d2-a979-7099-2aa5e66846e3 b9be970b-d5d2-a979-7099-2aa5e66846e3}" + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.233+1400 [INFO] server-e3f27379-9cff-8617-88f3-80539ce8e42f: aborting pipeline replication: peer="{Voter server-b9be970b-d5d2-a979-7099-2aa5e66846e3 b9be970b-d5d2-a979-7099-2aa5e66846e3}" +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.233+1400 [ERROR] server-e3f27379-9cff-8617-88f3-80539ce8e42f: failed to get log: index=23 error="log not found" + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.233+1400 [ERROR] server-e3f27379-9cff-8617-88f3-80539ce8e42f: failed to get log: index=23 error="log not found" +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.233+1400 [INFO] server-e3f27379-9cff-8617-88f3-80539ce8e42f: aborting pipeline replication: peer="{Voter server-31dc35fc-435f-67cd-0a44-c91d963b34d6 31dc35fc-435f-67cd-0a44-c91d963b34d6}" + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.233+1400 [INFO] server-e3f27379-9cff-8617-88f3-80539ce8e42f: aborting pipeline replication: peer="{Voter server-31dc35fc-435f-67cd-0a44-c91d963b34d6 31dc35fc-435f-67cd-0a44-c91d963b34d6}" +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.243+1400 [ERROR] server-e3f27379-9cff-8617-88f3-80539ce8e42f: failed to get log: index=23 error="log not found" + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.243+1400 [ERROR] server-e3f27379-9cff-8617-88f3-80539ce8e42f: failed to get log: index=23 error="log not found" + testing.go:153: 2024-06-12T04:32:34.244+1400 [INFO] creating new snapshot: path=/tmp/raft605465991/snapshots/2-23-1718116354244.tmp +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.244+1400 [ERROR] server-e3f27379-9cff-8617-88f3-80539ce8e42f: failed to get log: index=23 error="log not found" + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.244+1400 [ERROR] server-e3f27379-9cff-8617-88f3-80539ce8e42f: failed to get log: index=23 error="log not found" + testing.go:153: 2024-06-12T04:32:34.245+1400 [INFO] creating new snapshot: path=/tmp/raft669631066/snapshots/2-23-1718116354245.tmp +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.293+1400 [WARN] server-e3f27379-9cff-8617-88f3-80539ce8e42f: failed to contact: server-id=server-31dc35fc-435f-67cd-0a44-c91d963b34d6 time=52.036296ms + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.293+1400 [WARN] server-e3f27379-9cff-8617-88f3-80539ce8e42f: failed to contact: server-id=server-31dc35fc-435f-67cd-0a44-c91d963b34d6 time=52.036296ms +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.293+1400 [WARN] server-e3f27379-9cff-8617-88f3-80539ce8e42f: failed to contact: server-id=server-b9be970b-d5d2-a979-7099-2aa5e66846e3 time=51.997138ms + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.293+1400 [WARN] server-e3f27379-9cff-8617-88f3-80539ce8e42f: failed to contact: server-id=server-b9be970b-d5d2-a979-7099-2aa5e66846e3 time=51.997138ms +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.293+1400 [WARN] server-e3f27379-9cff-8617-88f3-80539ce8e42f: failed to contact quorum of nodes, stepping down + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.293+1400 [WARN] server-e3f27379-9cff-8617-88f3-80539ce8e42f: failed to contact quorum of nodes, stepping down +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.293+1400 [INFO] server-e3f27379-9cff-8617-88f3-80539ce8e42f: entering follower state: follower="Node at e3f27379-9cff-8617-88f3-80539ce8e42f [Follower]" leader= + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.293+1400 [INFO] server-e3f27379-9cff-8617-88f3-80539ce8e42f: entering follower state: follower="Node at e3f27379-9cff-8617-88f3-80539ce8e42f [Follower]" leader= +testLoggerAdapter verbose: cluster: 2024-06-12T04:32:34.293+1400 [ERROR] Restore failed: leadership lost while committing log + testing.go:149: cluster: 2024-06-12T04:32:34.293+1400 [ERROR] Restore failed: leadership lost while committing log +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.298+1400 [ERROR] server-e3f27379-9cff-8617-88f3-80539ce8e42f: failed to heartbeat to: peer=31dc35fc-435f-67cd-0a44-c91d963b34d6 error="command timed out" + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.298+1400 [ERROR] server-e3f27379-9cff-8617-88f3-80539ce8e42f: failed to heartbeat to: peer=31dc35fc-435f-67cd-0a44-c91d963b34d6 error="command timed out" +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.298+1400 [ERROR] server-e3f27379-9cff-8617-88f3-80539ce8e42f: failed to heartbeat to: peer=b9be970b-d5d2-a979-7099-2aa5e66846e3 error="command timed out" + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.298+1400 [ERROR] server-e3f27379-9cff-8617-88f3-80539ce8e42f: failed to heartbeat to: peer=b9be970b-d5d2-a979-7099-2aa5e66846e3 error="command timed out" +testLoggerAdapter verbose: server-b9be970b-d5d2-a979-7099-2aa5e66846e3: 2024-06-12T04:32:34.333+1400 [INFO] server-b9be970b-d5d2-a979-7099-2aa5e66846e3: copied to local snapshot: bytes=71 + testing.go:149: server-b9be970b-d5d2-a979-7099-2aa5e66846e3: 2024-06-12T04:32:34.333+1400 [INFO] server-b9be970b-d5d2-a979-7099-2aa5e66846e3: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-31dc35fc-435f-67cd-0a44-c91d963b34d6: 2024-06-12T04:32:34.336+1400 [INFO] server-31dc35fc-435f-67cd-0a44-c91d963b34d6: copied to local snapshot: bytes=71 + testing.go:149: server-31dc35fc-435f-67cd-0a44-c91d963b34d6: 2024-06-12T04:32:34.336+1400 [INFO] server-31dc35fc-435f-67cd-0a44-c91d963b34d6: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.336+1400 [WARN] server-e3f27379-9cff-8617-88f3-80539ce8e42f: installSnapshot rejected to: peer="{Voter server-31dc35fc-435f-67cd-0a44-c91d963b34d6 31dc35fc-435f-67cd-0a44-c91d963b34d6}" + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.336+1400 [WARN] server-e3f27379-9cff-8617-88f3-80539ce8e42f: installSnapshot rejected to: peer="{Voter server-31dc35fc-435f-67cd-0a44-c91d963b34d6 31dc35fc-435f-67cd-0a44-c91d963b34d6}" +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.363+1400 [ERROR] server-e3f27379-9cff-8617-88f3-80539ce8e42f: failed to heartbeat to: peer=31dc35fc-435f-67cd-0a44-c91d963b34d6 error="command timed out" + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.363+1400 [ERROR] server-e3f27379-9cff-8617-88f3-80539ce8e42f: failed to heartbeat to: peer=31dc35fc-435f-67cd-0a44-c91d963b34d6 error="command timed out" +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.369+1400 [ERROR] server-e3f27379-9cff-8617-88f3-80539ce8e42f: failed to heartbeat to: peer=b9be970b-d5d2-a979-7099-2aa5e66846e3 error="command timed out" + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.369+1400 [ERROR] server-e3f27379-9cff-8617-88f3-80539ce8e42f: failed to heartbeat to: peer=b9be970b-d5d2-a979-7099-2aa5e66846e3 error="command timed out" +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.439+1400 [ERROR] server-e3f27379-9cff-8617-88f3-80539ce8e42f: failed to heartbeat to: peer=b9be970b-d5d2-a979-7099-2aa5e66846e3 error="command timed out" + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.439+1400 [ERROR] server-e3f27379-9cff-8617-88f3-80539ce8e42f: failed to heartbeat to: peer=b9be970b-d5d2-a979-7099-2aa5e66846e3 error="command timed out" +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.518+1400 [ERROR] server-e3f27379-9cff-8617-88f3-80539ce8e42f: failed to heartbeat to: peer=b9be970b-d5d2-a979-7099-2aa5e66846e3 error="command timed out" + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.518+1400 [ERROR] server-e3f27379-9cff-8617-88f3-80539ce8e42f: failed to heartbeat to: peer=b9be970b-d5d2-a979-7099-2aa5e66846e3 error="command timed out" +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.618+1400 [ERROR] server-e3f27379-9cff-8617-88f3-80539ce8e42f: failed to heartbeat to: peer=b9be970b-d5d2-a979-7099-2aa5e66846e3 error="command timed out" + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.618+1400 [ERROR] server-e3f27379-9cff-8617-88f3-80539ce8e42f: failed to heartbeat to: peer=b9be970b-d5d2-a979-7099-2aa5e66846e3 error="command timed out" +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.745+1400 [ERROR] server-e3f27379-9cff-8617-88f3-80539ce8e42f: failed to install snapshot: id=2-23-1718116354218 error="command timed out" + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.745+1400 [ERROR] server-e3f27379-9cff-8617-88f3-80539ce8e42f: failed to install snapshot: id=2-23-1718116354218 error="command timed out" +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.745+1400 [ERROR] server-e3f27379-9cff-8617-88f3-80539ce8e42f: failed to send snapshot to: peer="{Voter server-b9be970b-d5d2-a979-7099-2aa5e66846e3 b9be970b-d5d2-a979-7099-2aa5e66846e3}" error="command timed out" + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.745+1400 [ERROR] server-e3f27379-9cff-8617-88f3-80539ce8e42f: failed to send snapshot to: peer="{Voter server-b9be970b-d5d2-a979-7099-2aa5e66846e3 b9be970b-d5d2-a979-7099-2aa5e66846e3}" error="command timed out" +testLoggerAdapter verbose: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.757+1400 [ERROR] server-e3f27379-9cff-8617-88f3-80539ce8e42f: failed to heartbeat to: peer=b9be970b-d5d2-a979-7099-2aa5e66846e3 error="command timed out" + testing.go:149: server-e3f27379-9cff-8617-88f3-80539ce8e42f: 2024-06-12T04:32:34.757+1400 [ERROR] server-e3f27379-9cff-8617-88f3-80539ce8e42f: failed to heartbeat to: peer=b9be970b-d5d2-a979-7099-2aa5e66846e3 error="command timed out" +panic: timed out waiting for shutdown + +goroutine 1152 [running]: +github.com/hashicorp/raft.(*cluster).Close.func1() + /build/reproducible-path/golang-github-hashicorp-raft-1.1.2/obj-arm-linux-gnueabihf/src/github.com/hashicorp/raft/testing.go:260 +0x24 +created by time.goFunc + /usr/lib/go-1.19/src/time/sleep.go:176 +0x2c +FAIL github.com/hashicorp/raft 16.212s ? github.com/hashicorp/raft/bench [no test files] - create-stamp debian/debhelper-build-stamp - fakeroot debian/rules binary -dh binary --buildsystem=golang --with=golang - dh_testroot -O--buildsystem=golang - dh_prep -O--buildsystem=golang - dh_auto_install --destdir=debian/golang-github-hashicorp-raft-dev/ -O--buildsystem=golang - dh_installdocs -O--buildsystem=golang - dh_installchangelogs -O--buildsystem=golang - dh_installinit -O--buildsystem=golang - dh_installsystemduser -O--buildsystem=golang - dh_perl -O--buildsystem=golang - dh_link -O--buildsystem=golang - dh_strip_nondeterminism -O--buildsystem=golang - dh_compress -O--buildsystem=golang - dh_fixperms -O--buildsystem=golang - dh_missing -O--buildsystem=golang - dh_installdeb -O--buildsystem=golang - dh_golang -O--buildsystem=golang - dh_gencontrol -O--buildsystem=golang -dpkg-gencontrol: warning: Depends field of package golang-github-hashicorp-raft-dev: substitution variable ${shlibs:Depends} used, but is not defined - dh_md5sums -O--buildsystem=golang - dh_builddeb -O--buildsystem=golang -dpkg-deb: building package 'golang-github-hashicorp-raft-dev' in '../golang-github-hashicorp-raft-dev_1.1.2-1_all.deb'. - dpkg-genbuildinfo --build=binary -O../golang-github-hashicorp-raft_1.1.2-1_armhf.buildinfo - dpkg-genchanges --build=binary -O../golang-github-hashicorp-raft_1.1.2-1_armhf.changes -dpkg-genchanges: info: binary-only upload (no source code included) - dpkg-source --after-build . -dpkg-buildpackage: info: binary-only upload (no source included) -dpkg-genchanges: info: including full source code in upload +FAIL +dh_auto_test: error: cd obj-arm-linux-gnueabihf && go test -vet=off -v -p 4 github.com/hashicorp/raft github.com/hashicorp/raft/bench returned exit code 1 +make: *** [debian/rules:9: build] Error 25 +dpkg-buildpackage: error: debian/rules build subprocess returned exit status 2 I: copying local configuration +E: Failed autobuilding of package +I: user script /srv/workspace/pbuilder/18637/tmp/hooks/C01_cleanup starting +debug output: disk usage on i-capture-the-hostname at Tue Jun 11 14:32:39 UTC 2024 +Filesystem Size Used Avail Use% Mounted on +tmpfs 7.7G 0 7.7G 0% /dev/shm + +I: user script /srv/workspace/pbuilder/18637/tmp/hooks/C01_cleanup finished I: unmounting dev/ptmx filesystem I: unmounting dev/pts filesystem I: unmounting dev/shm filesystem I: unmounting proc filesystem I: unmounting sys filesystem I: cleaning the build env -I: removing directory /srv/workspace/pbuilder/23716 and its subdirectories -I: Current time: Tue Jun 11 02:29:51 -12 2024 -I: pbuilder-time-stamp: 1718116191 +I: removing directory /srv/workspace/pbuilder/18637 and its subdirectories