mirror of
https://github.com/libp2p/go-libp2p.git
synced 2026-04-23 00:27:05 +08:00
ci: Out of the tarpit (#2923)
* Lint fixes * Use latest go version for go-check Fixes nil pointer issue in staticcheck * Add test_analysis helper script * Use custom go-test-template * Add some tests to the test_analysis script * Always upload test_results db * Attempt to fix test on windows * Better if statement * Try to fix flaky test * Disable caching setup-go on Windows * Better if statement * Tweak * Always upload summary and artifact * Close db * No extra newline
This commit is contained in:
Marco Munizaga
@@ -9,3 +9,9 @@ runs:
|
||||
shell: bash
|
||||
# This matches only tests with "NoCover" in their test name to avoid running all tests again.
|
||||
run: go test -tags nocover -run NoCover -v ./...
|
||||
- name: Install testing tools
|
||||
shell: bash
|
||||
run: cd scripts/test_analysis && go install ./cmd/gotest2sql
|
||||
- name: Install test_analysis
|
||||
shell: bash
|
||||
run: cd scripts/test_analysis && go install .
|
||||
|
||||
@@ -17,4 +17,5 @@ jobs:
|
||||
go-check:
|
||||
uses: ipdxco/unified-github-workflows/.github/workflows/go-check.yml@v1.0
|
||||
with:
|
||||
go-version: "1.22.x"
|
||||
go-generate-ignore-protoc-version-comments: true
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
name: Go Test
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
go-versions:
|
||||
required: false
|
||||
type: string
|
||||
default: '["this", "next"]'
|
||||
secrets:
|
||||
CODECOV_TOKEN:
|
||||
required: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
unit:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: ["ubuntu", "macos", "windows"]
|
||||
go: ${{ fromJSON(inputs.go-versions) }}
|
||||
env:
|
||||
GOTESTFLAGS: -cover -coverprofile=module-coverage.txt -coverpkg=./...
|
||||
GO386FLAGS: ""
|
||||
GORACEFLAGS: ""
|
||||
runs-on: ${{ fromJSON(vars[format('UCI_GO_TEST_RUNNER_{0}', matrix.os)] || format('"{0}-latest"', matrix.os)) }}
|
||||
name: ${{ matrix.os }} (go ${{ matrix.go }})
|
||||
steps:
|
||||
- name: Use msys2 on windows
|
||||
if: matrix.os == 'windows'
|
||||
# The executable for msys2 is also called bash.cmd
|
||||
# https://github.com/actions/virtual-environments/blob/main/images/win/Windows2019-Readme.md#shells
|
||||
# If we prepend its location to the PATH
|
||||
# subsequent 'shell: bash' steps will use msys2 instead of gitbash
|
||||
run: echo "C:/msys64/usr/bin" >> $GITHUB_PATH
|
||||
- name: Check out the repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
- name: Check out the latest stable version of Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: stable
|
||||
cache: ${{ matrix.os != 'windows' }} # Windows VMs are slow to use caching. Can add ~15m to the job
|
||||
- name: Read the Unified GitHub Workflows configuration
|
||||
id: config
|
||||
uses: ipdxco/unified-github-workflows/.github/actions/read-config@main
|
||||
- name: Read the go.mod file
|
||||
id: go-mod
|
||||
uses: ipdxco/unified-github-workflows/.github/actions/read-go-mod@main
|
||||
- name: Determine the Go version to use based on the go.mod file
|
||||
id: go
|
||||
env:
|
||||
MATRIX_GO: ${{ matrix.go }}
|
||||
GO_MOD_VERSION: ${{ fromJSON(steps.go-mod.outputs.json).Go }}
|
||||
run: |
|
||||
if [[ "$MATRIX_GO" == "this" ]]; then
|
||||
echo "version=$GO_MOD_VERSION.x" >> $GITHUB_OUTPUT
|
||||
elif [[ "$MATRIX_GO" == "next" ]]; then
|
||||
MAJOR="${GO_MOD_VERSION%.[0-9]*}"
|
||||
MINOR="${GO_MOD_VERSION#[0-9]*.}"
|
||||
echo "version=$MAJOR.$(($MINOR+1)).x" >> $GITHUB_OUTPUT
|
||||
elif [[ "$MATRIX_GO" == "prev" ]]; then
|
||||
MAJOR="${GO_MOD_VERSION%.[0-9]*}"
|
||||
MINOR="${GO_MOD_VERSION#[0-9]*.}"
|
||||
echo "version=$MAJOR.$(($MINOR-1)).x" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "version=$MATRIX_GO" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
- name: Enable shuffle flag for go test command
|
||||
if: toJSON(fromJSON(steps.config.outputs.json).shuffle) != 'false'
|
||||
run: |
|
||||
echo "GOTESTFLAGS=-shuffle=on $GOTESTFLAGS" >> $GITHUB_ENV
|
||||
echo "GO386FLAGS=-shuffle=on $GO386FLAGS" >> $GITHUB_ENV
|
||||
echo "GORACEFLAGS=-shuffle=on $GORACEFLAGS" >> $GITHUB_ENV
|
||||
- name: Enable verbose flag for go test command
|
||||
if: toJSON(fromJSON(steps.config.outputs.json).verbose) != 'false'
|
||||
run: |
|
||||
echo "GOTESTFLAGS=-v $GOTESTFLAGS" >> $GITHUB_ENV
|
||||
echo "GO386FLAGS=-v $GO386FLAGS" >> $GITHUB_ENV
|
||||
echo "GORACEFLAGS=-v $GORACEFLAGS" >> $GITHUB_ENV
|
||||
- name: Set extra flags for go test command
|
||||
if: fromJSON(steps.config.outputs.json).gotestflags != ''
|
||||
run: |
|
||||
echo "GOTESTFLAGS=${{ fromJSON(steps.config.outputs.json).gotestflags }} $GOTESTFLAGS" >> $GITHUB_ENV
|
||||
- name: Set extra flags for go test race command
|
||||
if: fromJSON(steps.config.outputs.json).goraceflags != ''
|
||||
run: |
|
||||
echo "GORACEFLAGS=${{ fromJSON(steps.config.outputs.json).goraceflags }} $GORACEFLAGS" >> $GITHUB_ENV
|
||||
- name: Set up the Go version read from the go.mod file
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ steps.go.outputs.version }}
|
||||
cache: ${{ matrix.os != 'windows' }} # Windows VMs are slow to use caching. Can add ~15m to the job
|
||||
- name: Display the Go version and environment
|
||||
run: |
|
||||
go version
|
||||
go env
|
||||
- name: Run repo-specific setup
|
||||
uses: ./.github/actions/go-test-setup
|
||||
if: hashFiles('./.github/actions/go-test-setup') != ''
|
||||
- name: Run tests
|
||||
id: test
|
||||
if: contains(fromJSON(steps.config.outputs.json).skipOSes, matrix.os) == false
|
||||
uses: protocol/multiple-go-modules@v1.4
|
||||
with:
|
||||
run: test_analysis ${{ env.GOTESTFLAGS }}
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: ${{ matrix.os }}_${{ matrix.go }}_test_results.db
|
||||
path: ./test_results.db
|
||||
- name: Add failure summary
|
||||
if: always()
|
||||
run: |
|
||||
echo "### Failure Summary" >> $GITHUB_STEP_SUMMARY
|
||||
test_analysis summarize >> $GITHUB_STEP_SUMMARY
|
||||
- name: Remove test results
|
||||
run: rm ./test_results.db
|
||||
- name: Run tests with race detector
|
||||
# speed things up. Windows and OSX VMs are slow
|
||||
if: matrix.os == 'ubuntu' &&
|
||||
fromJSON(steps.config.outputs.json).skipRace != true &&
|
||||
contains(fromJSON(steps.config.outputs.json).skipOSes, matrix.os) == false
|
||||
uses: protocol/multiple-go-modules@v1.4
|
||||
id: race
|
||||
with:
|
||||
run: test_analysis -race ${{ env.GORACEFLAGS }} ./...
|
||||
- name: Upload test results (Race)
|
||||
if: (steps.race.conclusion == 'success' || steps.race.conclusion == 'failure')
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: ${{ matrix.os }}_${{ matrix.go }}_test_results_race.db
|
||||
path: ./test_results.db
|
||||
- name: Add failure summary
|
||||
if: (steps.race.conclusion == 'success' || steps.race.conclusion == 'failure')
|
||||
run: |
|
||||
echo "# Tests with race detector failure summary" >> $GITHUB_STEP_SUMMARY
|
||||
test_analysis summarize >> $GITHUB_STEP_SUMMARY
|
||||
- name: Adding Link to Run Analysis
|
||||
run: echo "### [Test flakiness analysis](https://observablehq.com/d/d74435ea5bbf24c7?run-id=$GITHUB_RUN_ID)" >> $GITHUB_STEP_SUMMARY
|
||||
- name: Collect coverage files
|
||||
id: coverages
|
||||
run: echo "files=$(find . -type f -name 'module-coverage.txt' | tr -s '\n' ',' | sed 's/,$//')" >> $GITHUB_OUTPUT
|
||||
- name: Upload coverage to Codecov
|
||||
uses: codecov/codecov-action@54bcd8715eee62d40e33596ef5e8f0f48dbbccab # v4.1.0
|
||||
with:
|
||||
files: ${{ steps.coverages.outputs.files }}
|
||||
env_vars: OS=${{ matrix.os }}, GO=${{ steps.go.outputs.version }}
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
fail_ci_if_error: false
|
||||
@@ -15,7 +15,7 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
go-test:
|
||||
uses: libp2p/uci/.github/workflows/go-test.yml@v1.0
|
||||
uses: ./.github/workflows/go-test-template.yml
|
||||
with:
|
||||
go-versions: '["1.21.x", "1.22.x"]'
|
||||
secrets:
|
||||
|
||||
+25
-6
@@ -5,6 +5,8 @@ import (
|
||||
"crypto/rand"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/netip"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -488,10 +490,23 @@ func TestHostAddrsFactoryAddsCerthashes(t *testing.T) {
|
||||
h.Close()
|
||||
}
|
||||
|
||||
func newRandomPort(t *testing.T) string {
|
||||
t.Helper()
|
||||
// Find an available port
|
||||
c, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0})
|
||||
require.NoError(t, err)
|
||||
c.LocalAddr().Network()
|
||||
ipPort := netip.MustParseAddrPort(c.LocalAddr().String())
|
||||
port := strconv.Itoa(int(ipPort.Port()))
|
||||
require.NoError(t, c.Close())
|
||||
return port
|
||||
}
|
||||
|
||||
func TestWebRTCReuseAddrWithQUIC(t *testing.T) {
|
||||
port := newRandomPort(t)
|
||||
order := [][]string{
|
||||
{"/ip4/127.0.0.1/udp/54322/quic-v1", "/ip4/127.0.0.1/udp/54322/webrtc-direct"},
|
||||
{"/ip4/127.0.0.1/udp/54322/webrtc-direct", "/ip4/127.0.0.1/udp/54322/quic-v1"},
|
||||
{"/ip4/127.0.0.1/udp/" + port + "/quic-v1", "/ip4/127.0.0.1/udp/" + port + "/webrtc-direct"},
|
||||
{"/ip4/127.0.0.1/udp/" + port + "/webrtc-direct", "/ip4/127.0.0.1/udp/" + port + "/quic-v1"},
|
||||
// We do not support WebRTC automatically reusing QUIC addresses if port is not specified, yet.
|
||||
// {"/ip4/127.0.0.1/udp/0/webrtc-direct", "/ip4/127.0.0.1/udp/0/quic-v1"},
|
||||
}
|
||||
@@ -542,16 +557,18 @@ func TestWebRTCReuseAddrWithQUIC(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
swapPort := func(addrStrs []string, newPort string) []string {
|
||||
swapPort := func(addrStrs []string, oldPort, newPort string) []string {
|
||||
out := make([]string, 0, len(addrStrs))
|
||||
for _, addrStr := range addrStrs {
|
||||
out = append(out, strings.Replace(addrStr, "54322", newPort, 1))
|
||||
out = append(out, strings.Replace(addrStr, oldPort, newPort, 1))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
t.Run("setup with no reuseport. Should fail", func(t *testing.T) {
|
||||
h1, err := New(ListenAddrStrings(swapPort(order[0], "54323")...), Transport(quic.NewTransport), Transport(libp2pwebrtc.New), QUICReuse(quicreuse.NewConnManager, quicreuse.DisableReuseport()))
|
||||
oldPort := port
|
||||
newPort := newRandomPort(t)
|
||||
h1, err := New(ListenAddrStrings(swapPort(order[0], oldPort, newPort)...), Transport(quic.NewTransport), Transport(libp2pwebrtc.New), QUICReuse(quicreuse.NewConnManager, quicreuse.DisableReuseport()))
|
||||
require.NoError(t, err) // It's a bug/feature that swarm.Listen does not error if at least one transport succeeds in listening.
|
||||
defer h1.Close()
|
||||
// Check that webrtc did fail to listen
|
||||
@@ -560,7 +577,9 @@ func TestWebRTCReuseAddrWithQUIC(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("setup with autonat", func(t *testing.T) {
|
||||
h1, err := New(EnableAutoNATv2(), ListenAddrStrings(swapPort(order[0], "54324")...), Transport(quic.NewTransport), Transport(libp2pwebrtc.New), QUICReuse(quicreuse.NewConnManager, quicreuse.DisableReuseport()))
|
||||
oldPort := port
|
||||
newPort := newRandomPort(t)
|
||||
h1, err := New(EnableAutoNATv2(), ListenAddrStrings(swapPort(order[0], oldPort, newPort)...), Transport(quic.NewTransport), Transport(libp2pwebrtc.New), QUICReuse(quicreuse.NewConnManager, quicreuse.DisableReuseport()))
|
||||
require.NoError(t, err) // It's a bug/feature that swarm.Listen does not error if at least one transport succeeds in listening.
|
||||
defer h1.Close()
|
||||
// Check that webrtc did fail to listen
|
||||
|
||||
@@ -995,7 +995,7 @@ func (ids *idService) addConnWithLock(c network.Conn) {
|
||||
}
|
||||
|
||||
func signedPeerRecordFromMessage(msg *pb.Identify) (*record.Envelope, error) {
|
||||
if msg.SignedPeerRecord == nil || len(msg.SignedPeerRecord) == 0 {
|
||||
if len(msg.SignedPeerRecord) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
env, _, err := record.ConsumeEnvelope(msg.SignedPeerRecord, peer.PeerRecordEnvelopeDomain)
|
||||
|
||||
@@ -960,7 +960,7 @@ func waitForAddrInStream(t *testing.T, s <-chan ma.Multiaddr, expected ma.Multia
|
||||
}
|
||||
continue
|
||||
case <-time.After(timeout):
|
||||
t.Fatalf(failMsg)
|
||||
t.Fatal(failMsg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,6 +14,6 @@ func main() {
|
||||
return
|
||||
}
|
||||
if err := cmdlib.RunClient(os.Args[1], os.Args[2]); err != nil {
|
||||
log.Fatalf(err.Error())
|
||||
log.Fatal(err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,6 +14,6 @@ func main() {
|
||||
return
|
||||
}
|
||||
if err := cmdlib.RunServer(os.Args[1], nil); err != nil {
|
||||
log.Fatalf(err.Error())
|
||||
log.Fatal(err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
// gotest2sql inserts the output of go test -json ./... into a sqlite database
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
_ "github.com/glebarez/go-sqlite"
|
||||
)
|
||||
|
||||
type TestEvent struct {
|
||||
Time time.Time // encodes as an RFC3339-format string
|
||||
Action string
|
||||
Package string
|
||||
Test string
|
||||
Elapsed float64 // seconds
|
||||
Output string
|
||||
}
|
||||
|
||||
func main() {
|
||||
outputPath := flag.String("output", "", "output db file")
|
||||
verbose := flag.Bool("v", false, "Print test output to stdout")
|
||||
flag.Parse()
|
||||
|
||||
if *outputPath == "" {
|
||||
log.Fatal("-output path is required")
|
||||
}
|
||||
|
||||
db, err := sql.Open("sqlite", *outputPath)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// Create a table to store test results.
|
||||
_, err = db.Exec(`
|
||||
CREATE TABLE IF NOT EXISTS test_results (
|
||||
Time TEXT,
|
||||
Action TEXT,
|
||||
Package TEXT,
|
||||
Test TEXT,
|
||||
Elapsed REAL,
|
||||
Output TEXT,
|
||||
BatchInsertTime TEXT
|
||||
)`)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// Prepare the insert statement once
|
||||
insertTime := time.Now().Format(time.RFC3339Nano)
|
||||
stmt, err := tx.Prepare(`
|
||||
INSERT INTO test_results (Time, Action, Package, Test, Elapsed, Output, BatchInsertTime)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)`)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer stmt.Close() // Ensure the statement is closed after use
|
||||
|
||||
s := bufio.NewScanner(os.Stdin)
|
||||
for s.Scan() {
|
||||
line := s.Bytes()
|
||||
var ev TestEvent
|
||||
err = json.Unmarshal(line, &ev)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if *verbose && ev.Action == "output" {
|
||||
fmt.Print(ev.Output)
|
||||
}
|
||||
|
||||
_, err = stmt.Exec(
|
||||
ev.Time.Format(time.RFC3339Nano),
|
||||
ev.Action,
|
||||
ev.Package,
|
||||
ev.Test,
|
||||
ev.Elapsed,
|
||||
ev.Output,
|
||||
insertTime,
|
||||
)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Commit the transaction
|
||||
if err := tx.Commit(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
module github.com/libp2p/go-libp2p/scripts/test_analysis
|
||||
|
||||
go 1.22.1
|
||||
|
||||
require github.com/glebarez/go-sqlite v1.22.0
|
||||
|
||||
require (
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/google/uuid v1.5.0 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
golang.org/x/sys v0.15.0 // indirect
|
||||
modernc.org/libc v1.37.6 // indirect
|
||||
modernc.org/mathutil v1.6.0 // indirect
|
||||
modernc.org/memory v1.7.2 // indirect
|
||||
modernc.org/sqlite v1.28.0 // indirect
|
||||
)
|
||||
@@ -0,0 +1,23 @@
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ=
|
||||
github.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc=
|
||||
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
|
||||
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=
|
||||
github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
|
||||
github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
|
||||
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
modernc.org/libc v1.37.6 h1:orZH3c5wmhIQFTXF+Nt+eeauyd+ZIt2BX6ARe+kD+aw=
|
||||
modernc.org/libc v1.37.6/go.mod h1:YAXkAZ8ktnkCKaN9sw/UDeUVkGYJ/YquGO4FTi5nmHE=
|
||||
modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
|
||||
modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
|
||||
modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E=
|
||||
modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E=
|
||||
modernc.org/sqlite v1.28.0 h1:Zx+LyDDmXczNnEQdvPuEfcFVA2ZPyaD7UCZDjef3BHQ=
|
||||
modernc.org/sqlite v1.28.0/go.mod h1:Qxpazz0zH8Z1xCFyi5GSL3FzbtZ3fvbjmywNogldEW0=
|
||||
@@ -0,0 +1,226 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
_ "github.com/glebarez/go-sqlite"
|
||||
)
|
||||
|
||||
const dbPath = "./test_results.db"
|
||||
const retryCount = 4 // For a total of 5 runs
|
||||
|
||||
var coverRegex = regexp.MustCompile(`-cover`)
|
||||
|
||||
func main() {
|
||||
var t tester
|
||||
if len(os.Args) >= 2 {
|
||||
if os.Args[1] == "summarize" {
|
||||
md, err := t.summarize()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
fmt.Print(md)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
passThruFlags := os.Args[1:]
|
||||
err := t.runTests(passThruFlags)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
type tester struct {
|
||||
Dir string
|
||||
}
|
||||
|
||||
func (t *tester) runTests(passThruFlags []string) error {
|
||||
err := t.goTestAll(passThruFlags)
|
||||
if err == nil {
|
||||
// No failed tests, nothing to do
|
||||
return nil
|
||||
}
|
||||
log.Printf("Not all tests passed: %v", err)
|
||||
|
||||
failedTests, err := t.findFailedTests(context.Background())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf("Found %d failed tests. Retrying them %d times", len(failedTests), retryCount)
|
||||
hasOneNonFlakyFailure := false
|
||||
loggedFlaky := map[string]struct{}{}
|
||||
|
||||
for _, ft := range failedTests {
|
||||
isFlaky := false
|
||||
for i := 0; i < retryCount; i++ {
|
||||
log.Printf("Retrying %s.%s", ft.Package, ft.Test)
|
||||
if err := t.goTestPkgTest(ft.Package, ft.Test, filterOutFlags(passThruFlags, coverRegex)); err != nil {
|
||||
log.Printf("Failed to run %s.%s: %v", ft.Package, ft.Test, err)
|
||||
} else {
|
||||
isFlaky = true
|
||||
flakyName := ft.Package + "." + ft.Test
|
||||
if _, ok := loggedFlaky[flakyName]; !ok {
|
||||
loggedFlaky[flakyName] = struct{}{}
|
||||
log.Printf("Test %s.%s is flaky.", ft.Package, ft.Test)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !isFlaky {
|
||||
hasOneNonFlakyFailure = true
|
||||
}
|
||||
}
|
||||
|
||||
// A test consistently failed, so we should exit with a non-zero exit code.
|
||||
if hasOneNonFlakyFailure {
|
||||
return errors.New("one or more tests consistently failed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *tester) goTestAll(extraFlags []string) error {
|
||||
flags := []string{"./..."}
|
||||
flags = append(flags, extraFlags...)
|
||||
return t.goTest(flags)
|
||||
}
|
||||
|
||||
func (t *tester) goTestPkgTest(pkg, testname string, extraFlags []string) error {
|
||||
flags := []string{
|
||||
pkg, "-run", "^" + testname + "$", "-count", "1",
|
||||
}
|
||||
flags = append(flags, extraFlags...)
|
||||
return t.goTest(flags)
|
||||
}
|
||||
|
||||
func (t *tester) goTest(extraFlags []string) error {
|
||||
flags := []string{
|
||||
"test", "-json",
|
||||
}
|
||||
flags = append(flags, extraFlags...)
|
||||
cmd := exec.Command("go", flags...)
|
||||
cmd.Dir = t.Dir
|
||||
cmd.Stderr = os.Stderr
|
||||
|
||||
gotest2sql := exec.Command("gotest2sql", "-v", "-output", dbPath)
|
||||
gotest2sql.Dir = t.Dir
|
||||
gotest2sql.Stdin, _ = cmd.StdoutPipe()
|
||||
gotest2sql.Stdout = os.Stdout
|
||||
gotest2sql.Stderr = os.Stderr
|
||||
err := gotest2sql.Start()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = cmd.Run()
|
||||
return errors.Join(err, gotest2sql.Wait())
|
||||
}
|
||||
|
||||
type failedTest struct {
|
||||
Package string
|
||||
Test string
|
||||
}
|
||||
|
||||
func (t *tester) findFailedTests(ctx context.Context) ([]failedTest, error) {
|
||||
db, err := sql.Open("sqlite", t.Dir+dbPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
rows, err := db.QueryContext(ctx, "SELECT DISTINCT Package, Test FROM test_results where Action='fail' and Test != ''")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []failedTest
|
||||
for rows.Next() {
|
||||
var pkg, test string
|
||||
if err := rows.Scan(&pkg, &test); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, failedTest{pkg, test})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func filterOutFlags(flags []string, exclude *regexp.Regexp) []string {
|
||||
out := make([]string, 0, len(flags))
|
||||
for _, f := range flags {
|
||||
if !exclude.MatchString(f) {
|
||||
out = append(out, f)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// summarize returns a markdown string of the test results.
|
||||
func (t *tester) summarize() (string, error) {
|
||||
ctx := context.Background()
|
||||
var out strings.Builder
|
||||
|
||||
testFailures, err := t.findFailedTests(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
plural := "s"
|
||||
if len(testFailures) == 1 {
|
||||
plural = ""
|
||||
}
|
||||
out.WriteString(fmt.Sprintf("## %d Test Failure%s\n\n", len(testFailures), plural))
|
||||
|
||||
db, err := sql.Open("sqlite", t.Dir+dbPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
rows, err := db.QueryContext(ctx, `SELECT
|
||||
tr_output.Package,
|
||||
tr_output.Test,
|
||||
GROUP_CONCAT(tr_output.Output, "") AS Outputs
|
||||
FROM
|
||||
test_results tr_fail
|
||||
JOIN
|
||||
test_results tr_output
|
||||
ON
|
||||
tr_fail.Test = tr_output.Test
|
||||
AND tr_fail.BatchInsertTime = tr_output.BatchInsertTime
|
||||
AND tr_fail.Package = tr_output.Package
|
||||
WHERE
|
||||
tr_fail.Action = 'fail'
|
||||
AND tr_output.Test != ''
|
||||
GROUP BY
|
||||
tr_output.BatchInsertTime,
|
||||
tr_output.Package,
|
||||
tr_output.Test
|
||||
ORDER BY
|
||||
MIN(tr_output.Time);`)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for rows.Next() {
|
||||
var pkg, test, outputs string
|
||||
if err := rows.Scan(&pkg, &test, &outputs); err != nil {
|
||||
return "", err
|
||||
}
|
||||
_, err = out.WriteString(fmt.Sprintf(`<details>
|
||||
<summary>%s.%s</summary>
|
||||
<pre>
|
||||
%s
|
||||
</pre>
|
||||
</details>`, pkg, test, outputs))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
return out.String(), nil
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFailsOnConsistentFailure(t *testing.T) {
|
||||
tmpDir := t.TempDir() + "/"
|
||||
os.WriteFile(tmpDir+"/main.go", []byte(`package main
|
||||
func main() {}`), 0644)
|
||||
// Add a test that fails consistently.
|
||||
os.WriteFile(tmpDir+"/main_test.go", []byte(`package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
func TestConsistentFailure(t *testing.T) {
|
||||
t.Fatal("consistent failure")
|
||||
}`), 0644)
|
||||
os.WriteFile(tmpDir+"/go.mod", []byte(`module example.com/test`), 0644)
|
||||
|
||||
tstr := tester{Dir: tmpDir}
|
||||
err := tstr.runTests(nil)
|
||||
if err == nil {
|
||||
t.Fatal("Should have failed with a consistent failure")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPassesOnFlakyFailure(t *testing.T) {
|
||||
tmpDir := t.TempDir() + "/"
|
||||
os.WriteFile(tmpDir+"/main.go", []byte(`package main
|
||||
func main() {
|
||||
}`), 0644)
|
||||
// Add a test that fails the first time.
|
||||
os.WriteFile(tmpDir+"/main_test.go", []byte(`package main
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
func TestFlakyFailure(t *testing.T) {
|
||||
_, err := os.Stat("foo")
|
||||
if err != nil {
|
||||
os.WriteFile("foo", []byte("hello"), 0644)
|
||||
t.Fatal("flaky failure")
|
||||
}
|
||||
}`), 0644)
|
||||
os.WriteFile(tmpDir+"/go.mod", []byte(`module example.com/test`), 0644)
|
||||
|
||||
// Run the test.
|
||||
tstr := tester{Dir: tmpDir}
|
||||
err := tstr.runTests(nil)
|
||||
if err != nil {
|
||||
t.Fatal("Should have passed with a flaky test")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user