From 205fc6a3d76b01529817bc64d037340b558d2c05 Mon Sep 17 00:00:00 2001 From: Michael McLoughlin Date: Mon, 6 Jan 2020 22:56:38 -0500 Subject: [PATCH] tests: integration tests for third-party packages (#112) Closes #103 --- .github/workflows/ci.yml | 17 ++++ internal/test/utils.go | 36 +++++-- tests/thirdparty/config.go | 48 +++++++++ tests/thirdparty/config_test.go | 51 ++++++++++ tests/thirdparty/packages.json | 134 +++++++++++++++++++++++++ tests/thirdparty/packages_test.go | 161 ++++++++++++++++++++++++++++++ 6 files changed, 437 insertions(+), 10 deletions(-) create mode 100644 tests/thirdparty/config.go create mode 100644 tests/thirdparty/config_test.go create mode 100644 tests/thirdparty/packages.json create mode 100644 tests/thirdparty/packages_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c989c46..f016b00 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,3 +73,20 @@ jobs: run: ./script/bootstrap - name: Lint run: ./script/lint + + thirdparty: + strategy: + matrix: + go-version: [1.13.x] + platform: [ubuntu-latest] + runs-on: ${{ matrix.platform }} + steps: + - name: Install Go + uses: actions/setup-go@v1 + with: + go-version: ${{ matrix.go-version }} + - name: Checkout code + uses: actions/checkout@v1 + - name: Run Third-Party Tests + working-directory: ./tests/thirdparty + run: go test -v -pkgs packages.json diff --git a/internal/test/utils.go b/internal/test/utils.go index 9fc6651..8b8cc42 100644 --- a/internal/test/utils.go +++ b/internal/test/utils.go @@ -46,8 +46,31 @@ func TempDir(t *testing.T) (string, func()) { } } -// gobin returns a best guess path to the "go" binary. -func gobin() string { +// ExecCommand executes the command, logging the command and output and failing +// the test on error. +func ExecCommand(t *testing.T, cmd *exec.Cmd) { + t.Helper() + t.Logf("exec: %s", cmd.Args) + if cmd.Dir != "" { + t.Logf("dir: %s", cmd.Dir) + } + b, err := cmd.CombinedOutput() + t.Logf("output:\n%s\n", string(b)) + if err != nil { + t.Fatal(err) + } +} + +// Exec executes the named program with the given arguments, logging the command +// and output and failing the test on error. +func Exec(t *testing.T, name string, arg ...string) { + t.Helper() + cmd := exec.Command(name, arg...) + ExecCommand(t, cmd) +} + +// GoTool returns a best guess path to the "go" binary. +func GoTool() string { var exeSuffix string if runtime.GOOS == "windows" { exeSuffix = ".exe" @@ -61,14 +84,7 @@ func gobin() string { // goexec runs a "go" command and checks the output. func goexec(t *testing.T, arg ...string) { - t.Helper() - cmd := exec.Command(gobin(), arg...) - t.Logf("exec: %s", cmd.Args) - b, err := cmd.CombinedOutput() - t.Logf("output:\n%s\n", string(b)) - if err != nil { - t.Fatal(err) - } + Exec(t, GoTool(), arg...) } // Logger builds a logger that writes to the test object. diff --git a/tests/thirdparty/config.go b/tests/thirdparty/config.go new file mode 100644 index 0000000..113193b --- /dev/null +++ b/tests/thirdparty/config.go @@ -0,0 +1,48 @@ +// Package thirdparty executes integration tests based on third-party packages that use avo. +package thirdparty + +import ( + "encoding/json" + "io" + "os" + "path/filepath" +) + +// Package defines an integration test based on a third-party package using avo. +type Package struct { + ImportPath string `json:"import_path"` // package import path + Version string `json:"version"` // git sha, tag or branch + Generate [][]string `json:"generate"` // generate commands to run + Dir string `json:"dir"` // working directory for generate commands +} + +// Name returns the package name. +func (p Package) Name() string { + return filepath.Base(p.ImportPath) +} + +// CloneURL returns the git clone URL. +func (p Package) CloneURL() string { + return "https://" + p.ImportPath + ".git" +} + +// LoadPackages loads a list of package configurations from JSON format. +func LoadPackages(r io.Reader) ([]Package, error) { + var pkgs []Package + d := json.NewDecoder(r) + d.DisallowUnknownFields() + if err := d.Decode(&pkgs); err != nil { + return nil, err + } + return pkgs, nil +} + +// LoadPackagesFile loads a list of package configurations from a JSON file. +func LoadPackagesFile(filename string) ([]Package, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + return LoadPackages(f) +} diff --git a/tests/thirdparty/config_test.go b/tests/thirdparty/config_test.go new file mode 100644 index 0000000..50faef0 --- /dev/null +++ b/tests/thirdparty/config_test.go @@ -0,0 +1,51 @@ +package thirdparty + +import ( + "strings" + "testing" +) + +func TestPackageName(t *testing.T) { + p := Package{ImportPath: "github.com/username/repo"} + if p.Name() != "repo" { + t.Fail() + } +} + +func TestPackageCloneURL(t *testing.T) { + p := Package{ImportPath: "github.com/username/repo"} + if p.CloneURL() != "https://github.com/username/repo.git" { + t.Fail() + } +} + +func TestLoadPackages(t *testing.T) { + r := strings.NewReader(`[{"unknown_field": "value"}]`) + _, err := LoadPackages(r) + if err == nil { + t.Fatal("expected non-nil error") + } +} + +func TestLoadPackagesFile(t *testing.T) { + pkgs, err := LoadPackagesFile("packages.json") + if err != nil { + t.Fatal(err) + } + for _, pkg := range pkgs { + t.Log(pkg.ImportPath) + } + if len(pkgs) == 0 { + t.Fatal("no packages loaded") + } +} + +func TestLoadPackagesFileNotExist(t *testing.T) { + pkgs, err := LoadPackagesFile("does_not_exist") + if pkgs != nil { + t.Fatal("expected nil return") + } + if err == nil { + t.Fatal("expected non-nil error") + } +} diff --git a/tests/thirdparty/packages.json b/tests/thirdparty/packages.json new file mode 100644 index 0000000..3c91f2b --- /dev/null +++ b/tests/thirdparty/packages.json @@ -0,0 +1,134 @@ +[ + { + "import_path": "github.com/zeebo/xxh3", + "version": "65f423c10688c362d2a2ce6987b665c72ee7bddd", + "dir": "avo", + "generate": [ + [ + "go", + "run", + ".", + "-avx", + "-out", + "../vector_avx_amd64.s" + ], + [ + "go", + "run", + ".", + "-sse", + "-out", + "../vector_sse_amd64.s" + ] + ] + }, + { + "import_path": "github.com/dgryski/go-sip13", + "version": "25c5027a8c7bfa6dab4b577e53e5c9068f6e2152", + "generate": [ + [ + "go", + "run", + "_avo/asm.go", + "-out", + "sip13_amd64.s" + ] + ] + }, + { + "import_path": "github.com/orisano/wyhash", + "version": "32a3f7f6ba4797e2d87dab2969cc9dd63d39cce9", + "generate": [ + [ + "go", + "run", + "avo/gen.go", + "-out", + "blocks_amd64.s", + "-stubs", + "blocks_amd64.go" + ] + ] + }, + { + "import_path": "github.com/dgryski/go-bloomindex", + "version": "0902316dce158c154b958ee5cfc706c62af29a42", + "generate": [ + [ + "go", + "run", + "asm.go", + "-out", + "query_amd64.s" + ] + ] + }, + { + "import_path": "github.com/dgryski/go-marvin32", + "version": "7d18f4c6ea7c24b29d1c7a670f8ae40b0812f2e3", + "generate": [ + [ + "go", + "run", + "asm.go", + "-out", + "marvin_amd64.s" + ] + ] + }, + { + "import_path": "github.com/dgryski/go-speck", + "version": "5b36d4c08d8840c352a153bf37281434ad550ec0", + "generate": [ + [ + "go", + "run", + "asm.go", + "-out", + "speck_amd64.s" + ] + ] + }, + { + "import_path": "github.com/dgryski/go-chaskey", + "version": "ba454392bc5ab6daae103e15147185f8f4a27dcc", + "generate": [ + [ + "go", + "run", + "asm.go", + "-out", + "core_amd64.s" + ] + ] + }, + { + "import_path": "github.com/mkevac/gopherconrussia2019", + "version": "235b8b0156a20b4e078b88462e669730f99caeb1", + "dir": "simplesimd", + "generate": [ + [ + "go", + "run", + "asm.go", + "-out", + "simd.s", + "-stubs", + "stub.go" + ] + ] + }, + { + "import_path": "github.com/phoreproject/bls", + "version": "9d5f85bf4a9badf491a1b9b27fb3344b489bd2c4", + "generate": [ + [ + "go", + "run", + "asm/asm.go", + "-out", + "primitivefuncs_amd64.s" + ] + ] + } +] diff --git a/tests/thirdparty/packages_test.go b/tests/thirdparty/packages_test.go new file mode 100644 index 0000000..42118d6 --- /dev/null +++ b/tests/thirdparty/packages_test.go @@ -0,0 +1,161 @@ +package thirdparty + +import ( + "flag" + "os" + "os/exec" + "path/filepath" + "runtime" + "testing" + + "github.com/mmcloughlin/avo/internal/test" +) + +// Custom flags. +var ( + pkgsfilename = flag.String("pkgs", "", "packages configuration") + preserve = flag.Bool("preserve", false, "preserve working directories") + latest = flag.Bool("latest", false, "use latest versions of each package") +) + +// TestPackages runs integration tests on all packages specified by packages +// file given on the command line. +func TestPackages(t *testing.T) { + // Load packages. + if *pkgsfilename == "" { + t.Skip("no packages specified") + } + + pkgs, err := LoadPackagesFile(*pkgsfilename) + if err != nil { + t.Fatal(err) + } + + for _, pkg := range pkgs { + pkg := pkg // scopelint + t.Run(pkg.Name(), func(t *testing.T) { + dir, clean := test.TempDir(t) + if !*preserve { + defer clean() + } else { + t.Logf("working directory: %s", dir) + } + pt := PackageTest{ + T: t, + Package: pkg, + WorkDir: dir, + Latest: *latest, + } + pt.Run() + }) + } +} + +// PackageTest executes an integration test based on a given third-party package. +type PackageTest struct { + *testing.T + Package + + WorkDir string // working directory for the test + Latest bool // use latest version of the package + + repopath string // path the repo is cloned to +} + +// Run the test. +func (t *PackageTest) Run() { + t.checkout() + t.modinit() + t.replaceavo() + t.diff() + t.generate() + t.diff() + t.test() +} + +// checkout the code at the specified version. +func (t *PackageTest) checkout() { + // Clone repo. + dst := filepath.Join(t.WorkDir, t.Name()) + t.git("clone", "--quiet", t.CloneURL(), dst) + t.repopath = dst + + // Checkout specific version. + if t.Latest { + t.Log("using latest version") + return + } + t.git("-C", t.repopath, "checkout", "--quiet", t.Version) +} + +// modinit initializes the repo as a go module if it isn't one already. +func (t *PackageTest) modinit() { + // Check if module path already exists. + gomod := filepath.Join(t.repopath, "go.mod") + if _, err := os.Stat(gomod); err == nil { + t.Logf("already a module") + return + } + + // Initialize the module. + t.gotool("mod", "init", t.ImportPath) +} + +// replaceavo points all avo dependencies to the local version. +func (t *PackageTest) replaceavo() { + // Determine the path to avo. + _, self, _, ok := runtime.Caller(1) + if !ok { + t.Fatal("failed to determine path to avo") + } + avodir := filepath.Join(filepath.Dir(self), "..", "..") + + // Edit all go.mod files in the repo. + err := filepath.Walk(t.repopath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if filepath.Base(path) != "go.mod" { + return nil + } + t.gotool("mod", "edit", "-replace=github.com/mmcloughlin/avo="+avodir, path) + return nil + }) + if err != nil { + t.Fatal(err) + } +} + +// generate runs generate commands. +func (t *PackageTest) generate() { + if len(t.Generate) == 0 { + t.Fatal("no commands specified") + } + for _, args := range t.Generate { + cmd := exec.Command(args[0], args[1:]...) + cmd.Dir = filepath.Join(t.repopath, t.Dir) + test.ExecCommand(t.T, cmd) + } +} + +// diff runs git diff on the repository. +func (t *PackageTest) diff() { + t.git("-C", t.repopath, "diff") +} + +// test runs go test. +func (t *PackageTest) test() { + t.gotool("test", "./...") +} + +// git runs a git command. +func (t *PackageTest) git(arg ...string) { + test.Exec(t.T, "git", arg...) +} + +// gotool runs a go command. +func (t *PackageTest) gotool(arg ...string) { + cmd := exec.Command(test.GoTool(), arg...) + cmd.Dir = t.repopath + test.ExecCommand(t.T, cmd) +}