tests: integration tests for third-party packages (#112)

Closes #103

.github/workflows/ci.yml

@@ -73,3 +73,20 @@ jobs:
       run: ./script/bootstrap
     - name: Lint
       run: ./script/lint
+
+  thirdparty:
+    strategy:
+      matrix:
+        go-version: [1.13.x]
+        platform: [ubuntu-latest]
+    runs-on: ${{ matrix.platform }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Checkout code
+      uses: actions/checkout@v1
+    - name: Run Third-Party Tests
+      working-directory: ./tests/thirdparty
+      run: go test -v -pkgs packages.json

internal/test/utils.go

@@ -46,8 +46,31 @@ func TempDir(t *testing.T) (string, func()) {
 	}
 }
 
-// gobin returns a best guess path to the "go" binary.
-func gobin() string {
+// ExecCommand executes the command, logging the command and output and failing
+// the test on error.
+func ExecCommand(t *testing.T, cmd *exec.Cmd) {
+	t.Helper()
+	t.Logf("exec: %s", cmd.Args)
+	if cmd.Dir != "" {
+		t.Logf("dir: %s", cmd.Dir)
+	}
+	b, err := cmd.CombinedOutput()
+	t.Logf("output:\n%s\n", string(b))
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Exec executes the named program with the given arguments, logging the command
+// and output and failing the test on error.
+func Exec(t *testing.T, name string, arg ...string) {
+	t.Helper()
+	cmd := exec.Command(name, arg...)
+	ExecCommand(t, cmd)
+}
+
+// GoTool returns a best guess path to the "go" binary.
+func GoTool() string {
 	var exeSuffix string
 	if runtime.GOOS == "windows" {
 		exeSuffix = ".exe"
@@ -61,14 +84,7 @@ func gobin() string {
 
 // goexec runs a "go" command and checks the output.
 func goexec(t *testing.T, arg ...string) {
-	t.Helper()
-	cmd := exec.Command(gobin(), arg...)
-	t.Logf("exec: %s", cmd.Args)
-	b, err := cmd.CombinedOutput()
-	t.Logf("output:\n%s\n", string(b))
-	if err != nil {
-		t.Fatal(err)
-	}
+	Exec(t, GoTool(), arg...)
 }
 
 // Logger builds a logger that writes to the test object.

tests/thirdparty/config.go

@@ -0,0 +1,48 @@
+// Package thirdparty executes integration tests based on third-party packages that use avo.
+package thirdparty
+
+import (
+	"encoding/json"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+// Package defines an integration test based on a third-party package using avo.
+type Package struct {
+	ImportPath string     `json:"import_path"` // package import path
+	Version    string     `json:"version"`     // git sha, tag or branch
+	Generate   [][]string `json:"generate"`    // generate commands to run
+	Dir        string     `json:"dir"`         // working directory for generate commands
+}
+
+// Name returns the package name.
+func (p Package) Name() string {
+	return filepath.Base(p.ImportPath)
+}
+
+// CloneURL returns the git clone URL.
+func (p Package) CloneURL() string {
+	return "https://" + p.ImportPath + ".git"
+}
+
+// LoadPackages loads a list of package configurations from JSON format.
+func LoadPackages(r io.Reader) ([]Package, error) {
+	var pkgs []Package
+	d := json.NewDecoder(r)
+	d.DisallowUnknownFields()
+	if err := d.Decode(&pkgs); err != nil {
+		return nil, err
+	}
+	return pkgs, nil
+}
+
+// LoadPackagesFile loads a list of package configurations from a JSON file.
+func LoadPackagesFile(filename string) ([]Package, error) {
+	f, err := os.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	return LoadPackages(f)
+}

tests/thirdparty/config_test.go

@@ -0,0 +1,51 @@
+package thirdparty
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestPackageName(t *testing.T) {
+	p := Package{ImportPath: "github.com/username/repo"}
+	if p.Name() != "repo" {
+		t.Fail()
+	}
+}
+
+func TestPackageCloneURL(t *testing.T) {
+	p := Package{ImportPath: "github.com/username/repo"}
+	if p.CloneURL() != "https://github.com/username/repo.git" {
+		t.Fail()
+	}
+}
+
+func TestLoadPackages(t *testing.T) {
+	r := strings.NewReader(`[{"unknown_field": "value"}]`)
+	_, err := LoadPackages(r)
+	if err == nil {
+		t.Fatal("expected non-nil error")
+	}
+}
+
+func TestLoadPackagesFile(t *testing.T) {
+	pkgs, err := LoadPackagesFile("packages.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, pkg := range pkgs {
+		t.Log(pkg.ImportPath)
+	}
+	if len(pkgs) == 0 {
+		t.Fatal("no packages loaded")
+	}
+}
+
+func TestLoadPackagesFileNotExist(t *testing.T) {
+	pkgs, err := LoadPackagesFile("does_not_exist")
+	if pkgs != nil {
+		t.Fatal("expected nil return")
+	}
+	if err == nil {
+		t.Fatal("expected non-nil error")
+	}
+}

tests/thirdparty/packages.json

@@ -0,0 +1,134 @@
+[
+    {
+        "import_path": "github.com/zeebo/xxh3",
+        "version": "65f423c10688c362d2a2ce6987b665c72ee7bddd",
+        "dir": "avo",
+        "generate": [
+            [
+                "go",
+                "run",
+                ".",
+                "-avx",
+                "-out",
+                "../vector_avx_amd64.s"
+            ],
+            [
+                "go",
+                "run",
+                ".",
+                "-sse",
+                "-out",
+                "../vector_sse_amd64.s"
+            ]
+        ]
+    },
+    {
+        "import_path": "github.com/dgryski/go-sip13",
+        "version": "25c5027a8c7bfa6dab4b577e53e5c9068f6e2152",
+        "generate": [
+            [
+                "go",
+                "run",
+                "_avo/asm.go",
+                "-out",
+                "sip13_amd64.s"
+            ]
+        ]
+    },
+    {
+        "import_path": "github.com/orisano/wyhash",
+        "version": "32a3f7f6ba4797e2d87dab2969cc9dd63d39cce9",
+        "generate": [
+            [
+                "go",
+                "run",
+                "avo/gen.go",
+                "-out",
+                "blocks_amd64.s",
+                "-stubs",
+                "blocks_amd64.go"
+            ]
+        ]
+    },
+    {
+        "import_path": "github.com/dgryski/go-bloomindex",
+        "version": "0902316dce158c154b958ee5cfc706c62af29a42",
+        "generate": [
+            [
+                "go",
+                "run",
+                "asm.go",
+                "-out",
+                "query_amd64.s"
+            ]
+        ]
+    },
+    {
+        "import_path": "github.com/dgryski/go-marvin32",
+        "version": "7d18f4c6ea7c24b29d1c7a670f8ae40b0812f2e3",
+        "generate": [
+            [
+                "go",
+                "run",
+                "asm.go",
+                "-out",
+                "marvin_amd64.s"
+            ]
+        ]
+    },
+    {
+        "import_path": "github.com/dgryski/go-speck",
+        "version": "5b36d4c08d8840c352a153bf37281434ad550ec0",
+        "generate": [
+            [
+                "go",
+                "run",
+                "asm.go",
+                "-out",
+                "speck_amd64.s"
+            ]
+        ]
+    },
+    {
+        "import_path": "github.com/dgryski/go-chaskey",
+        "version": "ba454392bc5ab6daae103e15147185f8f4a27dcc",
+        "generate": [
+            [
+                "go",
+                "run",
+                "asm.go",
+                "-out",
+                "core_amd64.s"
+            ]
+        ]
+    },
+    {
+        "import_path": "github.com/mkevac/gopherconrussia2019",
+        "version": "235b8b0156a20b4e078b88462e669730f99caeb1",
+        "dir": "simplesimd",
+        "generate": [
+            [
+                "go",
+                "run",
+                "asm.go",
+                "-out",
+                "simd.s",
+                "-stubs",
+                "stub.go"
+            ]
+        ]
+    },
+    {
+        "import_path": "github.com/phoreproject/bls",
+        "version": "9d5f85bf4a9badf491a1b9b27fb3344b489bd2c4",
+        "generate": [
+            [
+                "go",
+                "run",
+                "asm/asm.go",
+                "-out",
+                "primitivefuncs_amd64.s"
+            ]
+        ]
+    }
+]

tests/thirdparty/packages_test.go

@@ -0,0 +1,161 @@
+package thirdparty
+
+import (
+	"flag"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/mmcloughlin/avo/internal/test"
+)
+
+// Custom flags.
+var (
+	pkgsfilename = flag.String("pkgs", "", "packages configuration")
+	preserve     = flag.Bool("preserve", false, "preserve working directories")
+	latest       = flag.Bool("latest", false, "use latest versions of each package")
+)
+
+// TestPackages runs integration tests on all packages specified by packages
+// file given on the command line.
+func TestPackages(t *testing.T) {
+	// Load packages.
+	if *pkgsfilename == "" {
+		t.Skip("no packages specified")
+	}
+
+	pkgs, err := LoadPackagesFile(*pkgsfilename)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, pkg := range pkgs {
+		pkg := pkg // scopelint
+		t.Run(pkg.Name(), func(t *testing.T) {
+			dir, clean := test.TempDir(t)
+			if !*preserve {
+				defer clean()
+			} else {
+				t.Logf("working directory: %s", dir)
+			}
+			pt := PackageTest{
+				T:       t,
+				Package: pkg,
+				WorkDir: dir,
+				Latest:  *latest,
+			}
+			pt.Run()
+		})
+	}
+}
+
+// PackageTest executes an integration test based on a given third-party package.
+type PackageTest struct {
+	*testing.T
+	Package
+
+	WorkDir string // working directory for the test
+	Latest  bool   // use latest version of the package
+
+	repopath string // path the repo is cloned to
+}
+
+// Run the test.
+func (t *PackageTest) Run() {
+	t.checkout()
+	t.modinit()
+	t.replaceavo()
+	t.diff()
+	t.generate()
+	t.diff()
+	t.test()
+}
+
+// checkout the code at the specified version.
+func (t *PackageTest) checkout() {
+	// Clone repo.
+	dst := filepath.Join(t.WorkDir, t.Name())
+	t.git("clone", "--quiet", t.CloneURL(), dst)
+	t.repopath = dst
+
+	// Checkout specific version.
+	if t.Latest {
+		t.Log("using latest version")
+		return
+	}
+	t.git("-C", t.repopath, "checkout", "--quiet", t.Version)
+}
+
+// modinit initializes the repo as a go module if it isn't one already.
+func (t *PackageTest) modinit() {
+	// Check if module path already exists.
+	gomod := filepath.Join(t.repopath, "go.mod")
+	if _, err := os.Stat(gomod); err == nil {
+		t.Logf("already a module")
+		return
+	}
+
+	// Initialize the module.
+	t.gotool("mod", "init", t.ImportPath)
+}
+
+// replaceavo points all avo dependencies to the local version.
+func (t *PackageTest) replaceavo() {
+	// Determine the path to avo.
+	_, self, _, ok := runtime.Caller(1)
+	if !ok {
+		t.Fatal("failed to determine path to avo")
+	}
+	avodir := filepath.Join(filepath.Dir(self), "..", "..")
+
+	// Edit all go.mod files in the repo.
+	err := filepath.Walk(t.repopath, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if filepath.Base(path) != "go.mod" {
+			return nil
+		}
+		t.gotool("mod", "edit", "-replace=github.com/mmcloughlin/avo="+avodir, path)
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+// generate runs generate commands.
+func (t *PackageTest) generate() {
+	if len(t.Generate) == 0 {
+		t.Fatal("no commands specified")
+	}
+	for _, args := range t.Generate {
+		cmd := exec.Command(args[0], args[1:]...)
+		cmd.Dir = filepath.Join(t.repopath, t.Dir)
+		test.ExecCommand(t.T, cmd)
+	}
+}
+
+// diff runs git diff on the repository.
+func (t *PackageTest) diff() {
+	t.git("-C", t.repopath, "diff")
+}
+
+// test runs go test.
+func (t *PackageTest) test() {
+	t.gotool("test", "./...")
+}
+
+// git runs a git command.
+func (t *PackageTest) git(arg ...string) {
+	test.Exec(t.T, "git", arg...)
+}
+
+// gotool runs a go command.
+func (t *PackageTest) gotool(arg ...string) {
+	cmd := exec.Command(test.GoTool(), arg...)
+	cmd.Dir = t.repopath
+	test.ExecCommand(t.T, cmd)
+}