tests/thirdparty: fix and run as github workflow (#198)

As part of fixing failing third-party tests, this PR significantly
rearchitects their specification and execution.

Third-party tests are now specified in a much more flexible format allowing
more customization on a per-package level. In addition, third-party tests are
now used to auto-generate a Github Actions workflow to perform the tests in
parallel. This not only gives faster feedback on PRs, but will also allow us
to more quickly narrow down on which packages are failing. An additional
workflow also confirms that local execution of third-party tests is consistent
with the Github Actions version. This workflow only runs when tests/thirdparty
itself is changed.

tests/thirdparty/config_test.go

@@ -5,33 +5,85 @@ import (
 	"testing"
 )
 
-func TestPackageName(t *testing.T) {
-	p := Package{ImportPath: "github.com/username/repo"}
-	if p.Name() != "repo" {
-		t.Fail()
+func TestValidateErrors(t *testing.T) {
+	cases := []struct {
+		Name           string
+		Item           interface{ Validate() error }
+		ErrorSubstring string
+	}{
+		{
+			Name:           "step_missing_name",
+			Item:           &Step{},
+			ErrorSubstring: "missing name",
+		},
+		{
+			Name: "step_missing_commands",
+			Item: &Step{
+				Name: "Setup",
+			},
+			ErrorSubstring: "missing commands",
+		},
+		{
+			Name: "package_missing_version",
+			Item: &Package{
+				Repository: GithubRepository{Owner: "octocat", Name: "hello-world"},
+			},
+			ErrorSubstring: "missing version",
+		},
+		{
+			Name: "package_missing_module",
+			Item: &Package{
+				Repository: GithubRepository{Owner: "octocat", Name: "hello-world"},
+				Version:    "v1.0.1",
+			},
+			ErrorSubstring: "missing module",
+		},
+		{
+			Name: "package_no_generate_commands",
+			Item: &Package{
+				Repository: GithubRepository{Owner: "octocat", Name: "hello-world"},
+				Version:    "v1.0.1",
+				Module:     "avo/go.mod",
+			},
+			ErrorSubstring: "no generate commands",
+		},
+		{
+			Name: "package_invalid_generate_commands",
+			Item: &Package{
+				Repository: GithubRepository{Owner: "octocat", Name: "hello-world"},
+				Version:    "v1.0.1",
+				Module:     "avo/go.mod",
+				Generate: []*Step{
+					{},
+				},
+			},
+			ErrorSubstring: "generate step: missing name",
+		},
+		{
+			Name: "packages_invalid_package",
+			Item: Packages{
+				{
+					Repository: GithubRepository{Owner: "octocat", Name: "hello-world"},
+				},
+			},
+			ErrorSubstring: "missing version",
+		},
+	}
+	for _, c := range cases {
+		c := c // scopelint
+		t.Run(c.Name, func(t *testing.T) {
+			err := c.Item.Validate()
+			if err == nil {
+				t.Fatal("expected error; got nil")
+			}
+			if !strings.Contains(err.Error(), c.ErrorSubstring) {
+				t.Fatalf("expected error message to contain %q; got %q", c.ErrorSubstring, err)
+			}
+		})
 	}
 }
 
-func TestPackageCloneURL(t *testing.T) {
-	p := Package{ImportPath: "github.com/username/repo"}
-	if p.CloneURL() != "https://github.com/username/repo.git" {
-		t.Fail()
-	}
-}
-
-func TestPackagesTestPath(t *testing.T) {
-	p := Package{}
-	if p.TestPath() != "./..." {
-		t.Fail()
-	}
-
-	p.Test = "./sub"
-	if p.TestPath() != "./sub" {
-		t.Fail()
-	}
-}
-
-func TestLoadPackages(t *testing.T) {
+func TestLoadPackagesBad(t *testing.T) {
 	r := strings.NewReader(`[{"unknown_field": "value"}]`)
 	_, err := LoadPackages(r)
 	if err == nil {
@@ -39,19 +91,6 @@ func TestLoadPackages(t *testing.T) {
 	}
 }
 
-func TestLoadPackagesFile(t *testing.T) {
-	pkgs, err := LoadPackagesFile("packages.json")
-	if err != nil {
-		t.Fatal(err)
-	}
-	for _, pkg := range pkgs {
-		t.Log(pkg.ImportPath)
-	}
-	if len(pkgs) == 0 {
-		t.Fatal("no packages loaded")
-	}
-}
-
 func TestLoadPackagesFileNotExist(t *testing.T) {
 	pkgs, err := LoadPackagesFile("does_not_exist")
 	if pkgs != nil {
@@ -61,3 +100,37 @@ func TestLoadPackagesFileNotExist(t *testing.T) {
 		t.Fatal("expected non-nil error")
 	}
 }
+
+func TestPackagesFileValid(t *testing.T) {
+	pkgs, err := LoadPackagesFile("packages.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, pkg := range pkgs {
+		t.Logf("read: %s", pkg.ID())
+	}
+	if len(pkgs) == 0 {
+		t.Fatal("no packages loaded")
+	}
+	if err := pkgs.Validate(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestPackagesFileStepsValid(t *testing.T) {
+	pkgs, err := LoadPackagesFile("packages.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	c := &Context{
+		AvoDirectory:        "avo",
+		RepositoryDirectory: "repo",
+	}
+	for _, pkg := range pkgs {
+		for _, s := range pkg.Steps(c) {
+			if err := s.Validate(); err != nil {
+				t.Errorf("package %s: %s", pkg.ID(), err)
+			}
+		}
+	}
+}