tests/thirdparty: add skip option (#228)

Add the ability to skip third-party tests by specifying a known issue.

tests/thirdparty/config.go

@@ -93,6 +93,11 @@ type Package struct {
 
 	// Test steps. If empty, defaults to "go test ./...".
 	Test []*Step `json:"test,omitempty"`
+
+	// If the package test has a known problem, record it by setting this to a
+	// non-zero avo issue number.  If set, the package will be skipped in
+	// testing.
+	KnownIssue int `json:"known_issue,omitempty"`
 }
 
 // ID returns an identifier for the package.
@@ -101,6 +106,17 @@ func (p *Package) ID() string {
 	return strings.ReplaceAll(pkgpath, "/", "-")
 }
 
+// Skip reports whether the package test should be skipped. If skipped, a known
+// issue will be set.
+func (p *Package) Skip() bool {
+	return p.KnownIssue != 0
+}
+
+// Reason returns the reason why the test is skipped.
+func (p *Package) Reason() string {
+	return fmt.Sprintf("https://github.com/mmcloughlin/avo/issues/%d", p.KnownIssue)
+}
+
 // defaults sets or removes default field values.
 func (p *Package) defaults(set bool) {
 	for _, stage := range []struct {

tests/thirdparty/make_workflow.go

@@ -93,6 +93,9 @@ func GenerateWorkflow(pkgs thirdparty.Packages) ([]byte, error) {
 		g.Indent()
 
 		g.Linef("runs-on: ubuntu-latest")
+		if pkg.Skip() {
+			g.Linef("if: false # skip: %s", pkg.Reason())
+		}
 		g.Linef("steps:")
 		g.Indent()
 

tests/thirdparty/metadata_test.go

@@ -54,3 +54,39 @@ func TestPackagesFileMetadata(t *testing.T) {
 		t.Fatal(err)
 	}
 }
+
+func TestPackagesFileKnownIssues(t *testing.T) {
+	test.RequiresNetwork(t)
+	ctx := context.Background()
+
+	pkgs, err := LoadPackagesFile("packages.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	g := github.NewClient(github.WithTokenFromEnvironment())
+
+	for _, pkg := range pkgs {
+		// Skipped packages must refer to an open issue.
+		if !pkg.Skip() {
+			continue
+		}
+
+		if pkg.KnownIssue == 0 {
+			t.Errorf("%s: skipped package must refer to known issue", pkg.ID())
+		}
+
+		issue, err := g.Issue(ctx, "mmcloughlin", "avo", pkg.KnownIssue)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if issue.State != "open" {
+			t.Errorf("%s: known issue in %s state", pkg.ID(), issue.State)
+		}
+
+		if pkg.Reason() != issue.HTMLURL {
+			t.Errorf("%s: expected skip reason to be the issue url %s", pkg.ID(), issue.HTMLURL)
+		}
+	}
+}

tests/thirdparty/packages_test.go

@@ -35,6 +35,9 @@ func TestPackages(t *testing.T) {
 	for _, pkg := range pkgs {
 		pkg := pkg // scopelint
 		t.Run(pkg.ID(), func(t *testing.T) {
+			if pkg.Skip() {
+				t.Skipf("skip: %s", pkg.Reason())
+			}
 			dir, clean := test.TempDir(t)
 			if !*preserve {
 				defer clean()