tests/thirdparty: add skip option (#228)

Add the ability to skip third-party tests by specifying a known issue.

internal/github/client.go

@@ -68,6 +68,24 @@ func (c *Client) Repository(ctx context.Context, owner, name string) (*Repositor
 	return repo, nil
 }
 
+// Issue gets information about the given Github issue.
+func (c *Client) Issue(ctx context.Context, owner, name string, number int) (*Issue, error) {
+	// Build request.
+	u := fmt.Sprintf("%s/repos/%s/%s/issues/%d", c.base, owner, name, number)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	// Execute.
+	issue := &Issue{}
+	if err := c.request(req, issue); err != nil {
+		return nil, err
+	}
+
+	return issue, nil
+}
+
 func (c *Client) request(req *http.Request, payload interface{}) (err error) {
 	// Add common headers.
 	if c.token != "" {

internal/github/client_test.go

@@ -25,3 +25,20 @@ func TestClientRepository(t *testing.T) {
 	}
 	t.Logf("repository = %s", j)
 }
+
+func TestClientIssue(t *testing.T) {
+	test.RequiresNetwork(t)
+
+	ctx := context.Background()
+	g := NewClient(WithHTTPClient(http.DefaultClient), WithTokenFromEnvironment())
+	r, err := g.Issue(ctx, "octocat", "hello-world", 42)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	j, err := json.MarshalIndent(r, "", "\t")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("issue = %s", j)
+}

internal/github/models.go

@@ -86,6 +86,70 @@ type Repository struct {
 	SubscribersCount int           `json:"subscribers_count"`
 }
 
+// Issue is a Github issue.
+type Issue struct {
+	URL               string            `json:"url"`
+	RepositoryURL     string            `json:"repository_url"`
+	LabelsURL         string            `json:"labels_url"`
+	CommentsURL       string            `json:"comments_url"`
+	EventsURL         string            `json:"events_url"`
+	HTMLURL           string            `json:"html_url"`
+	ID                int               `json:"id"`
+	NodeID            string            `json:"node_id"`
+	Number            int               `json:"number"`
+	Title             string            `json:"title"`
+	User              *User             `json:"user"`
+	Labels            []*Label          `json:"labels"`
+	State             string            `json:"state"`
+	Locked            bool              `json:"locked"`
+	Assignee          *User             `json:"assignee"`
+	Assignees         []*User           `json:"assignees"`
+	Comments          int               `json:"comments"`
+	CreatedAt         time.Time         `json:"created_at"`
+	UpdatedAt         time.Time         `json:"updated_at"`
+	ClosedAt          time.Time         `json:"closed_at"`
+	AuthorAssociation string            `json:"author_association"`
+	ActiveLockReason  string            `json:"active_lock_reason"`
+	PullRequestLinks  *PullRequestLinks `json:"pull_request"`
+	Body              string            `json:"body"`
+	ClosedBy          *User             `json:"closed_by"`
+	Reactions         *Reactions        `json:"reactions"`
+	TimelineURL       string            `json:"timeline_url"`
+}
+
+// Label is a Github label on an issue or PR.
+type Label struct {
+	ID          int    `json:"id"`
+	NodeID      string `json:"node_id"`
+	URL         string `json:"url"`
+	Name        string `json:"name"`
+	Color       string `json:"color"`
+	Default     bool   `json:"default"`
+	Description string `json:"description"`
+}
+
+// Reactions summarizes Github reactions.
+type Reactions struct {
+	URL        string `json:"url"`
+	TotalCount int    `json:"total_count"`
+	PlusOne    int    `json:"+1"`
+	MinusOne   int    `json:"-1"`
+	Laugh      int    `json:"laugh"`
+	Hooray     int    `json:"hooray"`
+	Confused   int    `json:"confused"`
+	Heart      int    `json:"heart"`
+	Rocket     int    `json:"rocket"`
+	Eyes       int    `json:"eyes"`
+}
+
+// PullRequestLinks are attached to an Issue object when it represents a PR.
+type PullRequestLinks struct {
+	URL      string `json:"url"`
+	HTMLURL  string `json:"html_url"`
+	DiffURL  string `json:"diff_url"`
+	PatchURL string `json:"patch_url"`
+}
+
 // User is a Github user.
 type User struct {
 	Login             string `json:"login"`

tests/thirdparty/config.go

@@ -93,6 +93,11 @@ type Package struct {
 
 	// Test steps. If empty, defaults to "go test ./...".
 	Test []*Step `json:"test,omitempty"`
+
+	// If the package test has a known problem, record it by setting this to a
+	// non-zero avo issue number.  If set, the package will be skipped in
+	// testing.
+	KnownIssue int `json:"known_issue,omitempty"`
 }
 
 // ID returns an identifier for the package.
@@ -101,6 +106,17 @@ func (p *Package) ID() string {
 	return strings.ReplaceAll(pkgpath, "/", "-")
 }
 
+// Skip reports whether the package test should be skipped. If skipped, a known
+// issue will be set.
+func (p *Package) Skip() bool {
+	return p.KnownIssue != 0
+}
+
+// Reason returns the reason why the test is skipped.
+func (p *Package) Reason() string {
+	return fmt.Sprintf("https://github.com/mmcloughlin/avo/issues/%d", p.KnownIssue)
+}
+
 // defaults sets or removes default field values.
 func (p *Package) defaults(set bool) {
 	for _, stage := range []struct {

tests/thirdparty/make_workflow.go

@@ -93,6 +93,9 @@ func GenerateWorkflow(pkgs thirdparty.Packages) ([]byte, error) {
 		g.Indent()
 
 		g.Linef("runs-on: ubuntu-latest")
+		if pkg.Skip() {
+			g.Linef("if: false # skip: %s", pkg.Reason())
+		}
 		g.Linef("steps:")
 		g.Indent()
 

tests/thirdparty/metadata_test.go

@@ -54,3 +54,39 @@ func TestPackagesFileMetadata(t *testing.T) {
 		t.Fatal(err)
 	}
 }
+
+func TestPackagesFileKnownIssues(t *testing.T) {
+	test.RequiresNetwork(t)
+	ctx := context.Background()
+
+	pkgs, err := LoadPackagesFile("packages.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	g := github.NewClient(github.WithTokenFromEnvironment())
+
+	for _, pkg := range pkgs {
+		// Skipped packages must refer to an open issue.
+		if !pkg.Skip() {
+			continue
+		}
+
+		if pkg.KnownIssue == 0 {
+			t.Errorf("%s: skipped package must refer to known issue", pkg.ID())
+		}
+
+		issue, err := g.Issue(ctx, "mmcloughlin", "avo", pkg.KnownIssue)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if issue.State != "open" {
+			t.Errorf("%s: known issue in %s state", pkg.ID(), issue.State)
+		}
+
+		if pkg.Reason() != issue.HTMLURL {
+			t.Errorf("%s: expected skip reason to be the issue url %s", pkg.ID(), issue.HTMLURL)
+		}
+	}
+}

tests/thirdparty/packages_test.go

@@ -35,6 +35,9 @@ func TestPackages(t *testing.T) {
 	for _, pkg := range pkgs {
 		pkg := pkg // scopelint
 		t.Run(pkg.ID(), func(t *testing.T) {
+			if pkg.Skip() {
+				t.Skipf("skip: %s", pkg.Reason())
+			}
 			dir, clean := test.TempDir(t)
 			if !*preserve {
 				defer clean()