avo/tests/thirdparty/config.go

// Package thirdparty executes integration tests based on third-party projects that use avo.
package thirdparty

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"time"
)

// GithubRepository specifies a repository on github.
type GithubRepository struct {
	Owner string `json:"owner"`
	Name  string `json:"name"`
}

func (r GithubRepository) String() string {
	return path.Join(r.Owner, r.Name)
}

// URL returns the Github repository URL.
func (r GithubRepository) URL() string {
	return fmt.Sprintf("https://github.com/%s", r)
}

// CloneURL returns the git clone URL.
func (r GithubRepository) CloneURL() string {
	return fmt.Sprintf("https://github.com/%s.git", r)
}

// Metadata about the repository.
type Metadata struct {
	// Repository description.
	Description string `json:"description,omitempty"`

	// Homepage URL. Not the same as the Github page.
	Homepage string `json:"homepage,omitempty"`

	// Number of Github stars.
	Stars int `json:"stars,omitempty"`
}

// Project defines an integration test based on a third-party project using avo.
type Project struct {
	// Repository for the project. At the moment, all projects are available on
	// github.
	Repository GithubRepository `json:"repository"`

	// Repository metadata.
	Metadata Metadata `json:"metadata"`

	// Default git branch. This is used when testing against the latest version.
	DefaultBranch string `json:"default_branch,omitempty"`

	// Version as a git sha, tag or branch.
	Version string `json:"version"`

	// If the project test has a known problem, record it by setting this to a
	// non-zero avo issue number.  If set, the project will be skipped in
	// testing.
	KnownIssue int `json:"known_issue,omitempty"`

	// Packages within the project to test.
	Packages []*Package `json:"packages"`
}

func (p *Project) defaults(set bool) {
	for _, pkg := range p.Packages {
		pkg.defaults(set)
	}
}

// Validate project definition.
func (p *Project) Validate() error {
	if p.DefaultBranch == "" {
		return errors.New("missing default branch")
	}
	if p.Version == "" {
		return errors.New("missing version")
	}
	if len(p.Packages) == 0 {
		return errors.New("missing packages")
	}
	for _, pkg := range p.Packages {
		if err := pkg.Validate(); err != nil {
			return fmt.Errorf("package %s: %w", pkg.Name(), err)
		}
	}
	return nil
}

// ID returns an identifier for the project.
func (p *Project) ID() string {
	return strings.ReplaceAll(p.Repository.String(), "/", "-")
}

// Skip reports whether the project test should be skipped. If skipped, a known
// issue will be set.
func (p *Project) Skip() bool {
	return p.KnownIssue != 0
}

// Reason returns the reason why the test is skipped.
func (p *Project) Reason() string {
	return fmt.Sprintf("https://github.com/mmcloughlin/avo/issues/%d", p.KnownIssue)
}

// Step represents a set of commands to run as part of the testing plan for a
// third-party package.
type Step struct {
	Name             string   `json:"name,omitempty"`
	WorkingDirectory string   `json:"dir,omitempty"`
	Commands         []string `json:"commands"`
}

// Validate step parameters.
func (s *Step) Validate() error {
	if s.Name == "" {
		return errors.New("missing name")
	}
	if len(s.Commands) == 0 {
		return errors.New("missing commands")
	}
	return nil
}

// Package defines an integration test for a package within a project.
type Package struct {
	// Sub-package within the project under test. Used as the root directory for
	// the test unless explicitly overridden.
	SubPackage string `json:"pkg,omitempty"`

	// Path to the module file for the avo generator package. This is necessary
	// so the integration test can insert replace directives to point at the avo
	// version under test.
	Module string `json:"module"`

	// Root directory for all steps in the test. All file path references will
	// be relative to this directory. If empty, the sub-package directory is
	// used, or otherwise the root of the repository.
	Root string `json:"root,omitempty"`

	// Setup steps. These run prior to the insertion of avo replace directives,
	// therefore should be used if it's necessary to initialize new go modules
	// within the repository.
	Setup []*Step `json:"setup,omitempty"`

	// Steps to run the avo code generator.
	Generate []*Step `json:"generate"`

	// Test steps. If empty, defaults to "go test ./...".
	Test []*Step `json:"test,omitempty"`
}

// defaults sets or removes default field values.
func (p *Package) defaults(set bool) {
	for _, stage := range []struct {
		Steps       []*Step
		DefaultName string
	}{
		{p.Setup, "Setup"},
		{p.Generate, "Generate"},
		{p.Test, "Test"},
	} {
		if len(stage.Steps) == 1 {
			stage.Steps[0].Name = applydefault(set, stage.Steps[0].Name, stage.DefaultName)
		}
	}
}

func applydefault(set bool, s, def string) string {
	switch {
	case set && s == "":
		return def
	case !set && s == def:
		return ""
	default:
		return s
	}
}

// Validate package definition.
func (p *Package) Validate() error {
	if p.Module == "" {
		return errors.New("missing module")
	}
	if len(p.Generate) == 0 {
		return errors.New("no generate commands")
	}

	stages := map[string][]*Step{
		"setup":    p.Setup,
		"generate": p.Generate,
		"test":     p.Test,
	}
	for name, steps := range stages {
		for _, s := range steps {
			if err := s.Validate(); err != nil {
				return fmt.Errorf("%s step: %w", name, err)
			}
		}
	}

	return nil
}

// Name of the package.
func (p *Package) Name() string {
	if p.IsRoot() {
		return "root"
	}
	return p.SubPackage
}

// IsRoot reports whether the package is the root of the containing project.
func (p *Package) IsRoot() bool {
	return p.SubPackage == ""
}

// WorkingDirectory returns the base directory for all steps in the test.
func (p *Package) WorkingDirectory() string {
	if p.Root != "" {
		return p.Root
	}
	if p.SubPackage != "" {
		return p.SubPackage
	}
	return ""
}

// Context specifies execution environment parameters for a third-party test.
type Context struct {
	// Path to the avo version under test.
	AvoDirectory string

	// Path to the checked out third-party repository.
	RepositoryDirectory string
}

// Steps generates the list of steps required to execute the integration test
// for this package. Context specifies execution environment parameters.
func (p *Package) Steps(c *Context) []*Step {
	var steps []*Step

	// Optional setup.
	steps = append(steps, p.Setup...)

	// Replace avo dependency.
	const invalid = "v0.0.0-00010101000000-000000000000"
	moddir := filepath.Dir(p.Module)
	modfile := filepath.Base(p.Module)
	steps = append(steps, &Step{
		Name:             "Avo Module Replacement",
		WorkingDirectory: moddir,
		Commands: []string{
			"go mod edit -modfile=" + modfile + " -require=github.com/mmcloughlin/avo@" + invalid,
			"go mod edit -modfile=" + modfile + " -replace=github.com/mmcloughlin/avo=" + c.AvoDirectory,
			"go mod tidy -modfile=" + modfile,
		},
	})

	// Run generation.
	steps = append(steps, p.Generate...)

	// Display changes.
	steps = append(steps, &Step{
		Name:     "Diff",
		Commands: []string{"git diff"},
	})

	// Tests.
	if len(p.Test) > 0 {
		steps = append(steps, p.Test...)
	} else {
		steps = append(steps, &Step{
			Name: "Test",
			Commands: []string{
				"go test ./...",
			},
		})
	}

	// Prepend sub-directory to every step.
	if dir := p.WorkingDirectory(); dir != "" {
		for _, s := range steps {
			s.WorkingDirectory = filepath.Join(dir, s.WorkingDirectory)
		}
	}

	return steps
}

// Test case for a given package within a project.
type Test struct {
	Project *Project
	Package *Package
}

// ID returns an identifier for the test case.
func (t *Test) ID() string {
	pkgpath := path.Join(t.Project.Repository.String(), t.Package.SubPackage)
	return strings.ReplaceAll(pkgpath, "/", "-")
}

// Projects is a collection of third-party integration tests.
type Projects []*Project

func (p Projects) defaults(set bool) {
	for _, prj := range p {
		prj.defaults(set)
	}
}

// Validate the project collection.
func (p Projects) Validate() error {
	// Projects are valid.
	for _, prj := range p {
		if err := prj.Validate(); err != nil {
			return fmt.Errorf("project %s: %w", prj.ID(), err)
		}
	}

	// No duplicate project IDs.
	pid := map[string]bool{}
	for _, prj := range p {
		id := prj.ID()
		if pid[id] {
			return fmt.Errorf("duplicate project %q", id)
		}
		pid[id] = true
	}

	// No duplicate test IDs.
	tid := map[string]bool{}
	for _, t := range p.Tests() {
		id := t.ID()
		if tid[id] {
			return fmt.Errorf("duplicate test %q", id)
		}
		tid[id] = true
	}

	return nil
}

// Tests returns all test cases for the projects collection.
func (p Projects) Tests() []*Test {
	var ts []*Test
	for _, prj := range p {
		for _, pkg := range prj.Packages {
			ts = append(ts, &Test{
				Project: prj,
				Package: pkg,
			})
		}
	}
	return ts
}

// Ranked returns a copy of the projects list ranked in desending order of
// popularity.
func (p Projects) Ranked() Projects {
	ranked := append(Projects(nil), p...)
	sort.SliceStable(ranked, func(i, j int) bool {
		return ranked[i].Metadata.Stars > ranked[j].Metadata.Stars
	})
	return ranked
}

// Top returns the top n most popular projects.
func (p Projects) Top(n int) Projects {
	top := p.Ranked()
	if len(top) > n {
		top = top[:n]
	}
	return top
}

// Suite defines a third-party test suite.
type Suite struct {
	// Projects to test.
	Projects Projects `json:"projects"`

	// Time of the last update to project metadata.
	MetadataLastUpdate time.Time `json:"metadata_last_update"`
}

func (s *Suite) defaults(set bool) {
	s.Projects.defaults(set)
}

// Validate the test suite.
func (s *Suite) Validate() error {
	if s.MetadataLastUpdate.IsZero() {
		return errors.New("empty metadata update time")
	}

	if s.MetadataLastUpdate.Location() != time.UTC {
		return errors.New("metadata update time not in UTC")
	}

	if err := s.Projects.Validate(); err != nil {
		return err
	}

	return nil
}

// LoadSuite loads a test suite from JSON format.
func LoadSuite(r io.Reader) (*Suite, error) {
	var s *Suite
	d := json.NewDecoder(r)
	d.DisallowUnknownFields()
	if err := d.Decode(&s); err != nil {
		return nil, err
	}
	s.defaults(true)
	return s, nil
}

// LoadSuiteFile loads a test suite from a JSON file.
func LoadSuiteFile(filename string) (*Suite, error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	return LoadSuite(f)
}

// StoreSuite writes a test suite in JSON format.
func StoreSuite(w io.Writer, s *Suite) error {
	e := json.NewEncoder(w)
	e.SetIndent("", "    ")
	s.defaults(false)
	err := e.Encode(s)
	s.defaults(true)
	return err
}

// StoreSuiteFile writes a test suite to a JSON file.
func StoreSuiteFile(filename string, s *Suite) error {
	f, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer f.Close()
	return StoreSuite(f, s)
}