code/avo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tests/thirdparty: package metadata (#223)

Browse Source 
Fetches third-party package metadata from Github.
This commit is contained in:
Michael McLoughlin
2021-11-07 16:13:33 -08:00
committed by GitHub
parent afe2d539b8
commit f355d27b13
9 changed files with 544 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/thirdparty.yml vendored
View File

@@ -31,4 +31,4 @@ jobs:
persist-credentials: false
- name: Run Third-Party Tests
working-directory: tests/thirdparty
run: go test -pkgs packages.json
run: go test -net -pkgs packages.json

107
internal/github/client.go Normal file
View File

@@ -0,0 +1,107 @@
// Package github provides a client for the Github REST API.
package github
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"os"
)
// Client for the Github REST API.
type Client struct {
client *http.Client
base string
token string
}
// Option configures a Github client.
type Option func(*Client)
// WithHTTPClient configures the HTTP client that should be used for Github API
// requests.
func WithHTTPClient(h *http.Client) Option {
return func(c *Client) { c.client = h }
}
// WithToken configures a Client with an authentication token for Github API
// requests.
func WithToken(token string) Option {
return func(c *Client) { c.token = token }
}
// WithTokenFromEnvironment configures a Client using the GITHUB_TOKEN
// environment variable.
func WithTokenFromEnvironment() Option {
return WithToken(os.Getenv("GITHUB_TOKEN"))
}
// NewClient initializes a client using the given HTTP client.
func NewClient(opts ...Option) *Client {
c := &Client{
client: http.DefaultClient,
base: "https://api.github.com",
}
for _, opt := range opts {
opt(c)
}
return c
}
// Repository gets information about the given Github repository.
func (c *Client) Repository(ctx context.Context, owner, name string) (*Repository, error) {
// Build request.
u := c.base + "/repos/" + owner + "/" + name
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
if err != nil {
return nil, err
}
// Execute.
repo := &Repository{}
if err := c.request(req, repo); err != nil {
return nil, err
}
return repo, nil
}
func (c *Client) request(req *http.Request, payload interface{}) (err error) {
// Add common headers.
if c.token != "" {
req.Header.Set("Authorization", "Bearer "+c.token)
}
req.Header.Set("Accept", "application/vnd.github.v3+json")
// Execute the request.
res, err := c.client.Do(req)
if err != nil {
return err
}
defer func() {
if errc := res.Body.Close(); errc != nil && err == nil {
err = errc
}
}()
// Check status.
if res.StatusCode != http.StatusOK {
return fmt.Errorf("http status %d: %s", res.StatusCode, http.StatusText(res.StatusCode))
}
// Parse response body.
d := json.NewDecoder(res.Body)
if err := d.Decode(payload); err != nil {
return err
}
// Should not have trailing data.
if d.More() {
return errors.New("unexpected extra data after JSON")
}
return nil
}

27
internal/github/client_test.go Normal file
View File

@@ -0,0 +1,27 @@
package github
import (
"context"
"encoding/json"
"net/http"
"testing"
"github.com/mmcloughlin/avo/internal/test"
)
func TestClientRepository(t *testing.T) {
test.RequiresNetwork(t)
ctx := context.Background()
g := NewClient(WithHTTPClient(http.DefaultClient), WithTokenFromEnvironment())
r, err := g.Repository(ctx, "golang", "go")
if err != nil {
t.Fatal(err)
}
j, err := json.MarshalIndent(r, "", "\t")
if err != nil {
t.Fatal(err)
}
t.Logf("repository = %s", j)
}

140
internal/github/models.go Normal file
View File

@@ -0,0 +1,140 @@
package github
import "time"
// Repository is a github repository.
type Repository struct {
ID int `json:"id"`
NodeID string `json:"node_id"`
Name string `json:"name"`
FullName string `json:"full_name"`
Private bool `json:"private"`
Owner *User `json:"owner"`
HTMLURL string `json:"html_url"`
Description string `json:"description"`
Fork bool `json:"fork"`
URL string `json:"url"`
ForksURL string `json:"forks_url"`
KeysURL string `json:"keys_url"`
CollaboratorsURL string `json:"collaborators_url"`
TeamsURL string `json:"teams_url"`
HooksURL string `json:"hooks_url"`
IssueEventsURL string `json:"issue_events_url"`
EventsURL string `json:"events_url"`
AssigneesURL string `json:"assignees_url"`
BranchesURL string `json:"branches_url"`
TagsURL string `json:"tags_url"`
BlobsURL string `json:"blobs_url"`
GitTagsURL string `json:"git_tags_url"`
GitRefsURL string `json:"git_refs_url"`
TreesURL string `json:"trees_url"`
StatusesURL string `json:"statuses_url"`
LanguagesURL string `json:"languages_url"`
StargazersURL string `json:"stargazers_url"`
ContributorsURL string `json:"contributors_url"`
SubscribersURL string `json:"subscribers_url"`
SubscriptionURL string `json:"subscription_url"`
CommitsURL string `json:"commits_url"`
GitCommitsURL string `json:"git_commits_url"`
CommentsURL string `json:"comments_url"`
IssueCommentURL string `json:"issue_comment_url"`
ContentsURL string `json:"contents_url"`
CompareURL string `json:"compare_url"`
MergesURL string `json:"merges_url"`
ArchiveURL string `json:"archive_url"`
DownloadsURL string `json:"downloads_url"`
IssuesURL string `json:"issues_url"`
PullsURL string `json:"pulls_url"`
MilestonesURL string `json:"milestones_url"`
NotificationsURL string `json:"notifications_url"`
LabelsURL string `json:"labels_url"`
ReleasesURL string `json:"releases_url"`
DeploymentsURL string `json:"deployments_url"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
PushedAt time.Time `json:"pushed_at"`
GitURL string `json:"git_url"`
SSHURL string `json:"ssh_url"`
CloneURL string `json:"clone_url"`
SvnURL string `json:"svn_url"`
Homepage string `json:"homepage"`
Size int `json:"size"`
StargazersCount int `json:"stargazers_count"`
WatchersCount int `json:"watchers_count"`
Language string `json:"language"`
HasIssues bool `json:"has_issues"`
HasProjects bool `json:"has_projects"`
HasDownloads bool `json:"has_downloads"`
HasWiki bool `json:"has_wiki"`
HasPages bool `json:"has_pages"`
ForksCount int `json:"forks_count"`
MirrorURL string `json:"mirror_url"`
Archived bool `json:"archived"`
Disabled bool `json:"disabled"`
OpenIssuesCount int `json:"open_issues_count"`
License *License `json:"license"`
AllowForking bool `json:"allow_forking"`
IsTemplate bool `json:"is_template"`
Topics []string `json:"topics"`
Visibility string `json:"visibility"`
Forks int `json:"forks"`
OpenIssues int `json:"open_issues"`
Watchers int `json:"watchers"`
DefaultBranch string `json:"default_branch"`
Organization *Organization `json:"organization"`
NetworkCount int `json:"network_count"`
SubscribersCount int `json:"subscribers_count"`
}
// User is a Github user.
type User struct {
Login string `json:"login"`
ID int `json:"id"`
NodeID string `json:"node_id"`
AvatarURL string `json:"avatar_url"`
GravatarID string `json:"gravatar_id"`
URL string `json:"url"`
HTMLURL string `json:"html_url"`
FollowersURL string `json:"followers_url"`
FollowingURL string `json:"following_url"`
GistsURL string `json:"gists_url"`
StarredURL string `json:"starred_url"`
SubscriptionsURL string `json:"subscriptions_url"`
OrganizationsURL string `json:"organizations_url"`
ReposURL string `json:"repos_url"`
EventsURL string `json:"events_url"`
ReceivedEventsURL string `json:"received_events_url"`
Type string `json:"type"`
SiteAdmin bool `json:"site_admin"`
}
// Organization is a Github organization.
type Organization struct {
Login string `json:"login"`
ID int `json:"id"`
NodeID string `json:"node_id"`
AvatarURL string `json:"avatar_url"`
GravatarID string `json:"gravatar_id"`
URL string `json:"url"`
HTMLURL string `json:"html_url"`
FollowersURL string `json:"followers_url"`
FollowingURL string `json:"following_url"`
GistsURL string `json:"gists_url"`
StarredURL string `json:"starred_url"`
SubscriptionsURL string `json:"subscriptions_url"`
OrganizationsURL string `json:"organizations_url"`
ReposURL string `json:"repos_url"`
EventsURL string `json:"events_url"`
ReceivedEventsURL string `json:"received_events_url"`
Type string `json:"type"`
SiteAdmin bool `json:"site_admin"`
}
// License is an open source license.
type License struct {
Key string `json:"key"`
Name string `json:"name"`
SPDXID string `json:"spdx_id"`
URL string `json:"url"`
NodeID string `json:"node_id"`
}

12
internal/test/utils.go
View File

@@ -2,6 +2,7 @@
package test
import (
"flag"
"io"
"io/ioutil"
"log"
@@ -12,6 +13,17 @@ import (
"testing"
)
var network = flag.Bool("net", false, "allow network access")
// RequiresNetwork declares that a test requires network access. The test is
// skipped if network access isn't enabled with the -net flag.
func RequiresNetwork(t *testing.T) {
t.Helper()
if !*network {
t.Skip("requires network: enable with -net flag")
}
}
// Assembles asserts that the given assembly code passes the go assembler.
func Assembles(t *testing.T, asm []byte) {
t.Helper()

77
tests/thirdparty/config.go vendored
View File

@@ -27,11 +27,23 @@ func (r GithubRepository) CloneURL() string {
return fmt.Sprintf("https://github.com/%s.git", r)
}
// Metadata about the repository.
type Metadata struct {
// Repository description.
Description string `json:"description,omitempty"`
// Homepage URL. Not the same as the Github page.
Homepage string `json:"homepage,omitempty"`
// Number of Github stars.
Stars int `json:"stars,omitempty"`
}
// Step represents a set of commands to run as part of the testing plan for a
// third-party package.
type Step struct {
Name string `json:"name"`
WorkingDirectory string `json:"dir"`
Name string `json:"name,omitempty"`
WorkingDirectory string `json:"dir,omitempty"`
Commands []string `json:"commands"`
}
@@ -52,13 +64,19 @@ type Package struct {
// available on github.
Repository GithubRepository `json:"repository"`
// Repository metadata.
Metadata Metadata `json:"metadata"`
// Default git branch. This is used when testing against the latest version.
DefaultBranch string `json:"default_branch,omitempty"`
// Version as a git sha, tag or branch.
Version string `json:"version"`
// Sub-package within the repository under test. All file path references
// will be relative to this directory. If empty the root of the repository
// is used.
SubPackage string `json:"pkg"`
SubPackage string `json:"pkg,omitempty"`
// Path to the module file for the avo generator package. This is necessary
// so the integration test can insert replace directives to point at the avo
@@ -68,13 +86,13 @@ type Package struct {
// Setup steps. These run prior to the insertion of avo replace directives,
// therefore should be used if it's necessary to initialize new go modules
// within the repository.
Setup []*Step `json:"setup"`
Setup []*Step `json:"setup,omitempty"`
// Steps to run the avo code generator.
Generate []*Step `json:"generate"` // generate commands to run
// Test steps. If empty, defaults to "go test ./...".
Test []*Step `json:"test"`
Test []*Step `json:"test,omitempty"`
}
// ID returns an identifier for the package.
@@ -83,9 +101,8 @@ func (p *Package) ID() string {
return strings.ReplaceAll(pkgpath, "/", "-")
}
// setdefaults fills in missing parameters to help make the input package
// descriptions less verbose.
func (p *Package) setdefaults() {
// defaults sets or removes default field values.
func (p *Package) defaults(set bool) {
for _, stage := range []struct {
Steps []*Step
DefaultName string
@@ -94,14 +111,28 @@ func (p *Package) setdefaults() {
{p.Generate, "Generate"},
{p.Test, "Test"},
} {
if len(stage.Steps) == 1 && stage.Steps[0].Name == "" {
stage.Steps[0].Name = stage.DefaultName
if len(stage.Steps) == 1 {
stage.Steps[0].Name = applydefault(set, stage.Steps[0].Name, stage.DefaultName)
}
}
}
func applydefault(set bool, s, def string) string {
switch {
case set && s == "":
return def
case !set && s == def:
return ""
default:
return s
}
}
// Validate package definition.
func (p *Package) Validate() error {
if p.DefaultBranch == "" {
return errors.New("missing default branch")
}
if p.Version == "" {
return errors.New("missing version")
}
@@ -193,9 +224,9 @@ func (p *Package) Steps(c *Context) []*Step {
// Packages is a collection of third-party integration tests.
type Packages []*Package
func (p Packages) setdefaults() {
func (p Packages) defaults(set bool) {
for _, pkg := range p {
pkg.setdefaults()
pkg.defaults(set)
}
}
@@ -217,7 +248,7 @@ func LoadPackages(r io.Reader) (Packages, error) {
if err := d.Decode(&pkgs); err != nil {
return nil, err
}
pkgs.setdefaults()
pkgs.defaults(true)
return pkgs, nil
}
@@ -230,3 +261,23 @@ func LoadPackagesFile(filename string) (Packages, error) {
defer f.Close()
return LoadPackages(f)
}
// StorePackages writes a list of package configurations in JSON format.
func StorePackages(w io.Writer, pkgs Packages) error {
e := json.NewEncoder(w)
e.SetIndent("", " ")
pkgs.defaults(false)
err := e.Encode(pkgs)
pkgs.defaults(true)
return err
}
// StorePackagesFile writes a list of package configurations to a JSON file.
func StorePackagesFile(filename string, pkgs Packages) error {
f, err := os.Create(filename)
if err != nil {
return err
}
defer f.Close()
return StorePackages(f, pkgs)
}

56
tests/thirdparty/config_test.go vendored
View File

@@ -1,6 +1,8 @@
package thirdparty
import (
"bytes"
"reflect"
"strings"
"testing"
)
@@ -24,35 +26,46 @@ func TestValidateErrors(t *testing.T) {
ErrorSubstring: "missing commands",
},
{
Name: "package_missing_version",
Name: "package_missing_default_branch",
Item: &Package{
Repository: GithubRepository{Owner: "octocat", Name: "hello-world"},
},
ErrorSubstring: "missing default branch",
},
{
Name: "package_missing_version",
Item: &Package{
Repository: GithubRepository{Owner: "octocat", Name: "hello-world"},
DefaultBranch: "main",
},
ErrorSubstring: "missing version",
},
{
Name: "package_missing_module",
Item: &Package{
Repository: GithubRepository{Owner: "octocat", Name: "hello-world"},
Version: "v1.0.1",
Repository: GithubRepository{Owner: "octocat", Name: "hello-world"},
DefaultBranch: "main",
Version: "v1.0.1",
},
ErrorSubstring: "missing module",
},
{
Name: "package_no_generate_commands",
Item: &Package{
Repository: GithubRepository{Owner: "octocat", Name: "hello-world"},
Version: "v1.0.1",
Module: "avo/go.mod",
Repository: GithubRepository{Owner: "octocat", Name: "hello-world"},
DefaultBranch: "main",
Version: "v1.0.1",
Module: "avo/go.mod",
},
ErrorSubstring: "no generate commands",
},
{
Name: "package_invalid_generate_commands",
Item: &Package{
Repository: GithubRepository{Owner: "octocat", Name: "hello-world"},
Version: "v1.0.1",
Module: "avo/go.mod",
Repository: GithubRepository{Owner: "octocat", Name: "hello-world"},
DefaultBranch: "main",
Version: "v1.0.1",
Module: "avo/go.mod",
Generate: []*Step{
{},
},
@@ -66,7 +79,7 @@ func TestValidateErrors(t *testing.T) {
Repository: GithubRepository{Owner: "octocat", Name: "hello-world"},
},
},
ErrorSubstring: "missing version",
ErrorSubstring: "missing default branch",
},
}
for _, c := range cases {
@@ -134,3 +147,26 @@ func TestPackagesFileStepsValid(t *testing.T) {
}
}
}
func TestPackagesFileRoundtrip(t *testing.T) {
pkgs, err := LoadPackagesFile("packages.json")
if err != nil {
t.Fatal(err)
}
// Write and read back.
buf := bytes.NewBuffer(nil)
if err := StorePackages(buf, pkgs); err != nil {
t.Fatal(err)
}
roundtrip, err := LoadPackages(buf)
if err != nil {
t.Fatal(err)
}
// Should be identical.
if !reflect.DeepEqual(pkgs, roundtrip) {
t.Fatal("roundtrip mismatch")
}
}

56
tests/thirdparty/metadata_test.go vendored Normal file
View File

@@ -0,0 +1,56 @@
package thirdparty
import (
"context"
"flag"
"testing"
"github.com/mmcloughlin/avo/internal/github"
"github.com/mmcloughlin/avo/internal/test"
)
var update = flag.Bool("update", false, "update package metadata")
func TestPackagesFileMetadata(t *testing.T) {
test.RequiresNetwork(t)
ctx := context.Background()
pkgs, err := LoadPackagesFile("packages.json")
if err != nil {
t.Fatal(err)
}
g := github.NewClient(github.WithTokenFromEnvironment())
for _, pkg := range pkgs {
// Fetch metadata.
r, err := g.Repository(ctx, pkg.Repository.Owner, pkg.Repository.Name)
if err != nil {
t.Fatal(err)
}
// Update, if requested.
if *update {
pkg.DefaultBranch = r.DefaultBranch
pkg.Metadata.Description = r.Description
pkg.Metadata.Homepage = r.Homepage
pkg.Metadata.Stars = r.StargazersCount
t.Logf("%s: metadata updated", pkg.ID())
}
// Check up to date. Potentially fast-changing properties not included.
uptodate := true
uptodate = pkg.DefaultBranch == r.DefaultBranch && uptodate
uptodate = pkg.Metadata.Description == r.Description && uptodate
uptodate = pkg.Metadata.Homepage == r.Homepage && uptodate
if !uptodate {
t.Errorf("%s: metadata out of date (use -update flag to fix)", pkg.ID())
}
}
if err := StorePackagesFile("packages.json", pkgs); err != nil {
t.Fatal(err)
}
}

91
tests/thirdparty/packages.json vendored
View File

@@ -4,6 +4,11 @@
"owner": "zeebo",
"name": "xxh3"
},
"metadata": {
"description": "XXH3 algorithm in Go",
"stars": 198
},
"default_branch": "master",
"version": "v1.0.0-rc1",
"module": "avo/go.mod",
"generate": [
@@ -21,6 +26,11 @@
"owner": "dgryski",
"name": "go-sip13"
},
"metadata": {
"description": "siphash 1-3",
"stars": 31
},
"default_branch": "master",
"version": "62edffca92457b3a66125c686137cc5f0fe81672",
"module": "_avo/go.mod",
"setup": [
@@ -55,6 +65,11 @@
"owner": "phoreproject",
"name": "bls"
},
"metadata": {
"description": "Go implementation of the BLS12-381 pairing",
"stars": 81
},
"default_branch": "master",
"version": "a88a5ae26844d7293359422888d7c7f69f43c845",
"module": "asm/go.mod",
"setup": [
@@ -90,6 +105,11 @@
"owner": "minio",
"name": "md5-simd"
},
"metadata": {
"description": "Accelerate aggregated MD5 hashing performance up to 8x for AVX512 and 4x for AVX2. Useful for server applications that need to compute many MD5 sums in parallel.",
"stars": 87
},
"default_branch": "master",
"version": "30ad8af83f6868c2a30c615f3edf1a9366bf3f89",
"module": "_gen/go.mod",
"generate": [
@@ -106,6 +126,11 @@
"owner": "zeebo",
"name": "blake3"
},
"metadata": {
"description": "Pure Go implementation of BLAKE3 with AVX2 and SSE4.1 acceleration",
"stars": 252
},
"default_branch": "master",
"version": "25dba572f0e78ec108f0dd79c9c15288f542d7d9",
"module": "avo/go.mod",
"generate": [
@@ -123,6 +148,11 @@
"owner": "klauspost",
"name": "reedsolomon"
},
"metadata": {
"description": "Reed-Solomon Erasure Coding in Go",
"stars": 1343
},
"default_branch": "master",
"version": "922778284547557265cff0f903ab5f4c27e40ae9",
"module": "_gen/go.mod",
"generate": [
@@ -139,6 +169,11 @@
"owner": "orisano",
"name": "wyhash"
},
"metadata": {
"description": "A pure-Go wyhash implementation.",
"stars": 21
},
"default_branch": "master",
"version": "32a3f7f6ba4797e2d87dab2969cc9dd63d39cce9",
"module": "avo/go.mod",
"setup": [
@@ -164,6 +199,11 @@
"owner": "klauspost",
"name": "compress"
},
"metadata": {
"description": "Optimized Go Compression Packages",
"stars": 2442
},
"default_branch": "master",
"version": "2adf487b3e02f95ce7efd6e4953fda0ae7ecd080",
"pkg": "s2",
"module": "_generate/go.mod",
@@ -181,6 +221,11 @@
"owner": "dgryski",
"name": "go-bloomindex"
},
"metadata": {
"description": "Bloom-filter based search index",
"stars": 107
},
"default_branch": "master",
"version": "0902316dce158c154b958ee5cfc706c62af29a42",
"module": "avo/go.mod",
"setup": [
@@ -221,6 +266,11 @@
"owner": "dgryski",
"name": "go-marvin32"
},
"metadata": {
"description": "Assembly-optimized Marvin32 hash function",
"stars": 12
},
"default_branch": "master",
"version": "7d18f4c6ea7c24b29d1c7a670f8ae40b0812f2e3",
"module": "avo/go.mod",
"setup": [
@@ -262,6 +312,11 @@
"owner": "dgryski",
"name": "go-speck"
},
"metadata": {
"description": "SPECK cipher",
"stars": 10
},
"default_branch": "master",
"version": "5b36d4c08d8840c352a153bf37281434ad550ec0",
"module": "avo/go.mod",
"setup": [
@@ -304,6 +359,11 @@
"owner": "dgryski",
"name": "go-chaskey"
},
"metadata": {
"description": "go-chaskey: an implementation of chaskey, an efficient MAC for microcontrollers",
"stars": 7
},
"default_branch": "master",
"version": "ba454392bc5ab6daae103e15147185f8f4a27dcc",
"module": "avo/go.mod",
"setup": [
@@ -346,6 +406,11 @@
"owner": "lukechampine",
"name": "us"
},
"metadata": {
"description": "An alternative interface to Sia",
"stars": 49
},
"default_branch": "master",
"version": "dff56a80f83653cb14eeeb57ba6ba3c3e942c412",
"pkg": "merkle/blake2b",
"module": "avo/go.mod",
@@ -380,6 +445,11 @@
"owner": "segmentio",
"name": "asm"
},
"metadata": {
"description": "Go library providing algorithms optimized to leverage the characteristics of modern CPUs",
"stars": 548
},
"default_branch": "main",
"version": "v1.0.0",
"module": "build/go.mod",
"generate": [
@@ -395,6 +465,11 @@
"owner": "ericlagergren",
"name": "lwcrypto"
},
"metadata": {
"description": "NIST Lightweight Cryptography finalists",
"stars": 2
},
"default_branch": "main",
"version": "0c42b05eddc34c58bf8e0cd4250c5cd2c256ea57",
"pkg": "ascon",
"module": "asm/go.mod",
@@ -413,6 +488,11 @@
"owner": "ericlagergren",
"name": "lwcrypto"
},
"metadata": {
"description": "NIST Lightweight Cryptography finalists",
"stars": 2
},
"default_branch": "main",
"version": "0c42b05eddc34c58bf8e0cd4250c5cd2c256ea57",
"pkg": "grain",
"module": "asm/go.mod",
@@ -431,6 +511,11 @@
"owner": "oasisprotocol",
"name": "curve25519-voi"
},
"metadata": {
"description": "High-performance Curve25519/ristretto255 for Go",
"stars": 32
},
"default_branch": "master",
"version": "d5a936accd94ef9da4c0fe9db0a6342dcdcfeadf",
"module": "internal/asm/amd64/go.mod",
"generate": [
@@ -447,6 +532,12 @@
"owner": "golang",
"name": "crypto"
},
"metadata": {
"description": "[mirror] Go supplementary cryptography libraries",
"homepage": "https://golang.org/x/crypto",
"stars": 2283
},
"default_branch": "master",
"version": "089bfa5675191fd96a44247682f76ebca03d7916",
"pkg": "curve25519",
"module": "internal/field/_asm/go.mod",