all: AVX-512 (#217)

Extends avo to support most AVX-512 instruction sets.

The instruction type is extended to support suffixes. The K family of opmask
registers is added to the register package, and the operand package is updated
to support the new operand types. Move instruction deduction in `Load` and
`Store` is extended to support KMOV* and VMOV* forms.

Internal code generation packages were overhauled. Instruction database loading
required various messy changes to account for the additional complexities of the
AVX-512 instruction sets. The internal/api package was added to introduce a
separation between instruction forms in the database, and the functions avo
provides to create them. This was required since with instruction suffixes there
is no longer a one-to-one mapping between instruction constructors and opcodes.

AVX-512 bloated generated source code size substantially, initially increasing
compilation and CI test times to an unacceptable level. Two changes were made to
address this:

1.  Instruction constructors in the `x86` package moved to an optab-based
    approach. This compiles substantially faster than the verbose code
    generation we had before.

2.  The most verbose code-generated tests are moved under build tags and
    limited to a stress test mode. Stress test builds are run on
    schedule but not in regular CI.

An example of AVX-512 accelerated 16-lane MD5 is provided to demonstrate and
test the new functionality.

Updates #20 #163 #229

Co-authored-by: Vaughn Iverson <vsivsi@yahoo.com>

internal/inst/stress_test.go

@@ -0,0 +1,9 @@
+// The godata test relies on a huge generated file, so we limit it to a
+// stress-test only build.
+
+//go:build stress
+// +build stress
+
+package inst
+
+//go:generate avogen -bootstrap -data ../data -output ztable_test.go godatatest

internal/inst/table.go

@@ -1,7 +1,11 @@
 package inst
 
+import (
+	"sort"
+	"strings"
+)
+
 //go:generate avogen -bootstrap -data ../data -output ztable.go godata
-//go:generate avogen -bootstrap -data ../data -output ztable_test.go godatatest
 
 // Lookup returns the instruction with the given opcode. Boolean return value
 // indicates whether the instruction was found.
@@ -13,3 +17,119 @@ func Lookup(opcode string) (Instruction, bool) {
 	}
 	return Instruction{}, false
 }
+
+// OperandTypes returns all the operand types that appear in the provided
+// instructions.
+func OperandTypes(is []Instruction) []string {
+	set := map[string]bool{}
+	for _, i := range is {
+		for _, f := range i.Forms {
+			for _, op := range f.Operands {
+				set[op.Type] = true
+			}
+		}
+	}
+	return sortedslice(set)
+}
+
+// ImplicitRegisters returns all the registers that appear as implicit operands
+// in the provided instructions.
+func ImplicitRegisters(is []Instruction) []string {
+	set := map[string]bool{}
+	for _, i := range is {
+		for _, f := range i.Forms {
+			for _, op := range f.ImplicitOperands {
+				set[op.Register] = true
+			}
+		}
+	}
+	return sortedslice(set)
+}
+
+// UniqueSuffixes returns all the non-empty suffixes that appear in the provided
+// instructions.
+func UniqueSuffixes(is []Instruction) []Suffix {
+	// Collect set.
+	set := map[Suffix]bool{}
+	for _, i := range is {
+		for _, f := range i.Forms {
+			for _, suffixes := range f.SupportedSuffixes() {
+				for _, suffix := range suffixes {
+					set[suffix] = true
+				}
+			}
+		}
+	}
+
+	// Convert to sorted slice.
+	suffixes := make([]Suffix, 0, len(set))
+	for suffix := range set {
+		suffixes = append(suffixes, suffix)
+	}
+
+	sort.Slice(suffixes, func(i, j int) bool {
+		return suffixes[i] < suffixes[j]
+	})
+
+	return suffixes
+}
+
+// SuffixesClasses returns all possible classes of suffix combinations.
+func SuffixesClasses(is []Instruction) map[string][]Suffixes {
+	classes := map[string][]Suffixes{}
+	for _, i := range is {
+		for _, f := range i.Forms {
+			class := f.SuffixesClass()
+			if _, ok := classes[class]; ok {
+				continue
+			}
+			classes[class] = f.SupportedSuffixes()
+		}
+	}
+	return classes
+}
+
+// ISAs returns all the unique ISAs seen in the given instructions.
+func ISAs(is []Instruction) []string {
+	set := map[string]bool{}
+	for _, i := range is {
+		for _, f := range i.Forms {
+			for _, isa := range f.ISA {
+				set[isa] = true
+			}
+		}
+	}
+	return sortedslice(set)
+}
+
+// ISACombinations returns all the unique combinations of ISAs seen in the given
+// instructions.
+func ISACombinations(is []Instruction) [][]string {
+	var combinations [][]string
+	seen := map[string]bool{}
+	for _, i := range is {
+		for _, f := range i.Forms {
+			isas := append([]string(nil), f.ISA...)
+			sort.Strings(isas)
+			key := strings.Join(isas, ",")
+
+			if !seen[key] {
+				combinations = append(combinations, isas)
+				seen[key] = true
+			}
+		}
+	}
+	return combinations
+}
+
+// sortedslice builds a sorted slice of strings from a set.
+func sortedslice(set map[string]bool) []string {
+	ss := make([]string, 0, len(set))
+	for s := range set {
+		ss = append(ss, s)
+	}
+
+	sort.Strings(ss)
+
+	return ss
+}

internal/inst/table_test.go

@@ -3,6 +3,7 @@ package inst_test
 import (
 	"io/ioutil"
 	"reflect"
+	"sort"
 	"strings"
 	"testing"
 
@@ -70,6 +71,84 @@ func TestInstructionProperties(t *testing.T) {
 	}
 }
 
+func TestHaveSuffixes(t *testing.T) {
+	for _, i := range inst.Instructions {
+		for _, f := range i.Forms {
+			if len(f.SupportedSuffixes()) == 0 {
+				t.Errorf("%s: no supported suffixes", i.Opcode)
+			}
+		}
+	}
+}
+
+func TestAcceptsSuffixes(t *testing.T) {
+	// Verify consistency between the AcceptsSuffixes and SupportedSuffixes methods.
+	for _, i := range inst.Instructions {
+		for _, f := range i.Forms {
+			expect := false
+			for _, suffixes := range f.SupportedSuffixes() {
+				if len(suffixes) > 0 {
+					expect = true
+				}
+			}
+
+			if got := f.AcceptsSuffixes(); got != expect {
+				t.Errorf("%s: AcceptsSuffixes() = %v; expect %v", i.Opcode, got, expect)
+			}
+		}
+	}
+}
+
+func TestSuffixesClasses(t *testing.T) {
+	// Verify that all instructions in a suffix class support the same suffixes.
+	reps := map[string][]inst.Suffixes{}
+	for _, i := range inst.Instructions {
+		for _, f := range i.Forms {
+			class := f.SuffixesClass()
+			expect, ok := reps[class]
+			if !ok {
+				t.Logf("new class %q: representative from instruction %s", class, i.Opcode)
+				reps[class] = f.SupportedSuffixes()
+				continue
+			}
+
+			got := f.SupportedSuffixes()
+			if !reflect.DeepEqual(expect, got) {
+				t.Fatalf("suffixes mismatch for class %q", class)
+			}
+		}
+	}
+}
+
+func TestSuffixesHaveSummaries(t *testing.T) {
+	set := map[inst.Suffix]bool{}
+	for _, i := range inst.Instructions {
+		for _, f := range i.Forms {
+			for _, suffixes := range f.SupportedSuffixes() {
+				for _, suffix := range suffixes {
+					set[suffix] = true
+				}
+			}
+		}
+	}
+
+	for suffix := range set {
+		if suffix.Summary() == "" {
+			t.Errorf("suffix %q missing summary", suffix)
+		}
+	}
+}
+
+func TestISASorted(t *testing.T) {
+	for _, i := range inst.Instructions {
+		for _, f := range i.Forms {
+			if !sort.StringsAreSorted(f.ISA) {
+				t.Fatalf("%s: isa not sorted", i.Opcode)
+			}
+		}
+	}
+}
+
 func TestAssembles(t *testing.T) {
 	g := gen.NewAsmTest(printer.NewArgvConfig())
 	b, err := g.Generate(inst.Instructions)
@@ -143,17 +222,86 @@ func TestCancellingInputs(t *testing.T) {
 			n := 0
 			for _, op := range f.Operands {
 				if op.Action.Read() {
-					n++
 					switch op.Type {
-					case "r8", "r16", "r32", "r64", "xmm", "ymm":
-						// pass
+					case "r8", "r16", "r32", "r64", "xmm", "ymm", "zmm":
+						n++
+					case "k":
+						// skip mask registers
 					default:
 						t.Errorf("%s: unexpected operand type %q for self-cancelling input", i.Opcode, op.Type)
 					}
 				}
 			}
-			if n != 2 {
-				t.Errorf("%s: expected two inputs for self-cancelling form", i.Opcode)
+			if n < 2 {
+				t.Log(f)
+				t.Errorf("%s: expected at least two inputs for self-cancelling form", i.Opcode)
+			}
+		}
+	}
+}
+
+func TestFlagOperandTypes(t *testing.T) {
+	for _, i := range inst.Instructions {
+		for _, f := range i.Forms {
+			// Check for memory operands.
+			mem := false
+			for _, op := range f.Operands {
+				if strings.HasPrefix(op.Type, "m") {
+					mem = true
+				}
+			}
+
+			// Broadcast applies to memory instructions only.
+			if f.Broadcast && !mem {
+				t.Errorf("%s: expect broadcast form to have memory operand", i.Opcode)
+			}
+
+			// Embedded rounding must be register-only.
+			if f.EmbeddedRounding && mem {
+				t.Errorf("%s: embedded-rounding only supported for register-only forms", i.Opcode)
+			}
+
+			// Suppress all exceptions is register only.
+			if f.SuppressAllExceptions && mem {
+				t.Errorf("%s: embedded-rounding only supported for register-only forms", i.Opcode)
+			}
+		}
+	}
+}
+
+func TestFlagCombinations(t *testing.T) {
+	for _, i := range inst.Instructions {
+		for _, f := range i.Forms {
+			if f.EmbeddedRounding && f.SuppressAllExceptions {
+				t.Errorf("%s: embedded-rounding cannot be combined with suppress-all-exceptions", i.Opcode)
+			}
+			if f.Broadcast && f.EmbeddedRounding {
+				t.Errorf("%s: broadcast cannot be combined with embedded-rounding", i.Opcode)
+			}
+			if f.Broadcast && f.SuppressAllExceptions {
+				t.Errorf("%s: broadcast cannot be combined with suppress-all-exceptions", i.Opcode)
+			}
+		}
+	}
+}
+
+func TestZeroingHasMask(t *testing.T) {
+	for _, i := range inst.Instructions {
+		for _, f := range i.Forms {
+			if !f.Zeroing {
+				continue
+			}
+
+			// Expect mask operand.
+			mask := false
+			for _, op := range f.Operands {
+				if op.Type == "k" {
+					mask = true
+				}
+			}
+
+			if !mask {
+				t.Errorf("%s: expect mask operand if zeroing is enabled", i.Opcode)
 			}
 		}
 	}

internal/inst/types.go

@@ -10,7 +10,7 @@ type Instruction struct {
 	Opcode  string // Golang assembly mnemonic
 	AliasOf string // Opcode of instruction that this is an alias for
 	Summary string // Description of the instruction
-	Forms   []Form // Accepted operand forms
+	Forms          // Accepted operand forms
 }
 
 // IsTerminal reports whether the instruction exits a function.
@@ -40,10 +40,13 @@ func (i Instruction) IsConditionalBranch() bool {
 	return i.IsBranch() && i.Opcode != "JMP"
 }
 
+// Forms is a collection of instruction forms.
+type Forms []Form
+
 // Arities returns the unique arities among the instruction forms.
-func (i Instruction) Arities() []int {
+func (fs Forms) Arities() []int {
 	s := map[int]bool{}
-	for _, f := range i.Forms {
+	for _, f := range fs {
 		s[f.Arity()] = true
 	}
 	a := make([]int, 0, len(s))
@@ -56,22 +59,22 @@ func (i Instruction) Arities() []int {
 
 // Arity is a convenience for returning the unique instruction arity when you
 // know it is not variadic. Panics for a variadic instruction.
-func (i Instruction) Arity() int {
-	if i.IsVariadic() {
+func (fs Forms) Arity() int {
+	if fs.IsVariadic() {
 		panic("variadic")
 	}
-	a := i.Arities()
+	a := fs.Arities()
 	return a[0]
 }
 
 // IsVariadic reports whether the instruction has more than one arity.
-func (i Instruction) IsVariadic() bool {
-	return len(i.Arities()) > 1
+func (fs Forms) IsVariadic() bool {
+	return len(fs.Arities()) > 1
 }
 
 // IsNiladic reports whether the instruction takes no operands.
-func (i Instruction) IsNiladic() bool {
-	a := i.Arities()
+func (fs Forms) IsNiladic() bool {
+	a := fs.Arities()
 	return len(a) == 1 && a[0] == 0
 }
 
@@ -86,12 +89,35 @@ type Form struct {
 	// Registers read or written but not explicitly passed to the instruction.
 	ImplicitOperands []ImplicitOperand
 
+	// Encoding type required for this instruction form.
+	EncodingType EncodingType
+
 	// CancellingInputs indicates this instruction form has no dependency on the
 	// input operands when they refer to the same register. The classic example of
 	// this is "XORQ RAX, RAX", in which case the output has no dependence on the
 	// value of RAX. Instruction forms with cancelling inputs have only two input
 	// operands, which have the same register type.
 	CancellingInputs bool
+
+	// Zeroing indicates whether the instruction form uses AVX-512 zeroing. This
+	// is the .Z suffix in Go, usually indicated with {z} operand suffix in
+	// Intel manuals.
+	Zeroing bool
+
+	// EmbeddedRounding indicates whether the instruction form uses AVX-512
+	// embedded rounding. This is the RN_SAE, RZ_SAE, RD_SAE and RU_SAE suffixes
+	// in Go, usually indicated with {er} in Intel manuals.
+	EmbeddedRounding bool
+
+	// SuppressAllExceptions indicates whether the instruction form uses AVX-512
+	// "suppress all exceptions". This is the SAE suffix in Go, usually
+	// indicated with {sae} in Intel manuals.
+	SuppressAllExceptions bool
+
+	// Broadcast indicates whether the instruction form uses AVX-512
+	// broadcast. This is the BCST suffix in Go, usually indicated by operand
+	// types like "m64bcst" in Intel manuals.
+	Broadcast bool
 }
 
 // Arity returns the number of operands this form expects.
@@ -108,6 +134,143 @@ func (f Form) Signature() []string {
 	return s
 }
 
+// Clone the instruction form.
+func (f Form) Clone() Form {
+	c := f
+	c.ISA = append([]string(nil), f.ISA...)
+	c.Operands = append([]Operand(nil), f.Operands...)
+	c.ImplicitOperands = append([]ImplicitOperand(nil), f.ImplicitOperands...)
+	return c
+}
+
+// AcceptsSuffixes reports whether this form takes any opcode suffixes.
+func (f Form) AcceptsSuffixes() bool {
+	return f.Broadcast || f.EmbeddedRounding || f.SuppressAllExceptions || f.Zeroing
+}
+
+// SuffixesClass returns a key representing the class of instruction suffixes it
+// accepts. All instructions sharing a suffix class accept the same suffixes.
+func (f Form) SuffixesClass() string {
+	if !f.AcceptsSuffixes() {
+		return "nil"
+	}
+	var parts []string
+	for _, flag := range []struct {
+		Name    string
+		Enabled bool
+	}{
+		{"er", f.EmbeddedRounding},
+		{"sae", f.SuppressAllExceptions},
+		{"bcst", f.Broadcast},
+		{"z", f.Zeroing},
+	} {
+		if flag.Enabled {
+			parts = append(parts, flag.Name)
+		}
+	}
+	return strings.Join(parts, "_")
+}
+
+// SupportedSuffixes returns the list of all possible suffix combinations
+// supported by this instruction form.
+func (f Form) SupportedSuffixes() []Suffixes {
+	suffixes := []Suffixes{
+		{},
+	}
+
+	add := func(ss ...Suffix) {
+		var exts []Suffixes
+		for _, s := range ss {
+			for _, suffix := range suffixes {
+				ext := append(Suffixes(nil), suffix...)
+				ext = append(ext, s)
+				exts = append(exts, ext)
+			}
+		}
+		suffixes = exts
+	}
+
+	if f.Broadcast {
+		add(BCST)
+	}
+
+	if f.EmbeddedRounding {
+		add(RN_SAE, RZ_SAE, RD_SAE, RU_SAE)
+	}
+
+	if f.SuppressAllExceptions {
+		add(SAE)
+	}
+
+	if f.Zeroing {
+		add(Z)
+	}
+
+	return suffixes
+}
+
+// Suffix is an opcode suffix.
+type Suffix string
+
+// Supported opcode suffixes in x86 assembly.
+const (
+	BCST   Suffix = "BCST"
+	RN_SAE Suffix = "RN_SAE"
+	RZ_SAE Suffix = "RZ_SAE"
+	RD_SAE Suffix = "RD_SAE"
+	RU_SAE Suffix = "RU_SAE"
+	SAE    Suffix = "SAE"
+	Z      Suffix = "Z"
+)
+
+func (s Suffix) String() string {
+	return string(s)
+}
+
+// Summary of the opcode suffix, for documentation purposes.
+func (s Suffix) Summary() string {
+	return suffixsummary[s]
+}
+
+var suffixsummary = map[Suffix]string{
+	BCST:   "Broadcast",
+	RN_SAE: "Round Towards Nearest",
+	RZ_SAE: "Round Towards Zero",
+	RD_SAE: "Round Towards Negative Infinity",
+	RU_SAE: "Round Towards Positive Infinity",
+	SAE:    "Suppress All Exceptions",
+	Z:      "Zeroing Masking",
+}
+
+// Suffixes is a list of opcode suffixes.
+type Suffixes []Suffix
+
+// String returns the dot-separated suffixes.
+func (s Suffixes) String() string { return s.Join(".") }
+
+// Join suffixes with the given separator.
+func (s Suffixes) Join(sep string) string {
+	return strings.Join(s.Strings(), sep)
+}
+
+// Strings returns the suffixes as strings.
+func (s Suffixes) Strings() []string {
+	var ss []string
+	for _, suffix := range s {
+		ss = append(ss, suffix.String())
+	}
+	return ss
+}
+
+// Summaries returns all the suffix summaries.
+func (s Suffixes) Summaries() []string {
+	var summaries []string
+	for _, suffix := range s {
+		summaries = append(summaries, suffix.Summary())
+	}
+	return summaries
+}
+
 // Operand is an operand to an instruction, describing the expected type and read/write action.
 type Operand struct {
 	Type   string
@@ -125,9 +288,10 @@ type Action uint8
 
 // Possible Action types.
 const (
-	R  Action = 0x1
-	W  Action = 0x2
-	RW Action = R | W
+	R Action = 1 << iota // Read
+	W                    // Write
+
+	RW Action = R | W // Read-Write
 )
 
 // ActionFromReadWrite builds an Action from boolean flags.
@@ -142,19 +306,24 @@ func ActionFromReadWrite(r, w bool) Action {
 	return a
 }
 
-// Contains reports whether a supports all actions in s.
-func (a Action) Contains(s Action) bool {
+// ContainsAll reports whether a supports all actions in s.
+func (a Action) ContainsAll(s Action) bool {
 	return (a & s) == s
 }
 
+// ContainsAny reports whether a supports any actions in s.
+func (a Action) ContainsAny(s Action) bool {
+	return (a & s) != 0
+}
+
 // Read reports whether a supports read.
 func (a Action) Read() bool {
-	return a.Contains(R)
+	return a.ContainsAll(R)
 }
 
 // Write reports whether a supports write.
 func (a Action) Write() bool {
-	return a.Contains(W)
+	return a.ContainsAll(W)
 }
 
 // String represents a as a human-readable string.
@@ -168,3 +337,14 @@ func (a Action) String() string {
 	}
 	return s
 }
+
+// EncodingType specifies a category of encoding types.
+type EncodingType uint8
+
+// Supported encoding types.
+const (
+	EncodingTypeLegacy EncodingType = 1 + iota
+	EncodingTypeREX
+	EncodingTypeVEX
+	EncodingTypeEVEX
+)

internal/inst/types_test.go

@@ -0,0 +1,82 @@
+package inst
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestFormSupportedSuffixes(t *testing.T) {
+	cases := []struct {
+		Form   Form
+		Expect []Suffixes
+	}{
+		{
+			Form: Form{},
+			Expect: []Suffixes{
+				{},
+			},
+		},
+		{
+			Form: Form{
+				Broadcast: true,
+			},
+			Expect: []Suffixes{
+				{BCST},
+			},
+		},
+		{
+			Form: Form{
+				EmbeddedRounding: true,
+			},
+			Expect: []Suffixes{
+				{RN_SAE},
+				{RZ_SAE},
+				{RD_SAE},
+				{RU_SAE},
+			},
+		},
+		{
+			Form: Form{
+				SuppressAllExceptions: true,
+			},
+			Expect: []Suffixes{
+				{SAE},
+			},
+		},
+		{
+			Form: Form{
+				Zeroing: true,
+			},
+			Expect: []Suffixes{
+				{Z},
+			},
+		},
+		{
+			Form: Form{
+				EmbeddedRounding: true,
+				Zeroing:          true,
+			},
+			Expect: []Suffixes{
+				{RN_SAE, Z},
+				{RZ_SAE, Z},
+				{RD_SAE, Z},
+				{RU_SAE, Z},
+			},
+		},
+		{
+			Form: Form{
+				Broadcast: true,
+				Zeroing:   true,
+			},
+			Expect: []Suffixes{
+				{BCST, Z},
+			},
+		},
+	}
+	for _, c := range cases {
+		got := c.Form.SupportedSuffixes()
+		if !reflect.DeepEqual(c.Expect, got) {
+			t.Errorf("%v.SupportedSuffixes() = %v; expect %v", c.Form, got, c.Expect)
+		}
+	}
+}