pkg/buildinfo: Use Go's buildinfo to derive version of development builds.

This allows the build commands in Makefile and tools/buildall.sh to be
simplified.

Official reproducible builds are now handled as a build variant, and the command
in Makefile no longer tries to produce reproducible builds.

Instructions in PACKAGING.md have been completely rewritten.

Makefile

@@ -1,25 +1,11 @@
 ELVISH_MAKE_BIN ?= $(or $(GOBIN),$(shell go env GOPATH)/bin)/elvish$(shell go env GOEXE)
 ELVISH_MAKE_BIN := $(subst \,/,$(ELVISH_MAKE_BIN))
-ELVISH_PLUGIN_SUPPORT ?= 0
-
-# Treat 0 as false and everything else as true (consistent with CGO_ENABLED).
-ifeq ($(ELVISH_PLUGIN_SUPPORT), 0)
-    REPRODUCIBLE := true
-else
-    REPRODUCIBLE := false
-endif
 
 default: test get
 
 get:
-	export CGO_ENABLED=$(ELVISH_PLUGIN_SUPPORT); \
-	if go env GOOS GOARCH | egrep -qx '(windows .*|linux (amd64|arm64))'; then \
-		export GOFLAGS=-buildmode=pie; \
-	fi; \
 	mkdir -p $(shell dirname $(ELVISH_MAKE_BIN))
-	go build -o $(ELVISH_MAKE_BIN) -trimpath -ldflags \
-		"-X src.elv.sh/pkg/buildinfo.VersionSuffix=-dev.$$(git rev-parse HEAD)$$(git diff HEAD --quiet || printf +%s `uname -n`) \
-		 -X src.elv.sh/pkg/buildinfo.Reproducible=$(REPRODUCIBLE)" ./cmd/elvish
+	go build -o $(ELVISH_MAKE_BIN) ./cmd/elvish
 
 generate:
 	go generate ./...

PACKAGING.md

@@ -1,124 +1,82 @@
 # Packager's Manual
 
 **Note**: The guidance here applies to the current development version and
-release versions starting from 0.16.0. The details for earlier versions are
+release versions starting from 0.19.0. The details for earlier versions are
 different.
 
-Elvish is a normal Go application, and doesn't require any special attention.
-Build the main package of `cmd/elvish`, and you should get a fully working
-binary.
+The main package of Elvish is `cmd/elvish`, and you can build it like any other
+Go application. None of the instructions below are strictly required.
 
-If you don't care about accurate version information or reproducible builds, you
-can now stop reading. If you do, there is a small amount of extra work to get
-them.
+## Supplying VCS information for development builds
 
-## Accurate version information
+When Elvish is built from a development branch, it will try to figure out its
+version from the VCS information Go compiler encoded. When that works,
+`elvish -version` will output something like this:
 
-The `pkg/buildinfo` package contains a constant, `Version`, and a variable,
-`VersionSuffix`, which are concatenated to form the full version used in the
-output of `elvish -version` and `elvish -buildinfo`. Their values are set as
-follows:
+```
+0.19.0-dev.0.20220320172241-5dc8c02a32cf
+```
 
--   At release tags, `Version` contains the version of the release, which is
-    identical to the tag name. `VersionSuffix` is empty.
+The version string follows the syntax of
+[Go module pseudo-version](https://go.dev/ref/mod#pseudo-versions), and consists
+of the following parts:
 
--   At development commits, `Version` contains the version of the next release.
-    `VersionSuffix` is set to `-dev.unknown`.
+-   `0.19.0-dev` identifies that this is a development build **before** the
+    0.19.0 release.
 
-The `VersionSuffix` variable can be overridden at build time, by passing
-`-ldflags "-X src.elv.sh/pkg/buildinfo.VersionSuffix=-foobar"` to `go build`,
-`go install` or `go get`. This is necessary in several scenarios, which are
-documented below.
+-   `.0` indicates that this is a pseudo-version, instead of a real version.
 
-### Packaging release versions
+-   `20220320172241` identifies the commit's creation time, in UTC.
 
-If you are using the standard Go toolchain and not applying any patches, there
-is nothing more to do; the default empty `VersionSuffix` suffices.
+-   `5dc8c02a32cf` is the 12-character prefix of the commit hash.
 
-If you are using a non-standard toolchain, or have applied any patches that can
-affect the resulting binary, you **must** override `VersionSuffix` with a string
-that starts with `+` and can uniquely identify your toolchain and patch. For
-official Linux distribution builds, this should identify your distribution, plus
-the version of the patch. Example:
+If that doesn't work for your build environment, the output of `elvish -version`
+will instead be:
 
 ```sh
-go build -ldflags "-X src.elv.sh/pkg/buildinfo.VersionSuffix=+deb1" ./cmd/elvish
+0.19.0-dev.unknown
 ```
 
-### Packaging development builds
-
-If you are packaging development builds, the default value of `VersionSuffix`,
-which is `-dev.unknown`, is likely not good enough, as it does not identify the
-commit Elvish is built from.
-
-You should override `VersionSuffix` with `-dev.$commit_hash`, where
-`$commit_hash` is the full commit hash, which can be obtained with
-`git rev-parse HEAD`. Example:
+If your build environment has the required information to build the
+pseudo-version string, you can supply it by overriding
+`src.elv.sh/pkg/buildinfo.VCSOverride` with the last two parts of the version
+string, commit's creation time and the 12-character prefix of the commit hash:
 
 ```sh
-go build -ldflags \
-  "-X src.elv.sh/pkg/buildinfo.VersionSuffix=-dev.$(git rev-parse HEAD)" \
-  ./cmd/elvish
+go build -ldflags '-X src.elv.sh/pkg/buildinfo.VCSOverride=20220320172241-5dc8c02a32cf' ./cmd/elvish
 ```
 
-If you have applied any patches that is not committed as a Git commit, you
-should also append a string that starts with `+` and can uniquely identify your
-patch.
-
-## Reproducible builds
-
-The idea of
-[reproducible build](https://en.wikipedia.org/wiki/Reproducible_builds) is that
-an Elvish binary from two different sources should be bit-to-bit identical, as
-long as they are built from the same version of the source code using the same
-version of the Go compiler.
-
-To make reproducible builds, you must do the following:
-
--   Pass `-trimpath` to the Go compiler.
-
--   For the following platforms, also pass `-buildmode=pie` to the Go compiler:
-
-    -   `GOOS=windows`, any `GOARCH`
+## Identifying the build variant
 
-    -   `GOOS=linux`, `GOARCH=amd64` or `GOARCH=arm64`
+You are encouraged to identify your build by overriding
+`src.elv.sh/pkg/buildinfo.BuildVariant` with something that identifies the
+distribution you are building for, and any patch level you have applied for
+Elvish. This will allow Elvish developers to easily identify any
+distribution-specific issue:
 
--   Disable cgo by setting the `CGO_ENABLED` environment variable to 0.
-
--   Follow the requirements above for putting
-    [accurate version information](#accurate-version-information) into the
-    binary, so that the user is able to uniquely identify the build by running
-    `elvish -version`.
+```
+go build -ldflags '-X src.elv.sh/pkg/buildinfo.BuildVariant=deb1' ./cmd/elvish
+```
 
-    The recommendation for how to set `VersionSuffix` when
-    [packaging development builds](#packaging-development-builds) becomes hard
-    requirements when packaging reproducible builds.
+## Official builds
 
-    In addition, if your distribution uses a patched version of the Go compiler
-    that changes its output, or if the build command uses any additional flags
-    (either via the command line or via any environment variables), you must
-    treat this as a patch on Elvish itself, and supply a version suffix
-    accordingly.
+A special build variant is `official`. This variant has a special meaning: the
+binary must be bit-by-bit identical to the official binaries, linked from
+https://elv.sh/get.
 
-If you follow these requirements when building Elvish, you can mark the build as
-a reproducible one by overriding `src.elv.sh/pkg/buildinfo.Reproducible` to
-`"true"`.
+The official binaries are built using the `tools/buildall.sh` script in the Git
+repo, using the docker image defined in https://github.com/elves/up. If you can
+fully mirror the environment **and** verify that the resulting binary is
+bit-by-bit identical to the official one, you can identify your build as
+`official`.
 
-Example when building a release version without any patches, on a platform where
-PIE is applicable:
+Reproducing the official binaries is completely optional. If your build setup is
+technically reproducible, but not identical with the official binaries, you can
+always use a distribution-specific variant, such as `deb1-reproducible`.
 
-```sh
-go build -buildmode=pie -trimpath \
-  -ldflags "-X src.elv.sh/pkg/buildinfo.Reproducible=true" \
-  ./cmd/elvish
-```
-
-Example when building a development version with a patch, on a platform where
-PIE is application:
-
-```sh
-go build -buildmode=pie -trimpath \
-  -ldflags "-X src.elv.sh/pkg/buildinfo.VersionSuffix=-dev.$(git rev-parse HEAD)+deb0 \
-            -X src.elv.sh/pkg/buildinfo.Reproducible=true" \
-  ./cmd/elvish
-```
+If you do want to reproduce the official binaries, realize that this is not a
+one-off configuration, but an ongoing commitment, since the environment for
+building the official binary will change over time (at a minimal, the Go version
+will be bumped from time to time). You must watch changes to them, update your
+build setup accordingly, and always verify that your build remains identical to
+official binaries.

README.md

@@ -23,14 +23,14 @@ User groups (all connected thanks to [Matrix](https://matrix.org)):
 Documentation for Elvish lives on the official website https://elv.sh,
 including:
 
--   [Learning material](https://elv.sh/learn);
+-   [Learning material](https://elv.sh/learn)
 
 -   [Reference docs](https://elv.sh/ref), including the
     [language reference](https://elv.sh/ref/language.html),
     [the `elvish` command](https://elv.sh/ref/command.html), and all the modules
-    in the standard library;
+    in the standard library
 
--   [Blog posts](https://elv.sh/blog), including release notes.
+-   [Blog posts](https://elv.sh/blog), including release notes
 
 The source for the documentation is in the
 [website](https://github.com/elves/elvish/tree/master/website) directory.
@@ -41,6 +41,7 @@ Most users do not need to build Elvish from source. Prebuilt binaries for the
 latest commit are provided for
 [Linux amd64](https://dl.elv.sh/linux-amd64/elvish-HEAD.tar.gz),
 [macOS amd64](https://dl.elv.sh/darwin-amd64/elvish-HEAD.tar.gz),
+[macOS arm64](https://dl.elv.sh/darwin-arm64/elvish-HEAD.tar.gz),
 [Windows amd64](https://dl.elv.sh/windows-amd64/elvish-HEAD.zip) and
 [many other platforms](https://elv.sh/get).
 
@@ -64,8 +65,8 @@ cd elvish
 make get
 ```
 
-This will install Elvish to `~/go/bin` (or `$GOPATH/bin` if you have set
-`$GOPATH`). You might want to add the directory to your `PATH`.
+This will install Elvish to `$GOBIN`, which defaults to `$GOPATH/bin` or
+`~/go/bin` if `$GOPATH` is not set.
 
 To install it elsewhere, override `ELVISH_MAKE_BIN` in the `make` command:
 
@@ -80,11 +81,11 @@ Elvish has experimental support for building and importing plugins, modules
 written in Go.
 
 However, since plugin support relies on dynamic linking, it is not enabled in
-the official prebuilt binaries. You need to build Elvish from source, with
-`ELVISH_PLUGIN_SUPPORT=1`:
+the official prebuilt binaries. You need to build Elvish from source, and make
+sure that CGo is enabled:
 
 ```sh
-make get ELVISH_PLUGIN_SUPPORT=1
+make get CGO_ENABLED=1
 ```
 
 To build a plugin, see this [example](https://github.com/elves/sample-plugin).

pkg/buildinfo/buildinfo.go

@@ -1,8 +1,7 @@
 // Package buildinfo contains build information.
 //
-// Build information should be set during compilation by passing
-// -ldflags "-X src.elv.sh/pkg/buildinfo.Var=value" to "go build" or
-// "go get".
+// Some of the exported fields may be set during compilation by passing -ldflags
+// "-X src.elv.sh/pkg/buildinfo.Var=value" to "go build".
 package buildinfo
 
 import (
@@ -10,36 +9,94 @@ import (
 	"fmt"
 	"os"
 	"runtime"
+	"runtime/debug"
+	"strings"
+	"time"
 
 	"src.elv.sh/pkg/prog"
 )
 
-// Version identifies the version of Elvish. On development commits, it
+// VersionBase identifies the version of Elvish. On the development branches, it
 // identifies the next release.
-const Version = "0.19.0"
+const VersionBase = "0.19.0"
 
-// VersionSuffix is appended to Version to build the full version string. It is public so it can be
-// overridden when building Elvish; see PACKAGING.md for details.
-var VersionSuffix = "-dev.unknown"
+// VCSOverride may be set during compilation to "time-commit" (e.g.
+// "20220320172241-5dc8c02a32cf") for identifying the version of development
+// builds.
+//
+// It is only needed if the automatic population of version information
+// implemented in devVersion fails.
+var VCSOverride string
 
-// Reproducible identifies whether the build is reproducible. This can be
-// overridden when building Elvish; see PACKAGING.md for details.
-var Reproducible = "false"
+// BuildVariant may be set during compilation to identify a particular
+// build variant, such as a build by a specific distribution, with modified
+// dependencies, or with a non-standard toolchain.
+//
+// If non-empty, it is appended to the version string, along with a "+" prefix.
+var BuildVariant string
 
 // Type contains all the build information fields.
 type Type struct {
-	Version      string `json:"version"`
-	Reproducible bool   `json:"reproducible"`
-	GoVersion    string `json:"goversion"`
+	Version   string `json:"version"`
+	GoVersion string `json:"goversion"`
 }
 
 func (Type) IsStructMap() {}
 
 // Value contains all the build information.
-var Value = Type{
-	Version:      Version + VersionSuffix,
-	Reproducible: Reproducible == "true",
-	GoVersion:    runtime.Version(),
+var Value Type
+
+func init() {
+	version := devVersion(VersionBase, VCSOverride, debug.ReadBuildInfo)
+	if BuildVariant != "" {
+		version += "+" + BuildVariant
+	}
+	Value = Type{
+		Version:   version,
+		GoVersion: runtime.Version(),
+	}
+}
+
+func devVersion(next, vcsOverride string, f func() (*debug.BuildInfo, bool)) string {
+	if vcsOverride != "" {
+		return next + "-dev.0." + vcsOverride
+	}
+	fallback := next + "-dev.unknown"
+	bi, ok := f()
+	if !ok {
+		return fallback
+	}
+	// If the main module's version is known, use it, but without the "v"
+	// prefix. This is the case when Elvish is built with "go install
+	// src.elv.sh/cmd/elvish@version".
+	if v := bi.Main.Version; v != "" && v != "(devel)" {
+		return strings.TrimPrefix(v, "v")
+	}
+	// If VCS information is available (i.e. when Elvish is built from a checked
+	// out repo), build the version string with it. Emulate the format of pseudo
+	// version (https://go.dev/ref/mod#pseudo-versions).
+	m := make(map[string]string)
+	for _, s := range bi.Settings {
+		if k := strings.TrimPrefix(s.Key, "vcs."); k != s.Key {
+			m[k] = s.Value
+		}
+	}
+	if m["revision"] == "" || m["time"] == "" || m["modified"] == "" {
+		return fallback
+	}
+	t, err := time.Parse(time.RFC3339Nano, m["time"])
+	if err != nil {
+		return fallback
+	}
+	revision := m["revision"]
+	if len(revision) > 12 {
+		revision = revision[:12]
+	}
+	version := fmt.Sprintf("%s-dev.0.%s-%s", next, t.Format("20060102150405"), revision)
+	if m["modified"] == "true" {
+		return version + "-dirty"
+	}
+	return version
 }
 
 // Program is the buildinfo subprogram.
@@ -64,7 +121,6 @@ func (p *Program) Run(fds [3]*os.File, _ []string) error {
 		} else {
 			fmt.Fprintln(fds[1], "Version:", Value.Version)
 			fmt.Fprintln(fds[1], "Go version:", Value.GoVersion)
-			fmt.Fprintln(fds[1], "Reproducible build:", Value.Reproducible)
 		}
 	case p.version:
 		if *p.json {

pkg/buildinfo/buildinfo_test.go

@@ -2,6 +2,7 @@ package buildinfo
 
 import (
 	"fmt"
+	"runtime/debug"
 	"testing"
 
 	. "src.elv.sh/pkg/prog/progtest"
@@ -14,10 +15,89 @@ func TestProgram(t *testing.T) {
 
 		ThatElvish("-buildinfo").WritesStdout(
 			fmt.Sprintf(
-				"Version: %v\nGo version: %v\nReproducible build: %v\n",
-				Value.Version, Value.GoVersion, Value.Reproducible)),
+				"Version: %v\nGo version: %v\n", Value.Version, Value.GoVersion)),
 		ThatElvish("-buildinfo", "-json").WritesStdout(mustToJSON(Value)+"\n"),
 
 		ThatElvish().ExitsWith(2).WritesStderr("internal error: no suitable subprogram\n"),
 	)
 }
+
+var devVersionTests = []struct {
+	name        string
+	vcsOverride string
+	bi          *debug.BuildInfo
+	want        string
+}{
+	// next is always "0.42.0"
+	{
+		"no BuildInfo",
+		"",
+		nil,
+		"0.42.0-dev.unknown",
+	},
+	{
+		"BuildInfo with Main.Version = (devel)",
+		"",
+		&debug.BuildInfo{Main: debug.Module{Version: "(devel)"}},
+		"0.42.0-dev.unknown",
+	},
+	{
+		"BuildInfo with non-empty Main.Version != (devel)",
+		"",
+		&debug.BuildInfo{Main: debug.Module{Version: "v0.42.0-dev.foobar"}},
+		"0.42.0-dev.foobar",
+	},
+	{
+		"BuildInfo with VCS data from clean checkout",
+		"",
+		&debug.BuildInfo{Settings: []debug.BuildSetting{
+			{Key: "vcs.revision", Value: "1234567890123456"},
+			{Key: "vcs.time", Value: "2022-04-01T23:59:58Z"},
+			{Key: "vcs.modified", Value: "false"},
+		}},
+		"0.42.0-dev.0.20220401235958-123456789012",
+	},
+	{
+		"BuildInfo with VCS data from dirty checkout",
+		"",
+		&debug.BuildInfo{Settings: []debug.BuildSetting{
+			{Key: "vcs.revision", Value: "1234567890123456"},
+			{Key: "vcs.time", Value: "2022-04-01T23:59:58Z"},
+			{Key: "vcs.modified", Value: "true"},
+		}},
+		"0.42.0-dev.0.20220401235958-123456789012-dirty",
+	},
+	{
+		"BuildInfo with unknown VCS timestamp format",
+		"",
+		&debug.BuildInfo{Settings: []debug.BuildSetting{
+			{Key: "vcs.revision", Value: "1234567890123456"},
+			{Key: "vcs.time", Value: "April First"},
+			{Key: "vcs.modified", Value: "false"},
+		}},
+		"0.42.0-dev.unknown",
+	},
+	{
+		"vcsOverride",
+		"20220401235958-123456789012",
+		nil,
+		"0.42.0-dev.0.20220401235958-123456789012",
+	},
+}
+
+func TestDevVersion(t *testing.T) {
+	for _, test := range devVersionTests {
+		t.Run(test.name, func(t *testing.T) {
+			f := func() (*debug.BuildInfo, bool) {
+				if test.bi == nil {
+					return nil, false
+				}
+				return test.bi, true
+			}
+			got := devVersion("0.42.0", test.vcsOverride, f)
+			if got != test.want {
+				t.Errorf("got %q, want %q", got, test.want)
+			}
+		})
+	}
+}

tools/buildall.sh

@@ -6,21 +6,15 @@
 # and building $DST_DIR/$GOOS-$GOARCH/elvish-$SUFFIX for each supported
 # combination of $GOOS and $GOARCH.
 #
-# It also creates and an archive for each binary file, and puts it in the same
+# It also creates an archive for each binary file, and puts it in the same
 # directory. For GOOS=windows, the archive is a .zip file. For all other GOOS,
 # the archive is a .tar.gz file.
 #
 # If the sha256sum command is available, this script also creates a sha256sum
 # file for each binary and archive file, and puts it in the same directory.
 #
-# The ELVISH_REPRODUCIBLE environment variable, if set, instructs the script to
-# mark the binary as a reproducible build. It must take one of the two following
-# values:
-#
-# - release: SRC_DIR must contain the source code for a tagged release.
-#
-# - dev: SRC_DIR must be a Git repository checked out from the latest master
-#        branch.
+# The value of the ELVISH_BUILD_VARIANT environment variable will be used to
+# override src.elv.sh/pkg/buildinfo.BuildVariant.
 #
 # This script is not whitespace-correct; avoid whitespaces in directory names.
 
@@ -39,18 +33,6 @@ SRC_DIR=$1
 DST_DIR=$2
 SUFFIX=$3
 
-LD_FLAGS=
-if test -n "$ELVISH_REPRODUCIBLE"; then
-    LD_FLAGS="-X src.elv.sh/pkg/buildinfo.Reproducible=true"
-    if test "$ELVISH_REPRODUCIBLE" = dev; then
-        LD_FLAGS="$LD_FLAGS -X src.elv.sh/pkg/buildinfo.VersionSuffix=-dev.$(git -C $SRC_DIR rev-parse HEAD)"
-    elif test "$ELVISH_REPRODUCIBLE" = release; then
-        : # nothing to do
-    else
-        echo "$ELVISH_REPRODUCIBLE must be 'dev' or 'release' when set"
-    fi
-fi
-
 export GOOS GOARCH GOFLAGS
 export CGO_ENABLED=0
 
@@ -94,7 +76,9 @@ buildone() {
     fi
 
     printf '%s' "Building for $GOOS-$GOARCH... "
-    go build -trimpath -ldflags "$LD_FLAGS"\
+    go build \
+      -trimpath \
+      -ldflags "-X src.elv.sh/pkg/buildinfo.BuildVariant=$ELVISH_BUILD_VARIANT" \
       -o $BIN_DIR/$BIN $SRC_DIR/cmd/elvish || {
         echo "Failed"
         return