Merge branch 'integrate-pr'

0.19.0-release-notes.md

@@ -56,3 +56,5 @@ and compiled, even if it is not executed:
 
 -   A new `compact` command that replaces consecutive runs of equal values with
     a single copy, similar to the Unix `uniq` command.
+
+-   A new `benchmark` command has been added ([#1586](https://b.elv.sh/1586)).

cmd/mvelvdoc/main.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+func main() {
+	for _, goFile := range os.Args[1:] {
+		bs, err := os.ReadFile(goFile)
+		handleErr("read file:", err)
+
+		var goLines, elvLines []string
+
+		lines := strings.Split(string(bs), "\n")
+		for i := 0; i < len(lines); i++ {
+			if !strings.HasPrefix(lines[i], "//elvdoc:") {
+				goLines = append(goLines, lines[i])
+				continue
+			}
+			if len(elvLines) > 0 {
+				elvLines = append(elvLines, "")
+			}
+			elvLines = append(elvLines, "#"+lines[i][2:])
+			i++
+			for i < len(lines) && strings.HasPrefix(lines[i], "//") {
+				elvLines = append(elvLines, "#"+lines[i][2:])
+				i++
+			}
+			i--
+		}
+
+		os.WriteFile(goFile, []byte(strings.Join(goLines, "\n")), 0o644)
+		if len(elvLines) > 0 {
+			elvFile := goFile[:len(goFile)-len(filepath.Ext(goFile))] + ".d.elv"
+			elvLines = append(elvLines, "")
+			os.WriteFile(elvFile, []byte(strings.Join(elvLines, "\n")), 0o644)
+		}
+	}
+}
+
+func handleErr(s string, err error) {
+	if err != nil {
+		log.Fatalln(s, err)
+	}
+}

pkg/elvdoc/elvdoc.go

@@ -0,0 +1,83 @@
+// Package elvdoc implements extraction of elvdoc, in-source documentation of
+// Elvish variables and functions.
+package elvdoc
+
+import (
+	"bufio"
+	"io"
+	"regexp"
+	"strings"
+)
+
+var (
+	// Groups:
+	// 1. Name
+	// 2. Signature (part inside ||)
+	fnRegexp = regexp.MustCompile(`^fn +([^ ]+) +\{(?:\|([^|]*)\|)?`)
+	// Groups:
+	// 1. Name
+	varRegexp = regexp.MustCompile(`^var +([^ ]+)`)
+)
+
+// Extract extracts elvdoc from Elvish source.
+func Extract(r io.Reader) (fnDocs, varDocs map[string]string, err error) {
+	fnDocs = make(map[string]string)
+	varDocs = make(map[string]string)
+
+	scanner := bufio.NewScanner(r)
+	var commentLines []string
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.HasPrefix(line, "# ") {
+			commentLines = append(commentLines, line)
+			continue
+		}
+		if m := fnRegexp.FindStringSubmatch(line); m != nil {
+			name, sig := m[1], m[2]
+			var sb strings.Builder
+			writeUsage(&sb, name, sig)
+			if len(commentLines) > 0 {
+				sb.WriteByte('\n')
+				writeCommentContent(&sb, commentLines)
+			}
+			fnDocs[name] = sb.String()
+		} else if m := varRegexp.FindStringSubmatch(line); m != nil {
+			name := m[1]
+			var sb strings.Builder
+			writeCommentContent(&sb, commentLines)
+			varDocs[name] = sb.String()
+		}
+		commentLines = commentLines[:0]
+	}
+
+	return fnDocs, varDocs, scanner.Err()
+}
+
+func writeUsage(sb *strings.Builder, name, sig string) {
+	sb.WriteString("```elvish\n")
+	sb.WriteString(name)
+	for _, field := range strings.Fields(sig) {
+		sb.WriteByte(' ')
+		if strings.HasPrefix(field, "&") {
+			sb.WriteString(field)
+		} else if strings.HasPrefix(field, "@") {
+			sb.WriteString("$" + field[1:] + "...")
+		} else {
+			sb.WriteString("$" + field)
+		}
+	}
+	sb.WriteString("\n```\n")
+}
+
+func writeCommentContent(sb *strings.Builder, lines []string) string {
+	for _, line := range lines {
+		// Every line starts with "# "
+		sb.WriteString(line[2:])
+		sb.WriteByte('\n')
+	}
+	return sb.String()
+}
+
+func Format(r io.Reader, w io.Writer) error {
+	return nil
+}

pkg/eval/builtin_fn_misc.go

@@ -5,7 +5,10 @@ package eval
 import (
 	"errors"
 	"fmt"
+	"math"
+	"math/big"
 	"net"
+	"strconv"
 	"sync"
 	"time"
 
@@ -36,8 +39,9 @@ func init() {
 		"deprecate": deprecate,
 
 		// Time
-		"sleep": sleep,
-		"time":  timeCmd,
+		"sleep":     sleep,
+		"time":      timeCmd,
+		"benchmark": benchmark,
 
 		"-ifaddrs": _ifaddrs,
 	})
@@ -500,6 +504,8 @@ func sleep(fm *Frame, duration any) error {
 // ~> put $t
 // ▶ (num 0.011030208)
 // ```
+//
+// @cf benchmark
 
 type timeOpt struct{ OnEnd Callable }
 
@@ -527,6 +533,193 @@ func timeCmd(fm *Frame, opts timeOpt, f Callable) error {
 	return err
 }
 
+//elvdoc:fn benchmark
+//
+// ```elvish
+// benchmark &min-runs=5 &min-time=1s &on-end=$nil &on-run-end=$nil $callable
+// ```
+//
+// Runs `$callable` repeatedly, and reports statistics about how long each run
+// takes.
+//
+// If the `&on-end` callback is not given, `benchmark` prints the average,
+// standard deviation, minimum and maximum of the time it took to run
+// `$callback`, and the number of runs. If the `&on-end` callback is given,
+// `benchmark` instead calls it with a map containing these metrics, keyed by
+// `avg`, `stddev`, `min`, `max` and `runs`. Each duration value (i.e. all
+// except `runs`) is given as the number of seconds.
+//
+// The number of runs is controlled by `&min-runs` and `&min-time`. The
+// `$callable` is run at least `&min-runs` times, **and** when the total
+// duration is at least `&min-time`.
+//
+// The `&min-runs` option must be a non-negative integer within the range of the
+// machine word.
+//
+// The `&min-time` option must be a string representing a non-negative duration,
+// specified as a sequence of decimal numbers with a unit suffix (the numbers
+// may have fractional parts), such as "300ms", "1.5h" and "1h45m7s". Valid time
+// units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
+//
+// If `&on-run-end` is given, it is called after each call to `$callable`, with
+// the time that call took, given as the number of seconds.
+//
+// If `$callable` throws an exception, `benchmark` terminates and propagates the
+// exception after the `&on-end` callback (or the default printing behavior)
+// finishes. The duration of the call that throws an exception is not passed to
+// `&on-run-end`, nor is it included when calculating the statistics for
+// `&on-end`. If the first call to `$callable` throws an exception and `&on-end`
+// is `$nil`, nothing is printed and any `&on-end` callback is not called.
+//
+// If `&on-run-end` is given and throws an exception, `benchmark` terminates and
+// propagates the exception after the `&on-end` callback (or the default
+// printing behavior) finishes, unless `$callable` has already thrown an
+// exception
+//
+// If `&on-end` throws an exception, the exception is propagated, unless
+// `$callable` or `&on-run-end` has already thrown an exception.
+//
+// Example:
+//
+// ```elvish-transcript
+// ~> benchmark { }
+// 98ns ± 382ns (min 0s, max 210.417µs, 10119226 runs)
+// ~> benchmark &on-end={|m| put $m[avg]} { }
+// ▶ (num 9.8e-08)
+// ~> benchmark &on-run-end={|d| echo $d} { sleep 0.3 }
+// 0.301123625
+// 0.30123775
+// 0.30119075
+// 0.300629166
+// 0.301260333
+// 301.088324ms ± 234.298µs (min 300.629166ms, max 301.260333ms, 5 runs)
+// ```
+//
+// @cf time
+
+type benchmarkOpts struct {
+	OnEnd    Callable
+	OnRunEnd Callable
+	MinRuns  int
+	MinTime  string
+	minTime  time.Duration
+}
+
+func (o *benchmarkOpts) SetDefaultOptions() {
+	o.MinRuns = 5
+	o.minTime = time.Second
+}
+
+func (opts *benchmarkOpts) parse() error {
+	if opts.MinRuns < 0 {
+		return errs.BadValue{What: "min-runs option",
+			Valid: "non-negative integer", Actual: strconv.Itoa(opts.MinRuns)}
+	}
+
+	if opts.MinTime != "" {
+		d, err := time.ParseDuration(opts.MinTime)
+		if err != nil {
+			return errs.BadValue{What: "min-time option",
+				Valid: "duration string", Actual: parse.Quote(opts.MinTime)}
+		}
+		if d < 0 {
+			return errs.BadValue{What: "min-time option",
+				Valid: "non-negative duration", Actual: parse.Quote(opts.MinTime)}
+		}
+		opts.minTime = d
+	}
+
+	return nil
+}
+
+// TimeNow is a reference to [time.Now] that can be overridden in tests.
+var TimeNow = time.Now
+
+func benchmark(fm *Frame, opts benchmarkOpts, f Callable) error {
+	if err := opts.parse(); err != nil {
+		return err
+	}
+
+	// Standard deviation is calculated using https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm
+	var (
+		min   = time.Duration(math.MaxInt64)
+		max   = time.Duration(math.MinInt64)
+		runs  int64
+		total time.Duration
+		m2    float64
+		err   error
+	)
+	for {
+		t0 := TimeNow()
+		err = f.Call(fm, NoArgs, NoOpts)
+		if err != nil {
+			break
+		}
+		dt := TimeNow().Sub(t0)
+
+		if min > dt {
+			min = dt
+		}
+		if max < dt {
+			max = dt
+		}
+		var oldDelta float64
+		if runs > 0 {
+			oldDelta = float64(dt) - float64(total)/float64(runs)
+		}
+		runs++
+		total += dt
+		if runs > 0 {
+			newDelta := float64(dt) - float64(total)/float64(runs)
+			m2 += oldDelta * newDelta
+		}
+
+		if opts.OnRunEnd != nil {
+			newFm := fm.Fork("on-run-end callback of benchmark")
+			err = opts.OnRunEnd.Call(newFm, []any{dt.Seconds()}, NoOpts)
+			if err != nil {
+				break
+			}
+		}
+
+		if runs >= int64(opts.MinRuns) && total >= opts.minTime {
+			break
+		}
+	}
+
+	if runs == 0 {
+		return err
+	}
+
+	avg := total / time.Duration(runs)
+	stddev := time.Duration(math.Sqrt(m2 / float64(runs)))
+	if opts.OnEnd == nil {
+		_, errOut := fmt.Fprintf(fm.ByteOutput(),
+			"%v ± %v (min %v, max %v, %d runs)\n", avg, stddev, min, max, runs)
+		if err == nil {
+			err = errOut
+		}
+	} else {
+		stats := vals.MakeMap(
+			"avg", avg.Seconds(), "stddev", stddev.Seconds(),
+			"min", min.Seconds(), "max", max.Seconds(), "runs", int64ToElv(runs))
+		newFm := fm.Fork("on-end callback of benchmark")
+		errOnEnd := opts.OnEnd.Call(newFm, []any{stats}, NoOpts)
+		if err == nil {
+			err = errOnEnd
+		}
+	}
+	return err
+}
+
+func int64ToElv(i int64) any {
+	if i <= int64(math.MaxInt) {
+		return int(i)
+	} else {
+		return big.NewInt(i)
+	}
+}
+
 //elvdoc:fn -ifaddrs
 //
 // ```elvish

pkg/eval/builtin_fn_misc_test.go

@@ -100,6 +100,96 @@ func TestTime(t *testing.T) {
 	)
 }
 
+func TestBenchmark(t *testing.T) {
+	var ticks []int64
+	testutil.Set(t, &TimeNow, func() time.Time {
+		if len(ticks) == 0 {
+			panic("mock TimeNow called more than len(ticks)")
+		}
+		v := ticks[0]
+		ticks = ticks[1:]
+		return time.Unix(v, 0)
+	})
+	setupTicks := func(ts ...int64) func(*Evaler) {
+		return func(_ *Evaler) { ticks = ts }
+	}
+
+	Test(t,
+		// Default output
+		That("benchmark &min-runs=2 &min-time=2s { }").
+			WithSetup(setupTicks(0, 1, 1, 3)).
+			Prints("1.5s ± 500ms (min 1s, max 2s, 2 runs)\n"),
+		// &on-end callback
+		That(
+			"var f = {|m| put $m[avg] $m[stddev] $m[min] $m[max] $m[runs]}",
+			"benchmark &min-runs=2 &min-time=2s &on-end=$f { }").
+			WithSetup(setupTicks(0, 1, 1, 3)).
+			Puts(1.5, 0.5, 1.0, 2.0, 2),
+
+		// &min-runs determining number of runs
+		That("benchmark &min-runs=4 &min-time=0s &on-end={|m| put $m[runs]} { }").
+			WithSetup(setupTicks(0, 1, 1, 3, 3, 4, 4, 6)).
+			Puts(4),
+		// &min-time determining number of runs
+		That("benchmark &min-runs=0 &min-time=10s &on-end={|m| put $m[runs]} { }").
+			WithSetup(setupTicks(0, 1, 1, 6, 6, 11)).
+			Puts(3),
+
+		// &on-run-end
+		That("benchmark &min-runs=3 &on-run-end=$put~ &on-end={|m| } { }").
+			WithSetup(setupTicks(0, 1, 1, 3, 3, 4)).
+			Puts(1.0, 2.0, 1.0),
+
+		// $callable throws exception
+		That(
+			"var i = 0",
+			"benchmark { set i = (+ $i 1); if (== $i 3) { fail failure } }").
+			WithSetup(setupTicks(0, 1, 1, 3, 3)).
+			Throws(FailError{"failure"}).
+			Prints("1.5s ± 500ms (min 1s, max 2s, 2 runs)\n"),
+		// $callable throws exception on first run
+		That("benchmark { fail failure }").
+			WithSetup(setupTicks(0)).
+			Throws(FailError{"failure"}).
+			Prints( /* nothing */ ""),
+		That("benchmark &on-end=$put~ { fail failure }").
+			WithSetup(setupTicks(0)).
+			Throws(FailError{"failure"}).
+			Puts( /* nothing */ ),
+
+		// &on-run-end throws exception
+		That("benchmark &on-run-end={|_| fail failure } { }").
+			WithSetup(setupTicks(0, 1)).
+			Throws(FailError{"failure"}).
+			Prints("1s ± 0s (min 1s, max 1s, 1 runs)\n"),
+
+		// &on-run throws exception
+		That("benchmark &min-runs=2 &min-time=0s &on-end={|_| fail failure } { }").
+			WithSetup(setupTicks(0, 1, 1, 3)).
+			Throws(FailError{"failure"}),
+
+		// Option errors
+		That("benchmark &min-runs=-1 { }").
+			Throws(errs.BadValue{What: "min-runs option",
+				Valid: "non-negative integer", Actual: "-1"}),
+		That("benchmark &min-time=abc { }").
+			Throws(errs.BadValue{What: "min-time option",
+				Valid: "duration string", Actual: "abc"}),
+		That("benchmark &min-time=-1s { }").
+			Throws(errs.BadValue{What: "min-time option",
+				Valid: "non-negative duration", Actual: "-1s"}),
+
+		// Test that output error is bubbled. We can't use
+		// testOutputErrorIsBubbled here, since the mock TimeNow requires setup.
+		That("benchmark &min-runs=0 &min-time=0s { } >&-").
+			WithSetup(setupTicks(0, 1)).
+			Throws(os.ErrInvalid),
+		That("benchmark &min-runs=0 &min-time=0s &on-end=$put~ { } >&-").
+			WithSetup(setupTicks(0, 1)).
+			Throws(ErrPortDoesNotSupportValueOutput),
+	)
+}
+
 func TestUseMod(t *testing.T) {
 	testutil.InTempDir(t)
 	must.WriteFile("mod.elv", "var x = value")

pkg/eval/errs/errs.go

@@ -18,10 +18,6 @@ type OutOfRange struct {
 
 // Error implements the error interface.
 func (e OutOfRange) Error() string {
-	if e.ValidHigh < e.ValidLow {
-		return fmt.Sprintf(
-			"out of range: %v has no valid value, but is %v", e.What, e.Actual)
-	}
 	return fmt.Sprintf(
 		"out of range: %s must be from %s to %s, but is %s",
 		e.What, e.ValidLow, e.ValidHigh, e.Actual)

pkg/eval/errs/errs_test.go

@@ -12,10 +12,6 @@ var errorMessageTests = []struct {
 		OutOfRange{What: "list index here", ValidLow: "0", ValidHigh: "2", Actual: "3"},
 		"out of range: list index here must be from 0 to 2, but is 3",
 	},
-	{
-		OutOfRange{What: "list index here", ValidLow: "1", ValidHigh: "0", Actual: "0"},
-		"out of range: list index here has no valid value, but is 0",
-	},
 	{
 		BadValue{What: "command", Valid: "callable", Actual: "number"},
 		"bad value: command must be callable, but is number",

pkg/eval/options.go

@@ -37,6 +37,16 @@ func scanOptions(rawOpts RawOptions, ptr any) error {
 		if !ok {
 			return UnknownOption{k}
 		}
+
+		// An option with no value (e.g., `&a-opt`) has `$true` as its default value. However, if
+		// the option struct member is a string we want an empty string as the default value.
+		switch b := v.(type) {
+		case bool:
+			if b && structValue.Field(fieldIdx).Type().Name() == "string" {
+				v = ""
+			}
+		}
+
 		err := vals.ScanToGo(v, structValue.Field(fieldIdx).Addr().Interface())
 		if err != nil {
 			return err