cue/syntax: cue.Attributes(true) not showing all the attributes in cue values

[FogDong]

What version of CUE are you using (cue version)?

$ cue version
master

Does this issue reproduce with the latest release?

yes

What did you do?

var test = `
step1: {} @step(1)
if true {
	step2: {} @step(2)
}
step3: {} @step(3)`

func main(){
        v := cuecontext.New().CompileString(test)
	node := v.Syntax(cue.Attributes(true))
	formatting, err := format.Node(node)
	if err != nil {
		panic(err)
	}
	fmt.Println(string(formatting))
}

What did you expect to see?

{
        step1: {} @step(1)
        if true {
                step2: {} @step(2)
        }
        step3: {} @step(3)
}

What did you see instead?

{
        step1: {}
        if true {
                step2: {} @step(2)
        }
        step3: {}
}

It's strange that only the attributes in comprehension can be shown, for your information, the result in v0.2.2 is like:

step1: {} @step(1)
step2: {} @step(2)
step3: {} @step(3)

I also checked the attributes in the node, it turns out that the attributes remain in the node but are lost in the result.

[myitcv]

Investigating now.

[myitcv]

Thanks for the report @FogDong. There appear to be a few issues going on here. Running the following against current tip (5a08d2f7a9a7):

# v0.2.2 go get
go get cuelang.org/[email protected]
go mod tidy

# v0.2.2 cue def -A
go run cuelang.org/go/cmd/cue def -A
stdout 'step1.*step\(1\)'
stdout 'step2.*step\(2\)'
stdout 'step3.*step\(3\)'

# v0.2.2 cue eval -A
go run cuelang.org/go/cmd/cue eval -A
stdout 'step1.*step\(1\)'
stdout 'step2.*step\(2\)'
stdout 'step3.*step\(3\)'

# v0.2.2 API
go run .
stdout 'step1.*step\(1\)'
stdout 'step2.*step\(2\)'
stdout 'step3.*step\(3\)'

# tip go get
go get cuelang.org/[email protected]
go mod tidy

# # tip cue def -A
# go run cuelang.org/go/cmd/cue def -A
# stdout 'step1.*step\(1\)'
# stdout 'step2.*step\(2\)'
# stdout 'step3.*step\(3\)'

# # tip cue eval -A
# go run cuelang.org/go/cmd/cue eval -A
# stdout 'step1.*step\(1\)'
# stdout 'step2.*step\(2\)'
# stdout 'step3.*step\(3\)'

# tip API
go run .
stdout 'step1.*step\(1\)'
stdout 'step2.*step\(2\)'
stdout 'step3.*step\(3\)'

-- go.mod --
module mod.com

go 1.18

require cuelang.org/go v0.2.2

-- main.go --
package main

import (
	"fmt"

	"cuelang.org/go/cue"
	"cuelang.org/go/cue/format"
	"cuelang.org/go/cue/load"
)

var test = `
step1: {} @step(1)
if true {
	step2: {} @step(2)
}
step3: {} @step(3)`

func main() {
	bps := load.Instances([]string{"."}, nil)
	v := cue.Build(bps)[0].Value()
	node := v.Syntax(cue.Attributes(true))
	formatting, err := format.Node(node)
	if err != nil {
		panic(err)
	}
	fmt.Println(string(formatting))
}
-- tools.go --
// +build tools

package tools

import _ "cuelang.org/go/cmd/cue"
-- x.cue --
package x

step1: {} @step(1)
if true {
	step2: {} @step(2)
}
step3: {} @step(3)

I would expect this to pass.

However I get something like:

# v0.2.2 go get (0.219s)
# v0.2.2 cue def -A (0.314s)
# v0.2.2 cue eval -A (0.317s)
# v0.2.2 API (0.206s)
# tip go get (0.148s)
# tip cue def -A (0.378s)
> go run cuelang.org/go/cmd/cue def -A
[stdout]
package x

step1: {}
if true {
        step2: {} @step(2)
}
step3: {}

> stdout 'step1.*step\(1\)'
FAIL: /tmp/testscript774895454/repro.txtar/script.txt:29: no match for `step1.*step\(1\)` found in stdout

# tip cue eval -A (0.376s)
> go run cuelang.org/go/cmd/cue eval -A
[stdout]
step1: {} @step(1)
step2: {}
step3: {} @step(3)

> stdout 'step1.*step\(1\)'
> stdout 'step2.*step\(2\)'
FAIL: /tmp/testscript818267885/repro.txtar/script.txt:36: no match for `step2.*step\(2\)` found in stdout

# tip API (0.296s)
> go run .
[stdout]
package x

step1: {}
if true {
        step2: {} @step(2)
}
step3: {}


> stdout 'step1.*step\(1\)'
FAIL: /tmp/testscript1659630887/repro.txtar/script.txt:41: no match for `step1.*step\(1\)` found in stdout

(strictly speaking I'm combining the results of three runs... but because testscript fails on the first error, I'm presenting the output as if it continued executing the script instead of aborting).

My analysis so far shows that with this input:

• cue def -A has been failing since 7f52c107b61980629c4c0d6751386a8ce360ddb9
• cue eval -A has been failing since 748a68589747fed6273d6338eb7e275a4fb116c6
• The Go API test has been failing since 845df056ca4facf04a7a844238d3f3da0d4dbe99

This definitely needs a fix or two or three.

[FogDong]

Thanks for the investigation. @myitcv

If this issue is fixed, can I get the attributes in:

go mod tidy
go run main.go
-- go.mod --
module test-cue/attr-1826

go 1.17

require cuelang.org/go v0.4.4-0.20220801114602-5a08d2f7a9a7

require (
	github.com/cockroachdb/apd/v2 v2.0.1 // indirect
	github.com/emicklei/proto v1.10.0 // indirect
	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
	github.com/google/uuid v1.2.0 // indirect
	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
	github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de // indirect
	github.com/pkg/errors v0.8.1 // indirect
	github.com/protocolbuffers/txtpbfmt v0.0.0-20220428173112-74888fd59c2b // indirect
	golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect
	golang.org/x/text v0.3.7 // indirect
	gopkg.in/yaml.v3 v3.0.1 // indirect
)
-- main.go --
package main

import (
	"fmt"

	"cuelang.org/go/cue"
	"cuelang.org/go/cue/cuecontext"
	"cuelang.org/go/cue/load"
)

func main() {
	ctx := cuecontext.New()
	bps := load.Instances([]string{"."}, nil)
	v := ctx.BuildInstance(bps[0])
	if err := showAttr(v); err != nil {
		panic(err)
	}
}

func showAttr(v cue.Value) error {
	st, err := v.Struct()
	if err != nil {
		return err
	}

	for i := 0; i < st.Len(); i++ {
		name := st.Field(i).Name
		attr := st.Field(i).Value.Attribute("step")
		fmt.Printf("name: %s, attr: %v \n", name, attr)
	}
	return nil
}
-- repro.txt --
-- stdout --
name: step1, attr: @step(1) 
name: step2, attr: @step() 
name: step3, attr: @step(3) 
-- stdout.golden --
name: step1, attr: @step(1) 
name: step2, attr: @step(2) 
name: step3, attr: @step(3) 
-- x.cue --
package x

step1: {} @step(1)
if true {
	step2: {} @step(2)
}
step3: {} @step(3)

[myitcv]

@FogDong your repro does not run for me. It gives:

> go mod tidy
[stderr]
test-cue/attr-1826 imports
        cuelang.org/go/cue/cuecontext imports
        cuelang.org/go/pkg imports
        cuelang.org/go/pkg/encoding/yaml imports
        cuelang.org/go/internal/encoding/yaml imports
        gopkg.in/yaml.v3 tested by
        gopkg.in/yaml.v3.test imports
        gopkg.in/check.v1 loaded from gopkg.in/[email protected],
        but go 1.16 would select v1.0.0-20180628173108-788fd7840127

To upgrade to the versions selected by go 1.16:
        go mod tidy -go=1.16 && go mod tidy -go=1.17
If reproducibility with go 1.16 is not needed:
        go mod tidy -compat=1.17
For other options, see:
        https://golang.org/doc/modules/pruning

[exit status 1]
FAIL: /tmp/testscript2233550218/repro.txtar/script.txt:1: unexpected go command failure

Does it run for you, using cmd/testscript?

The repro is also missing expectations; it looks like you were intending a comparison with stdout from the output of go run?

I've created a version which I assume to be what you intended:

go mod tidy
go run main.go
cmp stdout stdout.golden

-- go.mod --
module test-cue/attr-1826

go 1.18

require cuelang.org/go v0.4.4-0.20220801114602-5a08d2f7a9a7

-- main.go --
package main

import (
	"fmt"

	"cuelang.org/go/cue"
	"cuelang.org/go/cue/cuecontext"
	"cuelang.org/go/cue/load"
)

func main() {
	ctx := cuecontext.New()
	bps := load.Instances([]string{"."}, nil)
	v := ctx.BuildInstance(bps[0])
	if err := showAttr(v); err != nil {
		panic(err)
	}
}

func showAttr(v cue.Value) error {
	st, err := v.Struct()
	if err != nil {
		return err
	}

	for i := 0; i < st.Len(); i++ {
		name := st.Field(i).Name
		attr := st.Field(i).Value.Attribute("step")
		fmt.Printf("name: %s, attr: %v \n", name, attr)
	}
	return nil
}
-- x.cue --
package x

step1: {} @step(1)
if true {
	step2: {} @step(2)
}
step3: {} @step(3)
-- stdout.golden --
name: step1, attr: @step(1)
name: step2, attr: @step(2)
name: step3, attr: @step(3)

Your expectation is that this should pass with current tip, correct?

The current behaviour is:

> go mod tidy
> go run main.go
[stdout]
name: step1, attr: @step(1)
name: step2, attr: @step()
name: step3, attr: @step(3)

> cmp stdout stdout.golden
--- stdout
+++ stdout.golden
@@ -1,3 +0,0 @@
-name: step1, attr: @step(1)
-name: step2, attr: @step()
-name: step3, attr: @step(3)
@@ -0,0 +1,3 @@
+name: step1, attr: @step(1)
+name: step2, attr: @step(2)
+name: step3, attr: @step(3)

FAIL: /tmp/testscript2229779254/repro.txtar/script.txt:3: stdout and stdout.golden differ

Please confirm.

[FogDong]

Hi Paul, @myitcv

Yes your repro is correct, thanks for the fixing.

[myitcv]

Thanks for confirming, @FogDong. Updating the issue to use non-deprecated APIs, and handling the errors in any returned attributes. We can actually broaden this slightly to show a few more issues:

go mod tidy
go run .
cmp stdout stdout.golden

-- go.mod --
module mod.com

go 1.18

require cuelang.org/go v0.4.4-0.20220808083940-b4d1b16ccf89

-- main.cue --
package x

step1: int @step(1)
if true {
	step2: int @step(2)
}
for _ in [1] {
	step3: int @step(3)
}
step4: int @step(4)
-- main.go --
package main

import (
	"fmt"

	"cuelang.org/go/cue"
	"cuelang.org/go/cue/load"
)

func main() {
	bps := load.Instances([]string{"."}, nil)
	v := cue.Build(bps)[0].Value()
	iter, err := v.Fields()
	if err != nil {
		panic(err)
	}
	for iter.Next() {
		name := iter.Label()
		attr := iter.Value().Attribute("step")
		if err := attr.Err(); err != nil {
			fmt.Printf("name: %s does not have attr step: %v\n", name, err)
		} else {
			sval, err := attr.String(0)
			if err != nil {
				panic(err)
			}
			fmt.Printf("name: %s, attr: %v\n", name, sval)
		}
	}
}
-- stdout.golden --
name: step1, attr: 1
name: step2, attr: 2
name: step3, attr: 3
name: step4, attr: 4

The expectation is that this test should pass, however it gives:

> go mod tidy
> go run .
[stdout]
name: step1, attr: 1
name: step2 does not have attr step: attribute "step" does not exist
name: step3 does not have attr step: attribute "step" does not exist
name: step4, attr: 4

> cmp stdout stdout.golden
--- stdout
+++ stdout.golden
@@ -1,4 +1,4 @@
 name: step1, attr: 1
-name: step2 does not have attr step: attribute "step" does not exist
-name: step3 does not have attr step: attribute "step" does not exist
+name: step2, attr: 2
+name: step3, attr: 3
 name: step4, attr: 4

FAIL: /tmp/testscript2211182583/repro.txtar/script.txt:3: stdout and stdout.golden differ

I've bisected this to 748a68589747fed6273d6338eb7e275a4fb116c6.

Another apparent bug is that cue.Value.Source() retains all the attributes, but cue.Value.Syntax() does not:

# source
go mod tidy
go run . source
cmp stdout stdout.golden

# syntax
go mod tidy
go run . syntax
cmp stdout stdout.golden

-- go.mod --
module mod.com

go 1.18

require cuelang.org/go v0.4.4-0.20220808083940-b4d1b16ccf89

-- main.cue --
package x

step1: int @step(1)
if true {
	step2: int @step(2)
}
for _ in [1] {
	step3: int @step(3)
}
step4: int @step(4)
-- main.go --
package main

import (
	"fmt"
	"log"
	"os"

	"cuelang.org/go/cue"
	"cuelang.org/go/cue/ast"
	"cuelang.org/go/cue/format"
	"cuelang.org/go/cue/load"
)

func main() {
	bps := load.Instances([]string{"."}, nil)
	v := cue.Build(bps)[0].Value()
	var n ast.Node
        switch os.Args[1] {
	case "source": n = v.Source()
	case "syntax": n = v.Syntax()
        }
	b, err := format.Node(n)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("%s", b)
}
-- stdout.golden --
package x

step1: int @step(1)
if true {
	step2: int @step(2)
}
for _ in [1] {
	step3: int @step(3)
}
step4: int @step(4)

Again, the expectation is that this should pass but instead it gives:

# source (0.344s)
# syntax (0.305s)
> go mod tidy
> go run . syntax
[stdout]
package x

step1: int
if true {
        step2: int @step(2)
}
for _ in [1] {
        step3: int @step(3)
}
step4: int

> cmp stdout stdout.golden
--- stdout
+++ stdout.golden
@@ -1,10 +1,10 @@
 package x

-step1: int
+step1: int @step(1)
 if true {
        step2: int @step(2)
 }
 for _ in [1] {
        step3: int @step(3)
 }
-step4: int
+step4: int @step(4)

FAIL: /tmp/testscript3208190997/repro.txtar/script.txt:9: stdout and stdout.golden differ