diff --git a/.github/workflows/gen.yml b/.github/workflows/gen.yml index 27329d00eb..c3b6bdd4ad 100644 --- a/.github/workflows/gen.yml +++ b/.github/workflows/gen.yml @@ -1,13 +1,13 @@ -name: sqlc-pg-gen +name: gen on: workflow_dispatch: jobs: - gen: - name: sqlc-pg-gen - runs-on: ubuntu-22.04 + generate: + name: generate dialects + runs-on: ubuntu-24.04 services: postgres: - image: postgres:15.0-alpine + image: postgres:16 env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres @@ -20,20 +20,18 @@ jobs: - uses: actions/checkout@v7 - uses: actions/setup-go@v7 with: - go-version-file: go.mod + go-version-file: internal/goldeneye/go.mod check-latest: true - - run: go build -o sqlc-pg-gen ./internal/tools/sqlc-pg-gen - - run: mkdir -p gen/contrib - - run: ./sqlc-pg-gen gen + - run: go run ./cmd/goldeneye install clickhouse + working-directory: internal/goldeneye + - run: go run ./cmd/goldeneye generate + working-directory: internal/goldeneye env: - PG_USER: postgres - PG_HOST: localhost - PG_DATABASE: postgres - PG_PASSWORD: postgres - PG_PORT: ${{ job.services.postgres.ports['5432'] }} + POSTGRESQL_SERVER_URI: postgres://postgres:postgres@localhost:${{ job.services.postgres.ports['5432'] }}/postgres?sslmode=disable - name: Save results uses: actions/upload-artifact@v7 with: - name: sqlc-pg-gen-results - path: gen - + name: dialects + path: internal/engine/*/dialect + - name: Fail if the committed dialects differ + run: git add -N internal/engine && git diff --exit-code --stat -- internal/engine diff --git a/CLAUDE.md b/CLAUDE.md index 3059988024..dfce6cfb08 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -143,6 +143,24 @@ SQLC_TEST_CORE=1 go test ./internal/endtoend -run 'TestReplay/core' Go aborts a test binary on panic, so a case that panics the core analyzer ends the run early. Run a subset to get past one (`-run 'TestReplay/core/^select'`). +### Dialect Checks + +The dialect seeds under `/internal/engine//dialect/` are generated +from a live database by `/internal/goldeneye`, a nested module, and its tests +verify the committed files against one byte for byte. Engines whose database +is not available skip. + +```bash +cd internal/goldeneye +go run ./cmd/goldeneye install clickhouse # download the pinned clickhouse binary once +POSTGRESQL_SERVER_URI="postgres://postgres:postgres@127.0.0.1:5432/postgres?sslmode=disable" go test ./... +go run ./cmd/goldeneye generate postgresql # rewrite the files after a change +``` + +The checks are not part of CI. The `gen` workflow +(`.github/workflows/gen.yml`) generates the files on demand and uploads +them as an artifact. + ### Example Tests - **Location:** `/examples/` directory @@ -216,10 +234,13 @@ MYSQL_SERVER_URI="root:mysecretpassword@tcp(127.0.0.1:3306)/mysql?multiStatement - `/dolphin/` - MySQL parser (uses TiDB parser) - `/sqlite/` - SQLite parser - `/duckdb/` - DuckDB 2.0 parser (uses darkwing, the pure Go port of - DuckDB's PEG parser); its dialect seeds are generated by - `/internal/tools/sqlc-duckdb-gen` from a live DuckDB CLI + DuckDB's PEG parser) - `/dialect/` - The engine's type system and standard library, as - JSONL read by `/internal/core/seed` + JSONL read by `/internal/core/seed`; the generated parts come from + `/internal/goldeneye` +- `/internal/goldeneye/` - Nested module that generates the dialect seeds + under `/internal/engine//dialect/` from a live database and checks + the committed ones against it, one package per engine; see its README - `/internal/core/` - The analysis core: catalog, analyzer and dialect seeds - `/internal/compiler/` - Query compilation logic - `/internal/codegen/` - Code generation for different languages diff --git a/Makefile b/Makefile index b8745e57dc..52416e1f2a 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build build-endtoend test test-ci test-examples test-endtoend start psql mysqlsh proto +.PHONY: build build-endtoend test test-ci test-examples test-endtoend test-goldeneye start psql mysqlsh proto build: go build ./... @@ -26,8 +26,11 @@ test-ci: test-examples build-endtoend vet sqlc-dev: go build -o ~/bin/sqlc-dev ./cmd/sqlc/ -sqlc-pg-gen: - go build -o ~/bin/sqlc-pg-gen ./internal/tools/sqlc-pg-gen +goldeneye: + cd ./internal/goldeneye && go build -o ~/bin/goldeneye ./cmd/goldeneye + +test-goldeneye: + cd ./internal/goldeneye && go test ./... sqlc-gen-json: go build -o ~/bin/sqlc-gen-json ./cmd/sqlc-gen-json diff --git a/internal/engine/clickhouse/dialect/types.jsonl b/internal/engine/clickhouse/dialect/types.jsonl index 8ad86d1fde..eb1b00e1da 100644 --- a/internal/engine/clickhouse/dialect/types.jsonl +++ b/internal/engine/clickhouse/dialect/types.jsonl @@ -1,40 +1,66 @@ -{"name": "UInt8", "category": "N"} -{"name": "UInt16", "category": "N"} -{"name": "UInt32", "category": "N"} -{"name": "UInt64", "category": "N"} -{"name": "UInt128", "category": "N"} -{"name": "UInt256", "category": "N"} -{"name": "Int8", "category": "N"} -{"name": "Int16", "category": "N"} -{"name": "Int32", "category": "N"} -{"name": "Int64", "category": "N"} -{"name": "Int128", "category": "N"} -{"name": "Int256", "category": "N"} -{"name": "Float32", "category": "N"} -{"name": "Float64", "category": "N"} -{"name": "BFloat16", "category": "N"} -{"name": "Decimal", "category": "N"} -{"name": "Decimal32", "category": "N"} -{"name": "Decimal64", "category": "N"} -{"name": "Decimal128", "category": "N"} -{"name": "Decimal256", "category": "N"} -{"name": "Bool", "category": "B"} -{"name": "String", "category": "S"} -{"name": "FixedString", "category": "S"} -{"name": "UUID", "category": "S"} -{"name": "Date", "category": "D"} -{"name": "Date32", "category": "D"} -{"name": "DateTime", "category": "D"} -{"name": "DateTime64", "category": "D"} -{"name": "IPv4", "category": "S"} -{"name": "IPv6", "category": "S"} -{"name": "JSON", "category": "U"} -{"name": "Enum8", "category": "U"} -{"name": "Enum16", "category": "U"} -{"name": "Nullable", "category": "U"} -{"name": "LowCardinality", "category": "U"} -{"name": "Array", "category": "A"} -{"name": "Map", "category": "U"} -{"name": "Tuple", "category": "U"} -{"name": "Nested", "category": "U"} -{"name": "Nothing", "category": "U"} +{"name":"AggregateFunction","category":"U"} +{"name":"Array","category":"A"} +{"name":"BFloat16","category":"N"} +{"name":"Bool","category":"B","aliases":["boolean"]} +{"name":"Date","category":"D"} +{"name":"Date32","category":"D"} +{"name":"DateTime","category":"D","aliases":["TIMESTAMP"]} +{"name":"DateTime32","category":"D"} +{"name":"DateTime64","category":"D"} +{"name":"Decimal","category":"N","aliases":["DEC","FIXED","NUMERIC"]} +{"name":"Decimal128","category":"N"} +{"name":"Decimal256","category":"N"} +{"name":"Decimal32","category":"N"} +{"name":"Decimal64","category":"N"} +{"name":"Dynamic","category":"U"} +{"name":"Enum","category":"U"} +{"name":"Enum16","category":"U"} +{"name":"Enum8","category":"U"} +{"name":"FixedString","category":"S","aliases":["BINARY"]} +{"name":"Float32","category":"N","aliases":["FLOAT","REAL","SINGLE"]} +{"name":"Float64","category":"N","aliases":["DOUBLE","DOUBLE PRECISION"]} +{"name":"IPv4","category":"S","aliases":["INET4"]} +{"name":"IPv6","category":"S","aliases":["INET6"]} +{"name":"Int128","category":"N"} +{"name":"Int16","category":"N","aliases":["SMALLINT","SMALLINT SIGNED"]} +{"name":"Int256","category":"N"} +{"name":"Int32","category":"N","aliases":["INT","INT SIGNED","INTEGER","INTEGER SIGNED","MEDIUMINT","MEDIUMINT SIGNED"]} +{"name":"Int64","category":"N","aliases":["BIGINT","BIGINT SIGNED","SIGNED"]} +{"name":"Int8","category":"N","aliases":["BYTE","INT1","INT1 SIGNED","TINYINT","TINYINT SIGNED"]} +{"name":"IntervalDay","category":"T"} +{"name":"IntervalHour","category":"T"} +{"name":"IntervalMicrosecond","category":"T"} +{"name":"IntervalMillisecond","category":"T"} +{"name":"IntervalMinute","category":"T"} +{"name":"IntervalMonth","category":"T"} +{"name":"IntervalNanosecond","category":"T"} +{"name":"IntervalQuarter","category":"T"} +{"name":"IntervalSecond","category":"T"} +{"name":"IntervalWeek","category":"T"} +{"name":"IntervalYear","category":"T"} +{"name":"JSON","category":"U"} +{"name":"LineString","category":"U"} +{"name":"LowCardinality","category":"U"} +{"name":"Map","category":"U"} +{"name":"MultiLineString","category":"U"} +{"name":"MultiPolygon","category":"U"} +{"name":"Nested","category":"U"} +{"name":"Nothing","category":"U"} +{"name":"Nullable","category":"U"} +{"name":"Object","category":"U"} +{"name":"Point","category":"U"} +{"name":"Polygon","category":"U"} +{"name":"Ring","category":"U"} +{"name":"SimpleAggregateFunction","category":"U"} +{"name":"String","category":"S","aliases":["BINARY LARGE OBJECT","BINARY VARYING","BLOB","BYTEA","CHAR","CHAR LARGE OBJECT","CHAR VARYING","CHARACTER","CHARACTER LARGE OBJECT","CHARACTER VARYING","CLOB","GEOMETRY","LONGBLOB","LONGTEXT","MEDIUMBLOB","MEDIUMTEXT","NATIONAL CHAR","NATIONAL CHAR VARYING","NATIONAL CHARACTER","NATIONAL CHARACTER LARGE OBJECT","NATIONAL CHARACTER VARYING","NCHAR","NCHAR LARGE OBJECT","NCHAR VARYING","NVARCHAR","TEXT","TINYBLOB","TINYTEXT","VARBINARY","VARCHAR","VARCHAR2"]} +{"name":"Time","category":"D"} +{"name":"Time64","category":"D"} +{"name":"Tuple","category":"U"} +{"name":"UInt128","category":"N"} +{"name":"UInt16","category":"N","aliases":["SMALLINT UNSIGNED","YEAR"]} +{"name":"UInt256","category":"N"} +{"name":"UInt32","category":"N","aliases":["INT UNSIGNED","INTEGER UNSIGNED","MEDIUMINT UNSIGNED"]} +{"name":"UInt64","category":"N","aliases":["BIGINT UNSIGNED","BIT","SET","UNSIGNED"]} +{"name":"UInt8","category":"N","aliases":["INT1 UNSIGNED","TINYINT UNSIGNED"]} +{"name":"UUID","category":"S"} +{"name":"Variant","category":"U"} diff --git a/internal/engine/clickhouse/seed.go b/internal/engine/clickhouse/seed.go index 310b0476b4..b87a704e41 100644 --- a/internal/engine/clickhouse/seed.go +++ b/internal/engine/clickhouse/seed.go @@ -7,6 +7,15 @@ import ( "github.com/sqlc-dev/sqlc/internal/core/seed" ) +// The dialect directory describes ClickHouse's type system. types.jsonl is +// generated from system.data_type_families of the pinned ClickHouse release +// by goldeneye (internal/goldeneye), which also checks it against one; +// dialect.json and functions.jsonl are authored by hand, since ClickHouse +// publishes no function signatures. Regenerate from internal/goldeneye with: +// +// go run ./cmd/goldeneye install clickhouse +// go run ./cmd/goldeneye generate clickhouse +// //go:embed dialect var dialectFS embed.FS diff --git a/internal/engine/duckdb/seed.go b/internal/engine/duckdb/seed.go index 245de9d8eb..852b5d217b 100644 --- a/internal/engine/duckdb/seed.go +++ b/internal/engine/duckdb/seed.go @@ -9,10 +9,10 @@ import ( // The dialect directory describes DuckDB's type system. types.jsonl, // functions.jsonl and operators.jsonl are generated from a live DuckDB 2.0 -// CLI by sqlc-duckdb-gen (internal/tools/sqlc-duckdb-gen); dialect.json is -// authored by hand. Regenerate with: +// CLI by goldeneye (internal/goldeneye), which also checks them against one; +// dialect.json is authored by hand. Regenerate from internal/goldeneye with: // -// DUCKDB=/path/to/duckdb go run ./internal/tools/sqlc-duckdb-gen +// DUCKDB=/path/to/duckdb go run ./cmd/goldeneye generate duckdb // //go:embed dialect var dialectFS embed.FS diff --git a/internal/engine/postgresql/catalog.go b/internal/engine/postgresql/catalog.go index 313dcfdad3..08049b78f4 100644 --- a/internal/engine/postgresql/catalog.go +++ b/internal/engine/postgresql/catalog.go @@ -16,7 +16,7 @@ func NewCatalog() *catalog.Catalog { } // genPGCatalog and genInformationSchema build the schemas sqlc knows -// PostgreSQL by. Both read the dialect directory, which sqlc-pg-gen writes, +// PostgreSQL by. Both read the dialect directory, which goldeneye writes, // and which the analysis core seeds its catalog from as well. func genPGCatalog() *catalog.Schema { return systemSchema("pg_catalog", pgCatalogFuncs()) diff --git a/internal/engine/postgresql/dialect/functions.jsonl b/internal/engine/postgresql/dialect/functions.jsonl index 43d46acc2a..daf9060fd3 100644 --- a/internal/engine/postgresql/dialect/functions.jsonl +++ b/internal/engine/postgresql/dialect/functions.jsonl @@ -1695,11 +1695,6 @@ {"name":"pg_export_snapshot","returns":"text"} {"name":"pg_extension_config_dump","args":[{"type":"regclass"},{"type":"text"}],"returns":"void"} {"name":"pg_extension_update_paths","args":[{"name":"name","type":"name"}],"returns":"record"} -{"name":"pg_file_rename","args":[{"type":"text"},{"type":"text"}],"returns":"boolean"} -{"name":"pg_file_rename","args":[{"type":"text"},{"type":"text"},{"type":"text"}],"returns":"boolean"} -{"name":"pg_file_sync","args":[{"type":"text"}],"returns":"void"} -{"name":"pg_file_unlink","args":[{"type":"text"}],"returns":"boolean"} -{"name":"pg_file_write","args":[{"type":"text"},{"type":"text"},{"type":"boolean"}],"returns":"bigint"} {"name":"pg_filenode_relation","args":[{"type":"oid"},{"type":"oid"}],"returns":"regclass"} {"name":"pg_function_is_visible","args":[{"type":"oid"}],"returns":"boolean"} {"name":"pg_get_backend_memory_contexts","returns":"record"} @@ -1772,7 +1767,6 @@ {"name":"pg_lock_status","returns":"record"} {"name":"pg_log_backend_memory_contexts","args":[{"type":"integer"}],"returns":"boolean"} {"name":"pg_log_standby_snapshot","returns":"pg_lsn"} -{"name":"pg_logdir_ls","returns":"record"} {"name":"pg_logical_emit_message","args":[{"type":"boolean"},{"type":"text"},{"type":"bytea"}],"returns":"pg_lsn"} {"name":"pg_logical_emit_message","args":[{"type":"boolean"},{"type":"text"},{"type":"text"}],"returns":"pg_lsn"} {"name":"pg_logical_slot_get_binary_changes","args":[{"name":"slot_name","type":"name"},{"name":"upto_lsn","type":"pg_lsn"},{"name":"upto_nchanges","type":"integer"},{"name":"options","type":"text[]","mode":"v","has_default":true}],"returns":"record"} diff --git a/internal/engine/postgresql/extension.go b/internal/engine/postgresql/extension.go index 7a45f03bee..6d81f9f08c 100644 --- a/internal/engine/postgresql/extension.go +++ b/internal/engine/postgresql/extension.go @@ -9,7 +9,7 @@ import ( ) // loadExtension reads the named extension's function list from the dialect -// directory, where sqlc-pg-gen writes one directory per extension. An +// directory, where goldeneye writes one directory per extension. An // extension sqlc does not know is nil, which CREATE EXTENSION treats as // nothing to add. func loadExtension(name string) *catalog.Schema { diff --git a/internal/engine/postgresql/seed.go b/internal/engine/postgresql/seed.go index 0a6d628944..9cf184c50d 100644 --- a/internal/engine/postgresql/seed.go +++ b/internal/engine/postgresql/seed.go @@ -17,10 +17,11 @@ import ( // "WHERE bigint_col = 1" depends on. // // functions.jsonl and relations.jsonl are pg_catalog's functions and the -// tables and views of pg_catalog and information_schema, all written by -// sqlc-pg-gen. Both the analysis core and the catalog the legacy compiler -// builds read them. Each contrib extension sqlc knows is a directory under -// extensions/ holding the types and functions CREATE EXTENSION adds. +// tables and views of pg_catalog and information_schema, all generated from +// a live PostgreSQL 16 server by goldeneye (internal/goldeneye), which also +// checks them against one. Both the analysis core and the catalog the legacy +// compiler builds read them. Each contrib extension sqlc knows is a directory +// under extensions/ holding the types and functions CREATE EXTENSION adds. // //go:embed dialect var dialectFS embed.FS diff --git a/internal/goldeneye/README.md b/internal/goldeneye/README.md new file mode 100644 index 0000000000..3bd0a35787 --- /dev/null +++ b/internal/goldeneye/README.md @@ -0,0 +1,69 @@ +# goldeneye + +`goldeneye` generates the dialect seeds under `internal/engine//dialect` +from a live database, and verifies the committed ones against it. A dialect +is the JSONL that gives an engine its type system and standard library — +`types.jsonl`, `functions.jsonl`, `operators.jsonl`, `relations.jsonl` and +the `extensions/` bundles — read by `internal/core/seed`. Each engine package +here asks the database what it knows, writes the answer in that shape, and +the tests compare it with what is committed, byte for byte. A difference +means the committed dialect has drifted from the database. + +It is a nested Go module, so its only dependency beyond the standard library +is the PostgreSQL driver, and it never shares code with the analysis that +reads the files: the files are the contract. Run it from this directory: + +```bash +go run ./cmd/goldeneye install clickhouse # download the pinned clickhouse binary once +go run ./cmd/goldeneye check # check every engine whose database is available +go run ./cmd/goldeneye check postgresql # check one engine +go run ./cmd/goldeneye generate [engine] # rewrite the generated files from the database +go test ./... # the same checks as tests; engines without a database skip +``` + +`generate` and `check` say which engines they skipped for lack of a +database; naming an engine makes its database required. + +## What is generated, and what is not + +A generator owns only the files it produces; `dialect.json` is always written +by hand, and so are the lists an engine cannot describe. Both commands leave +the hand-written files alone, and the checks do not look at them. + +- **`postgresql`** reads a live server named by `POSTGRESQL_SERVER_URI`: + `functions.jsonl` is `pg_catalog`'s functions, `relations.jsonl` the tables + and views of `pg_catalog` and `information_schema`, and each contrib + extension gets a directory under `extensions/` holding the types and + functions `CREATE EXTENSION` adds, so the server needs contrib installed. + A function that one of those extensions puts in `pg_catalog`, as + `adminpack` does, belongs to the extension's directory rather than the + catalog's list. The server has to be the major release pinned in + `postgresql.Major`, since every release adds to the catalogs; the top-level + `types.jsonl` and `operators.jsonl` are hand-written. +- **`duckdb`** reads the DuckDB CLI named by `DUCKDB`, or `duckdb` on `PATH`: + `types.jsonl`, `functions.jsonl` and `operators.jsonl` come from + `duckdb_types()` and `duckdb_functions()`. The CLI has to be the DuckDB 2.0 + build darkwing is pinned against, which has no release to download yet. +- **`clickhouse`** needs no server: `types.jsonl` comes from + `system.data_type_families` of an ephemeral `clickhouse local` process, + every family that is not an alias becoming a type carrying the spellings + that alias it, with a category decided by its name. The binary is + downloaded once per pinned release by `install` into the user cache + directory, or supplied through the `CLICKHOUSE` environment variable; the + pinned release and the SHA-512 of each platform's download live in + `clickhouse/install.go`, and a download that does not match is discarded. + ClickHouse describes its functions no further than their names, so + `functions.jsonl` is hand-written. + +## Layout + +- `dialect/` — the record types the files are made of, mirrored from + `internal/core/seed`, and the helpers that write a generated set of files + into an engine directory or diff it against what is committed. +- `postgresql/`, `duckdb/`, `clickhouse/` — one package per engine, each + exposing `Locate`, `Version` and `Generate`, and a test that runs the check. +- `cmd/goldeneye/` — the command. + +The analysis checks — verifying the `analyze_*` cases under +`internal/endtoend/testdata` against what each database itself reports — are +meant to live here too, alongside the dialect checks. diff --git a/internal/goldeneye/clickhouse/clickhouse.go b/internal/goldeneye/clickhouse/clickhouse.go new file mode 100644 index 0000000000..5eb10d282c --- /dev/null +++ b/internal/goldeneye/clickhouse/clickhouse.go @@ -0,0 +1,137 @@ +// Package clickhouse generates the ClickHouse dialect seed under +// internal/engine/clickhouse/dialect from a clickhouse binary, run as an +// ephemeral `clickhouse local` process that needs no server. +// +// types.jsonl comes from system.data_type_families: every family that is not +// an alias becomes a type, the families that alias it become its aliases, +// and its category is decided by its name, since ClickHouse records no such +// thing. ClickHouse describes its functions no further than their names — +// system.functions carries no signatures — so functions.jsonl is written by +// hand and is not this package's business. +// +// The binary is downloaded once per pinned version by Install, or supplied +// through the CLICKHOUSE environment variable. +package clickhouse + +import ( + "context" + "encoding/json" + "fmt" + "os/exec" + "sort" + "strings" + + "github.com/sqlc-dev/sqlc/internal/goldeneye/dialect" +) + +// Engine is the name of the engine directory the dialect lives under. +const Engine = "clickhouse" + +// typeFamilies lists every data type ClickHouse knows and, for a spelling +// such as INT or VARCHAR, the type it stands for. +const typeFamilies = ` +SELECT name, alias_to +FROM system.data_type_families +ORDER BY name` + +// Version reports the release a binary is. +func Version(ctx context.Context, binary string) (string, error) { + out, err := exec.CommandContext(ctx, binary, "local", "--version").Output() + if err != nil { + return "", fmt.Errorf("clickhouse local --version: %w", err) + } + return strings.TrimSpace(string(out)), nil +} + +// Generate reads the dialect from the binary. +func Generate(ctx context.Context, binary string) (dialect.Files, error) { + results, err := local{binary: binary}.run(ctx, typeFamilies) + if err != nil { + return nil, err + } + if len(results) != 1 { + return nil, fmt.Errorf("clickhouse: expected one result set, got %d", len(results)) + } + var families []family + for _, row := range results[0].Data { + var f family + if err := json.Unmarshal(row["name"], &f.Name); err != nil { + return nil, fmt.Errorf("clickhouse: system.data_type_families: %w", err) + } + if err := json.Unmarshal(row["alias_to"], &f.AliasTo); err != nil { + return nil, fmt.Errorf("clickhouse: system.data_type_families: %w", err) + } + families = append(families, f) + } + types, err := dialect.JSONL(readTypes(families)) + if err != nil { + return nil, err + } + return dialect.Files{dialect.TypesFile: types}, nil +} + +type family struct { + Name string + AliasTo string +} + +// readTypes turns the families into types: one per family that is not an +// alias, carrying the spellings that alias it. The seed reads names without +// regard to case, so an alias that only differs in case from its type — +// bool for Bool, ENUM for Enum — is dropped. +func readTypes(families []family) []dialect.Type { + byName := map[string]*dialect.Type{} + var names []string + for _, f := range families { + if f.AliasTo != "" { + continue + } + byName[f.Name] = &dialect.Type{Name: f.Name, Category: category(f.Name)} + names = append(names, f.Name) + } + for _, f := range families { + if f.AliasTo == "" || strings.EqualFold(f.Name, f.AliasTo) { + continue + } + if t, ok := byName[f.AliasTo]; ok { + t.Aliases = append(t.Aliases, f.Name) + } + } + sort.Strings(names) + types := make([]dialect.Type, 0, len(names)) + for _, name := range names { + types = append(types, *byName[name]) + } + return types +} + +// category assigns a type the PostgreSQL category letter the seed package +// uses, by the family's name: N for numbers, B for booleans, S for strings +// and the identifiers stored as them, D for dates and times, T for time +// spans, A for arrays and U for everything else. +func category(name string) string { + switch { + case name == "Bool": + return "B" + case name == "Array": + return "A" + case strings.HasPrefix(name, "Interval"): + return "T" + case hasPrefix(name, "UInt", "Int", "Float", "BFloat", "Decimal"): + return "N" + case name == "String", name == "FixedString", name == "UUID", name == "IPv4", name == "IPv6": + return "S" + case hasPrefix(name, "Date", "Time"): + return "D" + } + return "U" +} + +func hasPrefix(name string, prefixes ...string) bool { + for _, prefix := range prefixes { + if strings.HasPrefix(name, prefix) { + return true + } + } + return false +} diff --git a/internal/goldeneye/clickhouse/clickhouse_test.go b/internal/goldeneye/clickhouse/clickhouse_test.go new file mode 100644 index 0000000000..0e074eaf21 --- /dev/null +++ b/internal/goldeneye/clickhouse/clickhouse_test.go @@ -0,0 +1,37 @@ +package clickhouse + +import ( + "context" + "testing" + + "github.com/sqlc-dev/sqlc/internal/goldeneye/dialect" +) + +// TestDialect verifies the committed ClickHouse dialect against what the +// pinned clickhouse binary reports. It skips unless the binary is installed. +func TestDialect(t *testing.T) { + binary, err := Locate() + if err != nil { + t.Skip(err) + } + ctx := context.Background() + version, err := Version(ctx, binary) + if err != nil { + t.Fatal(err) + } + files, err := Generate(ctx, binary) + if err != nil { + t.Fatal(err) + } + dir, err := dialect.Dir(Engine) + if err != nil { + t.Fatal(err) + } + report, err := dialect.Check(dir, files) + if err != nil { + t.Fatal(err) + } + if report != "" { + t.Errorf("%s does not match what %s reports:\n%s", dir, version, report) + } +} diff --git a/internal/goldeneye/clickhouse/install.go b/internal/goldeneye/clickhouse/install.go new file mode 100644 index 0000000000..fb29df46a0 --- /dev/null +++ b/internal/goldeneye/clickhouse/install.go @@ -0,0 +1,221 @@ +package clickhouse + +import ( + "archive/tar" + "compress/gzip" + "context" + "crypto/sha512" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" +) + +// DefaultVersion is the ClickHouse release the dialect is generated from. +// Bumping it is a deliberate change: the query tree format and type +// inference can shift between releases, so regenerate and review the dialect +// after changing it, and add the new release's assets to the table below. +const DefaultVersion = "25.8.2.29" + +// asset is one downloadable build of ClickHouse. Linux builds are published +// as clickhouse-common-static tarballs holding the binary at +// usr/bin/clickhouse; macOS builds are published as bare binaries. +type asset struct { + Version string + OS string + Arch string + Name string + SHA512 string +} + +// assets lists every build Install knows how to fetch, with the SHA-512 of +// the download. A version that is not in this table cannot be installed: +// verifying the download is the point of the table. +// +// The tarball checksums are the ones in the .sha512 files ClickHouse +// publishes next to them. ClickHouse publishes no checksum for the macOS +// binaries, so those were computed from the downloads. +var assets = []asset{ + {"25.8.2.29", "linux", "amd64", "clickhouse-common-static-25.8.2.29-amd64.tgz", "6ff0aa1ffac6e564970174422ecde0d645cdb96812247a6e544d39cad6d78a514265f90a2bc7b4bad49903cea96eddd16a415a45b2aeaf9164461be76331bdee"}, + {"25.8.2.29", "linux", "arm64", "clickhouse-common-static-25.8.2.29-arm64.tgz", "68204ca4d4e472790f808ee376251fae82e58066a31f35a40d15d442ce5988d697f18a1208d28b8bb8e2dfad4b20b7fcb5107e2178472abcd97251b8de7f058e"}, + {"25.8.2.29", "darwin", "amd64", "clickhouse-macos", "2805805ad2506e37a3e71b4ae9e797bdc010a9368dc28e99bcaaa2c70a72cfdd031c0fce8fc304248fad73211d28d645f75c6432cfae1e1e54d72d04e8626cd4"}, + {"25.8.2.29", "darwin", "arm64", "clickhouse-macos-aarch64", "4c9237e85c8d4e1aced2b339b32e086b4f23fa14f99754b056a7e33dda88c4fb5d52e09e29d20181ec5b580caadb00f36613802e71755ff34927535eb8babf79"}, +} + +// releaseTag returns the GitHub release tag for a version. ClickHouse tags +// its March and August releases as LTS and everything else as stable. +func releaseTag(version string) (string, error) { + parts := strings.Split(version, ".") + if len(parts) != 4 { + return "", fmt.Errorf("invalid ClickHouse version %q: want MAJOR.MINOR.PATCH.BUILD", version) + } + minor, err := strconv.Atoi(parts[1]) + if err != nil { + return "", fmt.Errorf("invalid ClickHouse version %q: %w", version, err) + } + suffix := "-stable" + if minor == 3 || minor == 8 { + suffix = "-lts" + } + return "v" + version + suffix, nil +} + +// releaseAsset finds the build for a platform in the table. +func releaseAsset(version, goos, goarch string) (asset, error) { + for _, a := range assets { + if a.Version == version && a.OS == goos && a.Arch == goarch { + return a, nil + } + } + for _, a := range assets { + if a.Version == version { + return asset{}, fmt.Errorf("no ClickHouse %s build is listed for %s/%s", version, goos, goarch) + } + } + return asset{}, fmt.Errorf("ClickHouse %s is not in the asset table; add its downloads and checksums to install.go", version) +} + +// url is the asset's download address on GitHub. +func (a asset) url() (string, error) { + tag, err := releaseTag(a.Version) + if err != nil { + return "", err + } + return "https://github.com/ClickHouse/ClickHouse/releases/download/" + tag + "/" + a.Name, nil +} + +// tarball reports whether the download is an archive rather than the binary. +func (a asset) tarball() bool { + return strings.HasSuffix(a.Name, ".tgz") +} + +// cachedBinary is where Install puts the binary for a version. +func cachedBinary(version string) (string, error) { + dir, err := os.UserCacheDir() + if err != nil { + return "", err + } + return filepath.Join(dir, "sqlc-clickhouse", version, "clickhouse"), nil +} + +// Locate finds a clickhouse binary: the CLICKHOUSE environment variable wins, +// then the cached copy of DefaultVersion. +func Locate() (string, error) { + if path := os.Getenv("CLICKHOUSE"); path != "" { + return path, nil + } + path, err := cachedBinary(DefaultVersion) + if err != nil { + return "", err + } + if _, err := os.Stat(path); err != nil { + return "", fmt.Errorf("clickhouse %s is not installed: run `go run ./cmd/goldeneye install clickhouse` in internal/goldeneye, or set CLICKHOUSE to a clickhouse binary", DefaultVersion) + } + return path, nil +} + +// Install downloads the clickhouse binary for a version into the cache and +// returns its path. It is a no-op when the version is already cached. The +// download is checked against the table's SHA-512 before it is installed. +func Install(ctx context.Context, version, goos, goarch string, progress io.Writer) (string, error) { + dest, err := cachedBinary(version) + if err != nil { + return "", err + } + if _, err := os.Stat(dest); err == nil { + return dest, nil + } + a, err := releaseAsset(version, goos, goarch) + if err != nil { + return "", err + } + url, err := a.url() + if err != nil { + return "", err + } + if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { + return "", err + } + + fmt.Fprintf(progress, "downloading %s\n", url) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return "", err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("downloading %s: %s", url, resp.Status) + } + + // Write next to the destination and rename so a partial or corrupt + // download never masquerades as an installed binary. + tmp, err := os.CreateTemp(filepath.Dir(dest), "clickhouse-*.partial") + if err != nil { + return "", err + } + defer os.Remove(tmp.Name()) + + // Hash every byte that comes off the wire, including the parts of a + // tarball after the binary, which extraction would otherwise not read. + sum := sha512.New() + body := io.TeeReader(resp.Body, sum) + var src io.Reader = body + if a.tarball() { + src, err = binaryInTarball(body) + if err != nil { + return "", fmt.Errorf("downloading %s: %w", url, err) + } + } + if _, err := io.Copy(tmp, src); err != nil { + tmp.Close() + return "", err + } + if _, err := io.Copy(io.Discard, body); err != nil { + tmp.Close() + return "", err + } + if err := tmp.Close(); err != nil { + return "", err + } + if got := hex.EncodeToString(sum.Sum(nil)); got != a.SHA512 { + return "", fmt.Errorf("downloading %s: SHA-512 mismatch: got %s, want %s", url, got, a.SHA512) + } + if err := os.Chmod(tmp.Name(), 0o755); err != nil { + return "", err + } + if err := os.Rename(tmp.Name(), dest); err != nil { + return "", err + } + return dest, nil +} + +// binaryInTarball positions a reader at the clickhouse binary inside a +// clickhouse-common-static tarball. +func binaryInTarball(r io.Reader) (io.Reader, error) { + gz, err := gzip.NewReader(r) + if err != nil { + return nil, err + } + tr := tar.NewReader(gz) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + return nil, errors.New("tarball does not contain usr/bin/clickhouse") + } + if err != nil { + return nil, err + } + if hdr.Typeflag == tar.TypeReg && strings.HasSuffix(hdr.Name, "/usr/bin/clickhouse") { + return tr, nil + } + } +} diff --git a/internal/goldeneye/clickhouse/local.go b/internal/goldeneye/clickhouse/local.go new file mode 100644 index 0000000000..2a6d88247c --- /dev/null +++ b/internal/goldeneye/clickhouse/local.go @@ -0,0 +1,82 @@ +package clickhouse + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" +) + +// local runs SQL through an ephemeral `clickhouse local` process. +type local struct { + binary string +} + +// resultSet is one JSON-format result printed by clickhouse local. Only +// statements that return rows print one; DDL and INSERT print nothing. +type resultSet struct { + Meta []resultColumn `json:"meta"` + Data []map[string]json.RawMessage `json:"data"` + Rows int `json:"rows"` +} + +type resultColumn struct { + Name string `json:"name"` + Type string `json:"type"` +} + +// run executes a multi-statement script in a fresh process with fresh +// storage and returns the result sets in statement order. Any statement +// failing fails the whole run with ClickHouse's own error message. +func (l local) run(ctx context.Context, script string) ([]resultSet, error) { + dir, err := os.MkdirTemp("", "sqlc-goldeneye-clickhouse-*") + if err != nil { + return nil, err + } + defer os.RemoveAll(dir) + + queries := filepath.Join(dir, "queries.sql") + if err := os.WriteFile(queries, []byte(script), 0o600); err != nil { + return nil, err + } + + // stdin must not be inherited: clickhouse local reads it as table data + // and blocks until it is closed. + cmd := exec.CommandContext(ctx, l.binary, "local", + "--multiquery", + "--queries-file", queries, + "--output-format", "JSON", + "--path", filepath.Join(dir, "data"), + ) + cmd.Stdin = nil + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + msg := strings.TrimSpace(stderr.String()) + if msg == "" { + msg = err.Error() + } + return nil, errors.New(msg) + } + + var results []resultSet + dec := json.NewDecoder(&stdout) + for { + var rs resultSet + err := dec.Decode(&rs) + if errors.Is(err, io.EOF) { + return results, nil + } + if err != nil { + return nil, fmt.Errorf("decoding clickhouse local output: %w", err) + } + results = append(results, rs) + } +} diff --git a/internal/goldeneye/cmd/goldeneye/main.go b/internal/goldeneye/cmd/goldeneye/main.go new file mode 100644 index 0000000000..c3af78df38 --- /dev/null +++ b/internal/goldeneye/cmd/goldeneye/main.go @@ -0,0 +1,188 @@ +// Command goldeneye generates the dialect seeds under +// internal/engine//dialect from a live database, and checks the +// committed ones against it. +// +// Usage, from internal/goldeneye: +// +// go run ./cmd/goldeneye install clickhouse # download the pinned clickhouse binary +// go run ./cmd/goldeneye generate [engine] # rewrite the generated files from the database +// go run ./cmd/goldeneye check [engine] # compare the committed files with the database +// +// Without an engine, generate and check cover every engine whose database +// is available and say which ones they skipped. `go test ./...` runs the +// same checks as tests. +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "os" + "runtime" + + "github.com/sqlc-dev/sqlc/internal/goldeneye/clickhouse" + "github.com/sqlc-dev/sqlc/internal/goldeneye/dialect" + "github.com/sqlc-dev/sqlc/internal/goldeneye/duckdb" + "github.com/sqlc-dev/sqlc/internal/goldeneye/postgresql" +) + +func main() { + if err := run(context.Background(), os.Args[1:], os.Stdout, os.Stderr); err != nil { + fmt.Fprintln(os.Stderr, "goldeneye:", err) + os.Exit(1) + } +} + +const usage = `usage: + goldeneye install clickhouse [-version V] + download the pinned clickhouse binary into the user cache directory + goldeneye generate [engine] + rewrite the generated dialect files from the database, for every available engine or one + goldeneye check [engine] + compare the committed dialect files with the database, for every available engine or one + +engines: clickhouse, duckdb, postgresql` + +// engine is one database goldeneye knows how to read a dialect from. +type engine struct { + name string + // locate finds the database — a binary or a connection URL — or says + // why it is not available. + locate func() (string, error) + // version describes the release the database is, for the log. + version func(context.Context, string) (string, error) + // generate reads the dialect from the database. + generate func(context.Context, string) (dialect.Files, error) +} + +var engines = []engine{ + {clickhouse.Engine, clickhouse.Locate, clickhouse.Version, clickhouse.Generate}, + {duckdb.Engine, duckdb.Locate, duckdb.Version, duckdb.Generate}, + {postgresql.Engine, postgresql.Locate, postgresql.Version, postgresql.Generate}, +} + +func run(ctx context.Context, args []string, stdout, stderr io.Writer) error { + if len(args) == 0 { + fmt.Fprintln(stderr, usage) + return errors.New("a command is required") + } + switch args[0] { + case "install": + return install(ctx, args[1:], stdout, stderr) + case "generate": + return forEach(ctx, args[1:], stderr, generate) + case "check": + return forEach(ctx, args[1:], stderr, check) + case "help", "-h", "--help": + fmt.Fprintln(stdout, usage) + return nil + } + fmt.Fprintln(stderr, usage) + return fmt.Errorf("unknown command %q", args[0]) +} + +func install(ctx context.Context, args []string, stdout, stderr io.Writer) error { + if len(args) == 0 || args[0] != clickhouse.Engine { + return errors.New("install takes the engine to install: clickhouse") + } + fs := flag.NewFlagSet("install", flag.ContinueOnError) + fs.SetOutput(stderr) + version := fs.String("version", clickhouse.DefaultVersion, "ClickHouse release to install") + if err := fs.Parse(args[1:]); err != nil { + return err + } + path, err := clickhouse.Install(ctx, *version, runtime.GOOS, runtime.GOARCH, stderr) + if err != nil { + return err + } + fmt.Fprintln(stdout, path) + return nil +} + +// forEach runs a command over the named engine, or over every engine that +// is available when none is named. An engine that is named has to be +// available; one that is not named is skipped with a note when it is not. +func forEach(ctx context.Context, args []string, stderr io.Writer, cmd func(context.Context, engine, string, io.Writer) error) error { + if len(args) > 1 { + return errors.New("at most one engine may be named") + } + name := "" + if len(args) == 1 { + name = args[0] + } + found := false + var failed []string + for _, e := range engines { + if name != "" && e.name != name { + continue + } + found = true + handle, err := e.locate() + if err != nil { + if name != "" { + return err + } + fmt.Fprintf(stderr, "skipping %s: %v\n", e.name, err) + continue + } + if err := cmd(ctx, e, handle, stderr); err != nil { + fmt.Fprintf(stderr, "%s: %v\n", e.name, err) + failed = append(failed, e.name) + } + } + if !found { + return fmt.Errorf("unknown engine %q", name) + } + if len(failed) > 0 { + return fmt.Errorf("%d engine(s) failed: %v", len(failed), failed) + } + return nil +} + +func generate(ctx context.Context, e engine, handle string, stderr io.Writer) error { + version, err := e.version(ctx, handle) + if err != nil { + return err + } + fmt.Fprintf(stderr, "%s: generating from %s\n", e.name, version) + files, err := e.generate(ctx, handle) + if err != nil { + return err + } + dir, err := dialect.Dir(e.name) + if err != nil { + return err + } + if err := dialect.Write(dir, files); err != nil { + return err + } + fmt.Fprintf(stderr, "%s: wrote %d file(s) to %s\n", e.name, len(files), dir) + return nil +} + +func check(ctx context.Context, e engine, handle string, stderr io.Writer) error { + version, err := e.version(ctx, handle) + if err != nil { + return err + } + fmt.Fprintf(stderr, "%s: checking against %s\n", e.name, version) + files, err := e.generate(ctx, handle) + if err != nil { + return err + } + dir, err := dialect.Dir(e.name) + if err != nil { + return err + } + report, err := dialect.Check(dir, files) + if err != nil { + return err + } + if report != "" { + return fmt.Errorf("%s does not match the database\n%s", dir, report) + } + fmt.Fprintf(stderr, "%s: ok, %d file(s) match\n", e.name, len(files)) + return nil +} diff --git a/internal/goldeneye/dialect/dialect.go b/internal/goldeneye/dialect/dialect.go new file mode 100644 index 0000000000..3db8aab463 --- /dev/null +++ b/internal/goldeneye/dialect/dialect.go @@ -0,0 +1,284 @@ +// Package dialect describes the files a dialect seed is made of — the JSONL +// records under internal/engine//dialect that give an engine its +// type system and standard library — and provides what a generator needs to +// write them and a check needs to compare them with what is committed. +// +// The record types mirror the ones in internal/core/seed, which reads the +// files. They are repeated here rather than imported so that this module +// never shares code with the analysis it checks: the files are the contract, +// and a change to their shape has to be made on both sides. +package dialect + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "sort" + "strings" +) + +// The files a dialect directory is made of. +const ( + SettingsFile = "dialect.json" + TypesFile = "types.jsonl" + OperatorsFile = "operators.jsonl" + CastsFile = "casts.jsonl" + FunctionsFile = "functions.jsonl" + RelationsFile = "relations.jsonl" +) + +// ExtensionsDir is the directory under a dialect holding one directory per +// extension the dialect knows, each a smaller bundle of the same files. +const ExtensionsDir = "extensions" + +// Type is a type the dialect defines. Aliases are spellings of the same type +// that a schema may use in a column definition. +type Type struct { + Name string `json:"name"` + Category string `json:"category"` + Aliases []string `json:"aliases,omitempty"` +} + +// Operator is a single operator overload. +type Operator struct { + Name string `json:"name"` + Left string `json:"left"` + Right string `json:"right"` + Result string `json:"result"` +} + +// Cast is a single cast between two types. Context is 'i'mplicit, +// 'a'ssignment or 'e'xplicit. +type Cast struct { + Source string `json:"source"` + Target string `json:"target"` + Context string `json:"context,omitempty"` +} + +// Function is a function the dialect ships with. Kind is 'f'unction, +// 'a'ggregate, 'w'indow or 'p'rocedure. +type Function struct { + Name string `json:"name"` + Kind string `json:"kind,omitempty"` + Args []Arg `json:"args,omitempty"` + Returns string `json:"returns"` + Nullable bool `json:"nullable,omitempty"` +} + +// Arg is one of a function's parameters. Mode is 'i'n, 'o'ut, 'b'oth, +// 't'able or 'v'ariadic, and defaults to in. +type Arg struct { + Name string `json:"name,omitempty"` + Type string `json:"type"` + Mode string `json:"mode,omitempty"` + HasDefault bool `json:"has_default,omitempty"` +} + +// Relation is a table or view the dialect ships with, such as one of +// PostgreSQL's system catalogs. Kind is 'r' for a table or 'v' for a view, +// and defaults to a table. +type Relation struct { + Catalog string `json:"catalog,omitempty"` + Schema string `json:"schema"` + Name string `json:"name"` + Kind string `json:"kind,omitempty"` + Columns []Column `json:"columns"` +} + +// Column is one of a relation's columns. +type Column struct { + Name string `json:"name"` + Type string `json:"type"` + NotNull bool `json:"not_null,omitempty"` + Array bool `json:"array,omitempty"` + Length int `json:"length,omitempty"` +} + +// Files is what a generator produces: the content of each file it writes, +// keyed by slash-separated path relative to the dialect directory, such as +// "functions.jsonl" or "extensions/hstore/types.jsonl". A generator owns +// only the files it produces; the rest of a dialect, dialect.json above all, +// is written by hand and is neither generated nor checked. +type Files map[string][]byte + +// JSONL encodes records one per line, the way the committed files are +// written: encoding/json's compact form, with a newline after each record. +func JSONL[T any](records []T) ([]byte, error) { + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + for _, record := range records { + if err := enc.Encode(record); err != nil { + return nil, err + } + } + return buf.Bytes(), nil +} + +// Dir returns the dialect directory of an engine, +// internal/engine//dialect, found relative to this source file so +// the working directory does not matter. +func Dir(engine string) (string, error) { + _, file, _, ok := runtime.Caller(0) + if !ok { + return "", errors.New("cannot locate the goldeneye source directory") + } + dir := filepath.Join(filepath.Dir(file), "..", "..", "engine", engine, "dialect") + if _, err := os.Stat(dir); err != nil { + return "", err + } + return filepath.Clean(dir), nil +} + +// Names lists the files in name order, so that a run is reported and +// written the same way every time. +func (f Files) Names() []string { + names := make([]string, 0, len(f)) + for name := range f { + names = append(names, name) + } + sort.Strings(names) + return names +} + +// Write writes the files into dir, creating extension directories as +// needed. Files the generator did not produce are left alone. +func Write(dir string, files Files) error { + for _, name := range files.Names() { + path := filepath.Join(dir, filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + if err := os.WriteFile(path, files[name], 0o644); err != nil { + return err + } + } + return nil +} + +// Check compares the files with the ones committed in dir and returns a +// report of every file that differs, or "" when each matches byte for byte. +func Check(dir string, files Files) (string, error) { + var report strings.Builder + for _, name := range files.Names() { + want, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(name))) + if errors.Is(err, os.ErrNotExist) { + fmt.Fprintf(&report, "%s: not committed\n", name) + continue + } + if err != nil { + return "", err + } + if bytes.Equal(want, files[name]) { + continue + } + fmt.Fprintf(&report, "%s (-committed +database)\n%s", name, Diff(string(want), string(files[name]))) + } + return report.String(), nil +} + +// Diff is a line diff of two texts, marking lines only in want with "-" and +// lines only in got with "+", showing two lines of context around each +// change and eliding the rest. The committed files run to thousands of +// lines and usually differ in a handful, so the common head and tail are +// stripped before the longest common subsequence of the middle is found. +func Diff(want, got string) string { + a := lines(want) + b := lines(got) + head := 0 + for head < len(a) && head < len(b) && a[head] == b[head] { + head++ + } + tail := 0 + for tail < len(a)-head && tail < len(b)-head && a[len(a)-1-tail] == b[len(b)-1-tail] { + tail++ + } + var edits []edit + for _, line := range a[:head] { + edits = append(edits, edit{' ', line}) + } + edits = append(edits, lcsEdits(a[head:len(a)-tail], b[head:len(b)-tail])...) + for _, line := range a[len(a)-tail:] { + edits = append(edits, edit{' ', line}) + } + + const context = 2 + show := make([]bool, len(edits)) + for i, e := range edits { + if e.kind == ' ' { + continue + } + for j := max(0, i-context); j <= min(len(edits)-1, i+context); j++ { + show[j] = true + } + } + var out strings.Builder + elided := false + for i, e := range edits { + if !show[i] { + elided = true + continue + } + if elided && out.Len() > 0 { + out.WriteString("...\n") + } + elided = false + fmt.Fprintf(&out, "%c %s\n", e.kind, e.line) + } + return out.String() +} + +type edit struct { + kind byte // ' ', '-' or '+' + line string +} + +func lines(s string) []string { + if s == "" { + return nil + } + return strings.Split(strings.TrimSuffix(s, "\n"), "\n") +} + +// lcsEdits is the edit script between a and b by longest common subsequence. +func lcsEdits(a, b []string) []edit { + lcs := make([][]int, len(a)+1) + for i := range lcs { + lcs[i] = make([]int, len(b)+1) + } + for i := len(a) - 1; i >= 0; i-- { + for j := len(b) - 1; j >= 0; j-- { + if a[i] == b[j] { + lcs[i][j] = lcs[i+1][j+1] + 1 + } else { + lcs[i][j] = max(lcs[i+1][j], lcs[i][j+1]) + } + } + } + var edits []edit + i, j := 0, 0 + for i < len(a) && j < len(b) { + switch { + case a[i] == b[j]: + edits = append(edits, edit{' ', a[i]}) + i++ + j++ + case lcs[i+1][j] >= lcs[i][j+1]: + edits = append(edits, edit{'-', a[i]}) + i++ + default: + edits = append(edits, edit{'+', b[j]}) + j++ + } + } + for ; i < len(a); i++ { + edits = append(edits, edit{'-', a[i]}) + } + for ; j < len(b); j++ { + edits = append(edits, edit{'+', b[j]}) + } + return edits +} diff --git a/internal/tools/sqlc-duckdb-gen/main.go b/internal/goldeneye/duckdb/duckdb.go similarity index 72% rename from internal/tools/sqlc-duckdb-gen/main.go rename to internal/goldeneye/duckdb/duckdb.go index 9629ba8979..ab0a614a62 100644 --- a/internal/tools/sqlc-duckdb-gen/main.go +++ b/internal/goldeneye/duckdb/duckdb.go @@ -1,47 +1,53 @@ -// Command sqlc-duckdb-gen writes the DuckDB dialect seed files — -// types.jsonl, functions.jsonl and operators.jsonl under -// internal/engine/duckdb/dialect — from a live DuckDB CLI, the same way -// sqlc-pg-gen writes PostgreSQL's from a live server. The CLI must be the +// Package duckdb generates the DuckDB dialect seed under +// internal/engine/duckdb/dialect — types.jsonl, functions.jsonl and +// operators.jsonl — from a live DuckDB CLI, the same way the postgresql +// package generates PostgreSQL's from a live server. The CLI must be the // DuckDB 2.0 build darkwing is pinned against; it is located through the // DUCKDB environment variable, falling back to "duckdb" on PATH. -// -// Usage: -// -// DUCKDB=/path/to/duckdb go run ./internal/tools/sqlc-duckdb-gen [engine-dir] -package main +package duckdb import ( "context" "encoding/json" - "flag" + "errors" "fmt" - "log" "os" "os/exec" - "path/filepath" "sort" "strings" - "github.com/sqlc-dev/sqlc/internal/core/seed" + "github.com/sqlc-dev/sqlc/internal/goldeneye/dialect" ) -func main() { - if err := run(context.Background()); err != nil { - log.Fatal(err) +// Engine is the name of the engine directory the dialect lives under. +const Engine = "duckdb" + +// Locate finds the DuckDB CLI: the DUCKDB environment variable wins, then +// "duckdb" on PATH. +func Locate() (string, error) { + if path := os.Getenv("DUCKDB"); path != "" { + return path, nil + } + path, err := exec.LookPath("duckdb") + if err != nil { + return "", errors.New("no duckdb CLI found: set DUCKDB to the DuckDB 2.0 binary darkwing is pinned against, or put duckdb on PATH") } + return path, nil } -func duckdbPath() string { - if path := os.Getenv("DUCKDB"); path != "" { - return path +// Version reports the release a CLI is. +func Version(ctx context.Context, binary string) (string, error) { + out, err := exec.CommandContext(ctx, binary, "--version").Output() + if err != nil { + return "", fmt.Errorf("duckdb --version: %w", err) } - return "duckdb" + return "DuckDB " + strings.TrimSpace(string(out)), nil } // query runs a SQL statement against an in-memory database and decodes the // CLI's JSON output into rows. -func query(ctx context.Context, sql string, rows any) error { - cmd := exec.CommandContext(ctx, duckdbPath(), "-json", ":memory:", "-c", sql) +func query(ctx context.Context, binary, sql string, rows any) error { + cmd := exec.CommandContext(ctx, binary, "-json", ":memory:", "-c", sql) out, err := cmd.Output() if err != nil { if exit, ok := err.(*exec.ExitError); ok { @@ -105,9 +111,9 @@ func categoryLetter(category *string) string { } } -func readTypes(ctx context.Context) ([]seed.Type, error) { +func readTypes(ctx context.Context, binary string) ([]dialect.Type, error) { var rows []typeRow - err := query(ctx, ` + err := query(ctx, binary, ` SELECT type_name, logical_type, type_category FROM duckdb_types() WHERE database_name = 'system' @@ -119,7 +125,7 @@ ORDER BY type_name`, &rows) // Group the dump's one-row-per-spelling by logical type: the spelling // matching the logical type id is the canonical name, the rest are // aliases. - grouped := map[string]*seed.Type{} + grouped := map[string]*dialect.Type{} var order []string for _, row := range rows { logical := strings.ToLower(row.LogicalType) @@ -128,7 +134,7 @@ ORDER BY type_name`, &rows) } t, ok := grouped[logical] if !ok { - t = &seed.Type{Name: logical, Category: categoryLetter(row.Category)} + t = &dialect.Type{Name: logical, Category: categoryLetter(row.Category)} grouped[logical] = t order = append(order, logical) } @@ -141,11 +147,11 @@ ORDER BY type_name`, &rows) } sort.Strings(order) - types := make([]seed.Type, 0, len(order)+1) + types := make([]dialect.Type, 0, len(order)+1) // "any" stands in for the generic parameters of DuckDB's polymorphic // functions (ANY, T, K, V); the analyzer resolves a call returning it // to the type of the call's first argument. - types = append(types, seed.Type{Name: "any", Category: "U"}) + types = append(types, dialect.Type{Name: "any", Category: "U"}) for _, name := range order { types = append(types, *grouped[name]) } @@ -154,7 +160,7 @@ ORDER BY type_name`, &rows) // typeNames is the set of names the seed declares, for filtering out function // overloads over generic or binder-internal types. -func typeNames(types []seed.Type) map[string]bool { +func typeNames(types []dialect.Type) map[string]bool { names := map[string]bool{} for _, t := range types { names[t.Name] = true @@ -211,9 +217,9 @@ func functionKind(functionType string) string { } } -func readFunctions(ctx context.Context, known map[string]bool) ([]seed.Function, []seed.Operator, error) { +func readFunctions(ctx context.Context, binary string, known map[string]bool) ([]dialect.Function, []dialect.Operator, error) { var rows []functionRow - err := query(ctx, ` + err := query(ctx, binary, ` SELECT DISTINCT function_name, function_type, parameter_types, varargs, return_type FROM duckdb_functions() WHERE database_name = 'system' @@ -224,8 +230,8 @@ ORDER BY function_name, parameter_types::VARCHAR, return_type`, &rows) return nil, nil, err } - var funcs []seed.Function - var operators []seed.Operator + var funcs []dialect.Function + var operators []dialect.Operator seenFunc := map[string]bool{} seenOp := map[string]bool{} for _, row := range rows { @@ -269,7 +275,7 @@ ORDER BY function_name, parameter_types::VARCHAR, return_type`, &rows) continue } seenOp[key] = true - operators = append(operators, seed.Operator{ + operators = append(operators, dialect.Operator{ Name: row.Name, Left: args[0], Right: args[1], @@ -278,7 +284,7 @@ ORDER BY function_name, parameter_types::VARCHAR, return_type`, &rows) continue } - fn := seed.Function{ + fn := dialect.Function{ Name: row.Name, Kind: functionKind(row.FunctionType), Returns: returns, @@ -287,14 +293,14 @@ ORDER BY function_name, parameter_types::VARCHAR, return_type`, &rows) Nullable: row.FunctionType == "aggregate" && !strings.HasPrefix(row.Name, "count"), } for _, arg := range args { - fn.Args = append(fn.Args, seed.Arg{Type: arg}) + fn.Args = append(fn.Args, dialect.Arg{Type: arg}) } if row.Varargs != nil { vararg, ok := seedTypeName(*row.Varargs, known) if !ok { continue } - fn.Args = append(fn.Args, seed.Arg{Type: vararg, Mode: "v"}) + fn.Args = append(fn.Args, dialect.Arg{Type: vararg, Mode: "v"}) } key := fn.Name + "\x00" + fn.Kind + "\x00" + strings.Join(args, "\x00") @@ -307,54 +313,25 @@ ORDER BY function_name, parameter_types::VARCHAR, return_type`, &rows) return funcs, operators, nil } -func writeJSONL[T any](path string, records []T) error { - out, err := os.Create(path) - if err != nil { - return err - } - defer out.Close() - enc := json.NewEncoder(out) - for _, record := range records { - if err := enc.Encode(record); err != nil { - return err - } - } - return nil -} - -func run(ctx context.Context) error { - flag.Parse() - - dir := flag.Arg(0) - if dir == "" { - dir = filepath.Join("internal", "engine", "duckdb") - } - dialectDir := filepath.Join(dir, "dialect") - - version, err := exec.CommandContext(ctx, duckdbPath(), "--version").Output() - if err != nil { - return fmt.Errorf("duckdb --version: %w", err) - } - log.Printf("generating from %s", strings.TrimSpace(string(version))) - - types, err := readTypes(ctx) +// Generate reads the dialect from the CLI. +func Generate(ctx context.Context, binary string) (dialect.Files, error) { + types, err := readTypes(ctx, binary) if err != nil { - return err + return nil, err } - funcs, operators, err := readFunctions(ctx, typeNames(types)) + funcs, operators, err := readFunctions(ctx, binary, typeNames(types)) if err != nil { - return err + return nil, err } - - if err := writeJSONL(filepath.Join(dialectDir, seed.TypesFile), types); err != nil { - return err + files := dialect.Files{} + if files[dialect.TypesFile], err = dialect.JSONL(types); err != nil { + return nil, err } - if err := writeJSONL(filepath.Join(dialectDir, seed.FunctionsFile), funcs); err != nil { - return err + if files[dialect.FunctionsFile], err = dialect.JSONL(funcs); err != nil { + return nil, err } - if err := writeJSONL(filepath.Join(dialectDir, seed.OperatorsFile), operators); err != nil { - return err + if files[dialect.OperatorsFile], err = dialect.JSONL(operators); err != nil { + return nil, err } - log.Printf("wrote %d types, %d functions, %d operators to %s", len(types), len(funcs), len(operators), dialectDir) - return nil + return files, nil } diff --git a/internal/goldeneye/duckdb/duckdb_test.go b/internal/goldeneye/duckdb/duckdb_test.go new file mode 100644 index 0000000000..57608c35e9 --- /dev/null +++ b/internal/goldeneye/duckdb/duckdb_test.go @@ -0,0 +1,37 @@ +package duckdb + +import ( + "context" + "testing" + + "github.com/sqlc-dev/sqlc/internal/goldeneye/dialect" +) + +// TestDialect verifies the committed DuckDB dialect against what the DuckDB +// CLI reports. It skips unless a CLI is found. +func TestDialect(t *testing.T) { + binary, err := Locate() + if err != nil { + t.Skip(err) + } + ctx := context.Background() + version, err := Version(ctx, binary) + if err != nil { + t.Fatal(err) + } + files, err := Generate(ctx, binary) + if err != nil { + t.Fatal(err) + } + dir, err := dialect.Dir(Engine) + if err != nil { + t.Fatal(err) + } + report, err := dialect.Check(dir, files) + if err != nil { + t.Fatal(err) + } + if report != "" { + t.Errorf("%s does not match what %s reports:\n%s", dir, version, report) + } +} diff --git a/internal/goldeneye/go.mod b/internal/goldeneye/go.mod new file mode 100644 index 0000000000..9c3940d0e2 --- /dev/null +++ b/internal/goldeneye/go.mod @@ -0,0 +1,11 @@ +module github.com/sqlc-dev/sqlc/internal/goldeneye + +go 1.26.0 + +require github.com/jackc/pgx/v5 v5.10.0 + +require ( + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + golang.org/x/text v0.29.0 // indirect +) diff --git a/internal/goldeneye/go.sum b/internal/goldeneye/go.sum new file mode 100644 index 0000000000..c0e505b7a6 --- /dev/null +++ b/internal/goldeneye/go.sum @@ -0,0 +1,26 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0= +github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/tools/sqlc-pg-gen/main.go b/internal/goldeneye/postgresql/postgresql.go similarity index 57% rename from internal/tools/sqlc-pg-gen/main.go rename to internal/goldeneye/postgresql/postgresql.go index d0816c7b39..b94ff5e5ef 100644 --- a/internal/tools/sqlc-pg-gen/main.go +++ b/internal/goldeneye/postgresql/postgresql.go @@ -1,22 +1,39 @@ -package main +// Package postgresql generates the PostgreSQL dialect seed under +// internal/engine/postgresql/dialect from a live server: pg_catalog's +// functions, the relations of pg_catalog and information_schema, and one +// directory per contrib extension holding the types and functions CREATE +// EXTENSION adds. types.jsonl and operators.jsonl at the top of the dialect +// are written by hand and are not this package's business. +// +// The server is named by POSTGRESQL_SERVER_URI and has to be the major +// release in Major, since each release adds to its catalogs. +package postgresql import ( + "bytes" "context" "encoding/json" - "flag" + "errors" "fmt" - "io" - "log" "os" - "path/filepath" + "path" "sort" + "strconv" "strings" - "github.com/sqlc-dev/sqlc/internal/core/seed" - "github.com/jackc/pgx/v5" + + "github.com/sqlc-dev/sqlc/internal/goldeneye/dialect" ) +// Engine is the name of the engine directory the dialect lives under. +const Engine = "postgresql" + +// Major is the PostgreSQL major release the dialect is generated from. +// Bumping it is a deliberate change: every release adds functions and +// catalog columns, so regenerate and review the dialect after changing it. +const Major = 16 + // https://dba.stackexchange.com/questions/255412/how-to-select-functions-that-belong-in-a-given-extension-in-postgresql // // Extension functions are added to the public schema @@ -59,10 +76,44 @@ WHERE d.deptype = 'e' ORDER BY t.oid; ` -func main() { - if err := run(context.Background()); err != nil { - log.Fatal(err) +// Locate returns the server to generate from, named by POSTGRESQL_SERVER_URI. +func Locate() (string, error) { + if url := os.Getenv("POSTGRESQL_SERVER_URI"); url != "" { + return url, nil + } + return "", errors.New("POSTGRESQL_SERVER_URI is not set") +} + +// Version reports the release a server is. +func Version(ctx context.Context, url string) (string, error) { + conn, err := pgx.Connect(ctx, url) + if err != nil { + return "", err + } + defer conn.Close(ctx) + var version string + if err := conn.QueryRow(ctx, "SELECT version()").Scan(&version); err != nil { + return "", err } + return version, nil +} + +// checkVersion refuses a server of another major release than the dialect +// is generated from, whose catalogs would differ from the committed ones +// without anything being wrong. +func checkVersion(ctx context.Context, conn *pgx.Conn) error { + var num string + if err := conn.QueryRow(ctx, "SELECT current_setting('server_version_num')").Scan(&num); err != nil { + return err + } + n, err := strconv.Atoi(num) + if err != nil { + return fmt.Errorf("server_version_num %q: %w", num, err) + } + if major := n / 10000; major != Major { + return fmt.Errorf("the dialect is generated from PostgreSQL %d, but the server is PostgreSQL %d", Major, major) + } + return nil } func clean(arg string) string { @@ -73,43 +124,35 @@ func clean(arg string) string { return arg } -// writeFunctions writes procs to destPath as JSONL, one function per line, in -// the form the seed package reads. Both the analysis core and the catalog the -// legacy compiler builds load the result. -func writeFunctions(procs []Proc, destPath string) error { - out, err := os.Create(destPath) - if err != nil { - return err - } - defer out.Close() - - enc := json.NewEncoder(out) +// encodeFunctions encodes procs as JSONL, one function per line, in the form +// the seed package reads. Both the analysis core and the catalog the legacy +// compiler builds load the result. +func encodeFunctions(procs []Proc) ([]byte, error) { + funcs := make([]dialect.Function, 0, len(procs)) for _, proc := range procs { - fn := seed.Function{Name: proc.Name, Returns: proc.ReturnTypeName()} + fn := dialect.Function{Name: proc.Name, Returns: proc.ReturnTypeName()} for _, arg := range proc.Args() { - a := seed.Arg{Name: arg.Name, Type: arg.TypeName(), HasDefault: arg.HasDefault} + a := dialect.Arg{Name: arg.Name, Type: arg.TypeName(), HasDefault: arg.HasDefault} if arg.Mode != "" && arg.Mode != "i" { a.Mode = arg.Mode } fn.Args = append(fn.Args, a) } - if err := enc.Encode(fn); err != nil { - return err - } + funcs = append(funcs, fn) } - return nil + return dialect.JSONL(funcs) } // readExtensionTypes reads the types an extension defines. -func readExtensionTypes(ctx context.Context, conn *pgx.Conn, extension string) ([]seed.Type, error) { +func readExtensionTypes(ctx context.Context, conn *pgx.Conn, extension string) ([]dialect.Type, error) { rows, err := conn.Query(ctx, extensionTypes, extension) if err != nil { return nil, err } defer rows.Close() - var types []seed.Type + var types []dialect.Type for rows.Next() { - var t seed.Type + var t dialect.Type if err := rows.Scan(&t.Name, &t.Category); err != nil { return nil, err } @@ -118,36 +161,18 @@ func readExtensionTypes(ctx context.Context, conn *pgx.Conn, extension string) ( return types, rows.Err() } -// writeTypes writes types to destPath as JSONL, one type per line, in the -// form the seed package reads. -func writeTypes(types []seed.Type, destPath string) error { - out, err := os.Create(destPath) - if err != nil { - return err - } - defer out.Close() - - enc := json.NewEncoder(out) - for _, t := range types { - if err := enc.Encode(t); err != nil { - return err - } - } - return nil -} - // writeRelations appends a schema's relations to out as JSONL, one relation // per line, in the form the seed package reads. -func writeRelations(out io.Writer, schemaName string, relations []Relation) error { +func writeRelations(out *bytes.Buffer, schemaName string, relations []Relation) error { enc := json.NewEncoder(out) for _, relation := range relations { - rec := seed.Relation{ + rec := dialect.Relation{ Catalog: relation.Catalog, Schema: schemaName, Name: relation.Name, } for _, col := range relation.Columns { - c := seed.Column{ + c := dialect.Column{ Name: col.Name, Type: col.Type, NotNull: col.IsNotNull, @@ -201,115 +226,63 @@ func preserveLegacyCatalogBehavior(allProcs []Proc) []Proc { return procs } -func databaseURL() string { - dburl := os.Getenv("DATABASE_URL") - if dburl != "" { - return dburl - } - pgUser := os.Getenv("PG_USER") - pgHost := os.Getenv("PG_HOST") - pgPort := os.Getenv("PG_PORT") - pgPass := os.Getenv("PG_PASSWORD") - pgDB := os.Getenv("PG_DATABASE") - if pgUser == "" { - pgUser = "postgres" - } - if pgPass == "" { - pgPass = "mysecretpassword" - } - if pgPort == "" { - pgPort = "5432" - } - if pgHost == "" { - pgHost = "127.0.0.1" - } - if pgDB == "" { - pgDB = "dinotest" - } - return fmt.Sprintf("postgres://%s:%s@%s:%s/%s?sslmode=disable", pgUser, pgPass, pgHost, pgPort, pgDB) -} - -func run(ctx context.Context) error { - flag.Parse() - - dir := flag.Arg(0) - if dir == "" { - dir = filepath.Join("internal", "engine", "postgresql") - } - - conn, err := pgx.Connect(ctx, databaseURL()) +// Generate reads the dialect from the server. It creates every contrib +// extension it describes, so the server's contrib package has to be +// installed. +func Generate(ctx context.Context, url string) (dialect.Files, error) { + conn, err := pgx.Connect(ctx, url) if err != nil { - return err + return nil, err } defer conn.Close(ctx) - - // The two schemas sqlc knows PostgreSQL by are written to the dialect - // directory rather than to Go: the engine and the analysis core both read - // them from there. Their relations share one file, keyed by schema. - dialectDir := filepath.Join(dir, "dialect") - relationsPath := filepath.Join(dialectDir, seed.RelationsFile) - schemas := []schemaToLoad{ - { - Name: "pg_catalog", - FuncsPath: filepath.Join(dialectDir, seed.FunctionsFile), - }, - { - Name: "information_schema", - }, + if err := checkVersion(ctx, conn); err != nil { + return nil, err } - relationsFile, err := os.Create(relationsPath) + files := dialect.Files{} + + // The two schemas sqlc knows PostgreSQL by. pg_catalog's functions are + // the dialect's standard library; the relations of both share one file, + // keyed by schema. + procs, err := readProcs(ctx, conn, "pg_catalog", extensions) if err != nil { - return err + return nil, err } - defer relationsFile.Close() - - for _, schema := range schemas { - procs, err := readProcs(ctx, conn, schema.Name) - if err != nil { - return err - } - - if schema.Name == "pg_catalog" { - procs = preserveLegacyCatalogBehavior(procs) - } - - relations, err := readRelations(ctx, conn, schema.Name) + if files[dialect.FunctionsFile], err = encodeFunctions(preserveLegacyCatalogBehavior(procs)); err != nil { + return nil, err + } + var relations bytes.Buffer + for _, schema := range []string{"pg_catalog", "information_schema"} { + rels, err := readRelations(ctx, conn, schema) if err != nil { - return err - } - - if schema.FuncsPath != "" { - if err := writeFunctions(procs, schema.FuncsPath); err != nil { - return err - } + return nil, err } - if err := writeRelations(relationsFile, schema.Name, relations); err != nil { - return err + if err := writeRelations(&relations, schema, rels); err != nil { + return nil, err } } + files[dialect.RelationsFile] = relations.Bytes() // Each extension is a directory of its own under the dialect, holding the // functions CREATE EXTENSION adds to the catalog. for _, extension := range extensions { if _, err := conn.Exec(ctx, fmt.Sprintf("CREATE EXTENSION IF NOT EXISTS %q", extension)); err != nil { - return fmt.Errorf("error creating %s: %s", extension, err) + return nil, fmt.Errorf("error creating %s: %s", extension, err) } rows, err := conn.Query(ctx, extensionFuncs, extension) if err != nil { - return err + return nil, err } procs, err := scanProcs(rows) if err != nil { - return err + return nil, err } types, err := readExtensionTypes(ctx, conn, extension) if err != nil { - return err + return nil, err } if len(procs) == 0 && len(types) == 0 { - log.Printf("nothing in %s, skipping", extension) continue } @@ -327,29 +300,18 @@ func run(ctx context.Context) error { return false }) - extensionDir := filepath.Join(dialectDir, "extensions", extension) - if err := os.MkdirAll(extensionDir, 0o755); err != nil { - return err - } + extensionDir := path.Join(dialect.ExtensionsDir, extension) if len(types) > 0 { - if err := writeTypes(types, filepath.Join(extensionDir, seed.TypesFile)); err != nil { - return fmt.Errorf("error generating extension %s: %w", extension, err) + if files[path.Join(extensionDir, dialect.TypesFile)], err = dialect.JSONL(types); err != nil { + return nil, fmt.Errorf("error generating extension %s: %w", extension, err) } } - if err := writeFunctions(procs, filepath.Join(extensionDir, seed.FunctionsFile)); err != nil { - return fmt.Errorf("error generating extension %s: %w", extension, err) + if files[path.Join(extensionDir, dialect.FunctionsFile)], err = encodeFunctions(procs); err != nil { + return nil, fmt.Errorf("error generating extension %s: %w", extension, err) } } - return nil -} - -type schemaToLoad struct { - // name is the name of a schema to load - Name string - // FuncsPath, when set, is the JSONL file this schema's functions are - // written to. - FuncsPath string + return files, nil } // https://www.postgresql.org/docs/current/contrib.html diff --git a/internal/goldeneye/postgresql/postgresql_test.go b/internal/goldeneye/postgresql/postgresql_test.go new file mode 100644 index 0000000000..e892b12a96 --- /dev/null +++ b/internal/goldeneye/postgresql/postgresql_test.go @@ -0,0 +1,37 @@ +package postgresql + +import ( + "context" + "testing" + + "github.com/sqlc-dev/sqlc/internal/goldeneye/dialect" +) + +// TestDialect verifies the committed PostgreSQL dialect against what the +// server reports. It skips unless POSTGRESQL_SERVER_URI names a server. +func TestDialect(t *testing.T) { + url, err := Locate() + if err != nil { + t.Skip(err) + } + ctx := context.Background() + version, err := Version(ctx, url) + if err != nil { + t.Fatal(err) + } + files, err := Generate(ctx, url) + if err != nil { + t.Fatal(err) + } + dir, err := dialect.Dir(Engine) + if err != nil { + t.Fatal(err) + } + report, err := dialect.Check(dir, files) + if err != nil { + t.Fatal(err) + } + if report != "" { + t.Errorf("%s does not match what %s reports:\n%s", dir, version, report) + } +} diff --git a/internal/tools/sqlc-pg-gen/proc.go b/internal/goldeneye/postgresql/proc.go similarity index 80% rename from internal/tools/sqlc-pg-gen/proc.go rename to internal/goldeneye/postgresql/proc.go index 9f0c741439..31933c7f99 100644 --- a/internal/tools/sqlc-pg-gen/proc.go +++ b/internal/goldeneye/postgresql/proc.go @@ -1,4 +1,4 @@ -package main +package postgresql import ( "context" @@ -19,6 +19,18 @@ FROM pg_catalog.pg_proc p LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace WHERE n.nspname::text = $1 AND pg_function_is_visible(p.oid) + -- A function that one of the extensions the dialect describes separately + -- put in the schema, as adminpack puts its own in pg_catalog, belongs to + -- that extension's directory: leaving it out here keeps the output the + -- same whether or not the extension is installed. + AND NOT EXISTS ( + SELECT 1 + FROM pg_catalog.pg_depend d + INNER JOIN pg_catalog.pg_extension e ON e.oid = d.refobjid + WHERE d.classid = 'pg_catalog.pg_proc'::regclass + AND d.objid = p.oid + AND d.deptype = 'e' + AND e.extname = ANY($2)) -- simply order all columns to keep subsequent runs stable ORDER BY 1, 2, 3, 4, 5; ` @@ -139,8 +151,10 @@ func scanProcs(rows pgx.Rows) ([]Proc, error) { return procs, rows.Err() } -func readProcs(ctx context.Context, conn *pgx.Conn, schemaName string) ([]Proc, error) { - rows, err := conn.Query(ctx, catalogFuncs, schemaName) +// readProcs reads the functions of a schema, leaving out the ones that +// belong to any of the named extensions. +func readProcs(ctx context.Context, conn *pgx.Conn, schemaName string, extensions []string) ([]Proc, error) { + rows, err := conn.Query(ctx, catalogFuncs, schemaName, extensions) if err != nil { return nil, err } diff --git a/internal/tools/sqlc-pg-gen/relation.go b/internal/goldeneye/postgresql/relation.go similarity index 99% rename from internal/tools/sqlc-pg-gen/relation.go rename to internal/goldeneye/postgresql/relation.go index af8d51eca1..ecd20e2d1f 100644 --- a/internal/tools/sqlc-pg-gen/relation.go +++ b/internal/goldeneye/postgresql/relation.go @@ -1,4 +1,4 @@ -package main +package postgresql import ( "context"