Rocky Config

[adapter]                # Warehouse / source connection
# …

[pipeline.<name>]        # One or more pipelines — discriminated by `type` (default: replication)
# …

# Optional globals:
[state]                  # Embedded state store backend
[cache.schemas]          # Schema (DESCRIBE) cache: enabled, ttl_seconds, replicate, trusted_max_age_seconds, strict_sources
[cost]                   # Cost model for `rocky optimize`
[hook.<event>]           # Lifecycle hooks (one per event)
# Governance (tags, grants, workspace bindings) is NOT a top-level table:
# it lives under [pipeline.<name>.target.governance]. A top-level
# [governance] is refused (deny_unknown_fields).

token     = "${DATABRICKS_TOKEN}"              # hard-required
namespace = "${ROCKY_NAMESPACE:-default}"      # default when unset
workspace = "${WORKSPACE_IDS:-}"               # default to empty string

[adapter]
type = "duckdb"
path = "playground.duckdb"        # omit for in-memory; required if also used for discovery

[adapter]
type      = "databricks"
host      = "${DATABRICKS_HOST}"                 # e.g. dbc-xxxx.cloud.databricks.com (no https://)
http_path = "${DATABRICKS_HTTP_PATH}"            # /sql/1.0/warehouses/<warehouse_id>

[adapter.auth]
token         = "${DATABRICKS_TOKEN}"            # PAT (tried first)
# client_id     = "${DATABRICKS_CLIENT_ID}"      # OAuth M2M (fallback)
# client_secret = "${DATABRICKS_CLIENT_SECRET}"

[adapter]
type     = "snowflake"
account  = "${SNOWFLAKE_ACCOUNT}"
username = "${SNOWFLAKE_USER}"

[adapter.auth]
# OAuth (pre-supplied token, highest priority):
# token = "${SNOWFLAKE_OAUTH_TOKEN}"

# RS256 key-pair JWT (preferred for service principals):
private_key_path = "${SNOWFLAKE_KEY_PATH}"

# Password (lowest priority):
# password = "${SNOWFLAKE_PASSWORD}"

[adapter]
type     = "postgres"                            # or "redshift" (beta)
host     = "${PGHOST}"                           # host or host:port (default 5432 / 5439)
database = "analytics"                           # the connected database = the only valid catalog
username = "${PGUSER}"
password = "${PGPASSWORD}"
# timeout_secs = 300                             # connect timeout + statement_timeout

[adapter.extra]                                  # unknown keys are refused
sslmode = "verify-full"                          # disable | prefer (default) | require | verify-full
# sslrootcert = "/etc/ssl/rds-ca.pem"            # extra PEM roots under verify-full
# port = 6543                                    # wins over host:port
# max_connections = 8
# merge_mode = "on_conflict"                     # postgres < 15 only; needs a unique index on unique_key
# late_binding_views = true                      # redshift only: views WITH NO SCHEMA BINDING

[adapter]
type     = "clickhouse"
host     = "${CLICKHOUSE_HOST}"                  # host or host:port (default 8123, 8443 with secure)
username = "${CLICKHOUSE_USER}"                  # default "default"
password = "${CLICKHOUSE_PASSWORD}"
# database = "default"                           # session default database for unqualified names
# timeout_secs = 300                             # per statement; also max_execution_time

[adapter.extra]                                  # unknown keys are refused
secure = true                                    # HTTPS, certificate always verified
# ca_cert = "/etc/ssl/ch-ca.pem"                 # extra PEM roots (needs secure = true)
# port = 9443                                    # wins over host:port

[adapter]
type     = "sqlserver"
host     = "${MSSQL_HOST}"                       # host, host,port or host:port (default 1433)
database = "analytics"                           # the connected database = the only valid catalog
# exactly ONE auth method:
username = "${MSSQL_USER}"                       # SQL auth (not on Fabric)
password = "${MSSQL_PASSWORD}"
# oauth_token = "${MSSQL_ACCESS_TOKEN}"          # Entra ID access token (not refreshed)
# client_id = "${AZURE_CLIENT_ID}"               # Entra ID service principal, with extra.tenant_id
# client_secret = "${AZURE_CLIENT_SECRET}"

[adapter.extra]                                  # unknown keys are refused
# encrypt = "mandatory"                          # mandatory (default) | strict (TDS 8.0) | optional
# trust_server_certificate = true                # local/test servers only
# ca_cert = "/etc/ssl/corp-ca.pem"
# flavor = "fabric"                              # Fabric Warehouse renderings
# tenant_id = "${AZURE_TENANT_ID}"
# port = 1433
# max_connections = 8

[adapter.fivetran]
type           = "fivetran"
kind           = "discovery"                     # required: fivetran has no data path
destination_id = "${FIVETRAN_DESTINATION_ID}"
api_key        = "${FIVETRAN_API_KEY}"
api_secret     = "${FIVETRAN_API_SECRET}"

[adapter.warehouse]
type = "databricks"
# …

[adapter.source]
type = "fivetran"
kind = "discovery"
# …

[pipeline.raw]
[pipeline.raw.source]
adapter = "source"                 # ref by name

[pipeline.raw.target]
adapter = "warehouse"

[pipeline.raw]
# type = "replication"               # optional, this is the default
strategy         = "incremental"     # or "full_refresh" or "merge"
timestamp_column = "_fivetran_synced"
# merge_keys     = ["id"]            # required when strategy = "merge"
# merge_keys_fallback = ["id"]       # used when merge_keys is unset
metadata_columns = [
    { name = "_loaded_by", type = "STRING", value = "'rocky'" },
]

[pipeline.raw.source]
# adapter = "default"                # optional — first adapter by default
[pipeline.raw.source.schema_pattern]
prefix     = "src__"
separator  = "__"
components = ["client", "regions...", "connector"]
# "name"    = single segment
# "name..." = variable-length (1+)
# Reserved: `table` and `id` are not allowed as component names
# (they're consumed by `--filter` and `[[table_overrides]]`).

[pipeline.raw.target]
catalog_template = "{client}_warehouse"
schema_template  = "staging__{regions}__{connector}"

# Per-`(connector, table)` overrides on top of the pipeline defaults.
# Per-field most-specific-match-wins: each field is supplied by the
# most-specific matching rule that sets it; unrelated rules don't
# clobber it. See `engine/crates/rocky-core/src/config.rs` ::
# `TableOverride` for the full surface.

# Skip diagnostic tables on every connector.
[[pipeline.raw.table_overrides]]
match.table = "_diagnostics_*"        # `*`/`?` glob in TOML only — CLI literals
enabled     = false

# `pii_users` on one specific connector needs a composite merge key.
[[pipeline.raw.table_overrides]]
match.connector = "stripe_main"        # matches conn.id OR conn.schema
match.table     = "pii_users"
merge_keys      = ["user_id", "tenant_id"]

# `audit_log` is append-only — keep it on incremental even when
# the pipeline default is `merge`.
[[pipeline.raw.table_overrides]]
match.connector  = "stripe_main"
match.table      = "audit_log"
strategy         = "incremental"
timestamp_column = "occurred_at"

[pipeline.silver]
type       = "transformation"
models     = "models/silver/**"   # glob, relative to rocky.toml; default "models/**"
depends_on = ["raw"]

# There is no contracts key: a model's contract is auto-discovered as the
# sibling `<model>.contract.toml`, or passed at the CLI via `--contracts <dir>`.

# A transformation target takes an adapter ref (plus optional governance)
# and nothing else — `catalog`/`schema` here are parse errors
# (`deny_unknown_fields`). Each model names its own destination in its
# sidecar `[target]`:
#
#   # models/silver/dim_customers.toml
#   [target]
#   catalog = "analytics"
#   schema  = "marts"
#   table   = "dim_customers"
[pipeline.silver.target]
adapter = "warehouse"

[[pipeline.silver.sources]]
schema = "raw"                     # catalog optional (two-part names)
table  = "orders"

[pipeline.silver.sources.freshness]
loaded_at_field = "_loaded_at"     # DATE / TIMESTAMP column
warn_after      = "12h"            # <N>s | <N>h | <N>d; at least one of the two
error_after     = "24h"            # must be >= warn_after (else E050)
filter          = "status <> 'test'"   # optional WHERE predicate; no `;`

[pipeline.qa]
type       = "quality"
depends_on = ["silver"]

# A quality target is an adapter ref only. The tables to check are listed
# in `[[pipeline.qa.tables]]` (omit `table` to check the whole schema).
[pipeline.qa.target]
adapter = "warehouse"

[[pipeline.qa.tables]]
catalog = "analytics"
schema  = "marts"

# Quality runs execute `row_count`, `custom`, and `assertions` only. Other
# check kinds are inert here — `rocky validate` warns (V034).
[pipeline.qa.checks]
row_count = true

[pipeline.dim_history]
type       = "snapshot"
unique_key = ["customer_id"]     # required: row identity in the source table
updated_at = "updated_at"        # required: change-detection column
depends_on = ["silver"]

[pipeline.dim_history.source]
catalog = "analytics"
schema  = "marts"
table   = "dim_customers"

[pipeline.dim_history.target]
catalog = "analytics_history"
schema  = "snapshots"
table   = "dim_customers_history"   # required: explicit history table

[pipeline.<name>.checks]
enabled      = true
row_count    = true                                      # source vs target row count
column_match = true                                      # source vs target column list
freshness    = { threshold_seconds = 86400 }             # max staleness of newest row
null_rate    = { columns = ["email"], threshold = 0.05, sample_percent = 10 }

[[pipeline.<name>.checks.custom]]
name      = "no_future_dates"
sql       = "SELECT COUNT(*) FROM {target} WHERE created_at > CURRENT_TIMESTAMP()"
threshold = 0                                            # max failing rows

[pipeline.<name>.contract]
required_columns = [
    { name = "id", type = "BIGINT", nullable = false },
]
protected_columns    = ["id", "email"]                   # may not be removed from source
allowed_type_changes = [
    { from = "INT", to = "BIGINT" },                     # widening allowlist
]

[pipeline.<name>.execution]
concurrency           = 16
fail_fast             = false
error_rate_abort_pct  = 50
table_retries         = 1
# adaptive_concurrency is planned but not yet a config field.
# The AIMD throttle primitive exists but isn't wired.

[pipeline.bronze.target.governance]
auto_create_catalogs = true
auto_create_schemas  = true

[pipeline.bronze.target.governance.tags]
managed_by = "rocky"

[[pipeline.bronze.target.governance.grants]]
principal   = "data-readers"
permissions = ["BROWSE", "USE CATALOG", "USE SCHEMA", "SELECT"]

[pipeline.bronze.target.governance.isolation]
enabled       = true
workspace_ids = "${WORKSPACE_IDS:-}"

# Embedded redb (default — no config needed)
[state]
backend = "local"

# S3-backed state sync
[state]
backend = "s3"
bucket  = "my-rocky-state"
prefix  = "prod/"
region  = "us-east-1"

# Tiered: local redb + S3 for durability
[state]
backend = "tiered"
# … S3 fields plus local path

# State-file namespacing (opt-in, default "none")
[state]
backend = "local"
namespacing = "pipeline"   # each pipeline gets its own .rocky-state/<pipeline>.redb