Databricks

POST https://{host}/api/2.0/sql/statements
Authorization: Bearer {token}
Content-Type: application/json

{
  "warehouse_id": "{warehouse_id}",
  "statement": "SELECT * FROM catalog.schema.table LIMIT 10",
  "wait_timeout": "30s",
  "disposition": "INLINE",
  "format": "JSON_ARRAY"
}

{
  "statement_id": "abc-123",
  "status": { "state": "SUCCEEDED" },
  "manifest": {
    "schema": {
      "columns": [
        { "name": "col1", "type_name": "STRING", "position": 0 }
      ]
    },
    "total_row_count": 10
  },
  "result": {
    "data_array": [["value1"], ["value2"]]
  }
}

GET https://{host}/api/2.0/sql/statements/{statement_id}
Authorization: Bearer {token}

POST https://{host}/api/2.0/sql/statements/{statement_id}/cancel

Authorization: Bearer {DATABRICKS_TOKEN}

POST https://{host}/oidc/v1/token
Content-Type: application/x-www-form-urlencoded

grant_type=client_credentials&
client_id={DATABRICKS_CLIENT_ID}&
client_secret={DATABRICKS_CLIENT_SECRET}&
scope=all-apis

{
  "access_token": "eyJ...",
  "token_type": "Bearer",
  "expires_in": 3600
}

if DATABRICKS_TOKEN is set and non-empty:
    use PAT auth
else if DATABRICKS_CLIENT_ID and DATABRICKS_CLIENT_SECRET are set:
    use OAuth M2M
else:
    error: no auth configured

PATCH https://{host}/api/2.1/unity-catalog/catalogs/{catalog_name}
Authorization: Bearer {token}
Content-Type: application/json

{
  "isolation_mode": "ISOLATED"
}

GET https://{host}/api/2.1/unity-catalog/bindings/catalog/{catalog_name}

{
  "bindings": [
    { "workspace_id": 12345, "binding_type": "BINDING_TYPE_READ_WRITE" }
  ]
}

PATCH https://{host}/api/2.1/unity-catalog/bindings/catalog/{catalog_name}
Content-Type: application/json

{
  "add": [
    { "workspace_id": 67890, "binding_type": "BINDING_TYPE_READ_WRITE" }
  ],
  "remove": [
    { "workspace_id": 11111 }
  ]
}

-- Create
CREATE CATALOG IF NOT EXISTS <catalog>

-- Tag (keys/values come from [governance.tags] in rocky.toml; 'managed_by' is always set)
ALTER CATALOG <catalog> SET TAGS (
    'managed_by' = '<pipeline_name>'
    -- plus any tags declared under [governance.tags]
)

-- Inspect
DESCRIBE CATALOG <catalog>
-- Returns rows: (info_name, info_value) — check for 'Catalog Name' row

-- Discover managed catalogs
SELECT catalog_name
FROM system.information_schema.catalog_tags
WHERE tag_name = 'managed_by' AND tag_value = '<pipeline_name>'

-- Create
CREATE SCHEMA IF NOT EXISTS <catalog>.<schema>

-- Tag (keys/values come from [governance.tags]; Rocky always sets 'managed_by')
ALTER SCHEMA <catalog>.<schema> SET TAGS (
    'layer' = 'raw',
    'connector' = '<connector>',
    'managed_by' = '<pipeline_name>'
    -- plus any tags declared under [governance.tags]
)

-- List schemas
SHOW SCHEMAS IN <catalog>

-- Full refresh
SELECT * FROM <source_catalog>.<source_schema>.<table>

-- Incremental (append rows newer than the stored watermark)
SELECT * FROM <source_catalog>.<source_schema>.<table>
WHERE _fivetran_synced > TIMESTAMP '<prior_watermark_literal>'

-- Get column info for comparison
DESCRIBE TABLE <catalog>.<schema>.<table>
-- Returns rows: (col_name, data_type, comment)

-- On a SAFE type widening (e.g. INT -> BIGINT): evolve in place
ALTER TABLE <target_catalog>.<target_schema>.<table> ALTER COLUMN <col> TYPE <wider_type>

-- On an UNSAFE type change: drop, then full refresh on next run
DROP TABLE IF EXISTS <target_catalog>.<target_schema>.<table>

-- Inspect current grants
SHOW GRANTS ON CATALOG <catalog>
-- Returns rows: (principal, action_type, object_type, object_name)

SHOW GRANTS ON SCHEMA <catalog>.<schema>

-- Grant (principal always backtick-quoted)
GRANT BROWSE ON CATALOG <catalog> TO `<principal>`
GRANT USE CATALOG ON CATALOG <catalog> TO `<principal>`
GRANT SELECT ON CATALOG <catalog> TO `<principal>`
GRANT USE SCHEMA ON SCHEMA <catalog>.<schema> TO `<principal>`

-- Revoke
REVOKE BROWSE ON CATALOG <catalog> FROM `<principal>`

-- Supported permission types for reconciliation:
-- BROWSE, USE CATALOG, USE SCHEMA, SELECT, MANAGE, MODIFY
-- Skip these (non-managed): OWNERSHIP, ALL PRIVILEGES, CREATE SCHEMA

-- Single table row count
SELECT COUNT(*) FROM <catalog>.<schema>.<table>

-- Batched row counts (UNION ALL, batches of 200)
SELECT 'cat1' AS c, 'sch1' AS s, 'tbl1' AS t, COUNT(*) AS cnt FROM cat1.sch1.tbl1
UNION ALL
SELECT 'cat1' AS c, 'sch1' AS s, 'tbl2' AS t, COUNT(*) AS cnt FROM cat1.sch1.tbl2
UNION ALL
...

-- Column introspection (batched by schema)
SELECT lower(table_schema), lower(table_name), lower(column_name)
FROM <catalog>.information_schema.columns
WHERE table_schema IN ('schema1', 'schema2', ...)
ORDER BY table_schema, table_name, ordinal_position

^[a-zA-Z0-9_]+$

^[a-zA-Z0-9_ \-\.@]+$