Debugging Cockroachdb Operator Migrations

export OPERATOR_NAMESPACE="<operator-namespace>"
export CRDB_NAMESPACE="<cockroachdb-namespace>"
export CRDB_HELM_RELEASE="<cockroachdb-release>"
export CRDB_MIGRATION_DIR="${CRDB_MIGRATION_DIR:-$(mktemp -d)}"

kubectl -n "$OPERATOR_NAMESPACE" get deploy cockroach-operator -o jsonpath='{.spec.template.spec.containers[0].image}{"\n"}'
kubectl get crd crdbclusters.crdb.cockroachlabs.com -o jsonpath='{.spec.versions[*].name}{"\n"}'
kubectl get crd crdbclusters.crdb.cockroachlabs.com -o yaml > "$CRDB_MIGRATION_DIR/crdbclusters-crd.yaml"
kubectl get crd crdbclusters.crdb.cockroachlabs.com -o json > "$CRDB_MIGRATION_DIR/crdbclusters-crd.json"
helm -n "$CRDB_NAMESPACE" history "$CRDB_HELM_RELEASE" || true

kubectl -n "$CRDB_NAMESPACE" get crdbcluster -o json | jq -r '
  .items[]
  | [.metadata.name, .apiVersion, (.metadata.labels["app.kubernetes.io/instance"] // ""), (.metadata.labels["crdb.io/migrate"] // ""), (.metadata.labels["crdb.cockroachlabs.com/migration"] // "")]
  | @tsv
'

export CRDBCLUSTER="<metadata.name-from-crdbcluster-list>"
test -n "$CRDBCLUSTER"

kubectl -n "$CRDB_NAMESPACE" get crdbcluster "$CRDBCLUSTER" -o yaml > "$CRDB_MIGRATION_DIR/crdbcluster.yaml"
kubectl -n "$CRDB_NAMESPACE" get crdbcluster "$CRDBCLUSTER" -o json > "$CRDB_MIGRATION_DIR/crdbcluster.json"

export CRDBCLUSTER_API_VERSION="$(jq -r '.apiVersion | split("/")[-1]' "$CRDB_MIGRATION_DIR/crdbcluster.json")"
export CRDBCLUSTER_SCHEMA_JSON="$CRDB_MIGRATION_DIR/crdbcluster-schema.json"
jq -e --arg version "$CRDBCLUSTER_API_VERSION" '
  .spec.versions[] | select(.name == $version) | .schema.openAPIV3Schema
' "$CRDB_MIGRATION_DIR/crdbclusters-crd.json" > "$CRDBCLUSTER_SCHEMA_JSON"

crdb_schema_has() {
  jq -e --arg path "$1" '
    def has_schema_path($schema; $parts):
      if ($parts | length) == 0 then true
      elif (($schema.properties? // {}) | has($parts[0])) then
        has_schema_path($schema.properties[$parts[0]]; $parts[1:])
      else false
      end;
    has_schema_path(.; $path | split("."))
  ' "$CRDBCLUSTER_SCHEMA_JSON" >/dev/null
}

crdb_first_schema_path() {
  for schema_path in "$@"; do
    if crdb_schema_has "$schema_path"; then
      printf '%s\n' "$schema_path"
      return 0
    fi
  done
  printf '\n'
}

export CRDB_MODE_PATH="$(crdb_first_schema_path spec.mode)"
export CRDB_MIGRATION_PATH="$(crdb_first_schema_path status.migration)"
export CRDB_CONDITIONS_PATH="$(crdb_first_schema_path status.conditions)"
export CRDB_OBSERVED_GENERATION_PATH="$(crdb_first_schema_path status.observedGeneration)"

printf '%s\n' \
  "apiVersion=$CRDBCLUSTER_API_VERSION" \
  "mode=$CRDB_MODE_PATH" \
  "migration=$CRDB_MIGRATION_PATH" \
  "conditions=$CRDB_CONDITIONS_PATH" \
  "observedGeneration=$CRDB_OBSERVED_GENERATION_PATH"

jq \
  --arg modePath "$CRDB_MODE_PATH" \
  --arg migrationPath "$CRDB_MIGRATION_PATH" \
  --arg conditionsPath "$CRDB_CONDITIONS_PATH" \
  --arg observedGenerationPath "$CRDB_OBSERVED_GENERATION_PATH" \
  '
  def value($path): if $path == "" then null else getpath($path | split(".")) end;
  {
  apiVersion,
  name: .metadata.name,
  schemaPaths: {
    mode: $modePath,
    migration: $migrationPath,
    conditions: $conditionsPath,
    observedGeneration: $observedGenerationPath
  },
  mode: value($modePath),
  migrateLabel: .metadata.labels["crdb.io/migrate"],
  migrationStatus: .metadata.labels["crdb.cockroachlabs.com/migration"],
  migration: value($migrationPath),
  conditions: value($conditionsPath),
  generation: .metadata.generation,
  observedGeneration: value($observedGenerationPath)
}' "$CRDB_MIGRATION_DIR/crdbcluster.json"

kubectl -n "$OPERATOR_NAMESPACE" logs -l app=cockroach-operator --tail=300 | grep -Ei 'migrationctrl|migration|phase|cert' || true
kubectl -n "$CRDB_NAMESPACE" get crdbnodes -o wide
kubectl -n "$CRDB_NAMESPACE" get crdbnodes -o yaml

kubectl -n "$CRDB_NAMESPACE" get sts <source-statefulset> -o json | jq '{
  replicas: .spec.replicas,
  readyReplicas: .status.readyReplicas,
  migrateLabel: .metadata.labels["crdb.io/migrate"],
  ownerReferences: .metadata.ownerReferences
}'

kubectl -n "$CRDB_NAMESPACE" get sts <source-statefulset> -o yaml
kubectl -n "$CRDB_NAMESPACE" get pods -l app.kubernetes.io/name=cockroachdb -o yaml | grep -E 'name:|ownerReferences:|kind:|uid:|controller:' -A8
kubectl -n "$CRDB_NAMESPACE" get pvc -o yaml | grep -E 'name:|ownerReferences:|kind:|uid:|controller:' -A8

kubectl -n "$CRDB_NAMESPACE" get crdbcluster.crdb.cockroachlabs.com "$CRDBCLUSTER" -o yaml
kubectl -n "$CRDB_NAMESPACE" get crdbcluster.v1beta1.crdb.cockroachlabs.com "$CRDBCLUSTER" -o yaml 2>&1 || true
kubectl -n "$OPERATOR_NAMESPACE" get svc cockroach-webhook-service
kubectl -n "$OPERATOR_NAMESPACE" get endpoints cockroach-webhook-service
kubectl get validatingwebhookconfigurations | grep cockroach