Diagnosing Cockroachdb Helm Deployments

attempt to grant extra privileges

kubectl auth can-i create clusterroles.rbac.authorization.k8s.io
kubectl auth can-i create clusterrolebindings.rbac.authorization.k8s.io
kubectl auth can-i get nodes

kubectl -n "$OPERATOR_NAMESPACE" get svc cockroach-webhook-service
kubectl -n "$OPERATOR_NAMESPACE" get endpoints cockroach-webhook-service
kubectl get validatingwebhookconfigurations | grep cockroach

kubectl get validatingwebhookconfiguration cockroach-webhook-config \
  -o jsonpath='{.webhooks[0].clientConfig.caBundle}' | base64 -d | openssl x509 -noout -dates -subject -issuer

jq \
  --arg modePath "$CRDB_MODE_PATH" \
  --arg desiredImagePath "$CRDB_DESIRED_IMAGE_PATH" \
  --arg observedGenerationPath "$CRDB_OBSERVED_GENERATION_PATH" \
  --arg reconciledPath "$CRDB_RECONCILED_PATH" \
  --arg readyNodesPath "$CRDB_READY_NODES_PATH" \
  --arg statusImagePath "$CRDB_STATUS_IMAGE_PATH" \
  --arg statusVersionPath "$CRDB_STATUS_VERSION_PATH" \
  --arg actionsPath "$CRDB_ACTIONS_PATH" \
  --arg conditionsPath "$CRDB_CONDITIONS_PATH" \
  '
  def value($path): if $path == "" then null else getpath($path | split(".")) end;
  {
  apiVersion,
  name: .metadata.name,
  schemaPaths: {
    mode: $modePath,
    desiredImage: $desiredImagePath,
    observedGeneration: $observedGenerationPath,
    reconciled: $reconciledPath,
    readyNodes: $readyNodesPath,
    statusImage: $statusImagePath,
    statusVersion: $statusVersionPath,
    actions: $actionsPath,
    conditions: $conditionsPath
  },
  mode: value($modePath),
  desiredImage: value($desiredImagePath),
  generation: .metadata.generation,
  observedGeneration: value($observedGenerationPath),
  reconciled: value($reconciledPath),
  readyNodes: value($readyNodesPath),
  statusImage: value($statusImagePath),
  statusVersion: value($statusVersionPath),
  actions: value($actionsPath),
  conditions: value($conditionsPath)
}' "$CRDB_DIAG_DIR/crdbcluster.json"

kubectl -n "$CRDB_NAMESPACE" get crdbnodes \
  -o 'custom-columns=NAME:.metadata.name,GENERATION:.metadata.generation,OBSERVED:.status.observedGeneration,DECOMMISSION:.status.decommission,REVISION:.metadata.annotations.crdb\.cockroachlabs\.com/clusterNodeRevision,NODE_ID:.status.nodeID'

kubectl -n "$CRDB_NAMESPACE" get pods -l app.kubernetes.io/name=cockroachdb -o wide
kubectl -n "$CRDB_NAMESPACE" describe pod <crdb-pod>
kubectl -n "$CRDB_NAMESPACE" logs <crdb-pod> -c cockroachdb --tail=200
kubectl -n "$CRDB_NAMESPACE" logs <crdb-pod> -c cockroachdb --previous
kubectl -n "$CRDB_NAMESPACE" get pod <crdb-pod> -o jsonpath='{.spec.containers[0].readinessProbe}{"\n"}'
kubectl -n "$CRDB_NAMESPACE" get pods -l app.kubernetes.io/name=cockroachdb \
  -o 'custom-columns=NAME:.metadata.name,IMAGE:.spec.containers[0].image,PHASE:.status.phase,READY:.status.containerStatuses[0].ready,NODE:.spec.nodeName'

jq \
  --arg desiredImagePath "$CRDB_DESIRED_IMAGE_PATH" \
  --arg statusImagePath "$CRDB_STATUS_IMAGE_PATH" \
  --arg statusVersionPath "$CRDB_STATUS_VERSION_PATH" \
  --arg actionsPath "$CRDB_ACTIONS_PATH" \
  --arg conditionsPath "$CRDB_CONDITIONS_PATH" \
  '
  def value($path): if $path == "" then null else getpath($path | split(".")) end;
  {
  apiVersion,
  name: .metadata.name,
  desiredImagePath: $desiredImagePath,
  desiredImage: value($desiredImagePath),
  statusImagePath: $statusImagePath,
  statusImage: value($statusImagePath),
  statusVersionPath: $statusVersionPath,
  statusVersion: value($statusVersionPath),
  actionsPath: $actionsPath,
  actions: value($actionsPath),
  conditionsPath: $conditionsPath,
  conditions: value($conditionsPath),
  annotations: .metadata.annotations
}' "$CRDB_DIAG_DIR/crdbcluster.json"

jq '.metadata.annotations' "$CRDB_DIAG_DIR/crdbcluster.json"
kubectl -n "$CRDB_NAMESPACE" get jobs
kubectl -n "$CRDB_NAMESPACE" describe job <version-checker-job>
kubectl -n "$CRDB_NAMESPACE" logs -l job-name=<version-checker-job>
kubectl -n "$CRDB_NAMESPACE" get pods -l app.kubernetes.io/name=cockroachdb \
  -o 'custom-columns=NAME:.metadata.name,IMAGE:.spec.containers[0].image,REVISION:.metadata.annotations.crdb\.cockroachlabs\.com/nodeRevision,PHASE:.status.phase'

kubectl -n "$CRDB_NAMESPACE" get service,endpoints -o wide
export CRDB_SERVICE="<sql-or-public-service-name-from-output>"
export CRDB_JOIN_SERVICE="<join-service-name-from-output>"
test -n "$CRDB_SERVICE"
test -n "$CRDB_JOIN_SERVICE"

kubectl -n "$CRDB_NAMESPACE" get service "$CRDB_SERVICE" -o yaml
kubectl -n "$CRDB_NAMESPACE" get service "$CRDB_JOIN_SERVICE" -o yaml
kubectl -n "$CRDB_NAMESPACE" get endpoints "$CRDB_SERVICE"
kubectl -n "$CRDB_NAMESPACE" get endpoints "$CRDB_JOIN_SERVICE"

kubectl -n "$CRDB_NAMESPACE" exec <crdb-pod> -c cockroachdb -- \
  nslookup "$CRDB_SERVICE.$CRDB_NAMESPACE.svc.cluster.local" 2>&1 || true

kubectl -n "$CRDB_NAMESPACE" exec <crdb-pod> -c cockroachdb -- \
  nslookup "$CRDB_JOIN_SERVICE.$CRDB_NAMESPACE.svc.cluster.local" 2>&1 || true

helm template <release> <chart> -n <namespace> -f values.yaml >/tmp/rendered.yaml
kubectl -n "$CRDB_NAMESPACE" get secret,configmap | grep -E 'cockroach|crdb|cert|ca|tls'
jq --arg certificatesPath "$CRDB_CERTIFICATES_PATH" '
  def value($path): if $path == "" then null else getpath($path | split(".")) end;
  {certificatesPath: $certificatesPath, certificates: value($certificatesPath)}
' "$CRDB_DIAG_DIR/crdbcluster.json"
kubectl -n "$CRDB_NAMESPACE" get pod <crdb-pod> -o jsonpath='{.spec.containers[*].name}{"\n"}'
kubectl -n "$CRDB_NAMESPACE" logs <crdb-pod> -c cert-reloader --tail=100

kubectl -n <namespace> describe pod <pod-name>
kubectl -n <namespace> get pvc -o wide
kubectl get storageclass
kubectl get nodes -L topology.kubernetes.io/region,topology.kubernetes.io/zone
kubectl -n <namespace> exec <crdb-pod> -c cockroachdb -- df -h /cockroach/cockroach-data

kubectl -n "$CRDB_NAMESPACE" exec <ready-crdb-pod> -c cockroachdb -- \
  /cockroach/cockroach node status --decommission

kubectl -n "$CRDB_NAMESPACE" get crdbnodes -o json | jq '[.items[] | select(.status.decommission != null and .status.decommission != "")] | {count: length, nodes: [.[] | {name: .metadata.name, decommission: .status.decommission}]}'

kubectl -n "$OPERATOR_NAMESPACE" logs -l app=cockroach-operator --tail=300 | grep -Ei 'decommission|drain|scale|blocking_ranges' || true

kubectl -n "$CRDB_NAMESPACE" patch crdbcluster "$CRDBCLUSTER" --type=merge -p '{"spec":{"mode":"Disabled"}}'

# Resume reconciliation:
kubectl -n "$CRDB_NAMESPACE" patch crdbcluster "$CRDBCLUSTER" --type=merge -p '{"spec":{"mode":"MutableOnly"}}'

kubectl -n "$OPERATOR_NAMESPACE" rollout restart deploy/cockroach-operator

helm -n "$CRDB_NAMESPACE" upgrade "$CRDB_HELM_RELEASE" <cockroachdb-chart> \
  --reuse-values \
  --set-string cockroachdb.crdbCluster.timestamp="$(date -u +%Y-%m-%dT%H:%M:%SZ)"