Profile Jmh Regressions

# Find the PR
gh pr list --head $(git rev-parse --abbrev-ref HEAD) --json number,url

# Read the latest benchmark comment (filtering by content to avoid noise)
gh pr view <NUMBER> --json comments --jq '.comments | map(select(.body | contains("## JMH LDBC Benchmark Comparison"))) | last | .body'

git merge-base HEAD origin/develop

BRANCH=$(git rev-parse --abbrev-ref HEAD | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9]/-/g' | sed 's/--*/-/g' | cut -c1-40)
SERVER_NAME="jmh-prof-${BRANCH}"
KEY_NAME="jmh-prof-key-${BRANCH}"

hcloud ssh-key create --name "$KEY_NAME" --public-key-from-file ~/.ssh/id_ed25519.pub
hcloud server create --name "$SERVER_NAME" --type ccx33 --image ubuntu-24.04 --location fsn1 --ssh-key "$KEY_NAME"

ssh-keygen -f ~/.ssh/known_hosts -R <IP>

ssh -o StrictHostKeyChecking=no root@<IP> \
  'apt-get update -qq && apt-get install -y -qq openjdk-21-jdk-headless git tmux > /dev/null 2>&1 && java -version'

ssh root@<IP> 'cd /tmp && \
  curl -sLO https://github.com/async-profiler/async-profiler/releases/download/v3.0/async-profiler-3.0-linux-x64.tar.gz && \
  tar xzf async-profiler-3.0-linux-x64.tar.gz && \
  echo 1 > /proc/sys/kernel/perf_event_paranoid && \
  echo 0 > /proc/sys/kernel/kptr_restrict'

rsync -az --exclude='.git' --exclude='target' --exclude='.idea' "$(git rev-parse --show-toplevel)/" root@<IP>:/root/ytdb/
ssh root@<IP> 'git config --global --add safe.directory /root/ytdb && \
  git config --global user.email "bench@test" && \
  git config --global user.name "bench" && \
  cd /root/ytdb && git init && git add -A && git commit -m "head" --quiet'

BASE_COMMIT=$(git merge-base HEAD origin/develop)
WORKTREE_DIR="/tmp/ytdb-base-profiling-$$"
rm -rf "$WORKTREE_DIR" && git worktree prune
git worktree add "$WORKTREE_DIR" "$BASE_COMMIT"

rsync -az --exclude='.git' --exclude='target' --exclude='.idea' "$WORKTREE_DIR/" root@<IP>:/root/ytdb-base/
ssh root@<IP> 'cd /root/ytdb-base && git init && git add -A && git commit -m "base" --quiet'

ssh root@<IP> '
  (cd /root/ytdb && chmod +x mvnw && ./mvnw -pl jmh-ldbc -am package -DskipTests -Dspotless.check.skip=true -q -Dmaven.repo.local=/root/.m2-head) &
  (cd /root/ytdb-base && chmod +x mvnw && ./mvnw -pl jmh-ldbc -am package -DskipTests -Dspotless.check.skip=true -q -Dmaven.repo.local=/root/.m2-base) &
  wait'

# Generate presigned URLs locally (boto3 required)
# IMPORTANT: Force HTTPS — the HETZNER_S3_ENDPOINT env var may contain http://
# but Hetzner servers cannot reach the S3 endpoint over plain HTTP (connection timeout).
python3 -c "
import boto3, os
endpoint = os.environ['HETZNER_S3_ENDPOINT']
if endpoint.startswith('http://'):
    endpoint = 'https://' + endpoint[len('http://'):]
s3 = boto3.client('s3',
    endpoint_url=endpoint,
    aws_access_key_id=os.environ['HETZNER_S3_ACCESS_KEY'],
    aws_secret_access_key=os.environ['HETZNER_S3_SECRET_KEY'])
for key in ['ldbc/ldbc-sf1-composite-merged-fk.tar.zst', 'ldbc/curated-params-v3.json', 'ldbc/factor-tables.json']:
    url = s3.generate_presigned_url('get_object',
        Params={'Bucket': 'bench-cache', 'Key': key},
        ExpiresIn=7200)
    print(f'{key}: {url}')
"

# Download and extract CSV dataset to HEAD
# Note: the tar archive contains a top-level sf1/ directory, so extract into ldbc-dataset/
# to get the expected ldbc-dataset/sf1/static/ and ldbc-dataset/sf1/dynamic/ layout.
ssh root@<IP> 'apt-get install -y -qq zstd > /dev/null 2>&1 && \
  mkdir -p /root/ytdb/jmh-ldbc/target/ldbc-dataset && \
  cd /root/ytdb/jmh-ldbc/target/ldbc-dataset && \
  curl -sS "<CSV_PRESIGNED_URL>" | zstd -dc | tar xf - && \
  echo "Dataset ready" && ls sf1/static/ sf1/dynamic/'

# Copy CSV dataset to BASE
ssh root@<IP> 'cp -r /root/ytdb/jmh-ldbc/target/ldbc-dataset /root/ytdb-base/jmh-ldbc/target/'

# Install canonical curated params into HEAD DB directory
ssh root@<IP> 'mkdir -p /root/ytdb/jmh-ldbc/target/ldbc-bench-db && \
  curl -sS -o /root/ytdb/jmh-ldbc/target/ldbc-bench-db/factor-tables.json "<FACTOR_TABLES_URL>" && \
  curl -sS -o /root/ytdb/jmh-ldbc/target/ldbc-bench-db/curated-params-v3.json "<CURATED_PARAMS_URL>" && \
  echo "HEAD curated params installed"'

# Install canonical curated params into BASE DB directory
ssh root@<IP> 'mkdir -p /root/ytdb-base/jmh-ldbc/target/ldbc-bench-db && \
  curl -sS -o /root/ytdb-base/jmh-ldbc/target/ldbc-bench-db/factor-tables.json "<FACTOR_TABLES_URL>" && \
  curl -sS -o /root/ytdb-base/jmh-ldbc/target/ldbc-bench-db/curated-params-v3.json "<CURATED_PARAMS_URL>" && \
  echo "BASE curated params installed"'

# HEAD (run first)
ssh root@<IP> 'rm -f /tmp/jmh*.lock && cd /root/ytdb/jmh-ldbc && java \
  --add-opens java.base/java.lang=ALL-UNNAMED \
  --add-opens java.base/java.lang.reflect=ALL-UNNAMED \
  --add-opens java.base/java.lang.invoke=ALL-UNNAMED \
  --add-opens java.base/java.io=ALL-UNNAMED \
  --add-opens java.base/java.nio=ALL-UNNAMED \
  --add-opens java.base/java.util=ALL-UNNAMED \
  --add-opens java.base/java.util.concurrent=ALL-UNNAMED \
  --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED \
  --add-opens java.base/java.net=ALL-UNNAMED \
  --add-opens java.base/sun.nio.ch=ALL-UNNAMED \
  --add-opens java.base/sun.nio.cs=ALL-UNNAMED \
  --add-opens java.base/sun.security.x509=ALL-UNNAMED \
  --add-opens jdk.unsupported/sun.misc=ALL-UNNAMED \
  -Xms4096m -Xmx4096m \
  -jar target/youtrackdb-jmh-ldbc-*.jar \
  "LdbcSingleThread.*ic5_newGroups" -f 1 -wi 0 -i 1 -r 1s -t 1'

# BASE (run after HEAD completes — JMH global lock prevents parallel runs)
ssh root@<IP> 'rm -f /tmp/jmh*.lock && cd /root/ytdb-base/jmh-ldbc && java \
  --add-opens java.base/java.lang=ALL-UNNAMED \
  --add-opens java.base/java.lang.reflect=ALL-UNNAMED \
  --add-opens java.base/java.lang.invoke=ALL-UNNAMED \
  --add-opens java.base/java.io=ALL-UNNAMED \
  --add-opens java.base/java.nio=ALL-UNNAMED \
  --add-opens java.base/java.util=ALL-UNNAMED \
  --add-opens java.base/java.util.concurrent=ALL-UNNAMED \
  --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED \
  --add-opens java.base/java.net=ALL-UNNAMED \
  --add-opens java.base/sun.nio.ch=ALL-UNNAMED \
  --add-opens java.base/sun.nio.cs=ALL-UNNAMED \
  --add-opens java.base/sun.security.x509=ALL-UNNAMED \
  --add-opens jdk.unsupported/sun.misc=ALL-UNNAMED \
  -Xms4096m -Xmx4096m \
  -jar target/youtrackdb-jmh-ldbc-*.jar \
  "LdbcSingleThread.*ic5_newGroups" -f 1 -wi 0 -i 1 -r 1s -t 1'

ssh root@<IP> 'cat > /root/run-bench.sh << '\''SCRIPT'\''
#!/bin/bash
DIR=$1        # /root/ytdb or /root/ytdb-base
BENCH=$2      # benchmark regex
shift 2
ARGS="$@"     # JMH args (shift+$@ preserves spaces in multi-word args)

JVM_ARGS="--add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED --add-opens java.base/java.lang.invoke=ALL-UNNAMED --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.util.concurrent=ALL-UNNAMED --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens java.base/java.net=ALL-UNNAMED --add-opens java.base/sun.nio.ch=ALL-UNNAMED --add-opens java.base/sun.nio.cs=ALL-UNNAMED --add-opens java.base/sun.security.x509=ALL-UNNAMED --add-opens jdk.unsupported/sun.misc=ALL-UNNAMED -Xms4096m -Xmx4096m"

cd $DIR/jmh-ldbc && java $JVM_ARGS \
  -jar target/youtrackdb-jmh-ldbc-*.jar \
  "$BENCH" $ARGS
SCRIPT
chmod +x /root/run-bench.sh'

ssh root@<IP> '/root/run-bench.sh /root/ytdb "<benchmark-regex>" "<jmh-args>"'
ssh root@<IP> '/root/run-bench.sh /root/ytdb-base "<benchmark-regex>" "<jmh-args>"'

ssh root@<IP> 'cat > /root/run-profile.sh << '\''SCRIPT'\''
#!/bin/bash
VERSION=$1    # head or base
DIR=$2        # /root/ytdb or /root/ytdb-base
BENCH=$3      # benchmark regex
shift 3
ARGS="$@"     # JMH args (shift+$@ preserves spaces in multi-word args)

JVM_ARGS="--add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED --add-opens java.base/java.lang.invoke=ALL-UNNAMED --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.util.concurrent=ALL-UNNAMED --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens java.base/java.net=ALL-UNNAMED --add-opens java.base/sun.nio.ch=ALL-UNNAMED --add-opens java.base/sun.nio.cs=ALL-UNNAMED --add-opens java.base/sun.security.x509=ALL-UNNAMED --add-opens jdk.unsupported/sun.misc=ALL-UNNAMED -Xms4096m -Xmx4096m"

mkdir -p /root/profiles/$VERSION

cd $DIR/jmh-ldbc && java $JVM_ARGS \
  -jar target/youtrackdb-jmh-ldbc-*.jar \
  "$BENCH" $ARGS \
  -prof "async:libPath=/tmp/async-profiler-3.0-linux-x64/lib/libasyncProfiler.so;output=collapsed;dir=/root/profiles/$VERSION;event=cpu"
SCRIPT
chmod +x /root/run-profile.sh'

ssh root@<IP> '/root/run-profile.sh <version> /root/ytdb<-base> "<benchmark-regex>" "<jmh-args>"'

# Filter out non-measurement stacks
grep -vE 'tearDown|WALVacuum|G1Conc|G1ParScan|GCThread|GangWorker|VMThread|CompilerThread|ServiceThread|SafepointSynchronize|SafepointCleanup|MonitorDeflation' <file.csv> > <file-filtered.csv>

awk -F";" '{split($NF, a, " "); method=a[1]; samples=a[2]; if(method != "") leaf[method]+=samples} END {for(m in leaf) print leaf[m], m}' <file-filtered.csv> | sort -rn | head -30