Source Analysis

mkdir -p workspace/raw/source/decompiled

# Example: JVM with cfr
java -jar cfr.jar target.jar --outputdir workspace/raw/source/decompiled/

# Example: Python with uncompyle6
uncompyle6 -o workspace/raw/source/decompiled/ target.pyc

# Example: .NET with ilspy
ilspycmd target.dll -o workspace/raw/source/decompiled/

# Example: Native with Ghidra headless
analyzeHeadless /tmp/ghidra_project proj -import target.bin -postScript ExportDecompiled.java workspace/raw/source/decompiled/

digraph source_tree_analysis {
    rankdir=TB;

    "Start (source tree)" [shape=doublecircle];
    "Phase ST1: Manifest discovery" [shape=box];
    "Phase ST2: Module enumeration" [shape=box];
    "Phase ST3: Prepare for per-unit analysis" [shape=box];
    "→ Phase 6 (shape-neutral)" [shape=box, style=filled, fillcolor="#f0f0f0"];

    "Start (source tree)" -> "Phase ST1: Manifest discovery";
    "Phase ST1: Manifest discovery" -> "Phase ST2: Module enumeration";
    "Phase ST2: Module enumeration" -> "Phase ST3: Prepare for per-unit analysis";
    "Phase ST3: Prepare for per-unit analysis" -> "→ Phase 6 (shape-neutral)";
}

# Source Tree Survey

**Language:** <primary>
**Manifest:** <path>
**Package name:** <from manifest>
**Source roots:** <list>
**Dependency count:** <count>
**Line count by language:** <table>

# Python: every directory with __init__.py plus every top-level .py file
find <source-root> -name '__init__.py' -exec dirname {} \;
find <source-root> -maxdepth 1 -name '*.py' -not -name '__init__.py'

# Rust: every directory with a mod.rs or every file in src/
find <crate>/src -name '*.rs'

# Go: every directory with at least one .go file
find <source-root> -name '*.go' -exec dirname {} \; | sort -u

# Swift: every directory under Sources/
find Sources -mindepth 2 -maxdepth 2 -type d

# Java/Kotlin: every directory with .java or .kt files
find src/main -name '*.java' -o -name '*.kt' -exec dirname {} \; | sort -u

{
  "modules": [
    {"path": "src/parser", "files": 7, "lines": 1240, "language": "rust"},
    {"path": "src/http", "files": 4, "lines": 680, "language": "rust"}
  ]
}

digraph bundle_pipeline {
    rankdir=TB;

    "Start (bundle target)" [shape=doublecircle];
    "Phase B1: Beautify bundle" [shape=box];
    "Phase B2: Find module boundaries" [shape=box];
    "Phase B3: Extract chunks" [shape=box];
    "Phase B4: Extract functions" [shape=box];
    "Phase B5: Extract strings" [shape=box];
    "→ Phase 6 (shape-neutral)" [shape=box, style=filled, fillcolor="#f0f0f0"];

    "Start (bundle target)" -> "Phase B1: Beautify bundle";
    "Phase B1: Beautify bundle" -> "Phase B2: Find module boundaries";
    "Phase B2: Find module boundaries" -> "Phase B3: Extract chunks";
    "Phase B3: Extract chunks" -> "Phase B4: Extract functions";
    "Phase B4: Extract functions" -> "Phase B5: Extract strings";
    "Phase B5: Extract strings" -> "→ Phase 6 (shape-neutral)";
}

LINE_COUNT=$(wc -l < "$BUNDLE")
BYTE_COUNT=$(wc -c < "$BUNDLE")
if [ "$LINE_COUNT" -lt 100 ] && [ "$BYTE_COUNT" -gt 100000 ]; then
  echo "Minified - needs beautification"
  mkdir -p workspace/raw/source
  BEAUTIFIED="workspace/raw/source/target.js"
  if [ ! -f "$BEAUTIFIED" ]; then
    npx js-beautify --type js -f "$BUNDLE" -o "$BEAUTIFIED"
  fi
  TARGET="$BEAUTIFIED"
else
  TARGET="$BUNDLE"
fi

# Size and line count
ls -lh "$TARGET"
wc -l "$TARGET"

# Detect bundler type
head -100 "$TARGET" > workspace/raw/source/analysis/bundle-header.txt
grep -m1 "webpack\|esbuild\|rollup\|parcel" "$TARGET" || echo "Unknown bundler"

# Module ID markers
grep -n "^/\*\*\*\*\*\*/ \"" "$TARGET" > manifests/webpack-modules.txt

# Export declarations
grep -n "__webpack_require__\.d\|__webpack_exports__" "$TARGET" | head -100 > manifests/webpack-exports.txt

# esbuild source comments
grep -n "^// " "$TARGET" | grep -v "^// $" > manifests/esbuild-markers.txt

# Function expression boundaries
grep -n "^(function\|^!function\|^var [a-z]* = function" "$TARGET" > manifests/iife-boundaries.txt

# Long comment separators
grep -n "^/\*\*\*\*\*\*\*\*" "$TARGET" > manifests/separator-comments.txt

# Export clusters
grep -n "^exports\.\|^module\.exports" "$TARGET" > manifests/export-points.txt

# Large gaps (blank lines often separate modules)
awk 'NF==0 {if(++blank>=3) print NR": --- blank section ---"} NF>0 {blank=0}' "$TARGET" | head -100

mkdir -p workspace/raw/source/chunks

# Webpack-style split
csplit -f workspace/raw/source/chunks/chunk- -n 4 "$TARGET" '/^\/\*\*\*\*\*\*\//' '{*}' 2>/dev/null

# Fall back to line-count splitting if no clear markers
if [ $(ls workspace/raw/source/chunks/ | wc -l) -lt 5 ]; then
  rm workspace/raw/source/chunks/*
  split -l 2000 "$TARGET" workspace/raw/source/chunks/chunk-
fi

# Rename to .js
for f in workspace/raw/source/chunks/chunk-*; do
  [ "${f%.js}" = "$f" ] && mv "$f" "${f}.js"
done

echo '{"chunks": [' > workspace/raw/source/manifests/chunk-manifest.json

for chunk in workspace/raw/source/chunks/*.js; do
  name=$(basename "$chunk")
  lines=$(wc -l < "$chunk")
  size=$(wc -c < "$chunk")

  # Extract first meaningful identifier
  first_func=$(grep -oP "function [a-zA-Z_][a-zA-Z0-9_]*" "$chunk" | head -1)
  first_class=$(grep -oP "class [a-zA-Z_][a-zA-Z0-9_]*" "$chunk" | head -1)

  echo "  {\"file\": \"$name\", \"lines\": $lines, \"bytes\": $size, \"first_func\": \"$first_func\", \"first_class\": \"$first_class\"},"
done >> workspace/raw/source/manifests/chunk-manifest.json

echo ']}' >> workspace/raw/source/manifests/chunk-manifest.json

mkdir -p workspace/raw/source/functions

# Named functions
grep -n "function [a-zA-Z_][a-zA-Z0-9_]*\s*(" "$TARGET" > workspace/raw/source/manifests/function-locations.txt

# Arrow function assignments
grep -n "const [a-zA-Z_][a-zA-Z0-9_]* = (" "$TARGET" >> workspace/raw/source/manifests/function-locations.txt
grep -n "const [a-zA-Z_][a-zA-Z0-9_]* = async" "$TARGET" >> workspace/raw/source/manifests/function-locations.txt

cat > /tmp/extract_funcs.js << 'EOF'
const fs = require('fs');
const code = fs.readFileSync(process.argv[2], 'utf8');

const funcPattern = /function\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*\([^)]*\)\s*\{/g;
let match;
let funcId = 0;
const outDir = process.argv[3] || 'workspace/raw/source/functions';

while ((match = funcPattern.exec(code)) !== null) {
  const name = match[1];
  const start = match.index;
  let depth = 1;
  let i = code.indexOf('{', start) + 1;
  while (depth > 0 && i < code.length) {
    if (code[i] === '{') depth++;
    if (code[i] === '}') depth--;
    i++;
  }
  const funcCode = code.slice(start, i);
  if (funcCode.split('\n').length > 10) {
    fs.writeFileSync(
      `${outDir}/func-${String(funcId++).padStart(4, '0')}-${name}.js`,
      funcCode
    );
  }
}
EOF
node /tmp/extract_funcs.js "$TARGET" workspace/raw/source/functions

echo '{"functions": [' > workspace/raw/source/manifests/function-manifest.json

for func in workspace/raw/source/functions/*.js; do
  name=$(basename "$func" .js)
  lines=$(wc -l < "$func")
  params=$(grep -oP "function[^(]*\([^)]*\)" "$func" | head -1)
  echo "  {\"file\": \"$name\", \"lines\": $lines, \"signature\": \"$params\"},"
done >> workspace/raw/source/manifests/function-manifest.json

echo ']}' >> workspace/raw/source/manifests/function-manifest.json

mkdir -p workspace/raw/source/manifests

# Extract all strings with line numbers (minimum 5 chars)
grep -noP '"[^"]{5,}"' "$TARGET" > workspace/raw/source/manifests/strings-raw.txt

# Categorize
grep -P '"https?://' workspace/raw/source/manifests/strings-raw.txt > workspace/raw/source/manifests/strings-urls.txt
grep -P '"[A-Z][A-Z_]{3,}"' workspace/raw/source/manifests/strings-raw.txt > workspace/raw/source/manifests/strings-constants.txt
grep -P '"--[a-z]' workspace/raw/source/manifests/strings-raw.txt > workspace/raw/source/manifests/strings-cli-flags.txt
grep -P '"Error|Failed|Warning' workspace/raw/source/manifests/strings-raw.txt > workspace/raw/source/manifests/strings-errors.txt

echo '{"string_locations": [' > workspace/raw/source/manifests/string-manifest.json

for chunk in workspace/raw/source/chunks/*.js; do
  name=$(basename "$chunk")
  echo "  {\"chunk\": \"$name\", \"strings\": [" >> workspace/raw/source/manifests/string-manifest.json
  grep -oP '"[^"]{10,}"' "$chunk" | head -20 | while read str; do
    echo "    $str," >> workspace/raw/source/manifests/string-manifest.json
  done
  echo "  ]}," >> workspace/raw/source/manifests/string-manifest.json
done

echo ']}' >> workspace/raw/source/manifests/string-manifest.json

wc -l "$UNIT"            # Know the size
head -50 "$UNIT"         # Get oriented
tail -20 "$UNIT"         # See the end
cat "$UNIT"              # READ THE ENTIRE FILE

# Domain indicators (informative strings are clues in any language)
grep -oE '"[a-zA-Z]{5,}"' "$UNIT" | sort | uniq -c | sort -rn | head -20

# Error messages
grep -oE '"[^"]*[Ee]rror[^"]*"' "$UNIT" | sort -u

# Common action verbs in method/function invocations
grep -oE '\.(read|write|send|receive|parse|validate|handle|process|create|delete|update)\(' "$UNIT" | sort | uniq -c

# JavaScript / TypeScript
grep -oE "function [a-zA-Z_][a-zA-Z0-9_]*" "$UNIT"
grep -oE "(const|let|var) [a-zA-Z_][a-zA-Z0-9_]* = (async )?(function|\()" "$UNIT"
grep -oE "class [A-Z][a-zA-Z0-9_]*" "$UNIT"
grep -oE "exports\.[a-zA-Z_]+|export (const|function|class|default)" "$UNIT"

# Python
grep -oE "^(async )?def [a-zA-Z_][a-zA-Z0-9_]*" "$UNIT"
grep -oE "^class [A-Z][a-zA-Z0-9_]*" "$UNIT"

# Go
grep -oE "^func (\([^)]+\) )?[A-Za-z_][A-Za-z0-9_]*" "$UNIT"
grep -oE "^type [A-Z][a-zA-Z0-9_]* " "$UNIT"

# Rust
grep -oE "^(pub )?fn [a-z_][a-z0-9_]*" "$UNIT"
grep -oE "^(pub )?(struct|enum|trait) [A-Z][a-zA-Z0-9_]*" "$UNIT"

# Swift
grep -oE "^(public |private |internal )?func [a-z_][a-zA-Z0-9_]*" "$UNIT"
grep -oE "^(public |private |internal )?(class|struct|enum|protocol|actor) [A-Z][a-zA-Z0-9_]*" "$UNIT"

# Java / Kotlin
grep -oE "^(public |private )?(class|interface|enum) [A-Z][a-zA-Z0-9_]*" "$UNIT"

# JavaScript / TypeScript
grep -oE "require\(['\"][^'\"]+['\"]\)|from ['\"][^'\"]+['\"]" "$UNIT"

# Python
grep -oE "^(from [a-zA-Z_][a-zA-Z0-9_.]* )?import [a-zA-Z_][a-zA-Z0-9_., ]*" "$UNIT"

# Rust
grep -oE "^use [a-zA-Z_][a-zA-Z0-9_:]+" "$UNIT"

# Swift
grep -oE "^import [A-Z][A-Za-z0-9_]*" "$UNIT"

# Java / Kotlin
grep -oE "^import [a-z][a-zA-Z0-9_.]+" "$UNIT"

# Go: import blocks span multiple lines; read them from the file header
head -50 "$UNIT" | awk '/^import [(\"]/,/^\)$/'

# I/O operations — common verbs across many languages
grep -nE "\.(read|write|get|set|load|save|fetch|send)\b" "$UNIT" | head -20

# Event / signal / pub-sub shapes (varies by language and framework):
#   Node EventEmitter:         .on('name', ...), .emit('name', ...)
#   Python pyee / blinker:      .connect(...), .send(...)
#   Rust tokio / crossbeam:    .notify(...), .subscribe(...)
#   Swift NotificationCenter:  .post(name:...), .addObserver(...)
#   Generic pub/sub:            .publish(...), .subscribe(...), .dispatch(...)
grep -nE "\.(on|emit|connect|notify|subscribe|publish|post|dispatch)\(" "$UNIT" | head -20

# Unit Analysis: <unit name>

## Behavioral Summary
[1-2 sentences: What does this unit DO from a user/system perspective?]

## Key Behaviors Identified

### [Behavior 1 Name]
- **What it does**: [behavioral description]
- **Inputs**: [what it accepts]
- **Outputs**: [what it produces]
- **Evidence**: [function names, strings that indicate this]

### [Behavior 2 Name]
- **What it does**: [behavioral description]
- **Inputs**: [what it accepts]
- **Outputs**: [what it produces]
- **Evidence**: [function names, strings that indicate this]

## Data Handled
- Reads: [what data it reads]
- Writes: [what data it writes]
- Transforms: [what transformations it performs]

## Error Conditions
- [Error 1]: "[exact error message]" - when [condition]
- [Error 2]: "[exact error message]" - when [condition]

## Dependencies
- External: [third-party packages or libraries]
- Internal: [other units it references]

## Domain Indicators
Key strings that reveal purpose:
[informative strings]

## Module Guess
This unit likely belongs to the **[module name]** module because:
- [reason 1]
- [reason 2]

## Needs Deep Dive?
- [ ] Yes - complex behavior requiring full spec
- [ ] No - utility code, can be summarized

## Key Claims

| Claim | Source Location | Confidence |
|-------|----------------|------------|
| [behavioral claim] | <unit>:{line} | inferred |