hyper-tps / distribute / hypertps-win64 / scripts / scan-binaries.sh

#!/usr/bin/env bash
# scan-binaries.sh — inventory binary files in this repo by extension + size
#
# Walks the working tree (excluding .git, node_modules, build, .godot)
# and classifies every file as binary or text. Reports:
#   1. Extensions sorted by total bytes (biggest first)
#   2. Top 30 largest binary files repo-wide
#   3. Files that are binary by content sniff but have an extension not
#      in our overlay-candidate allowlist (the things you might miss
#      when writing .gitattributes)
#   4. Files with no extension that are binary (also easy to miss)
#
# Detection: extension allowlist short-circuits as binary; unknown
# extensions sniff the first 8 KB for a NUL byte (the unix `file` utility's
# basic test). Text-shaped Godot resources (.tres, .tscn, .gdshader,
# .import, .uid, .gd) are explicitly classified as TEXT regardless of
# content, since they're collaborative-merge files.
#
# Usage:
#   ./scripts/scan-binaries.sh           # full report
#   ./scripts/scan-binaries.sh --suggest # also print suggested .gitattributes

set -euo pipefail

ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
cd "$ROOT"

# Files we KNOW are binary by extension. Short-circuits the slower NUL sniff.
BIN_EXT_REGEX='\.(glb|gltf|fbx|blend|blender|obj|stl|ply)$|'\
'\.(png|jpg|jpeg|tga|tiff|tif|psd|webp|gif|bmp|ico|heic)$|'\
'\.(exr|hdr)$|'\
'\.(wav|ogg|mp3|flac|opus|m4a|aac)$|'\
'\.(mp4|mov|webm|mkv|avi|m4v)$|'\
'\.(res|pck|so|dll|dylib|exe|bin|dat|raw|cubemap|cubemap_array)$|'\
'\.(zip|tar|gz|bz2|xz|7z|lz4|zst|rar)$|'\
'\.(pdf|fla|swf|woff|woff2|ttf|otf|eot)$'

# Files we explicitly want kept as TEXT for collaboration even though
# Godot binary resources LOOK like assets. Override binary detection.
TEXT_OVERRIDE_REGEX='\.(tres|tscn|gdshader|gdshaderinc|gd|gd\.uid|uid|import|md|txt|json|cfg|yml|yaml|toml|sh|js|ts|html|css|scss)$'

# Paths to skip outright (never reported). Used by both find -prune below
# and a defence-in-depth regex check on read.
EXCLUDE_REGEX='(^|/)(\.git|node_modules|build|\.godot|\.pgh-storage|by-arch|stage)(/|$)'

is_binary_by_sniff () {
  # Reads the first 8 KB; returns 0 (true) if a NUL byte is present.
  local f="$1"
  if head -c 8192 "$f" 2>/dev/null | LC_ALL=C grep -q $'\x00'; then
    return 0
  fi
  return 1
}

classify () {
  # Echoes "binary" or "text".
  local f="$1"
  if [[ "$f" =~ $TEXT_OVERRIDE_REGEX ]]; then
    echo "text"
    return
  fi
  if [[ "$f" =~ $BIN_EXT_REGEX ]]; then
    echo "binary"
    return
  fi
  if is_binary_by_sniff "$f"; then
    echo "binary"
    return
  fi
  echo "text"
}

ext_of () {
  local f="$1"
  local base="${f##*/}"
  if [[ "$base" == *.* ]]; then
    echo ".${base##*.}"
  else
    echo "(noext)"
  fi
}

# ── Collect ──────────────────────────────────────────────────

TMPDIR_INV="$(mktemp -d -t hyper-tps-scan.XXXXXX)"
trap 'rm -rf "$TMPDIR_INV"' EXIT
INVENTORY="$TMPDIR_INV/inventory.tsv"

echo "scanning $ROOT (this can take ~30s for a fresh checkout)..." >&2

while IFS= read -r -d $'\0' f; do
  rel="${f#./}"
  if [[ "$rel" =~ $EXCLUDE_REGEX ]]; then continue; fi
  if [[ -L "$rel" ]]; then continue; fi  # skip symlinks
  size=$(stat --printf='%s' "$rel" 2>/dev/null || echo 0)
  ext="$(ext_of "$rel")"
  kind="$(classify "$rel")"
  printf '%s\t%d\t%s\t%s\n' "$kind" "$size" "$ext" "$rel" >> "$INVENTORY"
done < <(find . \
    \( -path ./.git -o -path ./node_modules -o -path ./build \
       -o -path ./.godot -o -path ./.pgh-storage -o -path ./by-arch \
       -o -path ./stage \) -prune -o \
    -type f -print0)

# ── Reports ──────────────────────────────────────────────────

echo
echo "=== Extensions by total binary size (largest first) ==="
awk -F'\t' '$1 == "binary" { agg[$3] += $2; cnt[$3]++ }
            END { for (e in agg) printf "%12d  %5d  %s\n", agg[e], cnt[e], e }' \
  "$INVENTORY" | sort -rn | head -30

echo
echo "=== Top 30 largest binary files ==="
awk -F'\t' '$1 == "binary" { printf "%12d  %s\n", $2, $4 }' "$INVENTORY" | \
  sort -rn | head -30

echo
echo "=== Total bytes ==="
awk -F'\t' '{ tot[$1] += $2; n[$1]++ } END {
  for (k in tot) printf "  %-7s %12d bytes  %5d files\n", k, tot[k], n[k]
}' "$INVENTORY"

echo
echo "=== Binary files with no extension (easy to miss in .gitattributes) ==="
awk -F'\t' '$1 == "binary" && $3 == "(noext)" { printf "%12d  %s\n", $2, $4 }' \
  "$INVENTORY" | sort -rn | head -10
echo "  (none — good)" 2>/dev/null

# ── Suggested .gitattributes ─────────────────────────────────

if [[ "${1:-}" == "--suggest" ]]; then
  echo
  echo "=== Suggested .gitattributes (based on detected binary extensions) ==="
  awk -F'\t' '$1 == "binary" && $3 != "(noext)" { print $3 }' "$INVENTORY" | \
    sort -u | while read ext; do
      echo "*${ext} filter=hyper-overlay diff=hyper-overlay merge=hyper-overlay -text"
    done
fi

echo
echo "(complete inventory in $INVENTORY — copy somewhere if you want to keep it)"
mv "$INVENTORY" "/tmp/hyper-tps-binary-scan-$(date +%Y%m%d-%H%M%S).tsv"
trap - EXIT
echo "saved to /tmp/hyper-tps-binary-scan-*.tsv"