hypercore-kb / opencog-kb / language-learning / mvp / scripts / fetch-simplewiki.sh

#!/bin/bash
#
# fetch-simplewiki.sh -- Download and process Simple Wikipedia for corpus learning
#
# Uses existing wiki-scrub.pl and wiki-clean-en.sh from the learn repo.
# Output: corpus/incoming/stage1-definitions/*.txt (one file per ~1000 articles)
#
# Usage: bash mvp/scripts/fetch-simplewiki.sh
#

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
MVP_DIR="$(dirname "$SCRIPT_DIR")"
WIKI_TOOLS="$MVP_DIR/../../repos/learn/version-one/download/wikipedia"
OUTPUT_DIR="$MVP_DIR/corpus/incoming/stage1-definitions"
WORK_DIR="$MVP_DIR/corpus/.work/simplewiki"

DUMP_URL="https://dumps.wikimedia.org/simplewiki/latest/simplewiki-latest-pages-articles.xml.bz2"
DUMP_FILE="$WORK_DIR/simplewiki-latest-pages-articles.xml.bz2"

echo "═══ Simple Wikipedia Fetch Pipeline ═══"
echo "  Wiki tools: $WIKI_TOOLS"
echo "  Output dir: $OUTPUT_DIR"
echo "  Work dir:   $WORK_DIR"

# Check tools exist
if [ ! -f "$WIKI_TOOLS/wiki-scrub.pl" ]; then
    echo "ERROR: wiki-scrub.pl not found at $WIKI_TOOLS/"
    echo "  Run: git submodule update --init opencog-kb/repos/learn"
    exit 1
fi

mkdir -p "$WORK_DIR" "$OUTPUT_DIR"

# Step 1: Download dump (skip if already exists)
if [ -f "$DUMP_FILE" ]; then
    echo "  Dump already downloaded: $DUMP_FILE"
else
    echo "  Downloading Simple Wikipedia dump (~500MB)..."
    wget -q --show-progress -O "$DUMP_FILE" "$DUMP_URL"
fi

# Step 2: Extract and scrub wiki markup
SCRUBBED="$WORK_DIR/simplewiki-scrubbed.txt"
if [ -f "$SCRUBBED" ]; then
    echo "  Already scrubbed: $SCRUBBED"
else
    echo "  Extracting and scrubbing wiki markup..."
    bunzip2 < "$DUMP_FILE" | perl "$WIKI_TOOLS/wiki-scrub.pl" > "$SCRUBBED"
    echo "  Scrubbed: $(wc -l < "$SCRUBBED") lines"
fi

# Step 3: Post-cleanup (remove stubs, templates, etc.)
CLEANED="$WORK_DIR/simplewiki-cleaned.txt"
if [ -f "$CLEANED" ]; then
    echo "  Already cleaned: $CLEANED"
else
    echo "  Running post-cleanup..."
    # wiki-clean-en.sh expects input on stdin
    bash "$WIKI_TOOLS/wiki-clean-en.sh" < "$SCRUBBED" > "$CLEANED"
    echo "  Cleaned: $(wc -l < "$CLEANED") lines"
fi

# Step 4: Split into manageable files (~1000 articles per file)
# Articles are separated by blank lines in the scrubbed output.
echo "  Splitting into files..."
awk -v outdir="$OUTPUT_DIR" -v prefix="wiki-" '
BEGIN { file_num = 0; line_count = 0; article_count = 0; per_file = 1000 }
/^$/ { article_count++ }
{
    if (article_count >= per_file) {
        article_count = 0
        file_num++
    }
    fname = outdir "/" prefix sprintf("%04d", file_num) ".txt"
    print >> fname
    line_count++
}
END {
    printf "  Split into %d files (%d lines total)\n", file_num + 1, line_count
}
' "$CLEANED"

echo ""
echo "Done! Files in: $OUTPUT_DIR/"
echo "  $(ls "$OUTPUT_DIR" | wc -l) files"
echo "  $(wc -l "$OUTPUT_DIR"/*.txt 2>/dev/null | tail -1 | awk '{print $1}') total lines"
echo ""
echo "To process: (crawl-dir \"$OUTPUT_DIR\")"