#! /usr/bin/env perl
#
# wiki-scrub.pl
#
# Ad-hoc script to scrub wikipedia xml dumps, outputting only valid
# English-language sentences. This script removes wiki markup, URL's
# tables, images, & etc. It currently seems to be pretty darned
# bullet-proof, although it might handle multi-line refs incorrectly.
# Can also stumble over nested elements...
#
# Example usage:
# cat simplewiki-20080629.xml.bz2 | bunzip2 | ./wiki-scrub.pl
#
# This script creates individual files, one per article, placing these
# in a directory "wiki-stripped", which must already exist. This
# directory name is user-configurable, see immediately below.
#
# Copyright (c) 2008, 2013 Linas Vepstas <linas@linas.org>
#
# TODO: Remove }} .. there seem to be a bunch of these left over.
# also img pix stuff seems to stick around ...
#
#--------------------------------------------------------------------
#
# Directory where to dump the stripped pages.
# this directory must already exist
$page_out_directory = "wiki-stripped";
#--------------------------------------------------------------------
# Need to specify the binmodes, in order for \w to match utf8 chars
use utf8;
binmode STDIN, ':encoding(UTF-8)';
binmode STDOUT, ':encoding(UTF-8)';
$have_text = 0;
$have_infobox = 0;
$have_table = 0;
$have_ptable = 0;
$have_cmnt = 0;
$notfirst = 0;
$page_title = "";
$page_not_open = 1;
$start_processing = 0;
while (<>)
{
if (/<title>(.+?)<\/title>/) {
$page_title = $1;
close PAGE;
$page_not_open = 1;
}
if (/<text xml:space/) { $have_text = 1; $start_processing = 1; }
if (!$start_processing) { next; }
# End of a wiki page.
# If there are any badly-formed tables, etc. then reset the state
# variables as we exit from a wiki page.
if (/<\/text>/) {
$have_text = 0;
$have_infobox = 0;
$have_table = 0;
$have_ptable = 0;
$have_cmnt = 0;
$notfirst = 0;
$start_processing = 0;
}
chop;
# remove the text xml
s/.*<text xml:space="preserve">//;
# kill redirect pages
if (/#REDIRECT/) { $have_text = 0; next; }
if (/#redirect/) { $have_text = 0; next; }
# Remove stuff that's commented out. Don't be greedy(?)!
# Do this before most other processing.
s/<!--.+?-->//g;
if (/<!--/) { $have_text = 0; next; }
if (/-->/) { $have_text = 1; next; }
# kill photo galleries
if (/<gallery>/) { $have_text = 0; }
if (/<gallery .+?>/) { $have_text = 0; }
if (/<\/gallery>/) { $have_text = 1; next; }
# kill tables. These start with {| or |- and end with |}
# tables may be nested.
if (/^:*(\{\||\|\-)/) { $have_text = 0; $have_table++; }
if ($have_table && /^\|\}\s*/) {
$have_table --;
if (0 == $have_table) { $have_text = 1; }
next;
}
if ($have_table) { next; }
if (/<table/) { $have_text = 0; $have_ptable++; }
if (/<\/table/) {
$have_ptable --;
if (0 == $have_ptable) { $have_text = 1; }
next;
}
if ($have_ptable) { next; }
# kill stuff like this: 172||9||23||2||30||1||225||12
# or this: 118||2||||||||||||||118||2
# or this: !Total||105||37
# Sometimes it doesn't start or end with a number, and also there could be
# markup in between like: ||11||3||colspan="2"|-||11||3
s/(.*\|\|)+\d*$//;
# Ignore single-line templates e.g. {{template gorp}}
# Also nested ones e.g. {{math|{{aao|300|120|+}}}}
# Do this before processing multi-line templates
while(/\{\{[^{]+?\}\}/) { s/\{\{[^{]+?\}\}// };
# kill infoxes and other multi-line templates. These may have
# embedded templates.
# Don't be greedy -- some of these, like {{cite}}, have valid text
# both before and after.
my @cb = /\}\}/g;
if ($have_infobox && @cb) {
$have_infobox -= scalar @cb;
if (0 == $have_infobox) {
s/.*\}\}//;
}
}
my @ob = /\{\{/g;
if (@ob) {
if ($have_infobox) {
$have_infobox += scalar @ob;
} else {
$have_infobox += scalar @ob;
$notfirst = 0;
s/\{\{.+$//;
}
}
if ($have_infobox) {
if ($notfirst) { next; }
$notfirst = 1;
}
# remove single-line math markup. Don't be greedy(?)!
# Do this before multi-line math markup.
s/<math>.+?<\/math>//g;
# kill multi-line math markup
if (/<math>/) { $have_text = 0; }
if (/<\/math>/) { $have_text = 1; next; }
# ignore everything that isn't in a text section.
if (0 == $have_text) { next; }
# remove triple and double quotes (wiki bold, italic)
s/\'\'\'//g;
s/\'\'//g;
# remove refs, assumed to sit on one line. Don't be greedy(?)!
s/<ref.+?<\/ref>//g;
# multi-line refs seem to only have {{cite}} inside of them.
# The below seems to work, but should probably be convertedto work
# like multi-line templates.
s/<ref>//g;
s/<\/ref>//g;
s/<ref name.+?>//g;
# Ignore everything of the form ^[[en:title]] (these are tranlsated
# pages) These sometimes have {{Link FA|en}} after them.
if (/^\[\[\w[\w-]+?:.+?\]\]( \{\{Link FA\|\w+\}\})*$/) { next; }
# Ignore headers
if (/^==.+==\s*$/) { next; }
# remove quotes
s/"//g;
# Kill image tags of the form [[Image:Chemin.png|thumb|300px|blah]]
s/\[\[Image:.+?\]\]//g;
# Kill File tags of the form [[File:blah.jpg|thumb|right|350px| blah]]
s/\[\[File:.+?\]\]//g;
# kill wikilinks of the form [[the real link#ugh|The Stand-In Text]]
# also [[Wikipedia:special/blah|The Stand-In Text]]
s/\[\[[\p{Word}\p{Space}#:,–“„”‘!&€\.\/\-\$\*\w '\(\)]+?\|(.*?)\]\]/$1/g;
# Kill ordinary links -- [[Stuff more stuff]]
s/\[\[([\p{Word}\p{Space}:,–“„”‘!&€\.\/\-\+\$\*\w '\(\)]+?)\]\]/$1/g;
# kill weblinks i.e. [http:blah.com/whatever A Cool Site]
s/\[\S+ (.+?)\]/$1/g;
# kill weblinks with no text i.e. [http:blah.com/whatever]
s/\[http(.+?)\]//g;
# ignore misc html markup
s/<references\s*\/>//g;
s/<i>//g;
s/<i .+?>//g;
s/<\/i>//g;
s/<p>//g;
s/<p .+?>//g;
s/<\/p>//g;
s/<b>//g;
s/<b .+?>//g;
s/<\/b>//g;
s/<s>//g;
s/<\/s>//g;
s/<u>//g;
s/<\/u>//g;
s/<em>//g;
s/<\/em>//g;
s/<tt>//g;
s/<\/tt>//g;
s/<title>//g;
s/<\/title>//g;
s/<comment>//g;
s/<\/comment>//g;
s/<username>//g;
s/<\/username>//g;
s/<timestamp>//g;
s/<\/timestamp>//g;
s/<id>//g;
s/<\/id>//g;
s/<pre>//g;
s/<pre .+?>//g;
s/<\/pre>//g;
s/<big>//g;
s/<\/big>//g;
s/<small>//g;
s/<\/small>//g;
s/<center>//g;
s/<Center>//g;
s/<\/center>//g;
s/<inputbox>//g;
s/<\/inputbox>//g;
s/<charinsert>//g;
s/<\/charinsert>//g;
s/<timeline>//g;
s/<\/timeline>//g;
s/<cite>//g;
s/<cite .+?>//g;
s/<\/cite>//g;
s/<Cite>//g;
s/<Cite .+?>//g;
s/<\/Cite>//g;
s/<blockquote>//g;
s/<\/blockquote>//g;
s/<div .+?>//g;
s/<\/div>//g;
s/<font .+?>//g;
s/<\/font>//g;
s/<FONT .+?>//g;
s/<\/FONT>//g;
s/<span .+?>//g;
s/<\/span>//g;
s/<h1>//g;
s/<h1 .+?>//g;
s/<\/h1>//g;
s/<h2>//g;
s/<h2 .+?>//g;
s/<\/h2>//g;
s/<h3>//g;
s/<h3 .+?>//g;
s/<\/h3>//g;
s/<h4>//g;
s/<h4 .+?>//g;
s/<\/h4>//g;
s/<br>//g;
s/<BR>//g;
s/<\/br>//g;
s/<br\/>//g;
s/<br \/>//g;
s/<br .*?>//g;
s/<hr>//g;
s/<hr .+?>//g;
s/<includeonly>//g;
s/<\/includeonly>//g;
s/<noinclude>//g;
s/<\/noinclude>//g;
s/<Typo .+?\/>//g;
s/<nowiki>//g;
s/<nowiki \/>//g;
s/<\/nowiki>//g;
s/__NOTOC__//g;
s/<ul>//g;
s/<\/ul>//g;
s/<li>//g;
s/<li .?>//g;
s/<\/li>//g;
s/<tr>//g;
s/<tr .+?>//g;
s/<\/tr>//g;
s/<td>//g;
s/<td .+?>//g;
s/<\/td>//g;
s/<\/td .+?>//g;
# restore ordinary markup
s/&/&/g;
s/–/-/g;
# s/−/-/g;
s/</</g;
s/>/>/g;
s/°/°/g;
s/•/•/g;
s/ / /g;
# Make sure bulleted lists have a period at the end of them.
# But do try to avoid double-periods.
# Also don't add the period if it is empty.
if (/^\*\S+/ || /^#\S+/ || /^:\S+/ || /^-\S+/ || /^–\S+/) {
# Ignore }} append at the end, if any
# e.g. * Sommerdahl}}
s/\}+$//;
if (!/\.$/) { $_ = $_ . "."; }
}
# kill bullets
s/^\*\*\*//;
s/^\*\*//;
s/^\*//;
s/^#//;
s/^:::::://;
s/^::::://;
s/^:::://;
s/^::://;
s/^:://;
s/^://;
s/^-+//;
# Ignore plain }} lines
s/^\s*\}+$//;
# Ignore plain ]] lines
s/^\s*\]+$//;
# Trim
s/^\s+|\s+$//;
if ($page_not_open) {
$page_not_open = 0;
open PAGE, ">" . $page_out_directory . "/" . $page_title;
binmode PAGE, ':encoding(UTF-8)';
}
if (length $_) { print PAGE "$_\n"; }
}