#!/usr/bin/perl -w
use strict;
use XML::Parser;
$| = 1;

my ($title, @text);
my $n = 0;
my $t = my $t0 = time;
my $len = 0;

sub handle_start {
    my ($expat, $tag) = @_;
    if ($tag eq 'page') { $title = ""; @text = (); }
    elsif ($tag eq 'title') { $expat->setHandlers(Char => \&handle_char_title); }
    elsif ($tag eq 'text') { $expat->setHandlers(Char => \&handle_char_text); }
}

sub handle_char_title { $title .= $_[1]; }
sub handle_char_text { push @text, $_[1] if $_[1] =~ /\{\{\s*(ns:|(FULL|TALK|SUBJECT|ARTICLE|NAME)(PAGENAME|SPACE)E\s*\}\})/; }

sub handle_end {
    my ($expat, $tag) = @_;
    if ($tag eq 'page') {
	$n++;
	if (time >= $t+1) {
	    my $dt = time - $t0;
	    my $b = $expat->current_byte;
	    my $msg = sprintf "Processed %d pages (%.2f/s) = %d bytes (%.2f/s) in %d sec...", $n, $n/$dt, $b, $b/$dt, $dt;
	    $len = length $msg;
	    print STDERR "\r", $msg;
	    $t = time;
	}
	# my $text = join "", @text;
	# warn "\r[[$title]]: \"", substr($text, 0, 40), "...\"\n"; sleep 1;
	if (grep(/\{\{\s*ns:/, @text) and grep(/\{\{\s*(FULL|TALK|SUBJECT|ARTICLE|NAME)(PAGENAME|SPACE)E\s*\}\}/, @text)) {
	    print STDERR "\r", " " x $len, "\r" if $len; $len = 0;
	    print $title, "\n";
	}
	# elsif (grep(/^\#REDIRECT\b/i, @text)) { warn "[[$title]] is a redirect!", " " x 40, "\n"; }
    }
    elsif ($tag eq 'title' or $tag eq 'text') { $expat->setHandlers(Char => undef); }
}

my $parser = XML::Parser->new(ProtocolEncoding => 'UTF-8',
			      Handlers => {Start => \&handle_start,
					   End   => \&handle_end});
binmode STDIN, ':utf8';
binmode STDOUT, ':utf8';
binmode STDERR, ':utf8';
$parser->parse(\*STDIN);
warn "\n";

__END__

<mediawiki xmlns="http://www.mediawiki.org/xml/export-0.3/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.mediawiki.org/xml/export-0.3/ http://www.mediawiki.org/xml/export-0.3.xsd" version="0.3" xml:lang="en">
  <siteinfo>
    <sitename>Wikipedia</sitename>
    <base>http://en.wikipedia.org/wiki/Main_Page</base>
    <generator>MediaWiki 1.14alpha</generator>
    <case>first-letter</case>
      <namespaces>
      <namespace key="-2">Media</namespace>
      <namespace key="-1">Special</namespace>
      <namespace key="0" />
      <namespace key="1">Talk</namespace>
      <namespace key="2">User</namespace>
      <namespace key="3">User talk</namespace>
      <namespace key="4">Wikipedia</namespace>
      <namespace key="5">Wikipedia talk</namespace>
      <namespace key="6">Image</namespace>
      <namespace key="7">Image talk</namespace>
      <namespace key="8">MediaWiki</namespace>
      <namespace key="9">MediaWiki talk</namespace>
      <namespace key="10">Template</namespace>
      <namespace key="11">Template talk</namespace>
      <namespace key="12">Help</namespace>
      <namespace key="13">Help talk</namespace>
      <namespace key="14">Category</namespace>
      <namespace key="15">Category talk</namespace>
      <namespace key="100">Portal</namespace>
      <namespace key="101">Portal talk</namespace>
    </namespaces>
  </siteinfo>
  <page>
    <title>AmericanSamoa</title>
    <id>6</id>
    <revision>
      <id>133452270</id>
      <timestamp>2007-05-25T17:12:06Z</timestamp>
      <contributor>
        <username>Gurch</username>
        <id>241822</id>
      </contributor>
      <minor />
      <comment>Revert edit(s) by [[Special:Contributions/Ngaiklin|Ngaiklin]] to last version by [[Special:Contributions/Docu|Docu]]</comment>
      <text xml:space="preserve">#REDIRECT [[American Samoa]]{{R from CamelCase}}</text>
    </revision>
  </page>