#!/usr/bin/perl

use utf8;

binmode(STDIN, ":utf8");
binmode(STDOUT, ":utf8");
binmode(STDERR, ":utf8");

$full = "";
while(<>) {
	$line =$_;
	$line =~ s/[ \t]+/ /g;
	$oldline = "";
	while ($line ne $oldline) {
		$oldline = $line;
		while ($line =~ /\[\[(?:[^[|\]]+\|)?([^[|\}]+)\]\]/) {
			$line =~ s/\[\[(?:[^[|\]]+\|)?([^[|\}]+)\]\]/$1/;
		}
		$line =~ s/\[(?:http|ftp):\/\/(?:[^[\] ]+ )?([^[\]]+)\]/$1/;
	}

	$line =~ s/(^|[^'])''(?=[^']|$)/$1/g;
	$line =~ s/(^|[^'])'''''(?=[^']|$)/$1/g;
	$line =~ s/(^|[^'])'''(?=[^']|$)/$1/g;

	$line =~ s/<\/?blockquote>//g;
	$line =~ s/<\/?b>//g;
	$line =~ s/<\/?big>//g;
	$line =~ s/<\/?center>//g;
	$line =~ s/<\/?s>//g;
	$line =~ s/<\/?small>//g;
	$line =~ s/<\/?span(?: [^>]*)?>//g;
	$line =~ s/<\/?tt>//g;
	$line =~ s/<\/?u>//g;
	
	$line =~ s/<\/?su(?:b|p)>//g;

	$line =~ s/<\/?article>//g;
	$line =~ s/^ *=====(.+)===== *$/$1/;
	$line =~ s/^ *====(.+)==== *$/$1/;
	$line =~ s/^ *===(.+)=== *$/$1/;
	$line =~ s/^ *==(.+)== *$/$1/;
	$line =~ s/^ *=(.+)= *$/$1/;

#	$line =~ s/{{[^}]+\|([^}]*)}}/$1/g;

	## what normalisation do we want for list items?
	$line =~ s/^[:*#;]+//;
	$line =~ s/ +$//mg;
	$line =~ s/^ +//mg;

	$line =~ s/{{IAST\|([^}]+(?:}[^}]+)*)}}/$1/msg;
	$line =~ s/{{IPA[^}]+(?:}[^}]+)*}}/<ipa\/>/msg;
	$line =~ s/{{Javadoc[^}]+(?:}[^}]+)*}}/<javadoc\/>/msg;
	$line =~ s/{{Harvtxt[^}]+(?:}[^}]+)*}}/<citation\/>/msg;
	$line =~ s/{{lang\|[^}]+(?:}[^}]+)*}}/<foreign\/>/msg;
	$line =~ s/{{transl\|[^}]+(?:}[^}]+)*}}/<foreign\/>/msg;
	$line =~ s/{{[Nn]ihongo\|[^}]+(?:}[^}]+)*}}/<foreign\/>/msg;
	$line =~ s/<code[^>]*>(?:(?!<\/code>).)*<\/code>/<code\/>/msg;
	$line =~ s/<math[^>]*>(?:(?!<\/math>).)*<\/math>/<math\/>/msg;
	$line =~ s/<source[^>]*>(?:(?!<\/source>).)*<\/source>/<source\/>/msg;

	$line =~ s/[ \t]+/ /g;
	
	print $line;
}

