#!/usr/bin/perl

use utf8;

binmode(STDIN, ":utf8");
binmode(STDOUT, ":utf8");
binmode(STDERR, ":utf8");

$full = "";
while(<>) {
	$line =$_;
	$line =~ s/[ \t]+/ /g;
	$oldline = "";
	while ($line ne $oldline) {
		$oldline = $line;
		while ($line =~ /\[\[(?:[^[|\]]+\|)?([^[|\}]+)\]\]/) {
			$line =~ s/\[\[(?:[^[|\]]+\|)?([^[|\}]+)\]\]/$1/;
		}
		$line =~ s/\[(?:http|ftp):\/\/(?:[^[\] ]+ )?([^[\]]+)\]/$1/;
	}

	$line =~ s/(^|[^'])''(?=[^']|$)/$1/g;
	$line =~ s/(^|[^'])'''''(?=[^']|$)/$1/g;
	$line =~ s/(^|[^'])'''(?=[^']|$)/$1/g;

	$line =~ s/<\/?blockquote>/\n\n/g;
	$line =~ s/<\/?b>//g;
	$line =~ s/<\/?big>//g;
	$line =~ s/<\/?center>//g;
	$line =~ s/<\/?s>//g;
	$line =~ s/<\/?small>//g;
	$line =~ s/<\/?span(?: [^>]*)?>//g;
	$line =~ s/<\/?tt>//g;
	$line =~ s/<\/?u>//g;
	
	$line =~ s/<\/?su(?:b|p)>//g;

	$line =~ s/<\/?article>//g;
	$line =~ s/^ *=====(.+)===== *$/\n\n$1\n\n/;
	$line =~ s/^ *====(.+)==== *$/\n\n$1\n\n/;
	$line =~ s/^ *===(.+)=== *$/\n\n$1\n\n/;
	$line =~ s/^ *==(.+)== *$/\n\n$1\n\n/;
	$line =~ s/^ *=(.+)= *$/\n\n$1\n\n/;

#	$line =~ s/{{[^}]+\|([^}]*)}}/$1/g;

	## what normalisation do we want for list items?
	$line =~ s/^[:*#;]+/\n\n/;

	$full .= $line;
}

$full =~ s/ +$//mg;
$full =~ s/^ +//mg;
$full =~ s/\n/ /g;
$full =~ s/ {2,}/\n\n/g;

$full =~ s/{{IAST\|([^}]+(?:}[^}]+)*)}}/$1/msg;
$full =~ s/{{IPA[^}]+(?:}[^}]+)*}}/<ipa\/>/msg;
$full =~ s/{{Javadoc[^}]+(?:}[^}]+)*}}/<javadoc\/>/msg;
$full =~ s/{{Harvtxt[^}]+(?:}[^}]+)*}}/<citation\/>/msg;
$full =~ s/{{lang\|[^}]+(?:}[^}]+)*}}/<foreign\/>/msg;
$full =~ s/{{transl\|[^}]+(?:}[^}]+)*}}/<foreign\/>/msg;
$full =~ s/{{[Nn]ihongo\|[^}]+(?:}[^}]+)*}}/<foreign\/>/msg;
$full =~ s/<code[^>]*>(?:(?!<\/code>).)*<\/code>/<code\/>/msg;
$full =~ s/<math[^>]*>(?:(?!<\/math>).)*<\/math>/<math\/>/msg;
$full =~ s/<source[^>]*>(?:(?!<\/source>).)*<\/source>/<source\/>/msg;
print $full;
