#!/usr/bin/perl
use utf8;
$empty = 1;

while(<>) {
	$header = 0;
	if (/^$/) { #normalise to a single blank line between paragraphs
		next if $empty;
		$empty = 1;
	} else {
		$empty = 0;
		s/^\s+//; #strip space before
		s/[ 	]+$//; #and after
		s/(\S+)\/(\S+)/$1/g; #grab token, drop tag
		$header=1 if $2 =~ /-hl/; #treat headlines as 1 line paragraphs
		s/ (\p{Pe})( |$)/$1$2/g; #remove spaces before end punct
		s/( |^)(\p{Ps}) /$1$2/g; #remove spaces after start punct
		s/ ([;:.,\!\?])(?= |$)/$1/g; #remove spaces before terminating punct
		s/`` /"/g; #map opening LaTex quotes back to straight double
		s/ ''/"/g; #map closing quotes back to straight double
		s/([.?,\!;])\1$/$1/; #remove brown double punct artifact
		if ($header) { #treat headlines as 1 line paragraphs
			$_.="\n";
			$empty = 1;
		}
	}
	print;
}
print "\n" unless $empty;
