#!/usr/bin/perl


    use strict;
    use warnings;
    use Encode::Guess;

    my $max = 374542;
    my $current = 0;
    my $from = 1;
    my $to = 10000;
    my $dir = shift;
    my $tdir = shift;
    my $encodage = "";
    opendir(DIR, $dir) or die $!;

    while (my $file = readdir(DIR)) {

        # Use a regular expression to ignore files beginning with a period
        next if ($file =~ m/^\./);
	$current++;
	if ($current < $from) {next;}
	if ($current > $to) {next;}
	#print $current.":"."$file\n";
	open(FILE,$dir."/".$file);
    	binmode(FILE);
    	if(read(FILE,my $filestart, 500)) {
		my $enc = guess_encoding($filestart);
		if(ref($enc)) {
	    		#print "$file:\t",$enc->name,"\n";
			$encodage = $enc->name;
		}
		else {
	    		#print "Encoding of file $file can't be guessed\n";
			$encodage = "unknown";
		}
    	}
    	else {
		print "Cannot read from file $file\n";
    	}
	close FILE;
	open IN, $dir."/".$file;
	if ($encodage ne "utf8") {
		open OUT, ">:utf8",$tdir."/".$file;
	}
	else {
		open OUT, ">".$tdir."/".$file;
	}
	my $texto = 0;
	while(<IN>) {
		if (/^<p>/) {next}
		if (/^<\/p>/) {next}
		if (/^<h>/) {next}
		if (/^<\/h>/) {next}
		if (/^<l>/) {next}
		if (/^<\/l>/) {next}
		if (/<metadata no_ordem=\"(.*?)\"/) {
			my $nor = $1;
			$nor =~ s/\-/\_/;
			$nor =~ s/\*/star/;
			s/<metadata no_ordem=\"(.*?)\"/<text id=\"$nor\" no_ordem=\"$nor\"/;
			$texto++;
		}
		if (/<\/metadata>/) {
			s/<\/metadata>/<\/text>/;
			$texto--;
		}
		s/(&#33;)/!/g;
		s/(&quot;)|(&#34;)/"/g;
		s/(&#35;)/#/g;
		s/(&#36;)/\$/g;
		s/(&#37;)/\%/g;
		s/(&#38;)|(&#38;)/&/g;
		s/(&#39;)/'/g;
		s/(&#40;)/(/g;
		s/(&#41;)/)/g;
		s/(&#42;)/*/g;
		s/(&#43;)/+/g;
		s/(&#44;)/,/g;
		s/(&#45;)/-/g;
		s/(&#46;)/./g;
		s/(&#47;)/\//g;
		s/(&#48;)/0/g;
		s/(&#49;)/1/g;
		s/(&#50;)/2/g;
		s/(&#51;)/3/g;
		s/(&#52;)/4/g;
		s/(&#53;)/5/g;
		s/(&#54;)/6/g;
		s/(&#55;)/7/g;
		s/(&#56;)/8/g;
		s/(&#57;)/9/g;
		s/(&#58;)/:/g;
		s/(&#59;)/;/g;
		s/(&lt;)|(&#60;)/</g;
		s/(&#61;)/=/g;
		s/(&gt;)|(&#62;)/>/g;
s/(&#63;)/?/g;
s/(&#64;)/@/g;
s/(&#65;)/A/g;
s/(&#66;)/B/g;
s/(&#67;)/C/g;
s/(&#68;)/D/g;
s/(&#69;)/E/g;
s/(&#70;)/F/g;
s/(&#71;)/G/g;
s/(&#72;)/H/g;
s/(&#73;)/I/g;
s/(&#74;)/J/g;
s/(&#75;)/K/g;
s/(&#76;)/L/g;
s/(&#77;)/M/g;
s/(&#78;)/N/g;
s/(&#79;)/O/g;
s/(&#80;)/P/g;
s/(&#81;)/Q/g;
s/(&#82;)/R/g;
s/(&#83;)/S/g;
s/(&#84;)/T/g;
s/(&#85;)/U/g;
s/(&#86;)/V/g;
s/(&#87;)/W/g;
s/(&#88;)/X/g;
s/(&#89;)/Y/g;
s/(&#90;)/Z/g;
s/(&#91;)/[/g;
s/(&#94;)/^/g;
s/(&#95;)/_/g;
s/(&#96;)/`/g;
s/(&#97;)/a/g;
s/(&#98;)/b/g;
s/(&#99;)/c/g;
s/(&#100;)/d/g;
s/(&#101;)/e/g;
s/(&#102;)/f/g;
s/(&#103;)/g/g;
s/(&#104;)/h/g;
s/(&#105;)/i/g;
s/(&#106;)/j/g;
s/(&#107;)/k/g;
s/(&#108;)/l/g;
s/(&#109;)/m/g;
s/(&#110;)/n/g;
s/(&#111;)/o/g;
s/(&#112;)/p/g;
s/(&#113;)/q/g;
s/(&#114;)/r/g;
s/(&#115;)/s/g;
s/(&#116;)/t/g;
s/(&#117;)/u/g;
s/(&#118;)/v/g;
s/(&#119;)/w/g;
s/(&#120;)/x/g;
s/(&#121;)/y/g;
s/(&#122;)/z/g;
s/(&#123;)/{/g;
s/(&#124;)/|/g;
s/(&#125;)/}/g;
s/(&tilde;)|(&#126;)/~/g;
s/(&#128;)/€/g;
s/(&sbquo;)|(&#130;)/‚/g;
s/(&#131;)/ƒ/g;
s/(&dbquo;)|(&#132;)/„/g;
s/(&#133;)/…/g;
s/(&dagger;)|(&#134;)/†/g;
s/(&Dagger;)|(&#135;)/‡/g;
s/(&#136;)/ˆ/g;
s/(&permil;)|(&#137;)/‰/g;
s/(&#138;)/Š/g;
s/(&lsaquo;)|(&#139;)/‹/g;
s/(&#140;)/Œ/g;
s/(&lsquo;)|(&#145;)/‘/g;
s/(&rsquo;)|(&#146;)/’/g;
s/(&ldquo;)|(&#147;)/“/g;
s/(&rdquo;)|(&#148;)/”/g;
s/(&#149;)|(•)/2022/g;
s/(&ndash;)|(&#150;)/–/g;
s/(&mdash;)|(&#151;)/—/g;
s/(&tilde)|(&#152;)/˜/g;
s/(&trade;)|(&#153;)/™/g;
s/(&#154;)/š/g;
s/(&rsaquo;)|(&#155;)/›/g;
s/(&#156;)/œ/g;
s/(&#158;)/ž/g;
s/(&Yuml;)|(&#159;)/Ÿ/g;
s/(&nbsp;)|(&#160;)/A0/g;
s/(&iexcl;)|(&#161;)/¡/g;
s/(&cent;)|(&#162;)/¢/g;
s/(&pound;)|(&#163;)/£/g;
s/(&curren;)|(&#164;)/¤/g;
s/(&yen;)|(&#165;)/¥/g;
s/(&brvbar;)|(&#166;)/¦/g;
s/(&sect;)|(&#167;)/§/g;
s/(&uml;)|(&#168;)/¨/g;
s/(&copy;)|(&#169;)/©/g;
s/(&ordf;)|(&#170;)/ª/g;
s/(&laquo;)|(&#171;)/«/g;
s/(&not;)|(&#172;)/¬/g;
s/(&shy;)|(&#173;)/AD/g;
s/(&reg;)|(&#174;)/®/g;
s/(&macr;)|(&#175;)/¯/g;
s/(&deg;)|(&#176;)/°/g;
s/(&plusmn;)|(&#177;)/±/g;
s/(&sup2;)|(&#178;)/²/g;
s/(&sup3;)|(&#179;)/³/g;
s/(&#180;)/´/g;
s/(&micro;)|(&#181;)/µ/g;
s/(&para;)|(&#182;)/¶/g;
s/(&middot;)|(&#183;)/·/g;
s/(&cedil;)|(&#184;)/¸/g;
s/(&sup1;)|(&#185;)/¹/g;
s/(&ordm;)|(&#186;)/º/g;
s/(&raquo;)|(&#187;)/»/g;
s/(&frac14;)|(&#188;)/¼/g;
s/(&frac12;)|(&#189;)/½/g;
s/(&frac34;)|(&#190;)/¾/g;
s/(&#191;)/¿/g;
s/(&Agrave;)|(&#192;)/À/g;
s/(&Aacute;)|(&#193;)/Á/g;
s/(&Acirc;)|(&#194;)/Â/g;
s/(&Atilde;)|(&#195;)/Ã/g;
s/(&Auml;)|(&#196;)/Ä/g;
s/(&Aring;)|(&#197;)/Å/g;
s/(&AElig;)|(&#198;)/Æ/g;
s/(&Ccedil;)|(&#199;)/Ç/g;
s/(&Egrave;)|(&#200;)/È/g;
s/(&Eacute;)|(&#201;)/É/g;
s/(&Ecirc;)|(&#202;)/Ê/g;
s/(&Euml;)|(&#203;)/Ë/g;
s/(&Igrave;)|(&#204;)/Ì/g;
s/(&Iacute;)|(&#205;)/Í/g;
s/(&Icirc;)|(&#206;)/Î/g;
s/(&Iuml;)|(&#207;)/Ï/g;
s/(&ETH;)|(&#208;)/Ð/g;
s/(&Ntilde;)|(&#209;)/Ñ/g;
s/(&Ograve;)|(&#210;)/Ò/g;
s/(&Oacute;)|(&#211;)/Ó/g;
s/(&Ocirc;)|(&#212;)/Ô/g;
s/(&Otilde;)|(&#213;)/Õ/g;
s/(&Ouml;)|(&#214;)/Ö/g;
s/(&times;)|(&#215;)/×/g;
s/(&Oslash;)|(&#216;)/Ø/g;
s/(&Ugrave;)|(&#217;)/Ù/g;
s/(&Uacute;)|(&#218;)/Ú/g;
s/(&Ucirc;)|(&#219;)/Û/g;
s/(&Uuml;)|(&#220;)/Ü/g;
s/(&Yacute;)|(&#221;)/Ý/g;
s/(&THORN;)|(&#222;)/Þ/g;
s/(&szlig;)|(&#223;)/ß/g;
s/(&agrave;)|(&#224;)/à/g;
s/(&aacute;)|(&#225;)/á/g;
s/(&acirc;)|(&#226;)/â/g;
s/(&atilde;)|(&#227;)/ã/g;
s/(&auml;)|(&#228;)/ä/g;
s/(&aring;)|(&#229;)/å/g;
s/(&aelig;)|(&#230;)/æ/g;
s/(&ccedil;)|(&#231;)/ç/g;
s/(&egrave;)|(&#232;)/è/g;
s/(&eacute;)|(&#233;)/é/g;
s/(&ecirc;)|(&#234;)/ê/g;
s/(&euml;)|(&#235;)/ë/g;
s/(&igrave;)|(&#236;)/ì/g;
s/(&iacute;)|(&#237;)/í/g;
s/(&icirc;)|(&#238;)/î/g;
s/(&iuml;)|(&#239;)/ï/g;
s/(&eth;)|(&#240;)/ð/g;
s/(&ntilde;)|(&#241;)/ñ/g;
s/(&ograve;)|(&#242;)/ò/g;
s/(&oacute;)|(&#243;)/ó/g;
s/(&ocirc;)|(&#244;)/ô/g;
s/(&otilde;)|(&#245;)/õ/g;
s/(&ouml;)|(&#246;)/ö/g;
s/(&divide;)|(&#247;)/÷/g;
s/(&oslash;)|(&#248;)/ø/g;
s/(&ugrave;)|(&#249;)/ù/g;
s/(&uacute;)|(&#250;)/ú/g;
s/(&ucirc;)|(&#251;)/û/g;
s/(&uuml;)|(&#252;)/ü/g;
s/(&yacute;)|(&#253;)/ý/g;
s/(&thorn;)|(&#254;)/þ/g;
		s/(&yuml;)|(&#255;)/ÿ/g;
		print OUT;
	}
       close OUT;
	if ($texto > 0) {
		print $file." errors\n";
	}
    }
   closedir(DIR); 
    exit 0;


