use Encode;
while (<>) {
	if (/<text id=/) {
		if (/<text id=\"(.*?)\" no_ordem=\"(.*?)\" tema=\"(.*?)\" pais=\"(.*?)\" directoria=\"(.*?)\" ficheiro=\"(.*?)\"/) {
			print "$1\t$2\t$3\t$4\t$5\t$6\n";
		}
	}
}
exit;
while (<>) {
	($w,$p,$l) = split(/\t/);
	if (/(<text id=\".*?\" )(.*)/) {
		$id = $1;
		$rest = $2;
		$id =~ s/\-/\_/g;
		print $id.$rest."\n";
		#print "<s>\n";
	}
	elsif (/<\/text/ || /<s>/ || /<\/s>/) {
		print;		
	}
	elsif ($w eq "" || $p eq "" || $l eq "") {
		next;
	}
	else {
		print;
	}
}
exit;




#./261783.txt: text/plain; charset=unknown-8bit
while (<>) {
	/\.\/(\d+)\.txt\: (.*)/;
	$file = $1.".txt";
	$encodage = $2;
	if ($encodage =~ /utf\-8/) {
		system("cp "."files/".$file." utf-8/");
	}
	elsif ($encodage =~ /8859\-1/) {
		system("cp "."files/".$file." 8859-1/");
	}
}
exit;
$fichier = 1;
while (<>) {
	if (/<text id=/) {
		print $fichier."\n";
		#if ($fichier > 32767) {$fichier++;next;}
		#open F,">/media/USB\ DISK/files/".$fichier++.".txt";
		open F,">./files/".$fichier++.".txt";
		print F;
		while(<>) {
			print F;
			if (/<\/text>/) {close F;last;}
		} 
	}
}
exit;
$now = 1;
$debut = 1;
$fin = 10;
while (<>) {
	if ($now < $debut && $now > $fin) {
		if (/<\/text>/) {$now++}
		next;
	}
	print;
	if (/<\/text>/) {$now++}
	if ($now > $fin) {last}
}
exit;





while(<>) {
	if (/text id=\"(.*?)\"(.*)/) {
		$rest = $2;
		$id = $1;
		$id =~ s/\*/star/g;
		$id =~ s/\-/\_/g;
		print "<text id=\"".$id."\"".$rest."\n";
	}
	else {
		print;
	}

}
exit;

while (<>) {
	if (/text id=\"D1537\"/ || /text id=\"noCOD-1024096\"/ || /text id=\"V5137\"/ || /text id=\"V5219\"/ || /tema=\"\"/) {
		#print STDERR "catch\n";
		while(<>) {
			#print ">".$_;
			if (/<\/text>/) {goto SORTIR;}
		} 
	}
	print;
	SORTIR:
}
exit;

while (<>) {
	#$_ = encode('utf-8', $_);
	#if ($cnt-- < 1) {exit}
	s/\_nordem/ nordem/g;
	s/\_tema/ tema/g;
	s/\_pais/ pais/g;
	s/\_directoria/ directoria/g;
	s/\_ficheiro/ ficheiro/g;
	if (/nordem=\"(.*?)\"/) {
		$theid = $1;
		s/<metadata/<text id=\"$theid\"/g;
	}
	s/<\/metadata>/<\/text>/g;
	if (/<p>/) {next;}
	if (/<l>/) {next;}
	if (/<h>/) {next;}
	#if (/<unknown>/) {print; next;}
	#if (/metadata/) {print; next;}
	#if (/<|>/) {next;} # to check
	print $_;
}
exit;

$taille = 50001; # files
$cpteur = 1;
$nf = 1;
#open IN, "<:encoding(iso-8859-1)", "tempo/fullc3.tagged";
open IN, "tempo/fullc3.tagged";
open F, ">splits/split".$nf;
#open F, ">splits/split".$nf;

while(<IN>) {
	$_ = encode('utf8', $_);
	#goto SKIP;
	if (($cpteur % $taille) == 0) {
		close F;
		open F, ">splits/split".++$nf;	
		$cpteur = 1;		
	}	
	SKIP:
	print F $_;
	if (/<\/metadata>/) {$cpteur++}
}
close IN;
close F;
exit;


my $cnt = 2000;

# for sentence boundaries: SENT is replaced by SENT </s>  <s>
# </text> is replaced by </s> </text> read-write <s>
# revise top and bottom manually 
while (<>) {
	$data = encode('utf-8', $_);
	print $data;
	#$cnt--;
	#if ($cnt < 0) {exit;}
}
exit;

my $cnt = 30;

