#!/bin/perl # # ***** BEGIN LICENSE BLOCK ***** # Version: MPL 1.1/GPL 2.0/LGPL 2.1 # # The contents of this file are subject to the Mozilla Public License Version # 1.1 (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # http://www.mozilla.org/MPL/ # # Software distributed under the License is distributed on an "AS IS" basis, # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License # for the specific language governing rights and limitations under the # License. # # The Original Code is Mozilla MathML Project. # # The Initial Developer of the Original Code is # The University Of Queensland. # Portions created by the Initial Developer are Copyright (C) 2001 # the Initial Developer. All Rights Reserved. # # Contributor(s): # Roger B. Sidje - Original Author # # Alternatively, the contents of this file may be used under the terms of # either of the GNU General Public License Version 2 or later (the "GPL"), # or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), # in which case the provisions of the GPL or the LGPL are applicable instead # of those above. If you wish to allow use of your version of this file only # under the terms of either the GPL or the LGPL, and not to allow others to # use your version of this file under the terms of the MPL, indicate your # decision by deleting the provisions above and replace them with the notice # and other provisions required by the GPL or the LGPL. If you do not delete # the provisions above, a recipient may use your version of this file under # the terms of any one of the MPL, the GPL or the LGPL. # # ***** END LICENSE BLOCK ***** # RBS - Last Modified: March 14, 2001. # # Usage: perl encode.pl font-encoding-table.html [-t truetype | type1] # require 'getopts.pl'; our($opt_h, $opt_f, $opt_t); # require 'mathfont.pl'; my($DEBUG) = 1; # where is the MathML DTD file? my($DTDfile) = "C:\\Mozilla\\src\\mozilla\\layout\\mathml\\content\\src\\mathml.dtd"; # where to load the PUA file? my($PUAfile) = "C:\\Mozilla\\src\\mozilla\\layout\\mathml\\base\\src\\mathfontPUA.properties"; # where to save the PUA file if new assignments to the PUA are made? my($newPUAfile) = $PUAfile; # get the basename of the script my($progname) = $0 =~ /([^\/\\]+)$/; sub usage { my($comment) = @_; my($usage) = <<__USAGE__; Usage: perl $progname [-h] -f fontfile.html [-t truetype | type1] Purpose This script takes in input a Mozilla's MathFont Encoding Table and outputs the data needed to support the font, i.e., the data for the ucvmath module and the data for the MathFont Property File. Options -h (help, this message) -f fontfile.html The file that contains the font's encoding table. To see examples, launch 'view-source' on the .html files of the default fonts at: http://www.mozilla.org/projects/mathml/fonts/encoding/ These .html files are *exactly* the files passed to this script with this -f option. For example, to get the data currently used by Mozilla for the Math1 font, this script was executed as: encode.pl -f math1.html The script parses the .html file to extract the mapping enclosed in ... Not only does this ease maintenance, but it also means that the website reflects the latest updates. Note however that the MathFont Property Files are also hand-tuned to refine the results and to add further customization. Such refinements should not be done on the files generated by this script directly (see below). Otherwise you will overwrite and thus lose your changes if you run the script again. -t encodingtype This option should not be used if the TrueType and Type1 versions have the same encoding. If there are different encodings, use -t truetype : to process the TrueType encoding table inside the file -t type1 : to process the Type1 encoding table inside the file OUTPUT: Given the fontfile.html on input, three files are created on output: 1. a file called fontname-ucvmath.txt : this is the data to pass to "umaptable" for the ucvmath module. For example, running "encode.pl -f math1.html" produces math1-ucvmath.txt. Then, running "umaptable -uf < math1-ucvmath.txt > mathematica1.uf" will produce the resulting file that is currently in mozilla/intl/uconv/ucvmath. "umaptable" is a standalone C program that can be built from mozilla/intl/uconv/tools and the executable copied where you want. 2. fontname-properties.txt : this is the data for the MathFont Property File. Continuing with the example of Math1, a file called math1-properties.txt will be created. Then after hand-tuning (if necessary), the file is copied to its final destination: mozilla/layout/mathml/base/src/mathfontMath1.properties 3. fontname-encoding.html : the is a prettified output that shows the resulting mappings. For the example of Math1, this is the file that is at: http://www.mozilla.org/projects/mathml/fonts/encoding/math1-encoding.html 4. Furthermore, since new assignments can be made to the PUA during processing, the mathfontPUA.properties file is updated with any new assignments that may have been made. __USAGE__ $comment = "\nBad usage: $comment\n" if $comment; die "$comment\n$usage"; } # &usage("Missing arguments") if !&Getopts('hf:t:'); &usage("") if $opt_h; # help option # The file that contains MathFont Encoding Table(s) my($encoding_file) = $opt_f; &usage("Missing file") unless $opt_f; $encoding_file = "encoding\\" . $encoding_file; # The encoding type should be: # . "" (empty), if there is only one encoding for both TrueType and Type1 # . "truetype", to process the TrueType encoding table inside the file # . "type1", to process the Type1 encoding table inside the file my($encoding_type) = $opt_t; &usage("Unexpected type") unless $encoding_type eq "" || $encoding_type eq "type1" || $encoding_type eq "truetype"; # Processing starts here ####################################################################### # global variables -- in capitals # $UNIDATA{$unicode} holds a whitespace-separated list of entities and # annotated unicode points that all resolve to that $unicode point. # %UNIDATA is declared in mathfont.pl # $GLYPHDATA{$glyph} holds a whitespace-separated list of entities and # annotated unicode points that all resolve to that $glyph index. my(%GLYPHDATA); # $MAP{$glyph} holds the ultimate resolved unicode point of the $glyph # (could be a PUA value). my(%MAP); # load current assignments to the PUA &load_PUA($PUAfile); # parse the supplied encoding data my($fontfamily) = &parse_mapping_table($encoding_file, $encoding_type); # resolve all mappings &generate_mapping_data($fontfamily); # save the PUA to preserve any new assignments &save_PUA($PUAfile, $newPUAfile); # dump results &output_stretchy_data($fontfamily, $encoding_type); &output_mapping_data($fontfamily, $encoding_type); &output_encoding_map($fontfamily, $encoding_type); exit(0); ######################################################################### # parse_mapping_table: # # Parse a file containing the mapping between glyph indices and # unicode points. The file is formatted as: # # 0xNN 0xNNNN #optional trailing comment # 0xNN 0xNNNN:0 # 0xNN 0xNNNN:1 # -0xNN # 0xNN 0xNNNN:T, 0xABCD # 0xNN 0xNNNN:G, 0xABCD # 0xNN >PUA #required comment with &entity; # ...etc # # # Note that there is an enclosing: ... with the *required* # FontFamilyName and the optional type (exactly 'truetype' or 'type1'). # # The first field may contain a dash '-', in which case the line is skipped. # The second field is the glyph index. # The third field is a comma-delimited list of its associated unicode points # with annotations w.r.t their applicability ('T', 'L', 'M', 'B', 'R', '0'-'9'). # # Partial glyphs can apply to different chars. The annotation for '0' is # optional, i.e., it is assumed if no other annotation has been specified. # The list can run over many lines provided the preceding line ends # with a comma. # # If the keyword '>PUA' is present, or the unicode point is in plane 1, # a PUA code is assigned, and is associated with the required '&entity;' # that must be provided in the comment field. ######################################################################### sub parse_mapping_table { my($file, $type) = @_; local(*FILE); open(FILE, $file) || die "Cannot find $file\n"; my($data) = join("", ); close(FILE); my($fontfamily); $type = ':' . $type if $type; if ($data =~ m|(.+?)|s) { die "ERROR *** Bad mapping: mistmatching tags $type start:$1 end:$3" unless $1 eq $3; ($fontfamily, $data) = ($1, $2); die "ERROR *** No specified font type $1" if $fontfamily =~ /:/; } else { die "ERROR *** Bad mapping: data must be enclosed in the tag"; } my($isContinuation) = 0; my($line, $glyph, $comment, $oldline, $oldcomment); my (@lines) = split("\n", $data); foreach $line (@lines) { # skip bogus lines next if $line =~ m/^-/; # remove leading and trailing whitespace $line =~ s/^\s+//; $line =~ s/\s+$//; # cache comments in case the keyword '>PUA' is present $comment = ($line =~ m/#(.*)/) ? $1 : ""; $line =~ s/\s*#.*//; # see if this is the continuation of a longer line if ($isContinuation) { $line = $oldline . ' ' . $line; $comment = $oldcomment . ' ' . $comment; $isContinuation = 0; } # if the line ends with a comma, the next line is a continuation if ($line =~ m/,$/) { $oldline = $line; $oldcomment = $comment; $isContinuation = 1; next; } # get the mapping on the line next unless ($line =~ m/^(0x..)\s+(.+)$/); ($glyph, $data) = ($1, $2); # see if this a '>PUA' or plane 1 character if (($data eq '>PUA') || ($data =~ m/^0x1..../)) { my($entitylist) = ''; while ($comment =~ /&(.+?);/g) { $entitylist .= ' ' . $1; } $entitylist =~ s/^\s//; die "ERROR *** No entities found: $line" unless $entitylist; $GLYPHDATA{$glyph} = $entitylist; # continue to next line next; } # skip bogus lines where uncertainties still remain $data .= ' '; next unless $data =~ /^0x....\s*[, ]/ || $data =~ /^0x....:[TLMBRG0-9]\s*[, ]/; chop($data); # convert from comma-delimited to whitespace-delimited $data =~ s/\s*,\s*/ /g; # add explicit 0 at size0 $data =~ s/(0x....) /$1:0 /; $data =~ s/(0x....)$/$1:0/; # $GLYPHDATA{$glyph} is a list of referrers to that glyph $GLYPHDATA{$glyph} = $data; } # check for correctness &verify_mapping_table(); return $fontfamily; } ######################################################################### # verify_mapping_table: # # helper to check that some common mistakes were not made when # setting up the encoding table. ######################################################################### sub verify_mapping_table { my($glyph, $entry, $tmp); foreach $glyph (keys %GLYPHDATA) { my(@data) = split(' ', $GLYPHDATA{$glyph}); foreach $entry (@data) { # verify that $entry wasn't listed twice on distinct lines foreach $tmp (keys %GLYPHDATA) { next if $tmp eq $glyph; next unless $GLYPHDATA{$tmp} =~ /\b$entry\b/; die "ERROR *** Duplicate: $glyph $GLYPHDATA{$glyph} vs. $tmp $GLYPHDATA{$tmp}"; } # verify that $entry wasn't listed twice on the same line $tmp = $GLYPHDATA{$glyph}; $tmp =~ s/\b$entry\b//; next unless $tmp =~ /\b$entry\b/; die "ERROR *** Duplicate: $glyph $GLYPHDATA{$glyph}"; } } } ######################################################################### # generate_mapping_data: # # This routine resolves the unicode points (PUA and non-PUA) of all # elements. Upon completion, the assignments for all entities, annotated # unicode points, and glyph indices are known. ######################################################################### sub generate_mapping_data { my($font) = @_; my($glyph); foreach $glyph (sort keys %GLYPHDATA) { # $GLYPHDATA{$glyph} is a list of referrers to that glyph # see if one of the referrers is already resolved my($entry); my($unicode) = ''; my(@data) = split(' ', $GLYPHDATA{$glyph}); foreach $entry (@data) { # see if this entry is the unicode associated to size0 if ($entry =~ /(0x....):0/) { $unicode = $1; last; } # see if this entry is already associated to a unicode point $unicode = get_unicode($entry); last if $unicode; } # if we found one entry that was already resolved, make # everybody on the list resolve to that unicode point if ($unicode) { foreach $entry (@data) { next if $entry =~ /0x....:0/; if (defined $UNIDATA{$unicode}) { $UNIDATA{$unicode} .= " $entry" unless $UNIDATA{$unicode} =~ /\b$entry\b/; } else { $UNIDATA{$unicode} = $entry; } } } else { # make a new assignment to the PUA for this encoding point $unicode = &assign_PUA($GLYPHDATA{$glyph}); } # now, we know the unicode point of the glyph die "ERROR *** Duplicate $glyph" if defined $MAP{$glyph}; $MAP{$glyph} = $unicode; } # check that all went well verify_mapping_data(); } ######################################################################### # verify_mapping_data: # # helper to check the validity of the resolved mapping. ######################################################################### sub verify_mapping_data { my($unicode, $entry, $tmp); foreach $unicode (keys %UNIDATA) { my(@data) = split(' ', $UNIDATA{$unicode}); foreach $entry (@data) { # verify that $entry wasn't assigned twice on distinct slots foreach $tmp (keys %UNIDATA) { next if $tmp eq $unicode; # we don't care about different mappings outside the PUA because # these are not kept in the mathfontPUA.properties file next unless &is_pua($tmp) && &is_pua($unicode); next unless $UNIDATA{$tmp} =~ /\b$entry\b/; # if we reach here, different mappings in the PUA were found # for the same annotated code point, something is wrong die "ERROR *** Duplicate: $unicode $UNIDATA{$unicode} vs. $tmp $UNIDATA{$tmp}"; } # verify that $entry wasn't listed twice on the same slot $tmp = $UNIDATA{$unicode}; $tmp =~ s/\b$entry\b//; next unless $tmp =~ /\b$entry\b/; die "ERROR *** Duplicate: $unicode $UNIDATA{$unicode}"; } } } sub is_pua { my($value) = @_; if ($value =~ s/^0x//) { my($numeric) = hex($value); return 1 if $numeric >= 0xE000 && $numeric <= 0xF8FF; } return 0; } ######################################################################### # output_mapping_data: # # output the mapping data that goes in the ucvmath module ######################################################################### sub output_mapping_data { my($font, $type) = @_; $type = "-$type" if $type; my($file) = "$font-ucvmath$type.txt"; $file =~ s/ //g; $file = lc($file); print "Saving mapping data of $font in: $file\n"; local(*FILE); open(FILE, ">$file") || die "Cannot open $file\n"; my($glyph); foreach $glyph (sort keys %MAP) { print FILE "$glyph $MAP{$glyph}\n"; } close(FILE); } ######################################################################### # output_stretchy_data: # # output the list of stretchy data in the compact format expected by # the MathFont Property File. ######################################################################### sub output_stretchy_data { my($font, $type) = @_; $type = "-$type" if $type; my($file) = "$font-properties$type.txt"; $file =~ s/ //g; $file = lc($file); print "Saving properties of $font in: $file\n"; &load_DTD($DTDfile); my($pattern); my(@patterns) = qw { (\S+):[TL] (\S+):M (\S+):[BR] (\S+):G }; my($unicode, $glyph, $label); # construct the _transpose_ of the GLYPHDATA table so that upon # completion, $table{unicode} = list of its associated glyph # indices with the annotations flipped on the other side my(%table); foreach $glyph (%GLYPHDATA) { my(@data) = split(' ', $GLYPHDATA{$glyph}); foreach $unicode (@data) { # the '0x' prefix is not kept here next unless $unicode =~ m|0x(....):(.)|; ($unicode, $label) = ($1, $2); if (defined $table{$unicode}) { $table{$unicode} .= " $glyph:$label"; } else { $table{$unicode} = "$glyph:$label"; } } } # now, replace the glyph indices with their resolved unicode points my($parts, $sizes, $comment); local(*FILE); open(FILE, ">$file") || die "Cannot open $file\n"; foreach $unicode (sort keys %table) { my($isMutable) = 0; # partial glyphs $parts = ''; foreach $pattern (@patterns) { if ($table{$unicode} =~ m/$pattern/) { $glyph = $1; $parts .= &indirect_pua($MAP{$glyph}); $isMutable = 1; } else { $parts .= '\uFFFD'; } } # glyphs of larger sizes my($sizes) = "\\u$unicode"; # size0 is the char itself $label = '1'; while ($table{$unicode} =~ m/(\S+):$label/) { $glyph = $1; $sizes .= &indirect_pua($MAP{$glyph}); ++$label; $isMutable = 1; } # ignore this character if it is not mutable next unless $isMutable; # entry for the list of glyphs $comment = $ENTITY{"0x$unicode"}; if ($DEBUG) { print FILE "\\u$unicode = $parts$sizes $comment\n"; } else { print FILE "$comment\n\\u$unicode = $parts$sizes\n"; } } close(FILE); } sub indirect_pua { my($unicode) = @_; # see if this code is the PUA my($numeric) = hex($unicode); die "ERROR *** Forbidden mapping" if $numeric == 0xF8FF; if ($numeric >= 0xE000 && $numeric <= 0xF8FF) { # return the flag to indicate to lookup in the PUA return '\uF8FF'; } # re-use the existing unicode point $unicode =~ s/0x/\\u/; return $unicode; } ######################################################################### # output_encoding_map: # # output the encoding map in html format for visual comparison with the # graphical character map. ######################################################################### sub output_encoding_map { my($font, $type) = @_; $type = "-$type" if $type; my($shorttype) = ''; $shorttype = '-t1' if $type eq '-type1'; $shorttype = '-ttf' if $type eq '-truetype'; my($file) = "$font$shorttype-encoding.html"; $file =~ s/ //g; $file = lc($file); print "Saving visual encoding result of $font in: $file\n"; local(*FILE); open(FILE, ">$file") || die "Cannot open $file\n"; # header print FILE "\n" . "\n" . " $font$type - Visual Encoding Result\n" . " \n" . "\n" . "\n" . "

$font$type - Visual Encoding Result

\n\n"; # column indices print FILE "\n"; my($i, $j); for ($j = 0; $j <= 15; ++$j) { print FILE sprintf("", $j); } print FILE "\n"; # cmap array for ($i = 0; $i <= 15; ++$i) { print FILE sprintf("", $i); for ($j = 0; $j <= 15; ++$j) { my($glyph) = sprintf("0x%X%X", $i, $j); if (defined $GLYPHDATA{$glyph}) { my($unicode) = $MAP{$glyph}; my($data) = $GLYPHDATA{$glyph}; $unicode =~ s|0(x....)|&#$1;|; $data =~ s| |
|g; $data =~ s|(0x....):(.)|$1:$2|g; print FILE ""; } else { print FILE ''; } } print FILE "\n"; } # footer print FILE "
%X
%X " . "$MAP{$glyph}
" . "$unicode
" . $data . "		 
\n\n\n\n"; close(FILE); } ######################################################################### # load_DTD: # # load the mathml DTD so that we can comment outputs with entities ######################################################################### sub load_DTD { my($file) = @_; local(*FILE); open(FILE, $file) || die "Cannot open $file\n"; while () { while (/]+);'>/g) { my($entity, $unicode) = ($1, "0x$2"); $entity = chr(hex($unicode)) if hex($unicode) <= 0xFF; if (defined $ENTITY{$unicode}) { $ENTITY{$unicode} .= ", $entity"; } else { $ENTITY{$unicode} = "# $entity"; } } } close(FILE); }