################################################################
# #
# Romanizer #
# #
################################################################
package NLP::Romanizer;
use NLP::Chinese;
use NLP::UTF8;
use NLP::utilities;
use JSON;
$utf8 = NLP::UTF8;
$util = NLP::utilities;
$chinesePM = NLP::Chinese;
my $verbosePM = 0;
%empty_ht = ();
my $braille_capital_letter_indicator = "\xE2\xA0\xA0";
my $braille_number_indicator = "\xE2\xA0\xBC";
my $braille_decimal_point = "\xE2\xA0\xA8";
my $braille_comma = "\xE2\xA0\x82";
my $braille_solidus = "\xE2\xA0\x8C";
my $braille_numeric_space = "\xE2\xA0\x90";
my $braille_letter_indicator = "\xE2\xA0\xB0";
my $braille_period = "\xE2\xA0\xB2";
sub new {
local($caller) = @_;
my $object = {};
my $class = ref( $caller ) || $caller;
bless($object, $class);
return $object;
}
sub load_unicode_data {
local($this, *ht, $filename) = @_;
# ../../data/UnicodeData.txt
$n = 0;
if (open(IN, $filename)) {
while (<IN>) {
if (($unicode_value, $char_name, $general_category, $canon_comb_classes, $bidir_category, $char_decomp_mapping, $decimal_digit_value, $digit_value, $numeric_value, $mirrored, $unicode_1_0_name, $comment_field, $uc_mapping, $lc_mapping, $title_case_mapping) = split(";", $_)) {
$utf8_code = $utf8->unicode_hex_string2string($unicode_value);
$ht{UTF_TO_CHAR_NAME}->{$utf8_code} = $char_name;
$ht{UTF_NAME_TO_UNICODE}->{$char_name} = $unicode_value;
$ht{UTF_NAME_TO_CODE}->{$char_name} = $utf8_code;
$ht{UTF_TO_CAT}->{$utf8_code} = $general_category;
$ht{UTF_TO_NUMERIC}->{$utf8_code} = $numeric_value unless $numeric_value eq "";
$n++;
}
}
close(IN);
# print STDERR "Loaded $n entries from $filename\n";
} else {
print STDERR "Can't open $filename\n";
}
}
sub load_unicode_overwrite_romanization {
local($this, *ht, $filename) = @_;
# ../../data/UnicodeDataOverwrite.txt
$n = 0;
if (open(IN, $filename)) {
while (<IN>) {
next if /^#/;
$unicode_value = $util->slot_value_in_double_colon_del_list($_, "u");
$romanization = $util->slot_value_in_double_colon_del_list($_, "r");
$numeric = $util->slot_value_in_double_colon_del_list($_, "num");
$picture = $util->slot_value_in_double_colon_del_list($_, "pic");
$syllable_info = $util->slot_value_in_double_colon_del_list($_, "syllable-info");
$tone_mark = $util->slot_value_in_double_colon_del_list($_, "tone-mark");
$char_name = $util->slot_value_in_double_colon_del_list($_, "name");
$entry_processed_p = 0;
$utf8_code = $utf8->unicode_hex_string2string($unicode_value);
if ($unicode_value) {
$ht{UTF_TO_CHAR_ROMANIZATION}->{$utf8_code} = $romanization if $romanization;
$ht{UTF_TO_NUMERIC}->{$utf8_code} = $numeric if defined($numeric) && ($numeric ne "");
$ht{UTF_TO_PICTURE_DESCR}->{$utf8_code} = $picture if $picture;
$ht{UTF_TO_SYLLABLE_INFO}->{$utf8_code} = $syllable_info if $syllable_info;
$ht{UTF_TO_TONE_MARK}->{$utf8_code} = $tone_mark if $tone_mark;
$ht{UTF_TO_CHAR_NAME}->{$utf8_code} = $char_name if $char_name;
$entry_processed_p = 1 if $romanization || $numeric || $picture || $syllable_info || $tone_mark;
}
$n++ if $entry_processed_p;
}
close(IN);
} else {
print STDERR "Can't open $filename\n";
}
}
sub load_script_data {
local($this, *ht, $filename) = @_;
# ../../data/Scripts.txt
$n = 0;
if (open(IN, $filename)) {
while (<IN>) {
next unless $script_name = $util->slot_value_in_double_colon_del_list($_, "script-name");
$abugida_default_vowel_s = $util->slot_value_in_double_colon_del_list($_, "abugida-default-vowel");
$alt_script_name_s = $util->slot_value_in_double_colon_del_list($_, "alt-script-name");
$language_s = $util->slot_value_in_double_colon_del_list($_, "language");
$direction = $util->slot_value_in_double_colon_del_list($_, "direction"); # right-to-left
$font_family_s = $util->slot_value_in_double_colon_del_list($_, "font-family");
$ht{SCRIPT_P}->{$script_name} = 1;
$ht{SCRIPT_NORM}->{(uc $script_name)} = $script_name;
$ht{DIRECTION}->{$script_name} = $direction if $direction;
foreach $language (split(/,\s*/, $language_s)) {
$ht{SCRIPT_LANGUAGE}->{$script_name}->{$language} = 1;
$ht{LANGUAGE_SCRIPT}->{$language}->{$script_name} = 1;
}
foreach $alt_script_name (split(/,\s*/, $alt_script_name_s)) {
$ht{SCRIPT_NORM}->{$alt_script_name} = $script_name;
$ht{SCRIPT_NORM}->{(uc $alt_script_name)} = $script_name;
}
foreach $abugida_default_vowel (split(/,\s*/, $abugida_default_vowel_s)) {
$ht{SCRIPT_ABUDIGA_DEFAULT_VOWEL}->{$script_name}->{$abugida_default_vowel} = 1 if $abugida_default_vowel;
}
foreach $font_family (split(/,\s*/, $font_family_s)) {
$ht{SCRIPT_FONT}->{$script_name}->{$font_family} = 1 if $font_family;
}
$n++;
}
close(IN);
# print STDERR "Loaded $n entries from $filename\n";
} else {
print STDERR "Can't open $filename\n";
}
}
sub unicode_hangul_romanization {
local($this, $s, $pass_through_p) = @_;
$pass_through_p = 0 unless defined($pass_through_p);
@leads = split(/\s+/, "g gg n d dd r m b bb s ss - j jj c k t p h");
# @vowels = split(/\s+/, "a ae ya yai e ei ye yei o oa oai oi yo u ue uei ui yu w wi i");
@vowels = split(/\s+/, "a ae ya yae eo e yeo ye o wa wai oe yo u weo we wi yu eu yi i");
@tails = split(/\s+/, "- g gg gs n nj nh d l lg lm lb ls lt lp lh m b bs s ss ng j c k t p h");
$result = "";
@chars = $utf8->split_into_utf8_characters($s, "return only chars", *empty_ht);
foreach $char (@chars) {
$unicode = $utf8->utf8_to_unicode($char);
if (($unicode >= 0xAC00) && ($unicode <= 0xD7A3)) {
$code = $unicode - 0xAC00;
$lead_index = int($code / (28*21));
$vowel_index = int($code/28) % 21;
$tail_index = $code % 28;
$rom = $leads[$lead_index] . $vowels[$vowel_index] . $tails[$tail_index];
$rom =~ s/-//g;
$result .= $rom;
} elsif ($pass_through_p) {
$result .= $char;
}
}
return $result;
}
sub listify_comma_sep_string {
local($this, $s) = @_;
@result_list = ();
return @result_list unless $s =~ /\S/;
$s = $util->trim2($s);
my $elem;
while (($elem, $rest) = ($s =~ /^("(?:\\"|[^"])*"|'(?:\\'|[^'])*'|[^"', ]+),\s*(.*)$/)) {
push(@result_list, $util->dequote_string($elem));
$s = $rest;
}
push(@result_list, $util->dequote_string($s)) if $s =~ /\S/;
return @result_list;
}
sub braille_string_p {
local($this, $s) = @_;
return ($s =~ /^(\xE2[\xA0-\xA3][\x80-\xBF])+$/);
}
sub register_word_boundary_info {
local($this, *ht, $lang_code, $utf8_source_string, $utf8_target_string, $use_only_for_whole_word_p,
$use_only_at_start_of_word_p, $use_only_at_end_of_word_p,
$dont_use_at_start_of_word_p, $dont_use_at_end_of_word_p) = @_;
if ($use_only_for_whole_word_p) {
if ($lang_code) {
$ht{USE_ONLY_FOR_WHOLE_WORD_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_target_string} = 1;
} else {
$ht{USE_ONLY_FOR_WHOLE_WORD}->{$utf8_source_string}->{$utf8_target_string} = 1;
}
}
if ($use_only_at_start_of_word_p) {
if ($lang_code) {
$ht{USE_ONLY_AT_START_OF_WORD_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_target_string} = 1;
} else {
$ht{USE_ONLY_AT_START_OF_WORD}->{$utf8_source_string}->{$utf8_target_string} = 1;
}
}
if ($use_only_at_end_of_word_p) {
if ($lang_code) {
$ht{USE_ONLY_AT_END_OF_WORD_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_target_string} = 1;
} else {
$ht{USE_ONLY_AT_END_OF_WORD}->{$utf8_source_string}->{$utf8_target_string} = 1;
}
}
if ($dont_use_at_start_of_word_p) {
if ($lang_code) {
$ht{DONT_USE_AT_START_OF_WORD_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_target_string} = 1;
} else {
$ht{DONT_USE_AT_START_OF_WORD}->{$utf8_source_string}->{$utf8_target_string} = 1;
}
}
if ($dont_use_at_end_of_word_p) {
if ($lang_code) {
$ht{DONT_USE_AT_END_OF_WORD_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_target_string} = 1;
} else {
$ht{DONT_USE_AT_END_OF_WORD}->{$utf8_source_string}->{$utf8_target_string} = 1;
}
}
}
sub load_romanization_table {
local($this, *ht, $filename) = @_;
# ../../data/romanization-table.txt
$n = 0;
$line_number = 0;
if (open(IN, $filename)) {
while (<IN>) {
$line_number++;
next if /^#/;
if ($_ =~ /^::preserve\s/) {
$from_unicode = $util->slot_value_in_double_colon_del_list($_, "from");
$to_unicode = $util->slot_value_in_double_colon_del_list($_, "to");
if ($from_unicode =~ /^(?:U\+|\\u)[0-9A-F]{4,}$/i) {
$from_unicode =~ s/^(?:U\+|\\u)//;
$from_code_point = hex($from_unicode);
} else {
$from_code_point = "";
}
if ($to_unicode =~ /^(?:U\+|\\u)[0-9A-F]{4,}$/i) {
$to_unicode =~ s/^(?:U\+|\\u)//;
$to_code_point = hex($to_unicode);
} else {
$to_code_point = $from_code_point;
}
if ($from_code_point ne "") {
# print STDERR "Preserve code-points $from_unicode--$to_unicode = $from_code_point--$to_code_point\n";
foreach $code_point (($from_code_point .. $to_code_point)) {
$utf8_string = $utf8->unicode2string($code_point);
$ht{UTF_CHAR_MAPPING}->{$utf8_string}->{$utf8_string} = 1;
}
$n++;
}
next;
}
$utf8_source_string = $util->slot_value_in_double_colon_del_list($_, "s");
$utf8_target_string = $util->slot_value_in_double_colon_del_list($_, "t");
$utf8_alt_target_string_s = $util->slot_value_in_double_colon_del_list($_, "t-alt");
$use_alt_in_pointed_p = ($_ =~ /::use-alt-in-pointed\b/);
$use_only_for_whole_word_p = ($_ =~ /::use-only-for-whole-word\b/);
$use_only_at_start_of_word_p = ($_ =~ /::use-only-at-start-of-word\b/);
$use_only_at_end_of_word_p = ($_ =~ /::use-only-at-end-of-word\b/);
$dont_use_at_start_of_word_p = ($_ =~ /::dont-use-at-start-of-word\b/);
$dont_use_at_end_of_word_p = ($_ =~ /::dont-use-at-end-of-word\b/);
$use_only_in_lower_case_enviroment_p = ($_ =~ /::use-only-in-lower-case-enviroment\b/);
$word_external_punctuation_p = ($_ =~ /::word-external-punctuation\b/);
$utf8_source_string =~ s/\s*$//;
$utf8_target_string =~ s/\s*$//;
$utf8_alt_target_string_s =~ s/\s*$//;
$utf8_target_string =~ s/^"(.*)"$/$1/;
$utf8_target_string =~ s/^'(.*)'$/$1/;
@utf8_alt_targets = $this->listify_comma_sep_string($utf8_alt_target_string_s);
$numeric = $util->slot_value_in_double_colon_del_list($_, "num");
$numeric =~ s/\s*$//;
$annotation = $util->slot_value_in_double_colon_del_list($_, "annotation");
$annotation =~ s/\s*$//;
$lang_code = $util->slot_value_in_double_colon_del_list($_, "lcode");
$prob = $util->slot_value_in_double_colon_del_list($_, "p") || 1;
unless (($utf8_target_string eq "") && ($numeric =~ /\d/)) {
if ($lang_code) {
$ht{UTF_CHAR_MAPPING_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_target_string} = $prob;
} else {
$ht{UTF_CHAR_MAPPING}->{$utf8_source_string}->{$utf8_target_string} = $prob;
}
if ($word_external_punctuation_p) {
if ($lang_code) {
$ht{WORD_EXTERNAL_PUNCTUATION_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_target_string} = $prob;
} else {
$ht{WORD_EXTERNAL_PUNCTUATION}->{$utf8_source_string}->{$utf8_target_string} = $prob;
}
}
if ($this->braille_string_p($utf8_source_string)) {
if (($utf8_target_string =~ /^[a-z]+$/)
&& (! ($utf8_source_string =~ /^$braille_capital_letter_indicator/))) {
my $uc_utf8_source_string = "$braille_capital_letter_indicator$utf8_source_string";
my $uc_utf8_target_string = ucfirst $utf8_target_string;
if ($lang_code) {
$ht{UTF_CHAR_MAPPING_LANG_SPEC}->{$lang_code}->{$uc_utf8_source_string}->{$uc_utf8_target_string} = $prob;
} else {
$ht{UTF_CHAR_MAPPING}->{$uc_utf8_source_string}->{$uc_utf8_target_string} = $prob;
}
$this->register_word_boundary_info(*ht, $lang_code, $uc_utf8_source_string, $uc_utf8_target_string,
$use_only_for_whole_word_p, $use_only_at_start_of_word_p, $use_only_at_end_of_word_p,
$dont_use_at_start_of_word_p, $dont_use_at_end_of_word_p);
}
if (($utf8_target_string =~ /^[0-9]$/)
&& ($utf8_source_string =~ /^$braille_number_indicator./)) {
my $core_number_char = $utf8_source_string;
$core_number_char =~ s/$braille_number_indicator//;
$ht{BRAILLE_TO_DIGIT}->{$core_number_char} = $utf8_target_string;
}
}
}
if ($use_only_in_lower_case_enviroment_p) {
if ($lang_code) {
$ht{USE_ONLY_IN_LOWER_CASE_ENVIROMENT_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_target_string} = 1;
} else {
$ht{USE_ONLY_IN_LOWER_CASE_ENVIROMENT}->{$utf8_source_string}->{$utf8_target_string} = 1;
}
}
$this->register_word_boundary_info(*ht, $lang_code, $utf8_source_string, $utf8_target_string,
$use_only_for_whole_word_p, $use_only_at_start_of_word_p, $use_only_at_end_of_word_p,
$dont_use_at_start_of_word_p, $dont_use_at_end_of_word_p);
foreach $utf8_alt_target (@utf8_alt_targets) {
if ($lang_code) {
$ht{UTF_CHAR_ALT_MAPPING_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_alt_target} = $prob;
$ht{USE_ALT_IN_POINTED_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_alt_target} = 1 if $use_alt_in_pointed_p;
} else {
$ht{UTF_CHAR_ALT_MAPPING}->{$utf8_source_string}->{$utf8_alt_target} = $prob;
$ht{USE_ALT_IN_POINTED}->{$utf8_source_string}->{$utf8_alt_target} = 1 if $use_alt_in_pointed_p;
}
if ($use_only_for_whole_word_p) {
if ($lang_code) {
$ht{USE_ALT_ONLY_FOR_WHOLE_WORD_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_alt_target} = 1;
} else {
$ht{USE_ALT_ONLY_FOR_WHOLE_WORD}->{$utf8_source_string}->{$utf8_alt_target} = 1;
}
}
if ($use_only_at_start_of_word_p) {
if ($lang_code) {
$ht{USE_ALT_ONLY_AT_START_OF_WORD_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_alt_target} = 1;
} else {
$ht{USE_ALT_ONLY_AT_START_OF_WORD}->{$utf8_source_string}->{$utf8_alt_target} = 1;
}
}
if ($use_only_at_end_of_word_p) {
if ($lang_code) {
$ht{USE_ALT_ONLY_AT_END_OF_WORD_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_alt_target} = 1;
} else {
$ht{USE_ALT_ONLY_AT_END_OF_WORD}->{$utf8_source_string}->{$utf8_alt_target} = 1;
}
}
}
if ($numeric =~ /\d/) {
$ht{UTF_TO_NUMERIC}->{$utf8_source_string} = $numeric;
}
if ($annotation =~ /\S/) {
$ht{UTF_ANNOTATION}->{$utf8_source_string} = $annotation;
}
$n++;
}
close(IN);
# print STDERR "Loaded $n entries from $filename\n";
} else {
print STDERR "Can't open $filename\n";
}
}
sub char_name_to_script {
local($this, $char_name, *ht) = @_;
return $cached_result if $cached_result = $ht{CHAR_NAME_TO_SCRIPT}->{$char_name};
$orig_char_name = $char_name;
$char_name =~ s/\s+(CONSONANT|LETTER|LIGATURE|SIGN|SYLLABLE|SYLLABICS|VOWEL)\b.*$//;
my $script_name;
while ($char_name) {
last if $script_name = $ht{SCRIPT_NORM}->{(uc $char_name)};
$char_name =~ s/\s*\S+\s*$//;
}
$script_name = "" unless defined($script_name);
$ht{CHAR_NAME_TO_SCRIPT}->{$char_name} = $script_name;
return $script_name;
}
sub letter_plus_char_p {
local($this, $char_name) = @_;
return $cached_result if $cached_result = $ht{CHAR_NAME_LETTER_PLUS}->{$char_name};
my $letter_plus_p = ($char_name =~ /\b(?:LETTER|VOWEL SIGN|AU LENGTH MARK|CONSONANT SIGN|SIGN VIRAMA|SIGN PAMAAEH|SIGN COENG|SIGN AL-LAKUNA|SIGN ASAT|SIGN ANUSVARA|SIGN ANUSVARAYA|SIGN BINDI|TIPPI|SIGN NIKAHIT|SIGN CANDRABINDU|SIGN VISARGA|SIGN REAHMUK|SIGN NUKTA|SIGN DOT BELOW|HEBREW POINT)\b/) ? 1 : 0;
$ht{CHAR_NAME_LETTER_PLUS}->{$char_name} = $letter_plus_p;
return $letter_plus_p;
}
sub subjoined_char_p {
local($this, $char_name) = @_;
return $cached_result if $cached_result = $ht{CHAR_NAME_SUBJOINED}->{$char_name};
my $subjoined_p = (($char_name =~ /\b(?:SUBJOINED LETTER|VOWEL SIGN|AU LENGTH MARK|EMPHASIS MARK|CONSONANT SIGN|SIGN VIRAMA|SIGN PAMAAEH|SIGN COENG|SIGN ASAT|SIGN ANUSVARA|SIGN ANUSVARAYA|SIGN BINDI|TIPPI|SIGN NIKAHIT|SIGN CANDRABINDU|SIGN VISARGA|SIGN REAHMUK|SIGN DOT BELOW|HEBREW (POINT|PUNCTUATION GERESH)|ARABIC (?:DAMMA|DAMMATAN|FATHA|FATHATAN|HAMZA|KASRA|KASRATAN|MADDAH|SHADDA|SUKUN))\b/)) ? 1 : 0;
$ht{CHAR_NAME_SUBJOINED}->{$char_name} = $subjoined_p;
return $subjoined_p;
}
sub new_node_id {
local($this, *chart_ht) = @_;
my $n_nodes = $chart_ht{N_NODES};
$n_nodes++;
$chart_ht{N_NODES} = $n_nodes;
return $n_nodes;
}
sub add_node {
local($this, $s, $start, $end, *chart_ht, $type, $comment) = @_;
my $node_id = $this->new_node_id(*chart_ht);
# print STDERR "add_node($node_id, $start-$end): $s [$comment]\n" if $comment =~ /number/;
# print STDERR "add_node($node_id, $start-$end): $s [$comment]\n" if ($start >= 0) && ($start < 50);
$chart_ht{NODE_START}->{$node_id} = $start;
$chart_ht{NODE_END}->{$node_id} = $end;
$chart_ht{NODES_STARTING_AT}->{$start}->{$node_id} = 1;
$chart_ht{NODES_ENDING_AT}->{$end}->{$node_id} = 1;
$chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end}->{$node_id} = 1;
$chart_ht{NODE_TYPE}->{$node_id} = $type;
$chart_ht{NODE_COMMENT}->{$node_id} = $comment;
$chart_ht{NODE_ROMAN}->{$node_id} = $s;
return $node_id;
}
sub get_node_for_span {
local($this, $start, $end, *chart_ht) = @_;
return "" unless defined($chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end});
my @node_ids = sort { $a <=> $b } keys %{$chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end}};
return (@node_ids) ? $node_ids[0] : "";
}
sub get_node_for_span_and_type {
local($this, $start, $end, *chart_ht, $type) = @_;
return "" unless defined($chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end});
my @node_ids = sort { $a <=> $b } keys %{$chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end}};
foreach $node_id (@node_ids) {
return $node_id if $chart_ht{NODE_TYPE}->{$node_id} eq $type;
}
return "";
}
sub get_node_roman {
local($this, $node_id, *chart_id, $default) = @_;
$default = "" unless defined($default);
my $roman = $chart_ht{NODE_ROMAN}->{$node_id};
return (defined($roman)) ? $roman : $default;
}
sub set_node_id_slot_value {
local($this, $node_id, $slot, $value, *chart_id) = @_;
$chart_ht{NODE_SLOT}->{$node_id}->{$slot} = $value;
}
sub copy_slot_values {
local($this, $old_node_id, $new_node_id, *chart_id, @slots) = @_;
if (@slots) {
foreach $slot (keys %{$chart_ht{NODE_SLOT}->{$old_node_id}}) {
if (($slots[0] eq "all") || $util->member($slot, @slots)) {
my $value = $chart_ht{NODE_SLOT}->{$old_node_id}->{$slot};
$chart_ht{NODE_SLOT}->{$new_node_id}->{$slot} = $value if defined($value);
}
}
}
}
sub get_node_id_slot_value {
local($this, $node_id, $slot, *chart_id, $default) = @_;
$default = "" unless defined($default);
my $value = $chart_ht{NODE_SLOT}->{$node_id}->{$slot};
return (defined($value)) ? $value : $default;
}
sub get_node_for_span_with_slot_value {
local($this, $start, $end, $slot, *chart_id, $default) = @_;
$default = "" unless defined($default);
return $default unless defined($chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end});
my @node_ids = sort { $a <=> $b } keys %{$chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end}};
foreach $node_id (@node_ids) {
my $value = $chart_ht{NODE_SLOT}->{$node_id}->{$slot};
return $value if defined($value);
}
return $default;
}
sub get_node_for_span_with_slot {
local($this, $start, $end, $slot, *chart_id, $default) = @_;
$default = "" unless defined($default);
return $default unless defined($chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end});
my @node_ids = sort { $a <=> $b } keys %{$chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end}};
foreach $node_id (@node_ids) {
my $value = $chart_ht{NODE_SLOT}->{$node_id}->{$slot};
return $node_id if defined($value);
}
return $default;
}
sub register_new_complex_number_span_segment {
local($this, $start, $mid, $end, *chart_id, $line_number) = @_;
# e.g. 4 10 (= 40); 20 5 (= 25)
# might become part of larger complex number span, e.g. 4 1000 3 100 20 1
# print STDERR "register_new_complex_number_span_segment $start-$mid-$end\n" if $line_number == 43;
if (defined($old_start = $chart_ht{COMPLEX_NUMERIC_END_START}->{$mid})) {
undef($chart_ht{COMPLEX_NUMERIC_END_START}->{$mid});
$chart_ht{COMPLEX_NUMERIC_START_END}->{$old_start} = $end;
$chart_ht{COMPLEX_NUMERIC_END_START}->{$end} = $old_start;
} else {
$chart_ht{COMPLEX_NUMERIC_START_END}->{$start} = $end;
$chart_ht{COMPLEX_NUMERIC_END_START}->{$end} = $start;
}
}
sub romanize_by_token_with_caching {
local($this, $s, $lang_code, $output_style, *ht, *pinyin_ht, $initial_char_offset, $control, $line_number) = @_;
$control = "" unless defined($control);
my $return_chart_p = ($control =~ /return chart/i);
my $return_offset_mappings_p = ($control =~ /return offset mappings/i);
return $this->romanize($s, $lang_code, $output_style, *ht, *pinyin_ht, $initial_char_offset, $control, $line_number)
if $return_chart_p || $return_offset_mappings_p;
my $result = "";
my @separators = ();
my @tokens = ();
$s =~ s/\n$//; # Added May 2, 2019 as bug-fix (duplicate empty lines)
while (($sep, $token, $rest) = ($s =~ /^(\s*)(\S+)(.*)$/)) {
push(@separators, $sep);
push(@tokens, $token);
$s = $rest;
}
push(@separators, $s);
while (@tokens) {
my $sep = shift @separators;
my $token = shift @tokens;
$result .= $sep;
if ($token =~ /^[\x00-\x7F]*$/) { # all ASCII
$result .= $token;
} else {
my $rom_token = $ht{CACHED_ROMANIZATION}->{$lang_code}->{$token};
unless (defined($rom_token)) {
$rom_token = $this->romanize($token, $lang_code, $output_style, *ht, *pinyin_ht, $initial_char_offset, $control, $line_number);
$ht{CACHED_ROMANIZATION}->{$lang_code}->{$token} = $rom_token if defined($rom_token);
}
$result .= $rom_token;
}
}
my $sep = shift @separators;
$result .= $sep if defined($sep);
return $result;
}
sub romanize {
local($this, $s, $lang_code, $output_style, *ht, *pinyin_ht, $initial_char_offset, $control, $line_number, $initial_rom_char_offset) = @_;
my $orig_lang_code = $lang_code;
# Check whether the text (to be romanized) starts with a language code directive.
if (($line_lang_code) = ($s =~ /^::lcode\s+([a-z][a-z][a-z])\s/)) {
$lang_code = $line_lang_code;
}
$initial_char_offset = 0 unless defined($initial_char_offset);
$initial_rom_char_offset = 0 unless defined($initial_rom_char_offset);
$control = "" unless defined($control);
my $return_chart_p = ($control =~ /return chart/i);
my $return_offset_mappings_p = ($control =~ /return offset mappings/i);
$line_number = "" unless defined($line_number);
my @chars = $utf8->split_into_utf8_characters($s, "return only chars", *empty_ht);
my $n_characters = $#chars + 1;
%chart_ht = ();
$chart_ht{N_CHARS} = $n_characters;
$chart_ht{N_NODES} = 0;
my $char = "";
my $char_name = "";
my $prev_script = "";
my $current_script = "";
my $script_start = 0;
my $script_end = 0;
my $prev_letter_plus_script = "";
my $current_letter_plus_script = "";
my $letter_plus_script_start = 0;
my $letter_plus_script_end = 0;
my $log ="";
my $n_right_to_left_chars = 0;
my $n_left_to_right_chars = 0;
my $hebrew_word_start = ""; # used to identify Hebrew words with points
my $hebrew_word_contains_point = 0;
my $current_word_start = "";
my $current_word_script = "";
my $braille_all_caps_p = 0;
# prep
foreach $i ((0 .. ($#chars + 1))) {
if ($i <= $#chars) {
$char = $chars[$i];
$chart_ht{ORIG_CHAR}->{$i} = $char;
$char_name = $ht{UTF_TO_CHAR_NAME}->{$char} || "";
$chart_ht{CHAR_NAME}->{$i} = $char_name;
$current_script = $this->char_name_to_script($char_name, *ht);
$current_script_direction = $ht{DIRECTION}->{$current_script} || '';
if ($current_script_direction eq 'right-to-left') {
$n_right_to_left_chars++;
} elsif (($char =~ /^[a-z]$/i) || ! ($char =~ /^[\x00-\x7F]$/)) {
$n_left_to_right_chars++;
}
$chart_ht{CHAR_SCRIPT}->{$i} = $current_script;
$chart_ht{SCRIPT_SEGMENT_START}->{$i} = ""; # default value, to be updated later
$chart_ht{SCRIPT_SEGMENT_END}->{$i} = ""; # default value, to be updated later
$chart_ht{LETTER_TOKEN_SEGMENT_START}->{$i} = ""; # default value, to be updated later
$chart_ht{LETTER_TOKEN_SEGMENT_END}->{$i} = ""; # default value, to be updated later
$subjoined_char_p = $this->subjoined_char_p($char_name);
$chart_ht{CHAR_SUBJOINED}->{$i} = $subjoined_char_p;
$letter_plus_char_p = $this->letter_plus_char_p($char_name);
$chart_ht{CHAR_LETTER_PLUS}->{$i} = $letter_plus_char_p;
$current_letter_plus_script = ($letter_plus_char_p) ? $current_script : "";
$numeric_value = $ht{UTF_TO_NUMERIC}->{$char};
$numeric_value = "" unless defined($numeric_value);
$annotation = $ht{UTF_ANNOTATION}->{$char};
$annotation = "" unless defined($annotation);
$chart_ht{CHAR_NUMERIC_VALUE}->{$i} = $numeric_value;
$chart_ht{CHAR_ANNOTATION}->{$i} = $annotation;
$syllable_info = $ht{UTF_TO_SYLLABLE_INFO}->{$char} || "";
$chart_ht{CHAR_SYLLABLE_INFO}->{$i} = $syllable_info;
$tone_mark = $ht{UTF_TO_TONE_MARK}->{$char} || "";
$chart_ht{CHAR_TONE_MARK}->{$i} = $tone_mark;
} else {
$char = "";
$char_name = "";
$current_script = "";
$current_letter_plus_script = "";
}
if ($char_name =~ /^HEBREW (LETTER|POINT|PUNCTUATION GERESH) /) {
$hebrew_word_start = $i if $hebrew_word_start eq "";
$hebrew_word_contains_point = 1 if $char_name =~ /^HEBREW POINT /;
} elsif ($hebrew_word_start ne "") {
if ($hebrew_word_contains_point) {
foreach $j (($hebrew_word_start .. ($i-1))) {
$chart_ht{CHAR_PART_OF_POINTED_HEBREW_WORD}->{$j} = 1;
}
$chart_ht{CHAR_START_OF_WORD}->{$hebrew_word_start} = 1;
$chart_ht{CHAR_END_OF_WORD}->{($i-1)} = 1;
}
$hebrew_word_start = "";
$hebrew_word_contains_point = 0;
}
my $part_of_word_p = $current_script
&& ($this->letter_plus_char_p($char_name)
|| $this->subjoined_char_p($char_name)
|| ($char_name =~ /\b(LETTER|SYLLABLE|SYLLABICS|LIGATURE)\b/));
# Braille punctuation
my $end_offset = 0;
if ($char_name =~ /^Braille\b/i) {
if (($char =~ /^\s*$/) || ($char_name =~ /BLANK/)) {
$part_of_word_p = 0;
$braille_all_caps_p = 0;
} elsif ($chart_ht{NOT_PART_OF_WORD_P}->{$i}) {
$part_of_word_p = 0;
$braille_all_caps_p = 0;
} elsif ((keys %{$ht{WORD_EXTERNAL_PUNCTUATION_LANG_SPEC}->{$lang_code}->{$char}})
|| (keys %{$ht{WORD_EXTERNAL_PUNCTUATION}->{$char}})) {
$part_of_word_p = 0;
$braille_all_caps_p = 0;
} elsif (($i+1 <= $#chars)
&& ($s1 = $char . $chars[$i+1])
&& ((keys %{$ht{WORD_EXTERNAL_PUNCTUATION_LANG_SPEC}->{$lang_code}->{$s1}})
|| (keys %{$ht{WORD_EXTERNAL_PUNCTUATION}->{$s1}}))) {
$part_of_word_p = 0;
$braille_all_caps_p = 0;
$chart_ht{NOT_PART_OF_WORD_P}->{($i+1)} = 1;
} elsif (($i+2 <= $#chars)
&& ($s2 = $char . $chars[$i+1] . $chars[$i+2])
&& ((keys %{$ht{WORD_EXTERNAL_PUNCTUATION_LANG_SPEC}->{$lang_code}->{$s2}})
|| (keys %{$ht{WORD_EXTERNAL_PUNCTUATION}->{$s2}}))) {
$part_of_word_p = 0;
$braille_all_caps_p = 0;
$chart_ht{NOT_PART_OF_WORD_P}->{($i+1)} = 1;
$chart_ht{NOT_PART_OF_WORD_P}->{($i+2)} = 1;
} elsif (($i+1 <= $#chars)
&& ($char eq $braille_capital_letter_indicator)
&& ($chars[$i+1] eq $braille_capital_letter_indicator)) {
$braille_all_caps_p = 1;
} else {
$part_of_word_p = 1;
}
# last period in Braille text is also not part_of_word_p
if (($char eq $braille_period)
&& (($i == $#chars)
|| (($i < $#chars)
&& (! $this->braille_string_p($chars[$i+1]))))) {
$part_of_word_p = 0;
}
# period before other word-external punctuation is also not part_of_word_p
if (($i > 0)
&& ($chars[$i-1] eq $braille_period)
&& (! $part_of_word_p)
&& ($current_word_start ne "")) {
$end_offset = -1;
}
} else {
$braille_all_caps_p = 0;
}
$chart_ht{BRAILLE_ALL_CAPS_P}->{$i} = $braille_all_caps_p;
if (($current_word_start ne "")
&& ((! $part_of_word_p)
|| ($current_script ne $current_word_script))) {
# END OF WORD
$chart_ht{CHAR_START_OF_WORD}->{$current_word_start} = 1;
$chart_ht{CHAR_END_OF_WORD}->{($i-1+$end_offset)} = 1;
my $word = join("", @chars[$current_word_start .. ($i-1+$end_offset)]);
$chart_ht{WORD_START_END}->{$current_word_start}->{$i} = $word;
$chart_ht{WORD_END_START}->{$i+$end_offset}->{$current_word_start} = $word;
# print STDERR "Word ($current_word_start-$i+$end_offset): $word ($current_word_script)\n";
$current_word_start = "";
$current_word_script = "";
}
if ($part_of_word_p && ($current_word_start eq "")) {
# START OF WORD
$current_word_start = $i;
$current_word_script = $current_script;
}
# print STDERR "$i char: $char ($current_script)\n";
unless ($current_script eq $prev_script) {
if ($prev_script && ($i-1 >= $script_start)) {
my $script_end = $i;
$chart_ht{SCRIPT_SEGMENT_START_TO_END}->{$script_start} = $script_end;
$chart_ht{SCRIPT_SEGMENT_END_TO_START}->{$script_end} = $script_start;
foreach $i (($script_start .. $script_end)) {
$chart_ht{SCRIPT_SEGMENT_START}->{$i} = $script_start;
$chart_ht{SCRIPT_SEGMENT_END}->{$i} = $script_end;
}
# print STDERR "Script segment $script_start-$script_end: $prev_script\n";
}
$script_start = $i;
}
unless ($current_letter_plus_script eq $prev_letter_plus_script) {
if ($prev_letter_plus_script && ($i-1 >= $letter_plus_script_start)) {
my $letter_plus_script_end = $i;
$chart_ht{LETTER_TOKEN_SEGMENT_START_TO_END}->{$letter_plus_script_start} = $letter_plus_script_end;
$chart_ht{LETTER_TOKEN_SEGMENT_END_TO_START}->{$letter_plus_script_end} = $letter_plus_script_start;
foreach $i (($letter_plus_script_start .. $letter_plus_script_end)) {
$chart_ht{LETTER_TOKEN_SEGMENT_START}->{$i} = $letter_plus_script_start;
$chart_ht{LETTER_TOKEN_SEGMENT_END}->{$i} = $letter_plus_script_end;
}
# print STDERR "Script token segment $letter_plus_script_start-$letter_plus_script_end: $prev_letter_plus_script\n";
}
$letter_plus_script_start = $i;
}
$prev_script = $current_script;
$prev_letter_plus_script = $current_letter_plus_script;
}
$ht{STRING_IS_DOMINANTLY_RIGHT_TO_LEFT}->{$s} = 1 if $n_right_to_left_chars > $n_left_to_right_chars;
# main
my $i = 0;
while ($i <= $#chars) {
my $char = $chart_ht{ORIG_CHAR}->{$i};
my $current_script = $chart_ht{CHAR_SCRIPT}->{$i};
$chart_ht{CHART_CONTAINS_SCRIPT}->{$current_script} = 1;
my $script_segment_start = $chart_ht{SCRIPT_SEGMENT_START}->{$i};
my $script_segment_end = $chart_ht{SCRIPT_SEGMENT_END}->{$i};
my $char_name = $chart_ht{CHAR_NAME}->{$i};
my $subjoined_char_p = $chart_ht{CHAR_SUBJOINED}->{$i};
my $letter_plus_char_p = $chart_ht{CHAR_LETTER_PLUS}->{$i};
my $numeric_value = $chart_ht{CHAR_NUMERIC_VALUE}->{$i};
my $annotation = $chart_ht{CHAR_ANNOTATION}->{$i};
# print STDERR " $char_name annotation: $annotation\n" if $annotation;
my $tone_mark = $chart_ht{CHAR_TONE_MARK}->{$i};
my $found_char_mapping_p = 0;
my $prev_char_name = ($i >= 1) ? $chart_ht{CHAR_NAME}->{($i-1)} : "";
my $prev2_script = ($i >= 2) ? $chart_ht{CHAR_SCRIPT}->{($i-2)} : "";
my $prev_script = ($i >= 1) ? $chart_ht{CHAR_SCRIPT}->{($i-1)} : "";
my $next_script = ($i < $#chars) ? $chart_ht{CHAR_SCRIPT}->{($i+1)} : "";
my $next_char = ($i < $#chars) ? $chart_ht{ORIG_CHAR}->{($i+1)} : "";
my $next_char_name = $ht{UTF_TO_CHAR_NAME}->{$next_char} || "";
my $prev2_letter_plus_char_p = ($i >= 2) ? $chart_ht{CHAR_LETTER_PLUS}->{($i-2)} : 0;
my $prev_letter_plus_char_p = ($i >= 1) ? $chart_ht{CHAR_LETTER_PLUS}->{($i-1)} : 0;
my $next_letter_plus_char_p = ($i < $#chars) ? $chart_ht{CHAR_LETTER_PLUS}->{($i+1)} : 0;
my $next_index = $i + 1;
# Braille numeric mode
if ($char eq $braille_number_indicator) {
my $offset = 0;
my $numeric_value = "";
my $digit;
while ($i+$offset < $#chars) {
$offset++;
my $offset_char = $chart_ht{ORIG_CHAR}->{$i+$offset};
if (defined($digit = $ht{BRAILLE_TO_DIGIT}->{$offset_char})) {
$numeric_value .= $digit;
} elsif (($offset_char eq $braille_decimal_point)
|| ($ht{UTF_CHAR_MAPPING}->{$offset_char}->{"."})) {
$numeric_value .= ".";
} elsif ($offset_char eq $braille_comma) {
$numeric_value .= ",";
} elsif ($offset_char eq $braille_numeric_space) {
$numeric_value .= " ";
} elsif ($offset_char eq $braille_solidus) {
$numeric_value .= "/";
} elsif ($offset_char eq $braille_number_indicator) {
# stay in Braille numeric mode
} elsif ($offset_char eq $braille_letter_indicator) {
# consider as part of number, but without contributing to numeric_value
last;
} else {
$offset--;
last;
}
}
if ($offset) {
$next_index = $i + $offset + 1;
$node_id = $this->add_node($numeric_value, $i, $next_index, *chart_ht, "", "braille number");
$found_char_mapping_p = 1;
}
}
unless ($found_char_mapping_p) {
foreach $string_length (reverse(1 .. 6)) {
next if ($i + $string_length-1) > $#chars;
my $start_of_word_p = $chart_ht{CHAR_START_OF_WORD}->{$i} || 0;
my $end_of_word_p = $chart_ht{CHAR_END_OF_WORD}->{($i+$string_length-1)} || 0;
my $multi_char_substring = join("", @chars[$i..($i+$string_length-1)]);
my @mappings = keys %{$ht{UTF_CHAR_MAPPING_LANG_SPEC}->{$lang_code}->{$multi_char_substring}};
@mappings = keys %{$ht{UTF_CHAR_MAPPING}->{$multi_char_substring}} unless @mappings;
my @mappings_whole = ();
my @mappings_start_or_end = ();
my @mappings_other = ();
foreach $mapping (@mappings) {
next if $mapping =~ /\(__.*__\)/;
if ($ht{USE_ONLY_FOR_WHOLE_WORD_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$mapping}
|| $ht{USE_ONLY_FOR_WHOLE_WORD}->{$multi_char_substring}->{$mapping}) {
push(@mappings_whole, $mapping) if $start_of_word_p && $end_of_word_p;
} elsif ($ht{USE_ONLY_AT_START_OF_WORD_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$mapping}
|| $ht{USE_ONLY_AT_START_OF_WORD}->{$multi_char_substring}->{$mapping}) {
push(@mappings_start_or_end, $mapping) if $start_of_word_p;
} elsif ($ht{USE_ONLY_AT_END_OF_WORD_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$mapping}
|| $ht{USE_ONLY_AT_END_OF_WORD}->{$multi_char_substring}->{$mapping}) {
push(@mappings_start_or_end, $mapping) if $end_of_word_p;
} else {
push(@mappings_other, $mapping);
}
}
@mappings = @mappings_whole;
@mappings = @mappings_start_or_end unless @mappings;
@mappings = @mappings_other unless @mappings;
foreach $mapping (@mappings) {
next if $mapping =~ /\(__.*__\)/;
if ($ht{DONT_USE_AT_START_OF_WORD_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$mapping}
|| $ht{DONT_USE_AT_START_OF_WORD}->{$multi_char_substring}->{$mapping}) {
next if $start_of_word_p;
}
if ($ht{DONT_USE_AT_END_OF_WORD_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$mapping}
|| $ht{DONT_USE_AT_END_OF_WORD}->{$multi_char_substring}->{$mapping}) {
next if $end_of_word_p;
}
my $mapping2 = ($chart_ht{BRAILLE_ALL_CAPS_P}->{$i}) ? (uc $mapping) : $mapping;
$node_id = $this->add_node($mapping2, $i, $i+$string_length, *chart_ht, "", "multi-char-mapping");
$next_index = $i + $string_length;
$found_char_mapping_p = 1;
if ($annotation) {
@annotation_elems = split(/,\s*/, $annotation);
foreach $annotation_elem (@annotation_elems) {
if (($a_slot, $a_value) = ($annotation_elem =~ /^(\S+?):(\S+)\s*$/)) {
$this->set_node_id_slot_value($node_id, $a_slot, $a_value, *chart_ht);
} else {
$this->set_node_id_slot_value($node_id, $annotation_elem, 1, *chart_ht);
}
}
}
}
my @alt_mappings = keys %{$ht{UTF_CHAR_ALT_MAPPING_LANG_SPEC}->{$lang_code}->{$multi_char_substring}};
@alt_mappings = keys %{$ht{UTF_CHAR_ALT_MAPPING}->{$multi_char_substring}} unless @alt_mappings;
@alt_mappings = () if ($#alt_mappings == 0) && ($alt_mappings[0] eq "_NONE_");
foreach $alt_mapping (@alt_mappings) {
if ($chart_ht{CHAR_PART_OF_POINTED_HEBREW_WORD}->{$i}) {
next unless
$ht{USE_ALT_IN_POINTED_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$alt_mapping}
|| $ht{USE_ALT_IN_POINTED}->{$multi_char_substring}->{$alt_mapping};
}
if ($ht{USE_ALT_ONLY_FOR_WHOLE_WORD_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$alt_mapping}
|| $ht{USE_ALT_ONLY_FOR_WHOLE_WORD}->{$multi_char_substring}->{$alt_mapping}) {
next unless $start_of_word_p && $end_of_word_p;
}
if ($ht{USE_ALT_ONLY_AT_START_OF_WORD_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$alt_mapping}
|| $ht{USE_ALT_ONLY_AT_START_OF_WORD}->{$multi_char_substring}->{$alt_mapping}) {
next unless $start_of_word_p;
}
if ($ht{USE_ALT_ONLY_AT_END_OF_WORD_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$alt_mapping}
|| $ht{USE_ALT_ONLY_AT_END_OF_WORD}->{$multi_char_substring}->{$alt_mapping}) {
next unless $end_of_word_p;
}
my $alt_mapping2 = ($chart_ht{BRAILLE_ALL_CAPS_P}->{$i}) ? (uc $alt_mapping) : $alt_mapping;
$node_id = $this->add_node($alt_mapping2, $i, $i+$string_length, *chart_ht, "alt", "multi-char-mapping");
if ($annotation) {
@annotation_elems = split(/,\s*/, $annotation);
foreach $annotation_elem (@annotation_elems) {
if (($a_slot, $a_value) = ($annotation_elem =~ /^(\S+?):(\S+)\s*$/)) {
$this->set_node_id_slot_value($node_id, $a_slot, $a_value, *chart_ht);
} else {
$this->set_node_id_slot_value($node_id, $annotation_elem, 1, *chart_ht);
}
}
}
}
}
}
unless ($found_char_mapping_p) {
my $prev_node_id = $this->get_node_for_span($i-4, $i, *chart_ht)
|| $this->get_node_for_span($i-3, $i, *chart_ht)
|| $this->get_node_for_span($i-2, $i, *chart_ht)
|| $this->get_node_for_span($i-1, $i, *chart_ht);
my $prev_char_roman = ($prev_node_id) ? $this->get_node_roman($prev_node_id, *chart_id) : "";
my $prev_node_start = ($prev_node_id) ? $chart_ht{NODE_START}->{$prev_node_id} : "";
# Number
if (($numeric_value =~ /\d/)
&& (! ($char_name =~ /SUPERSCRIPT/))) {
my $prev_numeric_value = $this->get_node_for_span_with_slot_value($i-1, $i, "numeric-value", *chart_id);
my $sep = "";
$sep = " " if ($char_name =~ /^vulgar fraction /i) && ($prev_numeric_value =~ /\d/);
$node_id = $this->add_node("$sep$numeric_value", $i, $i+1, *chart_ht, "", "number");
$this->set_node_id_slot_value($node_id, "numeric-value", $numeric_value, *chart_ht);
if ((($prev_numeric_value =~ /\d/) && ($numeric_value =~ /\d\d/))
|| (($prev_numeric_value =~ /\d\d/) && ($numeric_value =~ /\d/))) {
# pull in any other parts of single digits
my $j = 1;
# pull in any single digits adjoining on left
if ($prev_numeric_value =~ /^\d$/) {
while (1) {
if (($i-$j-1 >= 0)
&& defined($digit_value = $this->get_node_for_span_with_slot_value($i-$j-1, $i-$j, "numeric-value", *chart_id))
&& ($digit_value =~ /^\d$/)) {
$j++;
} elsif (($i-$j-2 >= 0)
&& ($chart_ht{ORIG_CHAR}->{($i-$j-1)} =~ /^[.,]$/)
&& defined($digit_value = $this->get_node_for_span_with_slot_value($i-$j-2, $i-$j-1, "numeric-value", *chart_id))
&& ($digit_value =~ /^\d$/)) {
$j += 2;
} else {
last;
}
}
}
# pull in any single digits adjoining on right
my $k = 0;
if ($numeric_value =~ /^\d$/) {
while (1) {
if (defined($next_numeric_value = $chart_ht{CHAR_NUMERIC_VALUE}->{($i+$k+1)})
&& ($next_numeric_value =~ /^\d$/)) {
$k++;
} else {
last;
}
}
}
$this->register_new_complex_number_span_segment($i-$j, $i, $i+$k+1, *chart_ht, $line_number);
}
if ($chinesePM->string_contains_utf8_cjk_unified_ideograph_p($char)
&& ($tonal_translit = $chinesePM->tonal_pinyin($char, *pinyin_ht, ""))) {
$de_accented_translit = $util->de_accent_string($tonal_translit);
if ($numeric_value =~ /^(10000|1000000000000|10000000000000000)$/) {
$chart_ht{NODE_TYPE}->{$node_id} = "alt"; # keep, but demote
$alt_node_id = $this->add_node($de_accented_translit, $i, $i+1, *chart_ht, "", "CJK");
} else {
$alt_node_id = $this->add_node($de_accented_translit, $i, $i+1, *chart_ht, "alt", "CJK");
}
}
# ASCII
} elsif ($char =~ /^[\x00-\x7F]$/) {
$this->add_node($char, $i, $i+1, *chart_ht, "", "ASCII"); # ASCII character, incl. control characters
# Emoji, dingbats, pictographs
} elsif ($char =~ /^(\xE2[\x98-\x9E]|\xF0\x9F[\x8C-\xA7])/) {
$this->add_node($char, $i, $i+1, *chart_ht, "", "pictograph");
# Hangul (Korean)
} elsif (($char =~ /^[\xEA-\xED]/)
&& ($romanized_char = $this->unicode_hangul_romanization($char))) {
$this->add_node($romanized_char, $i, $i+1, *chart_ht, "", "Hangul");
# CJK (Chinese, Japanese, Korean)
} elsif ($chinesePM->string_contains_utf8_cjk_unified_ideograph_p($char)
&& ($tonal_translit = $chinesePM->tonal_pinyin($char, *pinyin_ht, ""))) {
$de_accented_translit = $util->de_accent_string($tonal_translit);
$this->add_node($de_accented_translit, $i, $i+1, *chart_ht, "", "CJK");
# Virama (cancel preceding vowel in Abudiga scripts)
} elsif ($char_name =~ /\bSIGN (?:VIRAMA|AL-LAKUNA|ASAT|COENG|PAMAAEH)\b/) {
# VIRAMA: cancel preceding default vowel (in Abudiga scripts)
if (($prev_script eq $current_script)
&& (($prev_char_roman_consonant, $prev_char_roman_vowel) = ($prev_char_roman =~ /^(.*[bcdfghjklmnpqrstvwxyz])([aeiou]+)$/i))
&& ($ht{SCRIPT_ABUDIGA_DEFAULT_VOWEL}->{$current_script}->{(lc $prev_char_roman_vowel)})) {
$this->add_node($prev_char_roman_consonant, $prev_node_start, $i+1, *chart_ht, "", "virama");
} else {
$this->add_node("", $i, $i+1, *chart_ht, "", "unexpected-virama");
}
# Nukta (special (typically foreign) variant)
} elsif ($char_name =~ /\bSIGN (?:NUKTA)\b/) {
# NUKTA (dot): indicates special (typically foreign) variant; normally covered by multi-mappings
if ($prev_script eq $current_script) {
my $node_id = $this->add_node($prev_char_roman, $prev_node_start, $i+1, *chart_ht, "", "nukta");
$this->copy_slot_values($prev_node_id, $node_id, *chart_id, "all");
$this->set_node_id_slot_value($node_id, "nukta", 1, *chart_ht);
} else {
$this->add_node("", $i, $i+1, *chart_ht, "", "unexpected-nukta");
}
# Zero-width character, incl. zero width space/non-joiner/joiner, left-to-right/right-to-left mark
} elsif ($char =~ /^\xE2\x80[\x8B-\x8F\xAA-\xAE]$/) {
if ($prev_node_id) {
my $node_id = $this->add_node($prev_char_roman, $prev_node_start, $i+1, *chart_ht, "", "zero-width-char");
$this->copy_slot_values($prev_node_id, $node_id, *chart_id, "all");
} else {
$this->add_node("", $i, $i+1, *chart_ht, "", "zero-width-char");
}
} elsif (($char =~ /^\xEF\xBB\xBF$/) && $prev_node_id) { # OK to leave byte-order-mark at beginning of line
my $node_id = $this->add_node($prev_char_roman, $prev_node_start, $i+1, *chart_ht, "", "zero-width-char");
$this->copy_slot_values($prev_node_id, $node_id, *chart_id, "all");
# Tone mark
} elsif ($tone_mark) {
if ($prev_script eq $current_script) {
my $node_id = $this->add_node($prev_char_roman, $prev_node_start, $i+1, *chart_ht, "", "tone-mark");
$this->copy_slot_values($prev_node_id, $node_id, *chart_id, "all");
$this->set_node_id_slot_value($node_id, "tone-mark", $tone_mark, *chart_ht);
} else {
$this->add_node("", $i, $i+1, *chart_ht, "", "unexpected-tone-mark");
}
# Diacritic
} elsif (($char_name =~ /\b(ACCENT|TONE|COMBINING DIAERESIS|COMBINING DIAERESIS BELOW|COMBINING MACRON|COMBINING VERTICAL LINE ABOVE|COMBINING DOT ABOVE RIGHT|COMBINING TILDE|COMBINING CYRILLIC|MUUSIKATOAN|TRIISAP)\b/) && ($ht{UTF_TO_CAT}->{$char} =~ /^Mn/)) {
if ($prev_script eq $current_script) {
my $node_id = $this->add_node($prev_char_roman, $prev_node_start, $i+1, *chart_ht, "", "diacritic");
$this->copy_slot_values($prev_node_id, $node_id, *chart_id, "all");
$diacritic = lc $char_name;
$diacritic =~ s/^.*(?:COMBINING CYRILLIC|COMBINING|SIGN)\s+//i;
$diacritic =~ s/^.*(ACCENT|TONE)/$1/i;
$diacritic =~ s/^\s*//;
$this->set_node_id_slot_value($node_id, "diacritic", $diacritic, *chart_ht);
# print STDERR "diacritic: $diacritic\n";
} else {
$this->add_node("", $i, $i+1, *chart_ht, "", "unexpected-diacritic");
}
# Romanize to find out more
} elsif ($char_name) {
if (defined($romanized_char = $this->romanize_char_at_position($i, $lang_code, $output_style, *ht, *chart_ht))) {
# print STDERR "ROM l.$line_number/$i: $romanized_char\n" if $line_number =~ /^[12]$/;
print STDOUT "ROM l.$line_number/$i: $romanized_char\n" if $verbosePM;
# Empty string mapping
if ($romanized_char eq "\"\"") {
$this->add_node("", $i, $i+1, *chart_ht, "", "empty-string-mapping");
# consider adding something for implausible romanizations of length 6+
# keep original character (instead of romanized_char lengthener, character-18b00 etc.)
} elsif (($romanized_char =~ /^(character|lengthener|modifier)/)) {
$this->add_node($char, $i, $i+1, *chart_ht, "", "nevermind-keep-original");
# Syllabic suffix in Abudiga languages, e.g. -m, -ng
} elsif (($romanized_char =~ /^\+(H|M|N|NG)$/i)
&& ($prev_script eq $current_script)
&& ($ht{SCRIPT_ABUDIGA_DEFAULT_VOWEL}->{$current_script}->{"a"})) {
my $core_suffix = $romanized_char;
$core_suffix =~ s/^\+//;
if ($prev_char_roman =~ /[aeiou]$/i) {
$this->add_node($core_suffix, $i, $i+1, *chart_ht, "", "syllable-end-consonant");
} else {
$this->add_node(join("", $prev_char_roman, "a", $core_suffix), $prev_node_start, $i+1, *chart_ht, "", "syllable-end-consonant-with-added-a");
$this->add_node(join("", "a", $core_suffix), $i, $i+1, *chart_ht, "backup", "syllable-end-consonant");
}
# Japanese special cases
} elsif ($char_name =~ /(?:HIRAGANA|KATAKANA) LETTER SMALL Y/) {
if (($prev_script eq $current_script)
&& (($prev_char_roman_consonant) = ($prev_char_roman =~ /^(.*[bcdfghjklmnpqrstvwxyz])i$/i))) {
unless ($this->get_node_for_span_and_type($prev_node_start, $i+1, *chart_ht, "")) {
$this->add_node("$prev_char_roman_consonant$romanized_char", $prev_node_start, $i+1, *chart_ht, "", "japanese-contraction");
}
} else {
$this->add_node($romanized_char, $i, $i+1, *chart_ht, "", "unexpected-japanese-contraction-character");
}
} elsif (($prev_script =~ /^(HIRAGANA|KATAKANA)$/i)
&& ($char_name eq "KATAKANA-HIRAGANA PROLONGED SOUND MARK") # Choonpu
&& (($prev_char_roman_vowel) = ($prev_char_roman =~ /([aeiou])$/i))) {
$this->add_node("$prev_char_roman$prev_char_roman_vowel", $prev_node_start, $i+1, *chart_ht, "", "japanese-vowel-lengthening");
} elsif (($current_script =~ /^(Hiragana|Katakana)$/i)
&& ($char_name =~ /^(HIRAGANA|KATAKANA) LETTER SMALL TU$/i) # Sokuon/Sukun
&& ($next_script eq $current_script)
&& ($romanized_next_char = $this->romanize_char_at_position_incl_multi($i+1, $lang_code, $output_style, *ht, *chart_ht))
&& (($doubled_consonant) = ($romanized_next_char =~ /^(ch|[bcdfghjklmnpqrstwz])/i))) {
# Note: $romanized_next_char could be part of a multi-character mapping
# print STDERR "current_script: $current_script char_name: $char_name next_script: $next_script romanized_next_char: $romanized_next_char doubled_consonant: $doubled_consonant\n";
$doubled_consonant = "t" if $doubled_consonant eq "ch";
$this->add_node($doubled_consonant, $i, $i+1, *chart_ht, "", "japanese-consonant-doubling");
# Greek small letter mu to micro-sign (instead of to "m") as used in abbreviations for microgram/micrometer/microliter/microsecond/micromolar/microfarad etc.
} elsif (($char_name eq "GREEK SMALL LETTER MU")
&& (! ($prev_script =~ /^GREEK$/))
&& ($i < $#chars)
&& ($chart_ht{ORIG_CHAR}->{($i+1)} =~ /^[cfgjlmstv]$/i)) {
$this->add_node("\xC2\xB5", $i, $i+1, *chart_ht, "", "greek-mu-to-micro-sign");
# Gurmukhi addak (doubles following consonant)
} elsif (($current_script eq "Gurmukhi")
&& ($char_name eq "GURMUKHI ADDAK")) {
if (($next_script eq $current_script)
&& ($romanized_next_char = $this->romanize_char_at_position_incl_multi($i+1, $lang_code, $output_style, *ht, *chart_ht))
&& (($doubled_consonant) = ($romanized_next_char =~ /^([bcdfghjklmnpqrstvwxz])/i))) {
$this->add_node($doubled_consonant, $i, $i+1, *chart_ht, "", "gurmukhi-consonant-doubling");
} else {
$this->add_node("'", $i, $i+1, *chart_ht, "", "gurmukhi-unexpected-addak");
}
# Subjoined character
} elsif ($subjoined_char_p
&& ($prev_script eq $current_script)
&& (($prev_char_roman_consonant, $prev_char_roman_vowel) = ($prev_char_roman =~ /^(.*[bcdfghjklmnpqrstvwxyz])([aeiou]+)$/i))
&& ($ht{SCRIPT_ABUDIGA_DEFAULT_VOWEL}->{$current_script}->{(lc $prev_char_roman_vowel)})) {
my $new_roman = "$prev_char_roman_consonant$romanized_char";
$this->add_node($new_roman, $prev_node_start, $i+1, *chart_ht, "", "subjoined-character");
# print STDERR " Subjoin l.$line_number/$i: $new_roman\n" if $line_number =~ /^[12]$/;
# Thai special case: written-pre-consonant-spoken-post-consonant
} elsif (($char_name =~ /THAI CHARACTER/)
&& ($prev_script eq $current_script)
&& ($chart_ht{CHAR_SYLLABLE_INFO}->{($i-1)} =~ /written-pre-consonant-spoken-post-consonant/i)
&& ($prev_char_roman =~ /^[aeiou]+$/i)
&& ($romanized_char =~ /^[bcdfghjklmnpqrstvwxyz]/)) {
$this->add_node("$romanized_char$prev_char_roman", $prev_node_start, $i+1, *chart_ht, "", "thai-vowel-consonant-swap");
# Thai special case: THAI CHARACTER O ANG (U+0E2D "\xE0\xB8\xAD")
} elsif ($char_name eq "THAI CHARACTER O ANG") {
if ($prev_script ne $current_script) {
$this->add_node("", $i, $i+1, *chart_ht, "", "thai-initial-o-ang-drop");
} elsif ($next_script ne $current_script) {
$this->add_node("", $i, $i+1, *chart_ht, "", "thai-final-o-ang-drop");
} else {
my $romanized_next_char = $this->romanize_char_at_position($i+1, $lang_code, $output_style, *ht, *chart_ht);
my $romanized_prev2_char = $this->romanize_char_at_position($i-2, $lang_code, $output_style, *ht, *chart_ht);
if (($prev_char_roman =~ /^[bcdfghjklmnpqrstvwxz]+$/i)
&& ($romanized_next_char =~ /^[bcdfghjklmnpqrstvwxz]+$/i)) {
$this->add_node("o", $i, $i+1, *chart_ht, "", "thai-middle-o-ang"); # keep between consonants
} elsif (($prev2_script eq $current_script)
&& 0
&& ($prev_char_name =~ /^THAI CHARACTER MAI [A-Z]+$/) # Thai tone
&& ($romanized_prev2_char =~ /^[bcdfghjklmnpqrstvwxz]+$/i)
&& ($romanized_next_char =~ /^[bcdfghjklmnpqrstvwxz]+$/i)) {
$this->add_node("o", $i, $i+1, *chart_ht, "", "thai-middle-o-ang"); # keep between consonant+tone-mark and consonant
} else {
$this->add_node("", $i, $i+1, *chart_ht, "", "thai-middle-o-ang-drop"); # drop next to vowel
}
}
# Romanization with space
} elsif ($romanized_char =~ /\s/) {
$this->add_node($char, $i, $i+1, *chart_ht, "", "space");
# Tibetan special cases
} elsif ($current_script eq "Tibetan") {
if ($subjoined_char_p
&& ($prev_script eq $current_script)
&& $prev_letter_plus_char_p
&& ($prev_char_roman =~ /^[bcdfghjklmnpqrstvwxyz]+$/i)) {
$this->add_node("$prev_char_roman$romanized_char", $prev_node_start, $i+1, *chart_ht, "", "subjoined-tibetan-character");
} elsif ($romanized_char =~ /^-A$/i) {
my $romanized_next_char = $this->romanize_char_at_position($i+1, $lang_code, $output_style, *ht, *chart_ht);
if (! $prev_letter_plus_char_p) {
$this->add_node("'", $i, $i+1, *chart_ht, "", "tibetan-frontal-dash-a");
} elsif (($prev_script eq $current_script)
&& ($next_script eq $current_script)
&& ($prev_char_roman =~ /[bcdfghjklmnpqrstvwxyz]$/)
&& ($romanized_next_char =~ /^[aeiou]/)) {
$this->add_node("a'", $i, $i+1, *chart_ht, "", "tibetan-medial-dash-a");
} elsif (($prev_script eq $current_script)
&& ($next_script eq $current_script)
&& ($prev_char_roman =~ /[aeiou]$/)
&& ($romanized_next_char =~ /[aeiou]/)) {
$this->add_node("'", $i, $i+1, *chart_ht, "", "tibetan-reduced-medial-dash-a");
} elsif (($prev_script eq $current_script)
&& (! ($prev_char_roman =~ /[aeiou]/))
&& (! $next_letter_plus_char_p)) {
$this->add_node("a", $i, $i+1, *chart_ht, "", "tibetan-final-dash-a");
} else {
$this->add_node("a", $i, $i+1, *chart_ht, "", "unexpected-tibetan-dash-a");
}
} elsif (($romanized_char =~ /^[AEIOU]/i)
&& ($prev_script eq $current_script)
&& ($prev_char_roman =~ /^A$/i)
&& (! $prev2_letter_plus_char_p)) {
$this->add_node($romanized_char, $prev_node_start, $i+1, *chart_ht, "", "tibetan-dropped-word-initial-a");
} else {
$this->add_node($romanized_char, $i, $i+1, *chart_ht, "", "standard-unicode-based-romanization");
}
# Khmer (for MUUSIKATOAN etc. see under "Diacritic" above)
} elsif (($current_script eq "Khmer")
&& (($char_roman_consonant, $char_roman_vowel) = ($romanized_char =~ /^(.*[bcdfghjklmnpqrstvwxyz])([ao]+)-$/i))) {
my $romanized_next_char = $this->romanize_char_at_position($i+1, $lang_code, $output_style, *ht, *chart_ht);
if (($next_script eq $current_script)
&& ($romanized_next_char =~ /^[aeiouy]/i)) {
$this->add_node($char_roman_consonant, $i, $i+1, *chart_ht, "", "khmer-vowel-drop");
} else {
$this->add_node("$char_roman_consonant$char_roman_vowel", $i, $i+1, *chart_ht, "", "khmer-standard-unicode-based-romanization");
}
# Abudiga add default vowel
} elsif ((@abudiga_default_vowels = sort keys %{$ht{SCRIPT_ABUDIGA_DEFAULT_VOWEL}->{$current_script}})
&& ($abudiga_default_vowel = $abudiga_default_vowels[0])
&& ($romanized_char =~ /^[bcdfghjklmnpqrstvwxyz]+$/i)) {
my $new_roman = join("", $romanized_char, $abudiga_default_vowel);
$this->add_node($new_roman, $i, $i+1, *chart_ht, "", "standard-unicode-based-romanization-plus-abudiga-default-vowel");
# print STDERR " Abudiga add default vowel l.$line_number/$i: $new_roman\n" if $line_number =~ /^[12]$/;
# Standard romanization
} else {
$node_id = $this->add_node($romanized_char, $i, $i+1, *chart_ht, "", "standard-unicode-based-romanization");
}
} else {
$this->add_node($char, $i, $i+1, *chart_ht, "", "unexpected-original");
}
} elsif (defined($romanized_char = $this->romanize_char_at_position($i, $lang_code, $output_style, *ht, *chart_ht))
&& ((length($romanized_char) <= 2)
|| ($ht{UTF_TO_CHAR_ROMANIZATION}->{$char}))) { # or from unicode_overwrite_romanization table
$romanized_char =~ s/^""$//;
$this->add_node($romanized_char, $i, $i+1, *chart_ht, "", "romanized-without-character-name");
} else {
$this->add_node($char, $i, $i+1, *chart_ht, "", "unexpected-original-without-character-name");
}
}
$i = $next_index;
}
$this->schwa_deletion(0, $n_characters, *chart_ht, $lang_code);
$this->default_vowelize_tibetan(0, $n_characters, *chart_ht, $lang_code, $line_number) if $chart_ht{CHART_CONTAINS_SCRIPT}->{"Tibetan"};
$this->assemble_numbers_in_chart(*chart_ht, $line_number);
if ($return_chart_p) {
} elsif ($return_offset_mappings_p) {
($result, $offset_mappings, $new_char_offset, $new_rom_char_offset) = $this->best_romanized_string(0, $n_characters, *chart_ht, $control, $initial_char_offset, $initial_rom_char_offset);
} else {
$result = $this->best_romanized_string(0, $n_characters, *chart_ht) unless $return_chart_p;
}
if ($verbosePM) {
my $logfile = "/nfs/isd/ulf/cgi-mt/amr-tmp/uroman-log.txt";
$util->append_to_file($logfile, $log) if $log && (-r $logfile);
}
return ($result, $offset_mappings) if $return_offset_mappings_p;
return *chart_ht if $return_chart_p;
return $result;
}
sub string_to_json_string {
local($this, $s) = @_;
utf8::decode($s);
my $j = JSON->new->utf8->encode([$s]);
$j =~ s/^\[(.*)\]$/$1/;
return $j;
}
sub chart_to_json_romanization_elements {
local($this, $chart_start, $chart_end, *chart_ht, $line_number) = @_;
my $result = "";
my $start = $chart_start;
my $end;
while ($start < $chart_end) {
$end = $this->find_end_of_rom_segment($start, $chart_end, *chart_ht);
my @best_romanizations;
if (($end && ($start < $end))
&& (@best_romanizations = $this->best_romanizations($start, $end, *chart_ht))) {
$orig_segment = $this->orig_string_at_span($start, $end, *chart_ht);
$next_start = $end;
} else {
$orig_segment = $chart_ht{ORIG_CHAR}->{$start};
@best_romanizations = ($orig);
$next_start = $start + 1;
}
$exclusive_end = $end - 1;
# $guarded_orig = $util->string_guard($orig_segment);
$guarded_orig = $this->string_to_json_string($orig_segment);
$result .= " { \"line\": $line_number, \"start\": $start, \"end\": $exclusive_end, \"orig\": $guarded_orig, \"roms\": [";
foreach $i ((0 .. $#best_romanizations)) {
my $rom = $best_romanizations[$i];
# my $guarded_rom = $util->string_guard($rom);
my $guarded_rom = $this->string_to_json_string($rom);
$result .= " { \"rom\": $guarded_rom";
# $result .= ", \"alt\": true" if $i >= 1;
$result .= " }";
$result .= "," if $i < $#best_romanizations;
}
$result .= " ] },\n";
$start = $next_start;
}
return $result;
}
sub default_vowelize_tibetan {
local($this, $chart_start, $chart_end, *chart_ht, $lang_code, $line_number) = @_;
# my $verbose = ($line_number == 103);
# print STDERR "\nStart default_vowelize_tibetan l.$line_number $chart_start-$chart_end\n" if $verbose;
my $token_start = $chart_start;
my $next_token_start = $chart_start;
while (($token_start = $next_token_start) < $chart_end) {
$next_token_start = $token_start + 1;
next unless $chart_ht{CHAR_LETTER_PLUS}->{$token_start};
my $current_script = $chart_ht{CHAR_SCRIPT}->{$token_start};
next unless ($current_script eq "Tibetan");
my $token_end = $chart_ht{LETTER_TOKEN_SEGMENT_START_TO_END}->{$token_start};
next unless $token_end;
next unless $token_end > $token_start;
$next_token_start = $token_end;
my $start = $token_start;
my $end;
my @node_ids = ();
while ($start < $token_end) {
$end = $this->find_end_of_rom_segment($start, $chart_end, *chart_ht);
last unless $end && ($end > $start);
my @alt_node_ids = sort { $a <=> $b } keys %{$chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end}};
last unless @alt_node_ids;
push(@node_ids, $alt_node_ids[0]);
$start = $end;
}
my $contains_vowel_p = 0;
my @romanizations = ();
foreach $node_id (@node_ids) {
my $roman = $chart_ht{NODE_ROMAN}->{$node_id};
$roman = "" unless defined($roman);
push(@romanizations, $roman);
$contains_vowel_p = 1 if $roman =~ /[aeiou]/i;
}
# print STDERR " old: $token_start-$token_end @romanizations\n" if $verbose;
unless ($contains_vowel_p) {
my $default_vowel_target_index;
if ($#node_ids <= 1) {
$default_vowel_target_index = 0;
} elsif ($romanizations[$#romanizations] eq "s") {
if ($romanizations[($#romanizations-1)] eq "y") {
$default_vowel_target_index = $#romanizations-1;
} else {
$default_vowel_target_index = $#romanizations-2;
}
} else {
$default_vowel_target_index = $#romanizations-1;
}
$romanizations[$default_vowel_target_index] .= "a";
my $old_node_id = $node_ids[$default_vowel_target_index];
my $old_start = $chart_ht{NODE_START}->{$old_node_id};
my $old_end = $chart_ht{NODE_END}->{$old_node_id};
my $old_roman = $chart_ht{NODE_ROMAN}->{$old_node_id};
my $new_roman = $old_roman . "a";
my $new_node_id = $this->add_node($new_roman, $old_start, $old_end, *chart_ht, "", "tibetan-default-vowel");
$this->copy_slot_values($old_node_id, $new_node_id, *chart_id, "all");
$chart_ht{NODE_TYPE}->{$old_node_id} = "backup"; # keep, but demote
}
if (($romanizations[0] eq "'")
&& ($#romanizations >= 1)
&& ($romanizations[1] =~ /^[o]$/)) {
my $old_node_id = $node_ids[0];
my $old_start = $chart_ht{NODE_START}->{$old_node_id};
my $old_end = $chart_ht{NODE_END}->{$old_node_id};
my $new_node_id = $this->add_node("", $old_start, $old_end, *chart_ht, "", "tibetan-delete-apostrophe");
$this->copy_slot_values($old_node_id, $new_node_id, *chart_id, "all");
$chart_ht{NODE_TYPE}->{$old_node_id} = "alt"; # keep, but demote
}
if (($#node_ids >= 1)
&& ($romanizations[$#romanizations] =~ /^[bcdfghjklmnpqrstvwxz]+y$/)) {
my $old_node_id = $node_ids[$#romanizations];
my $old_start = $chart_ht{NODE_START}->{$old_node_id};
my $old_end = $chart_ht{NODE_END}->{$old_node_id};
my $old_roman = $chart_ht{NODE_ROMAN}->{$old_node_id};
my $new_roman = $old_roman . "a";
my $new_node_id = $this->add_node($new_roman, $old_start, $old_end, *chart_ht, "", "tibetan-syllable-final-vowel");
$this->copy_slot_values($old_node_id, $new_node_id, *chart_id, "all");
$chart_ht{NODE_TYPE}->{$old_node_id} = "alt"; # keep, but demote
}
foreach $old_node_id (@node_ids) {
my $old_roman = $chart_ht{NODE_ROMAN}->{$old_node_id};
next unless $old_roman =~ /-a/;
my $old_start = $chart_ht{NODE_START}->{$old_node_id};
my $old_end = $chart_ht{NODE_END}->{$old_node_id};
my $new_roman = $old_roman;
$new_roman =~ s/-a/a/;
my $new_node_id = $this->add_node($new_roman, $old_start, $old_end, *chart_ht, "", "tibetan-syllable-delete-dash");
$this->copy_slot_values($old_node_id, $new_node_id, *chart_id, "all");
$chart_ht{NODE_TYPE}->{$old_node_id} = "alt"; # keep, but demote
}
}
}
sub schwa_deletion {
local($this, $chart_start, $chart_end, *chart_ht, $lang_code) = @_;
# delete word-final simple "a" in Devanagari (e.g. nepaala -> nepaal)
# see Wikipedia article "Schwa deletion in Indo-Aryan languages"
if ($chart_ht{CHART_CONTAINS_SCRIPT}->{"Devanagari"}) {
my $script_start = $chart_start;
my $next_script_start = $chart_start;
while (($script_start = $next_script_start) < $chart_end) {
$next_script_start = $script_start + 1;
my $current_script = $chart_ht{CHAR_SCRIPT}->{$script_start};
next unless ($current_script eq "Devanagari");
my $script_end = $chart_ht{SCRIPT_SEGMENT_START_TO_END}->{$script_start};
next unless $script_end;
next unless $script_end - $script_start >= 2;
$next_script_start = $script_end;
my $end_node_id = $this->get_node_for_span($script_end-1, $script_end, *chart_ht);
next unless $end_node_id;
my $end_roman = $chart_ht{NODE_ROMAN}->{$end_node_id};
next unless ($end_consonant) = ($end_roman =~ /^([bcdfghjklmnpqrstvwxz]+)a$/i);
my $prev_node_id = $this->get_node_for_span($script_end-4, $script_end-1, *chart_ht)
|| $this->get_node_for_span($script_end-3, $script_end-1, *chart_ht)
|| $this->get_node_for_span($script_end-2, $script_end-1, *chart_ht);
next unless $prev_node_id;
my $prev_roman = $chart_ht{NODE_ROMAN}->{$prev_node_id};
next unless $prev_roman =~ /[aeiou]/i;
# TO DO: check further back for vowel (e.g. if $prev_roman eq "r" due to vowel cancelation)
$chart_ht{NODE_TYPE}->{$end_node_id} = "alt"; # keep, but demote
# print STDERR "* Schwa deletion " . ($script_end-1) . "-$script_end $end_roman->$end_consonant\n";
$this->add_node($end_consonant, $script_end-1, $script_end, *chart_ht, "", "devanagari-with-deleted-final-schwa");
}
}
}
sub best_romanized_string {
local($this, $chart_start, $chart_end, *chart_ht, $control, $orig_char_offset, $rom_char_offset) = @_;
$control = "" unless defined($control);
my $current_orig_char_offset = $orig_char_offset || 0;
my $current_rom_char_offset = $rom_char_offset || 0;
my $return_offset_mappings_p = ($control =~ /\breturn offset mappings\b/);
my $result = "";
my $start = $chart_start;
my $end;
my @char_offsets = ("$current_orig_char_offset:$current_rom_char_offset");
while ($start < $chart_end) {
$end = $this->find_end_of_rom_segment($start, $chart_end, *chart_ht);
my $n_orig_chars_in_segment = 0;
my $n_rom_chars_in_segment = 0;
if ($end && ($start < $end)) {
my @best_romanizations = $this->best_romanizations($start, $end, *chart_ht);
my $best_romanization = (@best_romanizations) ? $best_romanizations[0] : undef;
if (defined($best_romanization)) {
$result .= $best_romanization;
if ($return_offset_mappings_p) {
$n_orig_chars_in_segment = $end-$start;
$n_rom_chars_in_segment = $utf8->length_in_utf8_chars($best_romanization);
}
$start = $end;
} else {
my $best_romanization = $chart_ht{ORIG_CHAR}->{$start};
$result .= $best_romanization;
$start++;
if ($return_offset_mappings_p) {
$n_orig_chars_in_segment = 1;
$n_rom_chars_in_segment = $utf8->length_in_utf8_chars($best_romanization);
}
}
} else {
my $best_romanization = $chart_ht{ORIG_CHAR}->{$start};
$result .= $best_romanization;
$start++;
if ($return_offset_mappings_p) {
$n_orig_chars_in_segment = 1;
$n_rom_chars_in_segment = $utf8->length_in_utf8_chars($best_romanization);
}
}
if ($return_offset_mappings_p) {
my $new_orig_char_offset = $current_orig_char_offset + $n_orig_chars_in_segment;
my $new_rom_char_offset = $current_rom_char_offset + $n_rom_chars_in_segment;
my $offset_mapping = "$new_orig_char_offset:$new_rom_char_offset";
push(@char_offsets, $offset_mapping);
$current_orig_char_offset = $new_orig_char_offset;
$current_rom_char_offset = $new_rom_char_offset;
}
}
return ($result, join(",", @char_offsets), $current_orig_char_offset, $current_rom_char_offset) if $return_offset_mappings_p;
return $result;
}
sub orig_string_at_span {
local($this, $start, $end, *chart_ht) = @_;
my $result = "";
foreach $i (($start .. ($end-1))) {
$result .= $chart_ht{ORIG_CHAR}->{$i};
}
return $result;
}
sub find_end_of_rom_segment {
local($this, $start, $chart_end, *chart_ht) = @_;
my @ends = sort { $a <=> $b } keys %{$chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}};
my $end_index = $#ends;
while (($end_index >= 0) && ($ends[$end_index] > $chart_end)) {
$end_index--;
}
if (($end_index >= 0)
&& defined($end = $ends[$end_index])
&& ($start < $end)) {
return $end;
} else {
return "";
}
}
sub best_romanizations {
local($this, $start, $end, *chart_ht) = @_;
@regular_romanizations = ();
@alt_romanizations = ();
@backup_romanizations = ();
foreach $node_id (sort { $a <=> $b } keys %{$chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end}}) {
my $type = $chart_ht{NODE_TYPE}->{$node_id};
my $roman = $chart_ht{NODE_ROMAN}->{$node_id};
if (! defined($roman)) {
# ignore
} elsif (($type eq "backup") && ! defined($backup_romanization)) {
push(@backup_romanizations, $roman) unless $util->member($roman, @backup_romanizations);
} elsif (($type eq "alt") && ! defined($alt_romanization)) {
push(@alt_romanizations, $roman) unless $util->member($roman, @alt_romanizations);
} else {
push(@regular_romanizations, $roman) unless $util->member($roman, @regular_romanizations);
}
}
@regular_alt_romanizations = sort @regular_romanizations;
foreach $alt_romanization (sort @alt_romanizations) {
push(@regular_alt_romanizations, $alt_romanization) unless $util->member($alt_romanization, @regular_alt_romanizations);
}
return @regular_alt_romanizations if @regular_alt_romanizations;
return sort @backup_romanizations;
}
sub join_alt_romanizations_for_viz {
local($this, @list) = @_;
my @viz_romanizations = ();
foreach $alt_rom (@list) {
if ($alt_rom eq "") {
push(@viz_romanizations, "-");
} else {
push(@viz_romanizations, $alt_rom);
}
}
return join(", ", @viz_romanizations);
}
sub markup_orig_rom_strings {
local($this, $chart_start, $chart_end, *ht, *chart_ht, *pinyin_ht, $last_group_id_index) = @_;
my $marked_up_rom = "";
my $marked_up_orig = "";
my $start = $chart_start;
my $end;
while ($start < $chart_end) {
my $segment_start = $start;
my $segment_end = $start+1;
my $end = $this->find_end_of_rom_segment($start, $chart_end, *chart_ht);
my $rom_segment = "";
my $orig_segment = "";
my $rom_title = "";
my $orig_title = "";
my $contains_alt_romanizations = 0;
if ($end) {
$segment_end = $end;
my @best_romanizations = $this->best_romanizations($start, $end, *chart_ht);
my $best_romanization = (@best_romanizations) ? $best_romanizations[0] : undef;
if (defined($best_romanization)) {
$rom_segment .= $best_romanization;
$orig_segment .= $this->orig_string_at_span($start, $end, *chart_ht);
$segment_end = $end;
if ($#best_romanizations >= 1) {
$rom_title .= $util->guard_html("Alternative romanizations: " . $this->join_alt_romanizations_for_viz(@best_romanizations) . "\n");
$contains_alt_romanizations = 1;
}
} else {
my $segment = $this->orig_string_at_span($start, $start+1, *chart_ht);
$rom_segment .= $segment;
$orig_segment .= $segment;
$segment_end = $start+1;
}
$start = $segment_end;
} else {
$rom_segment .= $chart_ht{ORIG_CHAR}->{$start};
$orig_segment .= $this->orig_string_at_span($start, $start+1, *chart_ht);
$segment_end = $start+1;
$start = $segment_end;
}
my $next_char = $chart_ht{ORIG_CHAR}->{$segment_end};
my $next_char_is_combining_p = $this->char_is_combining_char($next_char, *ht);
while ($next_char_is_combining_p
&& ($segment_end < $chart_end)
&& ($end = $this->find_end_of_rom_segment($segment_end, $chart_end, *chart_ht))
&& ($end > $segment_end)
&& (@best_romanizations = $this->best_romanizations($segment_end, $end, *chart_ht))
&& defined($best_romanization = $best_romanizations[0])) {
$orig_segment .= $this->orig_string_at_span($segment_end, $end, *chart_ht);
$rom_segment .= $best_romanization;
if ($#best_romanizations >= 1) {
$rom_title .= $util->guard_html("Alternative romanizations: " . $this->join_alt_romanizations_for_viz(@best_romanizations) . "\n");
$contains_alt_romanizations = 1;
}
$segment_end = $end;
$start = $segment_end;
$next_char = $chart_ht{ORIG_CHAR}->{$segment_end};
$next_char_is_combining_p = $this->char_is_combining_char($next_char, *ht);
}
foreach $i (($segment_start .. ($segment_end-1))) {
$orig_title .= "+‎ ‎" unless $orig_title eq "";
my $char = $chart_ht{ORIG_CHAR}->{$i};
my $numeric = $ht{UTF_TO_NUMERIC}->{$char};
$numeric = "" unless defined($numeric);
my $pic_descr = $ht{UTF_TO_PICTURE_DESCR}->{$char};
$pic_descr = "" unless defined($pic_descr);
if ($char =~ /^\xE4\xB7[\x80-\xBF]$/) {
$orig_title .= "$char_name\n";
} elsif (($char =~ /^[\xE3-\xE9][\x80-\xBF]{2,2}$/) && $chinesePM->string_contains_utf8_cjk_unified_ideograph_p($char)) {
my $unicode = $utf8->utf8_to_unicode($char);
$orig_title .= "CJK Unified Ideograph U+" . (uc sprintf("%04x", $unicode)) . "\n";
$orig_title .= "Chinese: $tonal_translit\n" if $tonal_translit = $chinesePM->tonal_pinyin($char, *pinyin_ht, "");
$orig_title .= "Number: $numeric\n" if $numeric =~ /\d/;
} elsif ($char_name = $ht{UTF_TO_CHAR_NAME}->{$char}) {
$orig_title .= "$char_name\n";
$orig_title .= "Number: $numeric\n" if $numeric =~ /\d/;
$orig_title .= "Picture: $pic_descr\n" if $pic_descr =~ /\S/;
} else {
my $unicode = $utf8->utf8_to_unicode($char);
if (($unicode >= 0xAC00) && ($unicode <= 0xD7A3)) {
$orig_title .= "Hangul syllable U+" . (uc sprintf("%04x", $unicode)) . "\n";
} else {
$orig_title .= "Unicode character U+" . (uc sprintf("%04x", $unicode)) . "\n";
}
}
}
(@non_ascii_roms) = ($rom_segment =~ /([\xC0-\xFF][\x80-\xBF]*)/g);
foreach $char (@non_ascii_roms) {
my $char_name = $ht{UTF_TO_CHAR_NAME}->{$char};
my $unicode = $utf8->utf8_to_unicode($char);
my $unicode_s = "U+" . (uc sprintf("%04x", $unicode));
if ($char_name) {
$rom_title .= "$char_name\n";
} else {
$rom_title .= "$unicode_s\n";
}
}
$last_group_id_index++;
$rom_title =~ s/\s*$//;
$rom_title =~ s/\n/
/g;
$orig_title =~ s/\s*$//;
$orig_title =~ s/\n/
‎/g;
$orig_title = "‭" . $orig_title . "‬";
my $rom_title_clause = ($rom_title eq "") ? "" : " title=\"$rom_title\"";
my $orig_title_clause = ($orig_title eq "") ? "" : " title=\"$orig_title\"";
my $alt_rom_clause = ($contains_alt_romanizations) ? "border-bottom:1px dotted;" : "";
$marked_up_rom .= "<span id=\"span-$last_group_id_index-1\" onmouseover=\"highlight_elems('span-$last_group_id_index','1');\" onmouseout=\"highlight_elems('span-$last_group_id_index','0');\" style=\"color:#00BB00;$alt_rom_clause\"$rom_title_clause>" . $util->guard_html($rom_segment) . "<\/span>";
$marked_up_orig .= "<span id=\"span-$last_group_id_index-2\" onmouseover=\"highlight_elems('span-$last_group_id_index','1');\" onmouseout=\"highlight_elems('span-$last_group_id_index','0');\"$orig_title_clause>" . $util->guard_html($orig_segment) . "<\/span>";
if (($last_char = $chart_ht{ORIG_CHAR}->{($segment_end-1)})
&& ($last_char_name = $ht{UTF_TO_CHAR_NAME}->{$last_char})
&& ($last_char_name =~ /^(FULLWIDTH COLON|FULLWIDTH COMMA|FULLWIDTH RIGHT PARENTHESIS|IDEOGRAPHIC COMMA|IDEOGRAPHIC FULL STOP|RIGHT CORNER BRACKET|BRAILLE PATTERN BLANK|TIBETAN MARK .*)$/)) {
$marked_up_orig .= "<wbr>";
$marked_up_rom .= "<wbr>";
}
}
return ($marked_up_rom, $marked_up_orig, $last_group_id_index);
}
sub romanizations_with_alternatives {
local($this, *ht, *chart_ht, *pinyin_ht, $chart_start, $chart_end) = @_;
$chart_start = 0 unless defined($chart_start);
$chart_end = $chart_ht{N_CHARS} unless defined($chart_end);
my $result = "";
my $start = $chart_start;
my $end;
# print STDOUT "romanizations_with_alternatives $chart_start-$chart_end\n";
while ($start < $chart_end) {
my $segment_start = $start;
my $segment_end = $start+1;
my $end = $this->find_end_of_rom_segment($start, $chart_end, *chart_ht);
my $rom_segment = "";
# print STDOUT " $start-$end\n";
if ($end) {
$segment_end = $end;
my @best_romanizations = $this->best_romanizations($start, $end, *chart_ht);
# print STDOUT " $start-$end @best_romanizations\n";
if (@best_romanizations) {
if ($#best_romanizations == 0) {
$rom_segment .= $best_romanizations[0];
} else {
$rom_segment .= "{" . join("|", @best_romanizations) . "}";
}
$segment_end = $end;
} else {
my $segment = $this->orig_string_at_span($start, $start+1, *chart_ht);
$rom_segment .= $segment;
$segment_end = $start+1;
}
$start = $segment_end;
} else {
$rom_segment .= $chart_ht{ORIG_CHAR}->{$start};
$segment_end = $start+1;
$start = $segment_end;
}
# print STDOUT " $start-$end ** $rom_segment\n";
$result .= $rom_segment;
}
return $result;
}
sub quick_romanize {
local($this, $s, $lang_code, *ht) = @_;
my $result = "";
my @chars = $utf8->split_into_utf8_characters($s, "return only chars", *empty_ht);
while (@chars) {
my $found_match_in_table_p = 0;
foreach $string_length (reverse(1..4)) {
next if ($string_length-1) > $#chars;
$multi_char_substring = join("", @chars[0..($string_length-1)]);
my @mappings = keys %{$ht{UTF_CHAR_MAPPING_LANG_SPEC}->{$lang_code}->{$multi_char_substring}};
@mappings = keys %{$ht{UTF_CHAR_MAPPING}->{$multi_char_substring}} unless @mappings;
if (@mappings) {
my $mapping = $mappings[0];
$result .= $mapping;
foreach $_ ((1 .. $string_length)) {
shift @chars;
}
$found_match_in_table_p = 1;
last;
}
}
unless ($found_match_in_table_p) {
$result .= $chars[0];
shift @chars;
}
}
return $result;
}
sub char_is_combining_char {
local($this, $c, *ht) = @_;
return 0 unless $c;
my $category = $ht{UTF_TO_CAT}->{$c};
return 0 unless $category;
return $category =~ /^M/;
}
sub mark_up_string_for_mouse_over {
local($this, $s, *ht, $control, *pinyin_ht) = @_;
$control = "" unless defined($control);
$no_ascii_p = ($control =~ /NO-ASCII/);
my $result = "";
@chars = $utf8->split_into_utf8_characters($s, "return only chars", *empty_ht);
while (@chars) {
$char = shift @chars;
$numeric = $ht{UTF_TO_NUMERIC}->{$char};
$numeric = "" unless defined($numeric);
$pic_descr = $ht{UTF_TO_PICTURE_DESCR}->{$char};
$pic_descr = "" unless defined($pic_descr);
$next_char = ($#chars >= 0) ? $chars[0] : "";
$next_char_is_combining_p = $this->char_is_combining_char($next_char, *ht);
if ($no_ascii_p
&& ($char =~ /^[\x00-\x7F]*$/)
&& ! $next_char_is_combining_p) {
$result .= $util->guard_html($char);
} elsif (($char =~ /^[\xE3-\xE9][\x80-\xBF]{2,2}$/) && $chinesePM->string_contains_utf8_cjk_unified_ideograph_p($char)) {
$unicode = $utf8->utf8_to_unicode($char);
$title = "CJK Unified Ideograph U+" . (uc sprintf("%04x", $unicode));
$title .= "
Chinese: $tonal_translit" if $tonal_translit = $chinesePM->tonal_pinyin($char, *pinyin_ht, "");
$title .= "
Number: $numeric" if $numeric =~ /\d/;
$result .= "<span title=\"$title\">" . $util->guard_html($char) . "<\/span>";
} elsif ($char_name = $ht{UTF_TO_CHAR_NAME}->{$char}) {
$title = $char_name;
$title .= "
Number: $numeric" if $numeric =~ /\d/;
$title .= "
Picture: $pic_descr" if $pic_descr =~ /\S/;
$char_plus = $char;
while ($next_char_is_combining_p) {
# combining marks (Mc:non-spacing, Mc:spacing combining, Me: enclosing)
$next_char_name = $ht{UTF_TO_CHAR_NAME}->{$next_char};
$title .= "
+ $next_char_name";
$char = shift @chars;
$char_plus .= $char;
$next_char = ($#chars >= 0) ? $chars[0] : "";
$next_char_is_combining_p = $this->char_is_combining_char($next_char, *ht);
}
$result .= "<span title=\"$title\">" . $util->guard_html($char_plus) . "<\/span>";
$result .= "<wbr>" if $char_name =~ /^(FULLWIDTH COLON|FULLWIDTH COMMA|FULLWIDTH RIGHT PARENTHESIS|IDEOGRAPHIC COMMA|IDEOGRAPHIC FULL STOP|RIGHT CORNER BRACKET)$/;
} elsif (($unicode = $utf8->utf8_to_unicode($char))
&& ($unicode >= 0xAC00) && ($unicode <= 0xD7A3)) {
$title = "Hangul syllable U+" . (uc sprintf("%04x", $unicode));
$result .= "<span title=\"$title\">" . $util->guard_html($char) . "<\/span>";
} else {
$result .= $util->guard_html($char);
}
}
return $result;
}
sub romanize_char_at_position_incl_multi {
local($this, $i, $lang_code, $output_style, *ht, *chart_ht) = @_;
my $char = $chart_ht{ORIG_CHAR}->{$i};
return "" unless defined($char);
my @mappings = keys %{$ht{UTF_CHAR_MAPPING_LANG_SPEC}->{$lang_code}->{$char}};
return $mappings[0] if @mappings;
@mappings = keys %{$ht{UTF_CHAR_MAPPING}->{$char}};
return $mappings[0] if @mappings;
return $this->romanize_char_at_position($i, $lang_code, $output_style, *ht, *chart_ht);
}
sub romanize_char_at_position {
local($this, $i, $lang_code, $output_style, *ht, *chart_ht) = @_;
my $char = $chart_ht{ORIG_CHAR}->{$i};
return "" unless defined($char);
return $char if $char =~ /^[\x00-\x7F]$/; # ASCII
my $romanization = $ht{UTF_TO_CHAR_ROMANIZATION}->{$char};
return $romanization if $romanization;
my $char_name = $chart_ht{CHAR_NAME}->{$i};
$romanization = $this->romanize_charname($char_name, $lang_code, $output_style, *ht, $char);
$ht{SUSPICIOUS_ROMANIZATION}->{$char_name}->{$romanization}
= ($ht{SUSPICIOUS_ROMANIZATION}->{$char_name}->{$romanization} || 0) + 1
unless (length($romanization) < 4)
|| ($romanization =~ /\s/)
|| ($romanization =~ /^[bcdfghjklmnpqrstvwxyz]{2,3}[aeiou]-$/) # Khmer ngo-/nyo-/pho- OK
|| ($romanization =~ /^[bcdfghjklmnpqrstvwxyz]{2,2}[aeiougw][aeiou]{1,2}$/) # Canadian, Ethiopic syllable OK
|| ($romanization =~ /^(allah|bbux|nyaa|nnya|quuv|rrep|shch|shur|syrx)$/i) # Arabic; Yi; Ethiopic syllable nyaa; Cyrillic letter shcha
|| (($char_name =~ /^(YI SYLLABLE|VAI SYLLABLE|ETHIOPIC SYLLABLE|CANADIAN SYLLABICS|CANADIAN SYLLABICS CARRIER)\s+(\S+)$/) && (length($romanization) <= 5));
# print STDERR "romanize_char_at_position $i $char_name :: $romanization\n" if $char_name =~ /middle/i;
return $romanization;
}
sub romanize_charname {
local($this, $char_name, $lang_code, $output_style, *ht, $char) = @_;
my $cached_result = $ht{ROMANIZE_CHARNAME}->{$char_name}->{$lang_code}->{$output_style};
# print STDERR "(C) romanize_charname($char_name): $cached_result\n" if $cached_result && ($char_name =~ /middle/i);
return $cached_result if defined($cashed_result);
$orig_char_name = $char_name;
$char_name =~ s/^.* LETTER\s+([A-Z]+)-\d+$/$1/; # HENTAIGANA LETTER A-3
$char_name =~ s/^.* LETTER\s+//;
$char_name =~ s/^.* SYLLABLE\s+B\d\d\d\s+//; # Linear B syllables
$char_name =~ s/^.* SYLLABLE\s+//;
$char_name =~ s/^.* SYLLABICS\s+//;
$char_name =~ s/^.* LIGATURE\s+//;
$char_name =~ s/^.* VOWEL SIGN\s+//;
$char_name =~ s/^.* CONSONANT SIGN\s+//;
$char_name =~ s/^.* CONSONANT\s+//;
$char_name =~ s/^.* VOWEL\s+//;
$char_name =~ s/ WITH .*$//;
$char_name =~ s/ WITHOUT .*$//;
$char_name =~ s/\s+(ABOVE|AGUNG|BAR|BARREE|BELOW|CEDILLA|CEREK|DIGRAPH|DOACHASHMEE|FINAL FORM|GHUNNA|GOAL|INITIAL FORM|ISOLATED FORM|KAWI|LELET|LELET RASWADI|LONSUM|MAHAPRANA|MEDIAL FORM|MURDA|MURDA MAHAPRANA|REVERSED|ROTUNDA|SASAK|SUNG|TAM|TEDUNG|TYPE ONE|TYPE TWO|WOLOSO)\s*$//;
$char_name =~ s/^([A-Z]+)\d+$/$1/; # Linear B syllables etc.
foreach $_ ((1 .. 3)) {
$char_name =~ s/^.*\b(?:ABKHASIAN|ACADEMY|AFRICAN|AIVILIK|AITON|AKHMIMIC|ALEUT|ALI GALI|ALPAPRAANA|ALTERNATE|ALTERNATIVE|AMBA|ARABIC|ARCHAIC|ASPIRATED|ATHAPASCAN|BASELINE|BLACKLETTER|BARRED|BASHKIR|BERBER|BHATTIPROLU|BIBLE-CREE|BIG|BINOCULAR|BLACKFOOT|BLENDED|BOTTOM|BROAD|BROKEN|CANDRA|CAPITAL|CARRIER|CHILLU|CLOSE|CLOSED|COPTIC|CROSSED|CRYPTOGRAMMIC|CURLED|CURLY|CYRILLIC|DANTAJA|DENTAL|DIALECT-P|DIAERESIZED|DOTLESS|DOUBLE|DOUBLE-STRUCK|EASTERN PWO KAREN|EGYPTOLOGICAL|FARSI|FINAL|FLATTENED|GLOTTAL|GREAT|GREEK|HALF|HIGH|INITIAL|INSULAR|INVERTED|IOTIFIED|JONA|KANTAJA|KASHMIRI|KHAKASSIAN|KHAMTI|KHANDA|KINNA|KIRGHIZ|KOMI|L-SHAPED|LATINATE|LITTLE|LONG|LONG-LEGGED|LOOPED|LOW|MAHAAPRAANA|MALAYALAM|MANCHU|MANDAILING|MATHEMATICAL|MEDIAL|MIDDLE-WELSH|MON|MONOCULAR|MOOSE-CREE|MULTIOCULAR|MUURDHAJA|N-CREE|NARROW|NASKAPI|NDOLE|NEUTRAL|NIKOLSBURG|NORTHERN|NUBIAN|NUNAVIK|NUNAVUT|OJIBWAY|OLD|OPEN|ORKHON|OVERLONG|PALI|PERSIAN|PHARYNGEAL|PRISHTHAMATRA|R-CREE|REDUPLICATION|REVERSED|ROMANIAN|ROUND|ROUNDED|RUDIMENTA|RUMAI PALAUNG|SANSKRIT|SANYAKA|SARA|SAYISI|SCRIPT|SEBATBEIT|SEMISOFT|SGAW KAREN|SHAN|SHARP|SHWE PALAUNG|SHORT|SIBE|SIDEWAYS|SIMALUNGUN|SMALL|SOGDIAN|SOFT|SOUTH-SLAVEY|SOUTHERN|SPIDERY|STIRRUP|STRAIGHT|STRETCHED|SUBSCRIPT|SWASH|TAI LAING|TAILED|TAILLESS|TAALUJA|TH-CREE|TALL|THREE-LEGGED|TURNED|TODO|TOP|TROKUTASTI|TUAREG|UKRAINIAN|UNBLENDED|VISIGOTHIC|VOCALIC|VOICED|VOICELESS|VOLAPUK|WAVY|WESTERN PWO KAREN|WEST-CREE|WESTERN|WIDE|WOODS-CREE|Y-CREE|YENISEI|YIDDISH)\s+//;
}
$char_name =~ s/\s+(ABOVE|AGUNG|BAR|BARREE|BELOW|CEDILLA|CEREK|DIGRAPH|DOACHASHMEE|FINAL FORM|GHUNNA|GOAL|INITIAL FORM|ISOLATED FORM|KAWI|LELET|LELET RASWADI|LONSUM|MAHAPRANA|MEDIAL FORM|MURDA|MURDA MAHAPRANA|REVERSED|ROTUNDA|SASAK|SUNG|TAM|TEDUNG|TYPE ONE|TYPE TWO|WOLOSO)\s*$//;
if ($char_name =~ /THAI CHARACTER/) {
$char_name =~ s/^THAI CHARACTER\s+//;
if ($char =~ /^\xE0\xB8[\x81-\xAE]/) {
# Thai consonants
$char_name =~ s/^([^AEIOU]*).*/$1/i;
} elsif ($char_name =~ /^SARA [AEIOU]/) {
# Thai vowels
$char_name =~ s/^SARA\s+//;
} else {
$char_name = $char;
}
}
if ($orig_char_name =~ /(HIRAGANA LETTER|KATAKANA LETTER|SYLLABLE|LIGATURE)/) {
$char_name = lc $char_name;
} elsif ($char_name =~ /\b(ANUSVARA|ANUSVARAYA|NIKAHIT|SIGN BINDI|TIPPI)\b/) {
$char_name = "+m";
} elsif ($char_name =~ /\bSCHWA\b/) {
$char_name = "e";
} elsif ($char_name =~ /\bIOTA\b/) {
$char_name = "i";
} elsif ($char_name =~ /\s/) {
} elsif ($orig_char_name =~ /KHMER LETTER/) {
$char_name .= "-";
} elsif ($orig_char_name =~ /CHEROKEE LETTER/) {
# use whole letter as is
} elsif ($orig_char_name =~ /KHMER INDEPENDENT VOWEL/) {
$char_name =~ s/q//;
} elsif ($orig_char_name =~ /LETTER/) {
$char_name =~ s/^[AEIOU]+([^AEIOU]+)$/$1/i;
$char_name =~ s/^([^-AEIOUY]+)[AEIOU].*/$1/i;
$char_name =~ s/^(Y)[AEIOU].*/$1/i if $orig_char_name =~ /\b(?:BENGALI|DEVANAGARI|GURMUKHI|GUJARATI|KANNADA|MALAYALAM|MODI|MYANMAR|ORIYA|TAMIL|TELUGU|TIBETAN)\b.*\bLETTER YA\b/;
$char_name =~ s/^(Y[AEIOU]+)[^AEIOU].*$/$1/i;
$char_name =~ s/^([AEIOU]+)[^AEIOU]+[AEIOU].*/$1/i;
}
my $result = ($orig_char_name =~ /\bCAPITAL\b/) ? (uc $char_name) : (lc $char_name);
# print STDERR "(R) romanize_charname($orig_char_name): $result\n" if $orig_char_name =~ /middle/i;
$ht{ROMANIZE_CHARNAME}->{$char_name}->{$lang_code}->{$output_style} = $result;
return $result;
}
sub assemble_numbers_in_chart {
local($this, *chart_ht, $line_number) = @_;
foreach $start (sort { $a <=> $b } keys %{$chart_ht{COMPLEX_NUMERIC_START_END}}) {
my $end = $chart_ht{COMPLEX_NUMERIC_START_END}->{$start};
my @numbers = ();
foreach $i (($start .. ($end-1))) {
my $orig_char = $chart_ht{ORIG_CHAR}->{$i};
my $node_id = $this->get_node_for_span_with_slot($i, $i+1, "numeric-value", *chart_id);
if (defined($node_id)) {
my $number = $chart_ht{NODE_ROMAN}->{$node_id};
if (defined($number)) {
push(@numbers, $number);
} elsif ($orig_char =~ /^[.,]$/) { # decimal point, comma separator
push(@numbers, $orig_char);
} else {
print STDERR "Found no romanization for node_id $node_id ($i-" . ($i+1) . ") in assemble_numbers_in_chart\n" if $verbosePM;
}
} else {
print STDERR "Found no node_id for span $i-" . ($i+1) . " in assemble_numbers_in_chart\n" if $verbosePM;
}
}
my $complex_number = $this->assemble_number(join("\xC2\xB7", @numbers), $line_number);
# print STDERR "assemble_numbers_in_chart l.$line_number $start-$end $complex_number (@numbers)\n";
$this->add_node($complex_number, $start, $end, *chart_ht, "", "complex-number");
}
}
sub assemble_number {
local($this, $s, $line_number) = @_;
# e.g. 10 9 100 7 10 8 = 1978
my $middot = "\xC2\xB7";
my @tokens = split(/$middot/, $s); # middle dot U+00B7
my $i = 0;
my @orig_tokens = @tokens;
# assemble single digit numbers, e.g. 1 7 5 -> 175
while ($i < $#tokens) {
if ($tokens[$i] =~ /^\d$/) {
my $j = $i+1;
while (($j <= $#tokens) && ($tokens[$j] =~ /^[0-9.,]$/)) {
$j++;
}
$j--;
if ($j>$i) {
my $new_token = join("", @tokens[$i .. $j]);
$new_token =~ s/,//g;
splice(@tokens, $i, $j-$i+1, $new_token);
}
}
$i++;
}
foreach $power ((10, 100, 1000, 10000, 100000, 1000000, 100000000, 1000000000, 1000000000000)) {
for (my $i=0; $i <= $#tokens; $i++) {
if ($tokens[$i] == $power) {
if (($i > 0) && ($tokens[($i-1)] < $power)) {
splice(@tokens, $i-1, 2, ($tokens[($i-1)] * $tokens[$i]));
$i--;
if (($i < $#tokens) && ($tokens[($i+1)] < $power)) {
splice(@tokens, $i, 2, ($tokens[$i] + $tokens[($i+1)]));
$i--;
}
}
}
# 400 30 (e.g. Egyptian)
my $gen_pattern = $power;
$gen_pattern =~ s/^1/\[1-9\]/;
if (($tokens[$i] =~ /^$gen_pattern$/) && ($i < $#tokens) && ($tokens[($i+1)] < $power)) {
splice(@tokens, $i, 2, ($tokens[$i] + $tokens[($i+1)]));
$i--;
}
}
last if $#tokens == 0;
}
my $result = join($middot, @tokens);
if ($verbosePM) {
my $logfile = "/nfs/isd/ulf/cgi-mt/amr-tmp/uroman-number-log.txt";
$util->append_to_file($logfile, "$s -> $result\n") if -r $logfile;
# print STDERR " assemble number l.$line_number @orig_tokens -> $result\n" if $line_number == 43;
}
return $result;
}
1;