original_docstring: Loads detokenization resources such as auto-attach, markup-attach etc.
          Example input file: data/detok-resource.txt
          This file is also loaded by the tokenizer to produce appropriate mt-style @...@ tokens.
docstring: Loads detokenization resources such as auto-attach, markup-attach etc.
  Example input file: data/detok-resource.txt
  This file is also loaded by the tokenizer to produce appropriate mt-style @...@ tokens.
docstring_tokens: Loads detokenization resources such as auto - attach markup - attach etc . Example input file : data / detok - resource . txt This file is also loaded by the tokenizer to produce appropriate mt - style @ ... @ tokens .
code: def load_resource(self, filename: Path, doc_lang_codes: List[str], verbose: bool = True) -> None:
          """Loads detokenization resources such as auto-attach, markup-attach etc.
          Example input file: data/detok-resource.txt
          This file is also loaded by the tokenizer to produce appropriate mt-style @...@ tokens."""
          try:
              with open(filename) as f_in:
                  line_number = 0
                  n_warnings = 0
                  n_entries = 0
                  resource_dict = ResourceDict()
                  for orig_line in f_in:
                      line_number += 1
                      line_without_comment = resource_dict.line_without_comment(orig_line)
                      if line_without_comment.strip() == '':
                          continue
                      lines = resource_dict.expand_resource_lines(line_without_comment)
                      for line in lines:
                          if re.match(r'^\uFEFF?\s*(?:#.*)?$', line):  # ignore empty or comment line
                              continue
                          if re.match(r'::(repair|punct-split|abbrev|misspelling)\b', line):
                              continue  # In tok-resources, only ::contraction entries are relevant.
                          # Check whether cost file line is well-formed. Following call will output specific warnings.
                          valid = double_colon_del_list_validation(line, str(line_number), str(filename),
                                                                   valid_slots=['alt-spelling',
                                                                                'attach-tag',
                                                                                'auto-attach',
                                                                                'char-split',
                                                                                'case-sensitive',
                                                                                'comment',
                                                                                'contraction',
                                                                                'country',
                                                                                'eng',
                                                                                'etym-lcode',
                                                                                'example',
                                                                                'except',
                                                                                'group',
                                                                                'ipa-trigger-left',
                                                                                'last-char-repeatable',
                                                                                'lcode',
                                                                                'lcode-not',
                                                                                'left-context',
                                                                                'left-context-not',
                                                                                'lexical',
                                                                                'markup-attach',
                                                                                'misspelling',
                                                                                'non-symbol',
                                                                                'nonstandard',
                                                                                'paired-delimiter',
                                                                                'plural',
                                                                                'priority',
                                                                                'right-context',
                                                                                'right-context-not',
                                                                                'sem-class',
                                                                                'side',
                                                                                'substandard',
                                                                                'syntax-checked',
                                                                                'tag',
                                                                                'target',
                                                                                'taxon',
                                                                                'token-category'],
                                                                   required_slot_dict={'attach-tag': [],
                                                                                       'auto-attach': ['side'],
                                                                                       'contraction': ['target'],
                                                                                       'ipa-trigger-left': [],
                                                                                       'lexical': [],
                                                                                       'markup-attach': [],
                                                                                       'non-symbol': []})
                          if not valid:
                              n_warnings += 1
                              continue
                          if m1 := re.match(r"::(\S+)", line):
                              head_slot = m1.group(1)
                          else:
                              continue
                          line_lang_code_s = slot_value_in_double_colon_del_list(line, 'lcode')
                          line_lang_codes = re.split(r'[;,\s*]', line_lang_code_s) if line_lang_code_s else []
                          if doc_lang_codes and line_lang_codes:
                              if not lists_share_element(doc_lang_codes, line_lang_codes):
                                  continue
                          s = slot_value_in_double_colon_del_list(line, head_slot)
                          lc_s = s.lower()
                          detokenization_entry = None
                          if head_slot == 'auto-attach':
                              side = slot_value_in_double_colon_del_list(line, 'side')
                              group = bool(slot_value_in_double_colon_del_list(line, 'group', False))
                              detokenization_entry = DetokenizationEntry(s, group, line_lang_codes)
                              if side == 'left' or side == 'both':
                                  for line_lang_code in (line_lang_codes if line_lang_codes else [None]):
                                      key = f'{line_lang_code} {lc_s}'
                                      if self.auto_attach_left_w_lc.get(key, False):
                                          lcode_clause = f' ::lcode {line_lang_code}' if line_lang_code else ''
                                          log.warning(f'Duplicate ::auto-attach {lc_s} ::side left{lcode_clause}')
                                      else:
                                          self.auto_attach_left_w_lc[key] = True
                                  self.auto_attach_left[lc_s].append(detokenization_entry)
                              if side == 'right' or side == 'both':
                                  for line_lang_code in (line_lang_codes if line_lang_codes else [None]):
                                      key = f'{line_lang_code} {lc_s}'
                                      if self.auto_attach_right_w_lc.get(key, False):
                                          lcode_clause = f' ::lcode {line_lang_code}' if line_lang_code else ''
                                          log.warning(f'Duplicate ::auto-attach {lc_s} ::side right{lcode_clause}')
                                      else:
                                          self.auto_attach_right_w_lc[key] = True
                                  self.auto_attach_right[lc_s].append(detokenization_entry)
                          elif head_slot == 'markup-attach':
                              paired_delimiter = bool(slot_value_in_double_colon_del_list(line, 'paired-delimiter',
                                                                                          False))
                              group = bool(slot_value_in_double_colon_del_list(line, 'group', False))
                              detokenization_entry = DetokenizationMarkupEntry(s, group=group,
                                                                               paired_delimiter=paired_delimiter)
                              if except_s := slot_value_in_double_colon_del_list(line, 'except'):
                                  detokenization_entry.exception_list = re.split(r'\s+', except_s)
                              self.markup_attach[lc_s].append(detokenization_entry)
                              self.markup_attach_re_elements.add(re.escape(lc_s) + ('+' if group else ''))
                          elif head_slot == 'attach_tag':
                              self.attach_tag = s
                          elif head_slot == 'contraction':
                              if ((not slot_value_in_double_colon_del_list(line, 'nonstandard'))
                                      and (not slot_value_in_double_colon_del_list(line, 'substandard'))):
                                  target = slot_value_in_double_colon_del_list(line, 'target')
                                  lc_target = target.lower()
                                  detokenization_entry = DetokenizationContractionEntry(target, s)
                                  self.contraction_dict[lc_target].append(detokenization_entry)
                          elif head_slot == 'lexical':
                              tag = slot_value_in_double_colon_del_list(line, 'tag')
                              if tag in ('DECONTRACTION-L', 'DECONTRACTION-R', 'DECONTRACTION-B'):
                                  detokenization_entry = DetokenizationEntry(s)
                                  if tag in ('DECONTRACTION-L', 'DECONTRACTION-B'):  # e.g. s', 'n'
                                      self.auto_attach_right[lc_s].append(detokenization_entry)
                                  if tag in ('DECONTRACTION-R', 'DECONTRACTION-B'):  # e.g. 's, 'n'
                                      self.auto_attach_left[lc_s].append(detokenization_entry)
                          if detokenization_entry:
                              detokenization_entry.case_sensitive = \
                                  bool(slot_value_in_double_colon_del_list(line, 'case-sensitive'))
                              if left_context_s := slot_value_in_double_colon_del_list(line, 'left-context'):
                                  try:
                                      detokenization_entry.left_context = \
                                          regex.compile(eval('r".*' + left_context_s + '$"'), flags=regex.VERSION1)
                                  except regex.error:
                                      log.warning(f'Regex compile error in l.{line_number} for {s} ::left-context '
                                                  f'{left_context_s}')
                              if left_context_not_s := slot_value_in_double_colon_del_list(line, 'left-context-not'):
                                  try:
                                      detokenization_entry.left_context_not = \
                                          regex.compile(eval('r".*(?<!' + left_context_not_s + ')$"'),
                                                        flags=regex.VERSION1)
                                  except regex.error:
                                      log.warning(f'Regex compile error in l.{line_number} for {s} ::left-context-not '
                                                  f'{left_context_not_s}')
                              if right_context_s := slot_value_in_double_colon_del_list(line, 'right-context'):
                                  try:
                                      detokenization_entry.right_context = \
                                          regex.compile(eval('r"' + right_context_s + '"'), flags=regex.VERSION1)
                                  except regex.error:
                                      log.warning(f'Regex compile error in l.{line_number} for {s} ::right-context '
                                                  f'{right_context_s}')
                              if right_context_not_s := slot_value_in_double_colon_del_list(line, 'right-context-not'):
                                  try:
                                      detokenization_entry.right_context_not = \
                                          regex.compile(eval('r"(?!' + right_context_not_s + ')"'),
                                                        flags=regex.VERSION1)
                                  except regex.error:
                                      log.warning(f'Regex compile error in l.{line_number} for {s} ::right-context-not '
                                                  f'{right_context_not_s}')
                              if lang_code_not_s := slot_value_in_double_colon_del_list(line, 'lcode-not'):
                                  detokenization_entry.lang_codes_not = re.split(r'[;,\s*]', lang_code_not_s)
                              n_entries += 1
                  if verbose:
                      log.info(f'Loaded {n_entries} entries from {line_number} lines in {filename}')
          except OSError:
              log.warning(f'Could not open general resource file {filename}')
code_tokens: def load_resource ( self , filename : Path , doc_lang_codes : List [ str ] , verbose : bool = True ) -> None : try : with open ( filename ) as f_in : line_number = 0 n_warnings = 0 n_entries = 0 resource_dict = ResourceDict ( ) for orig_line in f_in : line_number += 1 line_without_comment = resource_dict . line_without_comment ( orig_line ) if line_without_comment . strip ( ) == '' : continue lines = resource_dict . expand_resource_lines ( line_without_comment ) for line in lines : if re . match ( r'^\uFEFF?\s*(?:#.*)?$' , line ) : continue if re . match ( r'::(repair|punct-split|abbrev|misspelling)\b' , line ) : continue valid = double_colon_del_list_validation ( line , str ( line_number ) , str ( filename ) , valid_slots = [ 'alt-spelling' , 'attach-tag' , 'auto-attach' , 'char-split' , 'case-sensitive' , 'comment' , 'contraction' , 'country' , 'eng' , 'etym-lcode' , 'example' , 'except' , 'group' , 'ipa-trigger-left' , 'last-char-repeatable' , 'lcode' , 'lcode-not' , 'left-context' , 'left-context-not' , 'lexical' , 'markup-attach' , 'misspelling' , 'non-symbol' , 'nonstandard' , 'paired-delimiter' , 'plural' , 'priority' , 'right-context' , 'right-context-not' , 'sem-class' , 'side' , 'substandard' , 'syntax-checked' , 'tag' , 'target' , 'taxon' , 'token-category' ] , required_slot_dict = { 'attach-tag' : [ ] , 'auto-attach' : [ 'side' ] , 'contraction' : [ 'target' ] , 'ipa-trigger-left' : [ ] , 'lexical' : [ ] , 'markup-attach' : [ ] , 'non-symbol' : [ ] } ) if not valid : n_warnings += 1 continue if m1 := re . match ( r"::(\S+)" , line ) : head_slot = m1 . group ( 1 ) else : continue line_lang_code_s = slot_value_in_double_colon_del_list ( line , 'lcode' ) line_lang_codes = re . split ( r'[;,\s*]' , line_lang_code_s ) if line_lang_code_s else [ ] if doc_lang_codes and line_lang_codes : if not lists_share_element ( doc_lang_codes , line_lang_codes ) : continue s = slot_value_in_double_colon_del_list ( line , head_slot ) lc_s = s . lower ( ) detokenization_entry = None if head_slot == 'auto-attach' : side = slot_value_in_double_colon_del_list ( line , 'side' ) group = bool ( slot_value_in_double_colon_del_list ( line , 'group' , False ) ) detokenization_entry = DetokenizationEntry ( s , group , line_lang_codes ) if side == 'left' or side == 'both' : for line_lang_code in ( line_lang_codes if line_lang_codes else [ None ] ) : key = f'{line_lang_code} {lc_s}' if self . auto_attach_left_w_lc . get ( key , False ) : lcode_clause = f' ::lcode {line_lang_code}' if line_lang_code else '' log . warning ( f'Duplicate ::auto-attach {lc_s} ::side left{lcode_clause}' ) else : self . auto_attach_left_w_lc [ key ] = True self . auto_attach_left [ lc_s ] . append ( detokenization_entry ) if side == 'right' or side == 'both' : for line_lang_code in ( line_lang_codes if line_lang_codes else [ None ] ) : key = f'{line_lang_code} {lc_s}' if self . auto_attach_right_w_lc . get ( key , False ) : lcode_clause = f' ::lcode {line_lang_code}' if line_lang_code else '' log . warning ( f'Duplicate ::auto-attach {lc_s} ::side right{lcode_clause}' ) else : self . auto_attach_right_w_lc [ key ] = True self . auto_attach_right [ lc_s ] . append ( detokenization_entry ) elif head_slot == 'markup-attach' : paired_delimiter = bool ( slot_value_in_double_colon_del_list ( line , 'paired-delimiter' , False ) ) group = bool ( slot_value_in_double_colon_del_list ( line , 'group' , False ) ) detokenization_entry = DetokenizationMarkupEntry ( s , group = group , paired_delimiter = paired_delimiter ) if except_s := slot_value_in_double_colon_del_list ( line , 'except' ) : detokenization_entry . exception_list = re . split ( r'\s+' , except_s ) self . markup_attach [ lc_s ] . append ( detokenization_entry ) self . markup_attach_re_elements . add ( re . escape ( lc_s ) + ( '+' if group else '' ) ) elif head_slot == 'attach_tag' : self . attach_tag = s elif head_slot == 'contraction' : if ( ( not slot_value_in_double_colon_del_list ( line , 'nonstandard' ) ) and ( not slot_value_in_double_colon_del_list ( line , 'substandard' ) ) ) : target = slot_value_in_double_colon_del_list ( line , 'target' ) lc_target = target . lower ( ) detokenization_entry = DetokenizationContractionEntry ( target , s ) self . contraction_dict [ lc_target ] . append ( detokenization_entry ) elif head_slot == 'lexical' : tag = slot_value_in_double_colon_del_list ( line , 'tag' ) if tag in ( 'DECONTRACTION-L' , 'DECONTRACTION-R' , 'DECONTRACTION-B' ) : detokenization_entry = DetokenizationEntry ( s ) if tag in ( 'DECONTRACTION-L' , 'DECONTRACTION-B' ) : self . auto_attach_right [ lc_s ] . append ( detokenization_entry ) if tag in ( 'DECONTRACTION-R' , 'DECONTRACTION-B' ) : self . auto_attach_left [ lc_s ] . append ( detokenization_entry ) if detokenization_entry : detokenization_entry . case_sensitive = bool ( slot_value_in_double_colon_del_list ( line , 'case-sensitive' ) ) if left_context_s := slot_value_in_double_colon_del_list ( line , 'left-context' ) : try : detokenization_entry . left_context = regex . compile ( eval ( 'r".*' + left_context_s + '$"' ) , flags = regex . VERSION1 ) except regex . error : log . warning ( f'Regex compile error in l.{line_number} for {s} ::left-context ' f'{left_context_s}' ) if left_context_not_s := slot_value_in_double_colon_del_list ( line , 'left-context-not' ) : try : detokenization_entry . left_context_not = regex . compile ( eval ( 'r".*(?<!' + left_context_not_s + ')$"' ) , flags = regex . VERSION1 ) except regex . error : log . warning ( f'Regex compile error in l.{line_number} for {s} ::left-context-not ' f'{left_context_not_s}' ) if right_context_s := slot_value_in_double_colon_del_list ( line , 'right-context' ) : try : detokenization_entry . right_context = regex . compile ( eval ( 'r"' + right_context_s + '"' ) , flags = regex . VERSION1 ) except regex . error : log . warning ( f'Regex compile error in l.{line_number} for {s} ::right-context ' f'{right_context_s}' ) if right_context_not_s := slot_value_in_double_colon_del_list ( line , 'right-context-not' ) : try : detokenization_entry . right_context_not = regex . compile ( eval ( 'r"(?!' + right_context_not_s + ')"' ) , flags = regex . VERSION1 ) except regex . error : log . warning ( f'Regex compile error in l.{line_number} for {s} ::right-context-not ' f'{right_context_not_s}' ) if lang_code_not_s := slot_value_in_double_colon_del_list ( line , 'lcode-not' ) : detokenization_entry . lang_codes_not = re . split ( r'[;,\s*]' , lang_code_not_s ) n_entries += 1 if verbose : log . info ( f'Loaded {n_entries} entries from {line_number} lines in {filename}' ) except OSError : log . warning ( f'Could not open general resource file {filename}' )
short_docstring: Loads detokenization resources such as auto-attach, markup-attach etc.
short_docstring_tokens: Loads detokenization resources such as auto - attach markup - attach etc .
comment: """Loads detokenization resources such as auto-attach, markup-attach etc.
          Example input file: data/detok-resource.txt
          This file is also loaded by the tokenizer to produce appropriate mt-style @...@ tokens."""$SEP$# ignore empty or comment line$SEP$# In tok-resources, only ::contraction entries are relevant.$SEP$# Check whether cost file line is well-formed. Following call will output specific warnings.$SEP$# e.g. s', 'n'$SEP$# e.g. 's, 'n'