HEAD, COMPLEMENT, MODIFIER = range(3)
PTBPUNC = {'-LRB-', '-RRB-', '-LCB-', '-RCB-', '-LSB-', '-RSB-', '-NONE-'}
FRONTIERNTRE = re.compile(r' \)')
SUPERFLUOUSSPACERE = re.compile(r'\)\s+(?=\))')
INDEXRE = re.compile(r' ([0-9]+)\)')
STRTERMRE = re.compile(r' (?![0-9]+=)[^()]*\s*\)')
	def bitfanout(arg):
	def __new__(cls, label_or_str=None, children=None):
	def __init__(self, label_or_str, children=None):
	def __eq__(self, other):
	def __ne__(self, other):
	def __lt__(self, other):
	def __le__(self, other):
	def __gt__(self, other):
	def __ge__(self, other):
	def append(self, child):
	def extend(self, children):
	def insert(self, index, child):
	def pop(self, index=-1):
	def remove(self, child):
	def index(self, child):
	def __iter__(self):
	def __len__(self):
	def __bool__(self):
	def __mul__(self, _):
	def __rmul__(self, _):
	def __add__(self, _):
	def __radd__(self, _):
	def __getitem__(self, index):
	def __setitem__(self, index, value):
	def __delitem__(self, index):
	def leaves(self):
	def height(self):
	def subtrees(self, condition=None):
	def postorder(self, condition=None):
	def find(self, condition):
	def pos(self, nodes=False):
	def treepositions(self, order='preorder'):
	def leaf_treeposition(self, index):
	def treeposition_spanning_leaves(self, start, end):
	def convert(cls, val):
	def copy(self, deep=False):
	def _frozen_class(self):
	def freeze(self, leaf_freezer=None):
	def parse(cls, s, parse_label=None, parse_leaf=int,
			label_pattern=None, leaf_pattern=None):
	def _parse_error(cls, orig, match, expecting):
	def __repr__(self):
	def __str__(self):
	def pprint(self, margin=70, indent=0, brackets='()'):
	def _pprint_flat(self, brackets):
	def draw(self):
	def _repr_svg_(self):
	def __init__(self, label_or_str, children=None):
	def _addleaves(self):
	def leaves(self):
	def __setitem__(self, _index, _value):
	def __setslice__(self, _start, _stop, _value):
	def __delitem__(self, _index):
	def __delslice__(self, _start, _stop):
	def __iadd__(self, _):
	def __imul__(self, _):
	def append(self, _):
	def extend(self, _):
	def pop(self, _=None):
	def remove(self, _):
	def __hash__(self):
	def __init__(self, label_or_str, children=None):
	def _frozen_class(self):
	def _get_parent_index(self):
	def _get_left_sibling(self):
	def _get_right_sibling(self):
	def _get_treeposition(self):
	def _get_root(self):
	def _delparent(self, child, index):
	def _setparent(self, child, _index, dry_run=False):
	def __delitem__(self, index):
	def __setitem__(self, index, value):
	def append(self, child):
	def extend(self, children):
	def insert(self, index, child):
	def pop(self, index=-1):
	def remove(self, child):
	def detach(self):
	def disown(self):
	def spliceabove(self, label):
	def splicebelow(self, label):
	def prune(self):
	def __init__(self, label_or_str, children=None):
	def __init__(self, tree, sent):
	def __eq__(self, other):
	def __hash__(self):
	def __repr__(self):
	def __init__(self, tree, sent=None, abbr=False, highlight=(),
			highlightfunc=(), secedge=False):
	def __str__(self):
	def __repr__(self):
	def _repr_svg_(self):
	def nodecoords(self, tree, sent, highlight, highlightfunc, secedge):
		def findcell(m, matrix, startoflevel, children):
		def dumpmatrix():
	def svg(self, hscale=40, hmult=3, nodecolor='blue', leafcolor='red',
			funccolor='green', funcsep=None):
	def text(self, unicodelines=False, html=False, ansi=False,
				nodecolor='blue', leafcolor='red', funccolor='green',
				funcsep=None, morphsep=None, maxwidth=16, nodeprops=None):
		def crosscell(cur, x=vertline):
		def formatlabelnodeprops(text, nodeprops):
		def formatlabel(text):
	def tikzmatrix(self, nodecolor='blue', leafcolor='red',
			funccolor='green', funcsep=None):
	def tikznode(self, nodecolor='blue', leafcolor='red',
			funccolor='green', funcsep=None):
	def _tikzedges(self):
	def tikzqtree(self, nodecolor='blue', leafcolor='red'):
	def __init__(self, sent, pos, deps):
	def fromconll(cls, data):
	def svg(self):
		def levelheight(level):
		def under(e1, e1h, e2, e2h):
	def _repr_svg_(self):
def latexlabel(label):
def frontier(tree, sent, nodecolor='blue', leafcolor='red'):
def brackettree(treestr, detectdisc=True):
		def substleaf(x):
		def substleaf(x):
def discbrackettree(treestr):
	def substleaf(x):
def writebrackettree(tree, sent):
def writediscbrackettree(tree, sent, pretty=False):
def isdisc(node):
def escape(text):
def unescape(text):
def ptbescape(token):
def ptbunescape(token):
def ranges(seq):
class Tree(object):
class ImmutableTree(Tree):
class ParentedTree(Tree):
class ImmutableParentedTree(ImmutableTree, ParentedTree):
class DiscTree(ImmutableTree):
class DrawTree(object):
class DrawDependencies(object): FIELDS = tuple(range(6))
WORD, LEMMA, TAG, MORPH, FUNC, PARENT = FIELDS
EXPORTHEADER = '%% word\tlemma\ttag\tmorph\tedge\tparent\tsecedge\n'
EXPORTNONTERMINAL = re.compile(r'^#([5-9][0-9][0-9])$')
POSRE = re.compile(r'\(([^() ]+)\s+[^ ()]+\s*\)')
LEAVESRE = re.compile(r' (?=\))| ([^ ()]+)\s*(?=[\s)])')
ALPINOXML = re.compile(
		rb'<\?xml version="1.0" encoding="UTF-?8"\?>.*?</alpino_ds>\r?\n',
		flags=re.DOTALL | re.IGNORECASE)
ALPINOSENTID = re.compile(b'<sentence sentid="(.*?)">')
CONSUMED = True
NEWLB = re.compile(r'(?:.*[\n\r])?\s*')
READERS = OrderedDict((
		('export', NegraCorpusReader),
		('bracket', BracketCorpusReader),
		('discbracket', DiscBracketCorpusReader),
		('tiger', TigerXMLCorpusReader),
		('alpino', AlpinoCorpusReader),
		('alpinocompact', AlpinoCompactCorpusReader),
		('ftb', FTBXMLCorpusReader)))
WRITERS = ('export', 'bracket', 'discbracket',
		'conll', 'mst', 'tokens', 'wordpos')
class Item(object):
class CorpusReader(object):
class BracketCorpusReader(CorpusReader):
class DiscBracketCorpusReader(BracketCorpusReader):
class NegraCorpusReader(CorpusReader):
class TigerXMLCorpusReader(CorpusReader):
class AlpinoCorpusReader(CorpusReader):
class AlpinoCompactCorpusReader(AlpinoCorpusReader):
class FTBXMLCorpusReader(CorpusReader):
	def __init__(self, tree, sent, comment, block):
	def __init__(self, path, encoding='utf8', ensureroot=None, punct=None,
			headrules=None, removeempty=False,
			functions=None, morphology=None, lemmas=None,
			modifierrules=None):
	def itertrees(self, start=None, end=None):
	def trees(self):
	def sents(self):
	def tagged_sents(self):
	def blocks(self):
	def _read_blocks(self):
	def _parse(self, block):
	def _parsetree(self, block):
	def _word(self, block):
	def blocks(self):
	def _read_blocks(self):
	def _parse(self, block):
	def _parse(self, block):
		def substleaf(x):
	def blocks(self):
	def _read_blocks(self):
	def _parse(self, block):
	def blocks(self):
	def _read_blocks(self):
	def _parse(self, block):
	def blocks(self):
	def _read_blocks(self):
	def _parse(self, block):
	def _read_blocks(self):
	def __init__(self, *args, **kwargs):
	def blocks(self):
	def _read_blocks(self):
	def _parse(self, block):
def exporttree(block, functions=None, morphology=None, lemmas=None):
	def getchildren(parent):
def exportsplit(line):
def alpinotree(block, functions=None, morphology=None, lemmas=None):
	def getsubtree(node, parentid, morphology, lemmas):
def ftbtree(block, functions=None, morphology=None, lemmas=None):
	def getsubtree(node):
def writetree(tree, sent, key, fmt, comment=None, morphology=None,
		sentid=False):
def writeexporttree(tree, sent, key, comment, morphology):
	def collectsecedges(node):
def writealpinotree(tree, sent, key, commentstr):
	def addchildren(tree, sent, parent, cnt, depth=1, last=False):
def writedependencies(tree, sent, fmt):
def dependencies(root):
def _makedep(node, deps):
def deplen(deps):
def handlefunctions(action, tree, pos=True, root=False, morphology=None):
def handlemorphology(action, lemmaaction, preterminal, source, sent=None):
def incrementaltreereader(treeinput, morphology=None, functions=None,
		strict=False, robust=True, othertext=False):
def segmentbrackets(strict=False, robust=True):
	def tryparse(result, rest):
def segmentalpino(morphology, functions):
def segmentexport(morphology, functions, strict=False):
def numbase(key): def addfanoutmarkers(tree):
	"""Mark discontinuous constituents with '_n' where n = # gaps + 1."""
	for st in tree.subtrees():
		leaves = set(st.leaves())
		thisfanout = len([a for a in sorted(leaves) if a - 1 not in leaves])
		if thisfanout > 1 and not st.label.endswith('_%d' % thisfanout):
			st.label += '_%d' % thisfanout
	return tree def removefanoutmarkers(tree):
	"""Remove fanout marks."""
	for a in tree.subtrees(lambda x: '_' in x.label):
		a.label = a.label.rsplit('_', 1)[0]
	return tree def punctprune(tree, sent):
	"""Remove quotes and period at sentence beginning and end."""
	i = 0
	while i < len(sent) and sent[i] in PRUNEPUNCT:
		sent[i] = None
		i += 1
	i = len(sent) - 1
	while i < len(sent) and sent[i] in PRUNEPUNCT:
		sent[i] = None
		i -= 1
	if tree is None:
		sent[:] = [a for a in sent if a is not None]
	else:
		removeterminals(tree, sent, lambda a, _b: a is None) PUNCTUATION = frozenset('.,():-";?/!*&`[]<>{}|=\xc2\xab\xc2\xbb\xb7\xad\\'
		) | {'&bullet;', '..', '...', '....', '.....', '......',
		'!!', '!!!', '??', '???', "''", '``', ',,',
		'--', '---', '----', '-LRB-', '-RRB-', '-LCB-', '-RCB-',
		'-LSB-', '-RSB-', '#LRB#', '#RRB#'} PUNCTTAGS = {'.', ',', ':', "'", "''", '`', '``', '"',  # General
		'-LRB-', '-RRB-', '-LSB-', '-RSB-', '-LCB-', '-RCB-',  # PTB
		'$,', '$.', '$[', '$(',  # Negra/Tiger
		'let', 'LET[]', 'SPEC[symb]', 'TW[hoofd,vrij]',  # Alpino/Lassy
		'PONCT', 'PUNCT', 'COMMA', 'PAREN'}  # FTB/Grammatical Framework YEARRE = re.compile('^(?:19|20)[0-9]{2}$') FTBREGULARCOMPOUNDPATTERNS = {
		# an N, maybe with Det, Adj, and PPs:
		# "Organisation de coopération et de développement économique"
		# "Institut de formation des agents de voyages"
		# "pomme de terre"
		# "marché monétaire et obligataire"
		# "Bureau de recherches géologiques et minières"
		'N': re.compile(
			r'(D )?(A )*N( A( C A)?)*( P(\+D)?( D)?( A)* N( A( C A)?)*'
			r'(( C)? P(\+D)?( D)?( A)* N( A)*)*)?$'
			# r'|N( N)+$'  # Air France, maison mère ...
			# r'|(N|ET)( ET)+$'  # Wall Street, Zenith Data Structures,
			),
		'V': re.compile(
			r'V( V)*( P| A| D)*( N)+( P)*$'  # mettre en place, etc.
			r'|V( D)?( A)?( N)+$'),  # faire face, faire appel
		'P': re.compile(r'P(\+D)? (D )?(A )*N( P(\+D)?( D)?( A)* N( A( C A)?)*'
			r'(( C)? P(\+D)?( D)?( A)* N( A)*)*)? P(\+D)?$'),
		'ADV': re.compile(r'P(\+D)? (D )?(A )*N( P(\+D)?( D)?( A)* N( A)*)?$'),
		'A': re.compile(r'A (C A)?'),
		} FTBALLOWEDCOMPOUNDS = {
		'(MWN (N Fondation) (N France) (A active))',
		'(MWN (N Côte) (PONCT -) (P d\') (N Ivoire))',
		'(MWP (CL Il) (CL y) (V a))',
		'(MWN (N Jean) (PONCT -) (N Louis))',
		'(MWADV (ADV tout) (P de) (N suite))',
		'(MWP (P jusqu\') (P au))',
		'(MWN (A Haute) (PONCT -) (N Corse))',
		'(MWP (CL y) (A compris))',
		'(MWN (N Chalon) (PONCT -) (P sur) (PONCT -) (N Saône))',
		'(MWP (CL il) (CL y) (V a))',
		'(MWN (N Royaume) (PONCT -) (A uni))',
		'(MWN (N Seine) (PONCT -) (N Saint) (PONCT -) (N Denis))',
		'(MWN (N Roche) (PONCT -) (N la) (PONCT -) (N Molière))',
		'(MWN (N Air) (N France))',
		'(MWN (A Grande) (PONCT -) (N Bretagne))',
		'(MWN (N Union) (A soviétique))',
		}
import os
import re
from itertools import islice
from .tree import (Tree, ParentedTree, escape, unescape, ptbescape,
		HEAD, COMPLEMENT, MODIFIER)
from .treebank import writebrackettree, EXPORTNONTERMINAL
from .treetransforms import addfanoutmarkers, removefanoutmarkers
from .punctuation import punctprune, PUNCTUATION, PUNCTTAGS
from .lexicon import YEARRE, FTBREGULARCOMPOUNDPATTERNS, FTBALLOWEDCOMPOUNDS
				vp.append(np)
				return vp
			else:
				x = list(sbtrs[1:])
				sbtrs[:] = []
				vp.extend(x)
			return vp

		def make_PP(sbtrs):
			pp = ParentedTree('PP', [])
			pp_subtree = ParentedTree(sbtrs[0].label, sbtrs[0])
			pp_subtree.type = sbtrs[0].type
			pp_subtree.source = sbtrs[0].source
			pp.append(pp_subtree)
			if len(sbtrs) > 1:
				sbtrs_children = sbtrs[1:]
				np_subtree = make_NP(sbtrs_children)
				np_subtree.type = COMPLEMENT
				pp.append(np_subtree)
			return pp

		def make_NP(sbtrs):
			np = ParentedTree('NP', [])
			while sbtrs != []:
				if sbtrs[0].label in ('D', 'ET'):
					np_subtree = ParentedTree(sbtrs[0].label, sbtrs[0])
					np_subtree.type = sbtrs[0].type
					np_subtree.source = sbtrs[0].source
					np.append(np_subtree)
					sbtrs = sbtrs[1:]
				if sbtrs[0].label == 'N':
					np_subtree = ParentedTree(sbtrs[0].label, sbtrs[0])
					np_subtree.type = sbtrs[0].type
					np_subtree.source = sbtrs[0].source
					np.append(np_subtree)
					sbtrs = sbtrs[1:]
				elif sbtrs[0].label == 'A':
					if (len(sbtrs) > 2 and sbtrs[1].label == 'C'
							and sbtrs[2] == 'A'):
						ap = make_AP(sbtrs[0:3], 'A C A')
						np.append(ap)
						sbtrs = sbtrs[3:]
					else:
						ap = ParentedTree('AP', [])
						ap.type = MODIFIER
						ap_subtree = ParentedTree('A', sbtrs[0])
						ap_subtree.type = HEAD
						ap_subtree.source = sbtrs[0].source
						ap.append(ap_subtree)
						np.append(ap)
						sbtrs = sbtrs[1:]
				# if a prep is encountered
				# => treat all remaining sbtrs as a whole PP
				# (cf. closest attachment preferred)
				# (unhandled case: N1 (P N2) others
				# where others attaches to N1)
				elif sbtrs[0].label in ('P', 'P+D'):
					pp = make_PP(sbtrs)
					pp.type = COMPLEMENT
					np.append(pp)
					sbtrs = []
				elif sbtrs[0].label == 'C':
					coord = make_COORD(sbtrs)
					coord.type = MODIFIER
					np.append(coord)
					sbtrs = []
			return np

		def make_COORD(sbtrs):
			coord = ParentedTree('COORD', [])
			coord_subtree = ParentedTree(sbtrs[0].label, sbtrs[0])
			coord_subtree.type = sbtrs[0].type
			coord_subtree.source = sbtrs[0].source
			coord_subtree.type = HEAD
			coord.append(coord_subtree)
			# conjunction is supposed to be the first sbtree
			# if C P ... => coordination of PPs
			if strip(sbtrs[1].label) in ('P', 'P+D'):
				pp = make_PP(sbtrs[1:])
				pp.type = COMPLEMENT
				coord.append(pp)
			# otherwise = coordination of NPs (APs handled differently)
			else:
				np = make_NP(sbtrs[1:])
				np.type = COMPLEMENT
				coord.append(np)
			return coord

		def make_AP(sbtrs, cmpd_str):
			if cmpd_str == 'A':
				ap = ParentedTree('AP', [])
				ap_subtree = ParentedTree('A', sbtrs[0])
				ap_subtree.type = HEAD
				ap_subtree.source = sbtrs[0].source
				ap.append(ap_subtree)
			elif cmpd_str == 'A C A':
				ap = ParentedTree('AP', [])
				ap.type = MODIFIER
				ap_subtree = ParentedTree(sbtrs[0].label, sbtrs[0])
				ap_subtree.type = HEAD
				ap_subtree.source = sbtrs[0].source
				ap.append(ap_subtree)
				coord = ParentedTree('COORD', [])
				coord.type = MODIFIER
				coord_subtree = ParentedTree(sbtrs[1].label, sbtrs[1])
				coord_subtree.type = sbtrs[1].type
				coord_subtree.source = sbtrs[1].source
				coord.append(coord_subtree)
				ap2 = ParentedTree('AP', [])
				ap2.type = COMPLEMENT
				ap2_subtree = ParentedTree(sbtrs[2].label, sbtrs[2])
				ap2_subtree.type = HEAD
				ap2_subtree.source = sbtrs[2].source
				ap2.append(ap2_subtree)
				coord.append(ap2)
				ap.append(coord)
			return ap

		for sbtree in tree.subtrees():
			if (sbtree.label.startswith('MW')

					and writebrackettree(sbtree, sent).strip()