Generates linearized trees and tokens from the wsj tree format.

  It uses the linearized algorithm described in https://arxiv.org/abs/1412.7449.

  Args:
    tree_string: tree in wsj format

  Returns:
    tuple: (words, linearized tree)

def words_and_tags_from_wsj_tree(tree_string):
  """Generates linearized trees and tokens from the wsj tree format.

  It uses the linearized algorithm described in https://arxiv.org/abs/1412.7449.

  Args:
    tree_string: tree in wsj format

  Returns:
    tuple: (words, linearized tree)
  """
  stack, tags, words = [], [], []
  for tok in tree_string.strip().split():
    if tok[0] == "(":
      symbol = tok[1:]
      tags.append(symbol)
      stack.append(symbol)
    else:
      assert tok[-1] == ")"
      stack.pop()  # Pop the POS-tag.
      while tok[-2] == ")":
        tags.append("/" + stack.pop())
        tok = tok[:-1]
      words.append(tok[:-1])
  return str.join(" ", words), str.join(" ", tags[1:-1])