Format Mecab output into a nice data structure, based on Janome.

def detailed_tokens(tokenizer, text):
    """Format Mecab output into a nice data structure, based on Janome."""
    node = tokenizer.parseToNode(text)
    node = node.next  # first node is beginning of sentence and empty, skip it
    words = []
    while node.posid != 0:
        surface = node.surface
        base = surface  # a default value. Updated if available later.
        parts = node.feature.split(",")
        pos = ",".join(parts[0:4])
        if len(parts) > 7:
            # this information is only available for words in the tokenizer
            # dictionary
            base = parts[7]
        words.append(ShortUnitWord(surface, base, pos))
        node = node.next
    return words