#!/usr/bin/env python3
"""
filterCSV - Augments matching CSV files - according to iThoughts requirements

It can set colours for nodes, change their shape, or delete them - if the node
matches any one of a set of criteria the user specifies.

Reads from stdin
Writes to stdout
Commentary to stderr


Command line parameters are pairs of:

1) A specifier. This is a regular expression to match. (A special value 'all' matches
    any value)
2) An action or sequence of actions.

Actions can be:

* A 1 or 2 digit colour number - relative to the top left of iThoughts' colour palette.
* A colour RGB hexadecimal value.  Ex. FFFFFF is black, 000000 is white, FF0000 is red
* 'delete'.
* A shape - as named by iThoughts.
"""

import csv
import re
import sys
from collections import Counter
from string import hexdigits
import xml.etree.ElementTree as ElementTree

# from CSVTree import CSVTree

filterCSV_level = "1.2"
filterCSV_date = "29 April, 2020"


class ParameterParser:
    def getParameters(self):
        matchCriteria = []
        actionsLists = []

        output = []

        # Heading for parameters display
        output.append("\nfilterCSV " + filterCSV_level + " (" + filterCSV_date + ")\n")
        output.append("Criterion".ljust(40, " ") + " Actions")
        output.append("---------".ljust(40, " ") + " -------")

        # Read in pairs of parameters from command line
        parmNumber = 1
        for parmPair in range((len(sys.argv) - 1) // 2):
            # Handle match criterion
            matchCriterion = sys.argv[parmNumber]
            if matchCriterion == "all":
                matchCriteria.append(re.compile(".*"))
            else:
                matchCriteria.append(re.compile(matchCriterion))

            parmNumber += 1

            # Handle the actions that go with this match criterion
            actionsString = sys.argv[parmNumber].lower()
            actionList = actionsString.split()

            # For nextcolour/samecolour and nextshape/sameshape etc rewrite action as
            # the colour number or shape
            for actionNumber in range(len(actionList)):
                action = actionList[actionNumber]
                if action in ("nextcolour", "nextcolor", "nc"):
                    actionList[actionNumber] = str(
                        iThoughtsColours.getNextColourNumber()
                    )
                elif action in ("samecolour", "samecolor", "sc"):
                    actionList[actionNumber] = str(
                        iThoughtsColours.getSameColourNumber()
                    )
                elif action in ("nextshape", "ns"):
                    actionList[actionNumber] = iThoughtsShapes[
                        iThoughtsShapes.getNextShapeNumber()
                    ]
                elif action in ("sameshape", "ss"):
                    actionList[actionNumber] = iThoughtsShapes[
                        iThoughtsShapes.getSameShapeNumber()
                    ]

            # Add actions list to list of actions lists
            actionsLists.append(actionList)

            parmNumber += 1

            output.append(matchCriterion.ljust(40, " ") + " " + actionsString)

        output.append("\n")

        return matchCriteria, actionsLists, output


class TreeReader:
    def detectInputStreamType(self, firstLine):
        firstChar = firstLine[0]

        if firstLine.find("level0") > -1:
            return "iThoughtsCSV"
        elif firstChar == "<":
            return "XML"
        elif firstChar in ["#", "*"]:
            return "markdown"
        elif firstChar == '"':
            return "CSV"
        else:
            return "text"

    def findColumn(this, CSVArray, columnName):
        # Returns 0-indexed column if the column name appears in the first LineEnd
        # else -1
        if columnName in CSVArray[0]:
            return CSVArray[0].index(columnName)
        else:
            return -1

    def injectColumn(this, CSVArray, columnName):
        # Injects the named column at the beginning of each line,
        # with the heading given by columnName
        for rowNumber, _ in enumerate(CSVArray):
            if rowNumber == 0:
                CSVArray[0].insert(0, columnName)
            else:
                CSVArray[rowNumber].insert(0, "")

        return CSVArray

    def ensureMandatoryColumns(self, csvRows):
        # Add a shape column if one is not already present
        if self.findColumn(csvRows, "shape") == -1:
            csvRows = self.injectColumn(csvRows, "shape")

        # Add a position column if one is not already present
        if self.findColumn(csvRows, "position") == -1:
            csvRows = self.injectColumn(csvRows, "position")

        # Add a note column if one is not already present
        if self.findColumn(csvRows, "note") == -1:
            csvRows = self.injectColumn(csvRows, "note")

        # Add a colour column if one is not already present
        if self.findColumn(csvRows, "colour") == -1:
            csvRows = self.injectColumn(csvRows, "colour")

        # Establish the position of each column - to return
        colourColumn = self.findColumn(csvRows, "colour")
        levelColumn = self.findColumn(csvRows, "level")
        noteColumn = self.findColumn(csvRows, "note")

        shapeColumn = self.findColumn(csvRows, "shape")
        positionColumn = self.findColumn(csvRows, "position")

        return (
            csvRows,
            colourColumn,
            levelColumn,
            noteColumn,
            shapeColumn,
            positionColumn,
        )

    def ensureColumnsPopulated(self, csvRows):
        # Ensures every column in the header row is present in each subsequent row
        minimumColumnCount = len(csvRows[0])
        for rowNumber, row in enumerate(csvRows):
            extraCells = minimumColumnCount - len(row)
            if extraCells > 0:
                csvRows[rowNumber] += [""] * extraCells

        return csvRows

    def readiThoughtsCSVTree(self, inputFile):
        output = []
        # Build array of rows
        csvRows = []
        reader = csv.reader(inputFile)
        for row in reader:
            csvRows.append(row)

        return self.ensureMandatoryColumns(csvRows) + (output,)

    def readMarkdownOrTextTree(self, inputFile):
        output = []

        # Build array of rows
        csvRows = []

        # Work out what an indent would be - in numbers of characters per level
        indent = -1
        for line in inputFile:
            indent = len(line) - len(line.lstrip())
            if indent > 0:
                # This is the first line with whitespace at the beginning

                # Save the indentation whitespace
                indentCharacters = line[0:indent]

                # Print the detected indentation - so user can debug
                output.append(
                    "Indentation detected: "
                    + formatWhitespaceCharacters(indentCharacters)
                )

                break

        # Insert a header row, with attribute columns and a level0 column plus other
        # levels
        csvRows.append(
            [
                "level",
                "level0",
                "level1",
                "level2",
                "level3",
                "level4",
                "level5",
                "level6",
                "level7",
                "level8",
                "level9",
                "level10",
                "level11",
                "level12",
                "level13",
                "level14",
                "level15",
                "level16",
                "level17",
                "level18",
                "level19",
                "level20",
            ],
        )

        # Add a level column if one is not already present, and move the text into the
        # right level - for each data line
        for lineNumber, line in enumerate(inputFile):
            # A data line so work out how many levels deep it is indented
            lineIndent = len(line) - len(line.lstrip())
            if lineIndent % indent > 0:
                if lineIndent == 1:
                    output.append("Bad indentation: 1 white space character.")
                else:
                    output.append(
                        "Bad indentation:"
                        + str(lineIndent)
                        + " white space characters."
                    )

                output.append(
                    "Should be multiple of "
                    + str(indent)
                    + ". Rounding level down to "
                    + str(lineIndent / indent)
                    + "."
                )
                output.append("Line in error (" + str(rowNumber) + ") is: " + line)
                output.append(
                    "Leading white space characters: "
                    + formatWhitespaceCharacters(line[0:lineIndent])
                )
            else:
                lineIndentCharacters = line[0:lineIndent]
                if lineIndentCharacters != indentCharacters * (lineIndent // indent):
                    output.append("Bad indentation characters:")
                    output.append("Line in error (" + str(lineNumber) + ") is: " + line)
                    output.append(
                        "Leading white space characters: "
                        + formatWhitespaceCharacters(lineIndentCharacters)
                    )

            newRow = []

            level = lineIndent // indent
            newRow.append(str(level))

            # Insert blank cells - according to level
            for l in range(level):
                newRow.append("")

            # Clean the line text to remove any list item marker
            cleanedLine = line.lstrip().rstrip()
            if cleanedLine[0:2] == "* ":
                cleanedLine = cleanedLine[2:]

            # Add the cleaned line at the appropriate level
            newRow.append(cleanedLine)

            # Add this new row
            csvRows.append(newRow)

        return self.ensureMandatoryColumns(csvRows) + (output,)

    def readXMLTree(self, inputFile):
        output = []

        # Prepare the input text for namespace parsing and XML parsing
        XMLText = "\n".join(inputFile)

        # Create the XML parse tree
        self.XMLTree = ElementTree.fromstring(XMLText)

        # Hunt for the default namespace
        split1 = XMLText.split('xmlns="')
        if len(split1) == 1:
            output.append("No default namespace specification.")
            self.defaultXMLNamespace = ""
        else:
            self.defaultXMLNamespace = split1[1].split('"')[0]
            output.append(f"Default namespace is '{self.defaultXMLNamespace}'")

        # Hunt for other namespaces
        self.XMLNamespaces = {}
        split3 = XMLText.split("xmlns:")
        for fragment in range(len(split3)):
            if fragment > 0:
                split4 = split3[fragment].split('="')
                key = split4[0]
                split5 = split4[1].split('"')
                value = split5[0]
                self.XMLNamespaces[key] = value

        # Build array of rows
        csvRows = []

        # Insert a header row, with attribute columns and a level0 column plus other
        # levels
        csvRows.append(
            [
                "position",
                "colour",
                "shape",
                "level",
                "level0",
                "level1",
                "level2",
                "level3",
                "level4",
                "level5",
                "level6",
                "level7",
                "level8",
                "level9",
                "level10",
                "level11",
                "level12",
                "level13",
                "level14",
                "level15",
                "level16",
                "level17",
                "level18",
                "level19",
                "level20",
            ],
        )

        csvRows += self._readXMLTree(self.XMLTree, 0)

        return self.ensureMandatoryColumns(csvRows) + (output,)

    def resolveNamespaces(self, textToEdit):
        editedText = textToEdit.replace("{" + self.defaultXMLNamespace + "}", "")
        for key in self.XMLNamespaces:
            editedText = editedText.replace(
                "{" + self.XMLNamespaces[key] + "}", key + ":"
            )
        return editedText

    def _readXMLTree(self, XMLNode, level):
        csvRows = []

        # Edit the element tag - in case of namespaces
        editedElementName = self.resolveNamespaces(XMLNode.tag)

        # Create beginning of the row for the node itself - without attributes

        if XMLNode is self.XMLTree:
            # Is root node so create legend rows before it
            csvRows.append(
                [
                    "{-300,-120}",
                    iThoughtsColours.getColour(2),
                    "rectangle",
                    "0",
                    "element",
                ]
            )
            csvRows.append(
                ["{-300,-85}", iThoughtsColours.getColour(8), "rectangle", "0", "value"]
            )
            csvRows.append(
                [
                    "{-300,-50}",
                    iThoughtsColours.getColour(6),
                    "rectangle",
                    "0",
                    "attribute",
                ]
            )

            # Set position of root node to {0,0}
            elementCSVRow = [
                "{0,0}",
                iThoughtsColours.getColour(2),
                "rounded",
                f"{str(level)}",
            ]
        else:
            # Set position to blank for all but the root node
            elementCSVRow = [
                "",
                iThoughtsColours.getColour(2),
                "rounded",
                f"{str(level)}",
            ]

        # Add blank cells to position the cell with the element in
        if level > 0:
            levelBlankCells = [""] * level
            elementCSVRow += levelBlankCells

        # Add the (edited) element name
        elementCSVRow.append(editedElementName)

        # Add the row to the list of rows
        csvRows.append(elementCSVRow)

        for key, value in XMLNode.attrib.items():
            attributeCSVRow = [
                "",
                iThoughtsColours.getColour(6),
                "rounded",
                f"{str(level + 1)}",
            ]
            levelBlankCells = [""] * (level + 1)
            attributeCSVRow += levelBlankCells

            editedAttributeName = self.resolveNamespaces(key)
            attributeCSVRow += [f'{editedAttributeName}="{value}"']
            csvRows.append(attributeCSVRow)
        if XMLNode.text is not None:
            if XMLNode.text.lstrip().rstrip() != "":
                textCSVRow = [
                    "",
                    iThoughtsColours.getColour(8),
                    "rounded",
                    f"{str(level + 1)}",
                ]
                levelBlankCells = [""] * (level + 1)
                textCSVRow += levelBlankCells
                textCSVRow += [f"{XMLNode.text}"]
                csvRows.append(textCSVRow)

        for child in XMLNode:
            csvRows += self._readXMLTree(child, level + 1)

        return csvRows

    def createCSVArray(self):
        output = []

        # Read in whatever's in stdin - which might not be CSV
        inputFile = sys.stdin.readlines()

        # Detect the input stream type
        inputType = self.detectInputStreamType(inputFile[0])

        output.append(f"Input type detected as '{inputType}'.\n")

        # Depending on detected input type create a create CSV-like array,
        # including mandatory columns
        if inputType == "iThoughtsCSV":
            (
                csvRows,
                colourColumn,
                levelColumn,
                noteColumn,
                shapeColumn,
                positionColumn,
                output1,
            ) = self.readiThoughtsCSVTree(inputFile)

            output += output1

            csvRows = self.ensureColumnsPopulated(csvRows)

            return (
                csvRows,
                colourColumn,
                levelColumn,
                noteColumn,
                shapeColumn,
                positionColumn,
                output,
            )
        elif inputType in ["markdown", "text"]:
            (
                csvRows,
                colourColumn,
                levelColumn,
                noteColumn,
                shapeColumn,
                positionColumn,
                output1,
            ) = self.readMarkdownOrTextTree(inputFile)

            output += output1

            csvRows = self.ensureColumnsPopulated(csvRows)

            return (
                csvRows,
                colourColumn,
                levelColumn,
                noteColumn,
                shapeColumn,
                positionColumn,
                output,
            )
        elif inputType == "XML":
            (
                csvRows,
                colourColumn,
                levelColumn,
                noteColumn,
                shapeColumn,
                positionColumn,
                output1,
            ) = self.readXMLTree(inputFile)

            output += output1

            csvRows = self.ensureColumnsPopulated(csvRows)
            if len(self.XMLNamespaces) > 0:
                output.append("Namespace".ljust(20, " ") + " Namespace URL")
                output.append("---------".ljust(20, " ") + " -------------")
                for key in self.XMLNamespaces:
                    output.append(key.ljust(20, " ") + " " + self.XMLNamespaces[key])
                output.append("")

            return (
                csvRows,
                colourColumn,
                levelColumn,
                noteColumn,
                shapeColumn,
                positionColumn,
                output,
            )


class TreeWriter:
    def writeTreeAsCSV(self):

        CSVwriter = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL)

        # Write the header row
        headerRow = ["colour", "note", "position", "shape", "level"]
        levels = csvTree.calculateMaximumLevel() + 1

        for level in range(levels):
            headerRow.append("level" + str(level))

        CSVwriter.writerow(headerRow)

        # Write out the resulting CSV data
        outputArray = []
        outputArray = csvTree.writeCSVTree(outputArray)

        for row in outputArray:
            CSVwriter.writerow(row)


class iThoughtsColours:
    light_pastel = """FFB2B2 FFD8B2 FFFFB2 D8FFB2 B2FFB2 B2FFD8
                      B2FFFF B2D8FF B2B2FF D8B2FF FFB2FF FFB2D8""".split()

    dark_pastel = """B24747 B27C47 B2B247 7CB247 47B247 47B27C
                     47B2B2 477CB2 4747B2 7C47B2 B247B2 B2477C""".split()

    saturated = """FF0000 FF7F00 FFFF00 7FFF00 00FF00 00FF7F
                   00FFFF 007FFF 0000FF 7F00FF FF00FF FF007F""".split()

    grayscale = """000000 929292 a9a9a9 C0C0C0 D6D6D6 FFFFFF""".split()

    solarized = """002b36 073642 586e75 657b83 839496 93a1a1
                   eee8d5 fdf6e3 b58900 cb4b16 dc322f d33682
                   6c71c4 268bd2 2aa198 859900""".split()
    colours = light_pastel + dark_pastel + saturated + grayscale + solarized

    colourNumber = 0

    def __getitem__(self, key):
        """
        >>> ithoughts_colours = iThoughtsColours()
        >>> ithoughts_colours[0]
        'FFB2B2'
        >>> ithoughts_colours[-1]
        '859900'
        """
        return self.colours[key]

    def getColour(self, colourNumber):
        """Get colour based on colour number (which starts at 1)
        >>> ithoughts_colours = iThoughtsColours()
        >>> ithoughts_colours.getColour(1)
        'FFB2B2'
        >>> ithoughts_colours.getColour(58)
        '859900'
        """
        return self[colourNumber - 1]

    def __len__(self):
        """
        >>> len(iThoughtsColours())
        58
        """
        return len(self.colours)

    def getNextColourNumber(self):
        """Get the next colour number from the iThoughts palette
        >>> ithoughts_colours = iThoughtsColours()
        >>> ithoughts_colours.getNextColourNumber()
        1
        """
        self.colourNumber += 1
        return self.colourNumber

    def getSameColourNumber(self):
        """Get the same colour number from the iThoughts palette
        >>> ithoughts_colours = iThoughtsColours()
        >>> ithoughts_colours.getSameColourNumber()
        0
        """
        return self.colourNumber


class iThoughtsShapes:
    # Shapes as understood by iThoughts
    shapes = (
        "auto",
        "rectangle",
        "square",
        "rounded",
        "pill",
        "parallelogram",
        "diamond",
        "triangle",
        "oval",
        "circle",
        "underline",
        "none",
        "square bracket",
        "curved bracket",
    )

    shapeNumber = 0

    def __contains__(self, item):
        """Is item in self.shapes
        >>> ithoughts_shapes = iThoughtsShapes()
        >>> all(shape in ithoughts_shapes for shape
        ...     in ('auto', 'triangle', 'none', 'curved bracket'))
        True
        """
        return item in self.shapes

    def __getitem__(self, key):
        """Get shape based on shape number (which starts at 0)
        >>> ithoughts_shapes = iThoughtsShapes()
        >>> ithoughts_shapes[0]
        'auto'
        >>> ithoughts_shapes[-1]
        'curved bracket'
        """
        return self.shapes[key]

    def __len__(self):
        """
        >>> len(iThoughtsShapes())
        14
        """
        return len(self.shapes)

    def getNextShapeNumber(self):
        """Get the next shape number from the iThoughts list
        >>> ithoughts_shapes = iThoughtsShapes()
        >>> ithoughts_shapes.getNextShapeNumber()
        1
        """
        self.shapeNumber += 1
        return self.shapeNumber

    def getSameShapeNumber(self):
        """Get the same shape number from the iThoughts list
        >>> ithoughts_shapes = iThoughtsShapes()
        >>> ithoughts_shapes.getSameShapeNumber()
        0
        """
        return self.shapeNumber


class iThoughtsSpread:
    horizontalPosition = 0
    horizontalIncrement = 0
    verticalPosition = 0
    verticalIncrement = 0

    def resetHorizontalSpread(self, horizontalIncrement):
        self.horizontalPosition = 0
        self.horizontalIncrement = horizontalIncrement

    def nextHorizontal(self):
        x = self.horizontalPosition
        self.horizontalPosition += self.horizontalIncrement
        return x

    def resetVerticalSpread(self, verticalIncrement):
        self.verticalPosition = 0
        self.verticalIncrement = verticalIncrement

    def nextVertical(self):
        y = self.verticalPosition
        self.verticalPosition += self.verticalIncrement
        return y

    def getNextPosition(self):
        return self.nextHorizontal(), self.nextVertical()


class CSVTree:
    def __init__(self, shape, colour, note, level, position, cell):
        """
        >>> csv_tree = CSVTree("shape", "colour", "note", "level", 1, "cell")
        >>> csv_tree.data["shape"]
        'shape'
        """
        self.childNodes = []
        self.data = {
            "shape": shape,
            "colour": colour,
            "note": note,
            "level": level,
            "position": position,
            "cell": cell,
        }
        self.parent = None
        self.matched = False

    def addChild(self, childNode):
        self.childNodes.append(childNode)
        childNode.parent = self
        return childNode

    def deleteChild(self, childNode):
        self.childNodes.remove(childNode)

    def getChildren(self):
        return self.childNodes

    def isMatch(self, criterion):
        """
        Common method for establishing if a node matches some criterion, usually a regex
        """
        return (
            (re.search(criterion, self.data["shape"]) is not None)
            | (re.search(criterion, self.data["colour"]) is not None)
            | (re.search(criterion, self.data["note"]) is not None)
            | (re.search(criterion, self.data["cell"]) is not None)
        )

    def applyFilter(self, criterion, actionsList):
        """
        Apply filter to this node - if not tree root
        """
        propagate = True
        if self.data["level"] != -1:
            if self.isMatch(criterion):
                # Matched so apply all actions triggered by this match
                for action in actionsList:
                    if action == "delete":
                        # Delete the node
                        self.parent.deleteChild(self)

                        # Don't propagate
                        propagate = False
                    elif action[0] == "{":
                        # position specified
                        self.data["position"] = action

                        # Don't propagate
                        propagate = False
                    elif action == "note":
                        # Document the match in the note field
                        if self.data["note"] != "":
                            self.data["note"] += "\nMatched " + criterion.pattern
                        else:
                            self.data["note"] = "Matched " + criterion.pattern
                    elif action == "noshape":
                        # Remove any shape specification from the matched node
                        self.data["shape"] = ""
                    elif action == "nonote":
                        # Remove any note specification from the matched node
                        self.data["note"] = ""
                    elif action == "noposition":
                        # Remove any position specification from the matched node
                        self.data["position"] = ""
                    elif action == "nocolour":
                        # Remove any colour specification from the matched node
                        self.data["colour"] = ""
                    elif len(action) == 6 and all(c in hexdigits for c in action):
                        # 6-digit hexadecimal so is colour RGB value
                        self.data["colour"] = action
                    elif action in iThoughtsShapes:
                        # Is a shape
                        self.data["shape"] = action
                    elif action.isdigit():
                        # Attempt to parse as from the colour palette
                        colourNumber = int(action)

                        # We have an integer. If it is too big but not 6 digits
                        # we flag an error and don't do the update
                        if colourNumber > len(iThoughtsColours):
                            sys.stderr.write(
                                f"Erroneous colour value {colourNumber} "
                                f"(Pattern was: '{criterion.pattern}')." + "\n"
                            )

                        else:
                            self.data["colour"] = iThoughtsColours.getColour(
                                colourNumber
                            )
                    else:
                        sys.stderr.write(
                            f"Erroneous action value {action} "
                            f"(Pattern was: '{criterion.pattern}')." + "\n"
                        )

        # Apply filter to children, recursively - if propagation is indicated
        if propagate:
            for child in self.childNodes:
                child.applyFilter(criterion, actionsList)

    def writeCSVTree(self, outputArray):
        # Compose this node's line - if not root node
        level = int(self.data["level"])
        if level > -1:
            line = [
                self.data[key] for key in "colour note position shape level".split()
            ]
            line += [""] * int(self.data["level"])
            line.append(self.data["cell"])
            outputArray.append(line)

        # Print children, recursively
        for child in self.childNodes:
            child.writeCSVTree(outputArray)

        return outputArray

    def getCounter(self, counter=None):
        """
        >>> csv_tree = CSVTree("shape", "colour", "note", 1, 1, "cell")
        >>> csv_tree.getCounter()
        Counter({1: 1})
        >>> _ = csv_tree.addChild(CSVTree("shape", "colour", "note", 2, 7, "cell"))
        >>> csv_tree.getCounter()
        Counter({1: 1, 2: 1})
        """
        counter = counter or Counter()
        counter[self.data["level"]] += 1
        for child in self.childNodes:
            child.getCounter(counter)
        return counter

    def checkHierarchy(self, actionsList):
        sys.stderr.write("Beginning Level Check\n")
        sys.stderr.write("---------------------\n")

        self._checkHierarchy(-2, actionsList)

        sys.stderr.write("---------------------\n")
        sys.stderr.write("Completed Level Check\n")

    def _checkHierarchy(self, previousLevel, actionsList):
        level = int(self.data["level"])
        if level > previousLevel + 1:
            sys.stderr.write(self.data["cell"].ljust(40) + " Error: ")
            sys.stderr.write(f"Expected level {previousLevel + 1}.")
            sys.stderr.write(f" Found level {level}. ")

            firstAction = actionsList[0]

            if firstAction in ("repair", "repairnode"):
                # Repair just this node
                self.data["level"] = previousLevel + 1
                sys.stderr.write(
                    f"Repaired, setting level to {previousLevel + 1}." + "\n"
                )
            elif firstAction == "repairsubtree":
                # Repair the whole subtree
                sys.stderr.write(
                    "Repaired this node (setting its level to "
                    f"{previousLevel + 1}) and all its child nodes." + "\n"
                )
                self.repairSubtreeLevels(previousLevel)
            elif firstAction == "stop":
                # Terminate the check
                sys.stderr.write("Terminating.\n")
                sys.exit()
            else:
                sys.stderr.write("\n")
        else:
            if previousLevel > -2:
                sys.stderr.write(self.data["cell"].ljust(40) + " OK:   ")
                sys.stderr.write(f" Found level {level}." + "\n")
        for childNode in self.childNodes:
            childNode._checkHierarchy(previousLevel + 1, actionsList)

    def repairSubtreeLevels(self, parentLevel):
        # Repair this level
        self.data["level"] = parentLevel + 1

        # Repair lower levels
        for childNode in self.childNodes:
            childNode.repairSubtreeLevels(parentLevel + 1)

    def exportToMarkdown(self, actionsList):
        # Get the number of heading levels to allow before going to nested bulleted
        # lists
        headingLevels = int(actionsList[0])

        if len(actionsList) > 1:
            # Get first heading level
            startingLevel = int(actionsList[1])
        else:
            # Default to starting at heading level 1
            startingLevel = 1

        return self._exportToMarkdown(headingLevels - 1, startingLevel)

    def _exportToMarkdown(self, maxHeadingLevel, startingLevel):
        # Prime array of output lines
        output = []

        level = int(self.data["level"])
        if level > -1:
            note = self.data["note"]
            if level > maxHeadingLevel:
                output.append(
                    "  " * (level - maxHeadingLevel - startingLevel + 2)
                    + "* "
                    + self.data["cell"]
                )
                if note:
                    output.append(f"<br/><br/>{note}" + "\n")
            else:
                output.append(
                    "\n"
                    + "#" * (level + startingLevel)
                    + " "
                    + self.data["cell"]
                    + "\n"
                )
                if note:
                    output.append(note + "\n")

        for childNode in self.childNodes:
            output += childNode._exportToMarkdown(maxHeadingLevel, startingLevel)

        return output

    def calculateMaximumLevel(self, level=0):
        """
        maximum level is the max of this level and the levels of all childNodes,
        recursively through their descendents
        """
        thisLevel = int(self.data["level"])
        if thisLevel > level:
            level = thisLevel
        for childNode in self.childNodes:
            level = childNode.calculateMaximumLevel(level)
        return level

    def exportToHTML(self, actionsList):
        action = actionsList[0]

        # Prime array of output lines
        output = []

        if action == "table":
            # Work out how many levels are needed
            maximumLevel = self.calculateMaximumLevel()

            # Write table start
            output.append("<table>")

            # Write table rows
            output += self._exportToHTMLTable(maximumLevel, action)

            # Write table end
            output.append("</table>")
        else:
            # Write top-level list start
            output.append("<ul style='list-style-type: none;'>")

            # Write nested list
            level, freshOutput = self._exportToHTMLList(action)

            output += freshOutput

            # Write top-level list stop
            output.append("</ul>")

        return output

    def _exportToHTMLTable(self, maximumLevel, action):
        # Prime array of output lines
        output = []

        level = int(self.data["level"])
        # Determine the cell background colour
        colour = self.data["colour"] or "FFFFFF"
        # HTML table
        if level > -1:
            # Print table row start
            output.append("<tr>")

            # Print padding empty columns after the cell's column
            if level > 0:
                output.append("<td></td>" * level)

            # Print the cell itself, including styling
            output.append(
                "<td style='border: 1px solid black;border-radius: 5px;"
                + "background-color: #"
                + colour
                + ";'>"
                + self.data["cell"]
                + "</td>"
            )

            # Print padding empty columns after the cell's column
            if level < maximumLevel:
                output.append("<td></td>" * (maximumLevel - level))

            # Print any note as a final column
            note = self.data["note"]
            if note:
                output.append("<td>" + note + "</td>")

            # Print table row end
            output.append("</tr>")

        for childNode in self.childNodes:
            output += childNode._exportToHTMLTable(maximumLevel, action)

        return output

    def _exportToHTMLList(self, action):
        # Prime array of output lines
        output = []

        level = int(self.data["level"])

        # Determine the cell background colour
        colour = self.data["colour"]
        if colour == "":
            colour = "FFFFFF"

        # Determine if there is a note
        note = self.data["note"]

        # HTML nested list
        if level > -1:
            indent = "  " * (level + 1)
            # print list item
            output.append(
                indent
                + "<li style='padding: 10px;'><span style='border: 1px solid "
                + "black;padding: 5px;border-radius: 5px;background-color: #"
                + colour
                + ";'>"
                + self.data["cell"]
                + "</span>"
            )

            # Print any note
            if note:
                output.append(indent + "  <br/><br/>" + note)

            needListElements = len(self.childNodes) > 0
            if needListElements is True:
                output.append(indent + "<ul style='list-style-type: none;'>")

        returnedLevel = -2
        for childNode in self.childNodes:
            returnedLevel, freshOutput = childNode._exportToHTMLList(action)

            output += freshOutput

        if level > -1:
            if needListElements is True:
                output.append(indent + "</ul>")

            output.append(indent + "</li>")

        return [level, output]

    def exportToXML(self, actionsList):
        action = actionsList[0]

        if action == "freemind":
            # Freemind XML export
            return self.exportToFreemindXML(actionsList)
        elif action == "opml":
            return self.exportToOPMLXML(actionsList)

    def exportToFreemindXML(self, actionsList):
        # Export to XML in the format Freemind, MindNode and iThoughts accept

        # Prime array of output lines
        output = []

        # Warn if resulting XML would produce more than 1 Level 0 node
        if len(self.childNodes) > 1:
            sys.stderr.write(
                "Exported XML will have more than 1 root node. Some programs will get "
                "confused by this. Continuing.\n"
            )

        # Start the map
        output.append("<?xml version='1.0'?>")
        output.append("<map>")

        # Recursively print the nodes
        output += self._exportToFreemindXML(actionsList)

        # Finish the map
        output.append("</map>")

        return output

    def _exportToFreemindXML(self, actionslist):
        # Prime array of output lines
        output = []

        level = int(self.data["level"])

        if level > -1:
            # Print this node
            colour = self.data["colour"]
            cell = self.data["cell"]
            note = self.data["note"]
            indent = "  " * (level + 1)
            if colour == "":
                output.append(indent + "<node TEXT='" + cell + "'>")
            else:
                output.append(
                    indent
                    + "<node BACKGROUND_COLOR='#"
                    + colour
                    + "' TEXT='"
                    + cell
                    + "'>"
                )

            if note:
                output.append(indent + "  <richcontent TYPE='NOTE'><html>")
                output.append(indent + "    <head></head>")
                output.append(indent + "    <body>")
                output.append(indent + "      <p>")
                output.append(indent + "        " + note)
                output.append(indent + "      </p>")
                output.append(indent + "    </body>")

                output.append(indent + "  </richcontent>")

        for childNode in self.childNodes:
            output += childNode._exportToFreemindXML(actionsList)

        if level > -1:
            output.append(indent + "</node>")

        return output

    def exportToOPMLXML(self, actionsList):
        # Export to OPML XML

        # Prime array of output lines
        output = []

        # Warn if resulting XML would produce more than 1 Level 0 node
        if len(self.childNodes) > 1:
            sys.stderr.write(
                "Exported XML will have more than 1 root node. Some programs will get "
                "confused by this. Continuing.\n"
            )

        # Start the map
        output.append("<?xml version='1.0'?>")
        output.append("<opml version='2.0'>")
        output.append("  <head>")
        output.append("  </head>")
        output.append("  <body>")

        # Recursively output.append the nodes
        output += self._exportToOPMLXML(actionsList)

        # Finish the map
        output.append("  </body>")
        output.append("</opml>")

        return output

    def _exportToOPMLXML(self, actionslist):
        # Prime array of output lines
        output = []

        level = int(self.data["level"])

        if level > -1:
            # Print this node
            colour = self.data["colour"]
            cell = self.data["cell"]
            note = self.data["note"]
            indent = "  " * (level + 2)
            if colour == "":
                printLine = indent + "<outline text='" + cell + "'"
            else:
                printLine = (
                    indent
                    + "<outline BACKGROUND_COLOR='#"
                    + colour
                    + "' text='"
                    + cell
                    + "'"
                )

            if note:
                printLine = printLine + " Note='" + note + "'"

            output.append(printLine + ">")

        for childNode in self.childNodes:
            output += childNode._exportToOPMLXML(actionsList)

        if level > -1:
            output.append(indent + "</outline>")

        return output

    def processKeep(self, matchCriterion):
        # Reset all nodes' matched flags
        self._markUnmatched()

        # Apply tests to each node and mark it and all its children and ancestors
        # matched
        self._processKeep(matchCriterion)

        # Delete any unmatched nodes
        self._deleteUnmarked()

    def _markUnmatched(self):
        self.matched = False

        for childNode in self.childNodes:
            childNode._markUnmatched()

    def _markAncestorsMatched(self):
        if self.data["level"] > -1:
            self.matched = True
            self.parent._markAncestorsMatched()

    def _markSubtreeMatched(self):
        self.matched = True
        for childNode in self.childNodes:
            childNode._markSubtreeMatched()

    def _processKeep(self, matchCriterion):
        if self.isMatch(matchCriterion) is True:
            # mark self and all the ancestors matched
            self._markAncestorsMatched()

            # mark self and its subtree matched
            self._markSubtreeMatched()
        else:
            # Maybe children etc are matches
            for childNode in self.childNodes:
                childNode._processKeep(matchCriterion)

    def _deleteUnmarked(self):
        for childNode in self.childNodes:
            childNode._deleteUnmarked()
        if self.data["level"] > -1:
            if not self.matched:
                self.parent.deleteChild(self)

    def promoteLevel(self, actionslist):
        # Promote everything at the specified level, deleting parents
        promotedLevel = int(actionslist[0])
        if promotedLevel < 1:
            sys.stderr.write("Cannot promote level " + str(promotedLevel) + "\n")
            sys.exit()

        # Get nodes to promote
        nodesToPromote = self.findNodesAtLevel(promotedLevel)

        # Get their parents, removing duplicates
        parentsToDelete = []
        for node in nodesToPromote:
            if node.parent not in parentsToDelete:
                parentsToDelete.append(node.parent)

        # Promote each parent's children
        for parent in parentsToDelete:
            # Insert each child in turn
            parentsParent = parent.parent
            for childNode in parent.childNodes:
                parentsParent.addChild(childNode)

            # Delete the parent
            parentsParent.deleteChild(parent)

        # Repair all the levels in the tree
        self.repairSubtreeLevels(-2)

    def findNodesAtLevel(self, level):
        # returns a list of nodes at a particular level
        return self._findNodesAtLevel(level, [])

    def _findNodesAtLevel(self, level, nodes):
        # recursive helper routine to search the tree for nodes at a certain level
        if self.data["level"] == level:
            nodes.append(self)
        else:
            for childNode in self.childNodes:
                nodes = childNode._findNodesAtLevel(level, nodes)

        return nodes

    def writeStatistics(self, actionslist):
        # Write statistics in one of a number of formats

        # Prime array of output lines
        output = []

        # Get the statistics array
        statistics = self.getStatistics()

        # Output the statistics in the right format
        firstAction = actionslist[0]
        if firstAction == "csv":
            output.append("'Level','Nodes','Distinct Nodes'")

            for level in range(0, 21):
                levelNodes = statistics[0][level]
                if levelNodes == 0:
                    break
                distinctLevelNodes = statistics[1][level]
                output.append(
                    str(level) + "," + str(levelNodes) + "," + str(distinctLevelNodes)
                )
        elif firstAction == "html":
            output.append("<table>")
            output.append(
                "<tr>\n<th>Level</th>\n<th>Nodes</th>\n<th>Distinct Nodes</th>\n</tr>"
            )
            for level in range(0, 21):
                levelNodes = statistics[0][level]
                if levelNodes == 0:
                    break
                distinctLevelNodes = statistics[1][level]
                output.append("<tr>")
                output.append("<td>" + str(level) + "</td>")
                output.append("<td>" + str(levelNodes) + "</td>")
                output.append("<td>" + str(distinctLevelNodes) + "</td>")
                output.append("</tr>")
            output.append("</table>")
        elif firstAction == "markdown":
            output.append("|Level|Nodes|Distinct Nodes|")
            output.append("|-:|-:|-:|")
            for level in range(0, 21):
                levelNodes = statistics[0][level]
                if levelNodes == 0:
                    break
                distinctLevelNodes = statistics[1][level]
                output.append(
                    "|"
                    + str(level)
                    + "|"
                    + str(levelNodes)
                    + "|"
                    + str(distinctLevelNodes)
                    + "|"
                )
        elif firstAction == "text":
            output.append("Level Nodes Distinct Nodes")
            for level in range(0, 21):
                levelNodes = statistics[0][level]
                if levelNodes == 0:
                    break
                distinctLevelNodes = statistics[1][level]
                output.append(
                    str(level).rjust(5, " ")
                    + " "
                    + str(levelNodes).rjust(5, " ")
                    + " "
                    + str(distinctLevelNodes).rjust(14, " ")
                )
        else:
            sys.stderr.write("Invalid format for 'stats': " + firstAction + "\n")

        return output

    def getStatistics(self):
        # Prime statistics arrays
        nodesAtLevel = []
        distinctNodeValuesAtLevel = []
        for level in range(0, 21):
            nodesAtLevel.append(0)
            distinctNodeValuesAtLevel.append([])

        # Walk the tree, updating statistics
        [nodesAtLevel, distinctNodeValuesAtLevel] = self._getStatistics(
            nodesAtLevel, distinctNodeValuesAtLevel
        )

        # Coalesce the sets of node values into counts
        distinctNodesAtLevel = []
        for level in range(0, 21):
            distinctNodesAtLevel.append(len(distinctNodeValuesAtLevel[level]))

        # Return the statistics:
        #    Count of nodes at each level
        #    Count of unique node values at each level
        return [nodesAtLevel, distinctNodesAtLevel]

    def _getStatistics(self, nodesAtLevel, distinctNodeValuesAtLevel):
        level = int(self.data["level"])
        if level > -1:
            # Increment count of nodes at this level
            nodesAtLevel[level] += 1

            # Add cell value to the list for this level - if it's not already in it
            cellValue = self.data["cell"]
            if cellValue not in distinctNodeValuesAtLevel[level]:
                distinctNodeValuesAtLevel[level].append(cellValue)

        for childNode in self.childNodes:
            [nodesAtLevel, distinctNodeValuesAtLevel] = childNode._getStatistics(
                nodesAtLevel, distinctNodeValuesAtLevel
            )

        return [nodesAtLevel, distinctNodeValuesAtLevel]

    def doHorizontalSpread(self, actionsList):
        self._doSpread("horizontal", actionsList)

    def doVerticalSpread(self, actionsList):
        self._doSpread("vertical", actionsList)

    def _doSpread(self, direction, actionsList):
        # Spread the level 0 nodes - either vertically or horizontally

        # Get the increment
        action = actionsList[0]
        if action.isdigit():
            increment = int(action)
        else:
            sys.stderr.write("Increment value is not a positive integer.\n")
            sys.exit()

        # Set iThoughtsSpread variables up
        if direction == "horizontal":
            iThoughtsSpread.resetHorizontalSpread(increment)
            iThoughtsSpread.resetVerticalSpread(0)
        else:
            iThoughtsSpread.resetHorizontalSpread(0)
            iThoughtsSpread.resetVerticalSpread(increment)

        for childNode in self.childNodes:
            childNode.data["position"] = (
                "{"
                + str(iThoughtsSpread.nextHorizontal())
                + ","
                + str(iThoughtsSpread.nextVertical())
                + "}"
            )

    def dump(self, actionsList=None) -> str:
        """
        >>> csv_tree = CSVTree("", "", "", "-1", "", "")
        >>> csv_child = CSVTree("square", "", "", "0", "", "Foo")
        >>> csv_tree.addChild(csv_child)
        >>> print(csv_tree.dump([]).rstrip())
        -1
        square         0        Foo
        """
        s = "".join(f"{str(value)[:10]:<11}" for value in self.data.values())
        s = f"{'  ' * int(self.data['level'])}{s.rstrip()}" + "\n"
        return s + "".join(child.dump(actionsList) for child in self.getChildren())


def formatWhitespaceCharacters(whitespace):
    """
    >>> formatWhitespaceCharacters("a b \\t c d")
    '<tab><space><tab><space><tab><space><tab><space><tab>'
    """
    return "".join("<space>" if c == " " else "<tab>" for c in whitespace)


def writeOutput(printLines):
    for line in printLines:
        print(line, file=sys.stderr)


if __name__ == "__main__":
    iThoughtsColours = iThoughtsColours()

    iThoughtsShapes = iThoughtsShapes()

    iThoughtsSpread = iThoughtsSpread()

    matchCriteria, actionsLists, output = ParameterParser().getParameters()

    writeOutput(output)

    #  Convert stdin data into CSV array - in whatever form it was
    (
        csvRows,
        colourColumn,
        levelColumn,
        noteColumn,
        shapeColumn,
        positionColumn,
        output,
    ) = TreeReader().createCSVArray()

    writeOutput(output)

    # Build the tree from the CSV array
    currentLevel = -1
    csvTree = CSVTree("", "", "", currentLevel, "", "")
    currentNode = csvTree

    for rowNumber, row in enumerate(csvRows[1:]):
        # Extract information from this row
        level = int(row[levelColumn])
        shape = row[shapeColumn]
        colour = row[colourColumn]
        note = row[noteColumn]
        position = row[positionColumn]

        cellValue = row[levelColumn + level + 1]

        if level > currentLevel:
            # New child of previous node
            currentLevel = level
            currentNode = currentNode.addChild(
                CSVTree(shape, colour, note, level, position, cellValue)
            )
        elif level == currentLevel:
            # New sibling of previous node
            currentNode = currentNode.parent.addChild(
                CSVTree(shape, colour, note, level, position, cellValue)
            )
        else:
            # Not a sibling or child of previous node
            currentLevel = level

            # Look for the true parent by backing up the tree
            parentNode = currentNode
            while int(parentNode.data["level"]) >= int(level):
                parentNode = parentNode.parent

            # Add the new node - now we've found the parent to add it to
            currentNode = parentNode.addChild(
                CSVTree(shape, colour, note, level, position, cellValue)
            )

        currentNode.data["row"] = row

    # Apply battery of parameter pairs to do the colouring etc.
    # (A row could match more than one and a later one overrides an earlier one)
    # In some cases the "match criterion" is a command and the "actions list"
    # contains parameters for that command. e.g. "markdown"
    for parmPair, matchCriterion in enumerate(matchCriteria):
        actionsList = actionsLists[parmPair]
        if matchCriterion.pattern.lower() == "dump":
            sys.stderr.write(csvTree.dump(actionsList))
        elif actionsList[0] == "keep":
            csvTree.processKeep(matchCriterion)
        else:
            func = {
                "check": csvTree.checkHierarchy,
                "hspread": csvTree.doHorizontalSpread,
                "html": csvTree.exportToHTML,
                "markdown": csvTree.exportToMarkdown,
                "promote": csvTree.promoteLevel,
                "stats": csvTree.writeStatistics,
                "xml": csvTree.exportToXML,
                "vspread": csvTree.doVerticalSpread,
            }.get(matchCriterion.pattern.lower())
            if func:
                output = func(actionsList)
                if output:
                    print("\n".join(output))
                    sys.exit()
                # commands that generate no output fall through
            else:
                csvTree.applyFilter(matchCriterion, actionsList)

    TreeWriter().writeTreeAsCSV()

#!/usr/bin/env python3
"""
filterCSV - Augments matching CSV files - according to iThoughts requirements

It can set colours for nodes, change their shape, or delete them - if the node
matches any one of a set of criteria the user specifies.

Reads from stdin
Writes to stdout
Commentary to stderr


Command line parameters are pairs of:

1) A specifier. This is a regular expression to match. (A special value 'all' matches
    any value)
2) An action or sequence of actions.

Actions can be:

* A 1 or 2 digit colour number - relative to the top left of iThoughts' colour palette.
* A colour RGB hexadecimal value.  Ex. FFFFFF is black, 000000 is white, FF0000 is red
* 'delete'.
* A shape - as named by iThoughts.
"""

import csv
import re
import sys
from collections import Counter
from string import hexdigits
import xml.etree.ElementTree as ElementTree

# from CSVTree import CSVTree

filterCSV_level = "1.2"
filterCSV_date = "29 April, 2020"


class ParameterParser:
    def getParameters(self):
        matchCriteria = []
        actionsLists = []

        output = []

        # Heading for parameters display
        output.append("\nfilterCSV " + filterCSV_level + " (" + filterCSV_date + ")\n")
        output.append("Criterion".ljust(40, " ") + " Actions")
        output.append("---------".ljust(40, " ") + " -------")

        # Read in pairs of parameters from command line
        parmNumber = 1
        for parmPair in range((len(sys.argv) - 1) // 2):
            # Handle match criterion
            matchCriterion = sys.argv[parmNumber]
            if matchCriterion == "all":
                matchCriteria.append(re.compile(".*"))
            else:
                matchCriteria.append(re.compile(matchCriterion))

            parmNumber += 1

            # Handle the actions that go with this match criterion
            actionsString = sys.argv[parmNumber].lower()
            actionList = actionsString.split()

            # For nextcolour/samecolour and nextshape/sameshape etc rewrite action as
            # the colour number or shape
            for actionNumber in range(len(actionList)):
                action = actionList[actionNumber]
                if action in ("nextcolour", "nextcolor", "nc"):
                    actionList[actionNumber] = str(
                        iThoughtsColours.getNextColourNumber()
                    )
                elif action in ("samecolour", "samecolor", "sc"):
                    actionList[actionNumber] = str(
                        iThoughtsColours.getSameColourNumber()
                    )
                elif action in ("nextshape", "ns"):
                    actionList[actionNumber] = iThoughtsShapes[
                        iThoughtsShapes.getNextShapeNumber()
                    ]
                elif action in ("sameshape", "ss"):
                    actionList[actionNumber] = iThoughtsShapes[
                        iThoughtsShapes.getSameShapeNumber()
                    ]

            # Add actions list to list of actions lists
            actionsLists.append(actionList)

            parmNumber += 1

            output.append(matchCriterion.ljust(40, " ") + " " + actionsString)

        output.append("\n")

        return matchCriteria, actionsLists, output


class TreeReader:
    def detectInputStreamType(self, firstLine):
        firstChar = firstLine[0]

        if firstLine.find("level0") > -1:
            return "iThoughtsCSV"
        elif firstChar == "<":
            return "XML"
        elif firstChar in ["#", "*"]:
            return "markdown"
        elif firstChar == '"':
            return "CSV"
        else:
            return "text"

    def findColumn(this, CSVArray, columnName):
        # Returns 0-indexed column if the column name appears in the first LineEnd
        # else -1
        if columnName in CSVArray[0]:
            return CSVArray[0].index(columnName)
        else:
            return -1

    def injectColumn(this, CSVArray, columnName):
        # Injects the named column at the beginning of each line,
        # with the heading given by columnName
        for rowNumber, _ in enumerate(CSVArray):
            if rowNumber == 0:
                CSVArray[0].insert(0, columnName)
            else:
                CSVArray[rowNumber].insert(0, "")

        return CSVArray

    def ensureMandatoryColumns(self, csvRows):
        # Add a shape column if one is not already present
        if self.findColumn(csvRows, "shape") == -1:
            csvRows = self.injectColumn(csvRows, "shape")

        # Add a position column if one is not already present
        if self.findColumn(csvRows, "position") == -1:
            csvRows = self.injectColumn(csvRows, "position")

        # Add a note column if one is not already present
        if self.findColumn(csvRows, "note") == -1:
            csvRows = self.injectColumn(csvRows, "note")

        # Add a colour column if one is not already present
        if self.findColumn(csvRows, "colour") == -1:
            csvRows = self.injectColumn(csvRows, "colour")

        # Establish the position of each column - to return
        colourColumn = self.findColumn(csvRows, "colour")
        levelColumn = self.findColumn(csvRows, "level")
        noteColumn = self.findColumn(csvRows, "note")

        shapeColumn = self.findColumn(csvRows, "shape")
        positionColumn = self.findColumn(csvRows, "position")

        return (
            csvRows,
            colourColumn,
            levelColumn,
            noteColumn,
            shapeColumn,
            positionColumn,
        )

    def ensureColumnsPopulated(self, csvRows):
        # Ensures every column in the header row is present in each subsequent row
        minimumColumnCount = len(csvRows[0])
        for rowNumber, row in enumerate(csvRows):
            extraCells = minimumColumnCount - len(row)
            if extraCells > 0:
                csvRows[rowNumber] += [""] * extraCells

        return csvRows

    def readiThoughtsCSVTree(self, inputFile):
        output = []
        # Build array of rows
        csvRows = []
        reader = csv.reader(inputFile)
        for row in reader:
            csvRows.append(row)

        return self.ensureMandatoryColumns(csvRows) + (output,)

    def readMarkdownOrTextTree(self, inputFile):
        output = []

        # Build array of rows
        csvRows = []

        # Work out what an indent would be - in numbers of characters per level
        indent = -1
        for line in inputFile:
            indent = len(line) - len(line.lstrip())
            if indent > 0:
                # This is the first line with whitespace at the beginning

                # Save the indentation whitespace
                indentCharacters = line[0:indent]

                # Print the detected indentation - so user can debug
                output.append(
                    "Indentation detected: "
                    + formatWhitespaceCharacters(indentCharacters)
                )

                break

        # Insert a header row, with attribute columns and a level0 column plus other
        # levels
        csvRows.append(
            [
                "level",
                "level0",
                "level1",
                "level2",
                "level3",
                "level4",
                "level5",
                "level6",
                "level7",
                "level8",
                "level9",
                "level10",
                "level11",
                "level12",
                "level13",
                "level14",
                "level15",
                "level16",
                "level17",
                "level18",
                "level19",
                "level20",
            ],
        )

        # Add a level column if one is not already present, and move the text into the
        # right level - for each data line
        for lineNumber, line in enumerate(inputFile):
            # A data line so work out how many levels deep it is indented
            lineIndent = len(line) - len(line.lstrip())
            if lineIndent % indent > 0:
                if lineIndent == 1:
                    output.append("Bad indentation: 1 white space character.")
                else:
                    output.append(
                        "Bad indentation:"
                        + str(lineIndent)
                        + " white space characters."
                    )

                output.append(
                    "Should be multiple of "
                    + str(indent)
                    + ". Rounding level down to "
                    + str(lineIndent / indent)
                    + "."
                )
                output.append("Line in error (" + str(rowNumber) + ") is: " + line)
                output.append(
                    "Leading white space characters: "
                    + formatWhitespaceCharacters(line[0:lineIndent])
                )
            else:
                lineIndentCharacters = line[0:lineIndent]
                if lineIndentCharacters != indentCharacters * (lineIndent // indent):
                    output.append("Bad indentation characters:")
                    output.append("Line in error (" + str(lineNumber) + ") is: " + line)
                    output.append(
                        "Leading white space characters: "
                        + formatWhitespaceCharacters(lineIndentCharacters)
                    )

            newRow = []

            level = lineIndent // indent
            newRow.append(str(level))

            # Insert blank cells - according to level
            for l in range(level):
                newRow.append("")

            # Clean the line text to remove any list item marker
            cleanedLine = line.lstrip().rstrip()
            if cleanedLine[0:2] == "* ":
                cleanedLine = cleanedLine[2:]

            # Add the cleaned line at the appropriate level
            newRow.append(cleanedLine)

            # Add this new row
            csvRows.append(newRow)

        return self.ensureMandatoryColumns(csvRows) + (output,)

    def readXMLTree(self, inputFile):
        output = []

        # Prepare the input text for namespace parsing and XML parsing
        XMLText = "\n".join(inputFile)

        # Create the XML parse tree
        self.XMLTree = ElementTree.fromstring(XMLText)

        # Hunt for the default namespace
        split1 = XMLText.split('xmlns="')
        if len(split1) == 1:
            output.append("No default namespace specification.")
            self.defaultXMLNamespace = ""
        else:
            self.defaultXMLNamespace = split1[1].split('"')[0]
            output.append(f"Default namespace is '{self.defaultXMLNamespace}'")

        # Hunt for other namespaces
        self.XMLNamespaces = {}
        split3 = XMLText.split("xmlns:")
        for fragment in range(len(split3)):
            if fragment > 0:
                split4 = split3[fragment].split('="')
                key = split4[0]
                split5 = split4[1].split('"')
                value = split5[0]
                self.XMLNamespaces[key] = value

        # Build array of rows
        csvRows = []

        # Insert a header row, with attribute columns and a level0 column plus other
        # levels
        csvRows.append(
            [
                "position",
                "colour",
                "shape",
                "level",
                "level0",
                "level1",
                "level2",
                "level3",
                "level4",
                "level5",
                "level6",
                "level7",
                "level8",
                "level9",
                "level10",
                "level11",
                "level12",
                "level13",
                "level14",
                "level15",
                "level16",
                "level17",
                "level18",
                "level19",
                "level20",
            ],
        )

        csvRows += self._readXMLTree(self.XMLTree, 0)

        return self.ensureMandatoryColumns(csvRows) + (output,)

    def resolveNamespaces(self, textToEdit):
        editedText = textToEdit.replace("{" + self.defaultXMLNamespace + "}", "")
        for key in self.XMLNamespaces:
            editedText = editedText.replace(
                "{" + self.XMLNamespaces[key] + "}", key + ":"
            )
        return editedText

    def _readXMLTree(self, XMLNode, level):
        csvRows = []

        # Edit the element tag - in case of namespaces
        editedElementName = self.resolveNamespaces(XMLNode.tag)

        # Create beginning of the row for the node itself - without attributes

        if XMLNode is self.XMLTree:
            # Is root node so create legend rows before it
            csvRows.append(
                [
                    "{-300,-120}",
                    iThoughtsColours.getColour(2),
                    "rectangle",
                    "0",
                    "element",
                ]
            )
            csvRows.append(
                ["{-300,-85}", iThoughtsColours.getColour(8), "rectangle", "0", "value"]
            )
            csvRows.append(
                [
                    "{-300,-50}",
                    iThoughtsColours.getColour(6),
                    "rectangle",
                    "0",
                    "attribute",
                ]
            )

            # Set position of root node to {0,0}
            elementCSVRow = [
                "{0,0}",
                iThoughtsColours.getColour(2),
                "rounded",
                f"{str(level)}",
            ]
        else:
            # Set position to blank for all but the root node
            elementCSVRow = [
                "",
                iThoughtsColours.getColour(2),
                "rounded",
                f"{str(level)}",
            ]

        # Add blank cells to position the cell with the element in
        if level > 0:
            levelBlankCells = [""] * level
            elementCSVRow += levelBlankCells

        # Add the (edited) element name
        elementCSVRow.append(editedElementName)

        # Add the row to the list of rows
        csvRows.append(elementCSVRow)

        for key, value in XMLNode.attrib.items():
            attributeCSVRow = [
                "",
                iThoughtsColours.getColour(6),
                "rounded",
                f"{str(level + 1)}",
            ]
            levelBlankCells = [""] * (level + 1)
            attributeCSVRow += levelBlankCells

            editedAttributeName = self.resolveNamespaces(key)
            attributeCSVRow += [f'{editedAttributeName}="{value}"']
            csvRows.append(attributeCSVRow)
        if XMLNode.text is not None:
            if XMLNode.text.lstrip().rstrip() != "":
                textCSVRow = [
                    "",
                    iThoughtsColours.getColour(8),
                    "rounded",
                    f"{str(level + 1)}",
                ]
                levelBlankCells = [""] * (level + 1)
                textCSVRow += levelBlankCells
                textCSVRow += [f"{XMLNode.text}"]
                csvRows.append(textCSVRow)

        for child in XMLNode:
            csvRows += self._readXMLTree(child, level + 1)

        return csvRows

    def createCSVArray(self):
        output = []

        # Read in whatever's in stdin - which might not be CSV
        inputFile = sys.stdin.readlines()

        # Detect the input stream type
        inputType = self.detectInputStreamType(inputFile[0])

        output.append(f"Input type detected as '{inputType}'.\n")

        # Depending on detected input type create a create CSV-like array,
        # including mandatory columns
        if inputType == "iThoughtsCSV":
            (
                csvRows,
                colourColumn,
                levelColumn,
                noteColumn,
                shapeColumn,
                positionColumn,
                output1,
            ) = self.readiThoughtsCSVTree(inputFile)

            output += output1

            csvRows = self.ensureColumnsPopulated(csvRows)

            return (
                csvRows,
                colourColumn,
                levelColumn,
                noteColumn,
                shapeColumn,
                positionColumn,
                output,
            )
        elif inputType in ["markdown", "text"]:
            (
                csvRows,
                colourColumn,
                levelColumn,
                noteColumn,
                shapeColumn,
                positionColumn,
                output1,
            ) = self.readMarkdownOrTextTree(inputFile)

            output += output1

            csvRows = self.ensureColumnsPopulated(csvRows)

            return (
                csvRows,
                colourColumn,
                levelColumn,
                noteColumn,
                shapeColumn,
                positionColumn,
                output,
            )
        elif inputType == "XML":
            (
                csvRows,
                colourColumn,
                levelColumn,
                noteColumn,
                shapeColumn,
                positionColumn,
                output1,
            ) = self.readXMLTree(inputFile)

            output += output1

            csvRows = self.ensureColumnsPopulated(csvRows)
            if len(self.XMLNamespaces) > 0:
                output.append("Namespace".ljust(20, " ") + " Namespace URL")
                output.append("---------".ljust(20, " ") + " -------------")
                for key in self.XMLNamespaces:
                    output.append(key.ljust(20, " ") + " " + self.XMLNamespaces[key])
                output.append("")

            return (
                csvRows,
                colourColumn,
                levelColumn,
                noteColumn,
                shapeColumn,
                positionColumn,
                output,
            )


class TreeWriter:
    def writeTreeAsCSV(self):

        CSVwriter = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL)

        # Write the header row
        headerRow = ["colour", "note", "position", "shape", "level"]
        levels = csvTree.calculateMaximumLevel() + 1

        for level in range(levels):
            headerRow.append("level" + str(level))

        CSVwriter.writerow(headerRow)

        # Write out the resulting CSV data
        outputArray = []
        outputArray = csvTree.writeCSVTree(outputArray)

        for row in outputArray:
            CSVwriter.writerow(row)


class iThoughtsColours:
    light_pastel = """FFB2B2 FFD8B2 FFFFB2 D8FFB2 B2FFB2 B2FFD8
                      B2FFFF B2D8FF B2B2FF D8B2FF FFB2FF FFB2D8""".split()

    dark_pastel = """B24747 B27C47 B2B247 7CB247 47B247 47B27C
                     47B2B2 477CB2 4747B2 7C47B2 B247B2 B2477C""".split()

    saturated = """FF0000 FF7F00 FFFF00 7FFF00 00FF00 00FF7F
                   00FFFF 007FFF 0000FF 7F00FF FF00FF FF007F""".split()

    grayscale = """000000 929292 a9a9a9 C0C0C0 D6D6D6 FFFFFF""".split()

    solarized = """002b36 073642 586e75 657b83 839496 93a1a1
                   eee8d5 fdf6e3 b58900 cb4b16 dc322f d33682
                   6c71c4 268bd2 2aa198 859900""".split()
    colours = light_pastel + dark_pastel + saturated + grayscale + solarized

    colourNumber = 0

    def __getitem__(self, key):
        """
        >>> ithoughts_colours = iThoughtsColours()
        >>> ithoughts_colours[0]
        'FFB2B2'
        >>> ithoughts_colours[-1]
        '859900'
        """
        return self.colours[key]

    def getColour(self, colourNumber):
        """Get colour based on colour number (which starts at 1)
        >>> ithoughts_colours = iThoughtsColours()
        >>> ithoughts_colours.getColour(1)
        'FFB2B2'
        >>> ithoughts_colours.getColour(58)
        '859900'
        """
        return self[colourNumber - 1]

    def __len__(self):
        """
        >>> len(iThoughtsColours())
        58
        """
        return len(self.colours)

    def getNextColourNumber(self):
        """Get the next colour number from the iThoughts palette
        >>> ithoughts_colours = iThoughtsColours()
        >>> ithoughts_colours.getNextColourNumber()
        1
        """
        self.colourNumber += 1
        return self.colourNumber

    def getSameColourNumber(self):
        """Get the same colour number from the iThoughts palette
        >>> ithoughts_colours = iThoughtsColours()
        >>> ithoughts_colours.getSameColourNumber()
        0
        """
        return self.colourNumber


class iThoughtsShapes:
    # Shapes as understood by iThoughts
    shapes = (
        "auto",
        "rectangle",
        "square",
        "rounded",
        "pill",
        "parallelogram",
        "diamond",
        "triangle",
        "oval",
        "circle",
        "underline",
        "none",
        "square bracket",
        "curved bracket",
    )

    shapeNumber = 0

    def __contains__(self, item):
        """Is item in self.shapes
        >>> ithoughts_shapes = iThoughtsShapes()
        >>> all(shape in ithoughts_shapes for shape
        ...     in ('auto', 'triangle', 'none', 'curved bracket'))
        True
        """
        return item in self.shapes

    def __getitem__(self, key):
        """Get shape based on shape number (which starts at 0)
        >>> ithoughts_shapes = iThoughtsShapes()
        >>> ithoughts_shapes[0]
        'auto'
        >>> ithoughts_shapes[-1]
        'curved bracket'
        """
        return self.shapes[key]

    def __len__(self):
        """
        >>> len(iThoughtsShapes())
        14
        """
        return len(self.shapes)

    def getNextShapeNumber(self):
        """Get the next shape number from the iThoughts list
        >>> ithoughts_shapes = iThoughtsShapes()
        >>> ithoughts_shapes.getNextShapeNumber()
        1
        """
        self.shapeNumber += 1
        return self.shapeNumber

    def getSameShapeNumber(self):
        """Get the same shape number from the iThoughts list
        >>> ithoughts_shapes = iThoughtsShapes()
        >>> ithoughts_shapes.getSameShapeNumber()
        0
        """
        return self.shapeNumber


class iThoughtsSpread:
    horizontalPosition = 0
    horizontalIncrement = 0
    verticalPosition = 0
    verticalIncrement = 0

    def resetHorizontalSpread(self, horizontalIncrement):
        self.horizontalPosition = 0
        self.horizontalIncrement = horizontalIncrement

    def nextHorizontal(self):
        x = self.horizontalPosition
        self.horizontalPosition += self.horizontalIncrement
        return x

    def resetVerticalSpread(self, verticalIncrement):
        self.verticalPosition = 0
        self.verticalIncrement = verticalIncrement

    def nextVertical(self):
        y = self.verticalPosition
        self.verticalPosition += self.verticalIncrement
        return y

    def getNextPosition(self):
        return self.nextHorizontal(), self.nextVertical()


class CSVTree:
    def __init__(self, shape, colour, note, level, position, cell):
        """
        >>> csv_tree = CSVTree("shape", "colour", "note", "level", 1, "cell")
        >>> csv_tree.data["shape"]
        'shape'
        """
        self.childNodes = []
        self.data = {
            "shape": shape,
            "colour": colour,
            "note": note,
            "level": level,
            "position": position,
            "cell": cell,
        }
        self.parent = None
        self.matched = False

    def addChild(self, childNode):
        self.childNodes.append(childNode)
        childNode.parent = self
        return childNode

    def deleteChild(self, childNode):
        self.childNodes.remove(childNode)

    def getChildren(self):
        return self.childNodes

    def isMatch(self, criterion):
        """
        Common method for establishing if a node matches some criterion, usually a regex
        """
        return (
            (re.search(criterion, self.data["shape"]) is not None)
            | (re.search(criterion, self.data["colour"]) is not None)
            | (re.search(criterion, self.data["note"]) is not None)
            | (re.search(criterion, self.data["cell"]) is not None)
        )

    def applyFilter(self, criterion, actionsList):
        """
        Apply filter to this node - if not tree root
        """
        propagate = True
        if self.data["level"] != -1:
            if self.isMatch(criterion):
                # Matched so apply all actions triggered by this match
                for action in actionsList:
                    if action == "delete":
                        # Delete the node
                        self.parent.deleteChild(self)

                        # Don't propagate
                        propagate = False
                    elif action[0] == "{":
                        # position specified
                        self.data["position"] = action

                        # Don't propagate
                        propagate = False
                    elif action == "note":
                        # Document the match in the note field
                        if self.data["note"] != "":
                            self.data["note"] += "\nMatched " + criterion.pattern
                        else:
                            self.data["note"] = "Matched " + criterion.pattern
                    elif action == "noshape":
                        # Remove any shape specification from the matched node
                        self.data["shape"] = ""
                    elif action == "nonote":
                        # Remove any note specification from the matched node
                        self.data["note"] = ""
                    elif action == "noposition":
                        # Remove any position specification from the matched node
                        self.data["position"] = ""
                    elif action == "nocolour":
                        # Remove any colour specification from the matched node
                        self.data["colour"] = ""
                    elif len(action) == 6 and all(c in hexdigits for c in action):
                        # 6-digit hexadecimal so is colour RGB value
                        self.data["colour"] = action
                    elif action in iThoughtsShapes:
                        # Is a shape
                        self.data["shape"] = action
                    elif action.isdigit():
                        # Attempt to parse as from the colour palette
                        colourNumber = int(action)

                        # We have an integer. If it is too big but not 6 digits
                        # we flag an error and don't do the update
                        if colourNumber > len(iThoughtsColours):
                            sys.stderr.write(
                                f"Erroneous colour value {colourNumber} "
                                f"(Pattern was: '{criterion.pattern}')." + "\n"
                            )

                        else:
                            self.data["colour"] = iThoughtsColours.getColour(
                                colourNumber
                            )
                    else:
                        sys.stderr.write(
                            f"Erroneous action value {action} "
                            f"(Pattern was: '{criterion.pattern}')." + "\n"
                        )

        # Apply filter to children, recursively - if propagation is indicated
        if propagate:
            for child in self.childNodes:
                child.applyFilter(criterion, actionsList)

    def writeCSVTree(self, outputArray):
        # Compose this node's line - if not root node
        level = int(self.data["level"])
        if level > -1:
            line = [
                self.data[key] for key in "colour note position shape level".split()
            ]
            line += [""] * int(self.data["level"])
            line.append(self.data["cell"])
            outputArray.append(line)

        # Print children, recursively
        for child in self.childNodes:
            child.writeCSVTree(outputArray)

        return outputArray

    def getCounter(self, counter=None):
        """
        >>> csv_tree = CSVTree("shape", "colour", "note", 1, 1, "cell")
        >>> csv_tree.getCounter()
        Counter({1: 1})
        >>> _ = csv_tree.addChild(CSVTree("shape", "colour", "note", 2, 7, "cell"))
        >>> csv_tree.getCounter()
        Counter({1: 1, 2: 1})
        """
        counter = counter or Counter()
        counter[self.data["level"]] += 1
        for child in self.childNodes:
            child.getCounter(counter)
        return counter

    def checkHierarchy(self, actionsList):
        sys.stderr.write("Beginning Level Check\n")
        sys.stderr.write("---------------------\n")

        self._checkHierarchy(-2, actionsList)

        sys.stderr.write("---------------------\n")
        sys.stderr.write("Completed Level Check\n")

    def _checkHierarchy(self, previousLevel, actionsList):
        level = int(self.data["level"])
        if level > previousLevel + 1:
            sys.stderr.write(self.data["cell"].ljust(40) + " Error: ")
            sys.stderr.write(f"Expected level {previousLevel + 1}.")
            sys.stderr.write(f" Found level {level}. ")

            firstAction = actionsList[0]

            if firstAction in ("repair", "repairnode"):
                # Repair just this node
                self.data["level"] = previousLevel + 1
                sys.stderr.write(
                    f"Repaired, setting level to {previousLevel + 1}." + "\n"
                )
            elif firstAction == "repairsubtree":
                # Repair the whole subtree
                sys.stderr.write(
                    "Repaired this node (setting its level to "
                    f"{previousLevel + 1}) and all its child nodes." + "\n"
                )
                self.repairSubtreeLevels(previousLevel)
            elif firstAction == "stop":
                # Terminate the check
                sys.stderr.write("Terminating.\n")
                sys.exit()
            else:
                sys.stderr.write("\n")
        else:
            if previousLevel > -2:
                sys.stderr.write(self.data["cell"].ljust(40) + " OK:   ")
                sys.stderr.write(f" Found level {level}." + "\n")
        for childNode in self.childNodes:
            childNode._checkHierarchy(previousLevel + 1, actionsList)

    def repairSubtreeLevels(self, parentLevel):
        # Repair this level
        self.data["level"] = parentLevel + 1

        # Repair lower levels
        for childNode in self.childNodes:
            childNode.repairSubtreeLevels(parentLevel + 1)

    def exportToMarkdown(self, actionsList):
        # Get the number of heading levels to allow before going to nested bulleted
        # lists
        headingLevels = int(actionsList[0])

        if len(actionsList) > 1:
            # Get first heading level
            startingLevel = int(actionsList[1])
        else:
            # Default to starting at heading level 1
            startingLevel = 1

        return self._exportToMarkdown(headingLevels - 1, startingLevel)

    def _exportToMarkdown(self, maxHeadingLevel, startingLevel):
        # Prime array of output lines
        output = []

        level = int(self.data["level"])
        if level > -1:
            note = self.data["note"]
            if level > maxHeadingLevel:
                output.append(
                    "  " * (level - maxHeadingLevel - startingLevel + 2)
                    + "* "
                    + self.data["cell"]
                )
                if note:
                    output.append(f"<br/><br/>{note}" + "\n")
            else:
                output.append(
                    "\n"
                    + "#" * (level + startingLevel)
                    + " "
                    + self.data["cell"]
                    + "\n"
                )
                if note:
                    output.append(note + "\n")

        for childNode in self.childNodes:
            output += childNode._exportToMarkdown(maxHeadingLevel, startingLevel)

        return output

    def calculateMaximumLevel(self, level=0):
        """
        maximum level is the max of this level and the levels of all childNodes,
        recursively through their descendents
        """
        thisLevel = int(self.data["level"])
        if thisLevel > level:
            level = thisLevel
        for childNode in self.childNodes:
            level = childNode.calculateMaximumLevel(level)
        return level

    def exportToHTML(self, actionsList):
        action = actionsList[0]

        # Prime array of output lines
        output = []

        if action == "table":
            # Work out how many levels are needed
            maximumLevel = self.calculateMaximumLevel()

            # Write table start
            output.append("<table>")

            # Write table rows
            output += self._exportToHTMLTable(maximumLevel, action)

            # Write table end
            output.append("</table>")
        else:
            # Write top-level list start
            output.append("<ul style='list-style-type: none;'>")

            # Write nested list
            level, freshOutput = self._exportToHTMLList(action)

            output += freshOutput

            # Write top-level list stop
            output.append("</ul>")

        return output

    def _exportToHTMLTable(self, maximumLevel, action):
        # Prime array of output lines
        output = []

        level = int(self.data["level"])
        # Determine the cell background colour
        colour = self.data["colour"] or "FFFFFF"
        # HTML table
        if level > -1:
            # Print table row start
            output.append("<tr>")

            # Print padding empty columns after the cell's column
            if level > 0:
                output.append("<td></td>" * level)

            # Print the cell itself, including styling
            output.append(
                "<td style='border: 1px solid black;border-radius: 5px;"
                + "background-color: #"
                + colour
                + ";'>"
                + self.data["cell"]
                + "</td>"
            )

            # Print padding empty columns after the cell's column
            if level < maximumLevel:
                output.append("<td></td>" * (maximumLevel - level))

            # Print any note as a final column
            note = self.data["note"]
            if note:
                output.append("<td>" + note + "</td>")

            # Print table row end
            output.append("</tr>")

        for childNode in self.childNodes:
            output += childNode._exportToHTMLTable(maximumLevel, action)

        return output

    def _exportToHTMLList(self, action):
        # Prime array of output lines
        output = []

        level = int(self.data["level"])

        # Determine the cell background colour
        colour = self.data["colour"]
        if colour == "":
            colour = "FFFFFF"

        # Determine if there is a note
        note = self.data["note"]

        # HTML nested list
        if level > -1:
            indent = "  " * (level + 1)
            # print list item
            output.append(
                indent
                + "<li style='padding: 10px;'><span style='border: 1px solid "
                + "black;padding: 5px;border-radius: 5px;background-color: #"
                + colour
                + ";'>"
                + self.data["cell"]
                + "</span>"
            )

            # Print any note
            if note:
                output.append(indent + "  <br/><br/>" + note)

            needListElements = len(self.childNodes) > 0
            if needListElements is True:
                output.append(indent + "<ul style='list-style-type: none;'>")

        returnedLevel = -2
        for childNode in self.childNodes:
            returnedLevel, freshOutput = childNode._exportToHTMLList(action)

            output += freshOutput

        if level > -1:
            if needListElements is True:
                output.append(indent + "</ul>")

            output.append(indent + "</li>")

        return [level, output]

    def exportToXML(self, actionsList):
        action = actionsList[0]

        if action == "freemind":
            # Freemind XML export
            return self.exportToFreemindXML(actionsList)
        elif action == "opml":
            return self.exportToOPMLXML(actionsList)

    def exportToFreemindXML(self, actionsList):
        # Export to XML in the format Freemind, MindNode and iThoughts accept

        # Prime array of output lines
        output = []

        # Warn if resulting XML would produce more than 1 Level 0 node
        if len(self.childNodes) > 1:
            sys.stderr.write(
                "Exported XML will have more than 1 root node. Some programs will get "
                "confused by this. Continuing.\n"
            )

        # Start the map
        output.append("<?xml version='1.0'?>")
        output.append("<map>")

        # Recursively print the nodes
        output += self._exportToFreemindXML(actionsList)

        # Finish the map
        output.append("</map>")

        return output

    def _exportToFreemindXML(self, actionslist):
        # Prime array of output lines
        output = []

        level = int(self.data["level"])

        if level > -1:
            # Print this node
            colour = self.data["colour"]
            cell = self.data["cell"]
            note = self.data["note"]
            indent = "  " * (level + 1)
            if colour == "":
                output.append(indent + "<node TEXT='" + cell + "'>")
            else:
                output.append(
                    indent
                    + "<node BACKGROUND_COLOR='#"
                    + colour
                    + "' TEXT='"
                    + cell
                    + "'>"
                )

            if note:
                output.append(indent + "  <richcontent TYPE='NOTE'><html>")
                output.append(indent + "    <head></head>")
                output.append(indent + "    <body>")
                output.append(indent + "      <p>")
                output.append(indent + "        " + note)
                output.append(indent + "      </p>")
                output.append(indent + "    </body>")

                output.append(indent + "  </richcontent>")

        for childNode in self.childNodes:
            output += childNode._exportToFreemindXML(actionsList)

        if level > -1:
            output.append(indent + "</node>")

        return output

    def exportToOPMLXML(self, actionsList):
        # Export to OPML XML

        # Prime array of output lines
        output = []

        # Warn if resulting XML would produce more than 1 Level 0 node
        if len(self.childNodes) > 1:
            sys.stderr.write(
                "Exported XML will have more than 1 root node. Some programs will get "
                "confused by this. Continuing.\n"
            )

        # Start the map
        output.append("<?xml version='1.0'?>")
        output.append("<opml version='2.0'>")
        output.append("  <head>")
        output.append("  </head>")
        output.append("  <body>")

        # Recursively output.append the nodes
        output += self._exportToOPMLXML(actionsList)

        # Finish the map
        output.append("  </body>")
        output.append("</opml>")

        return output

    def _exportToOPMLXML(self, actionslist):
        # Prime array of output lines
        output = []

        level = int(self.data["level"])

        if level > -1:
            # Print this node
            colour = self.data["colour"]
            cell = self.data["cell"]
            note = self.data["note"]
            indent = "  " * (level + 2)
            if colour == "":
                printLine = indent + "<outline text='" + cell + "'"
            else:
                printLine = (
                    indent
                    + "<outline BACKGROUND_COLOR='#"
                    + colour
                    + "' text='"
                    + cell
                    + "'"
                )

            if note:
                printLine = printLine + " Note='" + note + "'"

            output.append(printLine + ">")

        for childNode in self.childNodes:
            output += childNode._exportToOPMLXML(actionsList)

        if level > -1:
            output.append(indent + "</outline>")

        return output

    def processKeep(self, matchCriterion):
        # Reset all nodes' matched flags
        self._markUnmatched()

        # Apply tests to each node and mark it and all its children and ancestors
        # matched
        self._processKeep(matchCriterion)

        # Delete any unmatched nodes
        self._deleteUnmarked()

    def _markUnmatched(self):
        self.matched = False

        for childNode in self.childNodes:
            childNode._markUnmatched()

    def _markAncestorsMatched(self):
        if self.data["level"] > -1:
            self.matched = True
            self.parent._markAncestorsMatched()

    def _markSubtreeMatched(self):
        self.matched = True
        for childNode in self.childNodes:
            childNode._markSubtreeMatched()

    def _processKeep(self, matchCriterion):
        if self.isMatch(matchCriterion) is True:
            # mark self and all the ancestors matched
            self._markAncestorsMatched()

            # mark self and its subtree matched
            self._markSubtreeMatched()
        else:
            # Maybe children etc are matches
            for childNode in self.childNodes:
                childNode._processKeep(matchCriterion)

    def _deleteUnmarked(self):
        for childNode in self.childNodes:
            childNode._deleteUnmarked()
        if self.data["level"] > -1:
            if not self.matched:
                self.parent.deleteChild(self)

    def promoteLevel(self, actionslist):
        # Promote everything at the specified level, deleting parents
        promotedLevel = int(actionslist[0])
        if promotedLevel < 1:
            sys.stderr.write("Cannot promote level " + str(promotedLevel) + "\n")
            sys.exit()

        # Get nodes to promote
        nodesToPromote = self.findNodesAtLevel(promotedLevel)

        # Get their parents, removing duplicates
        parentsToDelete = []
        for node in nodesToPromote:
            if node.parent not in parentsToDelete:
                parentsToDelete.append(node.parent)

        # Promote each parent's children
        for parent in parentsToDelete:
            # Insert each child in turn
            parentsParent = parent.parent
            for childNode in parent.childNodes:
                parentsParent.addChild(childNode)

            # Delete the parent
            parentsParent.deleteChild(parent)

        # Repair all the levels in the tree
        self.repairSubtreeLevels(-2)

    def findNodesAtLevel(self, level):
        # returns a list of nodes at a particular level
        return self._findNodesAtLevel(level, [])

    def _findNodesAtLevel(self, level, nodes):
        # recursive helper routine to search the tree for nodes at a certain level
        if self.data["level"] == level:
            nodes.append(self)
        else:
            for childNode in self.childNodes:
                nodes = childNode._findNodesAtLevel(level, nodes)

        return nodes

    def writeStatistics(self, actionslist):
        # Write statistics in one of a number of formats

        # Prime array of output lines
        output = []

        # Get the statistics array
        statistics = self.getStatistics()

        # Output the statistics in the right format
        firstAction = actionslist[0]
        if firstAction == "csv":
            output.append("'Level','Nodes','Distinct Nodes'")

            for level in range(0, 21):
                levelNodes = statistics[0][level]
                if levelNodes == 0:
                    break
                distinctLevelNodes = statistics[1][level]
                output.append(
                    str(level) + "," + str(levelNodes) + "," + str(distinctLevelNodes)
                )
        elif firstAction == "html":
            output.append("<table>")
            output.append(
                "<tr>\n<th>Level</th>\n<th>Nodes</th>\n<th>Distinct Nodes</th>\n</tr>"
            )
            for level in range(0, 21):
                levelNodes = statistics[0][level]
                if levelNodes == 0:
                    break
                distinctLevelNodes = statistics[1][level]
                output.append("<tr>")
                output.append("<td>" + str(level) + "</td>")
                output.append("<td>" + str(levelNodes) + "</td>")
                output.append("<td>" + str(distinctLevelNodes) + "</td>")
                output.append("</tr>")
            output.append("</table>")
        elif firstAction == "markdown":
            output.append("|Level|Nodes|Distinct Nodes|")
            output.append("|-:|-:|-:|")
            for level in range(0, 21):
                levelNodes = statistics[0][level]
                if levelNodes == 0:
                    break
                distinctLevelNodes = statistics[1][level]
                output.append(
                    "|"
                    + str(level)
                    + "|"
                    + str(levelNodes)
                    + "|"
                    + str(distinctLevelNodes)
                    + "|"
                )
        elif firstAction == "text":
            output.append("Level Nodes Distinct Nodes")
            for level in range(0, 21):
                levelNodes = statistics[0][level]
                if levelNodes == 0:
                    break
                distinctLevelNodes = statistics[1][level]
                output.append(
                    str(level).rjust(5, " ")
                    + " "
                    + str(levelNodes).rjust(5, " ")
                    + " "
                    + str(distinctLevelNodes).rjust(14, " ")
                )
        else:
            sys.stderr.write("Invalid format for 'stats': " + firstAction + "\n")

        return output

    def getStatistics(self):
        # Prime statistics arrays
        nodesAtLevel = []
        distinctNodeValuesAtLevel = []
        for level in range(0, 21):
            nodesAtLevel.append(0)
            distinctNodeValuesAtLevel.append([])

        # Walk the tree, updating statistics
        [nodesAtLevel, distinctNodeValuesAtLevel] = self._getStatistics(
            nodesAtLevel, distinctNodeValuesAtLevel
        )

        # Coalesce the sets of node values into counts
        distinctNodesAtLevel = []
        for level in range(0, 21):
            distinctNodesAtLevel.append(len(distinctNodeValuesAtLevel[level]))

        # Return the statistics:
        #    Count of nodes at each level
        #    Count of unique node values at each level
        return [nodesAtLevel, distinctNodesAtLevel]

    def _getStatistics(self, nodesAtLevel, distinctNodeValuesAtLevel):
        level = int(self.data["level"])
        if level > -1:
            # Increment count of nodes at this level
            nodesAtLevel[level] += 1

            # Add cell value to the list for this level - if it's not already in it
            cellValue = self.data["cell"]
            if cellValue not in distinctNodeValuesAtLevel[level]:
                distinctNodeValuesAtLevel[level].append(cellValue)

        for childNode in self.childNodes:
            [nodesAtLevel, distinctNodeValuesAtLevel] = childNode._getStatistics(
                nodesAtLevel, distinctNodeValuesAtLevel
            )

        return [nodesAtLevel, distinctNodeValuesAtLevel]

    def doHorizontalSpread(self, actionsList):
        self._doSpread("horizontal", actionsList)

    def doVerticalSpread(self, actionsList):
        self._doSpread("vertical", actionsList)

    def _doSpread(self, direction, actionsList):
        # Spread the level 0 nodes - either vertically or horizontally

        # Get the increment
        action = actionsList[0]
        if action.isdigit():
            increment = int(action)
        else:
            sys.stderr.write("Increment value is not a positive integer.\n")
            sys.exit()

        # Set iThoughtsSpread variables up
        if direction == "horizontal":
            iThoughtsSpread.resetHorizontalSpread(increment)
            iThoughtsSpread.resetVerticalSpread(0)
        else:
            iThoughtsSpread.resetHorizontalSpread(0)
            iThoughtsSpread.resetVerticalSpread(increment)

        for childNode in self.childNodes:
            childNode.data["position"] = (
                "{"
                + str(iThoughtsSpread.nextHorizontal())
                + ","
                + str(iThoughtsSpread.nextVertical())
                + "}"
            )

    def dump(self, actionsList=None) -> str:
        """
        >>> csv_tree = CSVTree("", "", "", "-1", "", "")
        >>> csv_child = CSVTree("square", "", "", "0", "", "Foo")
        >>> csv_tree.addChild(csv_child)
        >>> print(csv_tree.dump([]).rstrip())
        -1
        square         0        Foo
        """
        s = "".join(f"{str(value)[:10]:<11}" for value in self.data.values())
        s = f"{'  ' * int(self.data['level'])}{s.rstrip()}" + "\n"
        return s + "".join(child.dump(actionsList) for child in self.getChildren())


def formatWhitespaceCharacters(whitespace):
    """
    >>> formatWhitespaceCharacters("a b \\t c d")
    '<tab><space><tab><space><tab><space><tab><space><tab>'
    """
    return "".join("<space>" if c == " " else "<tab>" for c in whitespace)


def writeOutput(printLines):
    for line in printLines:
        print(line, file=sys.stderr)


if __name__ == "__main__":
    iThoughtsColours = iThoughtsColours()

    iThoughtsShapes = iThoughtsShapes()

    iThoughtsSpread = iThoughtsSpread()

    matchCriteria, actionsLists, output = ParameterParser().getParameters()

    writeOutput(output)

    #  Convert stdin data into CSV array - in whatever form it was
    (
        csvRows,
        colourColumn,
        levelColumn,
        noteColumn,
        shapeColumn,
        positionColumn,
        output,
    ) = TreeReader().createCSVArray()

    writeOutput(output)

    # Build the tree from the CSV array
    currentLevel = -1
    csvTree = CSVTree("", "", "", currentLevel, "", "")
    currentNode = csvTree

    for rowNumber, row in enumerate(csvRows[1:]):
        # Extract information from this row
        level = int(row[levelColumn])
        shape = row[shapeColumn]
        colour = row[colourColumn]
        note = row[noteColumn]
        position = row[positionColumn]

        cellValue = row[levelColumn + level + 1]

        if level > currentLevel:
            # New child of previous node
            currentLevel = level
            currentNode = currentNode.addChild(
                CSVTree(shape, colour, note, level, position, cellValue)
            )
        elif level == currentLevel:
            # New sibling of previous node
            currentNode = currentNode.parent.addChild(
                CSVTree(shape, colour, note, level, position, cellValue)
            )
        else:
            # Not a sibling or child of previous node
            currentLevel = level

            # Look for the true parent by backing up the tree
            parentNode = currentNode
            while int(parentNode.data["level"]) >= int(level):
                parentNode = parentNode.parent

            # Add the new node - now we've found the parent to add it to
            currentNode = parentNode.addChild(
                CSVTree(shape, colour, note, level, position, cellValue)
            )

        currentNode.data["row"] = row

    # Apply battery of parameter pairs to do the colouring etc.
    # (A row could match more than one and a later one overrides an earlier one)
    # In some cases the "match criterion" is a command and the "actions list"
    # contains parameters for that command. e.g. "markdown"
    for parmPair, matchCriterion in enumerate(matchCriteria):
        actionsList = actionsLists[parmPair]
        if matchCriterion.pattern.lower() == "dump":
            sys.stderr.write(csvTree.dump(actionsList))
        elif actionsList[0] == "keep":
            csvTree.processKeep(matchCriterion)
        else:
            func = {
                "check": csvTree.checkHierarchy,
                "hspread": csvTree.doHorizontalSpread,
                "html": csvTree.exportToHTML,
                "markdown": csvTree.exportToMarkdown,
                "promote": csvTree.promoteLevel,
                "stats": csvTree.writeStatistics,
                "xml": csvTree.exportToXML,
                "vspread": csvTree.doVerticalSpread,
            }.get(matchCriterion.pattern.lower())
            if func:
                output = func(actionsList)
                if output:
                    print("\n".join(output))
                    sys.exit()
                # commands that generate no output fall through
            else:
                csvTree.applyFilter(matchCriterion, actionsList)

    TreeWriter().writeTreeAsCSV()

#!/usr/bin/env python3 filterCSV - Augments matching CSV files - according to iThoughts requirements  It can set colours for nodes, change their shape, or delete them - if the node matches any one of a set of criteria the user specifies.  Reads from stdin Writes to stdout Commentary to stderr   Command line parameters are pairs of:  1) A specifier. This is a regular expression to match. (A special value 'all' matches     any value) 2) An action or sequence of actions.  Actions can be:  * A 1 or 2 digit colour number - relative to the top left of iThoughts' colour palette. * A colour RGB hexadecimal value.  Ex. FFFFFF is black, 000000 is white, FF0000 is red * 'delete'. * A shape - as named by iThoughts. # from CSVTree import CSVTree # Heading for parameters display # Read in pairs of parameters from command line # Handle match criterion # Handle the actions that go with this match criterion # For nextcolour/samecolour and nextshape/sameshape etc rewrite action as # the colour number or shape # Add actions list to list of actions lists # Returns 0-indexed column if the column name appears in the first LineEnd # else -1 # Injects the named column at the beginning of each line, # with the heading given by columnName # Add a shape column if one is not already present # Add a position column if one is not already present # Add a note column if one is not already present # Add a colour column if one is not already present # Establish the position of each column - to return # Ensures every column in the header row is present in each subsequent row # Build array of rows # Build array of rows # Work out what an indent would be - in numbers of characters per level # This is the first line with whitespace at the beginning # Save the indentation whitespace # Print the detected indentation - so user can debug # Insert a header row, with attribute columns and a level0 column plus other # levels # Add a level column if one is not already present, and move the text into the # right level - for each data line # A data line so work out how many levels deep it is indented # Insert blank cells - according to level # Clean the line text to remove any list item marker # Add the cleaned line at the appropriate level # Add this new row # Prepare the input text for namespace parsing and XML parsing # Create the XML parse tree # Hunt for the default namespace # Hunt for other namespaces # Build array of rows # Insert a header row, with attribute columns and a level0 column plus other # levels # Edit the element tag - in case of namespaces # Create beginning of the row for the node itself - without attributes # Is root node so create legend rows before it # Set position of root node to {0,0} # Set position to blank for all but the root node # Add blank cells to position the cell with the element in # Add the (edited) element name # Add the row to the list of rows # Read in whatever's in stdin - which might not be CSV # Detect the input stream type # Depending on detected input type create a create CSV-like array, # including mandatory columns # Write the header row # Write out the resulting CSV data FFB2B2 FFD8B2 FFFFB2 D8FFB2 B2FFB2 B2FFD8                       B2FFFF B2D8FF B2B2FF D8B2FF FFB2FF FFB2D8 B24747 B27C47 B2B247 7CB247 47B247 47B27C                      47B2B2 477CB2 4747B2 7C47B2 B247B2 B2477C FF0000 FF7F00 FFFF00 7FFF00 00FF00 00FF7F                    00FFFF 007FFF 0000FF 7F00FF FF00FF FF007F 000000 929292 a9a9a9 C0C0C0 D6D6D6 FFFFFF 002b36 073642 586e75 657b83 839496 93a1a1                    eee8d5 fdf6e3 b58900 cb4b16 dc322f d33682                    6c71c4 268bd2 2aa198 859900 >>> ithoughts_colours = iThoughtsColours()         >>> ithoughts_colours[0]         'FFB2B2'         >>> ithoughts_colours[-1]         '859900' Get colour based on colour number (which starts at 1)         >>> ithoughts_colours = iThoughtsColours()         >>> ithoughts_colours.getColour(1)         'FFB2B2'         >>> ithoughts_colours.getColour(58)         '859900' >>> len(iThoughtsColours())         58 Get the next colour number from the iThoughts palette         >>> ithoughts_colours = iThoughtsColours()         >>> ithoughts_colours.getNextColourNumber()         1 Get the same colour number from the iThoughts palette         >>> ithoughts_colours = iThoughtsColours()         >>> ithoughts_colours.getSameColourNumber()         0 # Shapes as understood by iThoughts Is item in self.shapes         >>> ithoughts_shapes = iThoughtsShapes()         >>> all(shape in ithoughts_shapes for shape         ...     in ('auto', 'triangle', 'none', 'curved bracket'))         True Get shape based on shape number (which starts at 0)         >>> ithoughts_shapes = iThoughtsShapes()         >>> ithoughts_shapes[0]         'auto'         >>> ithoughts_shapes[-1]         'curved bracket' >>> len(iThoughtsShapes())         14 Get the next shape number from the iThoughts list         >>> ithoughts_shapes = iThoughtsShapes()         >>> ithoughts_shapes.getNextShapeNumber()         1 Get the same shape number from the iThoughts list         >>> ithoughts_shapes = iThoughtsShapes()         >>> ithoughts_shapes.getSameShapeNumber()         0 >>> csv_tree = CSVTree("shape", "colour", "note", "level", 1, "cell")         >>> csv_tree.data["shape"]         'shape' Common method for establishing if a node matches some criterion, usually a regex Apply filter to this node - if not tree root # Matched so apply all actions triggered by this match # Delete the node # Don't propagate # position specified # Don't propagate # Document the match in the note field # Remove any shape specification from the matched node # Remove any note specification from the matched node # Remove any position specification from the matched node # Remove any colour specification from the matched node # 6-digit hexadecimal so is colour RGB value # Is a shape # Attempt to parse as from the colour palette # We have an integer. If it is too big but not 6 digits # we flag an error and don't do the update # Apply filter to children, recursively - if propagation is indicated # Compose this node's line - if not root node # Print children, recursively >>> csv_tree = CSVTree("shape", "colour", "note", 1, 1, "cell")         >>> csv_tree.getCounter()         Counter({1: 1})         >>> _ = csv_tree.addChild(CSVTree("shape", "colour", "note", 2, 7, "cell"))         >>> csv_tree.getCounter()         Counter({1: 1, 2: 1}) # Repair just this node # Repair the whole subtree # Terminate the check # Repair this level # Repair lower levels # Get the number of heading levels to allow before going to nested bulleted # lists # Get first heading level # Default to starting at heading level 1 # Prime array of output lines maximum level is the max of this level and the levels of all childNodes,         recursively through their descendents # Prime array of output lines # Work out how many levels are needed # Write table start # Write table rows # Write table end # Write top-level list start # Write nested list # Write top-level list stop # Prime array of output lines # Determine the cell background colour # HTML table # Print table row start # Print padding empty columns after the cell's column # Print the cell itself, including styling #" # Print padding empty columns after the cell's column # Print any note as a final column # Print table row end # Prime array of output lines # Determine the cell background colour # Determine if there is a note # HTML nested list # print list item #" # Print any note # Freemind XML export # Export to XML in the format Freemind, MindNode and iThoughts accept # Prime array of output lines # Warn if resulting XML would produce more than 1 Level 0 node # Start the map # Recursively print the nodes # Finish the map # Prime array of output lines # Print this node # Export to OPML XML # Prime array of output lines # Warn if resulting XML would produce more than 1 Level 0 node # Start the map # Recursively output.append the nodes # Finish the map # Prime array of output lines # Print this node # Reset all nodes' matched flags # Apply tests to each node and mark it and all its children and ancestors # matched # Delete any unmatched nodes # mark self and all the ancestors matched # mark self and its subtree matched # Maybe children etc are matches # Promote everything at the specified level, deleting parents # Get nodes to promote # Get their parents, removing duplicates # Promote each parent's children # Insert each child in turn # Delete the parent # Repair all the levels in the tree # returns a list of nodes at a particular level # recursive helper routine to search the tree for nodes at a certain level # Write statistics in one of a number of formats # Prime array of output lines # Get the statistics array # Output the statistics in the right format # Prime statistics arrays # Walk the tree, updating statistics # Coalesce the sets of node values into counts # Return the statistics: #    Count of nodes at each level #    Count of unique node values at each level # Increment count of nodes at this level # Add cell value to the list for this level - if it's not already in it # Spread the level 0 nodes - either vertically or horizontally # Get the increment # Set iThoughtsSpread variables up >>> csv_tree = CSVTree("", "", "", "-1", "", "")         >>> csv_child = CSVTree("square", "", "", "0", "", "Foo")         >>> csv_tree.addChild(csv_child)         >>> print(csv_tree.dump([]).rstrip())         -1         square         0        Foo >>> formatWhitespaceCharacters("a b \\t c d")     '<tab><space><tab><space><tab><space><tab><space><tab>' #  Convert stdin data into CSV array - in whatever form it was # Build the tree from the CSV array # Extract information from this row # New child of previous node # New sibling of previous node # Not a sibling or child of previous node # Look for the true parent by backing up the tree # Add the new node - now we've found the parent to add it to # Apply battery of parameter pairs to do the colouring etc. # (A row could match more than one and a later one overrides an earlier one) # In some cases the "match criterion" is a command and the "actions list" # contains parameters for that command. e.g. "markdown" # commands that generate no output fall through