make a single index file with 1 record per word which shows the word, file and linenums
        # storms, knowledge.csv - 3
        # string, rawData.csv - 1
        # structure, EVENT_SYSTEM-PC-FILE.CSV - 18, OBJECT_SYSTEM-PC-FILE.CSV - 4, sample-filelist-for-AIKIF.csv - 18 18
        # subgoal, goals.csv - 3 4 12 13 14, goal_types.csv - 8
        # subgoals, goal_types.csv - 9
        # subgoals;, goals.csv - 2

def consolidate(ipFile, opFile):
    """
    make a single index file with 1 record per word which shows the word, file and linenums
        # storms, knowledge.csv - 3
        # string, rawData.csv - 1
        # structure, EVENT_SYSTEM-PC-FILE.CSV - 18, OBJECT_SYSTEM-PC-FILE.CSV - 4, sample-filelist-for-AIKIF.csv - 18 18
        # subgoal, goals.csv - 3 4 12 13 14, goal_types.csv - 8
        # subgoals, goal_types.csv - 9
        # subgoals;, goals.csv - 2
    """
    curFile = ''
    curWord = ''
    curLineNums = ''
    indexedWords = {}
    with open(ipFile, "r", encoding='utf-8', errors='replace') as ip:
        for line in ip:
            cols = line.split(',')
            curFile = cols[0]
            curWord = cols[1]
            curLineNums = cols[2].strip()
            #DebugIndexing(curFile, curWord, curLineNums, line)
            if curWord in indexedWords:
                indexedWords[curWord] =  indexedWords[curWord] + ', ' + curFile + ' - ' + curLineNums
            else:
                indexedWords[curWord] = curFile + ' - ' + curLineNums
    with open(opFile, "w",encoding='utf-8', errors='replace') as op:
        op.write('word, filenames\n')  # this shows which words appear in which files
        word_keys = indexedWords.keys()
        for word in sorted(word_keys):
            if word != '':
                line_nums = indexedWords[word]
                op.write(word + ', ')
                for line_num in line_nums:
                    op.write(str(line_num))
            op.write('\n')