"""
Convert text files (.txt) to SCID (.scid) files to be used by SierraChart
Creates a new scid for every text file found in specified directory

Format (in .txt file) should be:    D,T,O,H,L,C,V,T

That is:    Date, Time, Open, High, Low, Close, Volume, Trades

Take note of the comma separation

Text files will be actively checked for updates every REFRESH seconds
"""

from datetime import datetime as dt
from time import sleep
from os import listdir
import struct

# SET THESE VARIABLES
# Convert the files in this folde to .scid files...
FILES_IN = "/home/sc/PRE-SCID"
# ...and output them here
FILES_OUT = "/home/sc/NOW-SCID"

# Set to True to continue updating (text) files or False to not
KEEP_UPDATING = True

# Set to how often to check if files should be updated
REFRESH = 0.25
# Set the extension of files to convert in FILES_IN (usually .csv or .txt)
FILES_IN_EXT = ".txt"

sizeHeader = 0x38
sizeRecord = 0x28
file_header = b"SCID8\x00\x00\x00\x28\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"


def excel_date(datetime):
    delta = datetime - dt(1899, 12, 30)
    return delta.days + delta.seconds / 86400


def write_struct(record, f):
    if len(record) == 8 and record[0][0].isnumeric():
        out = [excel_date(dt.strptime(
            f"{record[0]} {record[1]}", "%Y-%m-%d %H:%M:%S.%f"))]
        out.extend([float(s) for s in record[2:]])
        # note that SC packs dt, O, H, L, C, T, V not V, T so swap them
        wt = struct.pack("d4f4I", out[0], out[1], out[2], out[3], out[4], int(
            out[6]), int(out[5]), 0, 0)
        f.write(wt)


def create_base(f):
    record = None
    with open(FILES_OUT + f[:-3] + "scid", "wb") as scidfile:
        scidfile.write(file_header)

        with open(FILES_IN + f, "r") as txtfile:
            for line in txtfile:
                record = line.split(",")
                write_struct(record, scidfile)

    return f"{record[0]} {record[1]}"  # will update when date+time changes


def update_if_changed(files):
    for f in files:
        with open(FILES_IN + f, "r") as txtfile:
            record = txtfile.readlines()[-1].split(",")
            last_dt = f"{record[0]} {record[1]}"
            if files[f] == last_dt:
                continue
            files[f] = last_dt
            with open(FILES_OUT + f[: -3] + "scid", "ab") as scidfile:
                write_struct(record, scidfile)


if __name__ == "__main__":
    files = sorted(x for x in listdir(path=FILES_IN) if x[-4:] == FILES_IN_EXT)

    files = {x: "" for x in files}

    for f in files:
        files[f] = create_base(f)

    print("Created bases")
    if KEEP_UPDATING:
        print(f"Looking for updates every {REFRESH}s")
        # Test each input file every REFRESH seconds and update if the datetime has changed
        while True:
            sleep(REFRESH)
            update_if_changed(files)

"""
Convert text files (.txt) to SCID (.scid) files to be used by SierraChart
Creates a new scid for every text file found in specified directory

Format (in .txt file) should be:    D,T,O,H,L,C,V,T

That is:    Date, Time, Open, High, Low, Close, Volume, Trades

Take note of the comma separation

Text files will be actively checked for updates every REFRESH seconds
"""

from datetime import datetime as dt
from time import sleep
from os import listdir
import struct

# SET THESE VARIABLES
# Convert the files in this folde to .scid files...
FILES_IN = "/home/sc/PRE-SCID"
# ...and output them here
FILES_OUT = "/home/sc/NOW-SCID"

# Set to True to continue updating (text) files or False to not
KEEP_UPDATING = True

# Set to how often to check if files should be updated
REFRESH = 0.25
# Set the extension of files to convert in FILES_IN (usually .csv or .txt)
FILES_IN_EXT = ".txt"

sizeHeader = 0x38
sizeRecord = 0x28
file_header = b"SCID8\x00\x00\x00\x28\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"


def excel_date(datetime):
    delta = datetime - dt(1899, 12, 30)
    return delta.days + delta.seconds / 86400


def write_struct(record, f):
    if len(record) == 8 and record[0][0].isnumeric():
        out = [excel_date(dt.strptime(
            f"{record[0]} {record[1]}", "%Y-%m-%d %H:%M:%S.%f"))]
        out.extend([float(s) for s in record[2:]])
        # note that SC packs dt, O, H, L, C, T, V not V, T so swap them
        wt = struct.pack("d4f4I", out[0], out[1], out[2], out[3], out[4], int(
            out[6]), int(out[5]), 0, 0)
        f.write(wt)


def create_base(f):
    record = None
    with open(FILES_OUT + f[:-3] + "scid", "wb") as scidfile:
        scidfile.write(file_header)

        with open(FILES_IN + f, "r") as txtfile:
            for line in txtfile:
                record = line.split(",")
                write_struct(record, scidfile)

    return f"{record[0]} {record[1]}"  # will update when date+time changes


def update_if_changed(files):
    for f in files:
        with open(FILES_IN + f, "r") as txtfile:
            record = txtfile.readlines()[-1].split(",")
            last_dt = f"{record[0]} {record[1]}"
            if files[f] == last_dt:
                continue
            files[f] = last_dt
            with open(FILES_OUT + f[: -3] + "scid", "ab") as scidfile:
                write_struct(record, scidfile)


if __name__ == "__main__":
    files = sorted(x for x in listdir(path=FILES_IN) if x[-4:] == FILES_IN_EXT)

    files = {x: "" for x in files}

    for f in files:
        files[f] = create_base(f)

    print("Created bases")
    if KEEP_UPDATING:
        print(f"Looking for updates every {REFRESH}s")
        # Test each input file every REFRESH seconds and update if the datetime has changed
        while True:
            sleep(REFRESH)
            update_if_changed(files)

Convert text files (.txt) to SCID (.scid) files to be used by SierraChart Creates a new scid for every text file found in specified directory  Format (in .txt file) should be:    D,T,O,H,L,C,V,T  That is:    Date, Time, Open, High, Low, Close, Volume, Trades  Take note of the comma separation  Text files will be actively checked for updates every REFRESH seconds # SET THESE VARIABLES # Convert the files in this folde to .scid files... # ...and output them here # Set to True to continue updating (text) files or False to not # Set to how often to check if files should be updated # Set the extension of files to convert in FILES_IN (usually .csv or .txt) # note that SC packs dt, O, H, L, C, T, V not V, T so swap them # will update when date+time changes # Test each input file every REFRESH seconds and update if the datetime has changed