EOL = '\n'
COLSEP = '\t'
COLUMN_SEPARATOR = COLSEP
WEEKDAYS = [ 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun' ]
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'
    BOLD = '\033[1m'
    UNDERLINE = '\033[4m'
class bcolors:
class AttrDict(dict):
class OutputZipFile(zipfile.ZipFile):
class OutputFile:
class OutputDirectory:
def space_with_nbsp(text):
def split_xdid(xdid):
def br_with_n(text):
def get_log():
def log(s, minverbose=0, severity='INFO'):
def info(_s, _m=0):
def warn(_s, _m=0):
def error(_s, _m=0):
def summary(_s, _m=0):
def debug(s):
def progress(rest=None, every=1):
def args_parser(desc=""):
def get_args(desc="", parser=None):
def find_files(*paths, **kwargs):
def to_timet(y, mon=1, d=1, h=0, m=0, s=0):
def generate_zip_files(data):
def find_files_with_time(*paths, **kwargs):
def filetime(fn):
def iso8601(timet=None):
def datestr_to_datetime(s):
def parse_xdid(path):
def parse_pathname(path):
def parse_pubid(fn):
def construct_date(y, m, d):
def parse_iso8601(s):
def parse_seqnum(s):
def parse_date_from_filename(fn):
def clean_filename(fn):
def replace_ext(fn, newext):
    def __init__(self, *args, **kwargs):
def autoconvert(v):
def parse_tsv_data(contents, objname=None):
def parse_tsv(fn, objname=None):
def parse_tsv_rows(fn, objname=None):
    def __init__(self, fnzip, toplevel="", log=True):
    def write_file(self, fn, contents, timet=None):
    def write(self, data):
    def __del__(self):
    def __init__(self, outfp=None):
    def write_file(self, fn, contents, timet=None):
    def write(self, data):
    def write_row(self, fields):
    def write_html(self, fn, innerhtml, title=""):
def strip_toplevel(fn):
def disambiguate_fn(fn, all_filenames):
    def __init__(self, toplevel_dir):
    def exists(self, fn):
    def open_file(self, fn, mode='w'):
    def close_file(self, fn):
    def write_file(self, fn, contents, timet=None):
    def write_html(self, fn, innerhtml, title=""):
    def write_row(self, fn, headerstr, values):
def open_output(fnout=None):
def memoize(obj):
    def memoizer(*args, **kwargs):
def __dict_replace(s, d):
def escape(data, entities={}):
def consecutive(text): RECEIPTS_TSV = "gxd/receipts.tsv"
SIMILAR_TSV = "gxd/similar.tsv"
PUBLICATIONS_TSV = "gxd/publications.tsv"
PUZZLE_SOURCES_TSV = "gxd/sources.tsv"
RECENT_DOWNLOADS_TSV = "gxd/recent-downloads.tsv"
STATS_TSV = "pub/stats.tsv"
class Error(Exception):
def xd_receipts():
def xd_receipts_rows():
def xd_publications():
def xd_puzzle(xdid):
def xd_puzzles_dict():
def xd_puzzles(xdid=''):
def get_author(xdid=''):
def _puzzles():
def xd_puzzle_sources():
def xd_recent_downloads():
def delete_stats():
def stats():
def read_rows(tablename):
def append_row(tablename, row):
def get_last_receipt_id():
def xd_receipts_row(CaptureTime="", ReceivedTime="", ExternalSource="", InternalSource="", SourceFilename="", xdid=""):
def check_already_received(ExternalSource, SourceFilename):
def xd_sources_row(SourceFilename, ExternalSource, DownloadTime):
def xd_recent_download(pubid, dt):
def update_puzzles_row(xd):
def xd_similar(xdid=''):
def xd_similar_all():
def public_patterns():
def is_public(xdid): def get_args(desc="", parser=None):
    global g_args, g_scriptname
    g_scriptname = parse_pathname(sys.argv[0]).base

    if g_args:
        return g_args

    if not parser:
        parser = args_parser(desc)

    parser.add_argument('inputs', nargs='*', help='toplevel input(s)')
    parser.add_argument('-o', '--output', dest='output', action='store')
    parser.add_argument('-q', '--quiet', dest='verbose', action='store_const', const=-1, default=0)
    parser.add_argument('-v', '--verbose', dest='verbose', action='count', default=0)
    parser.add_argument('-d', '--debug', dest='debug', action='store_true', default=False, help='abort on exception')
    parser.add_argument('-c', '--corpus', dest='corpusdir', default='crosswords', help='corpus source')
    g_args = parser.parse_args()

    return g_args def log(s, minverbose=0, severity='INFO'):
    # This can be made in more Python way
    if g_logfp.isatty(): # Colors only for atty term
        if severity.lower() == 'warning':
            s = bcolors.WARNING + s + bcolors.ENDC
        if severity.lower() == 'error':
            s = bcolors.FAIL + s + bcolors.ENDC

    if g_logfp:
        g_logfp.write("%s: %s\n" % (severity.upper(), s))
    g_logs.append("%s: [%s] %s" % (g_currentProgress or g_scriptname, severity.upper(), s)) def error(_s, _m=0):
    log(_s, minverbose=_m, severity='error') def warn(_s, _m=0):
    log(_s, minverbose=_m, severity='warning') def summary(_s, _m=0):
    log(_s, minverbose=_m, severity='summary') def debug(s):
    if g_args.debug:
        print(" " + s) def open_output(fnout=None):
    assert g_args
    global g_logfp

    if not fnout:
        fnout = g_args.output

    if not fnout:
        outf = OutputFile(sys.stdout)
    elif fnout.endswith(".zip"):
        outf = OutputZipFile(fnout, parse_pathname(fnout).base)
    elif not parse_pathname(fnout).ext:  # extensionless assumed to be directories
        outf = OutputDirectory(fnout)
#        g_logfp = outf.open_file(g_scriptname + ".log")
    else:
        # make parent dirs
        try:
            os.makedirs(parse_pathname(fnout).path)
        except Exception as e:
            pass  # log("%s: %s" % (type(e), str(e)))
        outf = OutputFile(codecs.open(fnout, 'w', encoding="utf-8"))

    return outf def datestr_to_datetime(s):
    try:
        return datetime.date(*[int(x) for x in s.split("-")])
    except Exception as e:
        error("datestr_to_datetime(): %s" % str(e))
        if g_args.debug:
            raise
        dt = datetime.date.today()
    return dt def args_parser(desc=""):
    log("[%s]: %s" % (desc, " ".join(sys.argv)))
    return argparse.ArgumentParser(description=desc) RECEIPTS_TSV = "gxd/receipts.tsv"
SIMILAR_TSV = "gxd/similar.tsv"
PUBLICATIONS_TSV = "gxd/publications.tsv"
PUZZLE_SOURCES_TSV = "gxd/sources.tsv"
RECENT_DOWNLOADS_TSV = "gxd/recent-downloads.tsv"
STATS_TSV = "pub/stats.tsv"
class Error(Exception):
def xd_receipts():
def xd_receipts_rows():
def xd_publications():
def xd_puzzle(xdid):
def xd_puzzles_dict():
def xd_puzzles(xdid=''):
def get_author(xdid=''):
def _puzzles():
def xd_puzzle_sources():
def xd_recent_downloads():
def delete_stats():
def stats():
def read_rows(tablename):
def append_row(tablename, row):
def get_last_receipt_id():
def xd_receipts_row(CaptureTime="", ReceivedTime="", ExternalSource="", InternalSource="", SourceFilename="", xdid=""):
def check_already_received(ExternalSource, SourceFilename):
def xd_sources_row(SourceFilename, ExternalSource, DownloadTime):
def xd_recent_download(pubid, dt):
def update_puzzles_row(xd):
def xd_similar(xdid=''):
def xd_similar_all():
def public_patterns():
def is_public(xdid):
import urllib.request, urllib.error, urllib.parse
import datetime
import time
import re
from xdfile import utils, metadatabase as metadb
from xdfile.utils import get_args, log, error, warn, summary, debug, open_output, datestr_to_datetime, args_parser
from xdfile.metadatabase import xd_sources_header, xd_sources_row, xd_puzzle_sources, xd_recent_download, xd_recents_header
#!/usr/bin/env python3

# Usage:
#  $0 -o <output-zip> -r <recent-downloads.tsv>
#
#  Examines <input> filenames for each source and most recent date; downloads more recent puzzles and saves them to <output-zip>.
#




def construct_xdid(pubabbr, dt):
    return pubabbr + dt.strftime("%Y-%m-%d")


def get_dates_between(before_date, after_date, days_to_advance=1):
    if before_date > after_date:
        before_date, after_date = after_date, before_date

    days_diff = (after_date - before_date).days
    return [before_date + datetime.timedelta(days=x) for x in range(days_to_advance, days_diff, days_to_advance)]


def add_days(dt, ndays):
    return dt + datetime.timedelta(days=ndays)

def get_ungotten_dates(pubid, before_date, after_date, days_to_advance, ret=None):
    def prev_period(start_date, period=days_to_advance):
        return add_days(start_date, -period*2)

    if ret is None:
        ret = []

    newret = []
    pub_gotten = set()
    for puzrow in metadb.xd_puzzles(pubid):
        pub_gotten.add(datestr_to_datetime(puzrow.Date))

    if before_date > after_date:
        before_date, after_date = after_date, before_date

    before_before_date = prev_period(before_date)

    days_diff = (after_date - before_before_date).days

    for offset in range(days_to_advance, days_diff+1, days_to_advance):
        dt = add_days(before_before_date, offset)
        if dt not in pub_gotten:
            newret.append(dt)

    ret.extend(reversed(newret))

    if newret:
        return get_ungotten_dates(pubid, prev_period(before_before_date), before_before_date, days_to_advance, ret)
    else:
        return ret


def main():

    p = args_parser('download recent puzzles')