def open_output(fnout=None):
    assert g_args
    global g_logfp

    if not fnout:
        fnout = g_args.output

    if not fnout:
        outf = OutputFile(sys.stdout)
    elif fnout.endswith(".zip"):
        outf = OutputZipFile(fnout, parse_pathname(fnout).base)
    elif not parse_pathname(fnout).ext:  # extensionless assumed to be directories
        outf = OutputDirectory(fnout)
#        g_logfp = outf.open_file(g_scriptname + ".log")
    else:
        # make parent dirs
        try:
            os.makedirs(parse_pathname(fnout).path)
        except Exception as e:
            pass  # log("%s: %s" % (type(e), str(e)))
        outf = OutputFile(codecs.open(fnout, 'w', encoding="utf-8"))

    return outf def info(_s, _m=0):
    log(_s, minverbose=_m, severity='info') def log(s, minverbose=0, severity='INFO'):
    # This can be made in more Python way
    if g_logfp.isatty(): # Colors only for atty term
        if severity.lower() == 'warning':
            s = bcolors.WARNING + s + bcolors.ENDC
        if severity.lower() == 'error':
            s = bcolors.FAIL + s + bcolors.ENDC

    if g_logfp:
        g_logfp.write("%s: %s\n" % (severity.upper(), s))
    g_logs.append("%s: [%s] %s" % (g_currentProgress or g_scriptname, severity.upper(), s)) def find_files(*paths, **kwargs):
    for fn, data, dt in find_files_with_time(*paths, **kwargs):
        yield fn, data def get_args(desc="", parser=None):
    global g_args, g_scriptname
    g_scriptname = parse_pathname(sys.argv[0]).base

    if g_args:
        return g_args

    if not parser:
        parser = args_parser(desc)

    parser.add_argument('inputs', nargs='*', help='toplevel input(s)')
    parser.add_argument('-o', '--output', dest='output', action='store')
    parser.add_argument('-q', '--quiet', dest='verbose', action='store_const', const=-1, default=0)
    parser.add_argument('-v', '--verbose', dest='verbose', action='count', default=0)
    parser.add_argument('-d', '--debug', dest='debug', action='store_true', default=False, help='abort on exception')
    parser.add_argument('-c', '--corpus', dest='corpusdir', default='crosswords', help='corpus source')
    g_args = parser.parse_args()

    return g_args def parse_pathname(path):
    # Fix to proper split names like file.xml.1
    ext = os.extsep + os.extsep.join(os.path.basename(path).split(os.extsep)[1:])
    path, fn = os.path.split(path)
    ext = ext if fn else ''
    base = os.path.splitext(fn)[0]
    nt = namedtuple('Pathname', 'path base ext filename')
    return nt(path=path, base=base, ext=ext, filename=fn) def generate_zip_files(data):
    try:
        zf = zipfile.ZipFile(io.BytesIO(data))
        for zi in sorted(zf.infolist(), key=lambda x: x.filename):
            zipdt = to_timet(*zi.date_time)
            yield zi.filename, zf.read(zi), zipdt

    except zipfile.BadZipfile as e:
        error("generate_zip_files(): %s" % str(e)) def iso8601(timet=None):
    if not timet:
        timet = time.time()
    return datetime.datetime.fromtimestamp(int(timet)).isoformat(' ').split(' ')[0] def to_timet(y, mon=1, d=1, h=0, m=0, s=0):
    return time.mktime(datetime.datetime(y, mon, d, h, m, s).timetuple()) RECEIPTS_TSV = "gxd/receipts.tsv"
SIMILAR_TSV = "gxd/similar.tsv"
PUBLICATIONS_TSV = "gxd/publications.tsv"
PUZZLE_SOURCES_TSV = "gxd/sources.tsv"
RECENT_DOWNLOADS_TSV = "gxd/recent-downloads.tsv"
STATS_TSV = "pub/stats.tsv"
class Error(Exception):
def xd_receipts():
def xd_receipts_rows():
def xd_publications():
def xd_puzzle(xdid):
def xd_puzzles_dict():
def xd_puzzles(xdid=''):
def get_author(xdid=''):
def _puzzles():
def xd_puzzle_sources():
def xd_recent_downloads():
def delete_stats():
def stats():
def read_rows(tablename):
def append_row(tablename, row):
def get_last_receipt_id():
def xd_receipts_row(CaptureTime="", ReceivedTime="", ExternalSource="", InternalSource="", SourceFilename="", xdid=""):
def check_already_received(ExternalSource, SourceFilename):
def xd_sources_row(SourceFilename, ExternalSource, DownloadTime):
def xd_recent_download(pubid, dt):
def update_puzzles_row(xd):
def xd_similar(xdid=''):
def xd_similar_all():
def public_patterns():
def is_public(xdid): def xd_send_email(destaddr, fromaddr='admin@xd.saul.pw', subject='', body=''):
    client = boto3.client('ses', region_name=os.environ['REGION'])
    info("sending email to %s (subject '%s')" % (destaddr, subject))
    try:
        response = client.send_email(
                Source=fromaddr,
                Destination= {'ToAddresses': [ destaddr ] },
                Message={ 'Subject': { 'Data': subject },
                'Body': { 'Text': { 'Data': body } } })
        return response
    except Exception as e:
        error("xd_send_email(): %s" % str(e))
        return None
from xdfile.utils import open_output, info, log, find_files, get_args, parse_pathname, generate_zip_files, iso8601, to_timet
from xdfile.metadatabase import xd_sources_header, xd_sources_row
from xdfile.cloud import xd_send_email
import email.utils
import email
import mimetypes
import time
#!/usr/bin/env python3





def generate_email_files(msg):
    counter = 1
    upload_date = time.mktime(email.utils.parsedate(msg["Date"]))
    for part in msg.walk():
        # multipart/* are just containers
        if part.get_content_maintype() == 'multipart':
            continue
        # Applications should really sanitize the given filename so that an
        # email message can't be used to overwrite important files
        filename = part.get_filename()
        if not filename:
            ext = mimetypes.guess_extension(part.get_content_type())
            if not ext:
                # Use a generic bag-of-bits extension
                ext = '.bin'
            filename = 'part-%03d%s' % (counter, ext)
        counter += 1

        data = part.get_payload(decode=True)
        if parse_pathname(filename).ext == '.zip':
            for zipfn, zipdata, zipdt in generate_zip_files(data):
                yield zipfn, zipdata, zipdt
        else:
            yield filename, data, upload_date

def main():
    args = get_args('parse downloaded emails')
    outf = open_output()

    sources_tsv = ''
    for emailfn, emailcontents in find_files(*args.inputs):
        msg = email.message_from_bytes(emailcontents)
        upload_src = msg["From"]

        if not upload_src:
            continue

        email_sources_tsv = []
        email_files = generate_email_files(msg)
        for puzfn, puzdata, puzdt in email_files:
            # a basic sanity check of filesize
            # accommodate small puzzles and .pdf
            info("%s: %s from %s" % (puzfn, iso8601(puzdt), upload_src))

        summary("%s puzzles from %s" % (len(email_files), upload_src))

            if len(puzdata) > 1000 and len(puzdata) < 100000:

                email_sources_tsv.append(xd_sources_row(puzfn, upload_src, iso8601(puzdt)))