

def main():
    global downloaded_repos
    global args
    logo()
    parse_args()
    logfile_path = os.path.join(os.getcwd(), args.username, logfile)
    logfile_dir = os.path.dirname(logfile_path)
    if (not os.path.exists(logfile_dir)):
        os.mkdir(logfile_dir)
    __filehandler = logging.FileHandler(os.path.realpath(logfile_path))
    logger.addHandler(__filehandler)
    if (args.debug is True):
        logger.setLevel(logging.DEBUG)
    logger.info('Using username "{0}"'.format(args.username))
    logger.info('Downloading repos from http://www.github.com/{0}'.format(args.username))
    (repodata, rate_limit) = get_repo_data(args.username)
    if (args.repos is None):
        repos_json = json.loads(repodata)
    else:
        repos_json = ''
        tmp_list = []
        for r in args.repos.split(','):
            tmp_str = {
                'clone_url': 'https://github.com/{0}/{1}.git'.format(args.username, r),
                'full_name': os.path.realpath('{0}/{1}'.format(args.username, r)),
                'name': '{0}'.format(r),
            }
            tmp_list.append(tmp_str)
            if (args.debug is True):
                logger.debug('Adding repo: {0}'.format(tmp_str))
        repos_json_dumps = json.dumps(tmp_list)
        repos_json = json.loads(repos_json_dumps)
    if ((args.debug is True) and (repos_json is not None)):
        logger.debug('JSON response: {0}'.format(repos_json))
    if (args.threads > 1):
        q = Queue.Queue()
        threads = []
        for itm in repos_json:
            q.put(itm)
        while (not q.empty()):
            if (args.threads >= threading.activeCount()):
                q_itm = q.get()
                try:
                    t = threading.Thread(target=dl_worker, args=(q_itm,))
                    t.daemon = True
                    t.start()
                finally:
                    q.task_done()
        while (threading.activeCount() > 1):
            time.sleep(0.1)
        for thread in threads:
            thread.join()
        q.join()
    else:
        for itm in filename:
            dl_worker(itm)
    logger.info(('You may request up to %s more users this hour' % rate_limit))
    logger.info('Generating wordlists')
    if (args.debug is True):
        logger.debug('Creating a database in memory')
    db = sqlite3.connect(':memory:')
    cur = db.cursor()
    cur.execute('CREATE TABLE files (id INTEGER PRIMARY KEY, name                 varchar(255) UNIQUE, count int DEFAULT 1)')
    cur.execute('CREATE TABLE dirs (id INTEGER PRIMARY KEY, name                 varchar(255) UNIQUE, count int DEFAULT 1)')
    if (args.debug is True):
        logger.debug('Populating the database')
    for (root, dirs, files) in os.walk(('./%s' % args.username)):
        for m_file in files:
            try:
                cur.execute('INSERT INTO files (name) VALUES (?)', (os.path.join(root, m_file),))
            except sqlite3.IntegrityError:
                cur.execute('UPDATE files SET count = count + 1 WHERE                 name = ?', os.path.join(root, m_file))
        for m_dir in dirs:
            try:
                cur.execute('INSERT INTO dirs (name) VALUES (?)', (os.path.join(root, m_dir),))
            except sqlite3.IntegrityError:
                cur.execute(('UPDATE dirs SET count = count + 1                            WHERE name = ?' % os.path.join(root, m_dir)))
    if (args.debug is True):
        logger.debug('Generating the files wordlist')
    try:
        cur.execute('SELECT name FROM files ORDER BY count DESC')
        res = cur.fetchall()
    except sqlite3.OperationalError as e:
        logger.exception('Error getting file list from database: {0}'.format(e))
    filename = ('%s/%s-files.txt' % (args.username, args.username))
    logger.info(('Writing %s' % filename))
    try:
        fp = open(filename, 'w')
        for itm in res:
            encoded_itm = itm[0].encode('utf8')
            if isinstance(itm[0], basestring):
                encoded_itm = itm[0].encode('utf8')
            else:
                encoded_itm = unicode(itm[0]).encode('utf8')
            fp.write(('%s\n' % encoded_itm))
        fp.close()
    except (OSError, IOError) as e:
        logger.exception('Cannot write to "{0}": {1}'.format(filename, e.message))
    if (args.debug is True):
        logger.debug('Generating the dirs wordlist')
    cur.execute('SELECT name FROM dirs ORDER BY count DESC')
    res = cur.fetchall()
    filename = ('%s/%s-dirs.txt' % (args.username, args.username))
    logger.info(('Writing %s' % filename))
    try:
        fp = open(filename, 'w')
        for itm in res:
            fp.write(('%s\n' % itm[0]))
        fp.close()
    except (OSError, IOError) as e:
        logger.exception('Cannot write to "{0}": {1}'.format(filename, e.message))
    if cur:
        cur.close()
        del cur
    if db:
        db.close()
        del db
    logger.info(('Cloned %s repos from http://www.github.com/%s' % (downloaded_repos, args.username)))
    logger.info(('Logfile saved %s' % logfile_path))

dataset/ETHPy150Open wick2o/gitrecon/gitrecon.py main/VarMisuse@517/673 `repos_json`->`filename`