<reponame>Zoctan/AStock
#!/usr/bin/env python3
# -*- coding:utf-8 -*-

import pymongo
import os
from movie_spider.utils.time_util import *

# 菜鸟教程
# http://www.runoob.com/mongodb/mongodb-dropdatabase.html
# http://www.runoob.com/python3/python-mongodb.html
# 数据库名称
db_name = 'movie'
# 主机，端口
host, port = 'localhost', 27017
username, password = '<PASSWORD>', '<PASSWORD>'
# 数据库数据保存位置
db_directory = '/root/db'
# 客户端
client = pymongo.MongoClient("mongodb://{}:{}@{}:{}/".format(username, password, host, port))
# 数据库
db = client[db_name]


def dump_data():
    cmd = 'mongodump --host {} --port {}'.format(host, port)
    cmd += ' --db {} --out {}'.format(db_name, db_directory)
    os.system(cmd)


def restore_data(is_drop_before=False):
    cmd = 'mongorestore --host {} --port {}'.format(host, port)
    cmd += ' --db {} {}/{}'.format(db_name, db_directory, db_name)
    # 恢复前删除已有数据
    if is_drop_before:
        cmd += ' --drop'
    os.system(cmd)


def drop_db():
    db_list = client.list_database_names()
    if db_name in db_list:
        db.command('dropDatabase')
        print('数据库已删除')


def save(collection_name, data):
    collection = db[collection_name]
    if isinstance(data, list):
        collection.insert_many(data)
    else:
        collection.insert_one(data)


def update(collection_name, data_list):
    """
    更新电影信息 and 新增今天想看人数
    """
    # 汇总每次删除重新合并
    if collection_name == '汇总':
        db.汇总.remove()
    collection = db[collection_name]
    for data in data_list:
        query = {'name': data['name']}
        old_data = collection.find(query)
        # 数据不存在，先保存
        if old_data.count() == 0:
            save(collection_name, data)
        else:
            old_data = old_data.next()
            # 不更新同一天想看人数
            if old_data['wish'][-1]['time'] != get_today():
                old_data['wish'].extend(data['wish'])
            data['wish'] = old_data['wish']
            collection.update_one(query, {'$set': data})


def get_collections_data_list():
    """
    获取所有集合的所有数据
    """
    collection_name_list = db.list_collection_names()
    data_list = []
    for collection_name in collection_name_list:
        collection = db[collection_name]
        data_list.extend([i for i in collection.find()])
    print('共{}条数据'.format(len(data_list)))
    return data_list


def get_after_combined_data():
    """
    合并后数据
    """
    before_data_list = []
    collection_name_list = db.list_collection_names()
    for collection_name in collection_name_list:
        if collection_name == '汇总':
            continue
        collection = db[collection_name]
        before_data_list.append({'site': collection_name, 'list': [i for i in collection.find()]})
    # 0或1个集合数据
    if 0 <= len(before_data_list) <= 1:
        return before_data_list

    # 按数据量降序（电影多的在前）
    before_data_list = sorted(before_data_list, key=lambda data: len(data['list']), reverse=True)

    def get_same_movie(anchor, name):
        # 从非锚点site开始依次找出同名电影
        same_movie_list = []
        for i in range(len(before_data_list)):
            if i == anchor:
                continue
            movie_list = before_data_list[i]['list']
            for movie in movie_list:
                if name == movie['name']:
                    same_movie_list.append({'site': before_data_list[i]['site'], 'movie': movie})
        return same_movie_list

    common_attribute = ['_id', 'image', 'name', 'date', 'wish', 'detail']
    # 排除共有属性后的数据
    no_common_data = lambda _dict: {key: _dict[key] for key in _dict.keys() if key not in common_attribute}

    # 取数据更详细的
    def get_bigger(a, b):
        if a is None:
            if b is None:
                return ''
            else:
                return b
        else:
            if b is None:
                return a
            else:
                return a if len(a) > len(b) else b

    after_data_list = []
    # 以第1个为锚点
    anchor = 0
    anchor_site = before_data_list[anchor]['site']
    for data in before_data_list[anchor]['list']:
        # 共有属性
        after_data = {
            'image': data['image'],
            'name': data['name'],
            'date': data['date'],
            'from': [{
                'site': anchor_site,
                'wish': data['wish'],
                'detail': data['detail']
            }]
        }
        same_movie_list = get_same_movie(anchor, data['name'])
        # 存在同名电影
        if same_movie_list:
            for site_movie in same_movie_list:
                site = site_movie['site']
                movie = site_movie['movie']
                after_data['from'].append({
                    'site': site,
                    'wish': movie['wish'],
                    'detail': movie['detail']
                })
                # 非共有属性
                after_data.update(no_common_data(data))
                after_data.update(no_common_data(movie))

                date = get_bigger(movie['date'], data['date'])
                actor = get_bigger(movie.get('actor'), data.get('actor'))
                director = get_bigger(movie.get('director'), data.get('director'))
                type = get_bigger(movie.get('type'), data.get('type'))
                country = get_bigger(movie.get('country'), data.get('country'))
                after_data.update({'date': date,
                                   'type': type,
                                   'actor': actor,
                                   'director': director,
                                   'country': country})

                count_wish = lambda _index, _data: \
                    sum([i['wish'][_index]['count'] if len(i['wish']) >= -_index else 0 for i in _data['from']])
                # 最新的总想看人数
                today_total_wish = count_wish(-1, after_data)
                after_data.update({'total_wish': today_total_wish})
                # 想看人数环比昨日
                # 昨日想看人数
                yesterday_total_wish = count_wish(-2, after_data)
                # 昨日无数据则没有增长率
                if yesterday_total_wish == 0:
                    wish_change = 0.0
                else:
                    wish_change = round((today_total_wish - yesterday_total_wish) / yesterday_total_wish, 4)
                after_data.update({'wish_change': '{:.02f}%'.format(wish_change * 100)})
            # 排除没有导演的电影
            if after_data['director'] != '':
                if after_data.get('company'):
                    # 去除重复公司
                    company = '/'.join(set(after_data.get('company').split('/')))
                    after_data.update({'company': company})
                after_data_list.append(after_data)
    # 数据按想看人数降序
    after_data_list = sorted(after_data_list, key=lambda data: data['total_wish'], reverse=True)
    return after_data_list


def released_more_than(date, days):
    # 日期不全，默认没超过 days 时间
    if len(date) < 10:
        return False
    d = compare_time(date, get_today())
    return d < 0 and days <= -d


def delete_movies_which_released_more_than(days=30):
    """
    清除已经上映N天的电影
    """
    # 2018-11-21 < 2018-12-21
    query = {'date': {'$lte': get_today_ago(days)}}
    collection_name_list = db.list_collection_names()
    out_total, deleted_count = 0, 0
    for collection_name in collection_name_list:
        collection = db[collection_name]
        data_list = collection.find(query)
        for data in data_list:
            print(data)
            out_total += 1
        x = collection.delete_many(query)
        deleted_count += x.deleted_count
    print('共{}部电影已上映，已删除{}部'.format(out_total, deleted_count))


if __name__ == '__main__':
    dump_data()
    drop_db()
    restore_data(True)

#!/usr/bin/env python3
# -*- coding:utf-8 -*-

import pymongo
import os
from movie_spider.utils.time_util import *

# 菜鸟教程
# http://www.runoob.com/mongodb/mongodb-dropdatabase.html
# http://www.runoob.com/python3/python-mongodb.html
# 数据库名称
db_name = 'movie'
# 主机，端口
host, port = 'localhost', 27017
username, password = '<PASSWORD>', '<PASSWORD>'
# 数据库数据保存位置
db_directory = '/root/db'
# 客户端
client = pymongo.MongoClient("mongodb://{}:{}@{}:{}/".format(username, password, host, port))
# 数据库
db = client[db_name]


def dump_data():
    cmd = 'mongodump --host {} --port {}'.format(host, port)
    cmd += ' --db {} --out {}'.format(db_name, db_directory)
    os.system(cmd)


def restore_data(is_drop_before=False):
    cmd = 'mongorestore --host {} --port {}'.format(host, port)
    cmd += ' --db {} {}/{}'.format(db_name, db_directory, db_name)
    # 恢复前删除已有数据
    if is_drop_before:
        cmd += ' --drop'
    os.system(cmd)


def drop_db():
    db_list = client.list_database_names()
    if db_name in db_list:
        db.command('dropDatabase')
        print('数据库已删除')


def save(collection_name, data):
    collection = db[collection_name]
    if isinstance(data, list):
        collection.insert_many(data)
    else:
        collection.insert_one(data)


def update(collection_name, data_list):
    """
    更新电影信息 and 新增今天想看人数
    """
    # 汇总每次删除重新合并
    if collection_name == '汇总':
        db.汇总.remove()
    collection = db[collection_name]
    for data in data_list:
        query = {'name': data['name']}
        old_data = collection.find(query)
        # 数据不存在，先保存
        if old_data.count() == 0:
            save(collection_name, data)
        else:
            old_data = old_data.next()
            # 不更新同一天想看人数
            if old_data['wish'][-1]['time'] != get_today():
                old_data['wish'].extend(data['wish'])
            data['wish'] = old_data['wish']
            collection.update_one(query, {'$set': data})


def get_collections_data_list():
    """
    获取所有集合的所有数据
    """
    collection_name_list = db.list_collection_names()
    data_list = []
    for collection_name in collection_name_list:
        collection = db[collection_name]
        data_list.extend([i for i in collection.find()])
    print('共{}条数据'.format(len(data_list)))
    return data_list


def get_after_combined_data():
    """
    合并后数据
    """
    before_data_list = []
    collection_name_list = db.list_collection_names()
    for collection_name in collection_name_list:
        if collection_name == '汇总':
            continue
        collection = db[collection_name]
        before_data_list.append({'site': collection_name, 'list': [i for i in collection.find()]})
    # 0或1个集合数据
    if 0 <= len(before_data_list) <= 1:
        return before_data_list

    # 按数据量降序（电影多的在前）
    before_data_list = sorted(before_data_list, key=lambda data: len(data['list']), reverse=True)

    def get_same_movie(anchor, name):
        # 从非锚点site开始依次找出同名电影
        same_movie_list = []
        for i in range(len(before_data_list)):
            if i == anchor:
                continue
            movie_list = before_data_list[i]['list']
            for movie in movie_list:
                if name == movie['name']:
                    same_movie_list.append({'site': before_data_list[i]['site'], 'movie': movie})
        return same_movie_list

    common_attribute = ['_id', 'image', 'name', 'date', 'wish', 'detail']
    # 排除共有属性后的数据
    no_common_data = lambda _dict: {key: _dict[key] for key in _dict.keys() if key not in common_attribute}

    # 取数据更详细的
    def get_bigger(a, b):
        if a is None:
            if b is None:
                return ''
            else:
                return b
        else:
            if b is None:
                return a
            else:
                return a if len(a) > len(b) else b

    after_data_list = []
    # 以第1个为锚点
    anchor = 0
    anchor_site = before_data_list[anchor]['site']
    for data in before_data_list[anchor]['list']:
        # 共有属性
        after_data = {
            'image': data['image'],
            'name': data['name'],
            'date': data['date'],
            'from': [{
                'site': anchor_site,
                'wish': data['wish'],
                'detail': data['detail']
            }]
        }
        same_movie_list = get_same_movie(anchor, data['name'])
        # 存在同名电影
        if same_movie_list:
            for site_movie in same_movie_list:
                site = site_movie['site']
                movie = site_movie['movie']
                after_data['from'].append({
                    'site': site,
                    'wish': movie['wish'],
                    'detail': movie['detail']
                })
                # 非共有属性
                after_data.update(no_common_data(data))
                after_data.update(no_common_data(movie))

                date = get_bigger(movie['date'], data['date'])
                actor = get_bigger(movie.get('actor'), data.get('actor'))
                director = get_bigger(movie.get('director'), data.get('director'))
                type = get_bigger(movie.get('type'), data.get('type'))
                country = get_bigger(movie.get('country'), data.get('country'))
                after_data.update({'date': date,
                                   'type': type,
                                   'actor': actor,
                                   'director': director,
                                   'country': country})

                count_wish = lambda _index, _data: \
                    sum([i['wish'][_index]['count'] if len(i['wish']) >= -_index else 0 for i in _data['from']])
                # 最新的总想看人数
                today_total_wish = count_wish(-1, after_data)
                after_data.update({'total_wish': today_total_wish})
                # 想看人数环比昨日
                # 昨日想看人数
                yesterday_total_wish = count_wish(-2, after_data)
                # 昨日无数据则没有增长率
                if yesterday_total_wish == 0:
                    wish_change = 0.0
                else:
                    wish_change = round((today_total_wish - yesterday_total_wish) / yesterday_total_wish, 4)
                after_data.update({'wish_change': '{:.02f}%'.format(wish_change * 100)})
            # 排除没有导演的电影
            if after_data['director'] != '':
                if after_data.get('company'):
                    # 去除重复公司
                    company = '/'.join(set(after_data.get('company').split('/')))
                    after_data.update({'company': company})
                after_data_list.append(after_data)
    # 数据按想看人数降序
    after_data_list = sorted(after_data_list, key=lambda data: data['total_wish'], reverse=True)
    return after_data_list


def released_more_than(date, days):
    # 日期不全，默认没超过 days 时间
    if len(date) < 10:
        return False
    d = compare_time(date, get_today())
    return d < 0 and days <= -d


def delete_movies_which_released_more_than(days=30):
    """
    清除已经上映N天的电影
    """
    # 2018-11-21 < 2018-12-21
    query = {'date': {'$lte': get_today_ago(days)}}
    collection_name_list = db.list_collection_names()
    out_total, deleted_count = 0, 0
    for collection_name in collection_name_list:
        collection = db[collection_name]
        data_list = collection.find(query)
        for data in data_list:
            print(data)
            out_total += 1
        x = collection.delete_many(query)
        deleted_count += x.deleted_count
    print('共{}部电影已上映，已删除{}部'.format(out_total, deleted_count))


if __name__ == '__main__':
    dump_data()
    drop_db()
    restore_data(True)
#!/usr/bin/env python3 # -*- coding:utf-8 -*- # 菜鸟教程 # http://www.runoob.com/mongodb/mongodb-dropdatabase.html # http://www.runoob.com/python3/python-mongodb.html # 数据库名称 # 主机，端口 # 数据库数据保存位置 # 客户端 # 数据库 # 恢复前删除已有数据 更新电影信息 and 新增今天想看人数 # 汇总每次删除重新合并 # 数据不存在，先保存 # 不更新同一天想看人数 获取所有集合的所有数据 合并后数据 # 0或1个集合数据 # 按数据量降序（电影多的在前） # 从非锚点site开始依次找出同名电影 # 排除共有属性后的数据 # 取数据更详细的 # 以第1个为锚点 # 共有属性 # 存在同名电影 # 非共有属性 # 最新的总想看人数 # 想看人数环比昨日 # 昨日想看人数 # 昨日无数据则没有增长率 # 排除没有导演的电影 # 去除重复公司 # 数据按想看人数降序 # 日期不全，默认没超过 days 时间 清除已经上映N天的电影 # 2018-11-21 < 2018-12-21