import pymongo
from bson import ObjectId

__author__ = 'tomas'
from pymongo import MongoClient
import traceback

'''
This class initializes a Memex Mongo object and rebuilds the db collections if you want.
Warning: init_db will delete your collection when set to True
which_collection specifies whether to connect to the scrapy crawl data or common crawl data collection
to connect to common crawl specify this as cc_crawldata
'''

mongo_database_name = "MemexHack"
seed_urls_collection_name = "seed_urls"
broad_crawler_collection_name = "broad_crawler"
crawl_job_collection_name = "crawl_job"
workspace_collection_name = "workspace"
user_collection_name = "user"
role_collection_name = "role"
deep_crawler_collection_name = "deep_crawler"
deep_crawler_domains_collection_name = "deep_crawler_domains"
login_collection_name = "login"


class MongoInstance(object):

    def __init__(self, address="localhost", port=27017):
        self.client = MongoClient(address, port)
        db = self.client[mongo_database_name]
        self.db = db

        self.workspace_collection = db[workspace_collection_name]
        self.initialize()

    def get_workspace_collection(self):
        return self.db[workspace_collection_name]

    def get_seed_urls_collection(self):
        return self.db[seed_urls_collection_name]

    def get_broad_crawler_collection(self):
        return self.db[broad_crawler_collection_name]

    def get_crawl_job_collection(self):
        return self.db[crawl_job_collection_name]

    def get_user_collection(self):
        return self.db[user_collection_name]

    def get_role_collection(self):
        return self.db[role_collection_name]

    def get_deep_crawler_collection(self):
        return self.db[deep_crawler_collection_name]

    def get_deep_crawler_domains_collection(self):
        return self.db[deep_crawler_domains_collection_name]

    def get_login_collection(self):
        return self.db[login_collection_name]

    def initialize(self):
        collections = self.db.collection_names()

        if crawl_job_collection_name not in collections:
            self.db.create_collection(crawl_job_collection_name)

        if seed_urls_collection_name not in collections:
            self.db.create_collection(seed_urls_collection_name)
            self.db[seed_urls_collection_name].create_index([("workspaceId", pymongo.ASCENDING), ("url", pymongo.ASCENDING)], unique=True, drop_dups=True, name="seeds_url_ws_url_unique_index")

        if broad_crawler_collection_name not in collections:
            self.db.create_collection(broad_crawler_collection_name)
            self.db[broad_crawler_collection_name].create_index([("workspaceId", pymongo.ASCENDING), ("score", pymongo.DESCENDING), ("_id", pymongo.ASCENDING)], name="scoring_search_index")
            self.db[broad_crawler_collection_name].create_index([("workspaceId", pymongo.ASCENDING), ("url", pymongo.ASCENDING)], unique=True, drop_dups=True, name="broad_crawler_ws_url_unique_index")

        if deep_crawler_collection_name not in collections:
            self.db.create_collection(deep_crawler_collection_name)
            self.db[deep_crawler_collection_name].create_index([("workspaceId", pymongo.ASCENDING), ("url", pymongo.ASCENDING), ("_id", pymongo.ASCENDING)], name="url_search_index")
            self.db[deep_crawler_collection_name].create_index([("workspaceId", pymongo.ASCENDING), ("url", pymongo.ASCENDING)], unique=True, drop_dups=True, name="deep_crawler_ws_url_unique_index")

        if deep_crawler_domains_collection_name not in collections:
            self.db.create_collection(deep_crawler_domains_collection_name)
            self.db[deep_crawler_domains_collection_name].create_index([("workspaceId", pymongo.ASCENDING), ("jobId", pymongo.ASCENDING), ("domain", pymongo.ASCENDING)], name="domain_search_index")
            self.db[deep_crawler_domains_collection_name].create_index([("workspaceId", pymongo.ASCENDING), ("url", pymongo.ASCENDING)], unique=True, drop_dups=True, name="deep_crawler_domain_ws_url_unique_index")

        if login_collection_name not in collections:
            self.db.create_collection(login_collection_name)
            self.db[login_collection_name].create_index([("workspaceId", pymongo.ASCENDING)], name="login_index")

    def get_workspace_by_id(self, id):
        return self.workspace_collection.find_one({"_id": ObjectId(id)})




import pymongo
from bson import ObjectId

__author__ = 'tomas'
from pymongo import MongoClient
import traceback

'''
This class initializes a Memex Mongo object and rebuilds the db collections if you want.
Warning: init_db will delete your collection when set to True
which_collection specifies whether to connect to the scrapy crawl data or common crawl data collection
to connect to common crawl specify this as cc_crawldata
'''

mongo_database_name = "MemexHack"
seed_urls_collection_name = "seed_urls"
broad_crawler_collection_name = "broad_crawler"
crawl_job_collection_name = "crawl_job"
workspace_collection_name = "workspace"
user_collection_name = "user"
role_collection_name = "role"
deep_crawler_collection_name = "deep_crawler"
deep_crawler_domains_collection_name = "deep_crawler_domains"
login_collection_name = "login"


class MongoInstance(object):

    def __init__(self, address="localhost", port=27017):
        self.client = MongoClient(address, port)
        db = self.client[mongo_database_name]
        self.db = db

        self.workspace_collection = db[workspace_collection_name]
        self.initialize()

    def get_workspace_collection(self):
        return self.db[workspace_collection_name]

    def get_seed_urls_collection(self):
        return self.db[seed_urls_collection_name]

    def get_broad_crawler_collection(self):
        return self.db[broad_crawler_collection_name]

    def get_crawl_job_collection(self):
        return self.db[crawl_job_collection_name]

    def get_user_collection(self):
        return self.db[user_collection_name]

    def get_role_collection(self):
        return self.db[role_collection_name]

    def get_deep_crawler_collection(self):
        return self.db[deep_crawler_collection_name]

    def get_deep_crawler_domains_collection(self):
        return self.db[deep_crawler_domains_collection_name]

    def get_login_collection(self):
        return self.db[login_collection_name]

    def initialize(self):
        collections = self.db.collection_names()

        if crawl_job_collection_name not in collections:
            self.db.create_collection(crawl_job_collection_name)

        if seed_urls_collection_name not in collections:
            self.db.create_collection(seed_urls_collection_name)
            self.db[seed_urls_collection_name].create_index([("workspaceId", pymongo.ASCENDING), ("url", pymongo.ASCENDING)], unique=True, drop_dups=True, name="seeds_url_ws_url_unique_index")

        if broad_crawler_collection_name not in collections:
            self.db.create_collection(broad_crawler_collection_name)
            self.db[broad_crawler_collection_name].create_index([("workspaceId", pymongo.ASCENDING), ("score", pymongo.DESCENDING), ("_id", pymongo.ASCENDING)], name="scoring_search_index")
            self.db[broad_crawler_collection_name].create_index([("workspaceId", pymongo.ASCENDING), ("url", pymongo.ASCENDING)], unique=True, drop_dups=True, name="broad_crawler_ws_url_unique_index")

        if deep_crawler_collection_name not in collections:
            self.db.create_collection(deep_crawler_collection_name)
            self.db[deep_crawler_collection_name].create_index([("workspaceId", pymongo.ASCENDING), ("url", pymongo.ASCENDING), ("_id", pymongo.ASCENDING)], name="url_search_index")
            self.db[deep_crawler_collection_name].create_index([("workspaceId", pymongo.ASCENDING), ("url", pymongo.ASCENDING)], unique=True, drop_dups=True, name="deep_crawler_ws_url_unique_index")

        if deep_crawler_domains_collection_name not in collections:
            self.db.create_collection(deep_crawler_domains_collection_name)
            self.db[deep_crawler_domains_collection_name].create_index([("workspaceId", pymongo.ASCENDING), ("jobId", pymongo.ASCENDING), ("domain", pymongo.ASCENDING)], name="domain_search_index")
            self.db[deep_crawler_domains_collection_name].create_index([("workspaceId", pymongo.ASCENDING), ("url", pymongo.ASCENDING)], unique=True, drop_dups=True, name="deep_crawler_domain_ws_url_unique_index")

        if login_collection_name not in collections:
            self.db.create_collection(login_collection_name)
            self.db[login_collection_name].create_index([("workspaceId", pymongo.ASCENDING)], name="login_index")

    def get_workspace_by_id(self, id):
        return self.workspace_collection.find_one({"_id": ObjectId(id)})




This class initializes a Memex Mongo object and rebuilds the db collections if you want. Warning: init_db will delete your collection when set to True which_collection specifies whether to connect to the scrapy crawl data or common crawl data collection to connect to common crawl specify this as cc_crawldata