class Command(BaseCommand):

    def run(self):

        settings = self.project.settings

        logger.info("Getting file revisions...")
        file_revisions = self.project.get_disk_file_revisions()
        logger.info("%d file revisions" % len(file_revisions))

        snapshot = self.project.DiskSnapshot({'created_at' : time.time()})

        try:
            code_environment = CodeEnvironment(self.project,
                                               file_revisions,
                                               global_settings = self.settings,
                                               project_settings = settings)
            code_environment.analyze(snapshot=snapshot,
                                     save_if_empty=False)
        except KeyboardInterrupt:
            raise class Command(BaseCommand):

    def __init__(self,*args,**kwargs):
        super(Command,self).__init__(*args,**kwargs)
        if not hasattr(self,'project'):
            raise AttributeError("No project defined!")
        if not hasattr(self.project,'git'):
            raise AttributeError("Not a git project!")
        try:
            self.project.git.eager
        except self.project.git.DoesNotExist:
            raise AttributeError("Repository not found!") def group_snapshots_by_date(snapshots, period):

    available_periods = {
        'daily' : lambda dt : dt.strftime("%Y-%m-%d"),
        'weekly' : lambda dt: dt.strftime("%Y-%W"),
        'monthly' : lambda dt: dt.strftime("%Y-%m")
    }
    formatter = available_periods[period]

    grouped_snapshots = defaultdict(list)

    for snapshot in snapshots:
        dt = datetime.datetime.fromtimestamp(snapshot.committer_date_ts).date()
        key = formatter(dt)
        grouped_snapshots[key].append(snapshot)

    return grouped_snapshots def get_first_date_for_group(start_date,group_type,n):
    """
    :param start:   start date
    :n          :   how many groups we want to get
    :group_type :   daily, weekly, monthly
    """
    current_date = start_date
    if group_type == 'monthly':
        current_year = start_date.year
        current_month = start_date.month
        for i in range(n-1):
            current_month-=1
            if current_month == 0:
                current_month = 12
                current_year -= 1
        first_date = datetime.datetime(current_year,current_month,1)
    elif group_type == 'weekly':
        first_date=start_date-datetime.timedelta(days = start_date.weekday()+(n-1)*7)
    elif group_type == 'daily':
        first_date = start_date-datetime.timedelta(days = n-1)
    first_date = datetime.datetime(first_date.year,first_date.month,first_date.day,0,0,0)
    return first_date class GitBranch(BaseDocument):

    project = ForeignKeyField('Project',backref = 'git_branches')
    name = CharField(indexed = True,length = 100)
    hash = CharField(indexed = True,length = 64)
    remote = CharField(indexed = True,length = 100)
    last_analyzed_snapshot = ForeignKeyField('GitSnapshot')
    head_snapshot = ForeignKeyField('GitSnapshot')

    class Meta(BaseDocument.Meta):

        unique_together = [('project','name')] class GitSnapshot(BaseDocument):

    """
    """

    project = ForeignKeyField('Project',unique = False,backref = 'git_snapshots')
    snapshot = ForeignKeyField('Snapshot',unique = True,backref = 'git_snapshot')
    sha = CharField(indexed = True,length = 40)
    hash = CharField(indexed = True,length = 64)
    committer_date = DateTimeField(indexed = True)
    author_date = DateTimeField(indexed = True)
    author_name = CharField(length = 100)
    committer_date_ts = IntegerField(indexed = True)
    author_date_ts = IntegerField(indexed = True)
    tree_sha = CharField(indexed = True,length = 40)
    log = TextField(indexed = False)

    class Meta(BaseDocument.Meta):

        unique_together = [('project','sha')] class Diff(BaseDocument):

    """
    """

    #calculated as hash(snapshot_a.hash,snapshot_b.hash)
    hash = CharField(indexed = True,length = 64)
    configuration = CharField(indexed = True, length = 64)
    project = ForeignKeyField('Project',backref = 'diffs')
    snapshot_a = ForeignKeyField('Snapshot',backref = 'diffs_a')
    snapshot_b = ForeignKeyField('Snapshot',backref = 'diffs_b')

    def get_issues_count(self, by_severity=False):
        if isinstance(self.backend, SqlBackend):
            return self._get_issues_count_sql(by_severity=by_severity)
        raise NotImplementedError

    def _get_issues_count_sql(self, by_severity=False):

        diff_issue_occurrence_table = self.backend.get_table(DiffIssueOccurrence)
        issue_class_table = self.backend.get_table(self.project.IssueClass)
        project_issue_class_table = self.backend.get_table(ProjectIssueClass)
        issue_occurrence_table = self.backend.get_table(IssueOccurrence)
        issue_table = self.backend.get_table(Issue)

        s = select([diff_issue_occurrence_table.c.key,issue_class_table.c.severity,func.count().label('count')])\
            .select_from(diff_issue_occurrence_table\
            .join(issue_occurrence_table,diff_issue_occurrence_table.c.issue_occurrence == issue_occurrence_table.c.pk)\
            .join(issue_table)\
            .join(issue_class_table,and_(issue_table.c.analyzer == issue_class_table.c.analyzer,
                                         issue_table.c.code == issue_class_table.c.code))\
            .join(project_issue_class_table,and_(
                project_issue_class_table.c.issue_class == issue_class_table.c.pk,
                project_issue_class_table.c.enabled == True,
                project_issue_class_table.c.project == self.project.pk
                )))\
            .where(diff_issue_occurrence_table.c.diff == self.pk)\
            .group_by(diff_issue_occurrence_table.c.key,issue_class_table.c.severity)


        with self.backend.transaction():
            result  = self.backend.connection.execute(s).fetchall()

        if by_severity:
            counts = {'added' : {},'fixed' : {}}
            for row in result:
                if not row['severity'] in counts[row['key']]:
                    counts[row['key']][row['severity']] = 0
                counts[row['key']][row['severity']] += row['count']
        else:
            counts = {'added' : 0,'fixed' : 0}
            for row in result:
                counts[row['key']] += row['count']
        return counts

    def _summarize_issues_sql(self, include_filename=False, ignore=False):

        diff_issue_occurrence_table = self.backend.get_table(DiffIssueOccurrence)
        issue_occurrence_table = self.backend.get_table(IssueOccurrence)
        issue_table = self.backend.get_table(Issue)
        file_revision_table = self.backend.get_table(FileRevision)
        project_issue_class_table = self.backend.get_table(ProjectIssueClass)
        issue_class_table = self.backend.get_table(self.project.IssueClass)

        #we group by file revision path, issue code and analyzer
        group_columns = [file_revision_table.c.language,
                         file_revision_table.c.path,
                         diff_issue_occurrence_table.c['key'],
                         #we should not group by pk
#                         diff_issue_occurrence_table.c['pk'],
                         issue_table.c.code,
                         issue_table.c.analyzer]

        project_pk_type = self.backend.get_field_type(self.project.fields['pk'])

        #here we make sure that the given issue class is enabled for the project
        subselect = select([issue_class_table.c.pk])\
            .select_from(issue_class_table.join(project_issue_class_table))\
            .where(and_(
            issue_table.c.analyzer == issue_class_table.c.analyzer,
            issue_table.c.code == issue_class_table.c.code,
            issue_table.c.ignore == ignore,
            project_issue_class_table.c.project == expression.cast(self.project.pk,project_pk_type),
            project_issue_class_table.c.enabled == True))\

        #we perform a JOIN of the file revision table to the issue tables
        table = diff_issue_occurrence_table\
        .join(issue_occurrence_table,
              issue_occurrence_table.c.pk == diff_issue_occurrence_table.c.issue_occurrence)\
        .join(issue_table, and_(issue_occurrence_table.c.issue == issue_table.c.pk, issue_table.c.ignore == ignore))\
        .join(file_revision_table)

        #we select the aggregated issues for all file revisions in this snapshot
        s = select(group_columns+[func.count().label('count')])\
        .select_from(table)\
        .where(and_(exists(subselect),diff_issue_occurrence_table.c.diff == self.pk))\
        .group_by(*group_columns)\
        .order_by(file_revision_table.c.path)

        #we fetch the result
        with self.backend.transaction():
            result  = self.backend.connection.execute(s).fetchall()

        #we aggregate the issues by path fragments
        aggregator = lambda f: directory_splitter(f['path'],include_filename = include_filename)

        added_issues = []
        fixed_issues = []

        for row in result:
            if row['key'] == 'added':
                added_issues.append(row)
            else:
                fixed_issues.append(row)

        #we perform a map/reduce on the result
        map_reducer = IssuesMapReducer(aggregators = [aggregator],
                                       group_by = ['language','analyzer','code'])

        return {'added': map_reducer.mapreduce(added_issues),
                'fixed': map_reducer.mapreduce(fixed_issues)}

    def summarize_issues(self,include_filename = False, ignore=False):
        if isinstance(self.backend,SqlBackend):
            return self._summarize_issues_sql(include_filename=include_filename, ignore=ignore)
        raise NotImplementedError class Snapshot(BaseDocument):

    #calculated as by the creating object
    hash = CharField(indexed = True,length = 64)
    configuration = CharField(indexed = True, length = 64)
    project = ForeignKeyField('Project')
    file_revisions = ManyToManyField('FileRevision',backref = 'snapshots')
    analyzed = BooleanField(indexed = True)

    class Meta(Document.Meta):
        pass

    def load(self, data):
        """
        Imports a snapshot from a data structure
        """
        pass

    def export(self):
        """
        Exports a snapshot to a data structure
        """

    def summarize_issues(self, include_filename=False, ignore=False):
        if isinstance(self.backend, SqlBackend):
            return self._summarize_issues_sql(include_filename=include_filename, ignore=ignore)
        raise NotImplementedError

    def _summarize_issues_sql(self, include_filename=False, ignore=False):

        snapshot_file_revisions_table = self.backend.get_table(self.fields['file_revisions'].relationship_class)
        fr_table = self.backend.get_table(FileRevision)
        issue_table = self.backend.get_table(Issue)
        issue_occurrence_table = self.backend.get_table(IssueOccurrence)
        project_issue_class_table = self.backend.get_table(ProjectIssueClass)
        issue_class_table = self.backend.get_table(self.project.IssueClass)

        project_pk_type = self.backend.get_field_type(self.project.fields['pk'])
        snapshot_pk_type = self.backend.get_field_type(self.fields['pk'])

        #we group by file revision path, issue code and analyzer
        group_columns = [fr_table.c.language,fr_table.c.path,issue_table.c.code,issue_table.c.analyzer]

        #we perform a JOIN of the file revision table to the issue tables
        table = fr_table\
        .join(issue_occurrence_table,fr_table.c.pk == issue_occurrence_table.c.file_revision)\
        .join(issue_table, and_(issue_table.c.pk == issue_occurrence_table.c.issue, issue_table.c.ignore == ignore))

        #here we make sure that the given issue class is enabled for the project
        subselect = select([issue_class_table.c.pk])\
            .select_from(issue_class_table.join(project_issue_class_table))\
            .where(and_(
                issue_table.c.analyzer == issue_class_table.c.analyzer,
                issue_table.c.code == issue_class_table.c.code,
                issue_table.c.ignore == ignore,
                project_issue_class_table.c.project == expression.cast(self.project.pk,project_pk_type),
                project_issue_class_table.c.enabled == True))\

        file_revisions_select = select([snapshot_file_revisions_table.c.filerevision])\
                                .where(snapshot_file_revisions_table.c.snapshot == expression.cast(self.pk,snapshot_pk_type))

        #we select the aggregated issues for all file revisions in this snapshot
        s = select(group_columns+[func.count().label('count')])\
        .select_from(table)\
        .where(and_(exists(subselect),fr_table.c.pk.in_(file_revisions_select)))\
        .group_by(*group_columns)\
        .order_by(fr_table.c.path)

        #we fetch the result
        with self.backend.transaction():
            result = self.backend.connection.execute(s).fetchall()

        #we aggregate the issues by path fragments
        aggregator = lambda f:  directory_splitter(f['path'],include_filename = include_filename)

        #we perform a map/reduce on the result
        #the resulting items will contain the number of files and the number of issues in the file
        map_reducer = IssuesMapReducer(aggregators = [aggregator])
        return map_reducer.mapreduce(result) class Hasher(object):

    def __init__(self):
        self.digest = hashlib.md5()

    def add(self,value):
        if isinstance(value,unicode):
            v = value.encode("utf-8","ignore")
        elif isinstance(value,(str)):
            v = unicode(value,errors = 'replace').encode("utf-8","ignore")
        elif isinstance(value,(int,float,long,complex,bool)):
            v = unicode(str(value),errors = 'replace').encode("utf-8","ignore")
        elif isinstance(value,(tuple,list)):
            for v in value:
                self.add(v)
            return
        elif isinstance(value,dict):
            for key,v in sorted(value.items(),key = lambda x : x[0]):
                self.add(key)
                self.add(v)
            return
        elif value is None:
            v = '1bcdadabdf0de99dbdb747e951e967c5'
        else:
            raise AttributeError("Unhashable type: %s" % str(type(value)))

        self.digest.update(v)

    def digest(self):
        return self.digest def update(d,ud):
    """
    Recursively merge the values of ud into d.
    """
    if ud is None:
        return
    for key,value in ud.items():
        if not key in d:
            d[key] = value
        elif isinstance(value,dict):
            update(d[key],value)
        else:
            d[key] = value class CodeEnvironment(object):

    """
    Represents the full code environment of the project, including all code files.

    Responsibilities of the code environment:

    -Manage a list of file revisions
    -Manage a list of analyzers and aggregators
    -Manage project parameters that passed on to the analyzers
    """

    def __init__(self,
                 project,
                 global_settings,
                 project_settings,
                 raise_on_analysis_error = False,
                 env = None,
                 file_revisions = None,
                 ):
        self.project = project
        #global settings dictionary
        self.global_settings = global_settings
        self.project_settings = project_settings
        self.raise_on_analysis_error = raise_on_analysis_error
        self._env = env if env is not None else {}
        self._file_revisions = file_revisions

        self._active_analyzers = None
        self._active_aggregators = None
        self._analyzer_cache = {}

    def create_analysis_error(self, exception_str, location = None):
        issue_doc = {
            'code' : 'AnalysisError',
            'data' : {
                'exception' : exception_str
            },
            'location' : location,
            'fingerprint' : hashlib.sha256(exception_str).hexdigest(),
        }
        return issue_doc

    @property
    def file_revisions(self):
        return self._file_revisions

    @file_revisions.setter
    def file_revisions(self, file_revisions):
        self._file_revisions = file_revisions
        #we reset the analyzers and aggregators, as they depend
        #on the file revision information...
        self._active_analyzers = None
        self._active_aggregators = None
        self._analyzer_cache = {}

    @property
    def env(self):
        return self._env

    @property
    def analyzers(self):
        if self._active_analyzers is None:
            self._active_analyzers = self.get_active_analyzers()
        return self._active_analyzers

    @property
    def aggregators(self):
        if self._active_aggregators is None:
            self._active_aggregators = self.get_active_aggregators()
        return self._active_aggregators

    def get_language(self,file_revision):
        for language,language_pattern in self.global_settings.language_patterns.items():
            if 'patterns' in language_pattern and \
                apply_filter(file_revision['path'],language_pattern['patterns']):
                return language
        return None

    def filter_file_revisions(self,file_revisions):

        analyzer_filter = lambda filenames : filter_filenames_by_analyzers(filenames,
                                                                           self.global_settings.analyzers.values(),
                                                                           self.global_settings.language_patterns)

        filters = [analyzer_filter]

        if 'ignore' in self.project_settings:
            checkignore = self.project_settings['ignore']
            filters.append(lambda filenames : filter_filenames_by_checkignore(filenames,checkignore))

        file_revisions_by_path = {fr.path : fr for fr in file_revisions}
        filtered_paths = file_revisions_by_path.keys()

        for path_filter in filters:
            filtered_paths = path_filter(filtered_paths)

        return [file_revisions_by_path[path] for path in filtered_paths]

    def _get_active_objects(self, objs, disabled_by_default=False, obj_type='analyzers'):
        active_objs = {}
        project_settings = self.project_settings
        project_obj_settings = project_settings.get(obj_type,{})
        for name,params in objs.items():
            if 'enable' in project_obj_settings and not name in project_obj_settings['enable']:
                continue
            if 'disable' in project_obj_settings and name in project_obj_settings['disable']:
                continue
            if not name in project_obj_settings and disabled_by_default:
                continue
            obj_settings = params.copy()
            if not 'settings' in obj_settings:
                obj_settings['settings'] = {}
            project_obj_settings = project_obj_settings.get(name,{})
            if 'settings' in project_obj_settings:
                obj_settings['settings'].update(project_obj_settings['settings'])
            active_objs[name] = obj_settings
        return active_objs

    def get_active_aggregators(self, disabled_by_default=False):
        return self._get_active_objects(self.global_settings.aggregators, disabled_by_default=disabled_by_default, obj_type='aggregators')

    def get_active_analyzers(self, disabled_by_default=False):
        return self._get_active_objects(self.global_settings.analyzers, disabled_by_default=disabled_by_default, obj_type='analyzers')

    def init_analyzer(self,name,parameters):
        class_str = parameters['class']

        if class_str in self._analyzer_cache:
            return self._analyzer_cache[class_str]

        if isinstance(class_str,six.string_types):
            (module_name,separator,class_name) = class_str.rpartition(u".")
            module = __import__(module_name,globals(),locals(),[str(class_name)],-1)
            analyzer_class = getattr(module,class_name)
        else:
            analyzer_class = class_str

        try:
            analyzer = analyzer_class(self,
                                      settings = parameters.get('settings'),
                                      ignore = parameters.get('ignore')
                                      )
        except:
            logger.error("Cannot initialize analyzer {}".format(name))
            logger.error(traceback.format_exc())
            analyzer = None
        self._analyzer_cache[class_str] = analyzer
        return analyzer

    def diff_snapshots(self,snapshot_a,snapshot_b,save = True, diff=None):

        """
        Returns a list of
        """

        file_revisions_a = snapshot_a.file_revisions
        file_revisions_b = snapshot_b.file_revisions

        file_revisions_diff = diff_objects(file_revisions_a,
                                           file_revisions_b,
                                           file_revision_key,
                                           file_revision_comparator)

        #We just generate code objects and issues
        #for the modified file revisions, to save time when diffing.

        logger.debug("Generating list of modified file revisions...")
        modified_file_revisions_by_path = {}
        for fr_type in ('modified','added','deleted'):
            for fr in file_revisions_diff[fr_type]:
                if not fr.path in modified_file_revisions_by_path:
                    modified_file_revisions_by_path[fr.path] = fr

        logger.debug("Generating list of modified issues...")

        modified_file_revisions_a = [fr for fr in file_revisions_a
                                     if fr.path in modified_file_revisions_by_path]
        modified_file_revisions_b = [fr for fr in file_revisions_b
                                     if fr.path in modified_file_revisions_by_path]

        if modified_file_revisions_a:
            #to do: check the file revisions chunk-wise to avoid DB query errors
            issue_occurrences_a = self.project.backend.filter(IssueOccurrence,
                                         {
                                            'file_revision' : {'$in' : modified_file_revisions_a}
                                         },
                                      include = ('file_revision','issue'))
        else:
            issue_occurrences_a = []

        if modified_file_revisions_b:
            #to do: check the file revisions chunk-wise to avoid DB query errors
            issue_occurrences_b = self.project.backend.filter(IssueOccurrence,
                                           {
                                            'file_revision' : {'$in' : modified_file_revisions_b}
                                            },
                                      include = ('file_revision','issue'))
        else:
            issue_occurrences_b = []

        logger.debug("Diffing issues (%d in A, %d in B)" % (len(issue_occurrences_a),
                                                           len(issue_occurrences_b)))


        issue_occurrences_diff = diff_objects(issue_occurrences_a,
                                              issue_occurrences_b,
                                              issue_occurrence_key,
                                              issue_occurrence_comparator)

        logger.debug("Diffing summary...")
        summary_diff = self.diff_summaries(snapshot_a,snapshot_b)

        if diff is None:
            diff = Diff({'summary' : summary_diff,
                        'snapshot_a' : snapshot_a,
                        'project' : self.project,
                        'configuration' : self.project.configuration,
                        'snapshot_b' : snapshot_b})
            #we generate the hash value for this diff
            hasher = Hasher()
            hasher.add(diff.snapshot_a.hash)
            hasher.add(diff.snapshot_b.hash)
            diff.hash = hasher.digest.hexdigest()
        elif save:
            with self.project.backend.transaction():
                self.project.backend.filter(DiffFileRevision,{'diff' : diff}).delete()
                self.project.backend.filter(DiffIssueOccurrence,{'diff' : diff}).delete()
        if save:
            with self.project.backend.transaction():
                self.project.backend.save(diff)

        diff_file_revisions = []

        with self.project.backend.transaction():
            for key,file_revisions in file_revisions_diff.items():
                for file_revision in file_revisions:
                    hasher = Hasher()
                    hasher.add(file_revision.hash)
                    hasher.add(diff.hash)
                    hasher.add(key)
                    diff_file_revision = DiffFileRevision({
                                            'diff' : diff,
                                            'file_revision' : file_revision,
                                            'hash' : hasher.digest.hexdigest(),
                                            'key' : key})
                    if save:
                        self.project.backend.save(diff_file_revision)
                    diff_file_revisions.append(diff_file_revision)

        diff_issue_occurrences = []
        mapping = {'deleted' : 'fixed','added' : 'added'}
        with self.project.backend.transaction():
            for key,issue_occurrences in issue_occurrences_diff.items():
                if not key in mapping:
                    continue
                for issue_occurrence in issue_occurrences:
                    hasher = Hasher()
                    hasher.add(issue_occurrence.hash)
                    hasher.add(diff.hash)
                    hasher.add(mapping[key])
                    diff_issue_occurrence = DiffIssueOccurrence({
                            'diff' : diff,
                            'hash' : hasher.digest.hexdigest(),
                            'issue_occurrence' : issue_occurrence,
                            'key' : mapping[key]
                        })
                    if save:
                        self.project.backend.save(diff_issue_occurrence)
                    diff_issue_occurrences.append(diff_issue_occurrence)

        return diff,diff_file_revisions,diff_issue_occurrences

    def diff_summaries(self,snapshot_a,snapshot_b):

        summary = {}

        if not hasattr(snapshot_a,'summary') or not hasattr(snapshot_b,'summary'):
            return summary

        languages = set(snapshot_a.summary.keys()+snapshot_b.summary.keys())

        for language in languages:

            summary[language] = {}

            if not language in snapshot_a.summary or not language in snapshot_b.summary:
                continue

            language_summary_a = snapshot_a.summary[language]
            language_summary_b = snapshot_b.summary[language]

            for analyzer_name,analyzer_params in self.analyzers.items():

                if not analyzer_name in language_summary_a \
                   or not analyzer_name in language_summary_b:
                    continue

                summary[language][analyzer_name] = {}

                analyzer = self.init_analyzer(analyzer_name,analyzer_params)
                if analyzer is None:
                    continue

                for key in language_summary_a[analyzer_name]:
                    if not key in language_summary_b[analyzer_name]:
                        continue
                    result = analyzer.diff_summary(language_summary_a[analyzer_name][key],
                                                   language_summary_b[analyzer_name][key]
                                                  )
                    if result:
                        summary[language][analyzer_name][key] = result

        return summary

    def summarize(self,
                  file_revisions,
                  significance_limit = 0.01,
                  include_analysis_time = True):

        if not file_revisions:
            return {}

        results = defaultdict(lambda : defaultdict(lambda: defaultdict(dict)))
        file_revisions_by_key = defaultdict(lambda : {})

        for aggregator in self.aggregators.values():
            for file_revision in file_revisions:
                keys = aggregator['mapper'](file_revision)
                for key in keys:
                    if not file_revision['path'] in file_revisions_by_key[key]:
                        file_revisions_by_key[key][file_revision['path']] = file_revision

        for language in set([analyzer['language'] for analyzer in self.analyzers.values()]):
            for analyzer_name,analyzer_params in {name : analyzer
                    for name, analyzer in self.analyzers.items()
                    if analyzer['language'] == language}.items():

                    analyzer = self.init_analyzer(analyzer_name,analyzer_params)
                    if analyzer is None:
                        continue

                    for key in file_revisions_by_key:
                        try:
                            if hasattr(analyzer,'summarize_all'):
                                #If the analyzer has a `summarize_all` function we call it with the
                                #results from ALL analyzers and its own name.
                                results[language][analyzer_name][key] = analyzer\
                                                                        .summarize_all([f['results']
                                    for f in file_revisions_by_key[key].values()
                                    if 'results' in f and f['language'] == language]
                                    ,analyzer_name)
                            else:
                                results[language][analyzer_name][key] = analyzer.summarize([
                                    f['results'][analyzer_name]
                                    for f in file_revisions_by_key[key].values()
                                    if 'results' in f and f['language'] == language
                                    and analyzer_name in f['results']])
                        except Exception as e:
                            traceback.print_exc()
                            logger.error("Could not summarize results for analyzers %s and key %s" %
                                (analyzer_name,key))
                            raise
                            continue

            results[language] = dict(results[language])

        results = dict(results)
        return results

    def analyze_file_revisions(self,file_revisions):

        filtered_file_revisions =  self.filter_file_revisions(file_revisions)

        for file_revision in filtered_file_revisions:
            logger.info("Analyzing file revision "+file_revision['path'])
            file_revision.language = self.get_language(file_revision)
            file_revision.results = self.analyze_file_revision(file_revision,
                {analyzer_name : analyzer_params
                    for analyzer_name,analyzer_params in self.analyzers.items()
                    if analyzer_params['language'] == file_revision.language})

        return filtered_file_revisions

    def analyze_file_revision(self,file_revision,analyzers):

        analysis_time = {}
        results = {}

        for analyzer_name,analyzer_params in analyzers.items():
            try:

                analyzer = self.init_analyzer(analyzer_name,analyzer_params)
                if analyzer is None:
                    continue

                start = time.time()
                analyzer_results = analyzer.analyze(file_revision)
                stop = time.time()

                analysis_time[analyzer_name] = stop-start

                if analyzer_results:
                    results[analyzer_name] = analyzer_results
                else:
                    results[analyzer_name] = {}

            except Exception as e:
                if self.raise_on_analysis_error:
                    raise
                issue = self.create_analysis_error('An exception occurred during the analysis of this file.')
                logger.error(traceback.format_exc())
                results[analyzer_name] = {'issues' :  [issue]}

        results['analysis_time'] = dict(analysis_time)

        return results

    def analyze(self,file_revisions, save_if_empty = False, snapshot=None):

        """
        Handling dependencies:

        * First, genreate a list of file revisions for this snapshot
        * Then, check which ones of of them already exist
        * For the existing ones, check their dependencies
        * If any of the dependencies are outdated, add the dependent file revision to the analyze list

        How to handle hashes? Dependencies should be included in the hash sum.

        * Just load the files based on their SHA values
        * Check if dependencies match with the current set based on SHA values
        * If not, re-analyze the file revision
        * After analysis, calculate the hash value based on path, SHA and dependencies
        """

        logger.debug("Analyzing code environment...")

        if snapshot is None:
            snapshot = Snapshot()
            snapshot.configuration = self.project.configuration

        file_revisions_by_pk = dict([(fr.hash,fr) for fr in file_revisions])

        filtered_file_revisions = self.filter_file_revisions(file_revisions)
        filtered_file_revisions_by_pk = dict([(fr.hash,fr) for fr in filtered_file_revisions])

        excluded_file_revisions = [file_revisions_by_pk[pk]
                                    for pk in file_revisions_by_pk.keys()
                                    if not pk in filtered_file_revisions_by_pk
                                  ]

        logger.info("Excluding %d file revisions" % len(excluded_file_revisions))

        file_revisions = filtered_file_revisions
        file_revisions_by_pk = filtered_file_revisions_by_pk

        max_file_revisions = 10000

        if len(file_revisions) > max_file_revisions:

            logger.warning("Too many file revisions (%d) in snapshot, truncating at %d" %
                         (len(file_revisions),max_file_revisions))
            file_revisions_by_pk = dict(sorted(file_revisions_by_pk.items(),
                                               key = lambda x:x[0])[:max_file_revisions])
            file_revisions = file_revisions_by_pk.values()

        i = 0
        chunk_size = 50
        existing_file_revisions = []
        file_revisions_by_pk_keys = file_revisions_by_pk.keys()

        #we only check 50 keys at a time and then incrementally save them
        while i < len(file_revisions_by_pk_keys):
            file_revisions_by_pk_chunk = file_revisions_by_pk_keys[i:i+chunk_size]
            if not file_revisions_by_pk_chunk:
                break
            existing_file_revisions.extend(list(self.project.backend.filter(FileRevision,{
                    'project' : self.project,
                    'hash' : {'$in' : file_revisions_by_pk_chunk}
                    })))
            i+=chunk_size

        existing_file_revisions_by_pk = dict([(fr.hash,fr) for fr in existing_file_revisions])
        new_file_revisions = [file_revision for file_revision in file_revisions
                                if not file_revision.hash in existing_file_revisions_by_pk]

        new_file_revisions = []

        for file_revision in file_revisions:
            if not file_revision.hash in existing_file_revisions_by_pk:
                file_revision.configuration = self.project.configuration
                new_file_revisions.append(file_revision)
            elif existing_file_revisions_by_pk[file_revision.hash].configuration != self.project.configuration:
                #we replace the pk and configuration values of the new file_revision object, so that
                #it will overwrite the old version...
                file_revision.pk = existing_file_revisions_by_pk[file_revision.hash].pk
                file_revision.configuration = self.project.configuration
                new_file_revisions.append(file_revision)

        file_revisions_dict = {}

        for file_revision in existing_file_revisions+new_file_revisions:
            file_revisions_dict[file_revision.path] = file_revision

        logger.info("Analyzing %d new file revisions (%d are already analyzed)" % (
                len(new_file_revisions),
                len(existing_file_revisions)
                ))
        i = 0

        #We set the project information in the snapshot.
        snapshot.project = self.project
        snapshot.file_revisions = file_revisions_dict.values()
        self.env['snapshot'] = snapshot

        try:
            while i < len(new_file_revisions):
                j = i+10 if i+10 < len(new_file_revisions) else len(new_file_revisions)
                logger.info("Analyzing and saving: %d - %d (%d remaining)" %
                    (i, j, len(new_file_revisions) - i ))
                file_revisions_slice = new_file_revisions[i:j]
                analyzed_file_revisions = self.analyze_file_revisions(file_revisions_slice)
                logger.info("Annotating and saving file revisions...")
                self.save_file_revisions(snapshot,analyzed_file_revisions)
                i+=10
            logger.info("Summarizing file revisions...")
            snapshot.summary = self.summarize(file_revisions_dict.values())
        finally:
            del self.env['snapshot']

        snapshot.analyzed = True

        logger.info("Saving snapshot...")

        with self.project.backend.transaction():
            self.project.backend.save(snapshot)

        logger.info("Done analyzing snapshot %s" % snapshot.pk)

        return snapshot

    def save_file_revisions(self,snapshot,file_revisions):
        """
        We convert various items in the file revision to documents,
        so that we can easily search and retrieve them...
        """

        annotations = defaultdict(list)

        for file_revision in file_revisions:
            issues_results = {}

            for analyzer_name,results in file_revision.results.items():

                if 'issues' in results:
                    issues_results[analyzer_name] = results['issues']
                    del results['issues']
                    if len(issues_results) > 1000:
                        issues_results[analyzer_name] = [{
                                'code' : 'TooManyIssues',
                                'analyzer' : analyzer_name,
                            }]

            with self.project.backend.transaction():
                self.project.backend.save(file_revision)

            def location_sorter(issue):
                if issue['location'] and issue['location'][0] and issue['location'][0][0]:
                    return issue['location'][0][0][0]
                return 0

            with self.project.backend.transaction():
                for analyzer_name,issues in issues_results.items():
                    grouped_issues = group_issues_by_fingerprint(issues)
                    for issue_dict in grouped_issues:

                        hasher = Hasher()
                        hasher.add(analyzer_name)
                        hasher.add(issue_dict['code'])
                        hasher.add(issue_dict['fingerprint'])
                        issue_dict['hash'] = hasher.digest.hexdigest()

                        try:
                            #we check if the issue already exists
                            issue = self.project.backend.get(Issue,{'hash' : issue_dict['hash'],
                                                                    'project' : self.project
                                                                   })
                        except Issue.DoesNotExist:
                            #if not, we create it
                            d = issue_dict.copy()
                            d['analyzer'] = analyzer_name
                            if 'location' in d:
                                del d['location']
                            if 'occurrences' in d:
                                del d['occurrences']
                            issue = Issue(d)
                            issue.project = self.project
                            self.project.backend.save(issue)

                        for occurrence in issue_dict['occurrences']:
                            hasher = Hasher()
                            hasher.add(file_revision.hash)
                            hasher.add(issue.hash)
                            hasher.add(occurrence.get('from_row'))
                            hasher.add(occurrence.get('from_column'))
                            hasher.add(occurrence.get('to_row'))
                            hasher.add(occurrence.get('to_column'))
                            hasher.add(occurrence.get('sequence'))
                            occurrence['hash'] = hasher.digest.hexdigest()

                            try:
                                #we check if the occurrence already exists
                                occurrence = self.project.backend.get(IssueOccurrence,{'hash' : occurrence['hash'],
                                                                                       'issue' : issue
                                                                                      })
                            except IssueOccurrence.DoesNotExist:
                                #if not, we create it
                                occurrence = IssueOccurrence(occurrence)
                                occurrence.issue = issue
                                occurrence.file_revision = file_revision
                                self.project.backend.save(occurrence)
                            annotations['occurrences'].append(occurrence)
                        annotations['issues'].append(issue)

        return annotations
from checkmate.management.commands.analyze import Command as AnalyzeCommand
from .base import Command as BaseCommand
from ..lib.repository import group_snapshots_by_date,get_first_date_for_group
from ..models import GitBranch,GitSnapshot
from checkmate.lib.models import Diff, Snapshot
from checkmate.helpers.hashing import Hasher
from checkmate.helpers.settings import update
from checkmate.lib.code import CodeEnvironment
import time
import datetime
import traceback
import logging

    def analyze_grouped_snapshots(self,branch,group,grouped_snapshots):
        for key,snapshots in grouped_snapshots.items():
            last_snp,first_snp = snapshots[0],snapshots[-1]
            self.analyze_and_generate_diffs(branch,
                                            [first_snp,last_snp],
                                            [{'snapshot_a' : first_snp.sha,
                                              'snapshot_b' : last_snp.sha}])

    def analyze_and_generate_diffs(self,branch,snapshots,diff_list):

        analyzed_snapshots = {}
        new_analyzed_snapshots = {}

        if 'project_settings' in self.opts:
            project_settings = self.opts['project_settings']
        else:
            project_settings = self.project.settings

        #we look for a .checkmate.yml file in the main branch of the project
        #if we find it, we either replace the project settings, update them
        #or do nothing, depending on the desired configuration.
        try:
            git_settings = self.project.git.get_settings()
            if git_settings is not None:
                if git_settings.get('overwrite_project_settings',False):
                    project_settings = git_settings
                elif not project_settings.get('ignore_git_settings',False):
                    project_settings.update(git_settings)
        except ValueError:
            logger.warning("Error when loading checkmate settings from git...")
            git_settings = {}

        code_environment = CodeEnvironment(self.project,
                                           global_settings=self.settings,
                                           project_settings=project_settings)

        code_environment.env['branch'] = branch
        code_environment.env['project'] = self.project

        for git_snapshot in snapshots:
            try:
                query = {'sha' : git_snapshot.sha,
                         'project' : self.project}
                git_snapshot = self.backend.get(GitSnapshot,query,include = ('snapshot',))
                snapshot = git_snapshot.snapshot
                #we check if the snapshot is analyzed and if the analysis configuration matches
                if snapshot.analyzed and snapshot.configuration == self.project.configuration:
                    analyze_snapshot = False
                else:
                    analyze_snapshot = True
                    snapshot.configuration = self.project.configuration
            except GitSnapshot.DoesNotExist:
                analyze_snapshot = True
                snapshot = Snapshot()
                snapshot.configuration = self.project.configuration
            except GitSnapshot.MultipleDocumentsReturned:
                logger.error("Multiple snapshots returned, deleting...")
                with self.backend.transaction():
                    self.backend.filter(GitSnapshot,query).delete()
                analyze_snapshot = True

            if analyze_snapshot:

                try:
                    file_revisions = self.project.git.get_file_revisions(git_snapshot.sha)
                except:
                    logger.error("Cannot fetch file revisions for snapshot %s in project %s" % (git_snapshot.sha,self.project.pk) )
                    continue

                code_environment.file_revisions = file_revisions
                git_snapshot.snapshot = code_environment.analyze(file_revisions, save_if_empty = True, snapshot=snapshot)

                new_analyzed_snapshots[git_snapshot.sha] = git_snapshot

                with self.backend.transaction():
                    git_snapshot.snapshot.hash = git_snapshot.sha
                    self.backend.save(git_snapshot.snapshot)
                    git_snapshot.project = self.project
                    self.backend.save(git_snapshot)

            analyzed_snapshots[git_snapshot.sha] = git_snapshot

        snapshot_pairs = []
        for diff_params in diff_list:
            try:
                snapshot_pairs.append([analyzed_snapshots[diff_params['snapshot_a']]
                                      ,analyzed_snapshots[diff_params['snapshot_b']]])
            except KeyError:
                continue

        diffs = []

        for git_snapshot_a,git_snapshot_b in snapshot_pairs:
            diff = None
            try:
                query = {'snapshot_a' : git_snapshot_a.snapshot,
                         'snapshot_b' : git_snapshot_b.snapshot}
                diff = self.backend.get(Diff,query)
            except Diff.DoesNotExist:
                logger.debug("Generating a diff between snapshots %s and %s" % (git_snapshot_a.sha,
                                                                               git_snapshot_b.sha))
            #if the configuration of the diff does not match the project configuration, we regenerate it
            if diff is None or diff.configuration != self.project.configuration:
                if diff is not None:
                    diff.configuration = self.project.configuration
                diff,diff_file_revisions,diff_issue_occurrences = code_environment.diff_snapshots(
                                                            git_snapshot_a.snapshot,
                                                            git_snapshot_b.snapshot,
                                                            save = True,
                                                            diff = diff)

            diffs.append(diff)

        return analyzed_snapshots,diffs

    def update_branch(self,branch_name,head_snapshot = None):

        logger.debug("Updating info for branch %s" % branch_name)


        hasher = Hasher()