class Diff(BaseDocument):

    """
    """

    #calculated as hash(snapshot_a.hash,snapshot_b.hash)
    hash = CharField(indexed = True,length = 64)
    configuration = CharField(indexed = True, length = 64)
    project = ForeignKeyField('Project',backref = 'diffs')
    snapshot_a = ForeignKeyField('Snapshot',backref = 'diffs_a')
    snapshot_b = ForeignKeyField('Snapshot',backref = 'diffs_b')

    def get_issues_count(self, by_severity=False):
        if isinstance(self.backend, SqlBackend):
            return self._get_issues_count_sql(by_severity=by_severity)
        raise NotImplementedError

    def _get_issues_count_sql(self, by_severity=False):

        diff_issue_occurrence_table = self.backend.get_table(DiffIssueOccurrence)
        issue_class_table = self.backend.get_table(self.project.IssueClass)
        project_issue_class_table = self.backend.get_table(ProjectIssueClass)
        issue_occurrence_table = self.backend.get_table(IssueOccurrence)
        issue_table = self.backend.get_table(Issue)

        s = select([diff_issue_occurrence_table.c.key,issue_class_table.c.severity,func.count().label('count')])\
            .select_from(diff_issue_occurrence_table\
            .join(issue_occurrence_table,diff_issue_occurrence_table.c.issue_occurrence == issue_occurrence_table.c.pk)\
            .join(issue_table)\
            .join(issue_class_table,and_(issue_table.c.analyzer == issue_class_table.c.analyzer,
                                         issue_table.c.code == issue_class_table.c.code))\
            .join(project_issue_class_table,and_(
                project_issue_class_table.c.issue_class == issue_class_table.c.pk,
                project_issue_class_table.c.enabled == True,
                project_issue_class_table.c.project == self.project.pk
                )))\
            .where(diff_issue_occurrence_table.c.diff == self.pk)\
            .group_by(diff_issue_occurrence_table.c.key,issue_class_table.c.severity)


        with self.backend.transaction():
            result  = self.backend.connection.execute(s).fetchall()

        if by_severity:
            counts = {'added' : {},'fixed' : {}}
            for row in result:
                if not row['severity'] in counts[row['key']]:
                    counts[row['key']][row['severity']] = 0
                counts[row['key']][row['severity']] += row['count']
        else:
            counts = {'added' : 0,'fixed' : 0}
            for row in result:
                counts[row['key']] += row['count']
        return counts

    def _summarize_issues_sql(self, include_filename=False, ignore=False):

        diff_issue_occurrence_table = self.backend.get_table(DiffIssueOccurrence)
        issue_occurrence_table = self.backend.get_table(IssueOccurrence)
        issue_table = self.backend.get_table(Issue)
        file_revision_table = self.backend.get_table(FileRevision)
        project_issue_class_table = self.backend.get_table(ProjectIssueClass)
        issue_class_table = self.backend.get_table(self.project.IssueClass)

        #we group by file revision path, issue code and analyzer
        group_columns = [file_revision_table.c.language,
                         file_revision_table.c.path,
                         diff_issue_occurrence_table.c['key'],
                         #we should not group by pk
#                         diff_issue_occurrence_table.c['pk'],
                         issue_table.c.code,
                         issue_table.c.analyzer]

        project_pk_type = self.backend.get_field_type(self.project.fields['pk'])

        #here we make sure that the given issue class is enabled for the project
        subselect = select([issue_class_table.c.pk])\
            .select_from(issue_class_table.join(project_issue_class_table))\
            .where(and_(
            issue_table.c.analyzer == issue_class_table.c.analyzer,
            issue_table.c.code == issue_class_table.c.code,
            issue_table.c.ignore == ignore,
            project_issue_class_table.c.project == expression.cast(self.project.pk,project_pk_type),
            project_issue_class_table.c.enabled == True))\

        #we perform a JOIN of the file revision table to the issue tables
        table = diff_issue_occurrence_table\
        .join(issue_occurrence_table,
              issue_occurrence_table.c.pk == diff_issue_occurrence_table.c.issue_occurrence)\
        .join(issue_table, and_(issue_occurrence_table.c.issue == issue_table.c.pk, issue_table.c.ignore == ignore))\
        .join(file_revision_table)

        #we select the aggregated issues for all file revisions in this snapshot
        s = select(group_columns+[func.count().label('count')])\
        .select_from(table)\
        .where(and_(exists(subselect),diff_issue_occurrence_table.c.diff == self.pk))\
        .group_by(*group_columns)\
        .order_by(file_revision_table.c.path)

        #we fetch the result
        with self.backend.transaction():
            result  = self.backend.connection.execute(s).fetchall()

        #we aggregate the issues by path fragments
        aggregator = lambda f: directory_splitter(f['path'],include_filename = include_filename)

        added_issues = []
        fixed_issues = []

        for row in result:
            if row['key'] == 'added':
                added_issues.append(row)
            else:
                fixed_issues.append(row)

        #we perform a map/reduce on the result
        map_reducer = IssuesMapReducer(aggregators = [aggregator],
                                       group_by = ['language','analyzer','code'])

        return {'added': map_reducer.mapreduce(added_issues),
                'fixed': map_reducer.mapreduce(fixed_issues)}

    def summarize_issues(self,include_filename = False, ignore=False):
        if isinstance(self.backend,SqlBackend):
            return self._summarize_issues_sql(include_filename=include_filename, ignore=ignore)
        raise NotImplementedError class DiffFileRevision(BaseDocument):

    #calculated as hash(diff.hash,file_revision.hash,key)
    hash = CharField(indexed = True,length = 64)
    configuration = CharField(indexed = True,length = 64)
    diff = ForeignKeyField('Diff',backref = 'file_revisions')
    file_revision = ForeignKeyField('FileRevision',backref = 'diffs')
    key = EnumField(enums = ('added','deleted','modified')) class ProjectIssueClass(BaseDocument):

    project = ForeignKeyField('Project',backref = 'project_issue_classes')
    issue_class = ForeignKeyField('IssueClass',backref = 'project_issue_classes')
    enabled = BooleanField(default = True)

    class Meta(BaseDocument.Meta):

        unique_together = (('project','issue_class'),) class IssueClass(BaseDocument):

    class Severity:
        critical = 1
        potential_bug = 2
        minor = 3
        recommendation = 4

    hash = CharField(indexed = True,length = 64)
    title = CharField(indexed = True,length = 100)
    analyzer = CharField(indexed = True,length = 50)
    language = CharField(indexed = True,length = 50)
    code = CharField(indexed = True,length = 50)
    description = TextField(indexed = False)

    #obsolete
    occurrence_description = CharField(indexed = True,length = 2000)

    severity = IntegerField(indexed = True)
    categories = ManyToManyField('IssueCategory')

    class Meta(BaseDocument.Meta):
        unique_together = (('code','analyzer'),) class Issue(BaseDocument):

    """
    An `Issue` object represents an issue or problem with the code.
    It can be associated with one or multiple file revisions, code objects etc.

    An issue fingerprint should be a unique identifier for a given issue, hence if
    two issues have the same fingerprint they should be judged "identical".
    """

    class IgnoreReason:
        not_specified = 0
        not_relevant = 1
        false_positive = 2

    #calculated as hash(analyzer,code,fingerprint)
    hash = CharField(indexed = True,length = 64)
    configuration = CharField(indexed = True, length = 64)
    project = ForeignKeyField('Project',backref = 'issues',nullable = False)
    analyzer = CharField(indexed = True,length = 100,nullable = False)
    code = CharField(indexed = True,length = 100,nullable = False)
    fingerprint = CharField(indexed = True,length = 255,nullable = False)

    #determines if this issue should be ignored
    ignore = BooleanField(indexed=True, default=False, nullable=False, server_default=False)
    #gives a reason for the issue to be ignored (e.g. false_positive, )
    ignore_reason = IntegerField(indexed=True, nullable=True)
    #an optional comment for the ignore reason
    ignore_comment = CharField(indexed=False, length=255, nullable=True)

    class Meta(Document.Meta):
        unique_together = [('project','fingerprint','analyzer','code')]
        dbref_includes = ['code','analyzer'] class IssueOccurrence(BaseDocument):

    #can be uniquely identified by its filerevision.pk, issue.pk and from_row,to_row,from_column,to_column,sequence

    #calculated as hash(file_revision.hash,issue.hash,from_row,to_row,from_column,to_column,sequence)
    hash = CharField(indexed = True,length = 64)
    file_revision = ForeignKeyField('FileRevision',backref = 'issue_occurrences')
    issue = ForeignKeyField('Issue',backref = 'issue_occurrences')
    from_row = IntegerField()
    to_row = IntegerField()
    from_column = IntegerField()
    to_column = IntegerField()
    sequence = IntegerField(default = 0) class FileRevision(BaseDocument):

    #calculated as hash(path,sha)
    hash = CharField(indexed = True,length = 64)
    configuration = CharField(indexed = True, length = 64)
    project = ForeignKeyField('Project')
    path = CharField(indexed = True,length = 2000)
    language = CharField(indexed = True,length = 50)
    sha = CharField(indexed = True,length = 64)
    dependencies = ManyToManyField('FileRevision',backref = 'dependent_file_revisions')

    class Meta(Document.Meta):
        collection = "filerevision"

    def get_file_content(self):
        if hasattr(self,'_file_content'):
            if callable(self._file_content):
                return self._file_content()
            return self._file_content
        raise NotImplementedError class Snapshot(BaseDocument):

    #calculated as by the creating object
    hash = CharField(indexed = True,length = 64)
    configuration = CharField(indexed = True, length = 64)
    project = ForeignKeyField('Project')
    file_revisions = ManyToManyField('FileRevision',backref = 'snapshots')
    analyzed = BooleanField(indexed = True)

    class Meta(Document.Meta):
        pass

    def load(self, data):
        """
        Imports a snapshot from a data structure
        """
        pass

    def export(self):
        """
        Exports a snapshot to a data structure
        """

    def summarize_issues(self, include_filename=False, ignore=False):
        if isinstance(self.backend, SqlBackend):
            return self._summarize_issues_sql(include_filename=include_filename, ignore=ignore)
        raise NotImplementedError

    def _summarize_issues_sql(self, include_filename=False, ignore=False):

        snapshot_file_revisions_table = self.backend.get_table(self.fields['file_revisions'].relationship_class)
        fr_table = self.backend.get_table(FileRevision)
        issue_table = self.backend.get_table(Issue)
        issue_occurrence_table = self.backend.get_table(IssueOccurrence)
        project_issue_class_table = self.backend.get_table(ProjectIssueClass)
        issue_class_table = self.backend.get_table(self.project.IssueClass)

        project_pk_type = self.backend.get_field_type(self.project.fields['pk'])
        snapshot_pk_type = self.backend.get_field_type(self.fields['pk'])

        #we group by file revision path, issue code and analyzer
        group_columns = [fr_table.c.language,fr_table.c.path,issue_table.c.code,issue_table.c.analyzer]

        #we perform a JOIN of the file revision table to the issue tables
        table = fr_table\
        .join(issue_occurrence_table,fr_table.c.pk == issue_occurrence_table.c.file_revision)\
        .join(issue_table, and_(issue_table.c.pk == issue_occurrence_table.c.issue, issue_table.c.ignore == ignore))

        #here we make sure that the given issue class is enabled for the project
        subselect = select([issue_class_table.c.pk])\
            .select_from(issue_class_table.join(project_issue_class_table))\
            .where(and_(
                issue_table.c.analyzer == issue_class_table.c.analyzer,
                issue_table.c.code == issue_class_table.c.code,
                issue_table.c.ignore == ignore,
                project_issue_class_table.c.project == expression.cast(self.project.pk,project_pk_type),
                project_issue_class_table.c.enabled == True))\

        file_revisions_select = select([snapshot_file_revisions_table.c.filerevision])\
                                .where(snapshot_file_revisions_table.c.snapshot == expression.cast(self.pk,snapshot_pk_type))

        #we select the aggregated issues for all file revisions in this snapshot
        s = select(group_columns+[func.count().label('count')])\
        .select_from(table)\
        .where(and_(exists(subselect),fr_table.c.pk.in_(file_revisions_select)))\
        .group_by(*group_columns)\
        .order_by(fr_table.c.path)

        #we fetch the result
        with self.backend.transaction():
            result = self.backend.connection.execute(s).fetchall()

        #we aggregate the issues by path fragments
        aggregator = lambda f:  directory_splitter(f['path'],include_filename = include_filename)

        #we perform a map/reduce on the result
        #the resulting items will contain the number of files and the number of issues in the file
        map_reducer = IssuesMapReducer(aggregators = [aggregator])
        return map_reducer.mapreduce(result) class DiffIssueOccurrence(BaseDocument):

    #calculated as hash(diff.hash,issue_occurrence.hash,key)
    hash = CharField(indexed = True,length = 64)
    configuration = CharField(indexed = True,length = 64)
    diff = ForeignKeyField('Diff',backref = 'issue_occurrences')
    issue_occurrence = ForeignKeyField('IssueOccurrence',backref = 'diff_issue_occurrences')
    key = EnumField(enums = ('added','fixed')) class CodeEnvironment(object):

    """
    Represents the full code environment of the project, including all code files.

    Responsibilities of the code environment:

    -Manage a list of file revisions
    -Manage a list of analyzers and aggregators
    -Manage project parameters that passed on to the analyzers
    """

    def __init__(self,
                 project,
                 global_settings,
                 project_settings,
                 raise_on_analysis_error = False,
                 env = None,
                 file_revisions = None,
                 ):
        self.project = project
        #global settings dictionary
        self.global_settings = global_settings
        self.project_settings = project_settings
        self.raise_on_analysis_error = raise_on_analysis_error
        self._env = env if env is not None else {}
        self._file_revisions = file_revisions

        self._active_analyzers = None
        self._active_aggregators = None
        self._analyzer_cache = {}

    def create_analysis_error(self, exception_str, location = None):
        issue_doc = {
            'code' : 'AnalysisError',
            'data' : {
                'exception' : exception_str
            },
            'location' : location,
            'fingerprint' : hashlib.sha256(exception_str).hexdigest(),
        }
        return issue_doc

    @property
    def file_revisions(self):
        return self._file_revisions

    @file_revisions.setter
    def file_revisions(self, file_revisions):
        self._file_revisions = file_revisions
        #we reset the analyzers and aggregators, as they depend
        #on the file revision information...
        self._active_analyzers = None
        self._active_aggregators = None
        self._analyzer_cache = {}

    @property
    def env(self):
        return self._env

    @property
    def analyzers(self):
        if self._active_analyzers is None:
            self._active_analyzers = self.get_active_analyzers()
        return self._active_analyzers

    @property
    def aggregators(self):
        if self._active_aggregators is None:
            self._active_aggregators = self.get_active_aggregators()
        return self._active_aggregators

    def get_language(self,file_revision):
        for language,language_pattern in self.global_settings.language_patterns.items():
            if 'patterns' in language_pattern and \
                apply_filter(file_revision['path'],language_pattern['patterns']):
                return language
        return None

    def filter_file_revisions(self,file_revisions):

        analyzer_filter = lambda filenames : filter_filenames_by_analyzers(filenames,
                                                                           self.global_settings.analyzers.values(),
                                                                           self.global_settings.language_patterns)

        filters = [analyzer_filter]

        if 'ignore' in self.project_settings:
            checkignore = self.project_settings['ignore']
            filters.append(lambda filenames : filter_filenames_by_checkignore(filenames,checkignore))

        file_revisions_by_path = {fr.path : fr for fr in file_revisions}
        filtered_paths = file_revisions_by_path.keys()

        for path_filter in filters:
            filtered_paths = path_filter(filtered_paths)

        return [file_revisions_by_path[path] for path in filtered_paths]

    def _get_active_objects(self, objs, disabled_by_default=False, obj_type='analyzers'):
        active_objs = {}
        project_settings = self.project_settings
        project_obj_settings = project_settings.get(obj_type,{})
        for name,params in objs.items():
            if 'enable' in project_obj_settings and not name in project_obj_settings['enable']:
                continue
            if 'disable' in project_obj_settings and name in project_obj_settings['disable']:
                continue
            if not name in project_obj_settings and disabled_by_default:
                continue
            obj_settings = params.copy()
            if not 'settings' in obj_settings:
                obj_settings['settings'] = {}
            project_obj_settings = project_obj_settings.get(name,{})
            if 'settings' in project_obj_settings:
                obj_settings['settings'].update(project_obj_settings['settings'])
            active_objs[name] = obj_settings
        return active_objs

    def get_active_aggregators(self, disabled_by_default=False):
        return self._get_active_objects(self.global_settings.aggregators, disabled_by_default=disabled_by_default, obj_type='aggregators')

    def get_active_analyzers(self, disabled_by_default=False):
        return self._get_active_objects(self.global_settings.analyzers, disabled_by_default=disabled_by_default, obj_type='analyzers')

    def init_analyzer(self,name,parameters):
        class_str = parameters['class']

        if class_str in self._analyzer_cache:
            return self._analyzer_cache[class_str]

        if isinstance(class_str,six.string_types):
            (module_name,separator,class_name) = class_str.rpartition(u".")
            module = __import__(module_name,globals(),locals(),[str(class_name)],-1)
            analyzer_class = getattr(module,class_name)
        else:
            analyzer_class = class_str

        try:
            analyzer = analyzer_class(self,
                                      settings = parameters.get('settings'),
                                      ignore = parameters.get('ignore')
                                      )
        except:
            logger.error("Cannot initialize analyzer {}".format(name))
            logger.error(traceback.format_exc())
            analyzer = None
        self._analyzer_cache[class_str] = analyzer
        return analyzer

    def diff_snapshots(self,snapshot_a,snapshot_b,save = True, diff=None):

        """
        Returns a list of
        """

        file_revisions_a = snapshot_a.file_revisions
        file_revisions_b = snapshot_b.file_revisions

        file_revisions_diff = diff_objects(file_revisions_a,
                                           file_revisions_b,
                                           file_revision_key,
                                           file_revision_comparator)

        #We just generate code objects and issues
        #for the modified file revisions, to save time when diffing.

        logger.debug("Generating list of modified file revisions...")
        modified_file_revisions_by_path = {}
        for fr_type in ('modified','added','deleted'):
            for fr in file_revisions_diff[fr_type]:
                if not fr.path in modified_file_revisions_by_path:
                    modified_file_revisions_by_path[fr.path] = fr

        logger.debug("Generating list of modified issues...")

        modified_file_revisions_a = [fr for fr in file_revisions_a
                                     if fr.path in modified_file_revisions_by_path]
        modified_file_revisions_b = [fr for fr in file_revisions_b
                                     if fr.path in modified_file_revisions_by_path]

        if modified_file_revisions_a:
            #to do: check the file revisions chunk-wise to avoid DB query errors
            issue_occurrences_a = self.project.backend.filter(IssueOccurrence,
                                         {
                                            'file_revision' : {'$in' : modified_file_revisions_a}
                                         },
                                      include = ('file_revision','issue'))
        else:
            issue_occurrences_a = []

        if modified_file_revisions_b:
            #to do: check the file revisions chunk-wise to avoid DB query errors
            issue_occurrences_b = self.project.backend.filter(IssueOccurrence,
                                           {
                                            'file_revision' : {'$in' : modified_file_revisions_b}
                                            },
                                      include = ('file_revision','issue'))
        else:
            issue_occurrences_b = []

        logger.debug("Diffing issues (%d in A, %d in B)" % (len(issue_occurrences_a),
                                                           len(issue_occurrences_b)))


        issue_occurrences_diff = diff_objects(issue_occurrences_a,
                                              issue_occurrences_b,
                                              issue_occurrence_key,
                                              issue_occurrence_comparator)

        logger.debug("Diffing summary...")
        summary_diff = self.diff_summaries(snapshot_a,snapshot_b)

        if diff is None:
            diff = Diff({'summary' : summary_diff,
                        'snapshot_a' : snapshot_a,
                        'project' : self.project,
                        'configuration' : self.project.configuration,
                        'snapshot_b' : snapshot_b})
            #we generate the hash value for this diff
            hasher = Hasher()
            hasher.add(diff.snapshot_a.hash)
            hasher.add(diff.snapshot_b.hash)
            diff.hash = hasher.digest.hexdigest()
        elif save:
            with self.project.backend.transaction():
                self.project.backend.filter(DiffFileRevision,{'diff' : diff}).delete()
                self.project.backend.filter(DiffIssueOccurrence,{'diff' : diff}).delete()
        if save:
            with self.project.backend.transaction():
                self.project.backend.save(diff)

        diff_file_revisions = []

        with self.project.backend.transaction():
            for key,file_revisions in file_revisions_diff.items():
                for file_revision in file_revisions:
                    hasher = Hasher()
                    hasher.add(file_revision.hash)
                    hasher.add(diff.hash)
                    hasher.add(key)
                    diff_file_revision = DiffFileRevision({
                                            'diff' : diff,
                                            'file_revision' : file_revision,
                                            'hash' : hasher.digest.hexdigest(),
                                            'key' : key})
                    if save:
                        self.project.backend.save(diff_file_revision)
                    diff_file_revisions.append(diff_file_revision)

        diff_issue_occurrences = []
        mapping = {'deleted' : 'fixed','added' : 'added'}
        with self.project.backend.transaction():
            for key,issue_occurrences in issue_occurrences_diff.items():
                if not key in mapping:
                    continue
                for issue_occurrence in issue_occurrences:
                    hasher = Hasher()
                    hasher.add(issue_occurrence.hash)
                    hasher.add(diff.hash)
                    hasher.add(mapping[key])
                    diff_issue_occurrence = DiffIssueOccurrence({
                            'diff' : diff,
                            'hash' : hasher.digest.hexdigest(),
                            'issue_occurrence' : issue_occurrence,
                            'key' : mapping[key]
                        })
                    if save:
                        self.project.backend.save(diff_issue_occurrence)
                    diff_issue_occurrences.append(diff_issue_occurrence)

        return diff,diff_file_revisions,diff_issue_occurrences

    def diff_summaries(self,snapshot_a,snapshot_b):

        summary = {}

        if not hasattr(snapshot_a,'summary') or not hasattr(snapshot_b,'summary'):
            return summary

        languages = set(snapshot_a.summary.keys()+snapshot_b.summary.keys())

        for language in languages:

            summary[language] = {}

            if not language in snapshot_a.summary or not language in snapshot_b.summary:
                continue

            language_summary_a = snapshot_a.summary[language]
            language_summary_b = snapshot_b.summary[language]

            for analyzer_name,analyzer_params in self.analyzers.items():

                if not analyzer_name in language_summary_a \
                   or not analyzer_name in language_summary_b:
                    continue

                summary[language][analyzer_name] = {}

                analyzer = self.init_analyzer(analyzer_name,analyzer_params)
                if analyzer is None:
                    continue

                for key in language_summary_a[analyzer_name]:
                    if not key in language_summary_b[analyzer_name]:
                        continue
                    result = analyzer.diff_summary(language_summary_a[analyzer_name][key],
                                                   language_summary_b[analyzer_name][key]
                                                  )
                    if result:
                        summary[language][analyzer_name][key] = result

        return summary

    def summarize(self,
                  file_revisions,
                  significance_limit = 0.01,
                  include_analysis_time = True):

        if not file_revisions:
            return {}

        results = defaultdict(lambda : defaultdict(lambda: defaultdict(dict)))
        file_revisions_by_key = defaultdict(lambda : {})

        for aggregator in self.aggregators.values():
            for file_revision in file_revisions:
                keys = aggregator['mapper'](file_revision)
                for key in keys:
                    if not file_revision['path'] in file_revisions_by_key[key]:
                        file_revisions_by_key[key][file_revision['path']] = file_revision

        for language in set([analyzer['language'] for analyzer in self.analyzers.values()]):
            for analyzer_name,analyzer_params in {name : analyzer
                    for name, analyzer in self.analyzers.items()
                    if analyzer['language'] == language}.items():

                    analyzer = self.init_analyzer(analyzer_name,analyzer_params)
                    if analyzer is None:
                        continue

                    for key in file_revisions_by_key:
                        try:
                            if hasattr(analyzer,'summarize_all'):
                                #If the analyzer has a `summarize_all` function we call it with the
                                #results from ALL analyzers and its own name.
                                results[language][analyzer_name][key] = analyzer\
                                                                        .summarize_all([f['results']
                                    for f in file_revisions_by_key[key].values()
                                    if 'results' in f and f['language'] == language]
                                    ,analyzer_name)
                            else:
                                results[language][analyzer_name][key] = analyzer.summarize([
                                    f['results'][analyzer_name]
                                    for f in file_revisions_by_key[key].values()
                                    if 'results' in f and f['language'] == language
                                    and analyzer_name in f['results']])
                        except Exception as e:
                            traceback.print_exc()
                            logger.error("Could not summarize results for analyzers %s and key %s" %
                                (analyzer_name,key))
                            raise
                            continue

            results[language] = dict(results[language])

        results = dict(results)
        return results

    def analyze_file_revisions(self,file_revisions):

        filtered_file_revisions =  self.filter_file_revisions(file_revisions)

        for file_revision in filtered_file_revisions:
            logger.info("Analyzing file revision "+file_revision['path'])
            file_revision.language = self.get_language(file_revision)
            file_revision.results = self.analyze_file_revision(file_revision,
                {analyzer_name : analyzer_params
                    for analyzer_name,analyzer_params in self.analyzers.items()
                    if analyzer_params['language'] == file_revision.language})

        return filtered_file_revisions

    def analyze_file_revision(self,file_revision,analyzers):

        analysis_time = {}
        results = {}

        for analyzer_name,analyzer_params in analyzers.items():
            try:

                analyzer = self.init_analyzer(analyzer_name,analyzer_params)
                if analyzer is None:
                    continue

                start = time.time()
                analyzer_results = analyzer.analyze(file_revision)
                stop = time.time()

                analysis_time[analyzer_name] = stop-start

                if analyzer_results:
                    results[analyzer_name] = analyzer_results
                else:
                    results[analyzer_name] = {}

            except Exception as e:
                if self.raise_on_analysis_error:
                    raise
                issue = self.create_analysis_error('An exception occurred during the analysis of this file.')
                logger.error(traceback.format_exc())
                results[analyzer_name] = {'issues' :  [issue]}

        results['analysis_time'] = dict(analysis_time)

        return results

    def analyze(self,file_revisions, save_if_empty = False, snapshot=None):

        """
        Handling dependencies:

        * First, genreate a list of file revisions for this snapshot
        * Then, check which ones of of them already exist
        * For the existing ones, check their dependencies
        * If any of the dependencies are outdated, add the dependent file revision to the analyze list

        How to handle hashes? Dependencies should be included in the hash sum.

        * Just load the files based on their SHA values
        * Check if dependencies match with the current set based on SHA values
        * If not, re-analyze the file revision
        * After analysis, calculate the hash value based on path, SHA and dependencies
        """

        logger.debug("Analyzing code environment...")

        if snapshot is None:
            snapshot = Snapshot()
            snapshot.configuration = self.project.configuration

        file_revisions_by_pk = dict([(fr.hash,fr) for fr in file_revisions])

        filtered_file_revisions = self.filter_file_revisions(file_revisions)
        filtered_file_revisions_by_pk = dict([(fr.hash,fr) for fr in filtered_file_revisions])

        excluded_file_revisions = [file_revisions_by_pk[pk]
                                    for pk in file_revisions_by_pk.keys()
                                    if not pk in filtered_file_revisions_by_pk
                                  ]

        logger.info("Excluding %d file revisions" % len(excluded_file_revisions))

        file_revisions = filtered_file_revisions
        file_revisions_by_pk = filtered_file_revisions_by_pk

        max_file_revisions = 10000

        if len(file_revisions) > max_file_revisions:

            logger.warning("Too many file revisions (%d) in snapshot, truncating at %d" %
                         (len(file_revisions),max_file_revisions))
            file_revisions_by_pk = dict(sorted(file_revisions_by_pk.items(),
                                               key = lambda x:x[0])[:max_file_revisions])
            file_revisions = file_revisions_by_pk.values()

        i = 0
        chunk_size = 50
        existing_file_revisions = []
        file_revisions_by_pk_keys = file_revisions_by_pk.keys()

        #we only check 50 keys at a time and then incrementally save them
        while i < len(file_revisions_by_pk_keys):
            file_revisions_by_pk_chunk = file_revisions_by_pk_keys[i:i+chunk_size]
            if not file_revisions_by_pk_chunk:
                break
            existing_file_revisions.extend(list(self.project.backend.filter(FileRevision,{
                    'project' : self.project,
                    'hash' : {'$in' : file_revisions_by_pk_chunk}
                    })))
            i+=chunk_size

        existing_file_revisions_by_pk = dict([(fr.hash,fr) for fr in existing_file_revisions])
        new_file_revisions = [file_revision for file_revision in file_revisions
                                if not file_revision.hash in existing_file_revisions_by_pk]

        new_file_revisions = []

        for file_revision in file_revisions:
            if not file_revision.hash in existing_file_revisions_by_pk:
                file_revision.configuration = self.project.configuration
                new_file_revisions.append(file_revision)
            elif existing_file_revisions_by_pk[file_revision.hash].configuration != self.project.configuration:
                #we replace the pk and configuration values of the new file_revision object, so that
                #it will overwrite the old version...
                file_revision.pk = existing_file_revisions_by_pk[file_revision.hash].pk
                file_revision.configuration = self.project.configuration
                new_file_revisions.append(file_revision)

        file_revisions_dict = {}

        for file_revision in existing_file_revisions+new_file_revisions:
            file_revisions_dict[file_revision.path] = file_revision

        logger.info("Analyzing %d new file revisions (%d are already analyzed)" % (
                len(new_file_revisions),
                len(existing_file_revisions)
                ))
        i = 0

        #We set the project information in the snapshot.
        snapshot.project = self.project
        snapshot.file_revisions = file_revisions_dict.values()
        self.env['snapshot'] = snapshot

        try:
            while i < len(new_file_revisions):
                j = i+10 if i+10 < len(new_file_revisions) else len(new_file_revisions)
                logger.info("Analyzing and saving: %d - %d (%d remaining)" %
                    (i, j, len(new_file_revisions) - i ))
                file_revisions_slice = new_file_revisions[i:j]
                analyzed_file_revisions = self.analyze_file_revisions(file_revisions_slice)
                logger.info("Annotating and saving file revisions...")
                self.save_file_revisions(snapshot,analyzed_file_revisions)
                i+=10
            logger.info("Summarizing file revisions...")
            snapshot.summary = self.summarize(file_revisions_dict.values())
        finally:
            del self.env['snapshot']

        snapshot.analyzed = True

        logger.info("Saving snapshot...")

        with self.project.backend.transaction():
            self.project.backend.save(snapshot)

        logger.info("Done analyzing snapshot %s" % snapshot.pk)

        return snapshot

    def save_file_revisions(self,snapshot,file_revisions):
        """
        We convert various items in the file revision to documents,
        so that we can easily search and retrieve them...
        """

        annotations = defaultdict(list)

        for file_revision in file_revisions:
            issues_results = {}

            for analyzer_name,results in file_revision.results.items():

                if 'issues' in results:
                    issues_results[analyzer_name] = results['issues']
                    del results['issues']
                    if len(issues_results) > 1000:
                        issues_results[analyzer_name] = [{
                                'code' : 'TooManyIssues',
                                'analyzer' : analyzer_name,
                            }]

            with self.project.backend.transaction():
                self.project.backend.save(file_revision)

            def location_sorter(issue):
                if issue['location'] and issue['location'][0] and issue['location'][0][0]:
                    return issue['location'][0][0][0]
                return 0

            with self.project.backend.transaction():
                for analyzer_name,issues in issues_results.items():
                    grouped_issues = group_issues_by_fingerprint(issues)
                    for issue_dict in grouped_issues:

                        hasher = Hasher()
                        hasher.add(analyzer_name)
                        hasher.add(issue_dict['code'])
                        hasher.add(issue_dict['fingerprint'])
                        issue_dict['hash'] = hasher.digest.hexdigest()

                        try:
                            #we check if the issue already exists
                            issue = self.project.backend.get(Issue,{'hash' : issue_dict['hash'],
                                                                    'project' : self.project
                                                                   })
                        except Issue.DoesNotExist:
                            #if not, we create it
                            d = issue_dict.copy()
                            d['analyzer'] = analyzer_name
                            if 'location' in d:
                                del d['location']
                            if 'occurrences' in d:
                                del d['occurrences']
                            issue = Issue(d)
                            issue.project = self.project
                            self.project.backend.save(issue)

                        for occurrence in issue_dict['occurrences']:
                            hasher = Hasher()
                            hasher.add(file_revision.hash)
                            hasher.add(issue.hash)
                            hasher.add(occurrence.get('from_row'))
                            hasher.add(occurrence.get('from_column'))
                            hasher.add(occurrence.get('to_row'))
                            hasher.add(occurrence.get('to_column'))
                            hasher.add(occurrence.get('sequence'))
                            occurrence['hash'] = hasher.digest.hexdigest()

                            try:
                                #we check if the occurrence already exists
                                occurrence = self.project.backend.get(IssueOccurrence,{'hash' : occurrence['hash'],
                                                                                       'issue' : issue
                                                                                      })
                            except IssueOccurrence.DoesNotExist:
                                #if not, we create it
                                occurrence = IssueOccurrence(occurrence)
                                occurrence.issue = issue
                                occurrence.file_revision = file_revision
                                self.project.backend.save(occurrence)
                            annotations['occurrences'].append(occurrence)
                        annotations['issues'].append(issue)

        return annotations class ProjectBasedTest(TestCase):

    def setUp(self):

        self.tmpdir = tempfile.mkdtemp()
        self.tmp_project_path = os.path.join(self.tmpdir,"basic_project")
        os.makedirs(self.tmp_project_path)

        self.settings = Settings(
            analyzers = {},
            aggregators = {},
            commands = {}
        )

        self.settings.initialize()

        self.current_path = os.getcwd()

        os.chdir(self.tmp_project_path)
        init_command = InitCommand(None,None)
        init_command.run()

        assert os.path.exists(os.path.join(self.tmp_project_path,'.checkmate'))

        self.project,self.backend = get_project_and_backend(self.tmp_project_path,self.settings)

    def tearDown(self):
        os.chdir(self.current_path)
        shutil.rmtree(self.tmpdir)
from checkmate.lib.models import (Diff,
                                  DiffFileRevision,
                                  ProjectIssueClass,
                                  IssueClass,
                                  Issue,
                                  IssueOccurrence,
                                  FileRevision,
                                  Snapshot,
                                  DiffIssueOccurrence)
from checkmate.lib.code.environment import CodeEnvironment
from ..helpers import ProjectBasedTest



class TestDiffing(ProjectBasedTest):

    def test_simple_diff(self):
        snapshot_1 = Snapshot({'project' : self.project, 'hash' : 'snapshot_1'})
        file_revision_1 = FileRevision({'path' : 'foo.py','hash' : '1:foo.py', 'language' : 'python'})

        issue_class_1 = IssueClass({'code' : 'wildcard','analyzer' : 'example','severity' : 2})
        project_issue_class_1 = ProjectIssueClass({'project' : self.project,
                                                   'issue_class' : issue_class_1,
                                                   'enabled' : True})
        issue_1 = Issue({'code' : 'wildcard','analyzer' : 'example','fingerprint' : '1','project' : self.project})

        issue_1_occurrence = IssueOccurrence({'issue' : issue_1,
                                              'file_revision' : file_revision_1,
                                              'from_row' : 1,'from_column' : 0,
                                              'to_row' : 1,'to_column' : None})

        snapshot_1.file_revisions = [file_revision_1]

        with self.backend.transaction():
            self.backend.save(snapshot_1)
            self.backend.save(issue_class_1)
            self.backend.save(project_issue_class_1)
            self.backend.save(issue_1)
            self.backend.save(issue_1_occurrence)
            self.backend.commit()

        snapshot_2 = Snapshot({'project' : self.project, 'hash' : 'snapshot_2'})
        file_revision_2 = FileRevision({'path' : 'bar.py','hash' : '2:bar.py','language':'python'})
        snapshot_2.file_revisions = [file_revision_1,file_revision_2]
        issue_class_2 = IssueClass({'code' : 'format','analyzer' : 'example','severity' : 1})
        project_issue_class_2 = ProjectIssueClass({'project' : self.project,
                                                   'issue_class' : issue_class_2,
                                                   'enabled' : True})
        issue_2 = Issue({'code' : 'format','analyzer' : 'example','fingerprint' : '1','project' : self.project})
        issue_2_occurrence = IssueOccurrence({'issue' : issue_2,
                                              'file_revision' : file_revision_2,
                                              'from_row' : 2,'from_column' : 0,
                                              'to_row' : 3,'to_column' : None})

        with self.backend.transaction():
            self.backend.save(snapshot_2)
            self.backend.save(issue_class_2)
            self.backend.save(project_issue_class_2)
            self.backend.save(issue_2_occurrence)
            self.backend.save(issue_2)

        environment = CodeEnvironment(project = self.project,
                                      global_settings = self.settings,
                                      project_settings = {},
                                      file_revisions = [])

        #We make a simple diff
        diff,diff_file_revisions,diff_issue_occurrences = environment.diff_snapshots(snapshot_1,snapshot_2)

        #we expect one added file revision with 1 added issue occurrence
        assert len(diff_file_revisions) == 1
        assert len(diff_issue_occurrences) == 1
        assert diff_file_revisions[0].key == 'added'
        assert diff_file_revisions[0].file_revision == file_revision_2
        assert diff_issue_occurrences[0].key == 'added'
        assert diff_issue_occurrences[0].issue_occurrence == issue_2_occurrence


        snapshot_3 = Snapshot({'project' : self.project, 'hash' : 'snapshot_3'})