class ConsultationPlatform(models.Model):
    name = models.CharField(max_length=50)
    url = models.URLField(null=True, blank=True)
    connector = models.OneToOneField(connectors.models.Connector)

    def __unicode__(self):
        return self.name class Initiative(models.Model):
    external_id = models.IntegerField(editable=False)
    community_id = models.IntegerField(editable=True)
    name = models.CharField(max_length=50, editable=False)
    platform = models.ForeignKey(ConsultationPlatform, editable=True)
    social_network = models.ManyToManyField(SocialNetworkApp, blank=True)
    hashtag = models.CharField(unique=True, max_length=14, null=True,
                               help_text="Max length 14 characters (do not include '#')")
    url = models.URLField(editable=False)
    site_url = models.URLField(editable=True, default=None, null=True, blank=True)
    survey_url = models.URLField(editable=True, default=None, null=True, blank=True)
    users = models.IntegerField(editable=False, default=0)
    ideas = models.IntegerField(editable=False, default=0)
    votes = models.IntegerField(editable=False, default=0)
    comments = models.IntegerField(editable=False, default=0)
    active = models.BooleanField(default=False)
    language = models.CharField(max_length=5, default='en', choices=(('en', 'English'), ('es', 'Spanish'),
                                                                     ('it', 'Italian'),))
    notification_emails = models.BooleanField(default=False)

    def __unicode__(self):
        return self.name class Idea(BaseContent):
    title = models.CharField(max_length=255, null=True)
    text = models.TextField()
    url = models.URLField(null=True)
    location = models.ForeignKey(Location, null=True)
    # Social Network Metrics
    re_posts = models.IntegerField(default=0)       # e.g. Share in Facebook, RT in Twitter
    bookmarks = models.IntegerField(default=0)      # e.g. Favourite in Twitter
    # Additional Metric
    positive_votes = models.IntegerField(default=0)
    negative_votes = models.IntegerField(default=0)
    comments = models.IntegerField(default=0) class Comment(BaseContent):
    text = models.TextField()
    url = models.URLField(null=True)
    location = models.ForeignKey(Location, null=True)
    parent = models.CharField(max_length=10, choices=(('idea','Idea'),('comment','Comment'),))
    parent_idea = models.ForeignKey(Idea, null=True)
    parent_comment = models.ForeignKey('Comment', null=True)
    # Additional Metric
    positive_votes = models.IntegerField(default=0)
    negative_votes = models.IntegerField(default=0)
    comments = models.IntegerField(default=0) class SocialNetworkApp(models.Model):
    name = models.CharField(max_length=50)
    url = models.URLField(null=True, blank=True)
    connector = models.ForeignKey(connectors.models.SocialNetworkConnector)
    blocked = models.DateTimeField(null=True, editable=False, default=None)
    app_id = models.CharField(max_length=50)
    app_secret = models.CharField(max_length=50, null=True, blank=True)
    redirect_uri = models.URLField(null=True, blank=True)
    community = models.ForeignKey(SocialNetworkAppCommunity, null=True, blank=True)
    app_access_token = models.CharField(max_length=300, null=True, blank=True)
    callback_real_time_updates = models.URLField(null=True, blank=True)
    object_real_time_updates = models.CharField(max_length=100, null=True, blank=True, default='page')
    field_real_time_updates = models.CharField(max_length=50, null=True, blank=True, default='feed')
    token_real_time_updates = models.CharField(max_length=100, null=True, editable=False)
    subscribed_read_time_updates = models.BooleanField(default=False, editable=False)
    last_real_time_update_sig = models.CharField(max_length=100, null=True, editable=False)
    batch_requests = models.BooleanField(default=False)
    max_batch_requests = models.IntegerField(null=True, blank=True)

    def __unicode__(self):
        return self.name def save_sn_post(sn_app, post, initiative):
    if _is_story_from_admin(sn_app, post):
        if 'text' in post.keys():
            logger.info('The post will be ignored since it is a story and has been placed by the administrator. '
                        'Text: {}'.format(convert_to_utf8_str(post['text'])))
    else:
        hashtags = _extract_hashtags(post)
        if len(hashtags) > 0 and initiative:
            campaign = _get_campaign(hashtags, initiative)
            if campaign:
                try:
                    filters = {'sn_id': post['id']}
                    idea_attrs = {'sn_id': post['id'], 'source': 'social_network', 'datetime': post['datetime'],
                                  'title': post['title'], 'text': post['text'], 'url': post['url'],
                                  'comments': post['comments'], 'initiative': initiative, 'campaign': campaign,
                                  'positive_votes': post['positive_votes'], 'negative_votes': post['negative_votes'],
                                  'source_social': sn_app}
                    editable_fields = ('title', 'text', 'comments', 'positive_votes', 'negative_votes')
                    changeable_fields = ('title', 'text')
                    idea = update_or_create_content(sn_app, post, Idea, filters, idea_attrs, editable_fields,
                                                    'social_network', changeable_fields)
                    # Check if the post has an auto comment, if it has then delete the comment
                    auto_comment = _post_auto_commented(post)
                    if auto_comment:
                        params = {'app': sn_app, 'id_comment': auto_comment.sn_id, 'app_user': auto_comment.author}
                        try:
                            call_social_network_api(sn_app.connector, 'delete_comment', params)
                            auto_comment.exist = False
                            auto_comment.save()
                        except Exception as e:
                            logger.warning('An error occurred when trying to delete the auto comment placed '
                                           'on the post \'{}\'. Reason: {}'.format(convert_to_utf8_str(post['text']), e))
                    return {'idea': idea, 'initiative': initiative, 'campaign': campaign}
                except Exception as e:
                    logger.warning('An error occurred when trying to insert/update the post {} in the db. '
                                   'Reason: {}'.format(post, e))
                    logger.warning(traceback.format_exc())
            else:
                _send_notification_comment(sn_app, post, initiative, 'wrong_hashtag')
                if 'text' in post.keys():
                    logger.info('The post \'{}\' could not be created/updated. Reason: The campaign could not be '
                                'identified from the hashtags.'.format(convert_to_utf8_str(post['text'])))
                else:
                    logger.info('A post could not be created/updated. Reason: It seems it does not have hashtags')
        else:
            _send_notification_comment(sn_app, post, initiative, 'missing_hashtag')
            if 'text' in post.keys():
                logger.info('The post \'{}\' could not be created/updated. Reason: It seems it does not have hashtags'.
                            format(convert_to_utf8_str(post['text'])))
            else:
                logger.info('A post could not be created/updated. Reason: It seems it does not have hashtags')
    return None def save_sn_comment(sn_app, comment):
    try:
        #Ignore auto comments
        AutoComment.objects.get(sn_id=comment['id'])
        return
    except AutoComment.DoesNotExist:
        if comment['parent_type'] == 'post':
            comment.update({'parent_type': 'idea'})
        initiative = _find_initiative(comment['parent_type'], comment['parent_id'])
        if initiative:
            comment = do_create_update_comment(sn_app, initiative, comment, 'social_network')
            return {'comment': comment, 'initiative': initiative}
        else:
            logger.warning('An error occurred when trying to create/update the comment {}. '
                           'Reason: The initiative could not be found'.format(comment))
            return None def save_sn_vote(sn_app, vote):
    if vote['parent_type'] == 'post':
        vote.update({'user_id': vote['user_info']['id'], 'parent_type': 'idea'})
    else:
        vote.update({'user_id': vote['user_info']['id']})
    initiative = _find_initiative(vote['parent_type'], vote['parent_id'])
    if initiative:
        do_create_update_vote(sn_app, initiative, vote, 'social_network')
    else:
        logger.warning('An error occurred when trying to create the vote {}. '
                       'Reason: The initiative could not be found'.format(vote))
        return None def cud_initiative_votes(platform, initiative):
    # Fetch initiative's votes
    connector = platform.connector
    url_cb = get_url_cb(connector, 'get_votes_ideas_cb')
    #print ("<<<<<<<<<<<<< URL CB =" + str(url_cb))
    url = build_request_url(url_cb.url, url_cb.callback, {'initiative_id': initiative.external_id})
    resp = do_request(connector, url, url_cb.callback.method)
    votes = get_json_or_error(connector.name, url_cb.callback, resp)
    url_cb = get_url_cb(connector, 'get_votes_comments_cb')
    url = build_request_url(url_cb.url, url_cb.callback, {'initiative_id': initiative.external_id})
    resp = do_request(connector, url, url_cb.callback.method)
    votes += get_json_or_error(connector.name, url_cb.callback, resp)
    for vote in votes:
        do_create_update_vote(platform, initiative, vote, 'consultation_platform') def cud_initiative_ideas(platform, initiative):
    # Fetch ideas
    connector = platform.connector
    ideas_url_cb = get_url_cb(connector, 'get_ideas_cb')
    url = build_request_url(ideas_url_cb.url, ideas_url_cb.callback, {'initiative_id': initiative.external_id})
    resp = do_request(connector, url, ideas_url_cb.callback.method)
    ideas = get_json_or_error(connector.name, ideas_url_cb.callback, resp)
    # Iterate over initiative's ideas
    for idea in ideas:
        try:
            campaign = Campaign.objects.get(external_id=idea['campaign_info']['id'])
        except Campaign.DoesNotExist:
            logger.warning('Couldn\'t find the campaign with the id {} within the initiative {}. '
                           'Idea {} cannot be synchronized.'.format(idea['campaign_info']['id'], initiative.name,
                                                                    idea['id']))
            continue
        filters = {'cp_id': idea['id']}
        idea_attrs = {'cp_id': idea['id'], 'source': 'consultation_platform', 'datetime': idea['datetime'],
                      'title': idea['title'], 'text': idea['text'], 'url': idea['url'], 'comments': idea['comments'],
                      'initiative': initiative, 'campaign': campaign, 'positive_votes': idea['positive_votes'],
                      'negative_votes': idea['negative_votes'], 'source_consultation': platform}
        editable_fields = ('title', 'text', 'comments', 'positive_votes', 'negative_votes')
        changeable_fields = ('title', 'text', 'positive_votes', 'negative_votes')
        update_or_create_content(platform, idea, Idea, filters, idea_attrs, editable_fields, 'consultation_platform',
                                 changeable_fields) def cud_initiative_comments(platform, initiative):
    # Fetch initiative's comments
    connector = platform.connector
    url_cb = get_url_cb(connector, 'get_comments_cb')
    url = build_request_url(url_cb.url, url_cb.callback, {'initiative_id': initiative.external_id})
    resp = do_request(connector, url, url_cb.callback.method)
    comments = get_json_or_error(connector.name, url_cb.callback, resp)
    delayed_comments = []
    # Iterate over initiative's comments
    for comment in comments:
        comment_obj = do_create_update_comment(platform, initiative, comment, 'consultation_platform')
        if not comment_obj:
            delayed_comments.append(comment)
    for delayed_comment in delayed_comments:
        comment_obj = do_create_update_comment(platform, initiative, delayed_comment, 'consultation_platform')
        if not comment_obj:
            logger.warning('Comment {} couldn\'t be synchronized because its parent {} with id {} couldn\'t be found. '
                           '{}'.format(delayed_comment['id'], delayed_comment['parent_type'],
                                       delayed_comment['parent_id'], convert_to_utf8_str(delayed_comment['text']))) def invalidate_initiative_content(filters, update_attrs):
    # The invalidation process consist in assuming that all ideas, comments,
    # and votes don't exist anymore. Then as they are obtained from the external
    # platforms they will be marked as still existing
    Idea.objects.filter(**filters).update(**update_attrs)
    Comment.objects.filter(**filters).update(**update_attrs)
    Vote.objects.filter(**filters).update(**update_attrs) def do_push_content(obj, type, last_obj=None, batch_reqs=None):
    if obj.source == 'consultation_platform':
        # Push object to the initiative's social networks
        for social_network in obj.initiative.social_network.all():
            if _is_social_network_enabled(social_network):
            # ideas with no author email are anonymous and should not be synced
                if type == 'idea' and obj.author.email != "":
                    if social_network.batch_requests and obj.is_new:
                        if not social_network.name.lower() in batch_reqs.keys():
                            batch_reqs[social_network.name.lower()] = {'reqs': [], 'objs': []}
                        req = publish_idea_sn(obj, social_network, 'batch')
                        if req:
                            batch_reqs[social_network.name.lower()]['reqs'].\
                                append(req)
                            batch_reqs[social_network.name.lower()]['objs'].append(obj)
                        if len(batch_reqs[social_network.name.lower()]) == social_network.max_batch_requests:
                            ret = _do_batch_request(social_network, batch_reqs[social_network.name.lower()]['reqs'])
                            if ret:
                                _process_batch_request(ret, batch_reqs[social_network.name.lower()]['objs'], type)
                                batch_reqs[social_network.name.lower()]['reqs'] = []
                                batch_reqs[social_network.name.lower()]['objs'] = []
                        elif last_obj:
                            ret = _do_batch_request(social_network, batch_reqs[social_network.name.lower()]['reqs'])
                            if ret:
                                _process_batch_request(ret, batch_reqs[social_network.name.lower()]['objs'], type)
                    else:
                        publish_idea_sn(obj, social_network)
                elif type == 'comment':
                    if social_network.batch_requests and obj.is_new:
                        if not social_network.name.lower() in batch_reqs.keys():
                            batch_reqs[social_network.name.lower()] = {'reqs': [], 'objs': []}
                        req = publish_comment_sn(obj, social_network, 'batch')
                        if req:
                            batch_reqs[social_network.name.lower()]['reqs'].\
                                append(req)
                            batch_reqs[social_network.name.lower()]['objs'].append(obj)
                        if len(batch_reqs[social_network.name.lower()]) == social_network.max_batch_requests:
                            ret = _do_batch_request(social_network, batch_reqs[social_network.name.lower()]['reqs'])
                            if ret:
                                _process_batch_request(ret, batch_reqs[social_network.name.lower()]['objs'], type)
                                batch_reqs[social_network.name.lower()]['reqs'] = []
                                batch_reqs[social_network.name.lower()]['objs'] = []
                        elif last_obj:
                            ret = _do_batch_request(social_network, batch_reqs[social_network.name.lower()]['reqs'])
                            if ret:
                                _process_batch_request(ret, batch_reqs[social_network.name.lower()]['objs'], type)
                    else:
                        publish_comment_sn(obj, social_network)
                else:
                    logger.info('Objects of type {} are ignored and not synchronized'.format(type))
            else:
                logger.info('Still blocked to post on {}'.format(social_network.name))
        return batch_reqs
    else:
        # Push object to the initiative's consultation_platform
        if type == 'idea':
            publish_idea_cp(obj)
        elif type == 'comment':
            publish_comment_cp(obj) def do_delete_content(obj, type):
    initiative = obj.initiative
    if not obj.exist_cp and obj.exist_sn:
        # Delete object from the initiative's social networks
        for social_network in initiative.social_network.all():
            connector = social_network.connector
            app_user = SocialNetworkAppUser.objects.filter(email=obj.author.email, snapp=social_network)[0]
            if app_user:
                if type == 'idea':
                    params = {'app': social_network, 'id_post': obj.sn_id, 'app_user': app_user}
                    call_social_network_api(connector, 'delete_post', params)
                    logger.info('The idea {} does not exists anymore and thus it was deleted from {}'.
                                format(obj.id, social_network))
                else:
                    params = {'app': social_network, 'id_comment': obj.sn_id, 'app_user': app_user}
                    call_social_network_api(connector, 'delete_comment', params)
                    logger.info('The comment {} does not exists anymore and thus it was deleted from {}'.
                                format(obj.id, social_network))
        #obj.delete()
        _delete_obj(obj)
    elif not obj.exist_sn and obj.exist_cp:
        # Delete object from the initiative's consultation platform
        if type == 'idea':
            delete_post(obj.cp_id)
        else:
            delete_comment(obj.cp_id)
    else:
        _delete_obj(obj) def revalidate_initiative_content(filters, update_attrs):
    # The re-validation process consist in re-validating the ideas, comments,
    # and votes that were invalided before
    Idea.objects.filter(**filters).update(**update_attrs)
    Comment.objects.filter(**filters).update(**update_attrs)
    Vote.objects.filter(**filters).update(**update_attrs) def notify_new_campaigns(initiative):
    try:
        community = initiative.social_network.all()[0].community
        campaigns = Campaign.objects.filter(initiative = initiative, notified = False)
        for campaign in campaigns:
            for user in community.members.all():
                ctx = {
                    'user': user.name,
                    'campaign': campaign.name,
                    'group_url': community.url,
                    'initiative_url': initiative.url,
                    'initiative_name': initiative.name,
                    'initiative_site_url': initiative.site_url,
                    'initiative_community_id': initiative.community_id,                    
                    'hashtag' : campaign.hashtag
                }
                subject = initiative.name
                html_msg = get_template('app/email/email_new_campaign.html').render(Context(ctx))
                txt_msg = render_to_string('app/email/email_new_campaign.txt', ctx)
                ret = _send_notification_email(user.email, subject, html_msg)
            campaign.notified = True
            campaign.save()
    except Exception as e:
        logger.error('Error when try to notify new campaign: ' + str(e)) def count_other_platform_votes():
    ideas = Idea.objects.all()
    for idea in ideas:
        other_positive_votes = Vote.objects.filter(parent_idea_id=idea.id, value=1).exclude(source=idea.source).count()
        other_negative_votes = Vote.objects.filter(parent_idea_id=idea.id, value=-1).exclude(source=idea.source).count()
        msg = "other positive = {}, other negative = {}".format(other_positive_votes, other_negative_votes)
        idea.payload = msg
        idea.save()
    comments = Comment.objects.all()
    for comment in comments:
        other_positive_votes = Vote.objects.filter(parent_comment_id=comment.id, value=1).exclude(source=comment.source).count()
        other_negative_votes = Vote.objects.filter(parent_comment_id=comment.id, value=-1).exclude(source=comment.source).count()
        msg = "other positive = {}, other negative = {}".format(other_positive_votes, other_negative_votes)
        comment.payload = msg
        comment.save() def notify_new_users(initiative):
    try:
        snapp = initiative.social_network.all()[0]
        unnotified_users = SocialNetworkAppUser.objects.filter(welcome_msg_sent=False, snapp=snapp)
        for user in unnotified_users:
            try:
                ctx = {
                    'author' : user.name,
                    'initiative_name': initiative.name,
                    'initiative_site_url': initiative.site_url,
                    'initiative_community_id': initiative.community_id,
                    'survey_url': initiative.survey_url 
                }
                subject = initiative.name
                html_msg = get_template('app/email/email_new_user.html').render(Context(ctx))
                txt_msg = render_to_string('app/email/email_new_user.txt', ctx)
                ret = _send_notification_email(user.email, subject, html_msg)
                user.welcome_msg_sent = True
                user.save()
            except:
                pass
    except Exception as e:
        logger.error('Error when try to notify new users: ' + str(e)) def notify_join_group(initiative):
    try:
        #snapp = SocialNetworkApp.objects.all()[0]
        snapp = initiative.social_network.all()[0]
        users = SocialNetworkAppUser.objects.filter(snapp=snapp)
        members = get_community_members_list(snapp)
        community = snapp.community
        for user in users:
            if not is_a_community_member(user.snapp, user, members):
                now = timezone.now()
                delta = now - user.registration_timestamp
                logger.info("9nov User registration timestamp: " + str(user.registration_timestamp))
                logger.info("9nov Now timestamp: " + str(now))
                logger.info("9nov Dif: " + str(delta))
                if delta.total_seconds() <= 5*60:
                    logger.info("9nov Invitation to join group sent to " + convert_to_utf8_str(user.name))
                    ctx = {
                        'author' : user.name,
                        'initiative_name': initiative.name,
                        'initiative_site_url': initiative.site_url,
                        'initiative_community_id': initiative.community_id,
                        'community_url' : community.url
                    }
                    subject = initiative.name
                    html_msg = get_template('app/email/email_invitation_join_group.html').render(Context(ctx))
                    txt_msg = render_to_string('app/email/email_invitation_join_group.txt', ctx)
                    ret = _send_notification_email(user.email, subject, html_msg)
            else:
                logger.info('9nov This user is a community member ' + user.name)
    except Exception as e:
        logger.error('Error when try to notify users to join group: ' + str(e)) def check_reactivated_accounts_activity():
    snapps = SocialNetworkApp.objects.all()
    for snapp in snapps:
        try:
            #snapp = SocialNetworkApp.objects.all()[0]
            members = get_community_members_list(snapp)
            deactivated_ideas = Idea.objects.all().exclude(deactivation_time = None)
            for idea in deactivated_ideas:
                if idea.author.external_id in members:
                    idea.deactivation_time = None
                    idea.save()
            deactivated_comments = Comment.objects.all().exclude(deactivation_time = None)
            for comment in deactivated_comments:
                if comment.author.external_id in members:
                    comment.deactivation_time = None
                    comment.save()
        except Exception as e:
            ilogger.error('Error when trying to check reactivated accounts activity: ' + str(e)) def convert_to_utf8_str(arg):
    try:
        return smart_str(arg)
    except:
        # written by Michael Norton (http://docondev.blogspot.com/)
        if isinstance(arg, six.text_type):
            arg = arg.encode('utf-8')
        elif not isinstance(arg, bytes):
            arg = six.text_type(arg).encode('utf-8')
        elif isinstance(arg, bytes):
            arg = arg.decode('utf-8')
        return arg
        #return str(arg) def call_social_network_api(connector, method, params=None, error_handler_method=None):
    sn_class = connector.connector_class.title()
    sn_module = connector.connector_module.lower()
    sn = getattr(__import__(sn_module, fromlist=[sn_class]), sn_class)
    try:
        if params:
            return getattr(sn, method)(**params)
        else:
            return getattr(sn, method)()
    except Exception as e:
        if not error_handler_method:
            getattr(sn, 'error_handler')(method, e, params)
        else:
            getattr(sn, error_handler_method)(method, e, params) def updateAssembliesSessionKeys():
    assemblies = Assembly.objects.all()
    for assembly in assemblies:
        if assembly.keyHasExpired() or assembly.admin_session_key=="":
            assembly.updateSessionKey()
from app.models import ConsultationPlatform, Initiative, Idea, Comment, SocialNetworkApp
from app.sync import save_sn_post, save_sn_comment, save_sn_vote, cud_initiative_votes, cud_initiative_ideas, \
                     cud_initiative_comments, invalidate_initiative_content, do_push_content, do_delete_content, \
                     revalidate_initiative_content, notify_new_campaigns, count_other_platform_votes, notify_new_users, \
                     notify_join_group, check_reactivated_accounts_activity
from app.utils import convert_to_utf8_str, call_social_network_api
from appcivist.utils import updateAssembliesSessionKeys
from app.error import AppError
from celery import shared_task
from celery.utils.log import get_task_logger
from django.core.cache import cache
from django.db.models import Q
from hashlib import md5
import traceback
    logger.warning('Problem when trying to pull content of the initiative {} from {}. '
                   'The content was re-validated.'.format(initiative, platform))


def pull_data():
    # Pull data of the active initiatives from the consultation platforms
    # and social networks where they are running on
    initiatives = Initiative.objects.filter(active=True)
    for initiative in initiatives:
        invalidate_filters = {'initiative': initiative, 'is_new': False}
        try:
            invalidate_initiative_content(invalidate_filters, {'exist_cp': False})
            logger.info(u'Pulling content of the initiative {} from the platform {}'.format(initiative,
                                                                                           initiative.platform))
            _pull_content_consultation_platform(initiative.platform, initiative)
            for socialnetwork in initiative.social_network.all():
                if not socialnetwork.subscribed_read_time_updates:
                    try:
                        invalidate_initiative_content(invalidate_filters, {'exist_sn': False})
                        logger.info(u'Pulling content of the initiative {} from the platform {}'.format(initiative,
                                                                                                       socialnetwork))
                        _pull_content_social_network(socialnetwork, initiative)
                    except Exception as e:
                        _handle_pull_exceptions(initiative, socialnetwork, invalidate_filters, {'exist_sn': True})
                        raise AppError(e)
        except Exception as e:
            _handle_pull_exceptions(initiative, initiative.platform, invalidate_filters, {'exist_cp': True})
            raise AppError(e)

def push_data():
    batch_req_ideas = {}
    batch_req_comments = {}
    # Push ideas to consultation platforms and social networks
    logger.info('Pushing ideas to social networks and consultation platforms')
    existing_ideas = Idea.objects.filter(Q(exist_sn=True, sn_id__isnull=False) |
                                         Q(exist_cp=True, cp_id__isnull=False)).\
                                  filter(Q(has_changed=True) | Q(is_new=True)).\
                                  order_by('datetime')
    tot_ideas_to_sn = Idea.objects.filter(Q(exist_sn=True, sn_id__isnull=False) |
                                          Q(exist_cp=True, cp_id__isnull=False)).\
                                   filter(Q(has_changed=True) | Q(is_new=True)).\
                                   filter(source='consultation_platform').filter(is_new=True).count()
    count_ideas_to_sn = 0
    for idea in existing_ideas:
        try:
            if idea.initiative.active:
                if idea.source == 'social_network':
                    if not idea.source_social.subscribed_read_time_updates:
                        do_push_content(idea, 'idea')
                else:
                    if idea.is_new:  count_ideas_to_sn += 1
                    if count_ideas_to_sn == tot_ideas_to_sn:
                        batch_req_ideas = do_push_content(idea, 'idea', True, batch_req_ideas)
                    else:
                        batch_req_ideas = do_push_content(idea, 'idea', False, batch_req_ideas)
        except Exception as e:
            if idea.source == 'consultation_platform':
                logger.warning('Error when trying to publish the idea with the id={} on the social networks. '
                               .format(idea.id))
                logger.warning(traceback.format_exc())
            else:
                logger.warning('Error when trying to publish the idea with the id={} on the consultation platforms.'
                               .format(idea.id))
            raise AppError(e)

    # Push comments to consultation platforms and social networks
    logger.info('Pushing comments to social networks and consultation platforms')
    existing_comments = Comment.objects.filter(Q(exist_sn=True, sn_id__isnull=False) |
                                              Q(exist_cp=True, cp_id__isnull=False)).\
                                        filter(Q(has_changed=True) | Q(is_new=True)).\
                                        order_by('datetime')
    tot_comments_to_sn = Comment.objects.filter(Q(exist_sn=True, sn_id__isnull=False) |
                                                Q(exist_cp=True, cp_id__isnull=False)).\
                                         filter(Q(has_changed=True) | Q(is_new=True)).\
                                         filter(source='consultation_platform').filter(is_new=True).count()
    count_comments_to_sn = 0
    for comment in existing_comments:
        try:
            if comment.initiative.active:
                if comment.source == 'social_network':
                    if not comment.source_social.subscribed_read_time_updates:
                        do_push_content(comment, 'comment')
                else:
                    if comment.is_new: count_comments_to_sn += 1
                    if count_comments_to_sn == tot_comments_to_sn:
                        batch_req_comments = do_push_content(comment, 'comment', True, batch_req_comments)
                    else:
                        batch_req_comments = do_push_content(comment, 'comment', False, batch_req_comments)
        except Exception as e:
            if comment.source == 'consultation_platform':
                logger.warning('Error when trying to publish the comment with the id={} on the social networks.'
                               .format(comment.id))
            else:
                logger.warning('Error when trying to publish the comment with the id={} on the consultation platforms.'
                               .format(comment.id))
            raise AppError(e)
            
def delete_data():
    # Delete comments that don't exist anymore in their original social networks or consultation platforms
    logger.info('Checking whether exists comments that do not exist anymore')
    unexisting_comments = Comment.objects.filter(Q(exist_sn=False, sn_id__isnull=False) |
                                                 Q(exist_cp=False, cp_id__isnull=False))
    for comment in unexisting_comments:
        try:
            if comment.initiative.active:
                if comment.source == 'social_network':
                    if not comment.source_social.subscribed_read_time_updates:
                        do_delete_content(comment, 'comment')
                else:
                    do_delete_content(comment, 'comment')
        except Exception as e:
            if comment.source == 'consultation_platform':
                logger.warning('Error when trying to delete the comment with the id={} from {}. '
                               .format(comment.id, comment.source_consultation))
            else:
                logger.warning('Error when trying to delete the comment with the id={} from {}. '
                               .format(comment.id, comment.source_social))
            logger.warning(traceback.format_exc())
    # Delete ideas that don't exist anymore in their original social networks or consultation platforms
    logger.info('Checking whether exists ideas that do not exist anymore')

    unexisting_ideas = Idea.objects.filter(Q(exist_sn=False, sn_id__isnull=False) |