@default_index
def get_records_matching_field(field, id, index=None, doc_type=None, source=None):
    """ Checks if a record with a given ID exists in the index """

    query = {
        "size": 9999,
        'query': {
            "bool": {
                "must": [
                    {
                        "match": {
                            field: id
                        }
                    }
                ]
            }
        }
    }

    if doc_type:
        query["query"]["bool"]["must"].append({
            "match": {
                "doc_type": doc_type
            }
        })

    if source:
        query["_source"] = source

    return es.search(index=index, body=query) @default_index
def get_count_for_collection(doc_type, index=None):
    """

    :param doc_type: e.g. CFG_PUB_TYPE or CFG_DATA_TYPE
    :param index: name of index to use.
    :return: the number of records in that collection
    """
    return es.count(index=index, q='doc_type:'+doc_type) @default_index
def get_n_latest_records(n_latest, field="last_updated", index=None):
    """ Gets latest N records from the index """

    search = Search(using=es, index=index) \
        .filter("term", doc_type=CFG_PUB_TYPE) \
        .sort({field: {"order": "desc"}}) \
        .source(excludes=["authors", "keywords"])
    search = search[0:n_latest]

    query_result = search.execute().to_dict()
    return query_result['hits']['hits'] @default_index
def index_record_ids(record_ids, index=None):
    """ Index records given in the argument.

    :param record_ids: [list of ints] list of record ids e.g. [1, 5, 2, 3]
    :param index: [string] name of the index. If None a default is used
    :return: list of indexed publication and data recids
    """
    from hepdata.modules.records.utils.common import get_record_by_id

    docs = list(filter(None, [get_record_by_id(recid) for recid in record_ids]))

    existing_record_ids = [doc['recid'] for doc in docs]
    log.info('Indexing existing record IDs: {}'.format(existing_record_ids))

    to_index = []
    indexed_result = {CFG_DATA_TYPE: [], CFG_PUB_TYPE: []}

    for doc in docs:
        if 'related_publication' in doc:
            # Remove unnecessary fields if it's a data record
            for field in ['authors', '_additional_authors', '_first_author']:
                if field in doc:
                    del doc[field]

            enhance_data_document(doc)

            op_dict = {
                "index": {
                    "_index": index,
                    "_id": doc['recid'],
                    "routing": doc['related_publication']
                }
            }

            indexed_result[CFG_DATA_TYPE].append(doc['recid'])
            to_index.append(op_dict)

        else:

            if 'version' not in doc:
                log.warning('Skipping unfinished record ID {}'.format(doc['recid']))
                continue

            author_docs = prepare_author_for_indexing(doc)
            to_index += author_docs

            enhance_publication_document(doc)

            op_dict = {
                "index": {
                    "_index": index,
                    "_id": doc['recid'],
                    "routing": doc['recid']
                }
            }

            indexed_result[CFG_PUB_TYPE].append(doc['recid'])
            to_index.append(op_dict)

        if doc["last_updated"] is not None:
            doc["last_updated"] = parse(doc["last_updated"]).isoformat()
        to_index.append(doc)

    if to_index:
        result = es.bulk(index=index, body=to_index, refresh=True)
        if result['errors']:
            log.error('Bulk insert failed: %s' % result)

    return indexed_result def send_notification_email(recid, version, user, reviewers_notified, message=None, show_detail=True):
    """
    :param recid:
    :param user: user object
    :param reviewers_notified: whether reviewers have already been notified about this upload
    :param show_detail: whether to show the status and messages for each data table
    :param message:
    :return:
    """

    submission_participants = get_submission_participants_for_record(
        recid, roles=['uploader', 'reviewer'], status='primary'
    )

    if len(submission_participants) == 0:
        raise NoParticipantsException()

    site_url = current_app.config.get('SITE_URL', 'https://www.hepdata.net')

    record = get_record_by_id(recid)

    tables = []
    if show_detail:
        data_submissions = DataSubmission.query.filter_by(
            publication_recid=recid,
            version=version
        )

        for data_submission in data_submissions:
            table_data = {
                'name': data_submission.name,
                'status': 'todo'
            }
            review = DataReview.query.filter_by(
                publication_recid=recid, data_recid=data_submission.id, version=version
            ).first()
            if review:
                table_data['status'] = review.status
                table_data['messages'] = []
                for m in review.messages:
                    table_data['messages'].append({
                        'user': get_user_from_id(m.user).email,
                        'date': m.creation_date.strftime("%Y-%m-%d at %H:%M UTC"),
                        'message': m.message
                    })

            tables.append(table_data)

    for participant in submission_participants:
        invite_token = None
        if not participant.user_account:
            invite_token = participant.invitation_cookie

        message_body = render_template('hepdata_theme/email/submission_status.html',
                                       name=participant.full_name,
                                       actor=user.email,
                                       article=recid,
                                       message=message,
                                       invite_token=invite_token,
                                       role=participant.role,
                                       show_detail=show_detail,
                                       data_tables=tables,
                                       reviewers_notified=reviewers_notified,
                                       title=record['title'],
                                       site_url=site_url,
                                       link=site_url + "/record/{0}"
                                       .format(recid))

        if participant.role == 'reviewer' and not reviewers_notified:
            message_subject = '[HEPData] Submission {0} has a new upload available for you to review'.format(recid)
            hepsubmission = HEPSubmission.query.filter_by(publication_recid=recid, version=version).one()
            coordinator = User.query.get(hepsubmission.coordinator)
        else:
            message_subject = '[HEPData] Notification about submission {0}'.format(recid)

        create_send_email_task(participant.email,
                               message_subject,
                               message_body,
                               reply_to_address=user.email) def send_new_review_message_email(review, message, user):
    """
    Sends a message to all uploaders and reviewers to tell them that a
    comment has been made on a record.

    :param review:
    :param message:
    :param user:
    :return:
    """
    submission_participants = get_primary_submission_participants_for_record(review.publication_recid)

    table_information = DataSubmission.query.filter_by(id=review.data_recid).one()

    site_url = current_app.config.get('SITE_URL', 'https://www.hepdata.net')

    record = get_record_by_id(review.publication_recid)

    destinations = [participant.email for participant in submission_participants]
    full_names = [participant.full_name for participant in submission_participants]

    message_body = render_template(
        'hepdata_theme/email/review-message.html',
        name=', '.join(set(full_names)),
        actor=user.email,
        table_name=table_information.name,
        table_message=message.message,
        article=review.publication_recid,
        title=record['title'],
        site_url=site_url,
        link=site_url + "/record/{0}".format(review.publication_recid),
        table_link=site_url + "/record/{0}?table={1}".format(
            review.publication_recid, table_information.name.replace('+', '%2B'))
    )

    create_send_email_task(
        ','.join(set(destinations)),
        '[HEPData] Submission {0} ({1}) has a new review message'.format(
            review.publication_recid, table_information.name),
        message_body,
        user.email
    ) class NoParticipantsException(Exception):
    pass def send_question_email(question):
    reply_to = current_user.email

    submission = get_latest_hepsubmission(publication_recid=question.publication_recid)
    submission_participants = get_primary_submission_participants_for_record(question.publication_recid)

    if submission:
        destinations = [current_app.config['ADMIN_EMAIL']]
        for submission_participant in submission_participants:
            destinations.append(submission_participant.email)

        coordinator = User.query.get(submission.coordinator)
        if coordinator.id > 1:
            destinations.append(coordinator.email)

        if len(destinations) > 0:
            message_body = render_template(
                'hepdata_theme/email/question.html',
                inspire_id=submission.inspire_id,
                user_email=reply_to,
                site_url=current_app.config.get('SITE_URL', 'https://www.hepdata.net'),
                message=question.question)

            create_send_email_task(destination=','.join(set(destinations)),
                                   subject="[HEPData] Question for record ins{0}".format(submission.inspire_id),
                                   message=message_body, reply_to_address=reply_to) def send_coordinator_notification_email(recid, version, user, message=None):
    """
    :param recid:
    :param user: user object
    :param message: message to send
    :return:
    """

    hepsubmission = get_latest_hepsubmission(publication_recid=recid)
    coordinator = get_user_from_id(hepsubmission.coordinator)

    if not coordinator:
        raise NoParticipantsException()

    site_url = current_app.config.get('SITE_URL', 'https://www.hepdata.net')

    record = get_record_by_id(recid)

    name = coordinator.email
    coordinator_profile = UserProfile.get_by_userid(hepsubmission.coordinator)
    if coordinator_profile:
        name = coordinator_profile.full_name

    collaboration = _get_collaboration(hepsubmission.coordinator)

    message_body = render_template('hepdata_theme/email/passed_review.html',
                                   name=name,
                                   actor=user.email,
                                   collaboration=collaboration,
                                   article=recid,
                                   version=version,
                                   message=message,
                                   title=record['title'],
                                   site_url=site_url,
                                   link=site_url + "/record/{0}".format(recid),
                                   dashboard_link=site_url + "/dashboard"
                                   )

    create_send_email_task(coordinator.email,
                           '[HEPData] Submission {0} is ready to be finalised'.format(recid),
                           message_body,
                           reply_to_address=user.email) def get_inspire_record_information(inspire_rec_id):
    url = 'https://inspirehep.net/api/literature/{}'.format(inspire_rec_id)
    log.debug('Looking up: ' + url)
    req = resilient_requests('get', url)
    status = req.status_code

    if status == 200:
        content = req.json()

        parsed_content = {
            'title': get_title(content['metadata']),
            'doi': get_doi(content['metadata']),
            'authors': get_authors(content['metadata']),
            'type': get_type(content['metadata']),
            'abstract': get_abstract(content['metadata']),
            'creation_date': get_creation_date(content['metadata']),
            'arxiv_id': get_arxiv_id(content['metadata']),
            'collaborations': get_collaborations(content['metadata']),
            'keywords': get_keywords(content['metadata']),
            'journal_info': get_journal_info(content['metadata']),
            'year': get_year(content['metadata']),
            'subject_area': get_subject_area(content['metadata']),
        }

        if 'thesis' in parsed_content['type'] and 'thesis_info' in content['metadata'].keys():
            parsed_content = updated_parsed_content_for_thesis(content, parsed_content)
        elif 'thesis' in parsed_content['type'] and 'thesis_info' not in content['metadata'].keys():
            parsed_content['dissertation'] = {}

        status = 'success'

    else:
        parsed_content = deepcopy(parsed_content_defaults)

    return parsed_content, status RECORD_PLAIN_TEXT = {
    "passed": "passed review",
    "attention": "attention required",
    "todo": "to be reviewed"
}
JSON_LD_MIMETYPES = [
    'application/ld+json',
    'application/vnd.hepdata.ld+json'
]
        UPLOAD_MAX_SIZE = current_app.config.get('UPLOAD_MAX_SIZE', 52000000)
def returns_json(f):
    def decorated_function(*args, **kwargs):
def format_submission(recid, record, version, version_count, hepdata_submission,
                      data_table=None):
def format_tables(ctx, data_record_query, data_table, recid):
def format_resource(resource, contents, content_url):
def get_resource_mimetype(resource, contents):
def should_send_json_ld(request):
def get_commit_message(ctx, recid):
def create_breadcrumb_text(authors, ctx, record):
def submission_has_resources(hepsubmission):
def extract_journal_info(record):
def render_record(recid, record, version, output_format, light_mode=False):
def has_upload_permissions(recid, user, is_sandbox=False):
def has_coordinator_permissions(recid, user, is_sandbox=False):
def create_new_version(recid, user, notify_uploader=True, uploader_message=None):
def process_payload(recid, file, redirect_url, synchronous=False):
def process_saved_file(file_path, recid, userid, redirect_url, previous_status):
def save_zip_file(file, id):
def process_zip_archive(file_path, id, old_schema=False):
def check_and_convert_from_oldhepdata(input_directory, id, timestamp):
def move_files(submission_temp_path, submission_path):
def query_messages_for_data_review(data_review_record, messages):
def assign_or_create_review_status(data_table_metadata, publication_recid,
                                   version):
def determine_user_privileges(recid, ctx):
def process_data_tables(ctx, data_record_query, first_data_id,
                        data_table=None):
def truncate_author_list(record, length=10):
def get_all_ids(index=None, id_field='recid', last_updated=None, latest_first=False): def get_submission_participants_for_record(publication_recid, roles=None, **kwargs):
    """Gets the participants for a given publication record id

    :param int publication_recid: publication_recid of a submission.
    :param ``**kwargs``: Additional filter parameters to pass to `filter_by`.
    :return: List of participants relating to that record
    :rtype: list[SubmissionParticipant]
    """
    query = SubmissionParticipant.query.filter_by(
            publication_recid=publication_recid,
            **kwargs
    )

    if roles:
        query = query.filter(SubmissionParticipant.role.in_(roles))

    return query.all() def get_latest_hepsubmission(*args, **kwargs):
    """
    Gets the latest HEPSubmission record matching the given kwargs

    :return: the HEPSubmission object or None
    """

    hepsubmissions = HEPSubmission.query.filter_by(**kwargs).all()

    last = None
    for hepsubmission in hepsubmissions:
        if last is None:
            last = hepsubmission
        else:
            if hepsubmission.version > last.version:
                last = hepsubmission

    return last def update_action_for_submission_participant(recid, user_id, action=None):
    query = SubmissionParticipant.query.filter_by(
        publication_recid=recid, user_account=user_id)

    if action:
        query = query.filter_by(role=action)

    query.update(dict(action_date=datetime.utcnow()))
    try:
        db.session.commit()
    except Exception:
        db.session.rollback() class SubmissionParticipant(db.Model):

    """
    This table stores information about the reviewers and
    uploaders of a HEPData submission.
    """
    __tablename__ = "submissionparticipant"

    id = db.Column(db.Integer, primary_key=True,
                   nullable=False, autoincrement=True)

    publication_recid = db.Column(db.Integer)

    full_name = db.Column(db.String(128))
    email = db.Column(db.String(128))
    affiliation = db.Column(db.String(128))
    invitation_cookie = db.Column(UUIDType, default=uuid.uuid4)

    # when the user logs in with their cookie,
    # this user_account should be updated.
    user_account = db.Column(db.Integer, db.ForeignKey(User.id))

    # e.g., reviewer or uploader
    role = db.Column(db.String(32), default='')
    # e.g. primary or reserve reviewer/uploader
    status = db.Column(db.String(32), default='reserve')
    action_date = db.Column(db.DateTime, default=datetime.utcnow, nullable=True, index=True)
from datetime import datetime
from dateutil import parser
from invenio_accounts.models import User
from flask_login import login_required, login_user
from flask import Blueprint, send_file, abort, redirect
from flask_security.utils import verify_password
from sqlalchemy import or_, func
from yaml import CBaseLoader as Loader
from hepdata.config import CFG_DATA_TYPE, CFG_PUB_TYPE, SITE_URL
from hepdata.ext.elasticsearch.api import get_records_matching_field, get_count_for_collection, get_n_latest_records, \
    index_record_ids
from hepdata.modules.email.api import send_notification_email, send_new_review_message_email, NoParticipantsException, \
    send_question_email, send_coordinator_notification_email
from hepdata.modules.inspire_api.views import get_inspire_record_information
from hepdata.modules.records.api import request, determine_user_privileges, render_template, format_submission, \
    render_record, current_user, db, jsonify, get_user_from_id, get_record_contents, extract_journal_info, \
    user_allowed_to_perform_action, NoResultFound, OrderedDict, query_messages_for_data_review, returns_json, \
    process_payload, has_upload_permissions, has_coordinator_permissions, create_new_version, format_resource, \
    should_send_json_ld, JSON_LD_MIMETYPES, get_resource_mimetype
from hepdata.modules.submission.api import get_submission_participants_for_record
from hepdata.modules.submission.models import HEPSubmission, DataSubmission, \
    DataResource, DataReview, Message, Question
from hepdata.modules.records.utils.common import get_record_by_id, \
    default_time, IMAGE_TYPES, decode_string
from hepdata.modules.records.utils.data_processing_utils import \
    generate_table_structure, process_ctx
from hepdata.modules.records.utils.submission import create_data_review, \
    get_or_create_hepsubmission
from hepdata.modules.submission.api import get_latest_hepsubmission
from hepdata.modules.records.utils.workflow import \
    update_action_for_submission_participant
from hepdata.modules.stats.views import increment
from hepdata.modules.permissions.models import SubmissionParticipant
import logging
import json
import time
import yaml
                    return jsonify(ctx['json_ld']), status_code
                else:
                    return render_template('hepdata_records/related_record.html', ctx=ctx)

            else:
                return jsonify(
                    {"location": '/record/resource/{0}?view=true'.format(resource_obj.id), 'type': resource_obj.file_type,
                     'description': resource_obj.file_description, 'file_contents': decode_string(contents)})

    else:
        log.error("Unable to find resource %d.", resource_id)
        return abort(404)


@blueprint.route('/cli_upload', methods=['GET', 'POST'])
def cli_upload():
    """
    Used by the hepdata-cli tool to upload a submission.

    :return:
    """

    if request.method == 'GET':
        return redirect('/')

    # email must be provided
    if 'email' not in request.form.keys():
        return jsonify({"message": "User email is required: specify one using -e user-email."}), 400
    user_email = request.form['email']

    # password must be provided
    if 'pswd' not in request.form.keys():
        return jsonify({"message": "User password is required."}), 400
    user_pswd = request.form['pswd']

    # check user associated with this email exists and is active with a confirmed email address
    user = User.query.filter(func.lower(User.email) == user_email.lower(), User.active.is_(True), User.confirmed_at.isnot(None)).first()
    if user is None:
        return jsonify({"message": "Email {} does not match an active confirmed user.".format(user_email)}), 404
    elif user.password is None:
        return jsonify({"message": "Set HEPData password from {} first.".format(SITE_URL + '/lost-password/')}), 403
    elif verify_password(user_pswd, user.password) is False:
        return jsonify({"message": "Wrong password, please try again."}), 403
    else:
        login_user(user)

    # sandbox must be provided
    if 'sandbox' not in request.form.keys():
        return jsonify({"message": "sandbox (True or False) is required."}), 400
    str_sandbox = request.form['sandbox']
    is_sandbox = False if str_sandbox == 'False' else True if str_sandbox == 'True' else None

    # check that recid is an integer if provided
    recid = request.form['recid'] if 'recid' in request.form.keys() else None
    if recid and not recid.isdigit():
        return jsonify({
            "message": "recid {} should be an integer (HEPData ID not INSPIRE ID).".format(str(recid))
        }), 400

    invitation_cookie = request.form['invitation_cookie'] if 'invitation_cookie' in request.form.keys() else None

    # Check the user has upload permissions for this record
    if not has_upload_permissions(recid, user, is_sandbox):
        return jsonify({
            "message": "Email {} does not correspond to a confirmed uploader for this record.".format(str(user_email))
        }), 403

    if is_sandbox is True:
        if recid is None:
            return consume_sandbox_payload()  # '/sandbox/consume'
        else:
            # check that sandbox record exists and belongs to this user
            hepsubmission_record = get_latest_hepsubmission(publication_recid=recid, overall_status='sandbox')
            if hepsubmission_record is None:
                return jsonify({"message": "Sandbox record {} not found.".format(str(recid))}), 404
            else:
                return update_sandbox_payload(recid)  # '/sandbox/<int:recid>/consume'
    elif is_sandbox is False:
        # check that record exists and has 'todo' status
        hepsubmission_record = get_latest_hepsubmission(publication_recid=recid, overall_status='todo')
        if hepsubmission_record is None:
            return jsonify({"message": "Record {} not found.".format(str(recid))}), 404
        # check user is allowed to upload to this record and supplies the correct invitation cookie
        participant = SubmissionParticipant.query.filter_by(user_account=user.id, role='uploader', publication_recid=recid, status='primary').first()
        if participant and str(participant.invitation_cookie) != invitation_cookie:
            return jsonify({"message": "Invitation cookie did not match."}), 403
        return consume_data_payload(recid)  # '/<int:recid>/consume'


@blueprint.route('/<int:recid>/revise-submission', methods=['POST'])
@login_required
def revise_submission(recid):
    """
    This method creates a new version of a submission.

    :param recid: record id to attach the data to
    :return: For POST requests, returns JSONResponse either containing 'url'
             (for success cases) or 'message' (for error cases, which will
             give a 400 error). For GET requests, redirects to the record.
    """
    if not has_coordinator_permissions(recid, current_user):
        return jsonify({"message": "Current user is not a coordinator for this record."}), 403

    notify_uploader = request.values['notify-uploader'] == 'true'
    uploader_message = request.values['notify-uploader-message']
    return create_new_version(recid, current_user, notify_uploader, uploader_message)


@blueprint.route('/<int:recid>/consume', methods=['GET', 'POST'])
@login_required
def consume_data_payload(recid):
    """
    This method persists, then presents the loaded data back to the user.

    :param recid: record id to attach the data to
    :return: For POST requests, returns JSONResponse either containing 'url'
             (for success cases) or 'message' (for error cases, which will
             give a 400 error). For GET requests, redirects to the record.
    """


    if request.method == 'POST':