def addfile(project, filename, user, postdata, inputsource=None,returntype='xml'): #pylint: disable=too-many-return-statements
    """Add a new input file, this invokes the actual uploader"""


    def errorresponse(msg, code=403):
        if returntype == 'json':
            return withheaders(flask.make_response("{success: false, error: '" + msg + "'}"),'application/json',{'allow_origin': settings.ALLOW_ORIGIN})
        else:
            return withheaders(flask.make_response(msg,403),headers={'allow_origin': settings.ALLOW_ORIGIN}) #(it will have to be explicitly deleted by the client first)

    inputtemplate_id = flask.request.headers.get('Inputtemplate','')
    inputtemplate = None
    metadata = None


    printdebug('Handling addfile, postdata contains fields ' + ",".join(postdata.keys()) )

    if 'inputtemplate' in postdata:
        inputtemplate_id = postdata['inputtemplate']

    if inputtemplate_id:
        #An input template must always be provided
        for profile in settings.PROFILES:
            for t in profile.input:
                if t.id == inputtemplate_id:
                    inputtemplate = t
        if not inputtemplate:
            #Inputtemplate not found, send 404
            printlog("Specified inputtemplate (" + inputtemplate_id + ") not found!")
            return withheaders(flask.make_response("Specified inputtemplate (" + inputtemplate_id + ") not found!",404),headers={'allow_origin': settings.ALLOW_ORIGIN})
        printdebug('Inputtemplate explicitly provided: ' + inputtemplate.id )
    if not inputtemplate:
        #See if an inputtemplate is explicitly specified in the filename
        printdebug('Attempting to determine input template from filename ' + filename )
        if '/' in filename.strip('/'):
            raw = filename.split('/')
            inputtemplate = None
            for profile in settings.PROFILES:
                for it in profile.input:
                    if it.id == raw[0]:
                        inputtemplate = it
                        break
            if inputtemplate:
                filename = raw[1]
    if not inputtemplate:
        #Check if the specified filename can be uniquely associated with an inputtemplate
        for profile in settings.PROFILES:
            for t in profile.input:
                if t.filename == filename:
                    if inputtemplate:
                        #we found another one, not unique!! reset and break
                        inputtemplate = None
                        break
                    else:
                        #good, we found one, don't break cause we want to make sure there is only one
                        inputtemplate = t
        if not inputtemplate:
            printlog("No inputtemplate specified and filename " + filename + " does not uniquely match with any inputtemplate!")
            return errorresponse("No inputtemplate specified nor auto-detected for filename " + filename + "!",404)



    #See if other previously uploaded input files use this inputtemplate
    if inputtemplate.unique:
        nextseq = 0 #unique
    else:
        nextseq = 1 #will hold the next sequence number for this inputtemplate (in multi-mode only)

    for seq, inputfile in Project.inputindexbytemplate(project, user, inputtemplate): #pylint: disable=unused-variable
        if inputtemplate.unique:
            return errorresponse("You have already submitted a file of this type, you can only submit one. Delete it first. (Inputtemplate=" + inputtemplate.id + ", unique=True)")
        else:
            if seq >= nextseq:
                nextseq = seq + 1 #next available sequence number


    if not filename: #Actually, I don't think this can occur at this stage, but we'll leave it in to be sure (yes it can, when the entry shortcut is used!)
        if inputtemplate.filename:
            filename = inputtemplate.filename
        elif inputtemplate.extension:
            filename = str(nextseq) +'-' + str("%034x" % random.getrandbits(128)) + '.' + inputtemplate.extension
        else:
            filename = str(nextseq) +'-' + str("%034x" % random.getrandbits(128))

    #Make sure filename matches (only if not an archive)
    if inputtemplate.acceptarchive and (filename[-7:].lower() == '.tar.gz' or filename[-8:].lower() == '.tar.bz2' or filename[-4:].lower() == '.zip'):
        pass
    else:
        if inputtemplate.filename:
            if filename != inputtemplate.filename:
                filename = inputtemplate.filename
                #return flask.make_response("Specified filename must the filename dictated by the inputtemplate, which is " + inputtemplate.filename)
            #TODO LATER: add support for calling this with an actual number instead of #
        if inputtemplate.extension:
            if filename[-len(inputtemplate.extension) - 1:].lower() == '.' + inputtemplate.extension.lower():
                #good, extension matches (case independent). Let's just make sure the case is as defined exactly by the inputtemplate
                filename = filename[:-len(inputtemplate.extension) - 1] +  '.' + inputtemplate.extension
            else:
                filename = filename +  '.' + inputtemplate.extension
                #return flask.make_response("Specified filename does not have the extension dictated by the inputtemplate ("+inputtemplate.extension+")") #403

    if inputtemplate.onlyinputsource and (not 'inputsource' in postdata or not postdata['inputsource']):
        return errorresponse("Adding files for this inputtemplate must proceed through inputsource")

    if 'converter' in postdata and postdata['converter'] and not postdata['converter'] in [ x.id for x in inputtemplate.converters]:
        return errorresponse("Invalid converter specified: " + postdata['converter'])

    #Make sure the filename is secure
    validfilename = True
    DISALLOWED = ('/','&','|','<','>',';','"',"'","`","{","}","\n","\r","\b","\t")
    for c in filename:
        if c in DISALLOWED:
            validfilename = False
            break

    if not validfilename:
        return errorresponse("Filename contains invalid symbols! Do not use /,&,|,<,>,',`,\",{,} or ;")


    #Create the project (no effect if already exists)
    response = Project.create(project, user)
    if response is not None:
        return response


    printdebug("(Obtaining filename for uploaded file)")
    head = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
    head += "<clamupload>\n"
    if 'file' in flask.request.files:
        printlog("Adding client-side file " + flask.request.files['file'].filename + " to input files")
        sourcefile = flask.request.files['file'].filename
    elif 'url' in postdata and postdata['url']:
        #Download from URL
        printlog("Adding web-based URL " + postdata['url'] + " to input files")
        sourcefile = os.path.basename(postdata['url'])
    elif 'contents' in postdata and postdata['contents']:
        #In message
        printlog("Adding file " + filename + " with explicitly provided contents to input files")
        sourcefile = "editor"
    elif 'inputsource' in postdata and postdata['inputsource']:
        printlog("Adding file " + filename + " from preinstalled data to input files")
        if not inputsource:
            inputsource = None
            for s in inputtemplate.inputsources:
                if s.id.lower() == postdata['inputsource'].lower():
                    inputsource = s
            if not inputsource:
                return errorresponse("Specified inputsource '" + postdata['inputsource'] + "' does not exist for inputtemplate '"+inputtemplate.id+"'")
        sourcefile = os.path.basename(inputsource.path)
    elif 'accesstoken' in postdata and 'filename' in postdata:
        #XHR POST, data in body
        printlog("Adding client-side file " + filename + " to input files. Uploaded using XHR POST")
        sourcefile = postdata['filename']
    else:
        return errorresponse("No file, url or contents specified!")


    #============================ Generate metadata ========================================
    printdebug('(Generating and validating metadata)')
    if 'metafile' in flask.request.files:  #and (not isinstance(postdata['metafile'], dict) or len(postdata['metafile']) > 0)):
        #an explicit metadata file was provided, upload it:
        printlog("Metadata explicitly provided in file, uploading...")
        #Upload file from client to server
        metafile = Project.path(project, user) + 'input/.' + filename + '.METADATA'
        flask.request.files['metafile'].save(metafile)
        try:
            with io.open(metafile,'r',encoding='utf-8') as f:
                metadata = clam.common.data.CLAMMetaData.fromxml(f.read())
            errors, parameters = inputtemplate.validate(metadata, user)
            validmeta = True
        except Exception as e: #pylint: disable=broad-except
            printlog("Uploaded metadata is invalid! " + str(e))
            metadata = None
            errors = True
            parameters = []
            validmeta = False
    elif 'metadata' in postdata and postdata['metadata']:
        printlog("Metadata explicitly provided in message, uploading...")
        try:
            metadata = clam.common.data.CLAMMetaData.fromxml(postdata['metadata'])
            errors, parameters = inputtemplate.validate(metadata, user)
            validmeta = True
        except: #pylint: disable=bare-except
            printlog("Uploaded metadata is invalid!")
            metadata = None
            errors = True
            parameters = []
            validmeta = False
    elif 'inputsource' in postdata and postdata['inputsource']:
        printlog("Getting metadata from inputsource, uploading...")
        if inputsource.metadata:
            printlog("DEBUG: Validating metadata from inputsource")
            metadata = inputsource.metadata
            errors, parameters = inputtemplate.validate(metadata, user)
            validmeta = True
        else:
            printlog("DEBUG: No metadata provided with inputsource, looking for metadata files..")
            metafilename = os.path.dirname(inputsource.path)
            if metafilename: metafilename += '/'
            metafilename += '.' + os.path.basename(inputsource.path) + '.METADATA'
            if os.path.exists(metafilename):
                try:
                    metadata = clam.common.data.CLAMMetaData.fromxml(open(metafilename,'r').readlines())
                    errors, parameters = inputtemplate.validate(metadata, user)
                    validmeta = True
                except: #pylint: disable=bare-except
                    printlog("Uploaded metadata is invalid!")
                    metadata = None
                    errors = True
                    parameters = []
                    validmeta = False
            else:
                return withheaders(flask.make_response("No metadata found nor specified for inputsource " + inputsource.id ,500),headers={'allow_origin': settings.ALLOW_ORIGIN})
    else:
        errors, parameters = inputtemplate.validate(postdata, user)
        validmeta = True #will be checked later


    #  ----------- Check if archive are allowed -------------
    archive = False
    addedfiles = []
    if not errors and inputtemplate.acceptarchive: #pylint: disable=too-many-nested-blocks
        printdebug('(Archive test)')
        # -------- Are we an archive? If so, determine what kind
        archivetype = None
        if 'file' in flask.request.files:
            uploadname = sourcefile.lower()
            archivetype = None
            if uploadname[-4:] == '.zip':
                archivetype = 'zip'
            elif uploadname[-7:] == '.tar.gz':
                archivetype = 'tar.gz'
            elif uploadname[-4:] == '.tar':
                archivetype = 'tar'
            elif uploadname[-8:] == '.tar.bz2':
                archivetype = 'tar.bz2'
            xhrpost = False
        elif 'accesstoken' in postdata and 'filename' in postdata:
            xhrpost = True
            if postdata['filename'][-7:].lower() == '.tar.gz':
                uploadname = sourcefile.lower()
                archivetype = 'tar.gz'
            elif postdata['filename'][-8:].lower() == '.tar.bz2':
                uploadname = sourcefile.lower()
                archivetype = 'tar.bz2'
            elif postdata['filename'][-4:].lower() == '.tar':
                uploadname = sourcefile.lower()
                archivetype = 'tar'
            elif postdata['filename'][-4:].lower() == '.zip':
                uploadname = sourcefile.lower()
                archivetype = 'zip'

        if archivetype:
            # =============== upload archive ======================
            #random name
            archive = "%032x" % random.getrandbits(128) + '.' + archivetype

            #Upload file from client to server
            printdebug('(Archive transfer starting)')
            if not xhrpost:
                flask.request.files['file'].save(Project.path(project,user) + archive)
            elif xhrpost:
                with open(Project.path(project,user) + archive,'wb') as f:
                    while True:
                        chunk = flask.request.stream.read(16384)
                        if chunk:
                            f.write(chunk)
                        else:
                            break
            printdebug('(Archive transfer completed)')
            # =============== Extract archive ======================

            #Determine extraction command
            if archivetype == 'zip':
                cmd = 'unzip -u'
            elif archivetype == 'tar':
                cmd = 'tar -xvf'
            elif archivetype == 'tar.gz':
                cmd = 'tar -xvzf'
            elif archivetype == 'tar.bz2':
                cmd = 'tar -xvjf'
            else:
                raise Exception("Invalid archive format: " + archivetype) #invalid archive, shouldn't happen

            #invoke extractor
            printlog("Extracting '" + archive + "'" )
            try:
                process = subprocess.Popen(cmd + " " + archive, cwd=Project.path(project,user), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
            except: #pylint: disable=bare-except
                return withheaders(flask.make_response("Unable to extract archive",500),headers={'allow_origin': settings.ALLOW_ORIGIN})
            out, _ = process.communicate() #waits for process to end

            if sys.version < '3':
                if isinstance(out, str):
                    out = unicode(out,'utf-8') #pylint: disable=undefined-variable
            else:
                if isinstance(out, bytes):
                    out = str(out,'utf-8')

            #Read filename results

            firstline = True
            for line in out.split("\n"):
                line = line.strip()
                if line:
                    printdebug('(Extraction output: ' + line+')')
                    subfile = None
                    if archivetype[0:3] == 'tar':
                        subfile = line
                    elif archivetype == 'zip' and not firstline: #firstline contains archive name itself, skip it
                        colon = line.find(":")
                        if colon:
                            subfile =  line[colon + 1:].strip()
                    if subfile and os.path.isfile(Project.path(project, user) + subfile):
                        subfile_newname = clam.common.data.resolveinputfilename(os.path.basename(subfile), parameters, inputtemplate, nextseq+len(addedfiles), project)
                        printdebug('(Extracted file ' + subfile + ', moving to input/' + subfile_newname+')')
                        os.rename(Project.path(project, user) + subfile, Project.path(project, user) + 'input/' +  subfile_newname)
                        addedfiles.append(subfile_newname)
                firstline = False

            #all done, remove archive
            os.unlink(Project.path(project, user) + archive)

    if not archive:
        addedfiles = [clam.common.data.resolveinputfilename(filename, parameters, inputtemplate, nextseq, project)]

    fatalerror = None

    jsonoutput = {'success': False if errors else True, 'isarchive': archive}


    output = head
    for filename in addedfiles: #pylint: disable=too-many-nested-blocks
        output += "<upload source=\""+sourcefile +"\" filename=\""+filename+"\" inputtemplate=\"" + inputtemplate.id + "\" templatelabel=\""+inputtemplate.label+"\" format=\""+inputtemplate.formatclass.__name__+"\">\n"
        if not errors:
            output += "<parameters errors=\"no\">"
        else:
            output += "<parameters errors=\"yes\">"
            jsonoutput['error'] = 'There were parameter errors, file not uploaded: '
        for parameter in parameters:
            output += parameter.xml()
            if parameter.error:
                jsonoutput['error'] += parameter.error + ". "
        output += "</parameters>"



        if not errors:
            if not archive:
                #============================ Transfer file ========================================
                printdebug('(Start file transfer: ' +  Project.path(project, user) + 'input/' + filename+' )')
                if 'file' in flask.request.files:
                    printdebug('(Receiving data by uploading file)')
                    #Upload file from client to server
                    flask.request.files['file'].save(Project.path(project, user) + 'input/' + filename)
                elif 'url' in postdata and postdata['url']:
                    printdebug('(Receiving data via url)')
                    #Download file from 3rd party server to CLAM server
                    try:
                        r = requests.get(postdata['url'])
                    except:
                        raise flask.abort(404)
                    if not (r.status_code >= 200 and r.status_code < 300):
                        raise flask.abort(404)

                    CHUNK = 16 * 1024
                    f = open(Project.path(project, user) + 'input/' + filename,'wb')
                    for chunk in r.iter_content(chunk_size=CHUNK):
                        if chunk: # filter out keep-alive new chunks
                            f.write(chunk)
                            f.flush()
                    f.close()
                elif 'inputsource' in postdata and postdata['inputsource']:
                    #Copy (symlink!) from preinstalled data
                    printdebug('(Creating symlink to file ' + inputsource.path + ' <- ' + Project.path(project,user) + '/input/ ' + filename + ')')
                    os.symlink(inputsource.path, Project.path(project, user) + 'input/' + filename)
                elif 'contents' in postdata and postdata['contents']:
                    printdebug('(Receiving data via from contents variable)')
                    #grab encoding
                    encoding = 'utf-8'
                    for p in parameters:
                        if p.id == 'encoding':
                            encoding = p.value
                    #Contents passed in POST message itself
                    try:
                        f = io.open(Project.path(project, user) + 'input/' + filename,'w',encoding=encoding)
                        f.write(postdata['contents'])
                        f.close()
                    except UnicodeError:
                        return errorresponse("Input file " + str(filename) + " is not in the expected encoding!")
                elif 'accesstoken' in postdata and 'filename' in postdata:
                    printdebug('(Receiving data directly from post body)')
                    with open(Project.path(project,user) + 'input/' + filename,'wb') as f:
                        while True:
                            chunk = flask.request.stream.read(16384)
                            if chunk:
                                f.write(chunk)
                            else:
                                break

                printdebug('(File transfer completed)')



            #Create a file object
            file = clam.common.data.CLAMInputFile(Project.path(project, user), filename, False) #get CLAMInputFile without metadata (chicken-egg problem, this does not read the actual file contents!



            #============== Generate metadata ==============

            metadataerror = None
            if not metadata and not errors: #check if it has not already been set in another stage
                printdebug('(Generating metadata)')
                #for newly generated metadata
                try:
                    #Now we generate the actual metadata object (unsaved yet though). We pass our earlier validation results to prevent computing it again
                    validmeta, metadata, parameters = inputtemplate.generate(file, (errors, parameters ))
                    if validmeta:
                        #And we tie it to the CLAMFile object
                        file.metadata = metadata
                        #Add inputtemplate ID to metadata
                        metadata.inputtemplate = inputtemplate.id
                    else:
                        metadataerror = "Undefined error"
                except ValueError as msg:
                    validmeta = False
                    metadataerror = msg
                except KeyError as msg:
                    validmeta = False
                    metadataerror = msg
            elif validmeta:
                #for explicitly uploaded metadata
                metadata.file = file
                file.metadata = metadata
                metadata.inputtemplate = inputtemplate.id

            if metadataerror:
                printdebug('(Metadata could not be generated, ' + str(metadataerror) + ',  this usually indicated an error in service configuration)')
                #output += "<metadataerror />" #This usually indicates an error in service configuration!
                fatalerror = "<error type=\"metadataerror\">Metadata could not be generated for " + filename + ": " + str(metadataerror) + " (this usually indicates an error in service configuration!)</error>"
                jsonoutput['error'] = "Metadata could not be generated! " + str(metadataerror) + "  (this usually indicates an error in service configuration!)"
            elif validmeta:
                #=========== Convert the uploaded file (if requested) ==============

                conversionerror = False
                if 'converter' in postdata and postdata['converter']:
                    for c in inputtemplate.converters:
                        if c.id == postdata['converter']:
                            converter = c
                            break
                    if converter: #(should always be found, error already provided earlier if not)
                        printdebug('(Invoking converter)')
                        try:
                            success = converter.convertforinput(Project.path(project, user) + 'input/' + filename, metadata)
                        except: #pylint: disable=bare-except
                            success = False
                        if not success:
                            conversionerror = True
                            fatalerror = "<error type=\"conversion\">The file " + xmlescape(filename) + " could not be converted</error>"
                            jsonoutput['error'] = "The file could not be converted"
                            jsonoutput['success'] = False

                #====================== Validate the file itself ====================
                if not conversionerror:
                    valid = file.validate()

                    if valid:
                        printdebug('(Validation ok)')
                        output += "<valid>yes</valid>"

                        #Great! Everything ok, save metadata
                        metadata.save(Project.path(project, user) + 'input/' + file.metafilename())

                        #And create symbolic link for inputtemplates
                        linkfilename = os.path.dirname(filename)
                        if linkfilename: linkfilename += '/'
                        linkfilename += '.' + os.path.basename(filename) + '.INPUTTEMPLATE' + '.' + inputtemplate.id + '.' + str(nextseq)
                        os.symlink(Project.path(project, user) + 'input/' + filename, Project.path(project, user) + 'input/' + linkfilename)
                    else:
                        printdebug('(Validation error)')
                        #Too bad, everything worked out but the file itself doesn't validate.
                        #output += "<valid>no</valid>"
                        fatalerror = "<error type=\"validation\">The file " + xmlescape(filename) + " did not validate, it is not in the proper expected format.</error>"
                        jsonoutput['error'] = "The file " + filename.replace("'","") + " did not validate, it is not in the proper expected format."
                        jsonoutput['success'] = False
                        #remove upload
                        os.unlink(Project.path(project, user) + 'input/' + filename)


        output += "</upload>\n"

    output += "</clamupload>"



    if returntype == 'boolean':
        return jsonoutput['success']
    elif fatalerror:
        #fatal error return error message with 403 code
        printlog('Fatal Error during upload: ' + fatalerror)
        return errorresponse(head + fatalerror,403)
    elif errors:
        #parameter errors, return XML output with 403 code
        printdebug('There were parameter errors during upload!')
        if returntype == 'json':
            jsonoutput['xml'] = output #embed XML in JSON for complete client-side processing
            return withheaders(flask.make_response(json.dumps(jsonoutput)), 'application/json', {'allow_origin': settings.ALLOW_ORIGIN})
        else:
            return withheaders(flask.make_response(output,403),headers={'allow_origin': settings.ALLOW_ORIGIN})
    elif returntype == 'xml': #success
        printdebug('Returning xml')
        return withheaders(flask.make_response(output), 'text/xml', {'allow_origin': settings.ALLOW_ORIGIN})
    elif returntype == 'json': #success
        printdebug('Returning json')
        #everything ok, return JSON output (caller decides)
        jsonoutput['xml'] = output #embed XML in JSON for complete client-side processing
        return withheaders(flask.make_response(json.dumps(jsonoutput)), 'application/json', {'allow_origin': settings.ALLOW_ORIGIN})
    elif returntype == 'true_on_success':
        return True
    else:
        printdebug('Invalid return type')
        raise Exception("invalid return type")
def addfile ( project , filename , user , postdata , inputsource = None , returntype = 'xml' ) : def errorresponse ( msg , code = 403 ) : if returntype == 'json' : return withheaders ( flask . make_response ( "{success: false, error: '" + msg + "'}" ) , 'application/json' , { 'allow_origin' : settings . ALLOW_ORIGIN } ) else : return withheaders ( flask . make_response ( msg , 403 ) , headers = { 'allow_origin' : settings . ALLOW_ORIGIN } ) inputtemplate_id = flask . request . headers . get ( 'Inputtemplate' , '' ) inputtemplate = None metadata = None printdebug ( 'Handling addfile, postdata contains fields ' + "," . join ( postdata . keys ( ) ) ) if 'inputtemplate' in postdata : inputtemplate_id = postdata [ 'inputtemplate' ] if inputtemplate_id : for profile in settings . PROFILES : for t in profile . input : if t . id == inputtemplate_id : inputtemplate = t if not inputtemplate : printlog ( "Specified inputtemplate (" + inputtemplate_id + ") not found!" ) return withheaders ( flask . make_response ( "Specified inputtemplate (" + inputtemplate_id + ") not found!" , 404 ) , headers = { 'allow_origin' : settings . ALLOW_ORIGIN } ) printdebug ( 'Inputtemplate explicitly provided: ' + inputtemplate . id ) if not inputtemplate : printdebug ( 'Attempting to determine input template from filename ' + filename ) if '/' in filename . strip ( '/' ) : raw = filename . split ( '/' ) inputtemplate = None for profile in settings . PROFILES : for it in profile . input : if it . id == raw [ 0 ] : inputtemplate = it break if inputtemplate : filename = raw [ 1 ] if not inputtemplate : for profile in settings . PROFILES : for t in profile . input : if t . filename == filename : if inputtemplate : inputtemplate = None break else : inputtemplate = t if not inputtemplate : printlog ( "No inputtemplate specified and filename " + filename + " does not uniquely match with any inputtemplate!" ) return errorresponse ( "No inputtemplate specified nor auto-detected for filename " + filename + "!" , 404 ) if inputtemplate . unique : nextseq = 0 else : nextseq = 1 for seq , inputfile in Project . inputindexbytemplate ( project , user , inputtemplate ) : if inputtemplate . unique : return errorresponse ( "You have already submitted a file of this type, you can only submit one. Delete it first. (Inputtemplate=" + inputtemplate . id + ", unique=True)" ) else : if seq >= nextseq : nextseq = seq + 1 if not filename : if inputtemplate . filename : filename = inputtemplate . filename elif inputtemplate . extension : filename = str ( nextseq ) + '-' + str ( "%034x" % random . getrandbits ( 128 ) ) + '.' + inputtemplate . extension else : filename = str ( nextseq ) + '-' + str ( "%034x" % random . getrandbits ( 128 ) ) if inputtemplate . acceptarchive and ( filename [ - 7 : ] . lower ( ) == '.tar.gz' or filename [ - 8 : ] . lower ( ) == '.tar.bz2' or filename [ - 4 : ] . lower ( ) == '.zip' ) : pass else : if inputtemplate . filename : if filename != inputtemplate . filename : filename = inputtemplate . filename if inputtemplate . extension : if filename [ - len ( inputtemplate . extension ) - 1 : ] . lower ( ) == '.' + inputtemplate . extension . lower ( ) : filename = filename [ : - len ( inputtemplate . extension ) - 1 ] + '.' + inputtemplate . extension else : filename = filename + '.' + inputtemplate . extension if inputtemplate . onlyinputsource and ( not 'inputsource' in postdata or not postdata [ 'inputsource' ] ) : return errorresponse ( "Adding files for this inputtemplate must proceed through inputsource" ) if 'converter' in postdata and postdata [ 'converter' ] and not postdata [ 'converter' ] in [ x . id for x in inputtemplate . converters ] : return errorresponse ( "Invalid converter specified: " + postdata [ 'converter' ] ) validfilename = True DISALLOWED = ( '/' , '&' , '|' , '<' , '>' , ';' , '"' , "'" , "`" , "{" , "}" , "\n" , "\r" , "\b" , "\t" ) for c in filename : if c in DISALLOWED : validfilename = False break if not validfilename : return errorresponse ( "Filename contains invalid symbols! Do not use /,&,|,<,>,',`,\",{,} or ;" ) response = Project . create ( project , user ) if response is not None : return response printdebug ( "(Obtaining filename for uploaded file)" ) head = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" head += "<clamupload>\n" if 'file' in flask . request . files : printlog ( "Adding client-side file " + flask . request . files [ 'file' ] . filename + " to input files" ) sourcefile = flask . request . files [ 'file' ] . filename elif 'url' in postdata and postdata [ 'url' ] : printlog ( "Adding web-based URL " + postdata [ 'url' ] + " to input files" ) sourcefile = os . path . basename ( postdata [ 'url' ] ) elif 'contents' in postdata and postdata [ 'contents' ] : printlog ( "Adding file " + filename + " with explicitly provided contents to input files" ) sourcefile = "editor" elif 'inputsource' in postdata and postdata [ 'inputsource' ] : printlog ( "Adding file " + filename + " from preinstalled data to input files" ) if not inputsource : inputsource = None for s in inputtemplate . inputsources : if s . id . lower ( ) == postdata [ 'inputsource' ] . lower ( ) : inputsource = s if not inputsource : return errorresponse ( "Specified inputsource '" + postdata [ 'inputsource' ] + "' does not exist for inputtemplate '" + inputtemplate . id + "'" ) sourcefile = os . path . basename ( inputsource . path ) elif 'accesstoken' in postdata and 'filename' in postdata : printlog ( "Adding client-side file " + filename + " to input files. Uploaded using XHR POST" ) sourcefile = postdata [ 'filename' ] else : return errorresponse ( "No file, url or contents specified!" ) printdebug ( '(Generating and validating metadata)' ) if 'metafile' in flask . request . files : printlog ( "Metadata explicitly provided in file, uploading..." ) metafile = Project . path ( project , user ) + 'input/.' + filename + '.METADATA' flask . request . files [ 'metafile' ] . save ( metafile ) try : with io . open ( metafile , 'r' , encoding = 'utf-8' ) as f : metadata = clam . common . data . CLAMMetaData . fromxml ( f . read ( ) ) errors , parameters = inputtemplate . validate ( metadata , user ) validmeta = True except Exception as e : printlog ( "Uploaded metadata is invalid! " + str ( e ) ) metadata = None errors = True parameters = [ ] validmeta = False elif 'metadata' in postdata and postdata [ 'metadata' ] : printlog ( "Metadata explicitly provided in message, uploading..." ) try : metadata = clam . common . data . CLAMMetaData . fromxml ( postdata [ 'metadata' ] ) errors , parameters = inputtemplate . validate ( metadata , user ) validmeta = True except : printlog ( "Uploaded metadata is invalid!" ) metadata = None errors = True parameters = [ ] validmeta = False elif 'inputsource' in postdata and postdata [ 'inputsource' ] : printlog ( "Getting metadata from inputsource, uploading..." ) if inputsource . metadata : printlog ( "DEBUG: Validating metadata from inputsource" ) metadata = inputsource . metadata errors , parameters = inputtemplate . validate ( metadata , user ) validmeta = True else : printlog ( "DEBUG: No metadata provided with inputsource, looking for metadata files.." ) metafilename = os . path . dirname ( inputsource . path ) if metafilename : metafilename += '/' metafilename += '.' + os . path . basename ( inputsource . path ) + '.METADATA' if os . path . exists ( metafilename ) : try : metadata = clam . common . data . CLAMMetaData . fromxml ( open ( metafilename , 'r' ) . readlines ( ) ) errors , parameters = inputtemplate . validate ( metadata , user ) validmeta = True except : printlog ( "Uploaded metadata is invalid!" ) metadata = None errors = True parameters = [ ] validmeta = False else : return withheaders ( flask . make_response ( "No metadata found nor specified for inputsource " + inputsource . id , 500 ) , headers = { 'allow_origin' : settings . ALLOW_ORIGIN } ) else : errors , parameters = inputtemplate . validate ( postdata , user ) validmeta = True archive = False addedfiles = [ ] if not errors and inputtemplate . acceptarchive : printdebug ( '(Archive test)' ) archivetype = None if 'file' in flask . request . files : uploadname = sourcefile . lower ( ) archivetype = None if uploadname [ - 4 : ] == '.zip' : archivetype = 'zip' elif uploadname [ - 7 : ] == '.tar.gz' : archivetype = 'tar.gz' elif uploadname [ - 4 : ] == '.tar' : archivetype = 'tar' elif uploadname [ - 8 : ] == '.tar.bz2' : archivetype = 'tar.bz2' xhrpost = False elif 'accesstoken' in postdata and 'filename' in postdata : xhrpost = True if postdata [ 'filename' ] [ - 7 : ] . lower ( ) == '.tar.gz' : uploadname = sourcefile . lower ( ) archivetype = 'tar.gz' elif postdata [ 'filename' ] [ - 8 : ] . lower ( ) == '.tar.bz2' : uploadname = sourcefile . lower ( ) archivetype = 'tar.bz2' elif postdata [ 'filename' ] [ - 4 : ] . lower ( ) == '.tar' : uploadname = sourcefile . lower ( ) archivetype = 'tar' elif postdata [ 'filename' ] [ - 4 : ] . lower ( ) == '.zip' : uploadname = sourcefile . lower ( ) archivetype = 'zip' if archivetype : archive = "%032x" % random . getrandbits ( 128 ) + '.' + archivetype printdebug ( '(Archive transfer starting)' ) if not xhrpost : flask . request . files [ 'file' ] . save ( Project . path ( project , user ) + archive ) elif xhrpost : with open ( Project . path ( project , user ) + archive , 'wb' ) as f : while True : chunk = flask . request . stream . read ( 16384 ) if chunk : f . write ( chunk ) else : break printdebug ( '(Archive transfer completed)' ) if archivetype == 'zip' : cmd = 'unzip -u' elif archivetype == 'tar' : cmd = 'tar -xvf' elif archivetype == 'tar.gz' : cmd = 'tar -xvzf' elif archivetype == 'tar.bz2' : cmd = 'tar -xvjf' else : raise Exception ( "Invalid archive format: " + archivetype ) printlog ( "Extracting '" + archive + "'" ) try : process = subprocess . Popen ( cmd + " " + archive , cwd = Project . path ( project , user ) , stdout = subprocess . PIPE , stderr = subprocess . PIPE , shell = True ) except : return withheaders ( flask . make_response ( "Unable to extract archive" , 500 ) , headers = { 'allow_origin' : settings . ALLOW_ORIGIN } ) out , _ = process . communicate ( ) if sys . version < '3' : if isinstance ( out , str ) : out = unicode ( out , 'utf-8' ) else : if isinstance ( out , bytes ) : out = str ( out , 'utf-8' ) firstline = True for line in out . split ( "\n" ) : line = line . strip ( ) if line : printdebug ( '(Extraction output: ' + line + ')' ) subfile = None if archivetype [ 0 : 3 ] == 'tar' : subfile = line elif archivetype == 'zip' and not firstline : colon = line . find ( ":" ) if colon : subfile = line [ colon + 1 : ] . strip ( ) if subfile and os . path . isfile ( Project . path ( project , user ) + subfile ) : subfile_newname = clam . common . data . resolveinputfilename ( os . path . basename ( subfile ) , parameters , inputtemplate , nextseq + len ( addedfiles ) , project ) printdebug ( '(Extracted file ' + subfile + ', moving to input/' + subfile_newname + ')' ) os . rename ( Project . path ( project , user ) + subfile , Project . path ( project , user ) + 'input/' + subfile_newname ) addedfiles . append ( subfile_newname ) firstline = False os . unlink ( Project . path ( project , user ) + archive ) if not archive : addedfiles = [ clam . common . data . resolveinputfilename ( filename , parameters , inputtemplate , nextseq , project ) ] fatalerror = None jsonoutput = { 'success' : False if errors else True , 'isarchive' : archive } output = head for filename in addedfiles : output += "<upload source=\"" + sourcefile + "\" filename=\"" + filename + "\" inputtemplate=\"" + inputtemplate . id + "\" templatelabel=\"" + inputtemplate . label + "\" format=\"" + inputtemplate . formatclass . __name__ + "\">\n" if not errors : output += "<parameters errors=\"no\">" else : output += "<parameters errors=\"yes\">" jsonoutput [ 'error' ] = 'There were parameter errors, file not uploaded: ' for parameter in parameters : output += parameter . xml ( ) if parameter . error : jsonoutput [ 'error' ] += parameter . error + ". " output += "</parameters>" if not errors : if not archive : printdebug ( '(Start file transfer: ' + Project . path ( project , user ) + 'input/' + filename + ' )' ) if 'file' in flask . request . files : printdebug ( '(Receiving data by uploading file)' ) flask . request . files [ 'file' ] . save ( Project . path ( project , user ) + 'input/' + filename ) elif 'url' in postdata and postdata [ 'url' ] : printdebug ( '(Receiving data via url)' ) try : r = requests . get ( postdata [ 'url' ] ) except : raise flask . abort ( 404 ) if not ( r . status_code >= 200 and r . status_code < 300 ) : raise flask . abort ( 404 ) CHUNK = 16 * 1024 f = open ( Project . path ( project , user ) + 'input/' + filename , 'wb' ) for chunk in r . iter_content ( chunk_size = CHUNK ) : if chunk : f . write ( chunk ) f . flush ( ) f . close ( ) elif 'inputsource' in postdata and postdata [ 'inputsource' ] : printdebug ( '(Creating symlink to file ' + inputsource . path + ' <- ' + Project . path ( project , user ) + '/input/ ' + filename + ')' ) os . symlink ( inputsource . path , Project . path ( project , user ) + 'input/' + filename ) elif 'contents' in postdata and postdata [ 'contents' ] : printdebug ( '(Receiving data via from contents variable)' ) encoding = 'utf-8' for p in parameters : if p . id == 'encoding' : encoding = p . value try : f = io . open ( Project . path ( project , user ) + 'input/' + filename , 'w' , encoding = encoding ) f . write ( postdata [ 'contents' ] ) f . close ( ) except UnicodeError : return errorresponse ( "Input file " + str ( filename ) + " is not in the expected encoding!" ) elif 'accesstoken' in postdata and 'filename' in postdata : printdebug ( '(Receiving data directly from post body)' ) with open ( Project . path ( project , user ) + 'input/' + filename , 'wb' ) as f : while True : chunk = flask . request . stream . read ( 16384 ) if chunk : f . write ( chunk ) else : break printdebug ( '(File transfer completed)' ) file = clam . common . data . CLAMInputFile ( Project . path ( project , user ) , filename , False ) metadataerror = None if not metadata and not errors : printdebug ( '(Generating metadata)' ) try : validmeta , metadata , parameters = inputtemplate . generate ( file , ( errors , parameters ) ) if validmeta : file . metadata = metadata metadata . inputtemplate = inputtemplate . id else : metadataerror = "Undefined error" except ValueError as msg : validmeta = False metadataerror = msg except KeyError as msg : validmeta = False metadataerror = msg elif validmeta : metadata . file = file file . metadata = metadata metadata . inputtemplate = inputtemplate . id if metadataerror : printdebug ( '(Metadata could not be generated, ' + str ( metadataerror ) + ',  this usually indicated an error in service configuration)' ) fatalerror = "<error type=\"metadataerror\">Metadata could not be generated for " + filename + ": " + str ( metadataerror ) + " (this usually indicates an error in service configuration!)</error>" jsonoutput [ 'error' ] = "Metadata could not be generated! " + str ( metadataerror ) + "  (this usually indicates an error in service configuration!)" elif validmeta : conversionerror = False if 'converter' in postdata and postdata [ 'converter' ] : for c in inputtemplate . converters : if c . id == postdata [ 'converter' ] : converter = c break if converter : printdebug ( '(Invoking converter)' ) try : success = converter . convertforinput ( Project . path ( project , user ) + 'input/' + filename , metadata ) except : success = False if not success : conversionerror = True fatalerror = "<error type=\"conversion\">The file " + xmlescape ( filename ) + " could not be converted</error>" jsonoutput [ 'error' ] = "The file could not be converted" jsonoutput [ 'success' ] = False if not conversionerror : valid = file . validate ( ) if valid : printdebug ( '(Validation ok)' ) output += "<valid>yes</valid>" metadata . save ( Project . path ( project , user ) + 'input/' + file . metafilename ( ) ) linkfilename = os . path . dirname ( filename ) if linkfilename : linkfilename += '/' linkfilename += '.' + os . path . basename ( filename ) + '.INPUTTEMPLATE' + '.' + inputtemplate . id + '.' + str ( nextseq ) os . symlink ( Project . path ( project , user ) + 'input/' + filename , Project . path ( project , user ) + 'input/' + linkfilename ) else : printdebug ( '(Validation error)' ) fatalerror = "<error type=\"validation\">The file " + xmlescape ( filename ) + " did not validate, it is not in the proper expected format.</error>" jsonoutput [ 'error' ] = "The file " + filename . replace ( "'" , "" ) + " did not validate, it is not in the proper expected format." jsonoutput [ 'success' ] = False os . unlink ( Project . path ( project , user ) + 'input/' + filename ) output += "</upload>\n" output += "</clamupload>" if returntype == 'boolean' : return jsonoutput [ 'success' ] elif fatalerror : printlog ( 'Fatal Error during upload: ' + fatalerror ) return errorresponse ( head + fatalerror , 403 ) elif errors : printdebug ( 'There were parameter errors during upload!' ) if returntype == 'json' : jsonoutput [ 'xml' ] = output return withheaders ( flask . make_response ( json . dumps ( jsonoutput ) ) , 'application/json' , { 'allow_origin' : settings . ALLOW_ORIGIN } ) else : return withheaders ( flask . make_response ( output , 403 ) , headers = { 'allow_origin' : settings . ALLOW_ORIGIN } ) elif returntype == 'xml' : printdebug ( 'Returning xml' ) return withheaders ( flask . make_response ( output ) , 'text/xml' , { 'allow_origin' : settings . ALLOW_ORIGIN } ) elif returntype == 'json' : printdebug ( 'Returning json' ) jsonoutput [ 'xml' ] = output return withheaders ( flask . make_response ( json . dumps ( jsonoutput ) ) , 'application/json' , { 'allow_origin' : settings . ALLOW_ORIGIN } ) elif returntype == 'true_on_success' : return True else : printdebug ( 'Invalid return type' ) raise Exception ( "invalid return type" )
Add a new input file, this invokes the actual uploader