src/storage/s3/storage.c
/***********************************************************************************************************************************
General function for listing files to be used by other list routines
***********************************************************************************************************************************/
General function for listing files to be used by other list routines
General function for listing files to be used by other list routines
static void
storageS3ListInternal(
    StorageS3 *this, const String *path, const String *expression, bool recurse,
    void (*callback)(StorageS3 *this, void *callbackData, const String *name, StorageType type, const XmlNode *xml),
    void *callbackData)
{
    FUNCTION_LOG_BEGIN(logLevelDebug);
        FUNCTION_LOG_PARAM(STORAGE_S3, this);
        FUNCTION_LOG_PARAM(STRING, path);
        FUNCTION_LOG_PARAM(STRING, expression);
        FUNCTION_LOG_PARAM(BOOL, recurse);
        FUNCTION_LOG_PARAM(FUNCTIONP, callback);
        FUNCTION_LOG_PARAM_P(VOID, callbackData);
    FUNCTION_LOG_END();
    ASSERT(this != NULL);
    ASSERT(path != NULL);
    MEM_CONTEXT_TEMP_BEGIN()
    {
        const String *basePrefix;
        if (strSize(path) == 1)
            basePrefix = EMPTY_STR;
        else
            basePrefix = strNewFmt("%s/", strZ(strSub(path, 1)));
        const String *expressionPrefix = regExpPrefix(expression);
        const String *queryPrefix;
        if (expressionPrefix == NULL)
            queryPrefix = basePrefix;
        else
        {
            if (strEmpty(basePrefix))
                queryPrefix = expressionPrefix;
            else
                queryPrefix = strNewFmt("%s%s", strZ(basePrefix), strZ(expressionPrefix));
        }
        HttpQuery *query = httpQueryNewP();
        if (!recurse)
            httpQueryAdd(query, S3_QUERY_DELIMITER_STR, FSLASH_STR);
        httpQueryAdd(query, S3_QUERY_LIST_TYPE_STR, S3_QUERY_VALUE_LIST_TYPE_2_STR);
        if (!strEmpty(queryPrefix))
            httpQueryAdd(query, S3_QUERY_PREFIX_STR, queryPrefix);
        HttpRequest *request = NULL;
        do
        {
            MEM_CONTEXT_TEMP_BEGIN()
            {
                HttpResponse *response = NULL;
                if (request != NULL)
                {
                    response = storageS3ResponseP(request);
                    httpRequestFree(request);
                    request = NULL;
                }
                else
                    response = storageS3RequestP(this, HTTP_VERB_GET_STR, FSLASH_STR, query);
                XmlNode *xmlRoot = xmlDocumentRoot(xmlDocumentNewBuf(httpResponseContent(response)));
                const String *continuationToken = xmlNodeContent(
                    xmlNodeChild(xmlRoot, S3_XML_TAG_NEXT_CONTINUATION_TOKEN_STR, false));
                if (continuationToken != NULL)
                {
                    httpQueryPut(query, S3_QUERY_CONTINUATION_TOKEN_STR, continuationToken);
                    MEM_CONTEXT_PRIOR_BEGIN()
                    {
                        request = storageS3RequestAsyncP(this, HTTP_VERB_GET_STR, FSLASH_STR, query);
                    }
                    MEM_CONTEXT_PRIOR_END();
                }
                XmlNodeList *subPathList = xmlNodeChildList(xmlRoot, S3_XML_TAG_COMMON_PREFIXES_STR);
                for (unsigned int subPathIdx = 0; subPathIdx < xmlNodeLstSize(subPathList); subPathIdx++)
                {
                    const XmlNode *subPathNode = xmlNodeLstGet(subPathList, subPathIdx);
                    const String *subPath = xmlNodeContent(xmlNodeChild(subPathNode, S3_XML_TAG_PREFIX_STR, true));
                    subPath = strSubN(subPath, strSize(basePrefix), strSize(subPath) - strSize(basePrefix) - 1);
                    callback(this, callbackData, subPath, storageTypePath, NULL);
                }
                XmlNodeList *fileList = xmlNodeChildList(xmlRoot, S3_XML_TAG_CONTENTS_STR);
                for (unsigned int fileIdx = 0; fileIdx < xmlNodeLstSize(fileList); fileIdx++)
                {
                    const XmlNode *fileNode = xmlNodeLstGet(fileList, fileIdx);
                    const String *file = xmlNodeContent(xmlNodeChild(fileNode, S3_XML_TAG_KEY_STR, true));
                    file = strEmpty(basePrefix) ? file : strSub(file, strSize(basePrefix));
                    callback(this, callbackData, file, storageTypeFile, fileNode);
                }
            }
            MEM_CONTEXT_TEMP_END();
        }
        while (request != NULL);
    }
    MEM_CONTEXT_TEMP_END();
    FUNCTION_LOG_RETURN_VOID();
}
static void storageS3ListInternal ( StorageS3 * this , const String * path , const String * expression , bool recurse , void ( * callback ) ( StorageS3 * this , void * callbackData , const String * name , StorageType type , const XmlNode * xml ) , void * callbackData ) { FUNCTION_LOG_BEGIN ( logLevelDebug ) ; FUNCTION_LOG_PARAM ( STORAGE_S3 , this ) ; FUNCTION_LOG_PARAM ( STRING , path ) ; FUNCTION_LOG_PARAM ( STRING , expression ) ; FUNCTION_LOG_PARAM ( BOOL , recurse ) ; FUNCTION_LOG_PARAM ( FUNCTIONP , callback ) ; FUNCTION_LOG_PARAM_P ( VOID , callbackData ) ; FUNCTION_LOG_END ( ) ; ASSERT ( this != NULL ) ; ASSERT ( path != NULL ) ; MEM_CONTEXT_TEMP_BEGIN ( ) { const String * basePrefix ; if ( strSize ( path ) == 1 ) basePrefix = EMPTY_STR ; else basePrefix = strNewFmt ( " " , strZ ( strSub ( path , 1 ) ) ) ; const String * expressionPrefix = regExpPrefix ( expression ) ; const String * queryPrefix ; if ( expressionPrefix == NULL ) queryPrefix = basePrefix ; else { if ( strEmpty ( basePrefix ) ) queryPrefix = expressionPrefix ; else queryPrefix = strNewFmt ( " " , strZ ( basePrefix ) , strZ ( expressionPrefix ) ) ; } HttpQuery * query = httpQueryNewP ( ) ; if ( ! recurse ) httpQueryAdd ( query , S3_QUERY_DELIMITER_STR , FSLASH_STR ) ; httpQueryAdd ( query , S3_QUERY_LIST_TYPE_STR , S3_QUERY_VALUE_LIST_TYPE_2_STR ) ; if ( ! strEmpty ( queryPrefix ) ) httpQueryAdd ( query , S3_QUERY_PREFIX_STR , queryPrefix ) ; HttpRequest * request = NULL ; do { MEM_CONTEXT_TEMP_BEGIN ( ) { HttpResponse * response = NULL ; if ( request != NULL ) { response = storageS3ResponseP ( request ) ; httpRequestFree ( request ) ; request = NULL ; } else response = storageS3RequestP ( this , HTTP_VERB_GET_STR , FSLASH_STR , query ) ; XmlNode * xmlRoot = xmlDocumentRoot ( xmlDocumentNewBuf ( httpResponseContent ( response ) ) ) ; const String * continuationToken = xmlNodeContent ( xmlNodeChild ( xmlRoot , S3_XML_TAG_NEXT_CONTINUATION_TOKEN_STR , false ) ) ; if ( continuationToken != NULL ) { httpQueryPut ( query , S3_QUERY_CONTINUATION_TOKEN_STR , continuationToken ) ; MEM_CONTEXT_PRIOR_BEGIN ( ) { request = storageS3RequestAsyncP ( this , HTTP_VERB_GET_STR , FSLASH_STR , query ) ; } MEM_CONTEXT_PRIOR_END ( ) ; } XmlNodeList * subPathList = xmlNodeChildList ( xmlRoot , S3_XML_TAG_COMMON_PREFIXES_STR ) ; for ( unsigned int subPathIdx = 0 ; subPathIdx < xmlNodeLstSize ( subPathList ) ; subPathIdx ++ ) { const XmlNode * subPathNode = xmlNodeLstGet ( subPathList , subPathIdx ) ; const String * subPath = xmlNodeContent ( xmlNodeChild ( subPathNode , S3_XML_TAG_PREFIX_STR , true ) ) ; subPath = strSubN ( subPath , strSize ( basePrefix ) , strSize ( subPath ) - strSize ( basePrefix ) - 1 ) ; callback ( this , callbackData , subPath , storageTypePath , NULL ) ; } XmlNodeList * fileList = xmlNodeChildList ( xmlRoot , S3_XML_TAG_CONTENTS_STR ) ; for ( unsigned int fileIdx = 0 ; fileIdx < xmlNodeLstSize ( fileList ) ; fileIdx ++ ) { const XmlNode * fileNode = xmlNodeLstGet ( fileList , fileIdx ) ; const String * file = xmlNodeContent ( xmlNodeChild ( fileNode , S3_XML_TAG_KEY_STR , true ) ) ; file = strEmpty ( basePrefix ) ? file : strSub ( file , strSize ( basePrefix ) ) ; callback ( this , callbackData , file , storageTypeFile , fileNode ) ; } } MEM_CONTEXT_TEMP_END ( ) ; } while ( request != NULL ) ; } MEM_CONTEXT_TEMP_END ( ) ; FUNCTION_LOG_RETURN_VOID ( ) ; }
General function for listing files to be used by other list routines
General function for listing files to be used by other list routines
// Build the base prefix by stripping off the initial / // Get the expression prefix when possible to limit initial results // If there is an expression prefix then use it to build the query prefix, otherwise query prefix is base prefix // Create query // Add the delimiter to not recurse // Use list type 2 // Don't specify empty prefix because it is the default // Loop as long as a continuation token returned // Use an inner mem context here because we could potentially be retrieving millions of files so it is a good idea to // free memory at regular intervals // If there is an outstanding async request then wait for the response // Else get the response immediately from a sync request // If a continuation token exists then send an async request to get more data // Store request in the outer temp context // Get subpath list // Get subpath name // Strip off base prefix and final / // Add to list // Get file list // Get file name // Strip off the base prefix when present // Add to list