dbdimp.c
/*
** Extract the next token from a tokenization cursor.  The cursor must
** have been opened by a prior call to perl_tokenizer_Open().
*/
Extract the next token from a tokenization cursor.  The cursor must
have been opened by a prior call to perl_tokenizer_Open().
Extract the next token from a tokenization cursor . The cursor must have been opened by a prior call to perl_tokenizer_Open () .
static int perl_tokenizer_Next(
    sqlite3_tokenizer_cursor *pCursor,  
    const char **ppToken,               
    int *pnBytes,                       
    int *piStartOffset,                 
    int *piEndOffset,                   
    int *piPosition                     
){
    perl_tokenizer_cursor *c = (perl_tokenizer_cursor *) pCursor;
    int result;
    int n_retval;
    char *token;
    char *byteOffset;
    STRLEN n_a; 
    I32 hop;
    dTHX;
    dSP;
    ENTER;
    SAVETMPS;
    PUSHMARK(SP);
    PUTBACK;
    n_retval = call_sv(c->coderef, G_ARRAY);
    SPAGAIN;
    if (n_retval == 0) {
        result = SQLITE_DONE;
    }
    else {
        if (n_retval != 5) {
            warn("tokenizer cursor returned %d arguments", n_retval);
        }
        *piPosition    = POPi;
        *piEndOffset   = POPi;
        *piStartOffset = POPi;
        *pnBytes       = POPi;
        token          = POPpx;
        if (c->pInput) { 
            *pnBytes = strlen(token);
            hop            = *piStartOffset - c->lastCharOffset;
            byteOffset     = (char*)utf8_hop((U8*)c->lastByteOffset, hop);
            hop            = *piEndOffset - *piStartOffset;
            *piStartOffset = byteOffset - c->pInput;
            byteOffset     = (char*)utf8_hop((U8*)byteOffset, hop);
            *piEndOffset   = byteOffset - c->pInput;
            c->lastCharOffset = *piEndOffset,
            c->lastByteOffset = byteOffset;
        }
        if (*pnBytes > c->nTokenAllocated ){
            char *pNew;
            c->nTokenAllocated = *pnBytes + 20;
            pNew = sqlite3_realloc(c->pToken, c->nTokenAllocated);
            if( !pNew ) return SQLITE_NOMEM;
            c->pToken = pNew;
        }
        memcpy(c->pToken, token, *pnBytes);
        *ppToken  = c->pToken;
        result = SQLITE_OK;
    }
    PUTBACK;
    FREETMPS;
    LEAVE;
    return result;
}
static int perl_tokenizer_Next ( sqlite3_tokenizer_cursor * pCursor , const char * * ppToken , int * pnBytes , int * piStartOffset , int * piEndOffset , int * piPosition ) { perl_tokenizer_cursor * c = ( perl_tokenizer_cursor * ) pCursor ; int result ; int n_retval ; char * token ; char * byteOffset ; STRLEN n_a ; I32 hop ; dTHX ; dSP ; ENTER ; SAVETMPS ; PUSHMARK ( SP ) ; PUTBACK ; n_retval = call_sv ( c -> coderef , G_ARRAY ) ; SPAGAIN ; if ( n_retval == 0 ) { result = SQLITE_DONE ; } else { if ( n_retval != 5 ) { warn ( " " , n_retval ) ; } * piPosition = POPi ; * piEndOffset = POPi ; * piStartOffset = POPi ; * pnBytes = POPi ; token = POPpx ; if ( c -> pInput ) { * pnBytes = strlen ( token ) ; hop = * piStartOffset - c -> lastCharOffset ; byteOffset = ( char * ) utf8_hop ( ( U8 * ) c -> lastByteOffset , hop ) ; hop = * piEndOffset - * piStartOffset ; * piStartOffset = byteOffset - c -> pInput ; byteOffset = ( char * ) utf8_hop ( ( U8 * ) byteOffset , hop ) ; * piEndOffset = byteOffset - c -> pInput ; c -> lastCharOffset = * piEndOffset , c -> lastByteOffset = byteOffset ; } if ( * pnBytes > c -> nTokenAllocated ) { char * pNew ; c -> nTokenAllocated = * pnBytes + 20 ; pNew = sqlite3_realloc ( c -> pToken , c -> nTokenAllocated ) ; if ( ! pNew ) return SQLITE_NOMEM ; c -> pToken = pNew ; } memcpy ( c -> pToken , token , * pnBytes ) ; * ppToken = c -> pToken ; result = SQLITE_OK ; } PUTBACK ; FREETMPS ; LEAVE ; return result ; }
Extract the next token from a tokenization cursor.
Extract the next token from a tokenization cursor .
/* Cursor returned by perl_tokenizer_Open */ /* OUT: *ppToken is the token text */ /* OUT: Number of bytes in token */ /* OUT: Starting offset of token */ /* OUT: Ending offset of token */ /* OUT: Position integer of token */ /* this is required for older perls < 5.8.8 */ /* call the cursor */ /* if we get back an empty list, there is no more token */ /* otherwise, get token details from the return list */ /* if working with utf8 data */ /* recompute *pnBytes in bytes, not in chars */ /* recompute start/end offsets in bytes, not in chars */ /* remember where we are for next round */ /* make sure we have enough storage for copying the token */ /* need to copy the token into the C cursor before perl frees that
           memory */