dbdimp.c
/*
** Prepare to begin tokenizing a particular string.  The input
** string to be tokenized is supposed to be pInput[0..nBytes-1] ..
** except that nBytes passed by fts3 is -1 (don't know why) !
** This is passed to the tokenizer instance, which then returns a
** closure implementing the cursor (so the cursor is again a coderef).
*/
Prepare to begin tokenizing a particular string.  The input
string to be tokenized is supposed to be pInput[0..nBytes-1]
except that nBytes passed by fts3 is -1 (don't know why) .
This is passed to the tokenizer instance, which then returns a
closure implementing the cursor (so the cursor is again a coderef).
Prepare to begin tokenizing a particular string . The input string to be tokenized is supposed to be pInput [ 0 .. nBytes - 1 ] except that nBytes passed by fts3 is - 1 ( don ' t know why ) . This is passed to the tokenizer instance which then returns a closure implementing the cursor ( so the cursor is again a coderef ) .
static int perl_tokenizer_Open(
    sqlite3_tokenizer *pTokenizer,       
    const char *pInput, int nBytes,      
    sqlite3_tokenizer_cursor **ppCursor  
){
    dTHX;
    dSP;
    dMY_CXT;
    U32 flags;
    SV *perl_string;
    int n_retval;
    perl_tokenizer *t = (perl_tokenizer *)pTokenizer;
    perl_tokenizer_cursor *c;
    c = (perl_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));
    memset(c, 0, sizeof(*c));
    *ppCursor = &c->base;
    flags = SVs_TEMP; 
    if (MY_CXT.last_dbh_is_unicode) {
        c->lastByteOffset = c->pInput = pInput;
        c->lastCharOffset = 0;
        flags |= SVf_UTF8;
    }
    ENTER;
    SAVETMPS;
    if (nBytes < 0) { 
        nBytes = strlen(pInput);
    }
    perl_string = newSVpvn_flags(pInput, nBytes, flags);
    PUSHMARK(SP);
    XPUSHs(perl_string);
    PUTBACK;
    n_retval = call_sv(t->coderef, G_SCALAR);
    SPAGAIN;
    if (n_retval != 1) {
        warn("tokenizer returned %d arguments", n_retval);
    }
    c->coderef = newSVsv(POPs);
    PUTBACK;
    FREETMPS;
    LEAVE;
    return SQLITE_OK;
}
static int perl_tokenizer_Open ( sqlite3_tokenizer * pTokenizer , const char * pInput , int nBytes , sqlite3_tokenizer_cursor * * ppCursor ) { dTHX ; dSP ; dMY_CXT ; U32 flags ; SV * perl_string ; int n_retval ; perl_tokenizer * t = ( perl_tokenizer * ) pTokenizer ; perl_tokenizer_cursor * c ; c = ( perl_tokenizer_cursor * ) sqlite3_malloc ( sizeof ( * c ) ) ; memset ( c , 0 , sizeof ( * c ) ) ; * ppCursor = & c -> base ; flags = SVs_TEMP ; if ( MY_CXT . last_dbh_is_unicode ) { c -> lastByteOffset = c -> pInput = pInput ; c -> lastCharOffset = 0 ; flags |= SVf_UTF8 ; } ENTER ; SAVETMPS ; if ( nBytes < 0 ) { nBytes = strlen ( pInput ) ; } perl_string = newSVpvn_flags ( pInput , nBytes , flags ) ; PUSHMARK ( SP ) ; XPUSHs ( perl_string ) ; PUTBACK ; n_retval = call_sv ( t -> coderef , G_SCALAR ) ; SPAGAIN ; if ( n_retval != 1 ) { warn ( " " , n_retval ) ; } c -> coderef = newSVsv ( POPs ) ; PUTBACK ; FREETMPS ; LEAVE ; return SQLITE_OK ; }
Prepare to begin tokenizing a particular string.
Prepare to begin tokenizing a particular string .
/* Tokenizer object */ /* Input buffer */ /* OUT: Created tokenizer cursor */ /* allocate and initialize the cursor struct */ /* flags for creating the Perl SV containing the input string */ /* will call sv_2mortal */ /* special handling if working with utf8 strings */ /* data to keep track of byte offsets */ /* string passed to Perl needs to be flagged as utf8 */ /* build a Perl copy of the input string */ /* we get -1 from fts3. Don't know why ! */ /* call the tokenizer coderef */ /* store the cursor coderef returned by the tokenizer */