/*
Large portions of this file are copyright by the authors of Jackson under the Apache 2.0 or LGPL license.

The implementation was derived from the Jackson class 'org.codehaus.jackson.impl.ReaderBasedParser' and
modified for Rison.
*/

package com.bazaarvoice.jackson.rison;

import com.fasterxml.jackson.core.Base64Variant;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
import com.fasterxml.jackson.core.ObjectCodec;
import com.fasterxml.jackson.core.Version;
import com.fasterxml.jackson.core.base.ParserBase;
import com.fasterxml.jackson.core.io.IOContext;
import com.fasterxml.jackson.core.sym.CharsToNameCanonicalizer;
import com.fasterxml.jackson.core.util.ByteArrayBuilder;
import com.fasterxml.jackson.core.util.TextBuffer;

import java.io.IOException;
import java.io.Reader;
import java.io.Writer;

/**
 * This is a concrete implementation of {@link JsonParser}, which is
 * based on a {@link java.io.Reader} to handle low-level character
 * conversion tasks.
 */
public class RisonParser
        extends ParserBase
{

    public static final char INT_APOSTROPHE = '\'';

    /**
     * Enumeration that defines all configurable features for Rison parsers.
     */
    public enum Feature {
        /**
         * Whether to assume that the top-level value being encoded is an object,
         * and therefore to omit the containing '(' and ')'.
         *

         * Default setting is false.
         */
        O_RISON(false),

        /**
         * Whether to assume that the top-level value being encoded is an array,
         * and therefore to omit the containing '!(' and ')'.
         *

         * Default setting is false.
         */
        A_RISON(false)
        ;

        final boolean _defaultState;

        final int _mask;

        /**
         * Method that calculates bit set (flags) of all features that
         * are enabled by default.
         */
        public static int collectDefaults()
        {
            int flags = 0;
            for (Feature f : values()) {
                if (f.enabledByDefault()) {
                    flags |= f.getMask();
                }
            }
            return flags;
        }

        private Feature(boolean defaultState) {
            _defaultState = defaultState;
            _mask = (1 << ordinal());
        }

        public boolean enabledByDefault() { return _defaultState; }

        public int getMask() { return _mask; }
    }

    /*
    /**********************************************************
    /* Input configuration
    /**********************************************************
     */

    /**
     * Reader that can be used for reading more content, if one
     * buffer from input source, but in some cases pre-loaded buffer
     * is handed to the parser.
     */
    protected Reader _reader;

    /**
     * Current buffer from which data is read; generally data is read into
     * buffer from input source.
     */
    protected char[] _inputBuffer;

    /*
    /**********************************************************
    /* Configuration
    /**********************************************************
     */

    protected ObjectCodec _objectCodec;

    final protected CharsToNameCanonicalizer _symbols;

    final protected int _risonFeatures;

    /*
    /**********************************************************
    /* Parsing state
    /**********************************************************
     */

    /**
     * Flag that indicates that the current token has not yet
     * been fully processed, and needs to be finished for
     * some access (or skipped to obtain the next token)
     */
    protected boolean _tokenIncomplete = false;

    /*
   /**********************************************************
   /* Life-cycle
   /**********************************************************
    */

    public RisonParser(IOContext ctxt, int jsonFeatures, int risonFeatures, Reader r,
                       ObjectCodec codec, CharsToNameCanonicalizer st)
    {
        super(ctxt, jsonFeatures);
        _reader = r;
        _risonFeatures = risonFeatures;
        _inputBuffer = ctxt.allocTokenBuffer();
        _objectCodec = codec;
        _symbols = st;
    }

    @Override
    public Version version() {
        return PackageVersion.version();
    }

    /*
    /**********************************************************
    /* Configuration
    /**********************************************************
     */

    private boolean isRisonEnabled(Feature feature) {
        return (_risonFeatures & feature.getMask()) != 0;
    }

    /*
    /**********************************************************
    /* Base method defs, overrides
    /**********************************************************
     */

    @Override
    public ObjectCodec getCodec() {
        return _objectCodec;
    }

    @Override
    public void setCodec(ObjectCodec c) {
        _objectCodec = c;
    }

    @Override
    public int releaseBuffered(Writer w) throws IOException
    {
        int count = _inputEnd - _inputPtr;
        if (count < 1) {
            return 0;
        }
        // let's just advance ptr to end
        int origPtr = _inputPtr;
        w.write(_inputBuffer, origPtr, count);
        return count;
    }

    @Override
    public Object getInputSource() {
        return _reader;
    }

    @Override
    protected boolean loadMore() throws IOException
    {
        _currInputProcessed += _inputEnd;
        _currInputRowStart -= _inputEnd;

        if (_reader != null) {
            int count = _reader.read(_inputBuffer, 0, _inputBuffer.length);
            if (count > 0) {
                _inputPtr = 0;
                _inputEnd = count;
                return true;
            }
            // End of input
            _closeInput();
            // Should never return 0, so let's fail
            if (count == 0) {
                throw new IOException("Reader returned 0 characters when trying to read "+_inputEnd);
            }
        }
        return false;
    }

    protected char getNextChar(String eofMsg)
            throws IOException, JsonParseException
    {
        if (_inputPtr >= _inputEnd) {
            if (!loadMore()) {
                _reportInvalidEOF(eofMsg);
            }
        }
        return _inputBuffer[_inputPtr++];
    }

    @Override
    protected void _closeInput() throws IOException
    {
        /* 25-Nov-2008, tatus: As per [JACKSON-16] we are not to call close()
         *   on the underlying Reader, unless we "own" it, or auto-closing
         *   feature is enabled.
         *   One downside is that when using our optimized
         *   Reader (granted, we only do that for UTF-32...) this
         *   means that buffer recycling won't work correctly.
         */
        if (_reader != null) {
            if (_ioContext.isResourceManaged() || isEnabled(JsonParser.Feature.AUTO_CLOSE_SOURCE)) {
                _reader.close();
            }
            _reader = null;
        }
    }

    /**
     * Method called to release internal buffers owned by the base
     * reader. This may be called along with {@link #_closeInput} (for
     * example, when explicitly closing this reader instance), or
     * separately (if need be).
     */
    @Override
    protected void _releaseBuffers()
            throws IOException
    {
        super._releaseBuffers();
        char[] buf = _inputBuffer;
        if (buf != null) {
            _inputBuffer = null;
            _ioContext.releaseTokenBuffer(buf);
        }
    }

    /*
   /**********************************************************
   /* Public API, data access
   /**********************************************************
    */

    /**
     * Method for accessing textual representation of the current event;
     * if no current event (before first call to {@link #nextToken}, or
     * after encountering end-of-input), returns null.
     * Method can be called for any event.
     */
    @Override
    public String getText()
            throws IOException, JsonParseException
    {
        JsonToken t = _currToken;
        if (t == JsonToken.VALUE_STRING) {
            if (_tokenIncomplete) {
                _tokenIncomplete = false;
                _finishString(); // only strings can be incomplete
            }
            return _textBuffer.contentsAsString();
        }
        return _getText2(t);
    }

    protected String _getText2(JsonToken t)
    {
        if (t == null) {
            return null;
        }
        switch (t) {
            case FIELD_NAME:
                return _parsingContext.getCurrentName();

            case VALUE_STRING:
                // fall through
            case VALUE_NUMBER_INT:
            case VALUE_NUMBER_FLOAT:
                return _textBuffer.contentsAsString();
        }
        return t.asString();
    }

    @Override
    public char[] getTextCharacters()
            throws IOException, JsonParseException
    {
        if (_currToken != null) { // null only before/after document
            switch (_currToken) {

                case FIELD_NAME:
                    if (!_nameCopied) {
                        String name = _parsingContext.getCurrentName();
                        int nameLen = name.length();
                        if (_nameCopyBuffer == null) {
                            _nameCopyBuffer = _ioContext.allocNameCopyBuffer(nameLen);
                        } else if (_nameCopyBuffer.length < nameLen) {
                            _nameCopyBuffer = new char[nameLen];
                        }
                        name.getChars(0, nameLen, _nameCopyBuffer, 0);
                        _nameCopied = true;
                    }
                    return _nameCopyBuffer;

                case VALUE_STRING:
                    if (_tokenIncomplete) {
                        _tokenIncomplete = false;
                        _finishString(); // only strings can be incomplete
                    }
                    // fall through
                case VALUE_NUMBER_INT:
                case VALUE_NUMBER_FLOAT:
                    return _textBuffer.getTextBuffer();

                default:
                    return _currToken.asCharArray();
            }
        }
        return null;
    }

    @Override
    public int getTextLength()
            throws IOException, JsonParseException
    {
        if (_currToken != null) { // null only before/after document
            switch (_currToken) {

                case FIELD_NAME:
                    return _parsingContext.getCurrentName().length();
                case VALUE_STRING:
                    if (_tokenIncomplete) {
                        _tokenIncomplete = false;
                        _finishString(); // only strings can be incomplete
                    }
                    // fall through
                case VALUE_NUMBER_INT:
                case VALUE_NUMBER_FLOAT:
                    return _textBuffer.size();

                default:
                    return _currToken.asCharArray().length;
            }
        }
        return 0;
    }

    @Override
    public int getTextOffset() throws IOException, JsonParseException
    {
        // Most have offset of 0, only some may have other values:
        if (_currToken != null) {
            switch (_currToken) {
                case FIELD_NAME:
                    return 0;
                case VALUE_STRING:
                    if (_tokenIncomplete) {
                        _tokenIncomplete = false;
                        _finishString(); // only strings can be incomplete
                    }
                    // fall through
                case VALUE_NUMBER_INT:
                case VALUE_NUMBER_FLOAT:
                    return _textBuffer.getTextOffset();
            }
        }
        return 0;
    }

    @Override
    public Object getEmbeddedObject() throws IOException, JsonParseException {
        return null;
    }

    @Override
    public byte[] getBinaryValue(Base64Variant b64variant)
            throws IOException, JsonParseException
    {
        if (_currToken != JsonToken.VALUE_STRING &&
                (_currToken != JsonToken.VALUE_EMBEDDED_OBJECT || _binaryValue == null)) {
            _reportError("Current token ("+_currToken+") not VALUE_STRING or VALUE_EMBEDDED_OBJECT, can not access as binary");
        }
        /* To ensure that we won't see inconsistent data, better clear up
         * state...
         */
        if (_tokenIncomplete) {
            try {
                _binaryValue = _decodeBase64(b64variant);
            } catch (IllegalArgumentException iae) {
                throw _constructError("Failed to decode VALUE_STRING as base64 ("+b64variant+"): "+iae.getMessage());
            }
            /* let's clear incomplete only now; allows for accessing other
             * textual content in error cases
             */
            _tokenIncomplete = false;
        } else { // may actually require conversion...
            if (_binaryValue == null) {
                ByteArrayBuilder builder = _getByteArrayBuilder();
                _decodeBase64(getText(), builder, b64variant);
                _binaryValue = builder.toByteArray();
            }
        }
        return _binaryValue;
    }

    /*
   /**********************************************************
   /* Public API, traversal
   /**********************************************************
    */

    /**
     * @return Next token from the stream, if any found, or null
     *   to indicate end-of-input
     */
    @Override
    public JsonToken nextToken()
            throws IOException, JsonParseException
    {
        _numTypesValid = NR_UNKNOWN;

        if (_currToken == null && _parsingContext.inRoot()) {
            if (isRisonEnabled(Feature.A_RISON)) {
                _parsingContext = _parsingContext.createChildArrayContext(_tokenInputRow, _tokenInputCol);
                return (_currToken = JsonToken.START_ARRAY);
            } else if (isRisonEnabled(Feature.O_RISON)) {
                _parsingContext = _parsingContext.createChildObjectContext(_tokenInputRow, _tokenInputCol);
                return (_currToken = JsonToken.START_OBJECT);
            }
        }

        /* First: field names are special -- we will always tokenize
         * (part of) value along with field name to simplify
         * state handling. If so, can and need to use secondary token:
         */
        if (_currToken == JsonToken.FIELD_NAME) {
            return _nextAfterName();
        }
        if (_tokenIncomplete) {
            _skipString(); // only strings can be partial
        }
        int i = _nextChOrEnd();
        if (i < 0) { // end-of-input
            /* 19-Feb-2009, tatu: Should actually close/release things
             *    like input source, symbol table and recyclable buffers now.
             */
            close();
            if (_parsingContext.getParent().inRoot()) {
                if (isRisonEnabled(Feature.A_RISON) && _parsingContext.inArray()) {
                    _parsingContext = _parsingContext.getParent();
                    return (_currToken = JsonToken.END_ARRAY);
                }
                if (isRisonEnabled(Feature.O_RISON) && _parsingContext.inObject()) {
                    _parsingContext = _parsingContext.getParent();
                    return (_currToken = JsonToken.END_OBJECT);
                }
            }
            _handleEOF();
            return (_currToken = null);
        }

        /* First, need to ensure we know the starting location of token
         * after skipping leading white space
         */
        _tokenInputTotal = _currInputProcessed + _inputPtr - 1;
        _tokenInputRow = _currInputRow;
        _tokenInputCol = _inputPtr - _currInputRowStart - 1;

        // finally: clear any data retained so far
        _binaryValue = null;

        // Closing scope?
        if (i == ')') {
            if (_parsingContext.inArray()) {
                _parsingContext = _parsingContext.getParent();
                return (_currToken = JsonToken.END_ARRAY);
            }
            if (_parsingContext.inObject()) {
                _parsingContext = _parsingContext.getParent();
                return (_currToken = JsonToken.END_OBJECT);
            }
            _reportMismatchedEndMarker(i, ')');
        }

        // Nope: do we then expect a comma?
        if (_parsingContext.expectComma()) {
            if (i != INT_COMMA) {
                _reportUnexpectedChar(i, "was expecting comma to separate "+_parsingContext.getTypeDesc()+" entries");
            }
            i = _nextCh();
        }

        /* And should we now have a name? Always true for
         * Object contexts, since the intermediate 'expect-value'
         * state is never retained.
         */
        boolean inObject = _parsingContext.inObject();
        if (inObject) {
            // First, field name itself:
            String name;
            if (i != INT_APOSTROPHE) {
                name = _parseUnquotedFieldName(i);
            } else {
                name = _parseFieldName(i);
            }
            _parsingContext.setCurrentName(name);
            _currToken = JsonToken.FIELD_NAME;
            i = _nextCh();
            if (i != INT_COLON) {
                _reportUnexpectedChar(i, "was expecting a colon to separate field name and value");
            }
            i = _nextCh();
        }

        // Ok: we must have a value... what is it?

        JsonToken t;

        switch (i) {
            case INT_APOSTROPHE:
                _tokenIncomplete = true;
                t = JsonToken.VALUE_STRING;
                break;
            case '(':
                if (!inObject) {
                    _parsingContext = _parsingContext.createChildObjectContext(_tokenInputRow, _tokenInputCol);
                }
                t = JsonToken.START_OBJECT;
                break;
            case ')':
                // Error: ')' is not valid at this point; valid closers have
                // been handled earlier
                _reportUnexpectedChar(i, "expected a value");

            case INT_MINUS:
                /* Should we have separate handling for plus? Although
                * it is not allowed per se, it may be erroneously used,
                * and could be indicate by a more specific error message.
                */
            case INT_0:
            case '1':
            case '2':
            case '3':
            case '4':
            case '5':
            case '6':
            case '7':
            case '8':
            case INT_9:
                t = parseNumberText(i);
                break;

            case '!':
                i = _nextCh();
                switch (i) {
                    case '(':
                        if (!inObject) {
                            _parsingContext = _parsingContext.createChildArrayContext(_tokenInputRow, _tokenInputCol);
                        }
                        t = JsonToken.START_ARRAY;
                        break;
                    case 't':
                        t = JsonToken.VALUE_TRUE;
                        break;
                    case 'f':
                        t = JsonToken.VALUE_FALSE;
                        break;
                    case 'n':
                        t = JsonToken.VALUE_NULL;
                        break;
                    default:
                        t = _handleUnexpectedValue(i);
                        break;
                }
                break;
            default:
                if (IdentifierUtils.isIdStartLenient(i)) {
                    t = JsonToken.VALUE_STRING;
                    _inputPtr--;  // push back the first char
                    _parseUnquotedString();
                } else {
                    t = _handleUnexpectedValue(i);
                }
                break;
        }

        if (inObject) {
            _nextToken = t;
            return _currToken;
        }
        _currToken = t;
        return t;
    }

    private JsonToken _nextAfterName()
    {
        _nameCopied = false; // need to invalidate if it was copied
        JsonToken t = _nextToken;
        _nextToken = null;
        // Also: may need to start new context?
        if (t == JsonToken.START_ARRAY) {
            _parsingContext = _parsingContext.createChildArrayContext(_tokenInputRow, _tokenInputCol);
        } else if (t == JsonToken.START_OBJECT) {
            _parsingContext = _parsingContext.createChildObjectContext(_tokenInputRow, _tokenInputCol);
        }
        return (_currToken = t);
    }

    /*
    @Override
    public boolean nextFieldName(SerializableString str)
         throws IOException, JsonParseException
     */

    // note: identical to one in Utf8StreamParser
    @Override
    public String nextTextValue()
            throws IOException, JsonParseException
    {
        if (_currToken == JsonToken.FIELD_NAME) { // mostly copied from '_nextAfterName'
            _nameCopied = false;
            JsonToken t = _nextToken;
            _nextToken = null;
            _currToken = t;
            if (t == JsonToken.VALUE_STRING) {
                if (_tokenIncomplete) {
                    _tokenIncomplete = false;
                    _finishString();
                }
                return _textBuffer.contentsAsString();
            }
            if (t == JsonToken.START_ARRAY) {
                _parsingContext = _parsingContext.createChildArrayContext(_tokenInputRow, _tokenInputCol);
            } else if (t == JsonToken.START_OBJECT) {
                _parsingContext = _parsingContext.createChildObjectContext(_tokenInputRow, _tokenInputCol);
            }
            return null;
        }
        // !!! TODO: optimize this case as well
        return (nextToken() == JsonToken.VALUE_STRING) ? getText() : null;
    }

    // note: identical to one in Utf8StreamParser
    @Override
    public int nextIntValue(int defaultValue)
            throws IOException, JsonParseException
    {
        if (_currToken == JsonToken.FIELD_NAME) {
            _nameCopied = false;
            JsonToken t = _nextToken;
            _nextToken = null;
            _currToken = t;
            if (t == JsonToken.VALUE_NUMBER_INT) {
                return getIntValue();
            }
            if (t == JsonToken.START_ARRAY) {
                _parsingContext = _parsingContext.createChildArrayContext(_tokenInputRow, _tokenInputCol);
            } else if (t == JsonToken.START_OBJECT) {
                _parsingContext = _parsingContext.createChildObjectContext(_tokenInputRow, _tokenInputCol);
            }
            return defaultValue;
        }
        // !!! TODO: optimize this case as well
        return (nextToken() == JsonToken.VALUE_NUMBER_INT) ? getIntValue() : defaultValue;
    }

    // note: identical to one in Utf8StreamParser
    @Override
    public long nextLongValue(long defaultValue)
            throws IOException, JsonParseException
    {
        if (_currToken == JsonToken.FIELD_NAME) { // mostly copied from '_nextAfterName'
            _nameCopied = false;
            JsonToken t = _nextToken;
            _nextToken = null;
            _currToken = t;
            if (t == JsonToken.VALUE_NUMBER_INT) {
                return getLongValue();
            }
            if (t == JsonToken.START_ARRAY) {
                _parsingContext = _parsingContext.createChildArrayContext(_tokenInputRow, _tokenInputCol);
            } else if (t == JsonToken.START_OBJECT) {
                _parsingContext = _parsingContext.createChildObjectContext(_tokenInputRow, _tokenInputCol);
            }
            return defaultValue;
        }
        // !!! TODO: optimize this case as well
        return (nextToken() == JsonToken.VALUE_NUMBER_INT) ? getLongValue() : defaultValue;
    }

    // note: identical to one in Utf8StreamParser
    @Override
    public Boolean nextBooleanValue()
            throws IOException, JsonParseException
    {
        if (_currToken == JsonToken.FIELD_NAME) { // mostly copied from '_nextAfterName'
            _nameCopied = false;
            JsonToken t = _nextToken;
            _nextToken = null;
            _currToken = t;
            if (t == JsonToken.VALUE_TRUE) {
                return Boolean.TRUE;
            }
            if (t == JsonToken.VALUE_FALSE) {
                return Boolean.FALSE;
            }
            if (t == JsonToken.START_ARRAY) {
                _parsingContext = _parsingContext.createChildArrayContext(_tokenInputRow, _tokenInputCol);
            } else if (t == JsonToken.START_OBJECT) {
                _parsingContext = _parsingContext.createChildObjectContext(_tokenInputRow, _tokenInputCol);
            }
            return null;
        }
        switch (nextToken()) {
            case VALUE_TRUE:
                return Boolean.TRUE;
            case VALUE_FALSE:
                return Boolean.FALSE;
        }
        return null;
    }

    @Override
    public void close() throws IOException
    {
        super.close();
        _symbols.release();
    }

    /*
    /**********************************************************
    /* Internal methods, number parsing
    /* (note: in 1.8 and prior, part of "ReaderBasedNumericParser"
    /**********************************************************
     */

    /**
     * Initial parsing method for number values. It needs to be able
     * to parse enough input to be able to determine whether the
     * value is to be considered a simple integer value, or a more
     * generic decimal value: latter of which needs to be expressed
     * as a floating point number. The basic rule is that if the number
     * has no fractional or exponential part, it is an integer; otherwise
     * a floating point number.
     *

     * Because much of input has to be processed in any case, no partial
     * parsing is done: all input text will be stored for further
     * processing. However, actual numeric value conversion will be
     * deferred, since it is usually the most complicated and costliest
     * part of processing.
     */
    protected JsonToken parseNumberText(int ch)
            throws IOException, JsonParseException
    {
        /* Although we will always be complete with respect to textual
         * representation (that is, all characters will be parsed),
         * actual conversion to a number is deferred. Thus, need to
         * note that no representations are valid yet
         */
        boolean negative = (ch == INT_MINUS);
        int ptr = _inputPtr;
        int startPtr = ptr-1; // to include sign/digit already read
        final int inputLen = _inputEnd;

        dummy_loop:
        do { // dummy loop, to be able to break out
            if (negative) { // need to read the next digit
                if (ptr >= _inputEnd) {
                    break dummy_loop;
                }
                ch = _inputBuffer[ptr++];
                // First check: must have a digit to follow minus sign
                if (ch > INT_9 || ch < INT_0) {
                    _inputPtr = ptr;
                    return _handleInvalidNumberStart(ch, true);
                }
                /* (note: has been checked for non-negative already, in
                 * the dispatching code that determined it should be
                 * a numeric value)
                 */
            }
            // One special case, leading zero(es):
            if (ch == INT_0) {
                break dummy_loop;
            }

            /* First, let's see if the whole number is contained within
            * the input buffer unsplit. This should be the common case;
            * and to simplify processing, we will just reparse contents
            * in the alternative case (number split on buffer boundary)
            */

            int intLen = 1; // already got one

            // First let's get the obligatory integer part:

            int_loop:
            while (true) {
                if (ptr >= _inputEnd) {
                    break dummy_loop;
                }
                ch = (int) _inputBuffer[ptr++];
                if (ch < INT_0 || ch > INT_9) {
                    break int_loop;
                }
                ++intLen;
            }

            int fractLen = 0;

            // And then see if we get other parts
            if (ch == '.') { // yes, fraction
                fract_loop:
                while (true) {
                    if (ptr >= inputLen) {
                        break dummy_loop;
                    }
                    ch = (int) _inputBuffer[ptr++];
                    if (ch < INT_0 || ch > INT_9) {
                        break fract_loop;
                    }
                    ++fractLen;
                }
                // must be followed by sequence of ints, one minimum
                if (fractLen == 0) {
                    reportUnexpectedNumberChar(ch, "Decimal point not followed by a digit");
                }
            }

            int expLen = 0;
            if (ch == INT_e || ch == INT_E) { // and/or exponent
                if (ptr >= inputLen) {
                    break dummy_loop;
                }
                // Sign indicator?
                ch = (int) _inputBuffer[ptr++];
                if (ch == INT_MINUS || ch == INT_PLUS) { // yup, skip for now
                    if (ptr >= inputLen) {
                        break dummy_loop;
                    }
                    ch = (int) _inputBuffer[ptr++];
                }
                while (ch <= INT_9 && ch >= INT_0) {
                    ++expLen;
                    if (ptr >= inputLen) {
                        break dummy_loop;
                    }
                    ch = (int) _inputBuffer[ptr++];
                }
                // must be followed by sequence of ints, one minimum
                if (expLen == 0) {
                    reportUnexpectedNumberChar(ch, "Exponent indicator not followed by a digit");
                }
            }

            // Got it all: let's add to text buffer for parsing, access
            --ptr; // need to push back following separator
            _inputPtr = ptr;
            int len = ptr-startPtr;
            _textBuffer.resetWithShared(_inputBuffer, startPtr, len);
            return reset(negative, intLen, fractLen, expLen);
        } while (false);

        _inputPtr = negative ? (startPtr+1) : startPtr;
        return parseNumberText2(negative);
    }

    /**
     * Method called to parse a number, when the primary parse
     * method has failed to parse it, due to it being split on
     * buffer boundary. As a result code is very similar, except
     * that it has to explicitly copy contents to the text buffer
     * instead of just sharing the main input buffer.
     */
    private JsonToken parseNumberText2(boolean negative)
            throws IOException, JsonParseException
    {
        char[] outBuf = _textBuffer.emptyAndGetCurrentSegment();
        int outPtr = 0;

        // Need to prepend sign?
        if (negative) {
            outBuf[outPtr++] = '-';
        }

        // This is the place to do leading-zero check(s) too:
        int intLen = 0;
        char c = (_inputPtr < _inputEnd) ? _inputBuffer[_inputPtr++] : getNextChar("No digit following minus sign");
        if (c == '0') {
            c = _verifyNoLeadingZeroes();
        }
        boolean eof = false;

        // Ok, first the obligatory integer part:
        int_loop:
        while (c >= '0' && c <= '9') {
            ++intLen;
            if (outPtr >= outBuf.length) {
                outBuf = _textBuffer.finishCurrentSegment();
                outPtr = 0;
            }
            outBuf[outPtr++] = c;
            if (_inputPtr >= _inputEnd && !loadMore()) {
                // EOF is legal for main level int values
                c = CHAR_NULL;
                eof = true;
                break int_loop;
            }
            c = _inputBuffer[_inputPtr++];
        }
        // Also, integer part is not optional
        if (intLen == 0) {
            reportInvalidNumber("Missing integer part (next char "+_getCharDesc(c)+")");
        }

        int fractLen = 0;
        // And then see if we get other parts
        if (c == '.') { // yes, fraction
            outBuf[outPtr++] = c;

            fract_loop:
            while (true) {
                if (_inputPtr >= _inputEnd && !loadMore()) {
                    eof = true;
                    break fract_loop;
                }
                c = _inputBuffer[_inputPtr++];
                if (c < INT_0 || c > INT_9) {
                    break fract_loop;
                }
                ++fractLen;
                if (outPtr >= outBuf.length) {
                    outBuf = _textBuffer.finishCurrentSegment();
                    outPtr = 0;
                }
                outBuf[outPtr++] = c;
            }
            // must be followed by sequence of ints, one minimum
            if (fractLen == 0) {
                reportUnexpectedNumberChar(c, "Decimal point not followed by a digit");
            }
        }

        int expLen = 0;
        if (c == 'e' || c == 'E') { // exponent?
            if (outPtr >= outBuf.length) {
                outBuf = _textBuffer.finishCurrentSegment();
                outPtr = 0;
            }
            outBuf[outPtr++] = c;
            // Not optional, can require that we get one more char
            c = (_inputPtr < _inputEnd) ? _inputBuffer[_inputPtr++]
                    : getNextChar("expected a digit for number exponent");
            // Sign indicator?
            if (c == '-' || c == '+') {
                if (outPtr >= outBuf.length) {
                    outBuf = _textBuffer.finishCurrentSegment();
                    outPtr = 0;
                }
                outBuf[outPtr++] = c;
                // Likewise, non optional:
                c = (_inputPtr < _inputEnd) ? _inputBuffer[_inputPtr++]
                        : getNextChar("expected a digit for number exponent");
            }

            exp_loop:
            while (c <= INT_9 && c >= INT_0) {
                ++expLen;
                if (outPtr >= outBuf.length) {
                    outBuf = _textBuffer.finishCurrentSegment();
                    outPtr = 0;
                }
                outBuf[outPtr++] = c;
                if (_inputPtr >= _inputEnd && !loadMore()) {
                    eof = true;
                    break exp_loop;
                }
                c = _inputBuffer[_inputPtr++];
            }
            // must be followed by sequence of ints, one minimum
            if (expLen == 0) {
                reportUnexpectedNumberChar(c, "Exponent indicator not followed by a digit");
            }
        }

        // Ok; unless we hit end-of-input, need to push last char read back
        if (!eof) {
            --_inputPtr;
        }
        _textBuffer.setCurrentLength(outPtr);
        // And there we have it!
        return reset(negative, intLen, fractLen, expLen);
    }

    /**
     * Method called when we have seen one zero, and want to ensure
     * it is not followed by another
     */
    private char _verifyNoLeadingZeroes()
            throws IOException, JsonParseException
    {
        // Ok to have plain "0"
        if (_inputPtr >= _inputEnd && !loadMore()) {
            return '0';
        }
        char ch = _inputBuffer[_inputPtr];
        // if not followed by a number (probably '.'); return zero as is, to be included
        if (ch < '0' || ch > '9') {
            return '0';
        }
        if (!isEnabled(JsonParser.Feature.ALLOW_NUMERIC_LEADING_ZEROS)) {
            reportInvalidNumber("Leading zeroes not allowed");
        }
        // if so, just need to skip either all zeroes (if followed by number); or all but one (if non-number)
        ++_inputPtr; // Leading zero to be skipped
        if (ch == INT_0) {
            while (_inputPtr < _inputEnd || loadMore()) {
                ch = _inputBuffer[_inputPtr];
                if (ch < '0' || ch > '9') { // followed by non-number; retain one zero
                    return '0';
                }
                ++_inputPtr; // skip previous zero
                if (ch != '0') { // followed by other number; return
                    break;
                }
            }
        }
        return ch;
    }

    /**
     * Method called if expected numeric value (due to leading sign) does not
     * look like a number
     */
    protected JsonToken _handleInvalidNumberStart(int ch, boolean negative)
            throws IOException, JsonParseException
    {
        if (ch == 'I') {
            if (_inputPtr >= _inputEnd) {
                if (!loadMore()) {
                    _reportInvalidEOFInValue();
                }
            }
            ch = _inputBuffer[_inputPtr++];
            if (ch == 'N') {
                String match = negative ? "-INF" :"+INF";
                _matchToken(match, 3);
                if (isEnabled(JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS)) {
                    return resetAsNaN(match, negative ? Double.NEGATIVE_INFINITY : Double.POSITIVE_INFINITY);
                }
                _reportError("Non-standard token '"+match+"': enable JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS to allow");
            } else if (ch == 'n') {
                String match = negative ? "-Infinity" :"+Infinity";
                _matchToken(match, 3);
                if (isEnabled(JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS)) {
                    return resetAsNaN(match, negative ? Double.NEGATIVE_INFINITY : Double.POSITIVE_INFINITY);
                }
                _reportError("Non-standard token '"+match+"': enable JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS to allow");
            }
        }
        reportUnexpectedNumberChar(ch, "expected digit (0-9) to follow minus sign, for valid numeric value");
        return null;
    }

    /*
    /**********************************************************
    /* Internal methods, secondary parsing
    /**********************************************************
     */

    protected String _parseFieldName(int i)
            throws IOException, JsonParseException
    {
        /* First: let's try to see if we have a simple name: one that does
         * not cross input buffer boundary, and does not contain escape
         * sequences.
         */
        int ptr = _inputPtr;
        int hash = 0;
        final int inputLen = _inputEnd;

        if (ptr < inputLen) {
            do {
                int ch = _inputBuffer[ptr];
                if (ch == INT_APOSTROPHE) {
                    int start = _inputPtr;
                    _inputPtr = ptr+1; // to skip the quote
                    return _symbols.findSymbol(_inputBuffer, start, ptr - start, hash);
                } else if (ch == '!') {
                    break;  // escape, can't take the fast path
                }
                hash = (hash * 31) + ch;
                ++ptr;
            } while (ptr < inputLen);
        }

        int start = _inputPtr;
        _inputPtr = ptr;
        return _parseFieldName2(start, hash, INT_APOSTROPHE);
    }

    private String _parseFieldName2(int startPtr, int hash, int endChar)
            throws IOException, JsonParseException
    {
        _textBuffer.resetWithShared(_inputBuffer, startPtr, (_inputPtr - startPtr));

        /* Output pointers; calls will also ensure that the buffer is
         * not shared and has room for at least one more char.
         */
        char[] outBuf = _textBuffer.getCurrentSegment();
        int outPtr = _textBuffer.getCurrentSegmentSize();

        while (true) {
            if (_inputPtr >= _inputEnd) {
                if (!loadMore()) {
                    _reportInvalidEOF(": was expecting closing '"+((char) endChar)+"' for name");
                }
            }
            char c = _inputBuffer[_inputPtr++];
            if (c == endChar) {
                break;
            } else if (c == '!') {
                c = _decodeEscaped();
            }
            hash = (hash * 31) + c;
            // Ok, let's add char to output:
            outBuf[outPtr++] = c;

            // Need more room?
            if (outPtr >= outBuf.length) {
                outBuf = _textBuffer.finishCurrentSegment();
                outPtr = 0;
            }
        }
        _textBuffer.setCurrentLength(outPtr);
        {
            TextBuffer tb = _textBuffer;
            char[] buf = tb.getTextBuffer();
            int start = tb.getTextOffset();
            int len = tb.size();

            return _symbols.findSymbol(buf, start, len, hash);
        }
    }

    /**
     * Method called when we see non-white space character other
     * than double quote, when expecting a field name.
     * In standard mode will just throw an expection; but
     * in non-standard modes may be able to parse name.
     */
    protected String _parseUnquotedFieldName(int i)
            throws IOException, JsonParseException
    {
        // Also: first char must be a valid name char, but NOT be number
        boolean firstOk = IdentifierUtils.isIdStartLenient(i);
        if (!firstOk) {
            _reportUnexpectedChar(i, "was expecting either valid name character (for unquoted name) or single-quote (for quoted) to start field name");
        }
        int ptr = _inputPtr;
        int hash = 0;
        final int inputLen = _inputEnd;

        if (ptr < inputLen) {
            do {
                int ch = _inputBuffer[ptr];
                if (!IdentifierUtils.isIdCharLenient(ch)) {
                    int start = _inputPtr-1; // -1 to bring back first char
                    _inputPtr = ptr;
                    return _symbols.findSymbol(_inputBuffer, start, ptr - start, hash);
                }
                hash = (hash * 31) + ch;
                ++ptr;
            } while (ptr < inputLen);
        }
        int start = _inputPtr-1;
        _inputPtr = ptr;
        return _parseUnquotedFieldName2(start, hash);
    }

    /**
     * Method for handling cases where first non-space character
     * of an expected value token is not legal for standard JSON content.
     */
    protected JsonToken _handleUnexpectedValue(int i)
            throws IOException, JsonParseException
    {
        // Most likely an error, unless we are to allow single-quote-strings
        switch (i) {
            case 'N':
                _matchToken("NaN", 1);
                if (isEnabled(JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS)) {
                    return resetAsNaN("NaN", Double.NaN);
                }
                _reportError("Non-standard token 'NaN': enable JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS to allow");
                break;
            case '+': // note: '-' is taken as number
                if (_inputPtr >= _inputEnd) {
                    if (!loadMore()) {
                        _reportInvalidEOFInValue();
                    }
                }
                return _handleInvalidNumberStart(_inputBuffer[_inputPtr++], false);
        }
        _reportUnexpectedChar(i, "expected a valid value (number, String, array, object, 'true', 'false' or 'null')");
        return null;
    }

    private String _parseUnquotedFieldName2(int startPtr, int hash)
            throws IOException, JsonParseException
    {
        _textBuffer.resetWithShared(_inputBuffer, startPtr, (_inputPtr - startPtr));
        char[] outBuf = _textBuffer.getCurrentSegment();
        int outPtr = _textBuffer.getCurrentSegmentSize();

        while (true) {
            if (_inputPtr >= _inputEnd) {
                if (!loadMore()) { // acceptable for now (will error out later)
                    break;
                }
            }
            char c = _inputBuffer[_inputPtr];
            if (!IdentifierUtils.isIdCharLenient(c)) {
                break;
            }
            ++_inputPtr;
            hash = (hash * 31) + c;
            // Ok, let's add char to output:
            outBuf[outPtr++] = c;

            // Need more room?
            if (outPtr >= outBuf.length) {
                outBuf = _textBuffer.finishCurrentSegment();
                outPtr = 0;
            }
        }
        _textBuffer.setCurrentLength(outPtr);
        {
            TextBuffer tb = _textBuffer;
            char[] buf = tb.getTextBuffer();
            int start = tb.getTextOffset();
            int len = tb.size();

            return _symbols.findSymbol(buf, start, len, hash);
        }
    }

    protected void _parseUnquotedString()
            throws IOException, JsonParseException
    {
        /* First: let's try to see if we have simple String value: one
         * that does not cross input buffer boundary, and does not
         * contain escape sequences.
         */
        int ptr = _inputPtr;
        final int inputLen = _inputEnd;

        if (ptr < inputLen) {
            do {
                char ch = _inputBuffer[ptr];
                if (!IdentifierUtils.isIdCharLenient(ch)) {
                    _textBuffer.resetWithShared(_inputBuffer, _inputPtr, (ptr-_inputPtr));
                    _inputPtr = ptr;
                    // Yes, we got it all
                    return;
                }
                ++ptr;
            } while (ptr < inputLen);
        }

        /* Either ran out of input, or bumped into an escape
         * sequence...
         */
        _textBuffer.resetWithCopy(_inputBuffer, _inputPtr, (ptr-_inputPtr));
        _inputPtr = ptr;
        _parseUnquotedString2();
    }

    protected void _parseUnquotedString2()
            throws IOException, JsonParseException
    {
        char[] outBuf = _textBuffer.getCurrentSegment();
        int outPtr = _textBuffer.getCurrentSegmentSize();

        while (true) {
            if (_inputPtr >= _inputEnd && !loadMore()) {
                break;  // eof
            }
            char c = _inputBuffer[_inputPtr];
            if (!IdentifierUtils.isIdCharLenient(c)) {
                break;
            }
            _inputPtr++;
            // Need more room?
            if (outPtr >= outBuf.length) {
                outBuf = _textBuffer.finishCurrentSegment();
                outPtr = 0;
            }
            // Ok, let's add char to output:
            outBuf[outPtr++] = c;
        }
        _textBuffer.setCurrentLength(outPtr);
    }

    @Override
    protected void _finishString()
            throws IOException, JsonParseException
    {
        /* First: let's try to see if we have simple String value: one
         * that does not cross input buffer boundary, and does not
         * contain escape sequences.
         */
        int ptr = _inputPtr;
        final int inputLen = _inputEnd;

        if (ptr < inputLen) {
            do {
                int ch = _inputBuffer[ptr];
                if (ch == INT_APOSTROPHE) {
                    _textBuffer.resetWithShared(_inputBuffer, _inputPtr, (ptr-_inputPtr));
                    _inputPtr = ptr+1;
                    // Yes, we got it all
                    return;
                } else if (ch == '!') {
                    break;  // escape, can't take the fast path
                }
                ++ptr;
            } while (ptr < inputLen);
        }

        /* Either ran out of input, or bumped into an escape
         * sequence...
         */
        _textBuffer.resetWithCopy(_inputBuffer, _inputPtr, (ptr-_inputPtr));
        _inputPtr = ptr;
        _finishString2();
    }

    protected void _finishString2()
            throws IOException, JsonParseException
    {
        char[] outBuf = _textBuffer.getCurrentSegment();
        int outPtr = _textBuffer.getCurrentSegmentSize();

        while (true) {
            if (_inputPtr >= _inputEnd) {
                if (!loadMore()) {
                    _reportInvalidEOF(": was expecting closing quote for a string value");
                }
            }
            char c = _inputBuffer[_inputPtr++];
            if (c == INT_APOSTROPHE) {
                break;
            } else if (c == '!') {
                c = _decodeEscaped();
            }
            // Need more room?
            if (outPtr >= outBuf.length) {
                outBuf = _textBuffer.finishCurrentSegment();
                outPtr = 0;
            }
            // Ok, let's add char to output:
            outBuf[outPtr++] = c;
        }
        _textBuffer.setCurrentLength(outPtr);
    }

    /**
     * Method called to skim through rest of unparsed String value,
     * if it is not needed. This can be done bit faster if contents
     * need not be stored for future access.
     */
    protected void _skipString()
            throws IOException, JsonParseException
    {
        _tokenIncomplete = false;

        int inputPtr = _inputPtr;
        int inputLen = _inputEnd;
        char[] inputBuffer = _inputBuffer;

        while (true) {
            if (inputPtr >= inputLen) {
                _inputPtr = inputPtr;
                if (!loadMore()) {
                    _reportInvalidEOF(": was expecting closing quote for a string value");
                }
                inputPtr = _inputPtr;
                inputLen = _inputEnd;
            }
            char c = inputBuffer[inputPtr++];
            if (c == INT_APOSTROPHE) {
                _inputPtr = inputPtr;
                break;
            } else if (c == '!') {
                _inputPtr = inputPtr;
                _decodeEscaped();
                inputPtr = _inputPtr;
                inputLen = _inputEnd;
            }
        }
    }

    /*
    /**********************************************************
    /* Internal methods, other parsing
    /**********************************************************
     */

    private int _nextCh()
            throws IOException, JsonParseException
    {
        if (_inputPtr < _inputEnd || loadMore()) {
            return _inputBuffer[_inputPtr++];
        }
        throw _constructError("Unexpected end-of-input within/between " + _parsingContext.getTypeDesc() + " entries");
    }

    private int _nextChOrEnd()
            throws IOException, JsonParseException
    {
        if ((_inputPtr < _inputEnd) || loadMore()) {
            return _inputBuffer[_inputPtr++];
        }
        return -1;
    }

    @Override
    protected char _decodeEscaped()
            throws IOException, JsonParseException
    {
        if (_inputPtr >= _inputEnd) {
            if (!loadMore()) {
                _reportInvalidEOF(" in character escape sequence");
            }
        }
        char c = _inputBuffer[_inputPtr++];
        if (c == '!' || c == '\'') {
            return c;
        }
        return _handleUnrecognizedCharacterEscape(c);
    }

    /**
     * Helper method for checking whether input matches expected token
     */
    protected void _matchToken(String matchStr, int i)
            throws IOException, JsonParseException
    {
        final int len = matchStr.length();

        do {
            if (_inputPtr >= _inputEnd) {
                if (!loadMore()) {
                    _reportInvalidEOFInValue();
                }
            }
            if (_inputBuffer[_inputPtr] != matchStr.charAt(i)) {
                _reportInvalidToken(matchStr.substring(0, i), "'null', 'true', 'false' or NaN");
            }
            ++_inputPtr;
        } while (++i < len);

        // but let's also ensure we either get EOF, or non-alphanum char...
        if (_inputPtr >= _inputEnd) {
            if (!loadMore()) {
                return;
            }
        }
        char c = _inputBuffer[_inputPtr];
        if (c < '0' || c == ')') { // expected/allowed chars
            return;
        }
        // if letter, it's a problem tho
        if (IdentifierUtils.isIdCharLenient(c)) {
            _reportInvalidToken(matchStr.substring(0, i), "'null', 'true', 'false' or NaN");
        }
    }

    /*
    /**********************************************************
    /* Binary access
    /**********************************************************
     */

    /**
     * Efficient handling for incremental parsing of base64-encoded
     * textual content.
     */
    protected byte[] _decodeBase64(Base64Variant b64variant)
            throws IOException, JsonParseException
    {
        ByteArrayBuilder builder = _getByteArrayBuilder();

        //main_loop:
        while (true) {
            // first, we'll skip preceding white space, if any
            char ch;
            do {
                if (_inputPtr >= _inputEnd) {
                    loadMoreGuaranteed();
                }
                ch = _inputBuffer[_inputPtr++];
            } while (ch <= INT_SPACE);
            int bits = b64variant.decodeBase64Char(ch);
            if (bits < 0) {
                if (ch == INT_APOSTROPHE) { // reached the end, fair and square?
                    return builder.toByteArray();
                }
                bits = _decodeBase64Escape2(b64variant, ch, 0);
                if (bits < 0) { // white space to skip
                    continue;
                }
            }
            int decodedData = bits;

            // then second base64 char; can't get padding yet, nor ws

            if (_inputPtr >= _inputEnd) {
                loadMoreGuaranteed();
            }
            ch = _inputBuffer[_inputPtr++];
            bits = b64variant.decodeBase64Char(ch);
            if (bits < 0) {
                bits = _decodeBase64Escape2(b64variant, ch, 1);
            }
            decodedData = (decodedData << 6) | bits;

            // third base64 char; can be padding, but not ws
            if (_inputPtr >= _inputEnd) {
                loadMoreGuaranteed();
            }
            ch = _inputBuffer[_inputPtr++];
            bits = b64variant.decodeBase64Char(ch);

            // First branch: can get padding (-> 1 byte)
            if (bits < 0) {
                if (bits != Base64Variant.BASE64_VALUE_PADDING) {
                    // as per [JACKSON-631], could also just be 'missing'  padding
                    if (ch == INT_APOSTROPHE && !b64variant.usesPadding()) {
                        decodedData >>= 4;
                        builder.append(decodedData);
                        return builder.toByteArray();
                    }
                    bits = _decodeBase64Escape2(b64variant, ch, 2);
                }
                if (bits == Base64Variant.BASE64_VALUE_PADDING) {
                    // Ok, must get more padding chars, then
                    if (_inputPtr >= _inputEnd) {
                        loadMoreGuaranteed();
                    }
                    ch = _inputBuffer[_inputPtr++];
                    if (!b64variant.usesPaddingChar(ch)) {
                        throw reportInvalidBase64Char(b64variant, ch, 3, "expected padding character '"+b64variant.getPaddingChar()+"'");
                    }
                    // Got 12 bits, only need 8, need to shift
                    decodedData >>= 4;
                    builder.append(decodedData);
                    continue;
                }
                // otherwise we got escaped other char, to be processed below
            }
            // Nope, 2 or 3 bytes
            decodedData = (decodedData << 6) | bits;
            // fourth and last base64 char; can be padding, but not ws
            if (_inputPtr >= _inputEnd) {
                loadMoreGuaranteed();
            }
            ch = _inputBuffer[_inputPtr++];
            bits = b64variant.decodeBase64Char(ch);
            if (bits < 0) {
                if (bits != Base64Variant.BASE64_VALUE_PADDING) {
                    // as per [JACKSON-631], could also just be 'missing'  padding
                    if (ch == INT_APOSTROPHE && !b64variant.usesPadding()) {
                        decodedData >>= 2;
                        builder.appendTwoBytes(decodedData);
                        return builder.toByteArray();
                    }
                    bits = _decodeBase64Escape2(b64variant, ch, 3);
                }
                if (bits == Base64Variant.BASE64_VALUE_PADDING) {
                    // With padding we only get 2 bytes; but we have
                    // to shift it a bit so it is identical to triplet
                    // case with partial output.
                    // 3 chars gives 3x6 == 18 bits, of which 2 are
                    // dummies, need to discard:
                    decodedData >>= 2;
                    builder.appendTwoBytes(decodedData);
                    continue;
                }
                // otherwise we got escaped other char, to be processed below
            }
            // otherwise, our triplet is now complete
            decodedData = (decodedData << 6) | bits;
            builder.appendThreeBytes(decodedData);
        }
    }

    protected int _decodeBase64Escape2(Base64Variant b64variant, char ch, int index)
        throws IOException, JsonParseException
    {
        // if white space, skip if first triplet; otherwise errors
        if (ch <= INT_SPACE) {
            if (index == 0) { // whitespace only allowed to be skipped between triplets
                return -1;
            }
        }
        // 17-May-2011, tatu: As per [JACKSON-xxx], need to handle escaped chars
        if (ch != '!') {
            throw reportInvalidBase64Char(b64variant, ch, index);
        }
        char unescaped = _decodeEscaped();
        // otherwise try to find actual triplet value
        int bits = b64variant.decodeBase64Char(unescaped);
        if (bits < 0) {
            throw reportInvalidBase64Char(b64variant, unescaped, index);
        }
        return bits;
    }

    /*
    /**********************************************************
    /* Error reporting
    /**********************************************************
     */

    protected void _reportInvalidToken(String matchedPart, String msg)
            throws IOException, JsonParseException
    {
        StringBuilder sb = new StringBuilder(matchedPart);
        /* Let's just try to find what appears to be the token, using
         * regular Java identifier character rules. It's just a heuristic,
         * nothing fancy here.
         */
        while (true) {
            if (_inputPtr >= _inputEnd) {
                if (!loadMore()) {
                    break;
                }
            }
            char c = _inputBuffer[_inputPtr];
            if (!IdentifierUtils.isIdCharLenient(c)) {
                break;
            }
            ++_inputPtr;
            sb.append(c);
        }
        _reportError("Unrecognized token '"+sb.toString()+"': was expecting ");
    }
}


package com.bazaarvoice.jackson.rison;

/**
 * Utilities for dealing with Rison unquoted identifiers.
 */
class IdentifierUtils {

    // Ascii idstart and idchar that match the rison spec precisely.  non-ascii (>127) are also allowed.
    private static final int[] _idStartStrict = toIdSet(true, "_./~");
    private static final int[] _idCharStrict =  toIdSet(true, "_./~-0123456789");

    // Lenient idstart and idchar that accept strings encoded by http://mjtemplate.org/dist/mjt-0.9.2/rison.js
    // which allows a number of punctuation and whitespace characters that are excluded in the Rison spec.
    private static final int[] _idStartLenient = invert(toIdSet(false, " '!:(),*@$-0123456789"));
    private static final int[] _idCharLenient =  invert(toIdSet(false, " '!:(),*@$"));

    /**
     * Returns true if the specified character is a legal first character in an unquoted identifier.
     * Uses lenient rules appropriate for parsing.
     */
    public static boolean isIdStartLenient(int ch) {
        return ch > 127 || in(ch, _idStartLenient);
    }

    /**
     * Returns true if the specified character is a legal following character in an unquoted identifier.
     * Uses lenient rules appropriate for parsing.
     */
    public static boolean isIdCharLenient(int ch) {
        return ch > 127 || in(ch, _idCharLenient);
    }

    /**
     * Returns true if the specified character is a legal first character in an unquoted identifier.
     * Uses the strict definition described in the Rison specification, appropriate for encoding.
     */
    public static boolean isIdStartStrict(int ch) {
        return ch > 127 || in(ch, _idStartStrict);
    }

    /**
     * Returns true if the specified character is a legal following character in an unquoted identifier.
     * Uses the strict definition described in the Rison specification, appropriate for encoding.
     */
    public static boolean isIdCharStrict(int ch) {
        return ch > 127 || in(ch, _idCharStrict);
    }

    /**
     * Returns true if a string does not need to be quoted when serialized.
     * Uses the strict definition described in the Rison specification, appropriate for encoding.
     */
    public static boolean isIdStrict(String string) {
        int len = string.length();
        if (len == 0) {
            return false;
        }
        if (!isIdStartStrict(string.charAt(0))) {
            return false;
        }
        for (int i = 1; i < len; i++) {
            if (!isIdCharStrict(string.charAt(i))) {
                return false;
            }
        }
        return true;
    }

    /**
     * Returns true if the string represented by the specified character array does not
     * Uses the strict definition described in the Rison specification, appropriate for encoding.
     */
    public static boolean isIdStrict(char[] chars, int offset, int len) {
        if (len == 0) {
            return false;
        }
        int end = offset + len;
        if (!isIdStartStrict(chars[offset++])) {
            return false;
        }
        while (offset < end) {
            if (!isIdCharStrict(chars[offset++])) {
                return false;
            }
        }
        return true;
    }

    private static boolean in(int ch, int[] bits) {
        return (bits[ch >>> 5] & (1 << (ch & 31))) != 0;
    }

    /**
     * Encodes a set of ASCII characters (< 127) as a bitset of 4 32-bit values.
     */
    private static int[] toIdSet(boolean includeAlphaCharacters, String nonAlphaCharacters) {
        int[] bits = new int[4];  // 4*32=128 which covers all ASCII
        if (includeAlphaCharacters) {
            for (int ch = 'A'; ch <= 'Z'; ch++) {
                bits[ch >>> 5] |= (1 << (ch & 31));
            }
            for (int ch = 'a'; ch <= 'z'; ch++) {
                bits[ch >>> 5] |= (1 << (ch & 31));
            }
        }
        for (int i = 0; i < nonAlphaCharacters.length(); i++) {
            int ch = nonAlphaCharacters.charAt(i);
            if (ch >= 128) {
                throw new AssertionError(); // out of range
            }
            bits[ch >>> 5] |= (1 << (ch & 31));
        }
        return bits;
    }

    private static int[] invert(int[] bits) {
        for (int i = 0; i < bits.length; i++) {
            bits[i] = ~bits[i];
        }
        return bits;
    }
}


/*
Large portions of this file are copyright by the authors of Jackson under the Apache 2.0 or LGPL license.

The implementation was derived from the Jackson class 'org.codehaus.jackson.impl.WriterBasedGenerator' and
modified for Rison.
*/

package com.bazaarvoice.jackson.rison;

import com.fasterxml.jackson.core.Base64Variant;
import com.fasterxml.jackson.core.JsonGenerationException;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonStreamContext;
import com.fasterxml.jackson.core.ObjectCodec;
import com.fasterxml.jackson.core.SerializableString;
import com.fasterxml.jackson.core.Version;
import com.fasterxml.jackson.core.base.GeneratorBase;
import com.fasterxml.jackson.core.io.IOContext;
import com.fasterxml.jackson.core.io.NumberOutput;
import com.fasterxml.jackson.core.json.JsonWriteContext;
import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;

import java.io.IOException;
import java.io.Writer;
import java.math.BigDecimal;
import java.math.BigInteger;

/**
 * {@link JsonGenerator} that outputs Rison content using a {@link java.io.Writer}
 * which handles character encoding.
 */
public final class RisonGenerator
        extends GeneratorBase
{
    /**
     * Enumeration that defines all configurable features for Rison generators.
     */
    public enum Feature {
        /**
         * Whether to assume that the top-level value being encoded is an object,
         * and therefore to omit the containing '(' and ')'.
         *

         * Default setting is false.
         */
        O_RISON(false),

        /**
         * Whether to assume that the top-level value being encoded is an array,
         * and therefore to omit the containing '!(' and ')'.
         *

         * Default setting is false.
         */
        A_RISON(false)
        ;

        final boolean _defaultState;

        final int _mask;

        /**
         * Method that calculates bit set (flags) of all features that
         * are enabled by default.
         */
        public static int collectDefaults()
        {
            int flags = 0;
            for (Feature f : values()) {
                if (f.enabledByDefault()) {
                    flags |= f.getMask();
                }
            }
            return flags;
        }

        private Feature(boolean defaultState) {
            _defaultState = defaultState;
            _mask = (1 << ordinal());
        }

        public boolean enabledByDefault() { return _defaultState; }

        public int getMask() { return _mask; }
    }

    final protected static int SHORT_WRITE = 32;

    /*
   /**********************************************************
   /* Configuration
   /**********************************************************
    */

    final protected IOContext _ioContext;

    final protected Writer _writer;

    final protected int _risonFeatures;

    protected SerializableString _rootValueSeparator
            = DefaultPrettyPrinter.DEFAULT_ROOT_VALUE_SEPARATOR;

    /*
    /**********************************************************
    /* Output buffering
    /**********************************************************
     */

    /**
     * Intermediate buffer in which contents are buffered before
     * being written using {@link #_writer}.
     */
    protected char[] _outputBuffer;

    /**
     * Pointer to the first buffered character to output
     */
    protected int _outputHead = 0;

    /**
     * Pointer to the position right beyond the last character to output
     * (end marker; may point to position right beyond the end of the buffer)
     */
    protected int _outputTail = 0;

    /**
     * End marker of the output buffer; one past the last valid position
     * within the buffer.
     */
    protected int _outputEnd;

    /*
    /**********************************************************
    /* Life-cycle
    /**********************************************************
     */

    public RisonGenerator(IOContext ctxt, int jsonFeatures, int risonFeatures, ObjectCodec codec,
                          Writer w)
    {
        super(jsonFeatures, codec);
        _ioContext = ctxt;
        _writer = w;
        _risonFeatures = risonFeatures;
        _outputBuffer = ctxt.allocConcatBuffer();
        _outputEnd = _outputBuffer.length;
    }

    @Override
    public Version version() {
        return PackageVersion.version();
    }

    /*
   /**********************************************************
   /* Overridden configuration methods
   /**********************************************************
    */

    @Override
    public Object getOutputTarget() {
        return _writer;
    }

    private boolean isRisonEnabled(Feature feature) {
        return (_risonFeatures & feature.getMask()) != 0;
    }

    @Override
    public JsonGenerator setRootValueSeparator(SerializableString sep) {
        _rootValueSeparator = sep;
        return this;
    }

    /*
   /**********************************************************
   /* Overridden methods
   /**********************************************************
    */

    @Override
    public void writeFieldName(String name)  throws IOException, JsonGenerationException
    {
        int status = _writeContext.writeFieldName(name);
        if (status == JsonWriteContext.STATUS_EXPECT_VALUE) {
            _reportError("Can not write a field name, expecting a value");
        }
        _writeFieldName(name, (status == JsonWriteContext.STATUS_OK_AFTER_COMMA));
    }

    @Override
    public void writeFieldName(SerializableString name)
            throws IOException, JsonGenerationException
    {
        // Object is a value, need to verify it's allowed
        int status = _writeContext.writeFieldName(name.getValue());
        if (status == JsonWriteContext.STATUS_EXPECT_VALUE) {
            _reportError("Can not write a field name, expecting a value");
        }
        _writeFieldName(name, (status == JsonWriteContext.STATUS_OK_AFTER_COMMA));
    }

    /*
   /**********************************************************
   /* Output method implementations, structural
   /**********************************************************
    */

    private boolean omitArrayWrappers(JsonWriteContext writeContext) {
        return writeContext.inRoot() && isRisonEnabled(Feature.A_RISON);
    }

    private boolean omitObjectWrappers(JsonWriteContext writeContext) {
        return writeContext.inRoot() && isRisonEnabled(Feature.O_RISON);
    }

    @Override
    public void writeStartArray() throws IOException, JsonGenerationException
    {
        _verifyValueWrite("start an array");
        if (!omitArrayWrappers(_writeContext)) {
            if ((_outputTail + 1) >= _outputEnd) {
                _flushBuffer();
            }
            _outputBuffer[_outputTail++] = '!';
            _outputBuffer[_outputTail++] = '(';
        }
        _writeContext = _writeContext.createChildArrayContext();
    }

    @Override
    public void writeEndArray() throws IOException, JsonGenerationException
    {
        if (!_writeContext.inArray()) {
            _reportError("Current context not an ARRAY but "+_writeContext.getTypeDesc());
        }
        _writeContext = _writeContext.getParent();
        if (!omitArrayWrappers(_writeContext)) {
            if (_outputTail >= _outputEnd) {
                _flushBuffer();
            }
            _outputBuffer[_outputTail++] = ')';
        }
    }

    @Override
    public void writeStartObject() throws IOException, JsonGenerationException
    {
        _verifyValueWrite("start an object");
        if (!(omitObjectWrappers(_writeContext))) {
            if (_outputTail >= _outputEnd) {
                _flushBuffer();
            }
            _outputBuffer[_outputTail++] = '(';
        }
        _writeContext = _writeContext.createChildObjectContext();
    }

    @Override
    public void writeEndObject() throws IOException, JsonGenerationException
    {
        if (!_writeContext.inObject()) {
            _reportError("Current context not an object but "+_writeContext.getTypeDesc());
        }
        _writeContext = _writeContext.getParent();
        if (!omitObjectWrappers(_writeContext)) {
            if (_outputTail >= _outputEnd) {
                _flushBuffer();
            }
            _outputBuffer[_outputTail++] = ')';
        }
    }

    protected void _writeFieldName(String name, boolean commaBefore)
            throws IOException, JsonGenerationException
    {
        // for fast+std case, need to output up to 2 chars, comma, quote
        if ((_outputTail + 1) >= _outputEnd) {
            _flushBuffer();
        }
        if (commaBefore) {
            _outputBuffer[_outputTail++] = ',';
        }

        if (IdentifierUtils.isIdStrict(name)) {
            _writeRaw(name);
            return;
        }

        // we know there's room for at least one more char
        _outputBuffer[_outputTail++] = '\'';
        // The beef:
        _writeString(name);
        // and closing quotes; need room for one more char:
        if (_outputTail >= _outputEnd) {
            _flushBuffer();
        }
        _outputBuffer[_outputTail++] = '\'';
    }

    public void _writeFieldName(SerializableString name, boolean commaBefore)
            throws IOException, JsonGenerationException
    {
        // Can't take advantage of SerializableString.asQuotedChars() because JSON and Rison
        // have different rules for quoting characters within strings.
        _writeFieldName(name.getValue(), commaBefore);
    }

    /*
    /**********************************************************
    /* Output method implementations, textual
    /**********************************************************
     */

    @Override
    public void writeString(String text)
            throws IOException, JsonGenerationException
    {
        _verifyValueWrite("write text value");
        if (text == null) {
            _writeNull();
            return;
        }
        if (_outputTail >= _outputEnd) {
            _flushBuffer();
        }

        if (IdentifierUtils.isIdStrict(text)) {
            _writeRaw(text);
            return;
        }

        _outputBuffer[_outputTail++] = '\'';
        _writeString(text);
        // And finally, closing quotes
        if (_outputTail >= _outputEnd) {
            _flushBuffer();
        }
        _outputBuffer[_outputTail++] = '\'';
    }

    @Override
    public void writeString(char[] text, int offset, int len)
            throws IOException, JsonGenerationException
    {
        _verifyValueWrite("write text value");

        if (IdentifierUtils.isIdStrict(text, offset, len)) {
            _writeRaw(text, offset, len);
            return;
        }

        if (_outputTail >= _outputEnd) {
            _flushBuffer();
        }
        _outputBuffer[_outputTail++] = '\'';
        _writeString(text, offset, len);
        // And finally, closing quotes
        if (_outputTail >= _outputEnd) {
            _flushBuffer();
        }
        _outputBuffer[_outputTail++] = '\'';
    }

    @Override
    public void writeString(SerializableString sstr)
            throws IOException, JsonGenerationException
    {
        // Can't take advantage of SerializableString.asQuotedChars() because JSON and Rison
        // have different rules for quoting characters within strings.
        writeString(sstr.getValue());
    }

    @Override
    public void writeRawUTF8String(byte[] text, int offset, int length)
            throws IOException, JsonGenerationException
    {
        // could add support for buffering if we really want it...
        _reportUnsupportedOperation();
    }

    @Override
    public void writeUTF8String(byte[] text, int offset, int length)
            throws IOException, JsonGenerationException
    {
        // could add support for buffering if we really want it...
        _reportUnsupportedOperation();
    }

    /*
   /**********************************************************
   /* Output method implementations, unprocessed ("raw")
   /**********************************************************
    */

    @Override
    public void writeRaw(String text)
            throws IOException, JsonGenerationException
    {
        _reportUnsupportedOperation();
    }

    @Override
    public void writeRaw(String text, int start, int len)
            throws IOException, JsonGenerationException
    {
        _reportUnsupportedOperation();
    }

    @Override
    public void writeRaw(char[] text, int offset, int len)
            throws IOException, JsonGenerationException
    {
        _reportUnsupportedOperation();
    }

    @Override
    public void writeRaw(char c)
            throws IOException, JsonGenerationException
    {
        _reportUnsupportedOperation();
    }

    /*
    /**********************************************************
    /* Output method implementations, base64-encoded binary
    /**********************************************************
     */

    @Override
    public void writeBinary(Base64Variant b64variant, byte[] data, int offset, int len)
            throws IOException, JsonGenerationException
    {
        _verifyValueWrite("write binary value");
        // Starting quotes
        if (_outputTail >= _outputEnd) {
            _flushBuffer();
        }
        _outputBuffer[_outputTail++] = '\'';
        _writeBinary(b64variant, data, offset, offset+len);
        // and closing quotes
        if (_outputTail >= _outputEnd) {
            _flushBuffer();
        }
        _outputBuffer[_outputTail++] = '\'';
    }

    /*
    /**********************************************************
    /* Output method implementations, primitive
    /**********************************************************
     */

    @Override
    public void writeNumber(int i)
            throws IOException, JsonGenerationException
    {
        _verifyValueWrite("write number");
        if (_cfgNumbersAsStrings) {
            _writeQuotedInt(i);
            return;
        }
        // up to 10 digits and possible minus sign
        if ((_outputTail + 11) >= _outputEnd) {
            _flushBuffer();
        }
        _outputTail = NumberOutput.outputInt(i, _outputBuffer, _outputTail);
    }

    private void _writeQuotedInt(int i) throws IOException {
        if ((_outputTail + 13) >= _outputEnd) {
            _flushBuffer();
        }
        _outputBuffer[_outputTail++] = '\'';
        _outputTail = NumberOutput.outputInt(i, _outputBuffer, _outputTail);
        _outputBuffer[_outputTail++] = '\'';
    }

    @Override
    public void writeNumber(long l)
            throws IOException, JsonGenerationException
    {
        _verifyValueWrite("write number");
        if (_cfgNumbersAsStrings) {
            _writeQuotedLong(l);
            return;
        }
        if ((_outputTail + 21) >= _outputEnd) {
            // up to 20 digits, minus sign
            _flushBuffer();
        }
        _outputTail = NumberOutput.outputLong(l, _outputBuffer, _outputTail);
    }

    private void _writeQuotedLong(long l) throws IOException {
        if ((_outputTail + 23) >= _outputEnd) {
            _flushBuffer();
        }
        _outputBuffer[_outputTail++] = '\'';
        _outputTail = NumberOutput.outputLong(l, _outputBuffer, _outputTail);
        _outputBuffer[_outputTail++] = '\'';
    }

    @Override
    public void writeNumber(BigInteger value)
            throws IOException, JsonGenerationException
    {
        _verifyValueWrite("write number");
        if (value == null) {
            _writeNull();
        } else if (_cfgNumbersAsStrings) {
            _writeQuotedRaw(value);
        } else {
            _writeRaw(value.toString());
        }
    }

    @Override
    public void writeNumber(double d)
            throws IOException, JsonGenerationException
    {
        if (_cfgNumbersAsStrings ||
                // [JACKSON-139]
                (((Double.isNaN(d) || Double.isInfinite(d))
                        && isEnabled(JsonGenerator.Feature.QUOTE_NON_NUMERIC_NUMBERS)))) {
            writeString(Double.toString(d));
            return;
        }
        // What is the max length for doubles? 40 chars?
        _verifyValueWrite("write number");
        _writeRaw(formatDouble(d));
    }

    @Override
    public void writeNumber(float f)
            throws IOException, JsonGenerationException
    {
        if (_cfgNumbersAsStrings ||
                // [JACKSON-139]
                (((Float.isNaN(f) || Float.isInfinite(f))
                        && isEnabled(JsonGenerator.Feature.QUOTE_NON_NUMERIC_NUMBERS)))) {
            writeString(Float.toString(f));
            return;
        }
        // What is the max length for floats?
        _verifyValueWrite("write number");
        _writeRaw(formatFloat(f));
    }

    @Override
    public void writeNumber(BigDecimal value)
            throws IOException, JsonGenerationException
    {
        // Don't really know max length for big decimal, no point checking
        _verifyValueWrite("write number");
        if (value == null) {
            _writeNull();
        } else if (_cfgNumbersAsStrings) {
            _writeQuotedRaw(formatDecimal(value));
        } else {
            _writeRaw(formatDecimal(value));
        }
    }

    @Override
    public void writeNumber(String encodedValue)
            throws IOException, JsonGenerationException
    {
        _verifyValueWrite("write number");
        if (_cfgNumbersAsStrings) {
            _writeQuotedRaw(formatDecimal(encodedValue));
        } else {
            _writeRaw(formatDecimal(encodedValue));
        }
    }

    private void _writeQuotedRaw(Object value) throws IOException
    {
        if (_outputTail >= _outputEnd) {
            _flushBuffer();
        }
        _outputBuffer[_outputTail++] = '\'';
        _writeRaw(value.toString());
        if (_outputTail >= _outputEnd) {
            _flushBuffer();
        }
        _outputBuffer[_outputTail++] = '\'';
    }

    @Override
    public void writeBoolean(boolean state)
            throws IOException, JsonGenerationException
    {
        _verifyValueWrite("write boolean value");
        if ((_outputTail + 1) >= _outputEnd) {
            _flushBuffer();
        }
        _outputBuffer[_outputTail++] = '!';
        _outputBuffer[_outputTail++] = state ? 't' : 'f';
    }

    @Override
    public void writeNull()
            throws IOException, JsonGenerationException
    {
        _verifyValueWrite("write null value");
        _writeNull();
    }

    /*
    /**********************************************************
    /* Implementations for other methods
    /**********************************************************
     */

    @Override
    protected final void _verifyValueWrite(String typeMsg)
            throws IOException, JsonGenerationException
    {
        int status = _writeContext.writeValue();
        if (status == JsonWriteContext.STATUS_EXPECT_NAME) {
            _reportError("Can not "+typeMsg+", expecting field name");
        }
        char c;
        switch (status) {
            case JsonWriteContext.STATUS_OK_AFTER_COMMA:
                c = ',';
                break;
            case JsonWriteContext.STATUS_OK_AFTER_COLON:
                c = ':';
                break;
            case JsonWriteContext.STATUS_OK_AFTER_SPACE:
                if (_rootValueSeparator != null) {
                    _writeRaw(_rootValueSeparator.getValue());
                }
                return;
            case JsonWriteContext.STATUS_OK_AS_IS:
            default:
                return;
        }
        if (_outputTail >= _outputEnd) {
            _flushBuffer();
        }
        _outputBuffer[_outputTail++] = c;
    }

    /*
    /**********************************************************
    /* Low-level output handling
    /**********************************************************
     */

    @Override
    public void flush()
            throws IOException
    {
        _flushBuffer();
        if (_writer != null) {
            if (isEnabled(JsonGenerator.Feature.FLUSH_PASSED_TO_STREAM)) {
                _writer.flush();
            }
        }
    }

    @Override
    public void close()
            throws IOException
    {
        super.close();

        /* 05-Dec-2008, tatu: To add [JACKSON-27], need to close open
         *   scopes.
         */
        // First: let's see that we still have buffers...
        if (_outputBuffer != null
                && isEnabled(JsonGenerator.Feature.AUTO_CLOSE_JSON_CONTENT)) {
            while (true) {
                JsonStreamContext ctxt = getOutputContext();
                if (ctxt.inArray()) {
                    writeEndArray();
                } else if (ctxt.inObject()) {
                    writeEndObject();
                } else {
                    break;
                }
            }
        }
        _flushBuffer();

        /* 25-Nov-2008, tatus: As per [JACKSON-16] we are not to call close()
         *   on the underlying Reader, unless we "own" it, or auto-closing
         *   feature is enabled.
         *   One downside: when using UTF8Writer, underlying buffer(s)
         *   may not be properly recycled if we don't close the writer.
         */
        if (_writer != null) {
            if (_ioContext.isResourceManaged() || isEnabled(JsonGenerator.Feature.AUTO_CLOSE_TARGET)) {
                _writer.close();
            } else  if (isEnabled(JsonGenerator.Feature.FLUSH_PASSED_TO_STREAM)) {
                // If we can't close it, we should at least flush
                _writer.flush();
            }
        }
        // Internal buffer(s) generator has can now be released as well
        _releaseBuffers();
    }

    @Override
    protected void _releaseBuffers()
    {
        char[] buf = _outputBuffer;
        if (buf != null) {
            _outputBuffer = null;
            _ioContext.releaseConcatBuffer(buf);
        }
    }

    /*
    /**********************************************************
    /* Internal methods, low-level writing; text, default
    /**********************************************************
     */

    private void _writeRaw(String text)
            throws IOException, JsonGenerationException
    {
        // Nothing to check, can just output as is
        int len = text.length();
        int room = _outputEnd - _outputTail;

        if (room == 0) {
            _flushBuffer();
            room = _outputEnd - _outputTail;
        }
        // But would it nicely fit in? If yes, it's easy
        if (room >= len) {
            text.getChars(0, len, _outputBuffer, _outputTail);
            _outputTail += len;
        } else {
            _writeRawLong(text);
        }
    }

    private void _writeRaw(char[] text, int offset, int len)
            throws IOException, JsonGenerationException
    {
        // Only worth buffering if it's a short write?
        if (len < SHORT_WRITE) {
            int room = _outputEnd - _outputTail;
            if (len > room) {
                _flushBuffer();
            }
            System.arraycopy(text, offset, _outputBuffer, _outputTail, len);
            _outputTail += len;
            return;
        }
        // Otherwise, better just pass through:
        _flushBuffer();
        _writer.write(text, offset, len);
    }

    private void _writeRawLong(String text)
            throws IOException, JsonGenerationException
    {
        int room = _outputEnd - _outputTail;
        // If not, need to do it by looping
        text.getChars(0, room, _outputBuffer, _outputTail);
        _outputTail += room;
        _flushBuffer();
        int offset = room;
        int len = text.length() - room;

        while (len > _outputEnd) {
            int amount = _outputEnd;
            text.getChars(offset, offset+amount, _outputBuffer, 0);
            _outputHead = 0;
            _outputTail = amount;
            _flushBuffer();
            offset += amount;
            len -= amount;
        }
        // And last piece (at most length of buffer)
        text.getChars(offset, offset+len, _outputBuffer, 0);
        _outputHead = 0;
        _outputTail = len;
    }

    private void _writeString(String text)
            throws IOException, JsonGenerationException
    {
        /* One check first: if String won't fit in the buffer, let's
         * segment writes. No point in extending buffer to huge sizes
         * (like if someone wants to include multi-megabyte base64
         * encoded stuff or such)
         */
        final int len = text.length();
        if (len > _outputEnd) { // Let's reserve space for entity at begin/end
            _writeLongString(text);
            return;
        }

        // Ok: we know String will fit in buffer ok
        // But do we need to flush first?
        if ((_outputTail + len) > _outputEnd) {
            _flushBuffer();
        }
        text.getChars(0, len, _outputBuffer, _outputTail);

        _writeString2(len);
    }

    private void _writeString2(final int len)
            throws IOException, JsonGenerationException
    {
        // And then we'll need to verify need for escaping etc:
        int end = _outputTail + len;

        output_loop:
        while (_outputTail < end) {
            // Fast loop for chars not needing escaping
            escape_loop:
            while (true) {
                char c = _outputBuffer[_outputTail];
                if (c == '!' || c == '\'') {
                    break escape_loop;
                }
                if (++_outputTail >= end) {
                    break output_loop;
                }
            }

            // Ok, bumped into something that needs escaping.
            /* First things first: need to flush the buffer.
             * Inlined, as we don't want to lose tail pointer
             */
            int flushLen = (_outputTail - _outputHead);
            if (flushLen > 0) {
                _writer.write(_outputBuffer, _outputHead, flushLen);
            }
            /* In any case, tail will be the new start, so hopefully
             * we have room now.  prepend the '!' to escape the character (Rison)
             */
            _outputHead = _prependOrWrite(_outputBuffer, _outputTail, '!');
            _outputTail++;  // advance past the char-to-be-escaped
        }
    }

    /**
     * Method called to write "long strings", strings whose length exceeds
     * output buffer length.
     */
    private void _writeLongString(String text)
            throws IOException, JsonGenerationException
    {
        // First things first: let's flush the buffer to get some more room
        _flushBuffer();

        // Then we can write
        final int textLen = text.length();
        int offset = 0;
        do {
            int max = _outputEnd;
            int segmentLen = ((offset + max) > textLen)
                    ? (textLen - offset) : max;
            text.getChars(offset, offset+segmentLen, _outputBuffer, 0);
            _writeSegment(segmentLen);
            offset += segmentLen;
        } while (offset < textLen);
    }

    /**
     * Method called to output textual context which has been copied
     * to the output buffer prior to call. If any escaping is needed,
     * it will also be handled by the method.
     *

     * Note: when called, textual content to write is within output
     * buffer, right after buffered content (if any). That's why only
     * length of that text is passed, as buffer and offset are implied.
     */
    private void _writeSegment(int end)
            throws IOException, JsonGenerationException
    {
        int ptr = 0;
        int start = ptr;

        output_loop:
        while (ptr < end) {
            // Fast loop for chars not needing escaping
            char c;
            while (true) {
                c = _outputBuffer[ptr];
                if (c == '!' || c == '\'') {
                    break;
                }
                if (++ptr >= end) {
                    break;
                }
            }

            // Ok, bumped into something that needs escaping.
            /* First things first: need to flush the buffer.
             * Inlined, as we don't want to lose tail pointer
             */
            int flushLen = (ptr - start);
            if (flushLen > 0) {
                _writer.write(_outputBuffer, start, flushLen);
                if (ptr >= end) {
                    break output_loop;
                }
            }
            // So; either try to prepend (most likely), or write directly:
            start = _prependOrWrite(_outputBuffer, ptr, '!');
            ptr++;  // advance past the char-to-be-escaped

            // Flush the last 1 or 2 characters if this is the last time though the loop
            if (ptr == end) {
                _writer.write(_outputBuffer, start, ptr - start);
                break output_loop;
            }
        }
    }

    /**
     * This method called when the string content is already in
     * a char buffer, and need not be copied for processing.
     */
    private void _writeString(char[] text, int offset, int len)
            throws IOException, JsonGenerationException
    {
        /* Let's just find longest spans of non-escapable
        * content, and for each see if it makes sense
        * to copy them, or write through
        */
        len += offset; // -> len marks the end from now on
        while (offset < len) {
            int start = offset;

            while (true) {
                char c = text[offset];
                if (c == '!' || c == '\'') {
                    break;
                }
                if (++offset >= len) {
                    break;
                }
            }

            // Short span? Better just copy it to buffer first:
            int newAmount = offset - start;
            if (newAmount < SHORT_WRITE) {
                // Note: let's reserve room for escaped char (up to 6 chars)
                if ((_outputTail + newAmount) > _outputEnd) {
                    _flushBuffer();
                }
                if (newAmount > 0) {
                    System.arraycopy(text, start, _outputBuffer, _outputTail, newAmount);
                    _outputTail += newAmount;
                }
            } else { // Nope: better just write through
                _flushBuffer();
                _writer.write(text, start, newAmount);
            }
            // Was this the end?
            if (offset >= len) { // yup
                break;
            }
            // Nope, need to escape the char.
            char c = text[offset++];
            _appendCharacterEscape('!', c);
        }
    }

    /*
   /**********************************************************
   /* Internal methods, low-level writing; binary
   /**********************************************************
    */

    protected void _writeBinary(Base64Variant b64variant, byte[] input, int inputPtr, final int inputEnd)
            throws IOException, JsonGenerationException
    {
        // Encoding is by chunks of 3 input, 4 output chars, so:
        int safeInputEnd = inputEnd - 3;
        // Let's also reserve room for possible lf char each round
        int safeOutputEnd = _outputEnd - 6;
        int chunksBeforeLF = b64variant.getMaxLineLength() >> 2;

        // Ok, first we loop through all full triplets of data:
        while (inputPtr <= safeInputEnd) {
            if (_outputTail > safeOutputEnd) { // need to flush
                _flushBuffer();
            }
            // First, mash 3 bytes into lsb of 32-bit int
            int b24 = ((int) input[inputPtr++]) << 8;
            b24 |= ((int) input[inputPtr++]) & 0xFF;
            b24 = (b24 << 8) | (((int) input[inputPtr++]) & 0xFF);
            _outputTail = b64variant.encodeBase64Chunk(b24, _outputBuffer, _outputTail);
            if (--chunksBeforeLF <= 0) {
                // note: RISON does not escape newlines
                _outputBuffer[_outputTail++] = '\n';
                chunksBeforeLF = b64variant.getMaxLineLength() >> 2;
            }
        }

        // And then we may have 1 or 2 leftover bytes to encode
        int inputLeft = inputEnd - inputPtr; // 0, 1 or 2
        if (inputLeft > 0) { // yes, but do we have room for output?
            if (_outputTail > safeOutputEnd) { // don't really need 6 bytes but...
                _flushBuffer();
            }
            int b24 = ((int) input[inputPtr++]) << 16;
            if (inputLeft == 2) {
                b24 |= (((int) input[inputPtr]) & 0xFF) << 8;
            }
            _outputTail = b64variant.encodeBase64Partial(b24, inputLeft, _outputBuffer, _outputTail);
        }
    }

    /*
    /**********************************************************
    /* Internal methods, low-level writing, other
    /**********************************************************
     */

    private void _writeNull() throws IOException
    {
        if ((_outputTail + 1) >= _outputEnd) {
            _flushBuffer();
        }
        _outputBuffer[_outputTail++] = '!';
        _outputBuffer[_outputTail++] = 'n';
    }

    /*
   /**********************************************************
   /* Internal methods, low-level writing, escapes
   /**********************************************************
    */

    /**
     * Method called to try to either prepend character escape at front of
     * given buffer; or if not possible, to write it out directly.
     *
     * @return Pointer to start of prepended entity (if prepended); or 'ptr'
     *   if not.
     */
    private int _prependOrWrite(char[] buffer, int ptr, char esc)
            throws IOException, JsonGenerationException
    {
        if (ptr > 0) { // fits, just prepend
            buffer[--ptr] = esc;
        } else { // won't fit, write
            _writer.write(esc);
        }
        return ptr;
    }

    /**
     * Method called to append escape sequence for given character, at the
     * end of standard output buffer; or if not possible, write out directly.
     */
    private void _appendCharacterEscape(char esc, char ch)
            throws IOException, JsonGenerationException
    {
        if ((_outputTail + 1) >= _outputEnd) {
            _flushBuffer();
        }
        _outputBuffer[_outputTail++] = esc;
        _outputBuffer[_outputTail++] = ch;
    }

    protected final void _flushBuffer() throws IOException
    {
        int len = _outputTail - _outputHead;
        if (len > 0) {
            int offset = _outputHead;
            _outputTail = _outputHead = 0;
            _writer.write(_outputBuffer, offset, len);
        }
    }

    private String formatDouble(double d) {
        return Double.toString(d).toLowerCase();
    }

    private String formatFloat(float f) {
        return Float.toString(f).toLowerCase();
    }

    private String formatDecimal(Object number) {
        return number.toString().toLowerCase();
    }
}