<commit_before>#include "ffmpeg_video_source.h"
extern "C" {
#include <libavutil/imgutils.h>
}

namespace gg
{

VideoSourceFFmpeg::VideoSourceFFmpeg()
{
    // nop
}

VideoSourceFFmpeg::VideoSourceFFmpeg(std::string source_path,
                                     enum ColourSpace colour_space,
                                     bool use_refcount)
    : IVideoSource(colour_space)
    , _source_path(source_path)
    , _avformat_context(nullptr)
    , _avstream_idx(-1)
    , _use_refcount(use_refcount)
    , _avstream(nullptr)
    , _avcodec_context(nullptr)
    , _avframe(nullptr)
    , _avframe_converted(nullptr)
    , _sws_context(nullptr)
    , _daemon(nullptr)
{
    int ret = 0;
    std::string error_msg = "";

    av_register_all();

    if (avformat_open_input(&_avformat_context, _source_path.c_str(),
                            nullptr, nullptr) < 0)
    {
        error_msg.append("Could not open video source ")
                 .append(_source_path);
        throw VideoSourceError(error_msg);
    }

    if (avformat_find_stream_info(_avformat_context, nullptr) < 0)
        throw VideoSourceError("Could not find stream information");

    if (open_codec_context(&_avstream_idx, _avformat_context,
                           AVMEDIA_TYPE_VIDEO, error_msg) >= 0)
    {
        _avstream = _avformat_context->streams[_avstream_idx];
        _avcodec_context = _avstream->codec;

        /* allocate image where the decoded image will be put */
        _width = _avcodec_context->width;
        _height = _avcodec_context->height;
        _avpixel_format = _avcodec_context->pix_fmt;
        ret = av_image_alloc(_data_buffer, _data_buffer_linesizes,
                             _width, _height, _avpixel_format, 1);
        if (ret < 0)
            throw VideoSourceError("Could not allocate"
                                   " raw video buffer");
        _data_buffer_length = ret;
    }
    else
        throw VideoSourceError(error_msg);

    if (_avstream == nullptr)
        throw VideoSourceError("Could not find video stream in source");

    _avframe = av_frame_alloc();
    if (_avframe == nullptr)
        throw VideoSourceError("Could not allocate frame");

    /* initialize packet, set data to NULL, let the demuxer fill it */
    av_init_packet(&_avpacket);
    _avpacket.data = nullptr;
    _avpacket.size = 0;

    AVPixelFormat target_avpixel_format;
    switch(_colour)
    {
    case BGRA:
        target_avpixel_format = AV_PIX_FMT_BGRA;
        break;
    case I420:
        target_avpixel_format = AV_PIX_FMT_YUV420P;
        break;
    case UYVY:
        target_avpixel_format = AV_PIX_FMT_UYVY422;
        break;
    default:
        throw VideoSourceError("Target colour space not supported");
    }
    if (_avpixel_format != target_avpixel_format)
    {
        switch(_avpixel_format)
        {
        case AV_PIX_FMT_BGRA:
            _stride[0] = 4 * _width;
            break;
        case AV_PIX_FMT_UYVY422:
            _stride[0] = 2 * _width;
            break;
        case AV_PIX_FMT_YUV420P:
            // TODO: 1) is this correct? 2) odd width?
            _stride[0] = 1.5 * _width;
            break;
        default:
            throw VideoSourceError("Source colour space not supported");
        }
        _avframe_converted = av_frame_alloc();
        if (_avframe_converted == nullptr)
            throw VideoSourceError("Could not allocate conversion frame");
        _avframe_converted->format = target_avpixel_format;
        _avframe_converted->width  = _width;
        _avframe_converted->height = _height;
        int pixel_depth;
        switch(target_avpixel_format)
        {
        case AV_PIX_FMT_BGRA:
            pixel_depth = 32; // bits-per-pixel
            break;
        case AV_PIX_FMT_YUV420P:
            pixel_depth = 12; // bits-per-pixel
            /* TODO #54 - alignment by _pixel_depth causes problems
             * due to buffer size mismatches between EpiphanSDK and
             * FFmpeg, hence manually filling in linesizes (based on
             * debugging measurements), in conj. with
             * av_image_fill_pointers below
             */
            _avframe_converted->linesize[0] = _avframe_converted->width;
            _avframe_converted->linesize[1] = _avframe_converted->width / 2;
            _avframe_converted->linesize[2] = _avframe_converted->width / 2;
            break;
        case AV_PIX_FMT_UYVY422:
            pixel_depth = 16; // bits-per-pixel
            break;
        default:
            throw VideoSourceError("Colour space not supported");
        }
        ret = av_frame_get_buffer(_avframe_converted, pixel_depth);
        if (ret < 0)
            throw VideoSourceError("Could not allocate conversion buffer");

        _sws_context = sws_getContext(
                    _width, _height, _avpixel_format,
                    _width, _height, target_avpixel_format,
                    0, nullptr, nullptr, nullptr);
        if (_sws_context == nullptr)
            throw VideoSourceError("Could not allocate Sws context");
    }

    _daemon = new gg::BroadcastDaemon(this);
    _daemon->start(get_frame_rate());
}

VideoSourceFFmpeg::~VideoSourceFFmpeg()
{
    delete _daemon;

    if (_sws_context != nullptr)
        sws_freeContext(_sws_context);
    avcodec_close(_avcodec_context);
    avformat_close_input(&_avformat_context);
    av_frame_free(&_avframe);
    if (_avframe_converted != nullptr)
        av_frame_free(&_avframe_converted);
    av_free(_data_buffer[0]);
    _data_buffer_length = 0;
}

bool VideoSourceFFmpeg::get_frame_dimensions(int & width, int & height)
{
    if (this->_width > 0 and this->_height > 0)
    {
        width = this->_width;
        height = this->_height;
        return true;
    }
    else
        return false;
}

bool VideoSourceFFmpeg::get_frame(VideoFrame & frame)
{
    if (av_read_frame(_avformat_context, &_avpacket) < 0)
        return false;

    int ret = 0, got_frame;
    bool success = true;
    AVPacket orig_pkt = _avpacket;
    size_t passes = 0;
    do
    {
        ret = decode_packet(&got_frame, 0);
        if (ret < 0)
        {
            success = false;
            break;
        }
        _avpacket.data += ret;
        _avpacket.size -= ret;

        // need to convert pixel format?
        uint8_t ** data_ptr = nullptr;
        int * data_linesize = nullptr;
        if (_sws_context != nullptr)
        {
            ret = sws_scale(_sws_context,
                            _avframe->data, _stride,
                            0, _height,
                            _avframe_converted->data, _avframe_converted->linesize
                            );
            if (ret <= 0)
            {
                success = false;
                break;
            }
            data_ptr = _avframe_converted->data;
            data_linesize = _avframe_converted->linesize;
        }
        else
        {
            data_ptr = _avframe->data;
            data_linesize = _avframe->linesize;
        }

        /* copy decoded frame to destination buffer:
         * this is required since rawvideo expects non aligned data */
        av_image_copy(_data_buffer, _data_buffer_linesizes,
                      const_cast<const uint8_t **>(data_ptr), data_linesize,
                      _avpixel_format, _width, _height);

        passes++;
    }
    while (_avpacket.size > 0);
    av_packet_unref(&orig_pkt);

    if (not success)
        return false;

    // TODO - when are there multiple passes?
    if (passes != 1)
        return false;

    frame.init_from_specs(_data_buffer[0], _data_buffer_length, _width, _height);
    return true;
}

double VideoSourceFFmpeg::get_frame_rate()
{
    int num = _avstream->avg_frame_rate.num;
    int den = _avstream->avg_frame_rate.den;
    if (num == 0)
        return 0.0;
    return static_cast<double>(num) / den;
}

void VideoSourceFFmpeg::set_sub_frame(int x, int y, int width, int height)
{
    // TODO
}

void VideoSourceFFmpeg::get_full_frame()
{
    // TODO
}

int VideoSourceFFmpeg::open_codec_context(
        int * stream_idx, AVFormatContext * fmt_ctx,
        enum AVMediaType type, std::string & error_msg)
{
    int ret, stream_index;
    AVStream * st;
    AVCodecContext * dec_ctx = nullptr;
    AVCodec * dec = nullptr;
    AVDictionary * opts = nullptr;

    ret = av_find_best_stream(fmt_ctx, type, -1, -1, nullptr, 0);
    if (ret < 0)
    {
        error_msg.append("Could not find ")
                 .append(av_get_media_type_string(type))
                 .append(" stream in source ")
                 .append(_source_path);
        return ret;
    }
    else
    {
        stream_index = ret;
        st = fmt_ctx->streams[stream_index];

        /* find decoder for the stream */
        dec_ctx = st->codec;
        dec = avcodec_find_decoder(dec_ctx->codec_id);
        if (!dec)
        {
            error_msg.append("Failed to find ")
                     .append(av_get_media_type_string(type))
                     .append(" codec");
            return AVERROR(EINVAL);
        }

        /* Init the decoders, with or without reference counting */
        av_dict_set(&opts, "refcounted_frames", _use_refcount ? "1" : "0", 0);
        if ((ret = avcodec_open2(dec_ctx, dec, &opts)) < 0)
        {
            error_msg.append("Failed to open ")
                     .append(av_get_media_type_string(type))
                     .append(" codec");
            return ret;
        }
        *stream_idx = stream_index;
    }

    return 0;
}

int VideoSourceFFmpeg::decode_packet(int * got_frame, int cached)
{
    int ret = 0;
    int decoded = _avpacket.size;
    *got_frame = 0;
    if (_avpacket.stream_index == _avstream_idx)
    {
        /* decode video frame */
        ret = avcodec_decode_video2(_avcodec_context, _avframe, got_frame, &_avpacket);
        if (ret < 0)
            return ret;

        if (*got_frame)
        {
            if (_avframe->width != _width or _avframe->height != _height or
                _avframe->format != _avpixel_format)
                return -1;
        }
    }
    /* If we use frame reference counting, we own the data and need
     * to de-reference it when we don't use it anymore */
    if (*got_frame and _use_refcount)
        av_frame_unref(_avframe);
    return decoded;
}

}
<commit_msg>Issue #74: replaced data_ptr and data_linesize with avframe_ptr, which gets set conditionally<commit_after>#include "ffmpeg_video_source.h"
extern "C" {
#include <libavutil/imgutils.h>
}

namespace gg
{

VideoSourceFFmpeg::VideoSourceFFmpeg()
{
    // nop
}

VideoSourceFFmpeg::VideoSourceFFmpeg(std::string source_path,
                                     enum ColourSpace colour_space,
                                     bool use_refcount)
    : IVideoSource(colour_space)
    , _source_path(source_path)
    , _avformat_context(nullptr)
    , _avstream_idx(-1)
    , _use_refcount(use_refcount)
    , _avstream(nullptr)
    , _avcodec_context(nullptr)
    , _avframe(nullptr)
    , _avframe_converted(nullptr)
    , _sws_context(nullptr)
    , _daemon(nullptr)
{
    int ret = 0;
    std::string error_msg = "";

    av_register_all();

    if (avformat_open_input(&_avformat_context, _source_path.c_str(),
                            nullptr, nullptr) < 0)
    {
        error_msg.append("Could not open video source ")
                 .append(_source_path);
        throw VideoSourceError(error_msg);
    }

    if (avformat_find_stream_info(_avformat_context, nullptr) < 0)
        throw VideoSourceError("Could not find stream information");

    if (open_codec_context(&_avstream_idx, _avformat_context,
                           AVMEDIA_TYPE_VIDEO, error_msg) >= 0)
    {
        _avstream = _avformat_context->streams[_avstream_idx];
        _avcodec_context = _avstream->codec;

        /* allocate image where the decoded image will be put */
        _width = _avcodec_context->width;
        _height = _avcodec_context->height;
        _avpixel_format = _avcodec_context->pix_fmt;
        ret = av_image_alloc(_data_buffer, _data_buffer_linesizes,
                             _width, _height, _avpixel_format, 1);
        if (ret < 0)
            throw VideoSourceError("Could not allocate"
                                   " raw video buffer");
        _data_buffer_length = ret;
    }
    else
        throw VideoSourceError(error_msg);

    if (_avstream == nullptr)
        throw VideoSourceError("Could not find video stream in source");

    _avframe = av_frame_alloc();
    if (_avframe == nullptr)
        throw VideoSourceError("Could not allocate frame");

    /* initialize packet, set data to NULL, let the demuxer fill it */
    av_init_packet(&_avpacket);
    _avpacket.data = nullptr;
    _avpacket.size = 0;

    AVPixelFormat target_avpixel_format;
    switch(_colour)
    {
    case BGRA:
        target_avpixel_format = AV_PIX_FMT_BGRA;
        break;
    case I420:
        target_avpixel_format = AV_PIX_FMT_YUV420P;
        break;
    case UYVY:
        target_avpixel_format = AV_PIX_FMT_UYVY422;
        break;
    default:
        throw VideoSourceError("Target colour space not supported");
    }
    if (_avpixel_format != target_avpixel_format)
    {
        switch(_avpixel_format)
        {
        case AV_PIX_FMT_BGRA:
            _stride[0] = 4 * _width;
            break;
        case AV_PIX_FMT_UYVY422:
            _stride[0] = 2 * _width;
            break;
        case AV_PIX_FMT_YUV420P:
            // TODO: 1) is this correct? 2) odd width?
            _stride[0] = 1.5 * _width;
            break;
        default:
            throw VideoSourceError("Source colour space not supported");
        }
        _avframe_converted = av_frame_alloc();
        if (_avframe_converted == nullptr)
            throw VideoSourceError("Could not allocate conversion frame");
        _avframe_converted->format = target_avpixel_format;
        _avframe_converted->width  = _width;
        _avframe_converted->height = _height;
        int pixel_depth;
        switch(target_avpixel_format)
        {
        case AV_PIX_FMT_BGRA:
            pixel_depth = 32; // bits-per-pixel
            break;
        case AV_PIX_FMT_YUV420P:
            pixel_depth = 12; // bits-per-pixel
            /* TODO #54 - alignment by _pixel_depth causes problems
             * due to buffer size mismatches between EpiphanSDK and
             * FFmpeg, hence manually filling in linesizes (based on
             * debugging measurements), in conj. with
             * av_image_fill_pointers below
             */
            _avframe_converted->linesize[0] = _avframe_converted->width;
            _avframe_converted->linesize[1] = _avframe_converted->width / 2;
            _avframe_converted->linesize[2] = _avframe_converted->width / 2;
            break;
        case AV_PIX_FMT_UYVY422:
            pixel_depth = 16; // bits-per-pixel
            break;
        default:
            throw VideoSourceError("Colour space not supported");
        }
        ret = av_frame_get_buffer(_avframe_converted, pixel_depth);
        if (ret < 0)
            throw VideoSourceError("Could not allocate conversion buffer");

        _sws_context = sws_getContext(
                    _width, _height, _avpixel_format,
                    _width, _height, target_avpixel_format,
                    0, nullptr, nullptr, nullptr);
        if (_sws_context == nullptr)
            throw VideoSourceError("Could not allocate Sws context");
    }

    _daemon = new gg::BroadcastDaemon(this);
    _daemon->start(get_frame_rate());
}

VideoSourceFFmpeg::~VideoSourceFFmpeg()
{
    delete _daemon;

    if (_sws_context != nullptr)
        sws_freeContext(_sws_context);
    avcodec_close(_avcodec_context);
    avformat_close_input(&_avformat_context);
    av_frame_free(&_avframe);
    if (_avframe_converted != nullptr)
        av_frame_free(&_avframe_converted);
    av_free(_data_buffer[0]);
    _data_buffer_length = 0;
}

bool VideoSourceFFmpeg::get_frame_dimensions(int & width, int & height)
{
    if (this->_width > 0 and this->_height > 0)
    {
        width = this->_width;
        height = this->_height;
        return true;
    }
    else
        return false;
}

bool VideoSourceFFmpeg::get_frame(VideoFrame & frame)
{
    if (av_read_frame(_avformat_context, &_avpacket) < 0)
        return false;

    int ret = 0, got_frame;
    bool success = true;
    AVPacket orig_pkt = _avpacket;
    size_t passes = 0;
    do
    {
        ret = decode_packet(&got_frame, 0);
        if (ret < 0)
        {
            success = false;
            break;
        }
        _avpacket.data += ret;
        _avpacket.size -= ret;

        // need to convert pixel format?
        AVFrame * avframe_ptr = nullptr;
        if (_sws_context != nullptr)
        {
            ret = sws_scale(_sws_context,
                            _avframe->data, _stride,
                            0, _height,
                            _avframe_converted->data, _avframe_converted->linesize
                            );
            if (ret <= 0)
            {
                success = false;
                break;
            }

            avframe_ptr = _avframe_converted;
        }
        else
        {
            avframe_ptr = _avframe;
        }

        /* copy decoded frame to destination buffer:
         * this is required since rawvideo expects non aligned data */
        av_image_copy(_data_buffer, _data_buffer_linesizes,
                      const_cast<const uint8_t **>(avframe_ptr->data), avframe_ptr->linesize,
                      _avpixel_format, _width, _height);

        passes++;
    }
    while (_avpacket.size > 0);
    av_packet_unref(&orig_pkt);

    if (not success)
        return false;

    // TODO - when are there multiple passes?
    if (passes != 1)
        return false;

    frame.init_from_specs(_data_buffer[0], _data_buffer_length, _width, _height);
    return true;
}

double VideoSourceFFmpeg::get_frame_rate()
{
    int num = _avstream->avg_frame_rate.num;
    int den = _avstream->avg_frame_rate.den;
    if (num == 0)
        return 0.0;
    return static_cast<double>(num) / den;
}

void VideoSourceFFmpeg::set_sub_frame(int x, int y, int width, int height)
{
    // TODO
}

void VideoSourceFFmpeg::get_full_frame()
{
    // TODO
}

int VideoSourceFFmpeg::open_codec_context(
        int * stream_idx, AVFormatContext * fmt_ctx,
        enum AVMediaType type, std::string & error_msg)
{
    int ret, stream_index;
    AVStream * st;
    AVCodecContext * dec_ctx = nullptr;
    AVCodec * dec = nullptr;
    AVDictionary * opts = nullptr;

    ret = av_find_best_stream(fmt_ctx, type, -1, -1, nullptr, 0);
    if (ret < 0)
    {
        error_msg.append("Could not find ")
                 .append(av_get_media_type_string(type))
                 .append(" stream in source ")
                 .append(_source_path);
        return ret;
    }
    else
    {
        stream_index = ret;
        st = fmt_ctx->streams[stream_index];

        /* find decoder for the stream */
        dec_ctx = st->codec;
        dec = avcodec_find_decoder(dec_ctx->codec_id);
        if (!dec)
        {
            error_msg.append("Failed to find ")
                     .append(av_get_media_type_string(type))
                     .append(" codec");
            return AVERROR(EINVAL);
        }

        /* Init the decoders, with or without reference counting */
        av_dict_set(&opts, "refcounted_frames", _use_refcount ? "1" : "0", 0);
        if ((ret = avcodec_open2(dec_ctx, dec, &opts)) < 0)
        {
            error_msg.append("Failed to open ")
                     .append(av_get_media_type_string(type))
                     .append(" codec");
            return ret;
        }
        *stream_idx = stream_index;
    }

    return 0;
}

int VideoSourceFFmpeg::decode_packet(int * got_frame, int cached)
{
    int ret = 0;
    int decoded = _avpacket.size;
    *got_frame = 0;
    if (_avpacket.stream_index == _avstream_idx)
    {
        /* decode video frame */
        ret = avcodec_decode_video2(_avcodec_context, _avframe, got_frame, &_avpacket);
        if (ret < 0)
            return ret;

        if (*got_frame)
        {
            if (_avframe->width != _width or _avframe->height != _height or
                _avframe->format != _avpixel_format)
                return -1;
        }
    }
    /* If we use frame reference counting, we own the data and need
     * to de-reference it when we don't use it anymore */
    if (*got_frame and _use_refcount)
        av_frame_unref(_avframe);
    return decoded;
}

}
<|endoftext|>


<commit_before>#include "ffmpeg_video_target.h"
#include "except.h"

namespace gg
{

VideoTargetFFmpeg::VideoTargetFFmpeg(const std::string codec) :
    _codec(NULL),
    _codec_id(AV_CODEC_ID_NONE),
    _frame(NULL),
    _framerate(-1),
    _sws_context(NULL),
    _format_context(NULL),
    _stream(NULL),
    _frame_index(0)
{
    if (codec != "H265")
    {
        std::string msg;
        msg.append("Codec ")
           .append(codec)
           .append(" not recognised");
        throw VideoTargetError(msg);
    }
    _codec_id = AV_CODEC_ID_HEVC;

    av_register_all();
}

void VideoTargetFFmpeg::init(const std::string filepath, const float framerate)
{
    if (framerate <= 0)
        throw VideoTargetError("Negative fps does not make sense");
    if (framerate - (int) framerate > 0)
        throw VideoTargetError("Only integer framerates are supported");
    _framerate = (int) framerate;

    if (filepath.empty())
        throw VideoTargetError("Empty filepath specified");
    _filepath = filepath;

    /* allocate the output media context */
    avformat_alloc_output_context2(&_format_context, NULL, NULL, _filepath.c_str());
    if (_format_context == NULL)
        // Use MP4 as default if context cannot be deduced from file extension
        avformat_alloc_output_context2(&_format_context, NULL, "mp4", NULL);
    if (_format_context == NULL)
        throw VideoTargetError("Could not allocate output media context");

    _codec = avcodec_find_encoder(_codec_id);
    if (not _codec)
        throw VideoTargetError("Codec not found");

    _stream = avformat_new_stream(_format_context, _codec);
    if (_stream == NULL)
        throw VideoTargetError("Could not allocate stream");
    _stream->id = _format_context->nb_streams-1; // TODO isn't this wrong?
}

void VideoTargetFFmpeg::append(const VideoFrame_BGRA & frame)
{
    if (_framerate <= 0)
        throw VideoTargetError("Video target not initialised");

    // return value buffers
    int ret, got_output;

    // if first frame, initialise
    if (_frame == NULL)
    {
        // TODO - is _codec_context ever being modified after first frame?
        _stream->codec->codec_id = _codec_id;
        /* TODO: using default reduces filesize
         * but should we set it nonetheless?
         */
//        _stream->codec->bit_rate = 400000;
        _stream->codec->width = frame.cols();
        _stream->codec->height = frame.rows();
        _stream->time_base = (AVRational){ 1, _framerate };
        _stream->codec->time_base = _stream->time_base;
        _stream->codec->gop_size = 12;
        /* TODO emit one intra frame every twelve frames at most */
        _stream->codec->pix_fmt = AV_PIX_FMT_YUV420P;
        /* Some formats want stream headers to be separate. */
        if (_format_context->oformat->flags & AVFMT_GLOBALHEADER)
            _stream->codec->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;

        switch (_codec_id)
        {
        case AV_CODEC_ID_H264:
        case AV_CODEC_ID_HEVC:
            /* TODO will this work in real-time with a framegrabber ?
             * "slow" produces 2x larger file compared to "ultrafast",
             * but with a substantial visual quality degradation
             * (judged using the coloured chessboard pattern)
             * "fast" is a trade-off: visual quality looks similar
             * while file size is reasonable
             */
            av_opt_set(_stream->codec->priv_data, "preset", "fast", 0);
            /* Resolution must be a multiple of two, as required
             * by H264 and H265. Introduce a one-pixel padding for
             * non-complying dimension(s).
             */
            _stream->codec->width +=
                    _stream->codec->width % 2 == 0 ? 0 : 1;
            _stream->codec->height +=
                    _stream->codec->height % 2 == 0 ? 0 : 1;
            break;
        default:
            // nop
            break;
        }

        /* open it */
        if (avcodec_open2(_stream->codec, _codec, NULL) < 0)
            throw VideoTargetError("Could not open codec");

        _frame = av_frame_alloc();
        if (not _frame)
            throw VideoTargetError("Could not allocate video frame");
        _frame->format = _stream->codec->pix_fmt;
        _frame->width  = _stream->codec->width;
        _frame->height = _stream->codec->height;
        /* allocate the buffers for the frame data */
        ret = av_frame_get_buffer(_frame, 32);
        if (ret < 0)
            throw VideoTargetError("Could not allocate frame data");

        /* Now that all the parameters are set, we can open the audio and
         * video codecs and allocate the necessary encode buffers. */
        av_dump_format(_format_context, 0, _filepath.c_str(), 1);
        /* open the output file, if needed */
        if (!(_format_context->oformat->flags & AVFMT_NOFILE))
        {
            ret = avio_open(&_format_context->pb, _filepath.c_str(), AVIO_FLAG_WRITE);
            if (ret < 0)
            {
                std::string msg;
                msg.append("File ")
                   .append(_filepath)
                   .append(" could not be opened");
                throw VideoTargetError(msg);
            }
        }

        /* Write the stream header, if any. */
        ret = avformat_write_header(_format_context, NULL);
        if (ret < 0)
            throw VideoTargetError("Could not write header to file");

        /* Open context for converting BGRA pixels to YUV420p */
        _sws_context = sws_getContext(
                    frame.cols(), frame.rows(), AV_PIX_FMT_BGRA,
                    frame.cols(), frame.rows(), _stream->codec->pix_fmt,
                    0, NULL, NULL, NULL);
        if (_sws_context == NULL)
            throw VideoTargetError("Could not allocate Sws context");

        // To be incremented for each frame, used as pts
        _frame_index = 0;

        // Packet to be used when writing frames
        av_init_packet(&_packet);
        _packet.data = NULL;    // packet data will be allocated by the encoder
        _packet.size = 0;
    }

    /* when we pass a frame to the encoder, it may keep a reference to it
     * internally;
     * make sure we do not overwrite it here
     */
    ret = av_frame_make_writable(_frame);
    if (ret < 0)
        throw VideoTargetError("Could not make frame writeable");

    /* convert pixel format */
    const uint8_t * src_data_ptr[1] = { frame.data() }; // BGRA has one plane
    int bgra_stride[1] = { 4*frame.cols() };
    sws_scale(_sws_context,
              src_data_ptr, bgra_stride,
              0, frame.rows(),
              _frame->data, _frame->linesize
              );

    _frame->pts = _frame_index++;

    /* encode the image */
    encode_and_write(_frame, got_output);
}

void VideoTargetFFmpeg::encode_and_write(AVFrame * frame, int & got_output)
{
    int ret;

    ret = avcodec_encode_video2(_stream->codec, &_packet, frame, &got_output);
    if (ret < 0)
        throw VideoTargetError("Error encoding frame");

    if (got_output)
    {
        /* rescale output packet timestamp values from codec to stream timebase */
        av_packet_rescale_ts(&_packet, _stream->codec->time_base, _stream->time_base);
        // TODO - above time bases are the same, or not?
        _packet.stream_index = _stream->index;

        /* Write the compressed frame to the media file. */
        int ret = av_interleaved_write_frame(_format_context, &_packet);
        if (ret < 0)
            throw VideoTargetError("Could not interleaved write frame");
//        av_packet_unref(&packet); taken care of by av_interleaved_write_frame
    }
}

void VideoTargetFFmpeg::finalise()
{
    /* get the delayed frames */
    for (int got_output = 1; got_output; )
        encode_and_write(NULL, got_output);

    /* Write the trailer, if any. The trailer must be written before you
     * close the CodecContexts open when you wrote the header; otherwise
     * av_write_trailer() may try to use memory that was freed on
     * av_codec_close(). */
    av_write_trailer(_format_context);

    if (_stream->codec) avcodec_close(_stream->codec);
    if (_frame) av_frame_free(&_frame);
//  av_freep(&_frame->data[0]); no need for this because _frame never manages its own data
    if (_sws_context) sws_freeContext(_sws_context);

    if (!(_format_context->oformat->flags & AVFMT_NOFILE))
        /* Close the output file. */
        avio_closep(&_format_context->pb);
    /* free the stream */
    avformat_free_context(_format_context);

    // default values, for next init
    _codec = NULL;
    _codec_id = AV_CODEC_ID_NONE;
    _frame = NULL;
    _framerate = -1;
    _sws_context = NULL;
    _format_context = NULL;
    _stream = NULL;
    _frame_index = 0;
}

}
<commit_msg>Issue #21: FFmpeg target checking for *.mp4 file extension as this is what is passed on to FFmpeg<commit_after>#include "ffmpeg_video_target.h"
#include "except.h"

namespace gg
{

VideoTargetFFmpeg::VideoTargetFFmpeg(const std::string codec) :
    _codec(NULL),
    _codec_id(AV_CODEC_ID_NONE),
    _frame(NULL),
    _framerate(-1),
    _sws_context(NULL),
    _format_context(NULL),
    _stream(NULL),
    _frame_index(0)
{
    if (codec != "H265")
    {
        std::string msg;
        msg.append("Codec ")
           .append(codec)
           .append(" not recognised");
        throw VideoTargetError(msg);
    }
    _codec_id = AV_CODEC_ID_HEVC;

    av_register_all();
}

void VideoTargetFFmpeg::init(const std::string filepath, const float framerate)
{
    if (framerate <= 0)
        throw VideoTargetError("Negative fps does not make sense");
    if (framerate - (int) framerate > 0)
        throw VideoTargetError("Only integer framerates are supported");
    _framerate = (int) framerate;

    std::string filetype = "mp4";
    if (filepath.empty())
        throw VideoTargetError("Empty filepath specified");
    else if (filepath.length() <= filetype.length()+1) // +1 for the dot, i.e. .mp4
        throw VideoTargetError("Filepath not long enough to deduce output format");
    else if (0 != filepath.compare(filepath.length() - filetype.length(), filetype.length(), filetype))
    {
        std::string msg;
        msg.append("Filetype of ")
           .append(filepath)
           .append(" not supported, use *.")
           .append(filetype)
           .append(" instead");
        throw VideoTargetError(msg);
    }

    _filepath = filepath;

    /* allocate the output media context */
    avformat_alloc_output_context2(&_format_context, NULL, NULL, _filepath.c_str());
    if (_format_context == NULL)
        // Use MP4 as default if context cannot be deduced from file extension
        avformat_alloc_output_context2(&_format_context, NULL, "mp4", NULL);
    if (_format_context == NULL)
        throw VideoTargetError("Could not allocate output media context");

    _codec = avcodec_find_encoder(_codec_id);
    if (not _codec)
        throw VideoTargetError("Codec not found");

    _stream = avformat_new_stream(_format_context, _codec);
    if (_stream == NULL)
        throw VideoTargetError("Could not allocate stream");
    _stream->id = _format_context->nb_streams-1; // TODO isn't this wrong?
}

void VideoTargetFFmpeg::append(const VideoFrame_BGRA & frame)
{
    if (_framerate <= 0)
        throw VideoTargetError("Video target not initialised");

    // return value buffers
    int ret, got_output;

    // if first frame, initialise
    if (_frame == NULL)
    {
        // TODO - is _codec_context ever being modified after first frame?
        _stream->codec->codec_id = _codec_id;
        /* TODO: using default reduces filesize
         * but should we set it nonetheless?
         */
//        _stream->codec->bit_rate = 400000;
        _stream->codec->width = frame.cols();
        _stream->codec->height = frame.rows();
        _stream->time_base = (AVRational){ 1, _framerate };
        _stream->codec->time_base = _stream->time_base;
        _stream->codec->gop_size = 12;
        /* TODO emit one intra frame every twelve frames at most */
        _stream->codec->pix_fmt = AV_PIX_FMT_YUV420P;
        /* Some formats want stream headers to be separate. */
        if (_format_context->oformat->flags & AVFMT_GLOBALHEADER)
            _stream->codec->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;

        switch (_codec_id)
        {
        case AV_CODEC_ID_H264:
        case AV_CODEC_ID_HEVC:
            /* TODO will this work in real-time with a framegrabber ?
             * "slow" produces 2x larger file compared to "ultrafast",
             * but with a substantial visual quality degradation
             * (judged using the coloured chessboard pattern)
             * "fast" is a trade-off: visual quality looks similar
             * while file size is reasonable
             */
            av_opt_set(_stream->codec->priv_data, "preset", "fast", 0);
            /* Resolution must be a multiple of two, as required
             * by H264 and H265. Introduce a one-pixel padding for
             * non-complying dimension(s).
             */
            _stream->codec->width +=
                    _stream->codec->width % 2 == 0 ? 0 : 1;
            _stream->codec->height +=
                    _stream->codec->height % 2 == 0 ? 0 : 1;
            break;
        default:
            // nop
            break;
        }

        /* open it */
        if (avcodec_open2(_stream->codec, _codec, NULL) < 0)
            throw VideoTargetError("Could not open codec");

        _frame = av_frame_alloc();
        if (not _frame)
            throw VideoTargetError("Could not allocate video frame");
        _frame->format = _stream->codec->pix_fmt;
        _frame->width  = _stream->codec->width;
        _frame->height = _stream->codec->height;
        /* allocate the buffers for the frame data */
        ret = av_frame_get_buffer(_frame, 32);
        if (ret < 0)
            throw VideoTargetError("Could not allocate frame data");

        /* Now that all the parameters are set, we can open the audio and
         * video codecs and allocate the necessary encode buffers. */
        av_dump_format(_format_context, 0, _filepath.c_str(), 1);
        /* open the output file, if needed */
        if (!(_format_context->oformat->flags & AVFMT_NOFILE))
        {
            ret = avio_open(&_format_context->pb, _filepath.c_str(), AVIO_FLAG_WRITE);
            if (ret < 0)
            {
                std::string msg;
                msg.append("File ")
                   .append(_filepath)
                   .append(" could not be opened");
                throw VideoTargetError(msg);
            }
        }

        /* Write the stream header, if any. */
        ret = avformat_write_header(_format_context, NULL);
        if (ret < 0)
            throw VideoTargetError("Could not write header to file");

        /* Open context for converting BGRA pixels to YUV420p */
        _sws_context = sws_getContext(
                    frame.cols(), frame.rows(), AV_PIX_FMT_BGRA,
                    frame.cols(), frame.rows(), _stream->codec->pix_fmt,
                    0, NULL, NULL, NULL);
        if (_sws_context == NULL)
            throw VideoTargetError("Could not allocate Sws context");

        // To be incremented for each frame, used as pts
        _frame_index = 0;

        // Packet to be used when writing frames
        av_init_packet(&_packet);
        _packet.data = NULL;    // packet data will be allocated by the encoder
        _packet.size = 0;
    }

    /* when we pass a frame to the encoder, it may keep a reference to it
     * internally;
     * make sure we do not overwrite it here
     */
    ret = av_frame_make_writable(_frame);
    if (ret < 0)
        throw VideoTargetError("Could not make frame writeable");

    /* convert pixel format */
    const uint8_t * src_data_ptr[1] = { frame.data() }; // BGRA has one plane
    int bgra_stride[1] = { 4*frame.cols() };
    sws_scale(_sws_context,
              src_data_ptr, bgra_stride,
              0, frame.rows(),
              _frame->data, _frame->linesize
              );

    _frame->pts = _frame_index++;

    /* encode the image */
    encode_and_write(_frame, got_output);
}

void VideoTargetFFmpeg::encode_and_write(AVFrame * frame, int & got_output)
{
    int ret;

    ret = avcodec_encode_video2(_stream->codec, &_packet, frame, &got_output);
    if (ret < 0)
        throw VideoTargetError("Error encoding frame");

    if (got_output)
    {
        /* rescale output packet timestamp values from codec to stream timebase */
        av_packet_rescale_ts(&_packet, _stream->codec->time_base, _stream->time_base);
        // TODO - above time bases are the same, or not?
        _packet.stream_index = _stream->index;

        /* Write the compressed frame to the media file. */
        int ret = av_interleaved_write_frame(_format_context, &_packet);
        if (ret < 0)
            throw VideoTargetError("Could not interleaved write frame");
//        av_packet_unref(&packet); taken care of by av_interleaved_write_frame
    }
}

void VideoTargetFFmpeg::finalise()
{
    /* get the delayed frames */
    for (int got_output = 1; got_output; )
        encode_and_write(NULL, got_output);

    /* Write the trailer, if any. The trailer must be written before you
     * close the CodecContexts open when you wrote the header; otherwise
     * av_write_trailer() may try to use memory that was freed on
     * av_codec_close(). */
    av_write_trailer(_format_context);

    if (_stream->codec) avcodec_close(_stream->codec);
    if (_frame) av_frame_free(&_frame);
//  av_freep(&_frame->data[0]); no need for this because _frame never manages its own data
    if (_sws_context) sws_freeContext(_sws_context);

    if (!(_format_context->oformat->flags & AVFMT_NOFILE))
        /* Close the output file. */
        avio_closep(&_format_context->pb);
    /* free the stream */
    avformat_free_context(_format_context);

    // default values, for next init
    _codec = NULL;
    _codec_id = AV_CODEC_ID_NONE;
    _frame = NULL;
    _framerate = -1;
    _sws_context = NULL;
    _format_context = NULL;
    _stream = NULL;
    _frame_index = 0;
}

}
<|endoftext|>


<commit_before>#include <iostream>

// Inludes common necessary includes for development using depthai library
#include "depthai/depthai.hpp"

int main() {
    using namespace std;

    // Create pipeline
    dai::Pipeline pipeline;

    // Rotate color frames
    auto camRgb = pipeline.create<dai::node::ColorCamera>();
    camRgb->setPreviewSize(640, 400);
    camRgb->setResolution(dai::ColorCameraProperties::SensorResolution::THE_1080_P);
    camRgb->setInterleaved(false);

    auto manipRgb = pipeline.create<dai::node::ImageManip>();
    dai::RotatedRect rgbRr = {{camRgb->getPreviewWidth() / 2.0, camRgb->getPreviewHeight() / 2.0}, // center
                           {camRgb->getPreviewHeight(), camRgb->getPreviewWidth()}, // size
                            90}; // angle
    manipRgb->initialConfig.setCropRotatedRect(rgbRr, false);
    camRgb->preview.link(manipRgb->inputImage);

    auto manipRgbOut = pipeline.create<dai::node::XLinkOut>();
    manipRgbOut->setStreamName("manip_rgb");
    manipRgb->out.link(manipRgbOut->input);

    // Rotate mono frames
    auto monoLeft = pipeline.create<dai::node::MonoCamera>();
    monoLeft->setResolution(dai::MonoCameraProperties::SensorResolution::THE_400_P);
    monoLeft->setBoardSocket(dai::CameraBoardSocket::LEFT);

    auto manipLeft = pipeline.create<dai::node::ImageManip>();
    dai::RotatedRect rr = {{monoLeft->getResolutionWidth() / 2.0, monoLeft->getResolutionHeight() / 2.0}, // center
                           {monoLeft->getResolutionHeight(), monoLeft->getResolutionWidth()}, // size
                            90}; // angle
    manipLeft->initialConfig.setCropRotatedRect(rr, false);
    monoLeft->out.link(manipLeft->inputImage);

    auto manipLeftOut = pipeline.create<dai::node::XLinkOut>();
    manipLeftOut->setStreamName("manip_left");
    manipLeft->out.link(manipLeftOut->input);

    dai::Device device(pipeline);

    auto qLeft = device.getOutputQueue("manip_left", 8, false);
    auto qRgb = device.getOutputQueue("manip_rgb", 8, false);

    while(true) {
        auto inLeft = qLeft->tryGet<dai::ImgFrame>();
        if(inLeft) {
            cv::imshow("Left rotated", inLeft->getCvFrame());
        }

        auto inRgb = qRgb->tryGet<dai::ImgFrame>();
        if(inRgb) {
            cv::imshow("Color rotated", inRgb->getCvFrame());
        }

        int key = cv::waitKey(1);
        if(key == 'q' || key == 'Q') return 0;
    }
    return 0;
}<commit_msg>Fixed conversion problems<commit_after>#include <iostream>

// Inludes common necessary includes for development using depthai library
#include "depthai/depthai.hpp"

int main() {
    using namespace std;

    // Create pipeline
    dai::Pipeline pipeline;

    // Rotate color frames
    auto camRgb = pipeline.create<dai::node::ColorCamera>();
    camRgb->setPreviewSize(640, 400);
    camRgb->setResolution(dai::ColorCameraProperties::SensorResolution::THE_1080_P);
    camRgb->setInterleaved(false);

    auto manipRgb = pipeline.create<dai::node::ImageManip>();
    dai::RotatedRect rgbRr = {{camRgb->getPreviewWidth() / 2.0f, camRgb->getPreviewHeight() / 2.0f}, // center
                           {camRgb->getPreviewHeight() * 1.0f, camRgb->getPreviewWidth() * 1.0f}, // size
                            90}; // angle
    manipRgb->initialConfig.setCropRotatedRect(rgbRr, false);
    camRgb->preview.link(manipRgb->inputImage);

    auto manipRgbOut = pipeline.create<dai::node::XLinkOut>();
    manipRgbOut->setStreamName("manip_rgb");
    manipRgb->out.link(manipRgbOut->input);

    // Rotate mono frames
    auto monoLeft = pipeline.create<dai::node::MonoCamera>();
    monoLeft->setResolution(dai::MonoCameraProperties::SensorResolution::THE_400_P);
    monoLeft->setBoardSocket(dai::CameraBoardSocket::LEFT);

    auto manipLeft = pipeline.create<dai::node::ImageManip>();
    dai::RotatedRect rr = {{monoLeft->getResolutionWidth() / 2.0f, monoLeft->getResolutionHeight() / 2.0f}, // center
                           {monoLeft->getResolutionHeight() * 1.0f, monoLeft->getResolutionWidth()} * 1.0f, // size
                            90}; // angle
    manipLeft->initialConfig.setCropRotatedRect(rr, false);
    monoLeft->out.link(manipLeft->inputImage);

    auto manipLeftOut = pipeline.create<dai::node::XLinkOut>();
    manipLeftOut->setStreamName("manip_left");
    manipLeft->out.link(manipLeftOut->input);

    dai::Device device(pipeline);

    auto qLeft = device.getOutputQueue("manip_left", 8, false);
    auto qRgb = device.getOutputQueue("manip_rgb", 8, false);

    while(true) {
        auto inLeft = qLeft->tryGet<dai::ImgFrame>();
        if(inLeft) {
            cv::imshow("Left rotated", inLeft->getCvFrame());
        }

        auto inRgb = qRgb->tryGet<dai::ImgFrame>();
        if(inRgb) {
            cv::imshow("Color rotated", inRgb->getCvFrame());
        }

        int key = cv::waitKey(1);
        if(key == 'q' || key == 'Q') return 0;
    }
    return 0;
}<|endoftext|>


<commit_before>/**
 * Copyright (C) 2007 by INdT
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 *
 * @author Gustavo Sverzut Barbieri <gustavo.barbieri@openbossa.org>
 */

/**
 * @brief
 *
 * Reads ID3 tags from MP3 using id3lib.
 *
 * @todo: write a faster module to replace this one, using no external libs.
 */

#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

#define _XOPEN_SOURCE 600
#include <lightmediascanner_plugin.h>
#include <lightmediascanner_utils.h>
#include <lightmediascanner_db.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <id3/tag.h>

extern "C" {

static void
_id3lib_get_string(const ID3_Frame *frame, ID3_FieldID field_id, struct lms_string_size *s)
{
  ID3_Field* field;
  ID3_TextEnc enc;
  size_t size;

  field = frame->GetField(field_id);
  if (!field)
    return;

  enc = field->GetEncoding();
  field->SetEncoding(ID3TE_ISO8859_1);
  size = field->Size();
  if (size < 1)
    goto done;

  size++;
  s->str = (char *)malloc(size * sizeof(char));
  s->len = field->Get(s->str, size);
  if (s->len > 0)
    lms_strstrip(s->str, &s->len);

  if (s->len < 1) {
    free(s->str);
    s->str = NULL;
    s->len = 0;
  }

 done:
  field->SetEncoding(enc);
}

static unsigned int
_id3lib_get_string_as_int(const ID3_Frame *frame, ID3_FieldID field_id)
{
  char buf[32];
  ID3_Field* field;
  size_t size;

  field = frame->GetField(field_id);
  if (!field)
    return 0;

  size = field->Get(buf, sizeof(buf));
  if (size > 0)
    return atoi(buf);

  return 0;
}

static int
_id3lib_get_data(const ID3_Tag &tag, struct lms_audio_info *info)
{
  ID3_Tag::ConstIterator *itr;
  const ID3_Frame *frame;
  int todo;

  todo = 6; /* info fields left to parse: title, artist, album, genre,
               trackno, rating */

  itr = tag.CreateIterator();

  while ((frame = itr->GetNext()) != NULL && todo > 0) {
    ID3_FrameID fid;

    fid = frame->GetID();

    switch (fid) {
    case ID3FID_TITLE:
      if (!info->title.str) {
        _id3lib_get_string(frame, ID3FN_TEXT, &info->title);
        if (info->title.str)
          todo--;
      }
      break;

    case ID3FID_COMPOSER:
    case ID3FID_LEADARTIST:
    case ID3FID_BAND:
    case ID3FID_CONDUCTOR:
    case ID3FID_MIXARTIST:
      if (!info->artist.str) {
        _id3lib_get_string(frame, ID3FN_TEXT, &info->artist);
        if (info->artist.str)
          todo--;
      }
      break;

    case ID3FID_ALBUM:
      if (!info->album.str) {
        _id3lib_get_string(frame, ID3FN_TEXT, &info->album);
        if (info->album.str)
          todo--;
      }
      break;

    case ID3FID_CONTENTTYPE:
      if (!info->genre.str) {
        _id3lib_get_string(frame, ID3FN_TEXT, &info->genre);
        if (info->genre.str)
          todo--;
      }
      break;

    case ID3FID_TRACKNUM:
      if (!info->trackno) {
        info->trackno = _id3lib_get_string_as_int(frame, ID3FN_TEXT);
        if (info->trackno)
          todo--;
      }
      break;

    case ID3FID_POPULARIMETER:
      if (!info->rating) {
        info->rating = frame->GetField(ID3FN_RATING)->Get();
        if (info->rating)
          todo--;
      }
      break;

    default:
      break;
    }
  }

  delete itr;
  return 0;
}

static const char _name[] = "id3lib";
static const struct lms_string_size _exts[] = {
    LMS_STATIC_STRING_SIZE(".mp3")
};

struct plugin {
    struct lms_plugin plugin;
    lms_db_audio_t *audio_db;
};

static void *
_match(struct plugin *p, const char *path, int len, int base)
{
    int i;

    i = lms_which_extension(path, len, _exts, LMS_ARRAY_SIZE(_exts));
    if (i < 0)
      return NULL;
    else
      return (void*)(i + 1);
}

static int
_parse(struct plugin *plugin, sqlite3 *db, const struct lms_file_info *finfo, void *match)
{
    struct lms_audio_info info = {0};
    ID3_Tag tag;
    int r;

    tag.Link(finfo->path);
    r = _id3lib_get_data(tag, &info);
    if (r != 0)
      goto done;

    if (!info.title.str) {
      int ext_idx;

      ext_idx = ((int)match) - 1;
      info.title.len = finfo->path_len - finfo->base - _exts[ext_idx].len;
      info.title.str = (char *)malloc((info.title.len + 1) * sizeof(char));
      memcpy(info.title.str, finfo->path + finfo->base, info.title.len);
      info.title.str[info.title.len] = '\0';
    }

    info.id = finfo->id;
    r = lms_db_audio_add(plugin->audio_db, &info);

 done:
    if (info.title.str)
        free(info.title.str);
    if (info.artist.str)
        free(info.artist.str);
    if (info.album.str)
        free(info.album.str);
    if (info.genre.str)
        free(info.genre.str);

    return r;
}

static int
_setup(struct plugin *plugin, sqlite3 *db)
{
    plugin->audio_db = lms_db_audio_new(db);
    if (!plugin->audio_db)
        return -1;

    return 0;
}

static int
_start(struct plugin *plugin, sqlite3 *db)
{
    return lms_db_audio_start(plugin->audio_db);
}

static int
_finish(struct plugin *plugin, sqlite3 *db)
{
    if (plugin->audio_db)
        return lms_db_audio_free(plugin->audio_db);

    return 0;
}

static int
_close(struct plugin *plugin)
{
    free(plugin);
    return 0;
}

API struct lms_plugin *
lms_plugin_open(void)
{
    struct plugin *plugin;

    plugin = (struct plugin *)malloc(sizeof(*plugin));
    plugin->plugin.name = _name;
    plugin->plugin.match = (lms_plugin_match_fn_t)_match;
    plugin->plugin.parse = (lms_plugin_parse_fn_t)_parse;
    plugin->plugin.close = (lms_plugin_close_fn_t)_close;
    plugin->plugin.setup = (lms_plugin_setup_fn_t)_setup;
    plugin->plugin.start = (lms_plugin_start_fn_t)_start;
    plugin->plugin.finish = (lms_plugin_finish_fn_t)_finish;

    return (struct lms_plugin *)plugin;
}

}
<commit_msg>id3lib now uses playcnt.<commit_after>/**
 * Copyright (C) 2007 by INdT
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 *
 * @author Gustavo Sverzut Barbieri <gustavo.barbieri@openbossa.org>
 */

/**
 * @brief
 *
 * Reads ID3 tags from MP3 using id3lib.
 *
 * @todo: write a faster module to replace this one, using no external libs.
 */

#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

#define _XOPEN_SOURCE 600
#include <lightmediascanner_plugin.h>
#include <lightmediascanner_utils.h>
#include <lightmediascanner_db.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <id3/tag.h>

extern "C" {

static void
_id3lib_get_string(const ID3_Frame *frame, ID3_FieldID field_id, struct lms_string_size *s)
{
  ID3_Field* field;
  ID3_TextEnc enc;
  size_t size;

  field = frame->GetField(field_id);
  if (!field)
    return;

  enc = field->GetEncoding();
  field->SetEncoding(ID3TE_ISO8859_1);
  size = field->Size();
  if (size < 1)
    goto done;

  size++;
  s->str = (char *)malloc(size * sizeof(char));
  s->len = field->Get(s->str, size);
  if (s->len > 0)
    lms_strstrip(s->str, &s->len);

  if (s->len < 1) {
    free(s->str);
    s->str = NULL;
    s->len = 0;
  }

 done:
  field->SetEncoding(enc);
}

static unsigned int
_id3lib_get_string_as_int(const ID3_Frame *frame, ID3_FieldID field_id)
{
  char buf[32];
  ID3_Field* field;
  size_t size;

  field = frame->GetField(field_id);
  if (!field)
    return 0;

  size = field->Get(buf, sizeof(buf));
  if (size > 0)
    return atoi(buf);

  return 0;
}

static int
_id3lib_get_data(const ID3_Tag &tag, struct lms_audio_info *info)
{
  ID3_Tag::ConstIterator *itr;
  const ID3_Frame *frame;
  int todo;

  todo = 7; /* info fields left to parse: title, artist, album, genre,
               trackno, rating, playcnt */

  itr = tag.CreateIterator();

  while ((frame = itr->GetNext()) != NULL && todo > 0) {
    ID3_FrameID fid;

    fid = frame->GetID();

    switch (fid) {
    case ID3FID_TITLE:
      if (!info->title.str) {
        _id3lib_get_string(frame, ID3FN_TEXT, &info->title);
        if (info->title.str)
          todo--;
      }
      break;

    case ID3FID_COMPOSER:
    case ID3FID_LEADARTIST:
    case ID3FID_BAND:
    case ID3FID_CONDUCTOR:
    case ID3FID_MIXARTIST:
      if (!info->artist.str) {
        _id3lib_get_string(frame, ID3FN_TEXT, &info->artist);
        if (info->artist.str)
          todo--;
      }
      break;

    case ID3FID_ALBUM:
      if (!info->album.str) {
        _id3lib_get_string(frame, ID3FN_TEXT, &info->album);
        if (info->album.str)
          todo--;
      }
      break;

    case ID3FID_CONTENTTYPE:
      if (!info->genre.str) {
        _id3lib_get_string(frame, ID3FN_TEXT, &info->genre);
        if (info->genre.str)
          todo--;
      }
      break;

    case ID3FID_TRACKNUM:
      if (!info->trackno) {
        info->trackno = _id3lib_get_string_as_int(frame, ID3FN_TEXT);
        if (info->trackno)
          todo--;
      }
      break;

    case ID3FID_POPULARIMETER:
      if (!info->rating) {
        info->rating = frame->GetField(ID3FN_RATING)->Get();
        if (info->rating)
          todo--;
      }
      if (!info->playcnt) {
        info->playcnt = frame->GetField(ID3FN_COUNTER)->Get();
        if (info->playcnt)
          todo--;
      }
      break;

    default:
      break;
    }
  }

  delete itr;
  return 0;
}

static const char _name[] = "id3lib";
static const struct lms_string_size _exts[] = {
    LMS_STATIC_STRING_SIZE(".mp3")
};

struct plugin {
    struct lms_plugin plugin;
    lms_db_audio_t *audio_db;
};

static void *
_match(struct plugin *p, const char *path, int len, int base)
{
    int i;

    i = lms_which_extension(path, len, _exts, LMS_ARRAY_SIZE(_exts));
    if (i < 0)
      return NULL;
    else
      return (void*)(i + 1);
}

static int
_parse(struct plugin *plugin, sqlite3 *db, const struct lms_file_info *finfo, void *match)
{
    struct lms_audio_info info = {0};
    ID3_Tag tag;
    int r;

    tag.Link(finfo->path);
    r = _id3lib_get_data(tag, &info);
    if (r != 0)
      goto done;

    if (!info.title.str) {
      int ext_idx;

      ext_idx = ((int)match) - 1;
      info.title.len = finfo->path_len - finfo->base - _exts[ext_idx].len;
      info.title.str = (char *)malloc((info.title.len + 1) * sizeof(char));
      memcpy(info.title.str, finfo->path + finfo->base, info.title.len);
      info.title.str[info.title.len] = '\0';
    }

    info.id = finfo->id;
    r = lms_db_audio_add(plugin->audio_db, &info);

 done:
    if (info.title.str)
        free(info.title.str);
    if (info.artist.str)
        free(info.artist.str);
    if (info.album.str)
        free(info.album.str);
    if (info.genre.str)
        free(info.genre.str);

    return r;
}

static int
_setup(struct plugin *plugin, sqlite3 *db)
{
    plugin->audio_db = lms_db_audio_new(db);
    if (!plugin->audio_db)
        return -1;

    return 0;
}

static int
_start(struct plugin *plugin, sqlite3 *db)
{
    return lms_db_audio_start(plugin->audio_db);
}

static int
_finish(struct plugin *plugin, sqlite3 *db)
{
    if (plugin->audio_db)
        return lms_db_audio_free(plugin->audio_db);

    return 0;
}

static int
_close(struct plugin *plugin)
{
    free(plugin);
    return 0;
}

API struct lms_plugin *
lms_plugin_open(void)
{
    struct plugin *plugin;

    plugin = (struct plugin *)malloc(sizeof(*plugin));
    plugin->plugin.name = _name;
    plugin->plugin.match = (lms_plugin_match_fn_t)_match;
    plugin->plugin.parse = (lms_plugin_parse_fn_t)_parse;
    plugin->plugin.close = (lms_plugin_close_fn_t)_close;
    plugin->plugin.setup = (lms_plugin_setup_fn_t)_setup;
    plugin->plugin.start = (lms_plugin_start_fn_t)_start;
    plugin->plugin.finish = (lms_plugin_finish_fn_t)_finish;

    return (struct lms_plugin *)plugin;
}

}
<|endoftext|>


<commit_before>/**
 * @file
 *
 * @brief Write key sets using yaml-cpp
 *
 * @copyright BSD License (see LICENSE.md or https://www.libelektra.org)
 */

#include "write.hpp"
#include "yaml-cpp/yaml.h"

#include <kdbassert.h>
#include <kdbease.h>
#include <kdblogger.h>
#include <kdbplugin.h>

#include <fstream>

using namespace std;
using namespace kdb;

namespace
{

using KeySetPair = pair<KeySet, KeySet>;

/**
 * @brief This function checks if `element` is an array element of `parent`.
 *
 * @pre The key `child` must be below `parent`.
 *
 * @param parent This parameter specifies a parent key.
 * @param keys This variable stores a direct or indirect child of `parent`.
 *
 * @retval true If `element` is an array element
 * @retval false Otherwise
 */
bool isArrayElementOf (Key const & parent, Key const & child)
{
	char const * relative = elektraKeyGetRelativeName (*child, *parent);
	auto offsetIndex = ckdb::elektraArrayValidateBaseNameString (relative);
	if (offsetIndex <= 0) return false;
	// Skip `#`, underscores and digits
	relative += 2 * offsetIndex;
	// The next character has to be the separation char (`/`) or end of string
	if (relative[0] != '\0' && relative[0] != '/') return false;

	return true;
}

/**
 * @brief This function determines if the given key is an array parent.
 *
 * @param parent This parameter specifies a possible array parent.
 * @param keys This variable stores the key set of `parent`.
 *
 * @retval true If `parent` is the parent key of an array
 * @retval false Otherwise
 */
bool isArrayParent (Key const & parent, KeySet const & keys)
{
	for (auto const & key : keys)
	{
		if (!key.isBelow (parent)) continue;
		if (!isArrayElementOf (parent, key)) return false;
	}

	return true;
}

/**
 * @brief Split `keys` into two key sets, one for array parents and one for all other keys.
 *
 * @param keys This parameter contains the key set this function splits.
 *
 * @return A pair of key sets, where the first key set contains all array parents and the second key set contains all other keys
 */
KeySetPair splitArrayParentsOther (KeySet const & keys)
{
	KeySet arrayParents;
	KeySet others;

	keys.rewind ();
	Key previous;
	for (previous = keys.next (); keys.next (); previous = keys.current ())
	{
		bool const previousIsArray =
			previous.hasMeta ("array") ||
			(keys.current ().isBelow (previous) && keys.current ().getBaseName ()[0] == '#' && isArrayParent (previous, keys));

		(previousIsArray ? arrayParents : others).append (previous);
	}
	(previous.hasMeta ("array") ? arrayParents : others).append (previous);

	ELEKTRA_ASSERT (arrayParents.size () + others.size () == keys.size (),
			"Number of keys in split key sets: %zu ≠ number in given key set %zu", arrayParents.size () + others.size (),
			keys.size ());

	return make_pair (arrayParents, others);
}

/**
 * @brief This function splits `keys` into two key sets, one for array parents and elements, and the other one for all other keys.
 *
 * @param arrayParents This key set contains a (copy) of all array parents of `keys`.
 * @param keys This parameter contains the key set this function splits.
 *
 * @return A pair of key sets, where the first key set contains all array parents and elements,
 *         and the second key set contains all other keys
 */
KeySetPair splitArrayOther (KeySet const & arrayParents, KeySet const & keys)
{
	KeySet others = keys.dup ();
	KeySet arrays;

	for (auto const & parent : arrayParents)
	{
		arrays.append (others.cut (parent));
	}

	return make_pair (arrays, others);
}

/**
 * @brief This function returns a `NameIterator` starting at the first level that is not part of `parent`.
 *
 * @pre The parameter `key` must be a child of `parent`.
 *
 * @param key This is the key for which this function returns a relative iterator.
 * @param parent This key specifies the part of the name iterator that will not be part of the return value of this function.
 *
 * @returns A relative iterator that starts with the first part of the name of `key` not contained in `parent`.
 */
NameIterator relativeKeyIterator (Key const & key, Key const & parent)
{
	auto parentIterator = parent.begin ();
	auto keyIterator = key.begin ();
	while (parentIterator != parent.end () && keyIterator != key.end ())
	{
		parentIterator++;
		keyIterator++;
	}
	return keyIterator;
}

/**
 * @brief This function checks if a key name specifies an array key.
 *
 * If the key name contains a valid array index that is smaller than `unsigned long long`, then the function will also return this index.
 *
 * @param nameIterator This iterator specifies the name of the key.
 *
 * @retval (true, arrayIndex) if `name` specifies an array key, where `arrayIndex` specifies the index stored in the array key.
 * @retval (false, 0) otherwise
 */
std::pair<bool, unsigned long long> isArrayIndex (NameIterator const & nameIterator)
{
	string const name = *nameIterator;
	auto const offsetIndex = ckdb::elektraArrayValidateBaseNameString (name.c_str ());
	auto const isArrayElement = offsetIndex >= 1;
	return { isArrayElement, isArrayElement ? stoull (name.substr (offsetIndex)) : 0 };
}

/**
 * @brief This function creates a YAML node representing a key value.
 *
 * @param key This key specifies the data that should be saved in the YAML node returned by this function.
 *
 * @note Since YAML does not support non-empty binary data directly this function replaces data stored in binary keys with the string
 *       `Unsupported binary value!`. If you need support for binary data, please load the Base64 before you use YAML CPP.
 *
 * @returns A new YAML node containing the data specified in `key`
 */
YAML::Node createMetaDataNode (Key const & key)
{
	return key.hasMeta ("array") ?
		       YAML::Node (YAML::NodeType::Sequence) :
		       key.getBinarySize () == 0 ? YAML::Node (YAML::NodeType::Null) :
						   YAML::Node (key.isBinary () ? "Unsupported binary value!" : key.getString ());
}

/**
 * @brief This function creates a YAML Node containing a key value and optionally metadata.
 *
 * @param key This key specifies the data that should be saved in the YAML node returned by this function.
 *
 * @note Since YAML does not support non-empty binary data directly this function replaces data stored in binary keys with the string
 *       `Unsupported binary value!`. If you need support for binary data, please load the Base64 before you use YAML CPP.
 *
 * @returns A new YAML node containing the data and metadata specified in `key`
 */
YAML::Node createLeafNode (Key & key)
{

	YAML::Node metaNode{ YAML::Node (YAML::NodeType::Map) };
	YAML::Node dataNode = createMetaDataNode (key);

	key.rewindMeta ();
	while (Key meta = key.nextMeta ())
	{
		if (meta.getName () == "array" || meta.getName () == "binary") continue;
		if (meta.getName () == "type" && meta.getString () == "binary")
		{
			dataNode.SetTag ("tag:yaml.org,2002:binary");
			continue;
		}
		metaNode[meta.getName ()] = meta.getString ();
		ELEKTRA_LOG_DEBUG ("Add metakey “%s: %s”", meta.getName ().c_str (), meta.getString ().c_str ());
	}

	if (metaNode.size () <= 0)
	{
		ELEKTRA_LOG_DEBUG ("Return leaf node with value “%s”",
				   dataNode.IsNull () ? "~" : dataNode.IsSequence () ? "[]" : dataNode.as<string> ().c_str ());
		return dataNode;
	}

	YAML::Node node{ YAML::Node (YAML::NodeType::Sequence) };
	node.SetTag ("!elektra/meta");
	node.push_back (dataNode);
	node.push_back (metaNode);

#ifdef HAVE_LOGGER
	ostringstream data;
	data << node;
	ELEKTRA_LOG_DEBUG ("Return meta leaf node with value “%s”", data.str ().c_str ());
#endif

	return node;
}

/**
 * @brief This function adds `null` elements to the given YAML collection.
 *
 * @param sequence This node stores the collection to which this function adds `numberOfElements` empty elements.
 * @param numberOfElements This parameter specifies the number of empty element this function adds to `sequence`.
 */
void addEmptyArrayElements (YAML::Node & sequence, unsigned long long const numberOfElements)
{
	ELEKTRA_LOG_DEBUG ("Add %lld empty array elements", numberOfElements);
	for (auto missingFields = numberOfElements; missingFields > 0; missingFields--)
	{
		sequence.push_back ({});
	}
}

/**
 * @brief This function adds a key to a YAML node.
 *
 * @param data This node stores the data specified via `keyIterator`.
 * @param keyIterator This iterator specifies the current part of the key name this function adds to `data`.
 * @param key This parameter specifies the key that should be added to `data`.
 */
void addKeyNoArray (YAML::Node & data, NameIterator & keyIterator, Key & key)
{
	if (data.IsScalar ()) data = YAML::Node (YAML::NodeType::Undefined);

#ifdef HAVE_LOGGER
	ostringstream output;
	output << data;
	ELEKTRA_LOG_DEBUG ("Add key part “%s” to node “%s”", (*keyIterator).c_str (), output.str ().c_str ());
#endif

	if (keyIterator == key.end ())
	{
		ELEKTRA_LOG_DEBUG ("Create leaf node for key “%s”", key.getName ().c_str ());
		data = createLeafNode (key);
		return;
	}
	if (keyIterator == --key.end ())
	{
		data[*keyIterator] = createLeafNode (key);
		return;
	}

	YAML::Node node;

	node = (data[*keyIterator] && !data[*keyIterator].IsScalar ()) ? data[*keyIterator] : YAML::Node ();
	data[*keyIterator] = node;
	addKeyNoArray (node, ++keyIterator, key);
}

/**
 * @brief This function adds a key to a YAML node.
 *
 * @param data This node stores the data specified via `keyIterator`.
 * @param keyIterator This iterator specifies the current part of the key name this function adds to `data`.
 * @param key This parameter specifies the key that should be added to `data`.
 */
void addKeyArray (YAML::Node & data, NameIterator & keyIterator, Key & key)
{
	auto const isArrayAndIndex = isArrayIndex (keyIterator);
	auto const isArrayElement = isArrayAndIndex.first;
	auto const arrayIndex = isArrayAndIndex.second;

	if (data.IsScalar ()) data = YAML::Node (YAML::NodeType::Undefined);

#ifdef HAVE_LOGGER
	ostringstream output;
	output << data;
	ELEKTRA_LOG_DEBUG ("Add key part “%s” to node “%s”", (*keyIterator).c_str (), output.str ().c_str ());
#endif

	if (keyIterator == key.end ())
	{
		ELEKTRA_LOG_DEBUG ("Create leaf node for key “%s”", key.getName ().c_str ());
		data = createLeafNode (key);
		return;
	}
	if (keyIterator == --key.end ())
	{
		if (isArrayElement)
		{
			addEmptyArrayElements (data, arrayIndex - data.size ());
			data.push_back (createLeafNode (key));
		}
		else
		{
			data[*keyIterator] = createLeafNode (key);
		}

		return;
	}

	YAML::Node node;

	if (isArrayElement)
	{
		node = (data[arrayIndex] && !data[arrayIndex].IsScalar ()) ? data[arrayIndex] : YAML::Node ();
		data[arrayIndex] = node;
	}
	else
	{
		node = (data[*keyIterator] && !data[*keyIterator].IsScalar ()) ? data[*keyIterator] : YAML::Node ();
		data[*keyIterator] = node;
	}
	addKeyArray (node, ++keyIterator, key);
}

/**
 * @brief This function adds a key set to a YAML node.
 *
 * @param data This node stores the data specified via `mappings`.
 * @param mappings This keyset specifies all keys and values this function adds to `data`.
 * @param parent This key is the root of all keys stored in `mappings`.
 */
void addKeys (YAML::Node & data, KeySet const & mappings, Key const & parent, bool const isArray = false)
{
	for (auto key : mappings)
	{
		ELEKTRA_LOG_DEBUG ("Convert key “%s”: “%s”", key.getName ().c_str (),
				   key.getBinarySize () == 0 ? "NULL" : key.isString () ? key.getString ().c_str () : "binary value!");
		NameIterator keyIterator = relativeKeyIterator (key, parent);

		if (isArray)
		{
			addKeyArray (data, keyIterator, key);
		}
		else
		{
			addKeyNoArray (data, keyIterator, key);
		}

#ifdef HAVE_LOGGER
		ostringstream output;
		output << data;

		ELEKTRA_LOG_DEBUG ("Converted Data:");
		ELEKTRA_LOG_DEBUG ("__________");

		istringstream stream (output.str ());
		for (string line; std::getline (stream, line);)
		{
			ELEKTRA_LOG_DEBUG ("%s", line.c_str ());
		}

		ELEKTRA_LOG_DEBUG ("__________");
#endif
	}
}

} // end namespace

/**
 * @brief This function saves the key-value pairs stored in `mappings` as YAML data in the location specified via `parent`.
 *
 * @param mappings This key set stores the mappings that should be saved as YAML data.
 * @param parent This key specifies the path to the YAML data file that should be written.
 */
void yamlcpp::yamlWrite (KeySet const & mappings, Key const & parent)
{
	ofstream output (parent.getString ());
	auto data = YAML::Node ();

	KeySet arrayParents;
	KeySet arrays;
	KeySet nonArrays;
	tie (arrayParents, std::ignore) = splitArrayParentsOther (mappings);
	tie (arrays, nonArrays) = splitArrayOther (arrayParents, mappings);

	addKeys (data, arrays, parent, true);
	addKeys (data, nonArrays, parent);

#ifdef HAVE_LOGGER
	ELEKTRA_LOG_DEBUG ("Write Data:");
	ELEKTRA_LOG_DEBUG ("__________");

	ostringstream outputString;
	outputString << data;
	istringstream stream (outputString.str ());
	for (string line; std::getline (stream, line);)
	{
		ELEKTRA_LOG_DEBUG ("%s", line.c_str ());
	}

	ELEKTRA_LOG_DEBUG ("__________");
#endif

	output << data;
}
<commit_msg>YAML CPP: Declare variables as late as possible<commit_after>/**
 * @file
 *
 * @brief Write key sets using yaml-cpp
 *
 * @copyright BSD License (see LICENSE.md or https://www.libelektra.org)
 */

#include "write.hpp"
#include "yaml-cpp/yaml.h"

#include <kdbassert.h>
#include <kdbease.h>
#include <kdblogger.h>
#include <kdbplugin.h>

#include <fstream>

using namespace std;
using namespace kdb;

namespace
{

using KeySetPair = pair<KeySet, KeySet>;

/**
 * @brief This function checks if `element` is an array element of `parent`.
 *
 * @pre The key `child` must be below `parent`.
 *
 * @param parent This parameter specifies a parent key.
 * @param keys This variable stores a direct or indirect child of `parent`.
 *
 * @retval true If `element` is an array element
 * @retval false Otherwise
 */
bool isArrayElementOf (Key const & parent, Key const & child)
{
	char const * relative = elektraKeyGetRelativeName (*child, *parent);
	auto offsetIndex = ckdb::elektraArrayValidateBaseNameString (relative);
	if (offsetIndex <= 0) return false;
	// Skip `#`, underscores and digits
	relative += 2 * offsetIndex;
	// The next character has to be the separation char (`/`) or end of string
	if (relative[0] != '\0' && relative[0] != '/') return false;

	return true;
}

/**
 * @brief This function determines if the given key is an array parent.
 *
 * @param parent This parameter specifies a possible array parent.
 * @param keys This variable stores the key set of `parent`.
 *
 * @retval true If `parent` is the parent key of an array
 * @retval false Otherwise
 */
bool isArrayParent (Key const & parent, KeySet const & keys)
{
	for (auto const & key : keys)
	{
		if (!key.isBelow (parent)) continue;
		if (!isArrayElementOf (parent, key)) return false;
	}

	return true;
}

/**
 * @brief Split `keys` into two key sets, one for array parents and one for all other keys.
 *
 * @param keys This parameter contains the key set this function splits.
 *
 * @return A pair of key sets, where the first key set contains all array parents and the second key set contains all other keys
 */
KeySetPair splitArrayParentsOther (KeySet const & keys)
{
	KeySet arrayParents;
	KeySet others;

	keys.rewind ();
	Key previous;
	for (previous = keys.next (); keys.next (); previous = keys.current ())
	{
		bool const previousIsArray =
			previous.hasMeta ("array") ||
			(keys.current ().isBelow (previous) && keys.current ().getBaseName ()[0] == '#' && isArrayParent (previous, keys));

		(previousIsArray ? arrayParents : others).append (previous);
	}
	(previous.hasMeta ("array") ? arrayParents : others).append (previous);

	ELEKTRA_ASSERT (arrayParents.size () + others.size () == keys.size (),
			"Number of keys in split key sets: %zu ≠ number in given key set %zu", arrayParents.size () + others.size (),
			keys.size ());

	return make_pair (arrayParents, others);
}

/**
 * @brief This function splits `keys` into two key sets, one for array parents and elements, and the other one for all other keys.
 *
 * @param arrayParents This key set contains a (copy) of all array parents of `keys`.
 * @param keys This parameter contains the key set this function splits.
 *
 * @return A pair of key sets, where the first key set contains all array parents and elements,
 *         and the second key set contains all other keys
 */
KeySetPair splitArrayOther (KeySet const & arrayParents, KeySet const & keys)
{
	KeySet others = keys.dup ();
	KeySet arrays;

	for (auto const & parent : arrayParents)
	{
		arrays.append (others.cut (parent));
	}

	return make_pair (arrays, others);
}

/**
 * @brief This function returns a `NameIterator` starting at the first level that is not part of `parent`.
 *
 * @pre The parameter `key` must be a child of `parent`.
 *
 * @param key This is the key for which this function returns a relative iterator.
 * @param parent This key specifies the part of the name iterator that will not be part of the return value of this function.
 *
 * @returns A relative iterator that starts with the first part of the name of `key` not contained in `parent`.
 */
NameIterator relativeKeyIterator (Key const & key, Key const & parent)
{
	auto parentIterator = parent.begin ();
	auto keyIterator = key.begin ();
	while (parentIterator != parent.end () && keyIterator != key.end ())
	{
		parentIterator++;
		keyIterator++;
	}
	return keyIterator;
}

/**
 * @brief This function checks if a key name specifies an array key.
 *
 * If the key name contains a valid array index that is smaller than `unsigned long long`, then the function will also return this index.
 *
 * @param nameIterator This iterator specifies the name of the key.
 *
 * @retval (true, arrayIndex) if `name` specifies an array key, where `arrayIndex` specifies the index stored in the array key.
 * @retval (false, 0) otherwise
 */
std::pair<bool, unsigned long long> isArrayIndex (NameIterator const & nameIterator)
{
	string const name = *nameIterator;
	auto const offsetIndex = ckdb::elektraArrayValidateBaseNameString (name.c_str ());
	auto const isArrayElement = offsetIndex >= 1;
	return { isArrayElement, isArrayElement ? stoull (name.substr (offsetIndex)) : 0 };
}

/**
 * @brief This function creates a YAML node representing a key value.
 *
 * @param key This key specifies the data that should be saved in the YAML node returned by this function.
 *
 * @note Since YAML does not support non-empty binary data directly this function replaces data stored in binary keys with the string
 *       `Unsupported binary value!`. If you need support for binary data, please load the Base64 before you use YAML CPP.
 *
 * @returns A new YAML node containing the data specified in `key`
 */
YAML::Node createMetaDataNode (Key const & key)
{
	return key.hasMeta ("array") ?
		       YAML::Node (YAML::NodeType::Sequence) :
		       key.getBinarySize () == 0 ? YAML::Node (YAML::NodeType::Null) :
						   YAML::Node (key.isBinary () ? "Unsupported binary value!" : key.getString ());
}

/**
 * @brief This function creates a YAML Node containing a key value and optionally metadata.
 *
 * @param key This key specifies the data that should be saved in the YAML node returned by this function.
 *
 * @note Since YAML does not support non-empty binary data directly this function replaces data stored in binary keys with the string
 *       `Unsupported binary value!`. If you need support for binary data, please load the Base64 before you use YAML CPP.
 *
 * @returns A new YAML node containing the data and metadata specified in `key`
 */
YAML::Node createLeafNode (Key & key)
{

	YAML::Node metaNode{ YAML::Node (YAML::NodeType::Map) };
	YAML::Node dataNode = createMetaDataNode (key);

	key.rewindMeta ();
	while (Key meta = key.nextMeta ())
	{
		if (meta.getName () == "array" || meta.getName () == "binary") continue;
		if (meta.getName () == "type" && meta.getString () == "binary")
		{
			dataNode.SetTag ("tag:yaml.org,2002:binary");
			continue;
		}
		metaNode[meta.getName ()] = meta.getString ();
		ELEKTRA_LOG_DEBUG ("Add metakey “%s: %s”", meta.getName ().c_str (), meta.getString ().c_str ());
	}

	if (metaNode.size () <= 0)
	{
		ELEKTRA_LOG_DEBUG ("Return leaf node with value “%s”",
				   dataNode.IsNull () ? "~" : dataNode.IsSequence () ? "[]" : dataNode.as<string> ().c_str ());
		return dataNode;
	}

	YAML::Node node{ YAML::Node (YAML::NodeType::Sequence) };
	node.SetTag ("!elektra/meta");
	node.push_back (dataNode);
	node.push_back (metaNode);

#ifdef HAVE_LOGGER
	ostringstream data;
	data << node;
	ELEKTRA_LOG_DEBUG ("Return meta leaf node with value “%s”", data.str ().c_str ());
#endif

	return node;
}

/**
 * @brief This function adds `null` elements to the given YAML collection.
 *
 * @param sequence This node stores the collection to which this function adds `numberOfElements` empty elements.
 * @param numberOfElements This parameter specifies the number of empty element this function adds to `sequence`.
 */
void addEmptyArrayElements (YAML::Node & sequence, unsigned long long const numberOfElements)
{
	ELEKTRA_LOG_DEBUG ("Add %lld empty array elements", numberOfElements);
	for (auto missingFields = numberOfElements; missingFields > 0; missingFields--)
	{
		sequence.push_back ({});
	}
}

/**
 * @brief This function adds a key to a YAML node.
 *
 * @param data This node stores the data specified via `keyIterator`.
 * @param keyIterator This iterator specifies the current part of the key name this function adds to `data`.
 * @param key This parameter specifies the key that should be added to `data`.
 */
void addKeyNoArray (YAML::Node & data, NameIterator & keyIterator, Key & key)
{
	if (data.IsScalar ()) data = YAML::Node (YAML::NodeType::Undefined);

#ifdef HAVE_LOGGER
	ostringstream output;
	output << data;
	ELEKTRA_LOG_DEBUG ("Add key part “%s” to node “%s”", (*keyIterator).c_str (), output.str ().c_str ());
#endif

	if (keyIterator == key.end ())
	{
		ELEKTRA_LOG_DEBUG ("Create leaf node for key “%s”", key.getName ().c_str ());
		data = createLeafNode (key);
		return;
	}
	if (keyIterator == --key.end ())
	{
		data[*keyIterator] = createLeafNode (key);
		return;
	}

	YAML::Node node;

	node = (data[*keyIterator] && !data[*keyIterator].IsScalar ()) ? data[*keyIterator] : YAML::Node ();
	data[*keyIterator] = node;
	addKeyNoArray (node, ++keyIterator, key);
}

/**
 * @brief This function adds a key to a YAML node.
 *
 * @param data This node stores the data specified via `keyIterator`.
 * @param keyIterator This iterator specifies the current part of the key name this function adds to `data`.
 * @param key This parameter specifies the key that should be added to `data`.
 */
void addKeyArray (YAML::Node & data, NameIterator & keyIterator, Key & key)
{
	auto const isArrayAndIndex = isArrayIndex (keyIterator);
	auto const isArrayElement = isArrayAndIndex.first;
	auto const arrayIndex = isArrayAndIndex.second;

	if (data.IsScalar ()) data = YAML::Node (YAML::NodeType::Undefined);

#ifdef HAVE_LOGGER
	ostringstream output;
	output << data;
	ELEKTRA_LOG_DEBUG ("Add key part “%s” to node “%s”", (*keyIterator).c_str (), output.str ().c_str ());
#endif

	if (keyIterator == key.end ())
	{
		ELEKTRA_LOG_DEBUG ("Create leaf node for key “%s”", key.getName ().c_str ());
		data = createLeafNode (key);
		return;
	}
	if (keyIterator == --key.end ())
	{
		if (isArrayElement)
		{
			addEmptyArrayElements (data, arrayIndex - data.size ());
			data.push_back (createLeafNode (key));
		}
		else
		{
			data[*keyIterator] = createLeafNode (key);
		}

		return;
	}

	YAML::Node node;

	if (isArrayElement)
	{
		node = (data[arrayIndex] && !data[arrayIndex].IsScalar ()) ? data[arrayIndex] : YAML::Node ();
		data[arrayIndex] = node;
	}
	else
	{
		node = (data[*keyIterator] && !data[*keyIterator].IsScalar ()) ? data[*keyIterator] : YAML::Node ();
		data[*keyIterator] = node;
	}
	addKeyArray (node, ++keyIterator, key);
}

/**
 * @brief This function adds a key set to a YAML node.
 *
 * @param data This node stores the data specified via `mappings`.
 * @param mappings This keyset specifies all keys and values this function adds to `data`.
 * @param parent This key is the root of all keys stored in `mappings`.
 */
void addKeys (YAML::Node & data, KeySet const & mappings, Key const & parent, bool const isArray = false)
{
	for (auto key : mappings)
	{
		ELEKTRA_LOG_DEBUG ("Convert key “%s”: “%s”", key.getName ().c_str (),
				   key.getBinarySize () == 0 ? "NULL" : key.isString () ? key.getString ().c_str () : "binary value!");
		NameIterator keyIterator = relativeKeyIterator (key, parent);

		if (isArray)
		{
			addKeyArray (data, keyIterator, key);
		}
		else
		{
			addKeyNoArray (data, keyIterator, key);
		}

#ifdef HAVE_LOGGER
		ostringstream output;
		output << data;

		ELEKTRA_LOG_DEBUG ("Converted Data:");
		ELEKTRA_LOG_DEBUG ("__________");

		istringstream stream (output.str ());
		for (string line; std::getline (stream, line);)
		{
			ELEKTRA_LOG_DEBUG ("%s", line.c_str ());
		}

		ELEKTRA_LOG_DEBUG ("__________");
#endif
	}
}

} // end namespace

/**
 * @brief This function saves the key-value pairs stored in `mappings` as YAML data in the location specified via `parent`.
 *
 * @param mappings This key set stores the mappings that should be saved as YAML data.
 * @param parent This key specifies the path to the YAML data file that should be written.
 */
void yamlcpp::yamlWrite (KeySet const & mappings, Key const & parent)
{

	KeySet arrayParents;
	KeySet arrays;
	KeySet nonArrays;
	tie (arrayParents, std::ignore) = splitArrayParentsOther (mappings);
	tie (arrays, nonArrays) = splitArrayOther (arrayParents, mappings);

	auto data = YAML::Node ();
	addKeys (data, arrays, parent, true);
	addKeys (data, nonArrays, parent);

#ifdef HAVE_LOGGER
	ELEKTRA_LOG_DEBUG ("Write Data:");
	ELEKTRA_LOG_DEBUG ("__________");

	ostringstream outputString;
	outputString << data;
	istringstream stream (outputString.str ());
	for (string line; std::getline (stream, line);)
	{
		ELEKTRA_LOG_DEBUG ("%s", line.c_str ());
	}

	ELEKTRA_LOG_DEBUG ("__________");
#endif

	ofstream output (parent.getString ());
	output << data;
}
<|endoftext|>


<commit_before>/*
Copyright (c) 2014 Microsoft Corporation. All rights reserved.
Released under Apache 2.0 license as described in the file LICENSE.

Author: Leonardo de Moura
*/
#include <limits>
#include <utility>
#include "util/pair.h"
#include "library/attribute_manager.h"
#include "frontends/lean/token_table.h"

namespace lean {
static unsigned g_arrow_prec      = 25;
static unsigned g_decreasing_prec = 100;
static unsigned g_max_prec        = 1024;
static unsigned g_Max_prec        = 1024*1024;
static unsigned g_plus_prec       = 65;
static unsigned g_cup_prec        = 60;
unsigned get_max_prec() { return g_max_prec; }
unsigned get_Max_prec() { return g_Max_prec; }
unsigned get_arrow_prec() { return g_arrow_prec; }
unsigned get_decreasing_prec() { return g_decreasing_prec; }
static token_table update(token_table const & s, char const * token, char const * val,
                          optional<unsigned> expr_prec, optional<unsigned> tac_prec) {
    lean_assert(expr_prec || tac_prec);
    token_info info(token, val, 0, 0);
    if (token_info const * old_info = find(s, token)) {
        info = info.update_expr_precedence(old_info->expr_precedence());
        info = info.update_tactic_precedence(old_info->tactic_precedence());
    }
    if (expr_prec)
        info = info.update_expr_precedence(*expr_prec);
    if (tac_prec)
        info = info.update_tactic_precedence(*tac_prec);
    return insert(s, token, info);
}
token_table add_command_token(token_table const & s, char const * token) {
    return insert(s, token, token_info(token));
}
token_table add_command_token(token_table const & s, char const * token, char const * val) {
    return insert(s, token, token_info(token, val));
}
token_table add_token(token_table const & s, char const * token, unsigned prec) {
    return update(s, token, token, optional<unsigned>(prec), optional<unsigned>());
}
token_table add_token(token_table const & s, char const * token, char const * val, unsigned prec) {
    return update(s, token, val, optional<unsigned>(prec), optional<unsigned>());
}
token_table add_tactic_token(token_table const & s, char const * token, unsigned prec) {
    return update(s, token, token, optional<unsigned>(), optional<unsigned>(prec));
}
token_table add_tactic_token(token_table const & s, char const * token, char const * val, unsigned prec) {
    return update(s, token, val, optional<unsigned>(), optional<unsigned>(prec));
}
token_table const * find(token_table const & s, char c) {
    return s.find(c);
}
token_info const * value_of(token_table const & s) {
    return s.value();
}
optional<unsigned> get_expr_precedence(token_table const & s, char const * token) {
    auto it = find(s, token);
    return it ? optional<unsigned>(it->expr_precedence()) : optional<unsigned>();
}
optional<unsigned> get_tactic_precedence(token_table const & s, char const * token) {
    auto it = find(s, token);
    return it ? optional<unsigned>(it->tactic_precedence()) : optional<unsigned>();
}
bool is_token(token_table const & s, char const * token) {
    return static_cast<bool>(find(s, token));
}
void for_each(token_table const & s, std::function<void(char const *, token_info const &)> const & fn) {
    s.for_each([&](unsigned num, char const * keys, token_info const & info) {
            buffer<char> str;
            str.append(num, keys);
            str.push_back(0);
            fn(str.data(), info);
        });
}

static char const * g_lambda_unicode     = "\u03BB";
static char const * g_pi_unicode         = "\u03A0";
static char const * g_forall_unicode     = "\u2200";
static char const * g_arrow_unicode      = "\u2192";
static char const * g_cup                = "\u2294";
static char const * g_qed_unicode        = "∎";
static char const * g_decreasing_unicode = "↓";

void init_token_table(token_table & t) {
    pair<char const *, unsigned> builtin[] =
        {{"fun", 0}, {"Pi", 0}, {"let", 0}, {"in", 0}, {"at", 0}, {"have", 0}, {"assert", 0}, {"suppose", 0}, {"show", 0}, {"suffices", 0}, {"obtain", 0},
         {"do", 0}, {"if", 0}, {"then", 0}, {"else", 0}, {"by", 0}, {"hiding", 0}, {"replacing", 0}, {"renaming", 0},
         {"from", 0}, {"(", g_max_prec}, {")", 0}, {"{", g_max_prec}, {"}", 0}, {"_", g_max_prec},
         {"[", g_max_prec}, {"]", 0}, {"⦃", g_max_prec}, {"⦄", 0}, {".{", 0}, {"Type", g_max_prec},
         {"{|", g_max_prec}, {"|}", 0}, {"(:", g_max_prec}, {":)", 0},
         {"⊢", 0}, {"⟨", g_max_prec}, {"⟩", 0}, {"^", 0}, {"↑", 0}, {"▸", 0},
         {"using", 0}, {"|", 0}, {"!", g_max_prec}, {"?", 0},  {"with", 0}, {"...", 0}, {",", 0},
         {".", 0}, {":", 0}, {"::", 0}, {"calc", 0}, {"rewrite", 0}, {"xrewrite", 0}, {"krewrite", 0},
         {"esimp", 0}, {"fold", 0}, {"unfold", 0}, {"note", 0}, {"with_options", 0}, {"with_attributes", 0}, {"with_attrs", 0},
         {"generalize", 0}, {"as", 0}, {":=", 0}, {"--", 0}, {"#", 0}, {"#tactic", 0},
         {"(*", 0}, {"/-", 0}, {"begin", g_max_prec}, {"abstract", g_max_prec},
         {"proof", g_max_prec}, {"qed", 0}, {"@@", g_max_prec}, {"@", g_max_prec},
         {"sorry", g_max_prec}, {"+", g_plus_prec}, {g_cup, g_cup_prec}, {"->", g_arrow_prec}, {"<-", 0},
         {"?(", g_max_prec}, {"⌞", g_max_prec}, {"⌟", 0}, {"match", 0},
         {"<d", g_decreasing_prec}, {"renaming", 0}, {"extends", 0}, {nullptr, 0}};

    char const * commands[] =
        {"theorem", "axiom", "axioms", "variable", "protected", "private", "reveal",
         "definition", "meta_definition", "example", "coercion", "abbreviation", "noncomputable",
         "variables", "parameter", "parameters", "constant", "meta_constant", "constants",
         "[visible]", "[none]", "[parsing_only]",
         "evaluate", "check", "eval", "vm_eval", "[wf]", "[whnf]", "[priority", "[unfold_hints]",
         "print", "end", "namespace", "section", "prelude", "help",
         "import", "inductive", "record", "structure", "module", "universe", "universes", "local",
         "precedence", "reserve", "infixl", "infixr", "infix", "postfix", "prefix", "notation",
         "tactic_infixl", "tactic_infixr", "tactic_infix", "tactic_postfix", "tactic_prefix", "tactic_notation",
         "exit", "set_option", "open", "export", "override", "tactic_hint",
         "add_begin_end_tactic", "set_begin_end_tactic", "instance", "class",
         "multiple_instances", "find_decl", "attribute", "persistent", "inline",
         "include", "omit", "migrate", "init_quotient", "init_hits", "#erase_cache", "#projections", "#telescope_eq",
         "#compile", "#decl_stats", "#relevant_thms", "#simplify", "#normalizer", "#abstract_expr", "#unify",
         "#defeq_simplify", "#elab", "#compile", nullptr};

    pair<char const *, char const *> aliases[] =
        {{g_lambda_unicode, "fun"}, {"forall", "Pi"}, {g_forall_unicode, "Pi"}, {g_pi_unicode, "Pi"},
         {g_qed_unicode, "qed"}, {nullptr, nullptr}};

    pair<char const *, char const *> cmd_aliases[] =
        {{"lemma", "theorem"}, {"proposition", "theorem"}, {"premise", "variable"}, {"premises", "variables"},
         {"corollary", "theorem"}, {"hypothesis", "parameter"}, {"conjecture", "parameter"},
         {"record", "structure"}, {nullptr, nullptr}};

    auto it = builtin;
    while (it->first) {
        t = add_token(t, it->first, it->second);
        it++;
    }

    auto it2 = commands;
    while (*it2) {
        t = add_command_token(t, *it2);
        ++it2;
    }

    buffer<char const *> attrs;
    get_attribute_tokens(attrs);
    for (char const * attr : attrs)
        t = add_command_token(t, attr);

    auto it3 = aliases;
    while (it3->first) {
        t = add_token(t, it3->first, it3->second, 0);
        it3++;
    }
    t = add_token(t, g_arrow_unicode, "->", get_arrow_prec());
    t = add_token(t, "←", "<-", 0);
    t = add_token(t, g_decreasing_unicode, "<d", get_decreasing_prec());

    auto it4 = cmd_aliases;
    while (it4->first) {
        t = add_command_token(t, it4->first, it4->second);
        ++it4;
    }
}

static token_table * g_default_token_table = nullptr;

token_table mk_default_token_table() {
    return *g_default_token_table;
}

void initialize_token_table() {
    g_default_token_table = new token_table();
    init_token_table(*g_default_token_table);
}

void finalize_token_table() {
    delete g_default_token_table;
}

token_table mk_token_table() { return token_table(); }
}
<commit_msg>chore(frontends/lean/token_table): remove unnecessary keywords (fold and unfold)<commit_after>/*
Copyright (c) 2014 Microsoft Corporation. All rights reserved.
Released under Apache 2.0 license as described in the file LICENSE.

Author: Leonardo de Moura
*/
#include <limits>
#include <utility>
#include "util/pair.h"
#include "library/attribute_manager.h"
#include "frontends/lean/token_table.h"

namespace lean {
static unsigned g_arrow_prec      = 25;
static unsigned g_decreasing_prec = 100;
static unsigned g_max_prec        = 1024;
static unsigned g_Max_prec        = 1024*1024;
static unsigned g_plus_prec       = 65;
static unsigned g_cup_prec        = 60;
unsigned get_max_prec() { return g_max_prec; }
unsigned get_Max_prec() { return g_Max_prec; }
unsigned get_arrow_prec() { return g_arrow_prec; }
unsigned get_decreasing_prec() { return g_decreasing_prec; }
static token_table update(token_table const & s, char const * token, char const * val,
                          optional<unsigned> expr_prec, optional<unsigned> tac_prec) {
    lean_assert(expr_prec || tac_prec);
    token_info info(token, val, 0, 0);
    if (token_info const * old_info = find(s, token)) {
        info = info.update_expr_precedence(old_info->expr_precedence());
        info = info.update_tactic_precedence(old_info->tactic_precedence());
    }
    if (expr_prec)
        info = info.update_expr_precedence(*expr_prec);
    if (tac_prec)
        info = info.update_tactic_precedence(*tac_prec);
    return insert(s, token, info);
}
token_table add_command_token(token_table const & s, char const * token) {
    return insert(s, token, token_info(token));
}
token_table add_command_token(token_table const & s, char const * token, char const * val) {
    return insert(s, token, token_info(token, val));
}
token_table add_token(token_table const & s, char const * token, unsigned prec) {
    return update(s, token, token, optional<unsigned>(prec), optional<unsigned>());
}
token_table add_token(token_table const & s, char const * token, char const * val, unsigned prec) {
    return update(s, token, val, optional<unsigned>(prec), optional<unsigned>());
}
token_table add_tactic_token(token_table const & s, char const * token, unsigned prec) {
    return update(s, token, token, optional<unsigned>(), optional<unsigned>(prec));
}
token_table add_tactic_token(token_table const & s, char const * token, char const * val, unsigned prec) {
    return update(s, token, val, optional<unsigned>(), optional<unsigned>(prec));
}
token_table const * find(token_table const & s, char c) {
    return s.find(c);
}
token_info const * value_of(token_table const & s) {
    return s.value();
}
optional<unsigned> get_expr_precedence(token_table const & s, char const * token) {
    auto it = find(s, token);
    return it ? optional<unsigned>(it->expr_precedence()) : optional<unsigned>();
}
optional<unsigned> get_tactic_precedence(token_table const & s, char const * token) {
    auto it = find(s, token);
    return it ? optional<unsigned>(it->tactic_precedence()) : optional<unsigned>();
}
bool is_token(token_table const & s, char const * token) {
    return static_cast<bool>(find(s, token));
}
void for_each(token_table const & s, std::function<void(char const *, token_info const &)> const & fn) {
    s.for_each([&](unsigned num, char const * keys, token_info const & info) {
            buffer<char> str;
            str.append(num, keys);
            str.push_back(0);
            fn(str.data(), info);
        });
}

static char const * g_lambda_unicode     = "\u03BB";
static char const * g_pi_unicode         = "\u03A0";
static char const * g_forall_unicode     = "\u2200";
static char const * g_arrow_unicode      = "\u2192";
static char const * g_cup                = "\u2294";
static char const * g_qed_unicode        = "∎";
static char const * g_decreasing_unicode = "↓";

void init_token_table(token_table & t) {
    pair<char const *, unsigned> builtin[] =
        {{"fun", 0}, {"Pi", 0}, {"let", 0}, {"in", 0}, {"at", 0}, {"have", 0}, {"assert", 0}, {"suppose", 0}, {"show", 0}, {"suffices", 0}, {"obtain", 0},
         {"do", 0}, {"if", 0}, {"then", 0}, {"else", 0}, {"by", 0}, {"hiding", 0}, {"replacing", 0}, {"renaming", 0},
         {"from", 0}, {"(", g_max_prec}, {")", 0}, {"{", g_max_prec}, {"}", 0}, {"_", g_max_prec},
         {"[", g_max_prec}, {"]", 0}, {"⦃", g_max_prec}, {"⦄", 0}, {".{", 0}, {"Type", g_max_prec},
         {"{|", g_max_prec}, {"|}", 0}, {"(:", g_max_prec}, {":)", 0},
         {"⊢", 0}, {"⟨", g_max_prec}, {"⟩", 0}, {"^", 0}, {"↑", 0}, {"▸", 0},
         {"using", 0}, {"|", 0}, {"!", g_max_prec}, {"?", 0},  {"with", 0}, {"...", 0}, {",", 0},
         {".", 0}, {":", 0}, {"::", 0}, {"calc", 0}, {"rewrite", 0}, {"xrewrite", 0}, {"krewrite", 0},
         {"esimp", 0}, {"note", 0}, {"with_options", 0}, {"with_attributes", 0}, {"with_attrs", 0},
         {"generalize", 0}, {"as", 0}, {":=", 0}, {"--", 0}, {"#", 0}, {"#tactic", 0},
         {"(*", 0}, {"/-", 0}, {"begin", g_max_prec}, {"abstract", g_max_prec},
         {"proof", g_max_prec}, {"qed", 0}, {"@@", g_max_prec}, {"@", g_max_prec},
         {"sorry", g_max_prec}, {"+", g_plus_prec}, {g_cup, g_cup_prec}, {"->", g_arrow_prec}, {"<-", 0},
         {"?(", g_max_prec}, {"⌞", g_max_prec}, {"⌟", 0}, {"match", 0},
         {"<d", g_decreasing_prec}, {"renaming", 0}, {"extends", 0}, {nullptr, 0}};

    char const * commands[] =
        {"theorem", "axiom", "axioms", "variable", "protected", "private", "reveal",
         "definition", "meta_definition", "example", "coercion", "abbreviation", "noncomputable",
         "variables", "parameter", "parameters", "constant", "meta_constant", "constants",
         "[visible]", "[none]", "[parsing_only]",
         "evaluate", "check", "eval", "vm_eval", "[wf]", "[whnf]", "[priority", "[unfold_hints]",
         "print", "end", "namespace", "section", "prelude", "help",
         "import", "inductive", "record", "structure", "module", "universe", "universes", "local",
         "precedence", "reserve", "infixl", "infixr", "infix", "postfix", "prefix", "notation",
         "tactic_infixl", "tactic_infixr", "tactic_infix", "tactic_postfix", "tactic_prefix", "tactic_notation",
         "exit", "set_option", "open", "export", "override", "tactic_hint",
         "add_begin_end_tactic", "set_begin_end_tactic", "instance", "class",
         "multiple_instances", "find_decl", "attribute", "persistent", "inline",
         "include", "omit", "migrate", "init_quotient", "init_hits", "#erase_cache", "#projections", "#telescope_eq",
         "#compile", "#decl_stats", "#relevant_thms", "#simplify", "#normalizer", "#abstract_expr", "#unify",
         "#defeq_simplify", "#elab", "#compile", nullptr};

    pair<char const *, char const *> aliases[] =
        {{g_lambda_unicode, "fun"}, {"forall", "Pi"}, {g_forall_unicode, "Pi"}, {g_pi_unicode, "Pi"},
         {g_qed_unicode, "qed"}, {nullptr, nullptr}};

    pair<char const *, char const *> cmd_aliases[] =
        {{"lemma", "theorem"}, {"proposition", "theorem"}, {"premise", "variable"}, {"premises", "variables"},
         {"corollary", "theorem"}, {"hypothesis", "parameter"}, {"conjecture", "parameter"},
         {"record", "structure"}, {nullptr, nullptr}};

    auto it = builtin;
    while (it->first) {
        t = add_token(t, it->first, it->second);
        it++;
    }

    auto it2 = commands;
    while (*it2) {
        t = add_command_token(t, *it2);
        ++it2;
    }

    buffer<char const *> attrs;
    get_attribute_tokens(attrs);
    for (char const * attr : attrs)
        t = add_command_token(t, attr);

    auto it3 = aliases;
    while (it3->first) {
        t = add_token(t, it3->first, it3->second, 0);
        it3++;
    }
    t = add_token(t, g_arrow_unicode, "->", get_arrow_prec());
    t = add_token(t, "←", "<-", 0);
    t = add_token(t, g_decreasing_unicode, "<d", get_decreasing_prec());

    auto it4 = cmd_aliases;
    while (it4->first) {
        t = add_command_token(t, it4->first, it4->second);
        ++it4;
    }
}

static token_table * g_default_token_table = nullptr;

token_table mk_default_token_table() {
    return *g_default_token_table;
}

void initialize_token_table() {
    g_default_token_table = new token_table();
    init_token_table(*g_default_token_table);
}

void finalize_token_table() {
    delete g_default_token_table;
}

token_table mk_token_table() { return token_table(); }
}
<|endoftext|>