<commit_before>// This file is distributed under the MIT license.
// See the LICENSE file for details.

#include <algorithm>
#include <cstring>
#include <exception>
#include <fstream>
#include <iostream>
#include <limits>
#include <map>
#include <memory>
#include <sstream>
#include <utility>

#include <boost/algorithm/string.hpp>
#include <boost/fusion/include/adapt_struct.hpp>
#include <boost/iostreams/device/mapped_file.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/utility/string_ref.hpp>
#include <boost/filesystem.hpp>
#include <boost/optional.hpp>

#include <visionaray/detail/macros.h>
#include <visionaray/math/math.h>
#include <visionaray/texture/texture.h>

#include "image.h"
#include "obj_loader.h"

namespace qi = boost::spirit::qi;

//-------------------------------------------------------------------------------------------------
// boost::fusion-adapt some structs for parsing
//

namespace visionaray
{
namespace detail
{

struct face_index_t
{
    int vertex_index;
    boost::optional<int> tex_coord_index;
    boost::optional<int> normal_index;
};

} // detail
} // visionaray

BOOST_FUSION_ADAPT_STRUCT
(
    visionaray::detail::face_index_t,
    (int, vertex_index)
    (boost::optional<int>, tex_coord_index)
    (boost::optional<int>, normal_index)
)

BOOST_FUSION_ADAPT_STRUCT
(
    visionaray::vec2,
    (float, x)
    (float, y)
)

BOOST_FUSION_ADAPT_STRUCT
(
    visionaray::vec3,
    (float, x)
    (float, y)
    (float, z)
)

using boost::string_ref;

namespace boost
{
namespace spirit
{
namespace traits
{

template <typename Iterator, typename Enable>
struct assign_to_attribute_from_iterators<string_ref, Iterator, Enable>
{
    static void call(Iterator const& first, Iterator const& last, string_ref& attr)
    {
        attr = { first, static_cast<size_t>(last - first) };
    }
};

} // traits
} // spirit
}

namespace visionaray
{

typedef basic_triangle<3, float> triangle_type;

namespace detail
{

//-------------------------------------------------------------------------------------------------
// aabb of a list of triangles
//

aabb bounds(detail::triangle_list const& tris)
{
    aabb result( vec3(std::numeric_limits<float>::max()), -vec3(std::numeric_limits<float>::max()) );

    for (auto const& tri : tris)
    {
        auto v1 = tri.v1;
        auto v2 = tri.v1 + tri.e1;
        auto v3 = tri.v1 + tri.e2;

        result = combine(result, aabb(v1, v1));
        result = combine(result, aabb(v2, v2));
        result = combine(result, aabb(v3, v3));
    }

    return result;
}

} // detail

struct mtl
{
    vec3 ka;
    vec3 kd;
    vec3 ks;
    float ns;
    std::string map_kd;
};

void parse_mtl(std::string const& filename, std::map<std::string, mtl>* matlib)
{
    std::ifstream ifstr(filename.c_str(), std::ifstream::in);
    std::string line;
    std::map<std::string, mtl>::iterator it;

    while (ifstr.good() && !ifstr.eof() && std::getline(ifstr, line))
    {
        boost::algorithm::trim(line);
        std::string identifier;
        std::istringstream str(line);
        str >> identifier >> std::ws;

        if (identifier == "newmtl")
        {
            std::string name;
            str >> name;
            auto mtl_pair = std::make_pair(name, mtl());
            matlib->insert(mtl_pair);
            it = matlib->find(name);
        }
        else if (identifier == "Ka")
        {
            str >> (*it).second.ka.x >> std::ws >> (*it).second.ka.y >> std::ws >> (*it).second.ka.z >> std::ws;
        }
        else if (identifier == "Kd")
        {
            str >> (*it).second.kd.x >> std::ws >> (*it).second.kd.y >> std::ws >> (*it).second.kd.z >> std::ws;
        }
        else if (identifier == "Ks")
        {
            str >> (*it).second.ks.x >> std::ws >> (*it).second.ks.y >> std::ws >> (*it).second.ks.z >> std::ws;
        }
        else if (identifier == "Ns")
        {
            str >> (*it).second.ns >> std::ws;
        }
        else if (identifier == "map_Kd")
        {
            str >> (*it).second.map_kd;
        }
    }
}

detail::obj_scene load_obj(std::string const& filename)
{
    std::map<std::string, mtl> matlib;

    std::ifstream ifstr(filename.c_str(), std::ifstream::in);
    boost::iostreams::mapped_file_source file(filename);

    unsigned geom_id = 0;

    using namespace detail;

    obj_scene result;

    aligned_vector<vec3> vertices;
    aligned_vector<vec2> tex_coords;
    aligned_vector<vec3> normals;

    using face_vector = aligned_vector<face_index_t>;

    // intermediate containers

    vec3 v;
    vec2 vt;
    vec3 vn;
    face_vector faces;
    string_ref comment;
    string_ref mtl_file;
    string_ref mtl_name;

    // helper functions

    auto remap_index = [](int idx, int size) -> int
    {
        return idx < 0 ? static_cast<int>(size) + idx : idx - 1;
    };

    auto store_triangle = [&](int i1, int i2, int i3)
    {
        triangle_type tri;
        tri.prim_id = static_cast<unsigned>(result.primitives.size());
        tri.geom_id = geom_id;
        tri.v1 = vertices[i1];
        tri.e1 = vertices[i2] - tri.v1;
        tri.e2 = vertices[i3] - tri.v1;
        result.primitives.push_back(tri);
    };

    auto store_faces = [&](int vertices_size, int tex_coords_size, int normals_size)
    {
        size_t last = 2;
        auto i1 = remap_index(faces[0].vertex_index, vertices_size);

        while (last != faces.size())
        {
            // triangle
            auto i2 = remap_index(faces[last - 1].vertex_index, vertices_size);
            auto i3 = remap_index(faces[last].vertex_index, vertices_size);
            store_triangle(i1, i2, i3);

            // texture coordinates
            if (faces[0].tex_coord_index && faces[last - 1].tex_coord_index && faces[last].tex_coord_index)
            {
                auto ti1 = remap_index(*faces[0].tex_coord_index, tex_coords_size);
                auto ti2 = remap_index(*faces[last - 1].tex_coord_index, tex_coords_size);
                auto ti3 = remap_index(*faces[last].tex_coord_index, tex_coords_size);

                result.tex_coords.push_back( tex_coords[ti1] );
                result.tex_coords.push_back( tex_coords[ti2] );
                result.tex_coords.push_back( tex_coords[ti3] );
            }

            // normals
            if (faces[0].normal_index && faces[last - 1].normal_index && faces[last].normal_index)
            {
                auto ni1 = remap_index(*faces[0].normal_index, normals_size);
                auto ni2 = remap_index(*faces[last - 1].normal_index, normals_size);
                auto ni3 = remap_index(*faces[last].normal_index, normals_size);

                result.normals.push_back( normals[ni1] );
                result.normals.push_back( normals[ni2] );
                result.normals.push_back( normals[ni3] );
            }

            ++last;
        }
    };

    // obj grammar

    using It = string_ref::const_iterator;
    using space_type = decltype(qi::blank);

    qi::rule<It> r_unhandled                            = *(qi::char_ - qi::eol)                                                    >> qi::eol;
    qi::rule<It, string_ref()> r_text_to_eol            = qi::raw[*(qi::char_ - qi::eol)];

    qi::rule<It, string_ref()> r_comment                = "#" >> r_text_to_eol                                                      >> qi::eol;
    qi::rule<It, string_ref(), space_type> r_mtllib     = "mtllib" >> r_text_to_eol                                                 >> qi::eol;
    qi::rule<It, string_ref(), space_type> r_usemtl     = "usemtl" >> r_text_to_eol                                                 >> qi::eol;

    qi::rule<It, vec3(), space_type> r_v                = "v" >> qi::float_ >> qi::float_ >> qi::float_                             >> qi::eol;
    qi::rule<It, vec2(), space_type> r_vt               = "vt" >> qi::float_ >> qi::float_ >> -qi::float_                           >> qi::eol;// TODO: parse optional 3rd (w)
    qi::rule<It, vec3(), space_type> r_vn               = "vn" >> qi::float_ >> qi::float_ >> qi::float_                            >> qi::eol;

    qi::rule<It, detail::face_index_t()> r_face_vertex  = qi::int_ >> -qi::lit("/") >> -qi::int_ >> -qi::lit("/") >> -qi::int_;
    qi::rule<It, face_vector(), space_type> r_face      = "f" >> r_face_vertex >> r_face_vertex >> r_face_vertex >> *r_face_vertex  >> qi::eol;

    string_ref text(file.data(), file.size());
    auto it = text.cbegin();

    while (it != text.cend())
    {
        faces.clear();

        if ( qi::phrase_parse(it, text.cend(), r_comment, qi::blank, comment) )
        {
            VSNRAY_UNUSED(comment);
            continue;
        }
        else if ( qi::phrase_parse(it, text.cend(), r_mtllib, qi::blank, mtl_file) )
        {
            boost::filesystem::path p(filename);
            std::string mtl_dir = p.parent_path().string();

            std::string mtl_path = mtl_dir + "/" + std::string(mtl_file);

            parse_mtl(mtl_path, &matlib);
        }
        else if ( qi::phrase_parse(it, text.cend(), r_usemtl, qi::blank, mtl_name) )
        {
            auto mat_it = matlib.find(std::string(mtl_name));
            if (mat_it != matlib.end())
            {
                phong<float> mat;
                mat.set_cd( mat_it->second.kd );
                mat.set_kd( 1.0f );
                mat.set_ks( 1.0f );
                mat.set_specular_exp( mat_it->second.ns );
                result.materials.push_back(mat);

                typedef tex_list::value_type tex_type;
                boost::filesystem::path p(filename);
                std::string tex_filename = p.parent_path().string() + "/" + mat_it->second.map_kd;

                static const std::string extensions[] = { ".jpg", ".jpeg", ".JPG", ".JPEG" };
                auto tex_path = boost::filesystem::path(tex_filename);
                auto has_jpg_ext = ( std::find(extensions, extensions + 4, tex_path.extension()) != extensions + 4 );

                if (!mat_it->second.map_kd.empty() && boost::filesystem::exists(tex_filename) && has_jpg_ext)
                {
#if defined(VSNRAY_HAVE_JPEG)
                    jpeg_image jpg(tex_filename);

                    tex_type tex(jpg.width(), jpg.height());
                    tex.set_address_mode( Clamp );
                    tex.set_filter_mode( Linear );

                    auto data_ptr = reinterpret_cast<tex_type::value_type const*>(jpg.data());
                    tex.set_data(data_ptr);

                    result.textures.push_back(std::move(tex));
#endif
                }
                else
                {
                    result.textures.push_back(tex_type(0, 0));
                }
            }
            geom_id = result.materials.size() == 0 ? 0 : static_cast<unsigned>(result.materials.size() - 1);
        }
        else if ( qi::phrase_parse(it, text.cend(), r_v, qi::blank, v) )
        {
            vertices.push_back(v);
        }
        else if ( qi::phrase_parse(it, text.cend(), r_vt, qi::blank, vt) )
        {
            tex_coords.push_back(vt);
        }
        else if ( qi::phrase_parse(it, text.cend(), r_vn, qi::blank, vn) )
        {
            normals.push_back(vn);
        }
        else if ( qi::phrase_parse(it, text.cend(), r_face, qi::blank, faces) )
        {
            store_faces(static_cast<int>(vertices.size()), static_cast<int>(tex_coords.size()), static_cast<int>(normals.size()));
        }
        else if ( qi::phrase_parse(it, text.cend(), r_unhandled, qi::blank) )
        {
            continue;
        }
    }

// TODO

    result.normals.resize(0); // TODO: support for vertex normals
    if (result.normals.size() == 0) // have no default normals
    {
        for (auto const& tri : result.primitives)
        {
            vec3 n = normalize( cross(tri.e1, tri.e2) );
            result.normals.push_back(n);
        }
    }

    if (result.materials.size() == 0)
    {
        for (unsigned i = 0; i <= geom_id; ++i)
        {
            phong<float> m;
            m.set_cd( vec3(0.8f, 0.8f, 0.8f) );
            m.set_kd( 1.0f );
            m.set_ks( 1.0f );
            m.set_specular_exp( 32.0f );
            result.materials.push_back(m);
        }
    }
// TODO

    result.bbox = bounds(result.primitives);
    return result;
}

} // visionaray
<commit_msg>Bulk load vertices, tex coords and normals<commit_after>// This file is distributed under the MIT license.
// See the LICENSE file for details.

#include <algorithm>
#include <cstring>
#include <exception>
#include <fstream>
#include <iostream>
#include <limits>
#include <map>
#include <memory>
#include <sstream>
#include <utility>

#include <boost/algorithm/string.hpp>
#include <boost/fusion/include/adapt_struct.hpp>
#include <boost/iostreams/device/mapped_file.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/utility/string_ref.hpp>
#include <boost/filesystem.hpp>
#include <boost/optional.hpp>

#include <visionaray/detail/macros.h>
#include <visionaray/math/math.h>
#include <visionaray/texture/texture.h>

#include "image.h"
#include "obj_loader.h"

namespace qi = boost::spirit::qi;

//-------------------------------------------------------------------------------------------------
// boost::fusion-adapt some structs for parsing
//

namespace visionaray
{
namespace detail
{

struct face_index_t
{
    int vertex_index;
    boost::optional<int> tex_coord_index;
    boost::optional<int> normal_index;
};

} // detail
} // visionaray

BOOST_FUSION_ADAPT_STRUCT
(
    visionaray::detail::face_index_t,
    (int, vertex_index)
    (boost::optional<int>, tex_coord_index)
    (boost::optional<int>, normal_index)
)

BOOST_FUSION_ADAPT_STRUCT
(
    visionaray::vec2,
    (float, x)
    (float, y)
)

BOOST_FUSION_ADAPT_STRUCT
(
    visionaray::vec3,
    (float, x)
    (float, y)
    (float, z)
)

using boost::string_ref;

namespace boost
{
namespace spirit
{
namespace traits
{

template <typename Iterator, typename Enable>
struct assign_to_attribute_from_iterators<string_ref, Iterator, Enable>
{
    static void call(Iterator const& first, Iterator const& last, string_ref& attr)
    {
        attr = { first, static_cast<size_t>(last - first) };
    }
};

} // traits
} // spirit
}

namespace visionaray
{

typedef basic_triangle<3, float> triangle_type;

namespace detail
{

//-------------------------------------------------------------------------------------------------
// aabb of a list of triangles
//

aabb bounds(detail::triangle_list const& tris)
{
    aabb result( vec3(std::numeric_limits<float>::max()), -vec3(std::numeric_limits<float>::max()) );

    for (auto const& tri : tris)
    {
        auto v1 = tri.v1;
        auto v2 = tri.v1 + tri.e1;
        auto v3 = tri.v1 + tri.e2;

        result = combine(result, aabb(v1, v1));
        result = combine(result, aabb(v2, v2));
        result = combine(result, aabb(v3, v3));
    }

    return result;
}

} // detail

struct mtl
{
    vec3 ka;
    vec3 kd;
    vec3 ks;
    float ns;
    std::string map_kd;
};

void parse_mtl(std::string const& filename, std::map<std::string, mtl>* matlib)
{
    std::ifstream ifstr(filename.c_str(), std::ifstream::in);
    std::string line;
    std::map<std::string, mtl>::iterator it;

    while (ifstr.good() && !ifstr.eof() && std::getline(ifstr, line))
    {
        boost::algorithm::trim(line);
        std::string identifier;
        std::istringstream str(line);
        str >> identifier >> std::ws;

        if (identifier == "newmtl")
        {
            std::string name;
            str >> name;
            auto mtl_pair = std::make_pair(name, mtl());
            matlib->insert(mtl_pair);
            it = matlib->find(name);
        }
        else if (identifier == "Ka")
        {
            str >> (*it).second.ka.x >> std::ws >> (*it).second.ka.y >> std::ws >> (*it).second.ka.z >> std::ws;
        }
        else if (identifier == "Kd")
        {
            str >> (*it).second.kd.x >> std::ws >> (*it).second.kd.y >> std::ws >> (*it).second.kd.z >> std::ws;
        }
        else if (identifier == "Ks")
        {
            str >> (*it).second.ks.x >> std::ws >> (*it).second.ks.y >> std::ws >> (*it).second.ks.z >> std::ws;
        }
        else if (identifier == "Ns")
        {
            str >> (*it).second.ns >> std::ws;
        }
        else if (identifier == "map_Kd")
        {
            str >> (*it).second.map_kd;
        }
    }
}

detail::obj_scene load_obj(std::string const& filename)
{
    std::map<std::string, mtl> matlib;

    std::ifstream ifstr(filename.c_str(), std::ifstream::in);
    boost::iostreams::mapped_file_source file(filename);

    unsigned geom_id = 0;

    using namespace detail;

    obj_scene result;

    using vertex_vector     = aligned_vector<vec3>;
    using tex_coord_vector  = aligned_vector<vec2>;
    using normal_vector     = aligned_vector<vec3>;
    using face_vector       = aligned_vector<face_index_t>;

    vertex_vector       vertices;
    tex_coord_vector    tex_coords;
    normal_vector       normals;
    face_vector         faces;

    // intermediate containers

    string_ref comment;
    string_ref mtl_file;
    string_ref mtl_name;

    // helper functions

    auto remap_index = [](int idx, int size) -> int
    {
        return idx < 0 ? static_cast<int>(size) + idx : idx - 1;
    };

    auto store_triangle = [&](int i1, int i2, int i3)
    {
        triangle_type tri;
        tri.prim_id = static_cast<unsigned>(result.primitives.size());
        tri.geom_id = geom_id;
        tri.v1 = vertices[i1];
        tri.e1 = vertices[i2] - tri.v1;
        tri.e2 = vertices[i3] - tri.v1;
        result.primitives.push_back(tri);
    };

    auto store_faces = [&](int vertices_size, int tex_coords_size, int normals_size)
    {
        size_t last = 2;
        auto i1 = remap_index(faces[0].vertex_index, vertices_size);

        while (last != faces.size())
        {
            // triangle
            auto i2 = remap_index(faces[last - 1].vertex_index, vertices_size);
            auto i3 = remap_index(faces[last].vertex_index, vertices_size);
            store_triangle(i1, i2, i3);

            // texture coordinates
            if (faces[0].tex_coord_index && faces[last - 1].tex_coord_index && faces[last].tex_coord_index)
            {
                auto ti1 = remap_index(*faces[0].tex_coord_index, tex_coords_size);
                auto ti2 = remap_index(*faces[last - 1].tex_coord_index, tex_coords_size);
                auto ti3 = remap_index(*faces[last].tex_coord_index, tex_coords_size);

                result.tex_coords.push_back( tex_coords[ti1] );
                result.tex_coords.push_back( tex_coords[ti2] );
                result.tex_coords.push_back( tex_coords[ti3] );
            }

            // normals
            if (faces[0].normal_index && faces[last - 1].normal_index && faces[last].normal_index)
            {
                auto ni1 = remap_index(*faces[0].normal_index, normals_size);
                auto ni2 = remap_index(*faces[last - 1].normal_index, normals_size);
                auto ni3 = remap_index(*faces[last].normal_index, normals_size);

                result.normals.push_back( normals[ni1] );
                result.normals.push_back( normals[ni2] );
                result.normals.push_back( normals[ni3] );
            }

            ++last;
        }
    };

    // obj grammar

    using It = string_ref::const_iterator;
    using space_type = decltype(qi::blank);

    qi::rule<It> r_unhandled                                    = *(qi::char_ - qi::eol)                                                    >> qi::eol;
    qi::rule<It, string_ref()> r_text_to_eol                    = qi::raw[*(qi::char_ - qi::eol)];

    qi::rule<It, string_ref()> r_comment                        = "#" >> r_text_to_eol                                                      >> qi::eol;
    qi::rule<It, string_ref(), space_type> r_mtllib             = "mtllib" >> r_text_to_eol                                                 >> qi::eol;
    qi::rule<It, string_ref(), space_type> r_usemtl             = "usemtl" >> r_text_to_eol                                                 >> qi::eol;

    qi::rule<It, vec3(), space_type> r_v                        = "v" >> qi::float_ >> qi::float_ >> qi::float_                             >> qi::eol;
    qi::rule<It, vec2(), space_type> r_vt                       = "vt" >> qi::float_ >> qi::float_ >> -qi::float_                           >> qi::eol;// TODO: parse optional 3rd (w)
    qi::rule<It, vec3(), space_type> r_vn                       = "vn" >> qi::float_ >> qi::float_ >> qi::float_                            >> qi::eol;

    qi::rule<It, vertex_vector(), space_type> r_vertices        = r_v >> *r_v;
    qi::rule<It, tex_coord_vector(), space_type> r_tex_coords   = r_vt >> *r_vt;
    qi::rule<It, normal_vector(), space_type> r_normals         = r_vn >> *r_vn;

    qi::rule<It, detail::face_index_t()> r_face_vertex          = qi::int_ >> -qi::lit("/") >> -qi::int_ >> -qi::lit("/") >> -qi::int_;
    qi::rule<It, face_vector(), space_type> r_face              = "f" >> r_face_vertex >> r_face_vertex >> r_face_vertex >> *r_face_vertex  >> qi::eol;

    string_ref text(file.data(), file.size());
    auto it = text.cbegin();

    while (it != text.cend())
    {
        faces.clear();

        if ( qi::phrase_parse(it, text.cend(), r_comment, qi::blank, comment) )
        {
            VSNRAY_UNUSED(comment);
            continue;
        }
        else if ( qi::phrase_parse(it, text.cend(), r_mtllib, qi::blank, mtl_file) )
        {
            boost::filesystem::path p(filename);
            std::string mtl_dir = p.parent_path().string();

            std::string mtl_path = mtl_dir + "/" + std::string(mtl_file);

            parse_mtl(mtl_path, &matlib);
        }
        else if ( qi::phrase_parse(it, text.cend(), r_usemtl, qi::blank, mtl_name) )
        {
            auto mat_it = matlib.find(std::string(mtl_name));
            if (mat_it != matlib.end())
            {
                phong<float> mat;
                mat.set_cd( mat_it->second.kd );
                mat.set_kd( 1.0f );
                mat.set_ks( 1.0f );
                mat.set_specular_exp( mat_it->second.ns );
                result.materials.push_back(mat);

                typedef tex_list::value_type tex_type;
                boost::filesystem::path p(filename);
                std::string tex_filename = p.parent_path().string() + "/" + mat_it->second.map_kd;

                static const std::string extensions[] = { ".jpg", ".jpeg", ".JPG", ".JPEG" };
                auto tex_path = boost::filesystem::path(tex_filename);
                auto has_jpg_ext = ( std::find(extensions, extensions + 4, tex_path.extension()) != extensions + 4 );

                if (!mat_it->second.map_kd.empty() && boost::filesystem::exists(tex_filename) && has_jpg_ext)
                {
#if defined(VSNRAY_HAVE_JPEG)
                    jpeg_image jpg(tex_filename);

                    tex_type tex(jpg.width(), jpg.height());
                    tex.set_address_mode( Clamp );
                    tex.set_filter_mode( Linear );

                    auto data_ptr = reinterpret_cast<tex_type::value_type const*>(jpg.data());
                    tex.set_data(data_ptr);

                    result.textures.push_back(std::move(tex));
#endif
                }
                else
                {
                    result.textures.push_back(tex_type(0, 0));
                }
            }
            geom_id = result.materials.size() == 0 ? 0 : static_cast<unsigned>(result.materials.size() - 1);
        }
        else if ( qi::phrase_parse(it, text.cend(), r_vertices, qi::blank, vertices) )
        {
        }
        else if ( qi::phrase_parse(it, text.cend(), r_tex_coords, qi::blank, tex_coords) )
        {
        }
        else if ( qi::phrase_parse(it, text.cend(), r_normals, qi::blank, normals) )
        {
        }
        else if ( qi::phrase_parse(it, text.cend(), r_face, qi::blank, faces) )
        {
            store_faces(static_cast<int>(vertices.size()), static_cast<int>(tex_coords.size()), static_cast<int>(normals.size()));
        }
        else if ( qi::phrase_parse(it, text.cend(), r_unhandled, qi::blank) )
        {
            continue;
        }
    }

// TODO

    result.normals.resize(0); // TODO: support for vertex normals
    if (result.normals.size() == 0) // have no default normals
    {
        for (auto const& tri : result.primitives)
        {
            vec3 n = normalize( cross(tri.e1, tri.e2) );
            result.normals.push_back(n);
        }
    }

    if (result.materials.size() == 0)
    {
        for (unsigned i = 0; i <= geom_id; ++i)
        {
            phong<float> m;
            m.set_cd( vec3(0.8f, 0.8f, 0.8f) );
            m.set_kd( 1.0f );
            m.set_ks( 1.0f );
            m.set_specular_exp( 32.0f );
            result.materials.push_back(m);
        }
    }
// TODO

    result.bbox = bounds(result.primitives);
    return result;
}

} // visionaray
<|endoftext|>


<commit_before>// MemCheck.cpp (Oclgrind)
// Copyright (c) 2013-2015, James Price and Simon McIntosh-Smith,
// University of Bristol. All rights reserved.
//
// This program is provided under a three-clause BSD license. For full
// license terms please see the LICENSE file distributed with this
// source code.

#include "core/common.h"

#include "core/Context.h"
#include "core/Memory.h"
#include "core/WorkItem.h"

#include "llvm/IR/Instructions.h"
#include "llvm/IR/Type.h"

#include "MemCheck.h"

using namespace oclgrind;
using namespace std;

MemCheck::MemCheck(const Context *context)
 : Plugin(context)
{
}

void MemCheck::instructionExecuted(const WorkItem *workItem,
                                   const llvm::Instruction *instruction,
                                   const TypedValue& result)
{
  // Check static array bounds if load or store is executed
  const llvm::Value *PtrOp = nullptr;

  if (auto LI = llvm::dyn_cast<llvm::LoadInst>(instruction))
  {
    PtrOp = LI->getPointerOperand();
  }
  else if (auto SI = llvm::dyn_cast<llvm::StoreInst>(instruction))
  {
    PtrOp = SI->getPointerOperand();
  }
  else
  {
    return;
  }

  if (auto GEPI = llvm::dyn_cast<llvm::GetElementPtrInst>(PtrOp->stripPointerCasts()))
  {
    checkArrayAccess(workItem, GEPI);
  }
}

void MemCheck::memoryAtomicLoad(const Memory *memory,
                                const WorkItem *workItem,
                                AtomicOp op, size_t address, size_t size)
{
  checkLoad(memory, address, size);
}

void MemCheck::memoryAtomicStore(const Memory *memory,
                                 const WorkItem *workItem,
                                 AtomicOp op, size_t address, size_t size)
{
  checkStore(memory, address, size);
}

void MemCheck::memoryLoad(const Memory *memory, const WorkItem *workItem,
                          size_t address, size_t size)
{
  checkLoad(memory, address, size);
}

void MemCheck::memoryLoad(const Memory *memory, const WorkGroup *workGroup,
                          size_t address, size_t size)
{
  checkLoad(memory, address, size);
}

void MemCheck::memoryMap(const Memory *memory, size_t address,
                         size_t offset, size_t size, cl_map_flags flags)
{
  MapRegion map =
  {
    address, offset, size, memory->getPointer(address + offset),
    (flags == CL_MAP_READ ? MapRegion::READ : MapRegion::WRITE)
  };
  m_mapRegions.push_back(map);
}

void MemCheck::memoryStore(const Memory *memory, const WorkItem *workItem,
                           size_t address, size_t size,
                           const uint8_t *storeData)
{
  checkStore(memory, address, size);
}

void MemCheck::memoryStore(const Memory *memory, const WorkGroup *workGroup,
                           size_t address, size_t size,
                           const uint8_t *storeData)
{
  checkStore(memory, address, size);
}

void MemCheck::memoryUnmap(const Memory *memory, size_t address,
                           const void *ptr)
{
  for (auto region = m_mapRegions.begin();
            region != m_mapRegions.end();
            region++)
  {
    if (region->ptr == ptr)
    {
      m_mapRegions.erase(region);
      return;
    }
  }
}

void MemCheck::checkArrayAccess(const WorkItem *workItem,
                                const llvm::GetElementPtrInst *GEPI) const
{
  // Iterate through GEPI indices
  const llvm::Type *ptrType = GEPI->getPointerOperandType();

  for (auto opIndex = GEPI->idx_begin(); opIndex != GEPI->idx_end(); opIndex++)
  {
    int64_t index = workItem->getOperand(opIndex->get()).getSInt();

    if (ptrType->isArrayTy())
    {
      // Check index doesn't exceed size of array
      uint64_t size = ptrType->getArrayNumElements();

      if ((uint64_t)index >= size)
      {
        ostringstream info;
        info << "Index ("
             << index << ") exceeds static array size ("
             << size << ")";
        m_context->logError(info.str().c_str());
      }

      ptrType = ptrType->getArrayElementType();
    }
    else if (ptrType->isPointerTy())
    {
      ptrType = ptrType->getPointerElementType();
    }
    else if (ptrType->isVectorTy())
    {
      ptrType = ptrType->getVectorElementType();
    }
    else if (ptrType->isStructTy())
    {
      ptrType = ptrType->getStructElementType(index);
    }
  }
}

void MemCheck::checkLoad(const Memory *memory,
                         size_t address, size_t size) const
{
  if (!memory->isAddressValid(address, size))
  {
    logInvalidAccess(true, memory->getAddressSpace(), address, size);
    return;
  }

  if (memory->getBuffer(address)->flags & CL_MEM_WRITE_ONLY)
  {
    m_context->logError("Invalid read from write-only buffer");
  }

  // Check if memory location is currently mapped for writing
  for (auto region = m_mapRegions.begin();
            region != m_mapRegions.end();
            region++)
  {
    if (region->type == MapRegion::WRITE &&
        address < region->address + region->size &&
        address + size >= region->address)
    {
      m_context->logError("Invalid read from buffer mapped for writing");
    }
  }
}

void MemCheck::checkStore(const Memory *memory,
                          size_t address, size_t size) const
{
  if (!memory->isAddressValid(address, size))
  {
    logInvalidAccess(false, memory->getAddressSpace(), address, size);
    return;
  }

  if (memory->getBuffer(address)->flags & CL_MEM_READ_ONLY)
  {
    m_context->logError("Invalid write to read-only buffer");
  }

  // Check if memory location is currently mapped
  for (auto region = m_mapRegions.begin();
            region != m_mapRegions.end();
            region++)
  {
    if (address < region->address + region->size &&
        address + size >= region->address)
    {
      m_context->logError("Invalid write to mapped buffer");
    }
  }
}

void MemCheck::logInvalidAccess(bool read, unsigned addrSpace,
                                size_t address, size_t size) const
{
  Context::Message msg(ERROR, m_context);
  msg << "Invalid " << (read ? "read" : "write")
      << " of size " << size
      << " at " << getAddressSpaceName(addrSpace)
      << " memory address 0x" << hex << address << endl
      << msg.INDENT
      << "Kernel: " << msg.CURRENT_KERNEL << endl
      << "Entity: " << msg.CURRENT_ENTITY << endl
      << msg.CURRENT_LOCATION << endl;
  msg.send();
}
<commit_msg>Check full chain of GEP instructions for static arrays<commit_after>// MemCheck.cpp (Oclgrind)
// Copyright (c) 2013-2015, James Price and Simon McIntosh-Smith,
// University of Bristol. All rights reserved.
//
// This program is provided under a three-clause BSD license. For full
// license terms please see the LICENSE file distributed with this
// source code.

#include "core/common.h"

#include "core/Context.h"
#include "core/Memory.h"
#include "core/WorkItem.h"

#include "llvm/IR/Instructions.h"
#include "llvm/IR/Type.h"

#include "MemCheck.h"

using namespace oclgrind;
using namespace std;

MemCheck::MemCheck(const Context *context)
 : Plugin(context)
{
}

void MemCheck::instructionExecuted(const WorkItem *workItem,
                                   const llvm::Instruction *instruction,
                                   const TypedValue& result)
{
  // Check static array bounds if load or store is executed
  const llvm::Value *PtrOp = nullptr;

  if (auto LI = llvm::dyn_cast<llvm::LoadInst>(instruction))
  {
    PtrOp = LI->getPointerOperand();
  }
  else if (auto SI = llvm::dyn_cast<llvm::StoreInst>(instruction))
  {
    PtrOp = SI->getPointerOperand();
  }
  else
  {
    return;
  }

  // Walk up chain of GEP instructions leading to this access
  while (auto GEPI =
           llvm::dyn_cast<llvm::GetElementPtrInst>(PtrOp->stripPointerCasts()))
  {
    checkArrayAccess(workItem, GEPI);

    PtrOp = GEPI->getPointerOperand();
  }
}

void MemCheck::memoryAtomicLoad(const Memory *memory,
                                const WorkItem *workItem,
                                AtomicOp op, size_t address, size_t size)
{
  checkLoad(memory, address, size);
}

void MemCheck::memoryAtomicStore(const Memory *memory,
                                 const WorkItem *workItem,
                                 AtomicOp op, size_t address, size_t size)
{
  checkStore(memory, address, size);
}

void MemCheck::memoryLoad(const Memory *memory, const WorkItem *workItem,
                          size_t address, size_t size)
{
  checkLoad(memory, address, size);
}

void MemCheck::memoryLoad(const Memory *memory, const WorkGroup *workGroup,
                          size_t address, size_t size)
{
  checkLoad(memory, address, size);
}

void MemCheck::memoryMap(const Memory *memory, size_t address,
                         size_t offset, size_t size, cl_map_flags flags)
{
  MapRegion map =
  {
    address, offset, size, memory->getPointer(address + offset),
    (flags == CL_MAP_READ ? MapRegion::READ : MapRegion::WRITE)
  };
  m_mapRegions.push_back(map);
}

void MemCheck::memoryStore(const Memory *memory, const WorkItem *workItem,
                           size_t address, size_t size,
                           const uint8_t *storeData)
{
  checkStore(memory, address, size);
}

void MemCheck::memoryStore(const Memory *memory, const WorkGroup *workGroup,
                           size_t address, size_t size,
                           const uint8_t *storeData)
{
  checkStore(memory, address, size);
}

void MemCheck::memoryUnmap(const Memory *memory, size_t address,
                           const void *ptr)
{
  for (auto region = m_mapRegions.begin();
            region != m_mapRegions.end();
            region++)
  {
    if (region->ptr == ptr)
    {
      m_mapRegions.erase(region);
      return;
    }
  }
}

void MemCheck::checkArrayAccess(const WorkItem *workItem,
                                const llvm::GetElementPtrInst *GEPI) const
{
  // Iterate through GEPI indices
  const llvm::Type *ptrType = GEPI->getPointerOperandType();

  for (auto opIndex = GEPI->idx_begin(); opIndex != GEPI->idx_end(); opIndex++)
  {
    int64_t index = workItem->getOperand(opIndex->get()).getSInt();

    if (ptrType->isArrayTy())
    {
      // Check index doesn't exceed size of array
      uint64_t size = ptrType->getArrayNumElements();

      if ((uint64_t)index >= size)
      {
        ostringstream info;
        info << "Index ("
             << index << ") exceeds static array size ("
             << size << ")";
        m_context->logError(info.str().c_str());
      }

      ptrType = ptrType->getArrayElementType();
    }
    else if (ptrType->isPointerTy())
    {
      ptrType = ptrType->getPointerElementType();
    }
    else if (ptrType->isVectorTy())
    {
      ptrType = ptrType->getVectorElementType();
    }
    else if (ptrType->isStructTy())
    {
      ptrType = ptrType->getStructElementType(index);
    }
  }
}

void MemCheck::checkLoad(const Memory *memory,
                         size_t address, size_t size) const
{
  if (!memory->isAddressValid(address, size))
  {
    logInvalidAccess(true, memory->getAddressSpace(), address, size);
    return;
  }

  if (memory->getBuffer(address)->flags & CL_MEM_WRITE_ONLY)
  {
    m_context->logError("Invalid read from write-only buffer");
  }

  // Check if memory location is currently mapped for writing
  for (auto region = m_mapRegions.begin();
            region != m_mapRegions.end();
            region++)
  {
    if (region->type == MapRegion::WRITE &&
        address < region->address + region->size &&
        address + size >= region->address)
    {
      m_context->logError("Invalid read from buffer mapped for writing");
    }
  }
}

void MemCheck::checkStore(const Memory *memory,
                          size_t address, size_t size) const
{
  if (!memory->isAddressValid(address, size))
  {
    logInvalidAccess(false, memory->getAddressSpace(), address, size);
    return;
  }

  if (memory->getBuffer(address)->flags & CL_MEM_READ_ONLY)
  {
    m_context->logError("Invalid write to read-only buffer");
  }

  // Check if memory location is currently mapped
  for (auto region = m_mapRegions.begin();
            region != m_mapRegions.end();
            region++)
  {
    if (address < region->address + region->size &&
        address + size >= region->address)
    {
      m_context->logError("Invalid write to mapped buffer");
    }
  }
}

void MemCheck::logInvalidAccess(bool read, unsigned addrSpace,
                                size_t address, size_t size) const
{
  Context::Message msg(ERROR, m_context);
  msg << "Invalid " << (read ? "read" : "write")
      << " of size " << size
      << " at " << getAddressSpaceName(addrSpace)
      << " memory address 0x" << hex << address << endl
      << msg.INDENT
      << "Kernel: " << msg.CURRENT_KERNEL << endl
      << "Entity: " << msg.CURRENT_ENTITY << endl
      << msg.CURRENT_LOCATION << endl;
  msg.send();
}
<|endoftext|>


<commit_before>/* <x0/plugins/compress.cpp>
 *
 * This file is part of the x0 web server project and is released under LGPL-3.
 * http://www.xzero.ws/
 *
 * (c) 2009-2010 Christian Parpart <trapni@gentoo.org>
 */

#include "compress.h"
#include <x0/http/HttpPlugin.h>
#include <x0/http/HttpServer.h>
#include <x0/http/HttpRequest.h>
#include <x0/http/HttpResponse.h>
#include <x0/http/HttpRangeDef.h>
#include <x0/io/CompressFilter.h>
#include <x0/strutils.h>
#include <x0/Types.h>
#include <x0/sysconfig.h>

#include <boost/lexical_cast.hpp>
#include <boost/tokenizer.hpp>

#include <sstream>
#include <iostream>
#include <algorithm>

#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <fcntl.h>
#include <time.h>

#if !defined(HAVE_BZLIB_H) && !defined(HAVE_ZLIB_H)
#	warning No compression headers available!
#endif

/**
 * \ingroup plugins
 * \brief serves static files from server's local filesystem to client.
 */
class compress_plugin :
	public x0::HttpPlugin,
	public ICompressPlugin
{
private:
	struct context : public x0::ScopeValue
	{
		std::vector<std::string> content_types_;
		int level_;
		long long min_size_;
		long long max_size_;

		context() :
			content_types_(),				// no types
			level_(9),						// best compression
			min_size_(1024),				// 1 KB
			max_size_(128 * 1024 * 1024)	// 128 MB
		{
		}

		bool contains_mime(const std::string& value) const
		{
			for (auto i = content_types_.begin(), e = content_types_.end(); i != e; ++i)
				if (*i == value)
					return true;

			return false;
		}

		virtual void merge(const x0::ScopeValue *value)
		{
			//if (auto cx = dynamic_cast<const context *>(value))
			{
				; //! \todo
			}
		}
	};

	x0::HttpServer::RequestPostHook::Connection postProcess_;

public:
	compress_plugin(x0::HttpServer& srv, const std::string& name) :
		x0::HttpPlugin(srv, name)
	{
		using namespace std::placeholders;

		postProcess_ = server_.onPostProcess.connect<compress_plugin, &compress_plugin::postProcess>(this);

		declareCVar("CompressTypes", x0::HttpContext::server, &compress_plugin::setup_types);
		declareCVar("CompressLevel", x0::HttpContext::server, &compress_plugin::setup_level);
		declareCVar("CompressMinSize", x0::HttpContext::server, &compress_plugin::setup_minsize);
		declareCVar("CompressMaxSize", x0::HttpContext::server, &compress_plugin::setup_maxsize);
	}

	~compress_plugin() {
		server_.onPostProcess.disconnect(postProcess_);
		server().release(this);
	}

public: // ICompressPlugin
	virtual void setCompressTypes(const std::vector<std::string>& value)
	{
		server().acquire<context>(this)->content_types_ = value;
	}

	virtual void setCompressLevel(int value)
	{
		server().acquire<context>(this)->level_ = value;
	}

	virtual void setCompressMinSize(int value)
	{
		server().acquire<context>(this)->min_size_ = value;
	}

	virtual void setCompressMaxSize(int value)
	{
		server().acquire<context>(this)->max_size_ = value;
	}

private:
	std::error_code setup_types(const x0::SettingsValue& cvar, x0::Scope& s)
	{
		return cvar.load(s.acquire<context>(this)->content_types_);
	}

	std::error_code setup_level(const x0::SettingsValue& cvar, x0::Scope& s)
	{
		return cvar.load(s.acquire<context>(this)->level_);
	}

	std::error_code setup_minsize(const x0::SettingsValue& cvar, x0::Scope& s)
	{
		return cvar.load(s.acquire<context>(this)->min_size_);
	}

	std::error_code setup_maxsize(const x0::SettingsValue& cvar, x0::Scope& s)
	{
		return cvar.load(s.acquire<context>(this)->max_size_);
	}

	void postProcess(x0::HttpRequest *in, x0::HttpResponse *out)
	{
		if (out->headers.contains("Content-Encoding"))
			return; // do not double-encode content

		const context *cx = server_.resolveHost(in->hostid())->get<context>(this);
		if (!cx && !(cx = server().get<context>(this)))
			return;

		long long size = 0;
		if (out->headers.contains("Content-Length"))
			size = boost::lexical_cast<int>(out->headers["Content-Length"]);

		if (size < cx->min_size_)
			return;

		if (size > cx->max_size_)
			return;

		if (!cx->contains_mime(out->headers("Content-Type")))
			return;

		if (x0::BufferRef r = in->header("Accept-Encoding"))
		{
			typedef boost::tokenizer<boost::char_separator<char>> tokenizer;

			std::vector<std::string> items(x0::split<std::string>(r.str(), ", "));

#if defined(HAVE_BZLIB_H)
			if (std::find(items.begin(), items.end(), "bzip2") != items.end())
			{
				out->headers.push_back("Content-Encoding", "bzip2");
				out->filters.push_back(std::make_shared<x0::BZip2Filter>(cx->level_));
			}
			else
#endif
#if defined(HAVE_ZLIB_H)
			if (std::find(items.begin(), items.end(), "gzip") != items.end())
			{
				out->headers.push_back("Content-Encoding", "gzip");
				out->filters.push_back(std::make_shared<x0::GZipFilter>(cx->level_));
			}
			else if (std::find(items.begin(), items.end(), "deflate") != items.end())
			{
				out->headers.push_back("Content-Encoding", "deflate");
				out->filters.push_back(std::make_shared<x0::DeflateFilter>(cx->level_));
			}
			else
#endif
				return;

			// response might change according to Accept-Encoding
			if (!out->headers.contains("Vary"))
				out->headers.push_back("Vary", "Accept-Encoding");
			else
				out->headers["Vary"] += ",Accept-Encoding";

			// removing content-length implicitely enables chunked encoding
			out->headers.remove("Content-Length");
		}
	}
};

X0_EXPORT_PLUGIN(compress);
<commit_msg>[plugins] compress: honor chunked encoding and compress those, too (ignoring min-size)<commit_after>/* <x0/plugins/compress.cpp>
 *
 * This file is part of the x0 web server project and is released under LGPL-3.
 * http://www.xzero.ws/
 *
 * (c) 2009-2010 Christian Parpart <trapni@gentoo.org>
 */

#include "compress.h"
#include <x0/http/HttpPlugin.h>
#include <x0/http/HttpServer.h>
#include <x0/http/HttpRequest.h>
#include <x0/http/HttpResponse.h>
#include <x0/http/HttpRangeDef.h>
#include <x0/io/CompressFilter.h>
#include <x0/strutils.h>
#include <x0/Types.h>
#include <x0/sysconfig.h>

#include <boost/lexical_cast.hpp>
#include <boost/tokenizer.hpp>

#include <sstream>
#include <iostream>
#include <algorithm>

#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <fcntl.h>
#include <time.h>

#if !defined(HAVE_BZLIB_H) && !defined(HAVE_ZLIB_H)
#	warning No compression headers available!
#endif

/**
 * \ingroup plugins
 * \brief serves static files from server's local filesystem to client.
 */
class compress_plugin :
	public x0::HttpPlugin,
	public ICompressPlugin
{
private:
	struct context : public x0::ScopeValue
	{
		std::vector<std::string> content_types_;
		int level_;
		long long min_size_;
		long long max_size_;

		context() :
			content_types_(),				// no types
			level_(9),						// best compression
			min_size_(1024),				// 1 KB
			max_size_(128 * 1024 * 1024)	// 128 MB
		{
		}

		bool contains_mime(const std::string& value) const
		{
			for (auto i = content_types_.begin(), e = content_types_.end(); i != e; ++i)
				if (*i == value)
					return true;

			return false;
		}

		virtual void merge(const x0::ScopeValue *value)
		{
			//if (auto cx = dynamic_cast<const context *>(value))
			{
				; //! \todo
			}
		}
	};

	x0::HttpServer::RequestPostHook::Connection postProcess_;

public:
	compress_plugin(x0::HttpServer& srv, const std::string& name) :
		x0::HttpPlugin(srv, name)
	{
		using namespace std::placeholders;

		postProcess_ = server_.onPostProcess.connect<compress_plugin, &compress_plugin::postProcess>(this);

		declareCVar("CompressTypes", x0::HttpContext::server, &compress_plugin::setup_types);
		declareCVar("CompressLevel", x0::HttpContext::server, &compress_plugin::setup_level);
		declareCVar("CompressMinSize", x0::HttpContext::server, &compress_plugin::setup_minsize);
		declareCVar("CompressMaxSize", x0::HttpContext::server, &compress_plugin::setup_maxsize);
	}

	~compress_plugin() {
		server_.onPostProcess.disconnect(postProcess_);
		server().release(this);
	}

public: // ICompressPlugin
	virtual void setCompressTypes(const std::vector<std::string>& value)
	{
		server().acquire<context>(this)->content_types_ = value;
	}

	virtual void setCompressLevel(int value)
	{
		server().acquire<context>(this)->level_ = value;
	}

	virtual void setCompressMinSize(int value)
	{
		server().acquire<context>(this)->min_size_ = value;
	}

	virtual void setCompressMaxSize(int value)
	{
		server().acquire<context>(this)->max_size_ = value;
	}

private:
	std::error_code setup_types(const x0::SettingsValue& cvar, x0::Scope& s)
	{
		return cvar.load(s.acquire<context>(this)->content_types_);
	}

	std::error_code setup_level(const x0::SettingsValue& cvar, x0::Scope& s)
	{
		return cvar.load(s.acquire<context>(this)->level_);
	}

	std::error_code setup_minsize(const x0::SettingsValue& cvar, x0::Scope& s)
	{
		return cvar.load(s.acquire<context>(this)->min_size_);
	}

	std::error_code setup_maxsize(const x0::SettingsValue& cvar, x0::Scope& s)
	{
		return cvar.load(s.acquire<context>(this)->max_size_);
	}

	void postProcess(x0::HttpRequest *in, x0::HttpResponse *out)
	{
		if (out->headers.contains("Content-Encoding"))
			return; // do not double-encode content

		const context *cx = server_.resolveHost(in->hostid())->get<context>(this);
		if (!cx && !(cx = server().get<context>(this)))
			return;

		long long size = 0;
		if (out->headers.contains("Content-Length"))
			size = boost::lexical_cast<int>(out->headers["Content-Length"]);

		std::string te(out->headers["Transfer-Encoding"]);
		bool chunked = (te == "chunked");

		if (size < cx->min_size_ && !(size <= 0 && chunked))
			return;

		if (size > cx->max_size_)
			return;

		if (!cx->contains_mime(out->headers("Content-Type")))
			return;

		if (x0::BufferRef r = in->header("Accept-Encoding"))
		{
			typedef boost::tokenizer<boost::char_separator<char>> tokenizer;

			std::vector<std::string> items(x0::split<std::string>(r.str(), ", "));

#if defined(HAVE_BZLIB_H)
			if (std::find(items.begin(), items.end(), "bzip2") != items.end())
			{
				out->headers.push_back("Content-Encoding", "bzip2");
				out->filters.push_back(std::make_shared<x0::BZip2Filter>(cx->level_));
			}
			else
#endif
#if defined(HAVE_ZLIB_H)
			if (std::find(items.begin(), items.end(), "gzip") != items.end())
			{
				out->headers.push_back("Content-Encoding", "gzip");
				out->filters.push_back(std::make_shared<x0::GZipFilter>(cx->level_));
			}
			else if (std::find(items.begin(), items.end(), "deflate") != items.end())
			{
				out->headers.push_back("Content-Encoding", "deflate");
				out->filters.push_back(std::make_shared<x0::DeflateFilter>(cx->level_));
			}
			else
#endif
				return;

			// response might change according to Accept-Encoding
			if (!out->headers.contains("Vary"))
				out->headers.push_back("Vary", "Accept-Encoding");
			else
				out->headers["Vary"] += ",Accept-Encoding";

			// removing content-length implicitely enables chunked encoding
			out->headers.remove("Content-Length");
		}
	}
};

X0_EXPORT_PLUGIN(compress);
<|endoftext|>


<commit_before>// Copyright (c) 2018 The Gulden developers
// Authored by: Willem de Jonge (willem@isnapp.nl)
// Distributed under the GULDEN software license, see the accompanying
// file COPYING

#include "spvscanner.h"
#include "../net_processing.h"
#include "timedata.h"
#include "../validation/validation.h"
#include "wallet.h"

#include <algorithm>

// Maximum number of requests pending. This number should be high enough so that
// the requests can be reasonably distributed over our peers.
const int MAX_PENDING_REQUESTS = 512;

// Duration (seconds) of longest fork that can be handled
const int64_t MAX_FORK_DURATION = 2 * 24 * 3600;

// Persisting wallet db updates are limited as they are very expensive
// and updating at every processed block would give very poor performance.
// Therefore the state is only written to the db when a minimum time interval has passed,
// or a certain of number of blocks have been processed (whichever comes first).
// Note that due to this some blocks might be re-processed in case of abnormal termination,
// this is fine.

// Interval in seconds for writing the scan progress to the wallet db
const int64_t PERSIST_INTERVAL_SEC = 5;

// Blocks count after which scan processing state is written to the db.
const int PERSIST_BLOCK_COUNT = 500;

// limit UI update notifications (except when catched up)
const int UI_UPDATE_LIMIT = 50;

CSPVScanner::CSPVScanner(CWallet& _wallet) :
    wallet(_wallet),
    startTime(0),
    lastProcessed(nullptr),
    lastPersistTime(0)
{
    LOCK(cs_main);

    // init scan starting time to birth of first key
    startTime =  wallet.nTimeFirstKey;

    // forward scan starting time to last block processed if available
    CWalletDB walletdb(*wallet.dbw);
    CBlockLocator locator;
    int64_t lastBlockTime;
    if (walletdb.ReadLastSPVBlockProcessed(locator, lastBlockTime))
        startTime = lastBlockTime;

    // rewind scan starting time by maximum handled fork duration
    startTime = std::max(Params().GenesisBlock().GetBlockTime(), startTime - MAX_FORK_DURATION);
}

CSPVScanner::~CSPVScanner()
{
}

bool CSPVScanner::StartScan()
{
    return StartPartialHeaders(startTime, std::bind(&CSPVScanner::HeaderTipChanged, this, std::placeholders::_1));
}

const CBlockIndex* CSPVScanner::LastBlockProcessed() const
{
    return lastProcessed;
}

void CSPVScanner::RequestBlocks()
{
    LOCK2(cs_main, wallet.cs_wallet);

    // put lastProcessed and/or requestTip back on chain if forked
    while (!partialChain.Contains(requestTip)) {
        if (requestTip->nHeight > lastProcessed->nHeight) {
            CancelPriorityDownload(requestTip, std::bind(&CSPVScanner::ProcessPriorityRequest, this, std::placeholders::_1, std::placeholders::_2));
            requestTip = requestTip->pprev;
        }
        else { // so here requestTip == lastProcessed
            std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>();
            if (ReadBlockFromDisk(*pblock, lastProcessed, Params())) {
                wallet.BlockDisconnected(pblock);
            }
            else {
                // This block must be on disk, it was processed before.
                // So pruning has to keep at least as many blocks back as the longest fork we are willing to handle.
                assert(false);
            }
            UpdateLastProcessed(lastProcessed->pprev);
            requestTip = lastProcessed;
        }
    }

    // skip blocks that are before startTime
    const CBlockIndex* skip = lastProcessed;
    while (skip->GetBlockTime() < startTime && partialChain.Height() > skip->nHeight)
        skip = partialChain.Next(skip);
    if (skip != lastProcessed) {
        LogPrint(BCLog::WALLET, "Skipping %d old blocks for SPV scan, up to height %d\n", skip->nHeight - lastProcessed->nHeight, skip->nHeight);
        UpdateLastProcessed(skip);
        if (lastProcessed->nHeight > requestTip->nHeight) {
            requestTip = lastProcessed;
            startHeight = lastProcessed->nHeight;
        }
    }

    std::vector<const CBlockIndex*> blocksToRequest;

    // add requests for as long as nMaxPendingRequests is not reached and there are still heigher blocks in headerChain
    while (requestTip->nHeight - lastProcessed->nHeight < MAX_PENDING_REQUESTS &&
           partialChain.Height() > requestTip->nHeight) {

        requestTip = partialChain.Next(requestTip);
        blocksToRequest.push_back(requestTip);
    }

    if (!blocksToRequest.empty()) {
        LogPrint(BCLog::WALLET, "Requesting %d blocks for SPV, up to height %d\n", blocksToRequest.size(), requestTip->nHeight);
        AddPriorityDownload(blocksToRequest, std::bind(&CSPVScanner::ProcessPriorityRequest, this, std::placeholders::_1, std::placeholders::_2));
    }
}

void CSPVScanner::ProcessPriorityRequest(const std::shared_ptr<const CBlock> &block, const CBlockIndex *pindex)
{
    LOCK2(cs_main, wallet.cs_wallet);

    // if chainActive is up-to-date no SPV blocks need to be requested, we can update SPV to the activeChain
    if (chainActive.Tip() == partialChain.Tip()) {
        LogPrint(BCLog::WALLET, "chainActive is up-to-date, skipping SPV processing block %d\n", pindex->nHeight);
        if (lastProcessed != partialChain.Tip()) {
            UpdateLastProcessed(chainActive.Tip());
            requestTip = lastProcessed;
        }
    }

    // if this is the block we're waiting for process it and request new block(s)
    if (pindex->pprev == lastProcessed) {
        LogPrint(BCLog::WALLET, "SPV processing block %d\n", pindex->nHeight);

        // TODO handle mempool effects

        std::vector<CTransactionRef> vtxConflicted; // dummy for now
        wallet.BlockConnected(block, pindex, vtxConflicted);

        UpdateLastProcessed(pindex);

        RequestBlocks();

        if (partialChain.Height() == pindex->nHeight || pindex->nHeight % UI_UPDATE_LIMIT == 0)
            uiInterface.NotifySPVProgress(startHeight, pindex->nHeight, partialChain.Height());

        blocksSincePersist++;
    }
}

void CSPVScanner::HeaderTipChanged(const CBlockIndex* pTip)
{
    if (pTip)
    {
        // initialization on the first header tip notification
        if (lastProcessed == nullptr)
        {
            if (partialChain.Height() >= partialChain.HeightOffset()
                    && partialChain[partialChain.HeightOffset()]->GetBlockTime() <= startTime)
            {
                // use start of partial chain to init lastProcessed
                // forks are handled when requesting blocks which will also fast-forward to startTime
                // should the headerChain be very early
                lastProcessed = partialChain[partialChain.HeightOffset()];
            }
            else
            {
                // headerChain not usable, it does not start early enough or has no data. This should not happen.
                return;
            }
        }

        requestTip = lastProcessed;
        startHeight = lastProcessed->nHeight;

        LogPrint(BCLog::WALLET, "SPV init using %s (height = %d) as last processed block\n",
                 lastProcessed->GetBlockHashPoW2().ToString(), lastProcessed->nHeight);

        RequestBlocks();
    }
    else // pTip == nullptr => partial sync stopped
    {
        CancelAllPriorityDownloads();
        Persist();
    }
}

void CSPVScanner::UpdateLastProcessed(const CBlockIndex* pindex)
{
    lastProcessed = pindex;

    int64_t now = GetAdjustedTime();
    if (now - lastPersistTime > PERSIST_INTERVAL_SEC || blocksSincePersist >= PERSIST_BLOCK_COUNT)
        Persist();
}

void CSPVScanner::Persist()
{
    if (lastProcessed != nullptr)
    {
        CWalletDB walletdb(*wallet.dbw);
        walletdb.WriteLastSPVBlockProcessed(partialChain.GetLocatorPoW2(lastProcessed), lastProcessed->GetBlockTime());

        lastPersistTime = GetAdjustedTime();
        blocksSincePersist = 0;
    }
}
<commit_msg>Tighten conditions to continue  partial sync<commit_after>// Copyright (c) 2018 The Gulden developers
// Authored by: Willem de Jonge (willem@isnapp.nl)
// Distributed under the GULDEN software license, see the accompanying
// file COPYING

#include "spvscanner.h"
#include "../net_processing.h"
#include "timedata.h"
#include "../validation/validation.h"
#include "wallet.h"

#include <algorithm>

// Maximum number of requests pending. This number should be high enough so that
// the requests can be reasonably distributed over our peers.
const int MAX_PENDING_REQUESTS = 512;

// Duration (seconds) of longest fork that can be handled
const int64_t MAX_FORK_DURATION = 1 * 12 * 3600;

// Persisting wallet db updates are limited as they are very expensive
// and updating at every processed block would give very poor performance.
// Therefore the state is only written to the db when a minimum time interval has passed,
// or a certain of number of blocks have been processed (whichever comes first).
// Note that due to this some blocks might be re-processed in case of abnormal termination,
// this is fine.

// Interval in seconds for writing the scan progress to the wallet db
const int64_t PERSIST_INTERVAL_SEC = 5;

// Blocks count after which scan processing state is written to the db.
const int PERSIST_BLOCK_COUNT = 500;

// limit UI update notifications (except when catched up)
const int UI_UPDATE_LIMIT = 50;

CSPVScanner::CSPVScanner(CWallet& _wallet) :
    wallet(_wallet),
    startTime(0),
    lastProcessed(nullptr),
    lastPersistTime(0)
{
    LOCK(cs_main);

    // init scan starting time to birth of first key
    startTime =  wallet.nTimeFirstKey;

    // forward scan starting time to last block processed if available
    CWalletDB walletdb(*wallet.dbw);
    CBlockLocator locator;
    int64_t lastBlockTime;
    if (walletdb.ReadLastSPVBlockProcessed(locator, lastBlockTime))
        startTime = lastBlockTime;

    // rewind scan starting time by maximum handled fork duration
    startTime = std::max(Params().GenesisBlock().GetBlockTime(), startTime - MAX_FORK_DURATION);
}

CSPVScanner::~CSPVScanner()
{
}

bool CSPVScanner::StartScan()
{
    return StartPartialHeaders(startTime, std::bind(&CSPVScanner::HeaderTipChanged, this, std::placeholders::_1));
}

const CBlockIndex* CSPVScanner::LastBlockProcessed() const
{
    return lastProcessed;
}

void CSPVScanner::RequestBlocks()
{
    LOCK2(cs_main, wallet.cs_wallet);

    // put lastProcessed and/or requestTip back on chain if forked
    while (!partialChain.Contains(requestTip)) {
        if (requestTip->nHeight > lastProcessed->nHeight) {
            CancelPriorityDownload(requestTip, std::bind(&CSPVScanner::ProcessPriorityRequest, this, std::placeholders::_1, std::placeholders::_2));
            requestTip = requestTip->pprev;
        }
        else { // so here requestTip == lastProcessed
            std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>();
            if (ReadBlockFromDisk(*pblock, lastProcessed, Params())) {
                wallet.BlockDisconnected(pblock);
            }
            else {
                // This block must be on disk, it was processed before.
                // So pruning has to keep at least as many blocks back as the longest fork we are willing to handle.
                assert(false);
            }
            UpdateLastProcessed(lastProcessed->pprev);
            requestTip = lastProcessed;
        }
    }

    // skip blocks that are before startTime
    const CBlockIndex* skip = lastProcessed;
    while (skip->GetBlockTime() < startTime && partialChain.Height() > skip->nHeight)
        skip = partialChain.Next(skip);
    if (skip != lastProcessed) {
        LogPrint(BCLog::WALLET, "Skipping %d old blocks for SPV scan, up to height %d\n", skip->nHeight - lastProcessed->nHeight, skip->nHeight);
        UpdateLastProcessed(skip);
        if (lastProcessed->nHeight > requestTip->nHeight) {
            requestTip = lastProcessed;
            startHeight = lastProcessed->nHeight;
        }
    }

    std::vector<const CBlockIndex*> blocksToRequest;

    // add requests for as long as nMaxPendingRequests is not reached and there are still heigher blocks in headerChain
    while (requestTip->nHeight - lastProcessed->nHeight < MAX_PENDING_REQUESTS &&
           partialChain.Height() > requestTip->nHeight) {

        requestTip = partialChain.Next(requestTip);
        blocksToRequest.push_back(requestTip);
    }

    if (!blocksToRequest.empty()) {
        LogPrint(BCLog::WALLET, "Requesting %d blocks for SPV, up to height %d\n", blocksToRequest.size(), requestTip->nHeight);
        AddPriorityDownload(blocksToRequest, std::bind(&CSPVScanner::ProcessPriorityRequest, this, std::placeholders::_1, std::placeholders::_2));
    }
}

void CSPVScanner::ProcessPriorityRequest(const std::shared_ptr<const CBlock> &block, const CBlockIndex *pindex)
{
    LOCK2(cs_main, wallet.cs_wallet);

    // if chainActive is up-to-date no SPV blocks need to be requested, we can update SPV to the activeChain
    if (chainActive.Tip() == partialChain.Tip()) {
        LogPrint(BCLog::WALLET, "chainActive is up-to-date, skipping SPV processing block %d\n", pindex->nHeight);
        if (lastProcessed != partialChain.Tip()) {
            UpdateLastProcessed(chainActive.Tip());
            requestTip = lastProcessed;
        }
    }

    // if this is the block we're waiting for process it and request new block(s)
    if (pindex->pprev == lastProcessed) {
        LogPrint(BCLog::WALLET, "SPV processing block %d\n", pindex->nHeight);

        // TODO handle mempool effects

        std::vector<CTransactionRef> vtxConflicted; // dummy for now
        wallet.BlockConnected(block, pindex, vtxConflicted);

        UpdateLastProcessed(pindex);

        RequestBlocks();

        if (partialChain.Height() == pindex->nHeight || pindex->nHeight % UI_UPDATE_LIMIT == 0)
            uiInterface.NotifySPVProgress(startHeight, pindex->nHeight, partialChain.Height());

        blocksSincePersist++;
    }
}

void CSPVScanner::HeaderTipChanged(const CBlockIndex* pTip)
{
    if (pTip)
    {
        // initialization on the first header tip notification
        if (lastProcessed == nullptr)
        {
            if (partialChain.Height() >= partialChain.HeightOffset()
                    && partialChain[partialChain.HeightOffset()]->GetBlockTime() <= startTime)
            {
                // use start of partial chain to init lastProcessed
                // forks are handled when requesting blocks which will also fast-forward to startTime
                // should the headerChain be very early
                lastProcessed = partialChain[partialChain.HeightOffset()];
            }
            else
            {
                // headerChain not usable, it does not start early enough or has no data. This should not happen.
                return;
            }
        }

        requestTip = lastProcessed;
        startHeight = lastProcessed->nHeight;

        LogPrint(BCLog::WALLET, "SPV init using %s (height = %d) as last processed block\n",
                 lastProcessed->GetBlockHashPoW2().ToString(), lastProcessed->nHeight);

        RequestBlocks();
    }
    else // pTip == nullptr => partial sync stopped
    {
        CancelAllPriorityDownloads();
        Persist();
    }
}

void CSPVScanner::UpdateLastProcessed(const CBlockIndex* pindex)
{
    lastProcessed = pindex;

    int64_t now = GetAdjustedTime();
    if (now - lastPersistTime > PERSIST_INTERVAL_SEC || blocksSincePersist >= PERSIST_BLOCK_COUNT)
        Persist();
}

void CSPVScanner::Persist()
{
    if (lastProcessed != nullptr)
    {
        CWalletDB walletdb(*wallet.dbw);
        walletdb.WriteLastSPVBlockProcessed(partialChain.GetLocatorPoW2(lastProcessed), lastProcessed->GetBlockTime());

        lastPersistTime = GetAdjustedTime();
        blocksSincePersist = 0;
    }
}
<|endoftext|>


<commit_before>// Copyright (c) 2018 The Gulden developers
// Authored by: Willem de Jonge (willem@isnapp.nl)
// Distributed under the GULDEN software license, see the accompanying
// file COPYING

#include "spvscanner.h"
#include "../net_processing.h"
#include "../validation/validation.h"
#include "wallet.h"

#include <algorithm>

// Maximum number of requests pending. This number should be high enough so that
// the requests can be reasonably distributed over our peers.
const static int nMaxPendingRequests = 512;

// Seconds before oldest key to start scanning blocks.
const static int64_t startTimeGap = 3 * 3600;

CSPVScanner::CSPVScanner(CWallet& _wallet) :
    wallet(_wallet),
    startTime(0)
{
    LOCK(cs_main);

    CWalletDB walletdb(*wallet.dbw);
    CBlockLocator locator;
    if (!walletdb.ReadLastSPVBlockProcessed(locator))
        locator = chainActive.GetLocatorPoW2(chainActive.Genesis());

    lastProcessed = FindForkInGlobalIndex(headerChain, locator);
    requestTip = lastProcessed;

    // fixme (SPV): get actual seed time, now for testing hacked to birth of test wallet
    // was uint64_t seedTime = wallet.GetOldestKeyPoolTime();
    int64_t seedTime = 1527681623; // 05/30/2018 @ 12:00pm (UTC)

    startTime =  std::max(0LL, seedTime - startTimeGap);

    LogPrint(BCLog::WALLET, "Using %s (height = %d) as last processed SPV block\n",
             lastProcessed->GetBlockHashPoW2().ToString(), lastProcessed->nHeight);

    RegisterValidationInterface(this);
}

CSPVScanner::~CSPVScanner()
{
    UnregisterValidationInterface(this);
}

void CSPVScanner::StartScan()
{
    RequestBlocks();
}

void CSPVScanner::RequestBlocks()
{
    LOCK2(cs_main, wallet.cs_wallet);

    // put lastProcessed and/or requestTip back on chain if forked
    while (!headerChain.Contains(requestTip)) {
        if (requestTip->nHeight > lastProcessed->nHeight) {
            CancelPriorityDownload(requestTip);
            requestTip = requestTip->pprev;
        }
        else { // so here requestTip == lastProcessed
            std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>();
            if (!ReadBlockFromDisk(*pblock, lastProcessed, Params())) {
                wallet.BlockDisconnected(pblock);
            }
            else {
                // This block must be on disk, it was processed before.
                // So pruning has to keep at least as many blocks back as the longest fork we are willing to handle.
                assert(false);
            }
            UpdateLastProcessed(lastProcessed->pprev);
            requestTip = lastProcessed;
        }
    }

    // skip blocks that are before startTime
    const CBlockIndex* skip = lastProcessed;
    while (skip->GetBlockTime() < startTime && headerChain.Height() > skip->nHeight)
        skip = headerChain.Next(skip);
    if (skip != lastProcessed) {
        LogPrint(BCLog::WALLET, "Skipping %d old blocks for SPV scan, up to height %d\n", skip->nHeight - lastProcessed->nHeight, skip->nHeight);
        UpdateLastProcessed(skip);
        if (lastProcessed->nHeight > requestTip->nHeight)
            requestTip = lastProcessed;
    }

    std::vector<const CBlockIndex*> blocksToRequest;

    // add requests for as long as nMaxPendingRequests is not reached and there are still heigher blocks in headerChain
    while (requestTip->nHeight - lastProcessed->nHeight < nMaxPendingRequests &&
           headerChain.Height() > requestTip->nHeight) {

        requestTip = headerChain.Next(requestTip);
        blocksToRequest.push_back(requestTip);
    }

    if (!blocksToRequest.empty()) {
        LogPrint(BCLog::WALLET, "Requesting %d blocks for SPV, up to height %d\n", blocksToRequest.size(), requestTip->nHeight);
        AddPriorityDownload(blocksToRequest);
    }
}

void CSPVScanner::ProcessPriorityRequest(const std::shared_ptr<const CBlock> &block, const CBlockIndex *pindex)
{
    LOCK2(cs_main, wallet.cs_wallet);

    // if chainActive is up-to-date no SPV blocks need to be requested, we can update SPV to the activeChain
    if (chainActive.Tip() == headerChain.Tip()) {
        LogPrint(BCLog::WALLET, "chainActive is up-to-date, skipping SPV processing block %d\n", pindex->nHeight);
        if (lastProcessed != headerChain.Tip()) {
            UpdateLastProcessed(chainActive.Tip());
            requestTip = lastProcessed;
        }
    }

    // if this is the block we're waiting for process it and request new block(s)
    if (pindex->pprev == lastProcessed) {
        LogPrint(BCLog::WALLET, "SPV processing block %d\n", pindex->nHeight);

        // TODO handle mempool effects

        std::vector<CTransactionRef> vtxConflicted; // dummy for now
        wallet.BlockConnected(block, pindex, vtxConflicted);

        UpdateLastProcessed(pindex);

        RequestBlocks();
    }
}

void CSPVScanner::HeaderTipChanged(const CBlockIndex* pTip)
{
    RequestBlocks();
}

void CSPVScanner::UpdateLastProcessed(const CBlockIndex* pindex)
{
    lastProcessed = pindex;
    CWalletDB walletdb(*wallet.dbw);
    walletdb.WriteLastSPVBlockProcessed(headerChain.GetLocatorPoW2(lastProcessed));
}
<commit_msg>Fix build warning in spv<commit_after>// Copyright (c) 2018 The Gulden developers
// Authored by: Willem de Jonge (willem@isnapp.nl)
// Distributed under the GULDEN software license, see the accompanying
// file COPYING

#include "spvscanner.h"
#include "../net_processing.h"
#include "../validation/validation.h"
#include "wallet.h"

#include <algorithm>

// Maximum number of requests pending. This number should be high enough so that
// the requests can be reasonably distributed over our peers.
const static int nMaxPendingRequests = 512;

// Seconds before oldest key to start scanning blocks.
const static int64_t startTimeGap = 3 * 3600;

CSPVScanner::CSPVScanner(CWallet& _wallet) :
    wallet(_wallet),
    startTime(0)
{
    LOCK(cs_main);

    CWalletDB walletdb(*wallet.dbw);
    CBlockLocator locator;
    if (!walletdb.ReadLastSPVBlockProcessed(locator))
        locator = chainActive.GetLocatorPoW2(chainActive.Genesis());

    lastProcessed = FindForkInGlobalIndex(headerChain, locator);
    requestTip = lastProcessed;

    // fixme (SPV): get actual seed time, now for testing hacked to birth of test wallet
    // was uint64_t seedTime = wallet.GetOldestKeyPoolTime();
    int64_t seedTime = 1527681623; // 05/30/2018 @ 12:00pm (UTC)

    startTime =  std::max(int64_t(0), seedTime - startTimeGap);

    LogPrint(BCLog::WALLET, "Using %s (height = %d) as last processed SPV block\n",
             lastProcessed->GetBlockHashPoW2().ToString(), lastProcessed->nHeight);

    RegisterValidationInterface(this);
}

CSPVScanner::~CSPVScanner()
{
    UnregisterValidationInterface(this);
}

void CSPVScanner::StartScan()
{
    RequestBlocks();
}

void CSPVScanner::RequestBlocks()
{
    LOCK2(cs_main, wallet.cs_wallet);

    // put lastProcessed and/or requestTip back on chain if forked
    while (!headerChain.Contains(requestTip)) {
        if (requestTip->nHeight > lastProcessed->nHeight) {
            CancelPriorityDownload(requestTip);
            requestTip = requestTip->pprev;
        }
        else { // so here requestTip == lastProcessed
            std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>();
            if (!ReadBlockFromDisk(*pblock, lastProcessed, Params())) {
                wallet.BlockDisconnected(pblock);
            }
            else {
                // This block must be on disk, it was processed before.
                // So pruning has to keep at least as many blocks back as the longest fork we are willing to handle.
                assert(false);
            }
            UpdateLastProcessed(lastProcessed->pprev);
            requestTip = lastProcessed;
        }
    }

    // skip blocks that are before startTime
    const CBlockIndex* skip = lastProcessed;
    while (skip->GetBlockTime() < startTime && headerChain.Height() > skip->nHeight)
        skip = headerChain.Next(skip);
    if (skip != lastProcessed) {
        LogPrint(BCLog::WALLET, "Skipping %d old blocks for SPV scan, up to height %d\n", skip->nHeight - lastProcessed->nHeight, skip->nHeight);
        UpdateLastProcessed(skip);
        if (lastProcessed->nHeight > requestTip->nHeight)
            requestTip = lastProcessed;
    }

    std::vector<const CBlockIndex*> blocksToRequest;

    // add requests for as long as nMaxPendingRequests is not reached and there are still heigher blocks in headerChain
    while (requestTip->nHeight - lastProcessed->nHeight < nMaxPendingRequests &&
           headerChain.Height() > requestTip->nHeight) {

        requestTip = headerChain.Next(requestTip);
        blocksToRequest.push_back(requestTip);
    }

    if (!blocksToRequest.empty()) {
        LogPrint(BCLog::WALLET, "Requesting %d blocks for SPV, up to height %d\n", blocksToRequest.size(), requestTip->nHeight);
        AddPriorityDownload(blocksToRequest);
    }
}

void CSPVScanner::ProcessPriorityRequest(const std::shared_ptr<const CBlock> &block, const CBlockIndex *pindex)
{
    LOCK2(cs_main, wallet.cs_wallet);

    // if chainActive is up-to-date no SPV blocks need to be requested, we can update SPV to the activeChain
    if (chainActive.Tip() == headerChain.Tip()) {
        LogPrint(BCLog::WALLET, "chainActive is up-to-date, skipping SPV processing block %d\n", pindex->nHeight);
        if (lastProcessed != headerChain.Tip()) {
            UpdateLastProcessed(chainActive.Tip());
            requestTip = lastProcessed;
        }
    }

    // if this is the block we're waiting for process it and request new block(s)
    if (pindex->pprev == lastProcessed) {
        LogPrint(BCLog::WALLET, "SPV processing block %d\n", pindex->nHeight);

        // TODO handle mempool effects

        std::vector<CTransactionRef> vtxConflicted; // dummy for now
        wallet.BlockConnected(block, pindex, vtxConflicted);

        UpdateLastProcessed(pindex);

        RequestBlocks();
    }
}

void CSPVScanner::HeaderTipChanged(const CBlockIndex* pTip)
{
    RequestBlocks();
}

void CSPVScanner::UpdateLastProcessed(const CBlockIndex* pindex)
{
    lastProcessed = pindex;
    CWalletDB walletdb(*wallet.dbw);
    walletdb.WriteLastSPVBlockProcessed(headerChain.GetLocatorPoW2(lastProcessed));
}
<|endoftext|>


<commit_before>/*
    Copyright (C) 1995-1997  Peter Mattis, Spencer Kimball and Josh MacDonald

    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Library General Public
    License as published by the Free Software Foundation; either
    version 2 of the License, or (at your option) any later version.

    This library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Library General Public License for more details.

    You should have received a copy of the GNU Library General Public
    License along with this library; if not, write to the Free
    Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/

#include "cssysdef.h"

#include "csutil/stringconv.h"

#include <locale.h>

static float my_strtof (const char* s, const char** fail_pos)
{
  char* fp;
  float v;
#ifdef CS_HAVE_STRTOF
  v = strtof (s, &fp);
#else
  v = (float)strtof (s, &fp);
#endif
  if (fail_pos) *fail_pos = fp;
  return v;
}

typedef enum {
  G_ASCII_ALNUM  = 1 << 0,
  G_ASCII_ALPHA  = 1 << 1,
  G_ASCII_CNTRL  = 1 << 2,
  G_ASCII_DIGIT  = 1 << 3,
  G_ASCII_GRAPH  = 1 << 4,
  G_ASCII_LOWER  = 1 << 5,
  G_ASCII_PRINT  = 1 << 6,
  G_ASCII_PUNCT  = 1 << 7,
  G_ASCII_SPACE  = 1 << 8,
  G_ASCII_UPPER  = 1 << 9,
  G_ASCII_XDIGIT = 1 << 10
} GAsciiType;

static const uint16 ascii_table_data[256] = {
  0x004, 0x004, 0x004, 0x004, 0x004, 0x004, 0x004, 0x004,
  0x004, 0x104, 0x104, 0x004, 0x104, 0x104, 0x004, 0x004,
  0x004, 0x004, 0x004, 0x004, 0x004, 0x004, 0x004, 0x004,
  0x004, 0x004, 0x004, 0x004, 0x004, 0x004, 0x004, 0x004,
  0x140, 0x0d0, 0x0d0, 0x0d0, 0x0d0, 0x0d0, 0x0d0, 0x0d0,
  0x0d0, 0x0d0, 0x0d0, 0x0d0, 0x0d0, 0x0d0, 0x0d0, 0x0d0,
  0x459, 0x459, 0x459, 0x459, 0x459, 0x459, 0x459, 0x459,
  0x459, 0x459, 0x0d0, 0x0d0, 0x0d0, 0x0d0, 0x0d0, 0x0d0,
  0x0d0, 0x653, 0x653, 0x653, 0x653, 0x653, 0x653, 0x253,
  0x253, 0x253, 0x253, 0x253, 0x253, 0x253, 0x253, 0x253,
  0x253, 0x253, 0x253, 0x253, 0x253, 0x253, 0x253, 0x253,
  0x253, 0x253, 0x253, 0x0d0, 0x0d0, 0x0d0, 0x0d0, 0x0d0,
  0x0d0, 0x473, 0x473, 0x473, 0x473, 0x473, 0x473, 0x073,
  0x073, 0x073, 0x073, 0x073, 0x073, 0x073, 0x073, 0x073,
  0x073, 0x073, 0x073, 0x073, 0x073, 0x073, 0x073, 0x073,
  0x073, 0x073, 0x073, 0x0d0, 0x0d0, 0x0d0, 0x0d0, 0x004
  /* the upper 128 are all zeroes */
};

#define g_ascii_isdigit(c) \
  ((ascii_table_data[(unsigned char) (c)] & G_ASCII_DIGIT) != 0)
#define g_ascii_isspace(c) \
  ((ascii_table_data[(unsigned char) (c)] & G_ASCII_SPACE) != 0)
#define g_ascii_isxdigit(c) \
  ((ascii_table_data[(unsigned char) (c)] & G_ASCII_XDIGIT) != 0)

namespace CS
{
  namespace Utility
  {
    // This is actually g_ascii_strtod from glib.
    float strtof (const char* nptr, const char** endptr)
    {
      const char *fail_pos;
      float val;
      struct lconv *locale_data;
      const char *decimal_point;
      int decimal_point_len;
      const char *p, *decimal_point_pos;
      const char *end = NULL; /* Silence gcc */
      int strtod_errno;

      if (nptr == 0) return 0;

      fail_pos = NULL;

      locale_data = localeconv ();
      decimal_point = locale_data->decimal_point;
      decimal_point_len = strlen (decimal_point);

      CS_ASSERT (decimal_point_len != 0);

      decimal_point_pos = NULL;
      end = NULL;

      if (decimal_point[0] != '.' ||
	  decimal_point[1] != 0)
	{
	  p = nptr;
	  /* Skip leading space */
	  while (g_ascii_isspace (*p))
	    p++;

	  /* Skip leading optional sign */
	  if (*p == '+' || *p == '-')
	    p++;

	  if (p[0] == '0' &&
	      (p[1] == 'x' || p[1] == 'X'))
	    {
	      p += 2;
	      /* HEX - find the (optional) decimal point */

	      while (g_ascii_isxdigit (*p))
		p++;

	      if (*p == '.')
		decimal_point_pos = p++;

	      while (g_ascii_isxdigit (*p))
		p++;

	      if (*p == 'p' || *p == 'P')
		p++;
	      if (*p == '+' || *p == '-')
		p++;
	      while (g_ascii_isdigit (*p))
		p++;

	      end = p;
	    }
	  else if (g_ascii_isdigit (*p) || *p == '.')
	    {
	      while (g_ascii_isdigit (*p))
		p++;

	      if (*p == '.')
		decimal_point_pos = p++;

	      while (g_ascii_isdigit (*p))
		p++;

	      if (*p == 'e' || *p == 'E')
		p++;
	      if (*p == '+' || *p == '-')
		p++;
	      while (g_ascii_isdigit (*p))
		p++;

	      end = p;
	    }
	  /* For the other cases, we need not convert the decimal point */
	}

      if (decimal_point_pos)
	{
	  char *copy, *c;

	  /* We need to convert the '.' to the locale specific decimal point */
	  copy = (char*)cs_malloc (end - nptr + 1 + decimal_point_len);

	  c = copy;
	  memcpy (c, nptr, decimal_point_pos - nptr);
	  c += decimal_point_pos - nptr;
	  memcpy (c, decimal_point, decimal_point_len);
	  c += decimal_point_len;
	  memcpy (c, decimal_point_pos + 1, end - (decimal_point_pos + 1));
	  c += end - (decimal_point_pos + 1);
	  *c = 0;

	  errno = 0;
	  val = my_strtof (copy, &fail_pos);
	  strtod_errno = errno;

	  if (fail_pos)
	    {
	      if (fail_pos - copy > decimal_point_pos - nptr)
		fail_pos = (char *)nptr + (fail_pos - copy) - (decimal_point_len - 1);
	      else
		fail_pos = (char *)nptr + (fail_pos - copy);
	    }

	  cs_free (copy);

	}
      else if (end)
	{
	  char *copy;

	  copy = (char*)cs_malloc (end - (char *)nptr + 1);
	  memcpy (copy, nptr, end - nptr);
	  *(copy + (end - (char *)nptr)) = 0;

	  errno = 0;
	  val = my_strtof (copy, &fail_pos);
	  strtod_errno = errno;

	  if (fail_pos)
	    {
	      fail_pos = (char *)nptr + (fail_pos - copy);
	    }

	  cs_free (copy);
	}
      else
	{
	  errno = 0;
	  val = my_strtof (nptr, &fail_pos);
	  strtod_errno = errno;
	}

      if (endptr)
	*endptr = fail_pos;

      errno = strtod_errno;

      return val;
    }

  } // namespace Utility
} // namespace CS
<commit_msg>Fix blooper: strtod() should be used if strtof() is not available<commit_after>/*
    Copyright (C) 1995-1997  Peter Mattis, Spencer Kimball and Josh MacDonald

    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Library General Public
    License as published by the Free Software Foundation; either
    version 2 of the License, or (at your option) any later version.

    This library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Library General Public License for more details.

    You should have received a copy of the GNU Library General Public
    License along with this library; if not, write to the Free
    Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/

#include "cssysdef.h"

#include "csutil/stringconv.h"

#include <locale.h>

static float my_strtof (const char* s, const char** fail_pos)
{
  char* fp;
  float v;
#ifdef CS_HAVE_STRTOF
  v = strtof (s, &fp);
#else
  v = (float)strtod (s, &fp);
#endif
  if (fail_pos) *fail_pos = fp;
  return v;
}

typedef enum {
  G_ASCII_ALNUM  = 1 << 0,
  G_ASCII_ALPHA  = 1 << 1,
  G_ASCII_CNTRL  = 1 << 2,
  G_ASCII_DIGIT  = 1 << 3,
  G_ASCII_GRAPH  = 1 << 4,
  G_ASCII_LOWER  = 1 << 5,
  G_ASCII_PRINT  = 1 << 6,
  G_ASCII_PUNCT  = 1 << 7,
  G_ASCII_SPACE  = 1 << 8,
  G_ASCII_UPPER  = 1 << 9,
  G_ASCII_XDIGIT = 1 << 10
} GAsciiType;

static const uint16 ascii_table_data[256] = {
  0x004, 0x004, 0x004, 0x004, 0x004, 0x004, 0x004, 0x004,
  0x004, 0x104, 0x104, 0x004, 0x104, 0x104, 0x004, 0x004,
  0x004, 0x004, 0x004, 0x004, 0x004, 0x004, 0x004, 0x004,
  0x004, 0x004, 0x004, 0x004, 0x004, 0x004, 0x004, 0x004,
  0x140, 0x0d0, 0x0d0, 0x0d0, 0x0d0, 0x0d0, 0x0d0, 0x0d0,
  0x0d0, 0x0d0, 0x0d0, 0x0d0, 0x0d0, 0x0d0, 0x0d0, 0x0d0,
  0x459, 0x459, 0x459, 0x459, 0x459, 0x459, 0x459, 0x459,
  0x459, 0x459, 0x0d0, 0x0d0, 0x0d0, 0x0d0, 0x0d0, 0x0d0,
  0x0d0, 0x653, 0x653, 0x653, 0x653, 0x653, 0x653, 0x253,
  0x253, 0x253, 0x253, 0x253, 0x253, 0x253, 0x253, 0x253,
  0x253, 0x253, 0x253, 0x253, 0x253, 0x253, 0x253, 0x253,
  0x253, 0x253, 0x253, 0x0d0, 0x0d0, 0x0d0, 0x0d0, 0x0d0,
  0x0d0, 0x473, 0x473, 0x473, 0x473, 0x473, 0x473, 0x073,
  0x073, 0x073, 0x073, 0x073, 0x073, 0x073, 0x073, 0x073,
  0x073, 0x073, 0x073, 0x073, 0x073, 0x073, 0x073, 0x073,
  0x073, 0x073, 0x073, 0x0d0, 0x0d0, 0x0d0, 0x0d0, 0x004
  /* the upper 128 are all zeroes */
};

#define g_ascii_isdigit(c) \
  ((ascii_table_data[(unsigned char) (c)] & G_ASCII_DIGIT) != 0)
#define g_ascii_isspace(c) \
  ((ascii_table_data[(unsigned char) (c)] & G_ASCII_SPACE) != 0)
#define g_ascii_isxdigit(c) \
  ((ascii_table_data[(unsigned char) (c)] & G_ASCII_XDIGIT) != 0)

namespace CS
{
  namespace Utility
  {
    // This is actually g_ascii_strtod from glib.
    float strtof (const char* nptr, const char** endptr)
    {
      const char *fail_pos;
      float val;
      struct lconv *locale_data;
      const char *decimal_point;
      int decimal_point_len;
      const char *p, *decimal_point_pos;
      const char *end = NULL; /* Silence gcc */
      int strtod_errno;

      if (nptr == 0) return 0;

      fail_pos = NULL;

      locale_data = localeconv ();
      decimal_point = locale_data->decimal_point;
      decimal_point_len = strlen (decimal_point);

      CS_ASSERT (decimal_point_len != 0);

      decimal_point_pos = NULL;
      end = NULL;

      if (decimal_point[0] != '.' ||
	  decimal_point[1] != 0)
	{
	  p = nptr;
	  /* Skip leading space */
	  while (g_ascii_isspace (*p))
	    p++;

	  /* Skip leading optional sign */
	  if (*p == '+' || *p == '-')
	    p++;

	  if (p[0] == '0' &&
	      (p[1] == 'x' || p[1] == 'X'))
	    {
	      p += 2;
	      /* HEX - find the (optional) decimal point */

	      while (g_ascii_isxdigit (*p))
		p++;

	      if (*p == '.')
		decimal_point_pos = p++;

	      while (g_ascii_isxdigit (*p))
		p++;

	      if (*p == 'p' || *p == 'P')
		p++;
	      if (*p == '+' || *p == '-')
		p++;
	      while (g_ascii_isdigit (*p))
		p++;

	      end = p;
	    }
	  else if (g_ascii_isdigit (*p) || *p == '.')
	    {
	      while (g_ascii_isdigit (*p))
		p++;

	      if (*p == '.')
		decimal_point_pos = p++;

	      while (g_ascii_isdigit (*p))
		p++;

	      if (*p == 'e' || *p == 'E')
		p++;
	      if (*p == '+' || *p == '-')
		p++;
	      while (g_ascii_isdigit (*p))
		p++;

	      end = p;
	    }
	  /* For the other cases, we need not convert the decimal point */
	}

      if (decimal_point_pos)
	{
	  char *copy, *c;

	  /* We need to convert the '.' to the locale specific decimal point */
	  copy = (char*)cs_malloc (end - nptr + 1 + decimal_point_len);

	  c = copy;
	  memcpy (c, nptr, decimal_point_pos - nptr);
	  c += decimal_point_pos - nptr;
	  memcpy (c, decimal_point, decimal_point_len);
	  c += decimal_point_len;
	  memcpy (c, decimal_point_pos + 1, end - (decimal_point_pos + 1));
	  c += end - (decimal_point_pos + 1);
	  *c = 0;

	  errno = 0;
	  val = my_strtof (copy, &fail_pos);
	  strtod_errno = errno;

	  if (fail_pos)
	    {
	      if (fail_pos - copy > decimal_point_pos - nptr)
		fail_pos = (char *)nptr + (fail_pos - copy) - (decimal_point_len - 1);
	      else
		fail_pos = (char *)nptr + (fail_pos - copy);
	    }

	  cs_free (copy);

	}
      else if (end)
	{
	  char *copy;

	  copy = (char*)cs_malloc (end - (char *)nptr + 1);
	  memcpy (copy, nptr, end - nptr);
	  *(copy + (end - (char *)nptr)) = 0;

	  errno = 0;
	  val = my_strtof (copy, &fail_pos);
	  strtod_errno = errno;

	  if (fail_pos)
	    {
	      fail_pos = (char *)nptr + (fail_pos - copy);
	    }

	  cs_free (copy);
	}
      else
	{
	  errno = 0;
	  val = my_strtof (nptr, &fail_pos);
	  strtod_errno = errno;
	}

      if (endptr)
	*endptr = fail_pos;

      errno = strtod_errno;

      return val;
    }

  } // namespace Utility
} // namespace CS
<|endoftext|>