<commit_before>/*
 * This file is open source software, licensed to you under the terms
 * of the Apache License, Version 2.0 (the "License").  See the NOTICE file
 * distributed with this work for additional information regarding copyright
 * ownership.  You may not use this file except in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
/*
 * Copyright 2016 ScyllaDB
 */

#pragma once

#include <seastar/core/file.hh>
#include <seastar/core/shared_ptr.hh>

#include <deque>
#include <atomic>

namespace seastar {
class io_queue;

namespace internal {

// Given a properly aligned vector of iovecs, ensures that it respects the
// IOV_MAX limit, by trimming if necessary. The modified vector still satisfied
// the alignment requirements.
// Returns the final total length of all iovecs.
size_t sanitize_iovecs(std::vector<iovec>& iov, size_t disk_alignment) noexcept;

}

class posix_file_handle_impl : public seastar::file_handle_impl {
    int _fd;
    std::atomic<unsigned>* _refcount;
    dev_t _device_id;
    open_flags _open_flags;
    uint32_t _memory_dma_alignment;
    uint32_t _disk_read_dma_alignment;
    uint32_t _disk_write_dma_alignment;
public:
    posix_file_handle_impl(int fd, open_flags f, std::atomic<unsigned>* refcount, dev_t device_id,
            uint32_t memory_dma_alignment,
            uint32_t disk_read_dma_alignment,
            uint32_t disk_write_dma_alignment)
            : _fd(fd), _refcount(refcount), _device_id(device_id), _open_flags(f)
            , _memory_dma_alignment(memory_dma_alignment)
            , _disk_read_dma_alignment(disk_read_dma_alignment)
            , _disk_write_dma_alignment(disk_write_dma_alignment) {
    }
    virtual ~posix_file_handle_impl();
    posix_file_handle_impl(const posix_file_handle_impl&) = delete;
    posix_file_handle_impl(posix_file_handle_impl&&) = delete;
    virtual shared_ptr<file_impl> to_file() && override;
    virtual std::unique_ptr<seastar::file_handle_impl> clone() const override;
};

class posix_file_impl : public file_impl {
    std::atomic<unsigned>* _refcount = nullptr;
    dev_t _device_id;
    io_queue* _io_queue;
    open_flags _open_flags;
public:
    int _fd;
    posix_file_impl(int fd, open_flags, file_open_options options, dev_t device_id,
            uint32_t block_size);
    posix_file_impl(int fd, open_flags, std::atomic<unsigned>* refcount, dev_t device_id,
            uint32_t memory_dma_alignment,
            uint32_t disk_read_dma_alignment,
            uint32_t disk_write_dma_alignment);
    virtual ~posix_file_impl() override;
    future<size_t> write_dma(uint64_t pos, const void* buffer, size_t len, const io_priority_class& pc) noexcept override;
    future<size_t> write_dma(uint64_t pos, std::vector<iovec> iov, const io_priority_class& pc) noexcept override;
    future<size_t> read_dma(uint64_t pos, void* buffer, size_t len, const io_priority_class& pc) noexcept override;
    future<size_t> read_dma(uint64_t pos, std::vector<iovec> iov, const io_priority_class& pc) noexcept override;
    future<> flush(void) noexcept override;
    future<struct stat> stat(void) noexcept override;
    future<> truncate(uint64_t length) noexcept override;
    future<> discard(uint64_t offset, uint64_t length) noexcept override;
    virtual future<> allocate(uint64_t position, uint64_t length) noexcept override;
    future<uint64_t> size() noexcept override;
    virtual future<> close() noexcept override;
    virtual std::unique_ptr<seastar::file_handle_impl> dup() override;
    virtual subscription<directory_entry> list_directory(std::function<future<> (directory_entry de)> next) override;
    virtual future<temporary_buffer<uint8_t>> dma_read_bulk(uint64_t offset, size_t range_size, const io_priority_class& pc) noexcept override;

    open_flags flags() const {
        return _open_flags;
    }
private:
    void query_dma_alignment(uint32_t block_size);

    /**
     * Try to read from the given position where the previous short read has
     * stopped. Check the EOF condition.
     *
     * The below code assumes the following: short reads due to I/O errors
     * always end at address aligned to HW block boundary. Therefore if we issue
     * a new read operation from the next position we are promised to get an
     * error (different from EINVAL). If we've got a short read because we have
     * reached EOF then the above read would either return a zero-length success
     * (if the file size is aligned to HW block size) or an EINVAL error (if
     * file length is not aligned to HW block size).
     *
     * @param pos offset to read from
     * @param len number of bytes to read
     * @param pc the IO priority class under which to queue this operation
     *
     * @return temporary buffer with read data or zero-sized temporary buffer if
     *         pos is at or beyond EOF.
     * @throw appropriate exception in case of I/O error.
     */
    future<temporary_buffer<uint8_t>>
    read_maybe_eof(uint64_t pos, size_t len, const io_priority_class& pc);
};

// The Linux XFS implementation is challenged wrt. append: a write that changes
// eof will be blocked by any other concurrent AIO operation to the same file, whether
// it changes file size or not. Furthermore, ftruncate() will also block and be blocked
// by AIO, so attempts to game the system and call ftruncate() have to be done very carefully.
//
// Other Linux filesystems may have different locking rules, so this may need to be
// adjusted for them.
class append_challenged_posix_file_impl : public posix_file_impl, public enable_shared_from_this<append_challenged_posix_file_impl> {
    // File size as a result of completed kernel operations (writes and truncates)
    uint64_t _committed_size;
    // File size as a result of seastar API calls
    uint64_t _logical_size;
    // Pending operations
    enum class opcode {
        invalid,
        read,
        write,
        truncate,
        flush,
    };
    struct op {
        opcode type;
        uint64_t pos;
        size_t len;
        std::function<future<> ()> run;
    };
    // Queue of pending operations; processed from front to end to avoid
    // starvation, but can issue concurrent operations.
    std::deque<op> _q;
    unsigned _max_size_changing_ops = 0;
    unsigned _current_non_size_changing_ops = 0;
    unsigned _current_size_changing_ops = 0;
    bool _fsync_is_exclusive = true;

    // Set when the user is closing the file
    enum class state { open, draining, closing, closed };
    state _closing_state = state::open;

    bool _sloppy_size = false;
    uint64_t _sloppy_size_hint;
    // Fulfiled when _done and I/O is complete
    promise<> _completed;
private:
    void commit_size(uint64_t size) noexcept;
    bool must_run_alone(const op& candidate) const noexcept;
    bool size_changing(const op& candidate) const noexcept;
    bool may_dispatch(const op& candidate) const noexcept;
    void dispatch(op& candidate) noexcept;
    void optimize_queue() noexcept;
    void process_queue() noexcept;
    bool may_quit() const noexcept;
    void enqueue_op(op&& op);
    template <typename... T, typename Func>
    future<T...> enqueue(opcode type, uint64_t pos, size_t len, Func&& func) noexcept {
        try {
            auto pr = make_lw_shared(promise<T...>());
            auto fut = pr->get_future();
            auto op_func = [func = std::move(func), pr = std::move(pr)] () mutable {
                return futurize_invoke(std::move(func)).then_wrapped([pr = std::move(pr)] (future<T...> f) mutable {
                    f.forward_to(std::move(*pr));
                });
            };
            enqueue_op({type, pos, len, op_func});
            return fut;
        } catch (...) {
            return make_exception_future<T...>(std::current_exception());
        }
    }
public:
    append_challenged_posix_file_impl(int fd, open_flags, file_open_options options, unsigned max_size_changing_ops, bool fsync_is_exclusive, dev_t device_id, size_t block_size);
    ~append_challenged_posix_file_impl() override;
    future<size_t> read_dma(uint64_t pos, void* buffer, size_t len, const io_priority_class& pc) noexcept override;
    future<size_t> read_dma(uint64_t pos, std::vector<iovec> iov, const io_priority_class& pc) noexcept override;
    future<size_t> write_dma(uint64_t pos, const void* buffer, size_t len, const io_priority_class& pc) noexcept override;
    future<size_t> write_dma(uint64_t pos, std::vector<iovec> iov, const io_priority_class& pc) noexcept override;
    future<> flush() noexcept override;
    future<struct stat> stat() noexcept override;
    future<> truncate(uint64_t length) noexcept override;
    future<uint64_t> size() noexcept override;
    future<> close() noexcept override;
};

class blockdev_file_impl : public posix_file_impl {
public:
    blockdev_file_impl(int fd, open_flags, file_open_options options, dev_t device_id, size_t block_size);
    future<> truncate(uint64_t length) noexcept override;
    future<> discard(uint64_t offset, uint64_t length) noexcept override;
    future<uint64_t> size() noexcept override;
    virtual future<> allocate(uint64_t position, uint64_t length) noexcept override;
};

}
<commit_msg>append-challenged-file: Switch onto noncopyable function<commit_after>/*
 * This file is open source software, licensed to you under the terms
 * of the Apache License, Version 2.0 (the "License").  See the NOTICE file
 * distributed with this work for additional information regarding copyright
 * ownership.  You may not use this file except in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
/*
 * Copyright 2016 ScyllaDB
 */

#pragma once

#include <seastar/core/file.hh>
#include <seastar/core/shared_ptr.hh>

#include <deque>
#include <atomic>

namespace seastar {
class io_queue;

namespace internal {

// Given a properly aligned vector of iovecs, ensures that it respects the
// IOV_MAX limit, by trimming if necessary. The modified vector still satisfied
// the alignment requirements.
// Returns the final total length of all iovecs.
size_t sanitize_iovecs(std::vector<iovec>& iov, size_t disk_alignment) noexcept;

}

class posix_file_handle_impl : public seastar::file_handle_impl {
    int _fd;
    std::atomic<unsigned>* _refcount;
    dev_t _device_id;
    open_flags _open_flags;
    uint32_t _memory_dma_alignment;
    uint32_t _disk_read_dma_alignment;
    uint32_t _disk_write_dma_alignment;
public:
    posix_file_handle_impl(int fd, open_flags f, std::atomic<unsigned>* refcount, dev_t device_id,
            uint32_t memory_dma_alignment,
            uint32_t disk_read_dma_alignment,
            uint32_t disk_write_dma_alignment)
            : _fd(fd), _refcount(refcount), _device_id(device_id), _open_flags(f)
            , _memory_dma_alignment(memory_dma_alignment)
            , _disk_read_dma_alignment(disk_read_dma_alignment)
            , _disk_write_dma_alignment(disk_write_dma_alignment) {
    }
    virtual ~posix_file_handle_impl();
    posix_file_handle_impl(const posix_file_handle_impl&) = delete;
    posix_file_handle_impl(posix_file_handle_impl&&) = delete;
    virtual shared_ptr<file_impl> to_file() && override;
    virtual std::unique_ptr<seastar::file_handle_impl> clone() const override;
};

class posix_file_impl : public file_impl {
    std::atomic<unsigned>* _refcount = nullptr;
    dev_t _device_id;
    io_queue* _io_queue;
    open_flags _open_flags;
public:
    int _fd;
    posix_file_impl(int fd, open_flags, file_open_options options, dev_t device_id,
            uint32_t block_size);
    posix_file_impl(int fd, open_flags, std::atomic<unsigned>* refcount, dev_t device_id,
            uint32_t memory_dma_alignment,
            uint32_t disk_read_dma_alignment,
            uint32_t disk_write_dma_alignment);
    virtual ~posix_file_impl() override;
    future<size_t> write_dma(uint64_t pos, const void* buffer, size_t len, const io_priority_class& pc) noexcept override;
    future<size_t> write_dma(uint64_t pos, std::vector<iovec> iov, const io_priority_class& pc) noexcept override;
    future<size_t> read_dma(uint64_t pos, void* buffer, size_t len, const io_priority_class& pc) noexcept override;
    future<size_t> read_dma(uint64_t pos, std::vector<iovec> iov, const io_priority_class& pc) noexcept override;
    future<> flush(void) noexcept override;
    future<struct stat> stat(void) noexcept override;
    future<> truncate(uint64_t length) noexcept override;
    future<> discard(uint64_t offset, uint64_t length) noexcept override;
    virtual future<> allocate(uint64_t position, uint64_t length) noexcept override;
    future<uint64_t> size() noexcept override;
    virtual future<> close() noexcept override;
    virtual std::unique_ptr<seastar::file_handle_impl> dup() override;
    virtual subscription<directory_entry> list_directory(std::function<future<> (directory_entry de)> next) override;
    virtual future<temporary_buffer<uint8_t>> dma_read_bulk(uint64_t offset, size_t range_size, const io_priority_class& pc) noexcept override;

    open_flags flags() const {
        return _open_flags;
    }
private:
    void query_dma_alignment(uint32_t block_size);

    /**
     * Try to read from the given position where the previous short read has
     * stopped. Check the EOF condition.
     *
     * The below code assumes the following: short reads due to I/O errors
     * always end at address aligned to HW block boundary. Therefore if we issue
     * a new read operation from the next position we are promised to get an
     * error (different from EINVAL). If we've got a short read because we have
     * reached EOF then the above read would either return a zero-length success
     * (if the file size is aligned to HW block size) or an EINVAL error (if
     * file length is not aligned to HW block size).
     *
     * @param pos offset to read from
     * @param len number of bytes to read
     * @param pc the IO priority class under which to queue this operation
     *
     * @return temporary buffer with read data or zero-sized temporary buffer if
     *         pos is at or beyond EOF.
     * @throw appropriate exception in case of I/O error.
     */
    future<temporary_buffer<uint8_t>>
    read_maybe_eof(uint64_t pos, size_t len, const io_priority_class& pc);
};

// The Linux XFS implementation is challenged wrt. append: a write that changes
// eof will be blocked by any other concurrent AIO operation to the same file, whether
// it changes file size or not. Furthermore, ftruncate() will also block and be blocked
// by AIO, so attempts to game the system and call ftruncate() have to be done very carefully.
//
// Other Linux filesystems may have different locking rules, so this may need to be
// adjusted for them.
class append_challenged_posix_file_impl : public posix_file_impl, public enable_shared_from_this<append_challenged_posix_file_impl> {
    // File size as a result of completed kernel operations (writes and truncates)
    uint64_t _committed_size;
    // File size as a result of seastar API calls
    uint64_t _logical_size;
    // Pending operations
    enum class opcode {
        invalid,
        read,
        write,
        truncate,
        flush,
    };
    struct op {
        opcode type;
        uint64_t pos;
        size_t len;
        noncopyable_function<future<> ()> run;
    };
    // Queue of pending operations; processed from front to end to avoid
    // starvation, but can issue concurrent operations.
    std::deque<op> _q;
    unsigned _max_size_changing_ops = 0;
    unsigned _current_non_size_changing_ops = 0;
    unsigned _current_size_changing_ops = 0;
    bool _fsync_is_exclusive = true;

    // Set when the user is closing the file
    enum class state { open, draining, closing, closed };
    state _closing_state = state::open;

    bool _sloppy_size = false;
    uint64_t _sloppy_size_hint;
    // Fulfiled when _done and I/O is complete
    promise<> _completed;
private:
    void commit_size(uint64_t size) noexcept;
    bool must_run_alone(const op& candidate) const noexcept;
    bool size_changing(const op& candidate) const noexcept;
    bool may_dispatch(const op& candidate) const noexcept;
    void dispatch(op& candidate) noexcept;
    void optimize_queue() noexcept;
    void process_queue() noexcept;
    bool may_quit() const noexcept;
    void enqueue_op(op&& op);
    template <typename... T, typename Func>
    future<T...> enqueue(opcode type, uint64_t pos, size_t len, Func&& func) noexcept {
        try {
            auto pr = make_lw_shared(promise<T...>());
            auto fut = pr->get_future();
            auto op_func = [func = std::move(func), pr = std::move(pr)] () mutable {
                return futurize_invoke(std::move(func)).then_wrapped([pr = std::move(pr)] (future<T...> f) mutable {
                    f.forward_to(std::move(*pr));
                });
            };
            enqueue_op({type, pos, len, std::move(op_func)});
            return fut;
        } catch (...) {
            return make_exception_future<T...>(std::current_exception());
        }
    }
public:
    append_challenged_posix_file_impl(int fd, open_flags, file_open_options options, unsigned max_size_changing_ops, bool fsync_is_exclusive, dev_t device_id, size_t block_size);
    ~append_challenged_posix_file_impl() override;
    future<size_t> read_dma(uint64_t pos, void* buffer, size_t len, const io_priority_class& pc) noexcept override;
    future<size_t> read_dma(uint64_t pos, std::vector<iovec> iov, const io_priority_class& pc) noexcept override;
    future<size_t> write_dma(uint64_t pos, const void* buffer, size_t len, const io_priority_class& pc) noexcept override;
    future<size_t> write_dma(uint64_t pos, std::vector<iovec> iov, const io_priority_class& pc) noexcept override;
    future<> flush() noexcept override;
    future<struct stat> stat() noexcept override;
    future<> truncate(uint64_t length) noexcept override;
    future<uint64_t> size() noexcept override;
    future<> close() noexcept override;
};

class blockdev_file_impl : public posix_file_impl {
public:
    blockdev_file_impl(int fd, open_flags, file_open_options options, dev_t device_id, size_t block_size);
    future<> truncate(uint64_t length) noexcept override;
    future<> discard(uint64_t offset, uint64_t length) noexcept override;
    future<uint64_t> size() noexcept override;
    virtual future<> allocate(uint64_t position, uint64_t length) noexcept override;
};

}
<|endoftext|>


<commit_before>// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include "cf-handler.h"
#include <dirent.h>
#include <stdio.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <vespa/defaults.h>
#include <vespa/config/common/configsystem.h>
#include <vespa/config/common/exceptions.h>

#include <vespa/log/log.h>
LOG_SETUP(".cf-handler");

CfHandler::CfHandler() : _subscriber() {}

CfHandler::~CfHandler()
{
}

void
CfHandler::subscribe(const std::string & configId, uint64_t timeoutMS)
{
    _handle = _subscriber.subscribe<LogforwarderConfig>(configId, timeoutMS);
}

namespace {
std::string
cfFilePath() {
    std::string path = vespa::Defaults::underVespaHome("var/db/vespa/splunk");
    DIR *dp = opendir(path.c_str());
    if (dp == NULL) {
        if (errno != ENOTDIR || mkdir(path.c_str(), 0755) != 0) {
            perror(path.c_str());
        }
    }
    if (dp != NULL) closedir(dp);
    path += "/deploymentclient.conf";
    return path;
}
}

void
CfHandler::doConfigure()
{
    std::unique_ptr<LogforwarderConfig> cfg(_handle->getConfig());
    const LogforwarderConfig& config(*cfg);

    std::string path = cfFilePath();
    std::string tmpPath = path + ".new";
    FILE *fp = fopen(tmpPath.c_str(), "w");
    if (fp == NULL) return;

    fprintf(fp, "[deployment-client]\n");
    fprintf(fp, "clientName = %s\n", config.clientName.c_str());
    fprintf(fp, "\n");
    fprintf(fp, "[target-broker:deploymentServer]\n");
    fprintf(fp, "targetUri = %s\n", config.deploymentServer.c_str());

    fclose(fp);
    rename(tmpPath.c_str(), path.c_str());
}

void
CfHandler::check()
{
    if (_subscriber.nextConfig(0)) {
        doConfigure();
    }
}

constexpr uint64_t CONFIG_TIMEOUT_MS = 30 * 1000;

void
CfHandler::start(const char *configId)
{
    LOG(debug, "Reading configuration with id '%s'", configId);
    try {
        subscribe(configId, CONFIG_TIMEOUT_MS);
        doConfigure();
    } catch (config::ConfigTimeoutException & ex) {
        LOG(warning, "Timout getting config, please check your setup. Will exit and restart: %s", ex.getMessage().c_str());
        exit(EXIT_FAILURE);
    } catch (config::InvalidConfigException& ex) {
        LOG(error, "Fatal: Invalid configuration, please check your setup: %s", ex.getMessage().c_str());
        exit(EXIT_FAILURE);
    } catch (config::ConfigRuntimeException& ex) {
        LOG(error, "Fatal: Could not get config, please check your setup: %s", ex.getMessage().c_str());
        exit(EXIT_FAILURE);
    }
}
<commit_msg>fix logic<commit_after>// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include "cf-handler.h"
#include <dirent.h>
#include <stdio.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <vespa/defaults.h>
#include <vespa/config/common/configsystem.h>
#include <vespa/config/common/exceptions.h>

#include <vespa/log/log.h>
LOG_SETUP(".cf-handler");

CfHandler::CfHandler() : _subscriber() {}

CfHandler::~CfHandler()
{
}

void
CfHandler::subscribe(const std::string & configId, uint64_t timeoutMS)
{
    _handle = _subscriber.subscribe<LogforwarderConfig>(configId, timeoutMS);
}

namespace {
std::string
cfFilePath() {
    std::string path = vespa::Defaults::underVespaHome("var/db/vespa/splunk");
    DIR *dp = opendir(path.c_str());
    if (dp == NULL) {
        if (errno != ENOENT || mkdir(path.c_str(), 0755) != 0) {
            perror(path.c_str());
        }
    } else {
        closedir(dp);
    }
    path += "/deploymentclient.conf";
    return path;
}
}

void
CfHandler::doConfigure()
{
    std::unique_ptr<LogforwarderConfig> cfg(_handle->getConfig());
    const LogforwarderConfig& config(*cfg);

    std::string path = cfFilePath();
    std::string tmpPath = path + ".new";
    FILE *fp = fopen(tmpPath.c_str(), "w");
    if (fp == NULL) return;

    fprintf(fp, "[deployment-client]\n");
    fprintf(fp, "clientName = %s\n", config.clientName.c_str());
    fprintf(fp, "\n");
    fprintf(fp, "[target-broker:deploymentServer]\n");
    fprintf(fp, "targetUri = %s\n", config.deploymentServer.c_str());

    fclose(fp);
    rename(tmpPath.c_str(), path.c_str());
}

void
CfHandler::check()
{
    if (_subscriber.nextConfig(0)) {
        doConfigure();
    }
}

constexpr uint64_t CONFIG_TIMEOUT_MS = 30 * 1000;

void
CfHandler::start(const char *configId)
{
    LOG(debug, "Reading configuration with id '%s'", configId);
    try {
        subscribe(configId, CONFIG_TIMEOUT_MS);
    } catch (config::ConfigTimeoutException & ex) {
        LOG(warning, "Timout getting config, please check your setup. Will exit and restart: %s", ex.getMessage().c_str());
        exit(EXIT_FAILURE);
    } catch (config::InvalidConfigException& ex) {
        LOG(error, "Fatal: Invalid configuration, please check your setup: %s", ex.getMessage().c_str());
        exit(EXIT_FAILURE);
    } catch (config::ConfigRuntimeException& ex) {
        LOG(error, "Fatal: Could not get config, please check your setup: %s", ex.getMessage().c_str());
        exit(EXIT_FAILURE);
    }
}
<|endoftext|>


<commit_before>/*******************************************************************************
 * This file is part of "Patrick's Programming Library", Version 7 (PPL7).
 * Web: http://www.pfp.de/ppl/
 *
 * $Author$
 * $Revision$
 * $Date$
 * $Id$
 *
 *******************************************************************************
 * Copyright (c) 2013, Patrick Fedick <patrick@pfp.de>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *    1. Redistributions of source code must retain the above copyright notice, this
 *       list of conditions and the following disclaimer.
 *    2. Redistributions in binary form must reproduce the above copyright notice,
 *       this list of conditions and the following disclaimer in the documentation
 *       and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER AND CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
 * THE POSSIBILITY OF SUCH DAMAGE.
 *******************************************************************************/

#include "prolog.h"
#ifdef HAVE_STDIO_H
#include <stdio.h>
#endif
#ifdef HAVE_STDLIB_H
#include <stdlib.h>
#endif
#ifdef HAVE_STRING_H
#include <string.h>
#endif
#ifdef HAVE_LIBMHASH
#define PROTOTYPES 1
#undef HAVE__BOOL
#include <mutils/mhash.h>
#endif

#ifdef HAVE_OPENSSL
#include <openssl/evp.h>
#endif

#ifdef HAVE_LIBZ
#include <zlib.h>
#endif

#include "ppl7.h"
#include "ppl7-crypto.h"

namespace ppl7 {

bool __OpenSSLDigestAdded = false;

Mutex __OpenSSLGlobalMutex;

void InitOpenSSLDigest()
{
#ifndef HAVE_OPENSSL
	throw UnsupportedFeatureException("OpenSSL");
#else
	__OpenSSLGlobalMutex.lock();
	::OpenSSL_add_all_digests();
	__OpenSSLDigestAdded=true;
	__OpenSSLGlobalMutex.unlock();
#endif
}

Digest::Digest()
{
	bytecount=0;
	m=NULL;
	ret=NULL;
	ctx=NULL;
#ifndef HAVE_OPENSSL
	throw UnsupportedFeatureException("OpenSSL");
#else
	if (!__OpenSSLDigestAdded) {
		InitOpenSSLDigest();
	}
#endif
}

Digest::~Digest()
{
#ifdef HAVE_OPENSSL
	free(ret);
	if (ctx) EVP_MD_CTX_destroy((EVP_MD_CTX*)ctx);
#endif
}

Digest::Digest(const String &name)
{
	bytecount=0;
	m=NULL;
	ret=NULL;
	ctx=NULL;
#ifndef HAVE_OPENSSL
	throw UnsupportedFeatureException("OpenSSL");
#else
	if (!__OpenSSLDigestAdded) {
		InitOpenSSLDigest();
	}
	setAlgorithm(name);
#endif
}

Digest::Digest(Algorithm algorithm)
{
	bytecount=0;
	m=NULL;
	ret=NULL;
	ctx=NULL;
#ifndef HAVE_OPENSSL
	throw UnsupportedFeatureException("OpenSSL");
#else
	if (!__OpenSSLDigestAdded) {
		InitOpenSSLDigest();
	}
	setAlgorithm(algorithm);
#endif
}

void Digest::setAlgorithm(const String &name)
{
#ifndef HAVE_OPENSSL
	throw UnsupportedFeatureException("OpenSSL");
#else
	m=EVP_get_digestbyname((const char*)name);
	if (!m) {
		throw InvalidAlgorithmException("%s",(const char*)name);
	}
	if (!ctx) {
		ctx=EVP_MD_CTX_create();
		if (!ctx) throw OutOfMemoryException();
	} else {
		reset();
	}
	EVP_DigestInit_ex((EVP_MD_CTX*)ctx,(const EVP_MD*)m, NULL);
#endif
}

void Digest::setAlgorithm(Algorithm algorithm)
{
#ifndef HAVE_OPENSSL
	throw UnsupportedFeatureException("OpenSSL");
#else

	switch(algorithm) {
#ifndef OPENSSL_NO_MD4
		case Algo_MD4: m=EVP_md4(); break;
#endif
#ifndef OPENSSL_NO_MD5
		case Algo_MD5: m=EVP_md5(); break;
#endif
#ifndef OPENSSL_NO_SHA
		case Algo_SHA1: m=EVP_sha1(); break;
		case Algo_ECDSA: m=EVP_ecdsa(); break;

#endif
#ifndef OPENSSL_NO_SHA256
		case Algo_SHA224: m=EVP_sha224(); break;
		case Algo_SHA256: m=EVP_sha256(); break;
#endif
#ifndef OPENSSL_NO_SHA512
		case Algo_SHA384: m=EVP_sha384(); break;
		case Algo_SHA512: m=EVP_sha512(); break;
#endif
#ifndef OPENSSL_NO_WHIRLPOOL
#if OPENSSL_VERSION_NUMBER >= 0x10001000L
		case Algo_WHIRLPOOL: m=EVP_whirlpool(); break;
#endif
#endif
#ifndef OPENSSL_NO_RIPEMD
		case Algo_RIPEMD160: m=EVP_ripemd160(); break;
#endif

		default: throw InvalidAlgorithmException();
	}
	if (!m) {
		throw InvalidAlgorithmException("%i",algorithm);
	}
	if (!ctx) {
		ctx=EVP_MD_CTX_create();
		if (!ctx) throw OutOfMemoryException();
	} else {
		reset();
	}
	EVP_DigestInit_ex((EVP_MD_CTX*)ctx,(const EVP_MD*)m, NULL);
#endif
}

void Digest::addData(const void *data, size_t size)
{
#ifndef HAVE_OPENSSL
	throw UnsupportedFeatureException("OpenSSL");
#else
	if (!m) throw NoAlgorithmSpecifiedException();
	EVP_DigestUpdate((EVP_MD_CTX*)ctx,data,size);
	bytecount+=size;
#endif
}

void Digest::addData(const ByteArrayPtr &data)
{
	addData(data.ptr(),data.size());
}

void Digest::addData(const String &data)
{
	addData(data.getPtr(),data.size());
}

void Digest::addData(const WideString &data)
{
	addData(data.getPtr(),data.size());
}

void Digest::addData(FileObject &file)
{
	file.seek(0);
	size_t bsize=1024*1024*1;		// We allocate 1 MB maximum
	ppluint64 fsize=file.size();
	if (fsize<bsize) bsize=fsize;	// or filesize if file is < 1 MB
	void *buffer=malloc(bsize);
	if (!buffer) {
		throw OutOfMemoryException();
	}
	ppluint64 rest=fsize;
	try {
		while(rest) {
			size_t bytes=rest;
			if (bytes>bsize) bytes=bsize;
			if (!file.read(buffer,bytes)) {
				throw ReadException();
			}
			addData(buffer,bytes);
			rest-=bytes;
		}
	} catch (...) {
		free(buffer);
		throw;
	}
	free(buffer);

}

void Digest::addFile(const String &filename)
{
	File ff;
	ff.open(filename,File::READ);
	addData(ff);
}

ppluint64 Digest::bytesHashed() const
{
	return bytecount;
}

void Digest::saveDigest(ByteArray &result)
{
#ifndef HAVE_OPENSSL
	throw UnsupportedFeatureException("OpenSSL");
#else
	result=getDigest();
#endif
}

void Digest::saveDigest(String &result)
{
#ifndef HAVE_OPENSSL
	throw UnsupportedFeatureException("OpenSSL");
#else
	ByteArray ba=getDigest();
	result=ba.toHex();
#endif
}

void Digest::saveDigest(WideString &result)
{
#ifndef HAVE_OPENSSL
	throw UnsupportedFeatureException("OpenSSL");
#else
	ByteArray ba=getDigest();
	result=ba.toHex();
#endif
}

ByteArray Digest::getDigest()
{
#ifndef HAVE_OPENSSL
	throw UnsupportedFeatureException("OpenSSL");
#else
	unsigned int len;
	if (!ret) {
		ret=(unsigned char*)malloc(EVP_MAX_MD_SIZE);
		if (!ret) throw OutOfMemoryException();
	}
	EVP_DigestFinal((EVP_MD_CTX*)ctx,ret,&len);
	EVP_MD_CTX_cleanup((EVP_MD_CTX*)ctx);
	EVP_DigestInit_ex((EVP_MD_CTX*)ctx,(const EVP_MD*)m, NULL);
	bytecount=0;
	return ByteArray(ret,len);
#endif
}

void Digest::reset()
{
#ifndef HAVE_OPENSSL
	throw UnsupportedFeatureException("OpenSSL");
#else
	if (!m) throw NoAlgorithmSpecifiedException();
	if (!ctx) throw NoAlgorithmSpecifiedException();
	EVP_MD_CTX_cleanup((EVP_MD_CTX*)ctx);
	EVP_DigestInit((EVP_MD_CTX*)ctx,(const EVP_MD*)m);
	bytecount=0;
#endif
}

ByteArray Digest::hash(const ByteArrayPtr &data, Algorithm algorithm)
{
	Digest dig(algorithm);
	dig.addData(data);
	return dig.getDigest();
}

ByteArray Digest::hash(const ByteArrayPtr &data, const String &algorithmName)
{
	Digest dig(algorithmName);
	dig.addData(data);
	return dig.getDigest();
}

ByteArray Digest::md4(const ByteArrayPtr &data)
{
	return Digest::hash(data,Algo_MD4);
}

ByteArray Digest::md5(const ByteArrayPtr &data)
{
	return Digest::hash(data,Algo_MD5);
}

ByteArray Digest::sha1(const ByteArrayPtr &data)
{
	return Digest::hash(data,Algo_SHA1);
}

ByteArray Digest::sha224(const ByteArrayPtr &data)
{
	return Digest::hash(data,Algo_SHA224);
}

ByteArray Digest::sha256(const ByteArrayPtr &data)
{
	return Digest::hash(data,Algo_SHA256);
}

ByteArray Digest::sha384(const ByteArrayPtr &data)
{
	return Digest::hash(data,Algo_SHA384);
}

ByteArray Digest::sha512(const ByteArrayPtr &data)
{
	return Digest::hash(data,Algo_SHA512);
}

ByteArray Digest::ecdsa(const ByteArrayPtr &data)
{
	return Digest::hash(data,Algo_ECDSA);
}

ppluint32 Digest::crc32(const ByteArrayPtr &data)
{
#ifdef HAVE_LIBZ
	uLong crc=::crc32(0L,Z_NULL,0);
	return ::crc32(crc,(const Bytef*)data.ptr(),(uInt)data.size());
#endif
	return Crc32(data.ptr(),data.size());
}

ppluint32 Digest::adler32(const ByteArrayPtr &data)
{
     const unsigned char *buffer = (const unsigned char *)data.ptr();
     size_t buflength=data.size();
     ppluint32 s1 = 1;
     ppluint32 s2 = 0;

     for (size_t n = 0; n < buflength; n++) {
        s1 = (s1 + buffer[n]) % 65521;
        s2 = (s2 + s1) % 65521;
     }
     return (s2 << 16) | s1;
  }
}
<commit_msg>take care that InitOpenSSLDigest calls OpenSSL_add_all_digests only once in multithreaded environment<commit_after>/*******************************************************************************
 * This file is part of "Patrick's Programming Library", Version 7 (PPL7).
 * Web: http://www.pfp.de/ppl/
 *
 * $Author$
 * $Revision$
 * $Date$
 * $Id$
 *
 *******************************************************************************
 * Copyright (c) 2013, Patrick Fedick <patrick@pfp.de>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *    1. Redistributions of source code must retain the above copyright notice, this
 *       list of conditions and the following disclaimer.
 *    2. Redistributions in binary form must reproduce the above copyright notice,
 *       this list of conditions and the following disclaimer in the documentation
 *       and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER AND CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
 * THE POSSIBILITY OF SUCH DAMAGE.
 *******************************************************************************/

#include "prolog.h"
#ifdef HAVE_STDIO_H
#include <stdio.h>
#endif
#ifdef HAVE_STDLIB_H
#include <stdlib.h>
#endif
#ifdef HAVE_STRING_H
#include <string.h>
#endif
#ifdef HAVE_LIBMHASH
#define PROTOTYPES 1
#undef HAVE__BOOL
#include <mutils/mhash.h>
#endif

#ifdef HAVE_OPENSSL
#include <openssl/evp.h>
#endif

#ifdef HAVE_LIBZ
#include <zlib.h>
#endif

#include "ppl7.h"
#include "ppl7-crypto.h"

namespace ppl7 {

bool __OpenSSLDigestAdded = false;

Mutex __OpenSSLGlobalMutex;

void InitOpenSSLDigest()
{
#ifndef HAVE_OPENSSL
	throw UnsupportedFeatureException("OpenSSL");
#else
	__OpenSSLGlobalMutex.lock();
	if (!__OpenSSLDigestAdded) {
		::OpenSSL_add_all_digests();
		__OpenSSLDigestAdded=true;
	}
	__OpenSSLGlobalMutex.unlock();
#endif
}

Digest::Digest()
{
	bytecount=0;
	m=NULL;
	ret=NULL;
	ctx=NULL;
#ifndef HAVE_OPENSSL
	throw UnsupportedFeatureException("OpenSSL");
#else
	if (!__OpenSSLDigestAdded) {
		InitOpenSSLDigest();
	}
#endif
}

Digest::~Digest()
{
#ifdef HAVE_OPENSSL
	free(ret);
	if (ctx) EVP_MD_CTX_destroy((EVP_MD_CTX*)ctx);
#endif
}

Digest::Digest(const String &name)
{
	bytecount=0;
	m=NULL;
	ret=NULL;
	ctx=NULL;
#ifndef HAVE_OPENSSL
	throw UnsupportedFeatureException("OpenSSL");
#else
	if (!__OpenSSLDigestAdded) {
		InitOpenSSLDigest();
	}
	setAlgorithm(name);
#endif
}

Digest::Digest(Algorithm algorithm)
{
	bytecount=0;
	m=NULL;
	ret=NULL;
	ctx=NULL;
#ifndef HAVE_OPENSSL
	throw UnsupportedFeatureException("OpenSSL");
#else
	if (!__OpenSSLDigestAdded) {
		InitOpenSSLDigest();
	}
	setAlgorithm(algorithm);
#endif
}

void Digest::setAlgorithm(const String &name)
{
#ifndef HAVE_OPENSSL
	throw UnsupportedFeatureException("OpenSSL");
#else
	m=EVP_get_digestbyname((const char*)name);
	if (!m) {
		throw InvalidAlgorithmException("%s",(const char*)name);
	}
	if (!ctx) {
		ctx=EVP_MD_CTX_create();
		if (!ctx) throw OutOfMemoryException();
	} else {
		reset();
	}
	EVP_DigestInit_ex((EVP_MD_CTX*)ctx,(const EVP_MD*)m, NULL);
#endif
}

void Digest::setAlgorithm(Algorithm algorithm)
{
#ifndef HAVE_OPENSSL
	throw UnsupportedFeatureException("OpenSSL");
#else

	switch(algorithm) {
#ifndef OPENSSL_NO_MD4
		case Algo_MD4: m=EVP_md4(); break;
#endif
#ifndef OPENSSL_NO_MD5
		case Algo_MD5: m=EVP_md5(); break;
#endif
#ifndef OPENSSL_NO_SHA
		case Algo_SHA1: m=EVP_sha1(); break;
		case Algo_ECDSA: m=EVP_ecdsa(); break;

#endif
#ifndef OPENSSL_NO_SHA256
		case Algo_SHA224: m=EVP_sha224(); break;
		case Algo_SHA256: m=EVP_sha256(); break;
#endif
#ifndef OPENSSL_NO_SHA512
		case Algo_SHA384: m=EVP_sha384(); break;
		case Algo_SHA512: m=EVP_sha512(); break;
#endif
#ifndef OPENSSL_NO_WHIRLPOOL
#if OPENSSL_VERSION_NUMBER >= 0x10001000L
		case Algo_WHIRLPOOL: m=EVP_whirlpool(); break;
#endif
#endif
#ifndef OPENSSL_NO_RIPEMD
		case Algo_RIPEMD160: m=EVP_ripemd160(); break;
#endif

		default: throw InvalidAlgorithmException();
	}
	if (!m) {
		throw InvalidAlgorithmException("%i",algorithm);
	}
	if (!ctx) {
		ctx=EVP_MD_CTX_create();
		if (!ctx) throw OutOfMemoryException();
	} else {
		reset();
	}
	EVP_DigestInit_ex((EVP_MD_CTX*)ctx,(const EVP_MD*)m, NULL);
#endif
}

void Digest::addData(const void *data, size_t size)
{
#ifndef HAVE_OPENSSL
	throw UnsupportedFeatureException("OpenSSL");
#else
	if (!m) throw NoAlgorithmSpecifiedException();
	EVP_DigestUpdate((EVP_MD_CTX*)ctx,data,size);
	bytecount+=size;
#endif
}

void Digest::addData(const ByteArrayPtr &data)
{
	addData(data.ptr(),data.size());
}

void Digest::addData(const String &data)
{
	addData(data.getPtr(),data.size());
}

void Digest::addData(const WideString &data)
{
	addData(data.getPtr(),data.size());
}

void Digest::addData(FileObject &file)
{
	file.seek(0);
	size_t bsize=1024*1024*1;		// We allocate 1 MB maximum
	ppluint64 fsize=file.size();
	if (fsize<bsize) bsize=fsize;	// or filesize if file is < 1 MB
	void *buffer=malloc(bsize);
	if (!buffer) {
		throw OutOfMemoryException();
	}
	ppluint64 rest=fsize;
	try {
		while(rest) {
			size_t bytes=rest;
			if (bytes>bsize) bytes=bsize;
			if (!file.read(buffer,bytes)) {
				throw ReadException();
			}
			addData(buffer,bytes);
			rest-=bytes;
		}
	} catch (...) {
		free(buffer);
		throw;
	}
	free(buffer);

}

void Digest::addFile(const String &filename)
{
	File ff;
	ff.open(filename,File::READ);
	addData(ff);
}

ppluint64 Digest::bytesHashed() const
{
	return bytecount;
}

void Digest::saveDigest(ByteArray &result)
{
#ifndef HAVE_OPENSSL
	throw UnsupportedFeatureException("OpenSSL");
#else
	result=getDigest();
#endif
}

void Digest::saveDigest(String &result)
{
#ifndef HAVE_OPENSSL
	throw UnsupportedFeatureException("OpenSSL");
#else
	ByteArray ba=getDigest();
	result=ba.toHex();
#endif
}

void Digest::saveDigest(WideString &result)
{
#ifndef HAVE_OPENSSL
	throw UnsupportedFeatureException("OpenSSL");
#else
	ByteArray ba=getDigest();
	result=ba.toHex();
#endif
}

ByteArray Digest::getDigest()
{
#ifndef HAVE_OPENSSL
	throw UnsupportedFeatureException("OpenSSL");
#else
	unsigned int len;
	if (!ret) {
		ret=(unsigned char*)malloc(EVP_MAX_MD_SIZE);
		if (!ret) throw OutOfMemoryException();
	}
	EVP_DigestFinal((EVP_MD_CTX*)ctx,ret,&len);
	EVP_MD_CTX_cleanup((EVP_MD_CTX*)ctx);
	EVP_DigestInit_ex((EVP_MD_CTX*)ctx,(const EVP_MD*)m, NULL);
	bytecount=0;
	return ByteArray(ret,len);
#endif
}

void Digest::reset()
{
#ifndef HAVE_OPENSSL
	throw UnsupportedFeatureException("OpenSSL");
#else
	if (!m) throw NoAlgorithmSpecifiedException();
	if (!ctx) throw NoAlgorithmSpecifiedException();
	EVP_MD_CTX_cleanup((EVP_MD_CTX*)ctx);
	EVP_DigestInit((EVP_MD_CTX*)ctx,(const EVP_MD*)m);
	bytecount=0;
#endif
}

ByteArray Digest::hash(const ByteArrayPtr &data, Algorithm algorithm)
{
	Digest dig(algorithm);
	dig.addData(data);
	return dig.getDigest();
}

ByteArray Digest::hash(const ByteArrayPtr &data, const String &algorithmName)
{
	Digest dig(algorithmName);
	dig.addData(data);
	return dig.getDigest();
}

ByteArray Digest::md4(const ByteArrayPtr &data)
{
	return Digest::hash(data,Algo_MD4);
}

ByteArray Digest::md5(const ByteArrayPtr &data)
{
	return Digest::hash(data,Algo_MD5);
}

ByteArray Digest::sha1(const ByteArrayPtr &data)
{
	return Digest::hash(data,Algo_SHA1);
}

ByteArray Digest::sha224(const ByteArrayPtr &data)
{
	return Digest::hash(data,Algo_SHA224);
}

ByteArray Digest::sha256(const ByteArrayPtr &data)
{
	return Digest::hash(data,Algo_SHA256);
}

ByteArray Digest::sha384(const ByteArrayPtr &data)
{
	return Digest::hash(data,Algo_SHA384);
}

ByteArray Digest::sha512(const ByteArrayPtr &data)
{
	return Digest::hash(data,Algo_SHA512);
}

ByteArray Digest::ecdsa(const ByteArrayPtr &data)
{
	return Digest::hash(data,Algo_ECDSA);
}

ppluint32 Digest::crc32(const ByteArrayPtr &data)
{
#ifdef HAVE_LIBZ
	uLong crc=::crc32(0L,Z_NULL,0);
	return ::crc32(crc,(const Bytef*)data.ptr(),(uInt)data.size());
#endif
	return Crc32(data.ptr(),data.size());
}

ppluint32 Digest::adler32(const ByteArrayPtr &data)
{
     const unsigned char *buffer = (const unsigned char *)data.ptr();
     size_t buflength=data.size();
     ppluint32 s1 = 1;
     ppluint32 s2 = 0;

     for (size_t n = 0; n < buflength; n++) {
        s1 = (s1 + buffer[n]) % 65521;
        s2 = (s2 + s1) % 65521;
     }
     return (s2 << 16) | s1;
  }
}
<|endoftext|>


<commit_before>/*
 * Copyright 2009 Colin Percival, 2011 ArtForz, 2012-2013 pooler
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 * This file was originally written by Colin Percival as part of the Tarsnap
 * online backup system.
 */

#include "crypto/scrypt.h"
//#include "util.h"
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <openssl/sha.h>

#if defined(USE_SSE2) && !defined(USE_SSE2_ALWAYS)
#ifdef _MSC_VER
// MSVC 64bit is unable to use inline asm
#include <intrin.h>
#else
// GCC Linux or i686-w64-mingw32
#include <cpuid.h>
#endif
#endif

static inline uint32_t be32dec(const void *pp)
{
	const uint8_t *p = (uint8_t const *)pp;
	return ((uint32_t)(p[3]) + ((uint32_t)(p[2]) << 8) +
	    ((uint32_t)(p[1]) << 16) + ((uint32_t)(p[0]) << 24));
}

static inline void be32enc(void *pp, uint32_t x)
{
	uint8_t *p = (uint8_t *)pp;
	p[3] = x & 0xff;
	p[2] = (x >> 8) & 0xff;
	p[1] = (x >> 16) & 0xff;
	p[0] = (x >> 24) & 0xff;
}

typedef struct HMAC_SHA256Context {
	SHA256_CTX ictx;
	SHA256_CTX octx;
} HMAC_SHA256_CTX;

/* Initialize an HMAC-SHA256 operation with the given key. */
static void
HMAC_SHA256_Init(HMAC_SHA256_CTX *ctx, const void *_K, size_t Klen)
{
	unsigned char pad[64];
	unsigned char khash[32];
	const unsigned char *K = (const unsigned char *)_K;
	size_t i;

	/* If Klen > 64, the key is really SHA256(K). */
	if (Klen > 64) {
		SHA256_Init(&ctx->ictx);
		SHA256_Update(&ctx->ictx, K, Klen);
		SHA256_Final(khash, &ctx->ictx);
		K = khash;
		Klen = 32;
	}

	/* Inner SHA256 operation is SHA256(K xor [block of 0x36] || data). */
	SHA256_Init(&ctx->ictx);
	memset(pad, 0x36, 64);
	for (i = 0; i < Klen; i++)
		pad[i] ^= K[i];
	SHA256_Update(&ctx->ictx, pad, 64);

	/* Outer SHA256 operation is SHA256(K xor [block of 0x5c] || hash). */
	SHA256_Init(&ctx->octx);
	memset(pad, 0x5c, 64);
	for (i = 0; i < Klen; i++)
		pad[i] ^= K[i];
	SHA256_Update(&ctx->octx, pad, 64);

	/* Clean the stack. */
	memset(khash, 0, 32);
}

/* Add bytes to the HMAC-SHA256 operation. */
static void
HMAC_SHA256_Update(HMAC_SHA256_CTX *ctx, const void *in, size_t len)
{
	/* Feed data to the inner SHA256 operation. */
	SHA256_Update(&ctx->ictx, in, len);
}

/* Finish an HMAC-SHA256 operation. */
static void
HMAC_SHA256_Final(unsigned char digest[32], HMAC_SHA256_CTX *ctx)
{
	unsigned char ihash[32];

	/* Finish the inner SHA256 operation. */
	SHA256_Final(ihash, &ctx->ictx);

	/* Feed the inner hash to the outer SHA256 operation. */
	SHA256_Update(&ctx->octx, ihash, 32);

	/* Finish the outer SHA256 operation. */
	SHA256_Final(digest, &ctx->octx);

	/* Clean the stack. */
	memset(ihash, 0, 32);
}

/**
 * PBKDF2_SHA256(passwd, passwdlen, salt, saltlen, c, buf, dkLen):
 * Compute PBKDF2(passwd, salt, c, dkLen) using HMAC-SHA256 as the PRF, and
 * write the output to buf.  The value dkLen must be at most 32 * (2^32 - 1).
 */
void
PBKDF2_SHA256(const uint8_t *passwd, size_t passwdlen, const uint8_t *salt,
    size_t saltlen, uint64_t c, uint8_t *buf, size_t dkLen)
{
	HMAC_SHA256_CTX PShctx, hctx;
	size_t i;
	uint8_t ivec[4];
	uint8_t U[32];
	uint8_t T[32];
	uint64_t j;
	int k;
	size_t clen;

	/* Compute HMAC state after processing P and S. */
	HMAC_SHA256_Init(&PShctx, passwd, passwdlen);
	HMAC_SHA256_Update(&PShctx, salt, saltlen);

	/* Iterate through the blocks. */
	for (i = 0; i * 32 < dkLen; i++) {
		/* Generate INT(i + 1). */
		be32enc(ivec, (uint32_t)(i + 1));

		/* Compute U_1 = PRF(P, S || INT(i)). */
		memcpy(&hctx, &PShctx, sizeof(HMAC_SHA256_CTX));
		HMAC_SHA256_Update(&hctx, ivec, 4);
		HMAC_SHA256_Final(U, &hctx);

		/* T_i = U_1 ... */
		memcpy(T, U, 32);

		for (j = 2; j <= c; j++) {
			/* Compute U_j. */
			HMAC_SHA256_Init(&hctx, passwd, passwdlen);
			HMAC_SHA256_Update(&hctx, U, 32);
			HMAC_SHA256_Final(U, &hctx);

			/* ... xor U_j ... */
			for (k = 0; k < 32; k++)
				T[k] ^= U[k];
		}

		/* Copy as many bytes as necessary into buf. */
		clen = dkLen - i * 32;
		if (clen > 32)
			clen = 32;
		memcpy(&buf[i * 32], T, clen);
	}

	/* Clean PShctx, since we never called _Final on it. */
	memset(&PShctx, 0, sizeof(HMAC_SHA256_CTX));
}

#define ROTL(a, b) (((a) << (b)) | ((a) >> (32 - (b))))

static inline void xor_salsa8(uint32_t B[16], const uint32_t Bx[16])
{
	uint32_t x00,x01,x02,x03,x04,x05,x06,x07,x08,x09,x10,x11,x12,x13,x14,x15;
	int i;

	x00 = (B[ 0] ^= Bx[ 0]);
	x01 = (B[ 1] ^= Bx[ 1]);
	x02 = (B[ 2] ^= Bx[ 2]);
	x03 = (B[ 3] ^= Bx[ 3]);
	x04 = (B[ 4] ^= Bx[ 4]);
	x05 = (B[ 5] ^= Bx[ 5]);
	x06 = (B[ 6] ^= Bx[ 6]);
	x07 = (B[ 7] ^= Bx[ 7]);
	x08 = (B[ 8] ^= Bx[ 8]);
	x09 = (B[ 9] ^= Bx[ 9]);
	x10 = (B[10] ^= Bx[10]);
	x11 = (B[11] ^= Bx[11]);
	x12 = (B[12] ^= Bx[12]);
	x13 = (B[13] ^= Bx[13]);
	x14 = (B[14] ^= Bx[14]);
	x15 = (B[15] ^= Bx[15]);
	for (i = 0; i < 8; i += 2) {
		/* Operate on columns. */
		x04 ^= ROTL(x00 + x12,  7);  x09 ^= ROTL(x05 + x01,  7);
		x14 ^= ROTL(x10 + x06,  7);  x03 ^= ROTL(x15 + x11,  7);

		x08 ^= ROTL(x04 + x00,  9);  x13 ^= ROTL(x09 + x05,  9);
		x02 ^= ROTL(x14 + x10,  9);  x07 ^= ROTL(x03 + x15,  9);

		x12 ^= ROTL(x08 + x04, 13);  x01 ^= ROTL(x13 + x09, 13);
		x06 ^= ROTL(x02 + x14, 13);  x11 ^= ROTL(x07 + x03, 13);

		x00 ^= ROTL(x12 + x08, 18);  x05 ^= ROTL(x01 + x13, 18);
		x10 ^= ROTL(x06 + x02, 18);  x15 ^= ROTL(x11 + x07, 18);

		/* Operate on rows. */
		x01 ^= ROTL(x00 + x03,  7);  x06 ^= ROTL(x05 + x04,  7);
		x11 ^= ROTL(x10 + x09,  7);  x12 ^= ROTL(x15 + x14,  7);

		x02 ^= ROTL(x01 + x00,  9);  x07 ^= ROTL(x06 + x05,  9);
		x08 ^= ROTL(x11 + x10,  9);  x13 ^= ROTL(x12 + x15,  9);

		x03 ^= ROTL(x02 + x01, 13);  x04 ^= ROTL(x07 + x06, 13);
		x09 ^= ROTL(x08 + x11, 13);  x14 ^= ROTL(x13 + x12, 13);

		x00 ^= ROTL(x03 + x02, 18);  x05 ^= ROTL(x04 + x07, 18);
		x10 ^= ROTL(x09 + x08, 18);  x15 ^= ROTL(x14 + x13, 18);
	}
	B[ 0] += x00;
	B[ 1] += x01;
	B[ 2] += x02;
	B[ 3] += x03;
	B[ 4] += x04;
	B[ 5] += x05;
	B[ 6] += x06;
	B[ 7] += x07;
	B[ 8] += x08;
	B[ 9] += x09;
	B[10] += x10;
	B[11] += x11;
	B[12] += x12;
	B[13] += x13;
	B[14] += x14;
	B[15] += x15;
}

void scrypt_1024_1_1_256_sp_generic(const char *input, char *output, char *scratchpad)
{
	uint8_t B[128];
	uint32_t X[32];
	uint32_t *V;
	uint32_t i, j, k;

	V = (uint32_t *)(((uintptr_t)(scratchpad) + 63) & ~ (uintptr_t)(63));

	PBKDF2_SHA256((const uint8_t *)input, 80, (const uint8_t *)input, 80, 1, B, 128);

	for (k = 0; k < 32; k++)
		X[k] = le32dec(&B[4 * k]);

	for (i = 0; i < 1024; i++) {
		memcpy(&V[i * 32], X, 128);
		xor_salsa8(&X[0], &X[16]);
		xor_salsa8(&X[16], &X[0]);
	}
	for (i = 0; i < 1024; i++) {
		j = 32 * (X[16] & 1023);
		for (k = 0; k < 32; k++)
			X[k] ^= V[j + k];
		xor_salsa8(&X[0], &X[16]);
		xor_salsa8(&X[16], &X[0]);
	}

	for (k = 0; k < 32; k++)
		le32enc(&B[4 * k], X[k]);

	PBKDF2_SHA256((const uint8_t *)input, 80, B, 128, 1, (uint8_t *)output, 32);
}

#if defined(USE_SSE2)
// By default, set to generic scrypt function. This will prevent crash in case when scrypt_detect_sse2() wasn't called
void (*scrypt_1024_1_1_256_sp_detected)(const char *input, char *output, char *scratchpad) = &scrypt_1024_1_1_256_sp_generic;

void scrypt_detect_sse2()
{
#if defined(USE_SSE2_ALWAYS)
    printf("scrypt: using scrypt-sse2 as built.\n");
#else // USE_SSE2_ALWAYS
    // 32bit x86 Linux or Windows, detect cpuid features
    unsigned int cpuid_edx=0;
#if defined(_MSC_VER)
    // MSVC
    int x86cpuid[4];
    __cpuid(x86cpuid, 1);
    cpuid_edx = (unsigned int)buffer[3];
#else // _MSC_VER
    // Linux or i686-w64-mingw32 (gcc-4.6.3)
    unsigned int eax, ebx, ecx;
    __get_cpuid(1, &eax, &ebx, &ecx, &cpuid_edx);
#endif // _MSC_VER

    if (cpuid_edx & 1<<26)
    {
        scrypt_1024_1_1_256_sp_detected = &scrypt_1024_1_1_256_sp_sse2;
        printf("scrypt: using scrypt-sse2 as detected.\n");
    }
    else
    {
        scrypt_1024_1_1_256_sp_detected = &scrypt_1024_1_1_256_sp_generic;
        printf("scrypt: using scrypt-generic, SSE2 unavailable.\n");
    }
#endif // USE_SSE2_ALWAYS
}
#endif

void scrypt_1024_1_1_256(const char *input, char *output)
{
	char scratchpad[SCRYPT_SCRATCHPAD_SIZE];
    scrypt_1024_1_1_256_sp(input, output, scratchpad);
}
<commit_msg>Delete scrypt.cpp<commit_after><|endoftext|>


<commit_before>#include <cudaobjdetect.hpp>

extern "C"
struct HOGPtr HOG_ctorCuda(
        struct SizeWrapper win_size, struct SizeWrapper block_size,
        struct SizeWrapper block_stride, struct SizeWrapper cell_size, int nbins)
{
    return rescueObjectFromPtr(cuda::HOG::create(
            win_size, block_size, block_stride, cell_size, nbins));
}

extern "C"
void HOG_setWinSigmaCuda(struct HOGPtr ptr, double val)
{
    ptr->setWinSigma(val);
}

extern "C"
double HOG_getWinSigmaCuda(struct HOGPtr ptr)
{
    return ptr->getWinSigma();
}

extern "C"
void HOG_setL2HysThresholdCuda(struct HOGPtr ptr, double val)
{
    ptr->setL2HysThreshold(val);
}

extern "C"
double HOG_getL2HysThresholdCuda(struct HOGPtr ptr)
{
    return ptr->getL2HysThreshold();
}

extern "C"
void HOG_setGammaCorrectionCuda(struct HOGPtr ptr, bool val)
{
    ptr->setGammaCorrection(val);
}

extern "C"
bool HOG_getGammaCorrectionCuda(struct HOGPtr ptr)
{
    return ptr->getGammaCorrection();
}

extern "C"
void HOG_setNumLevelsCuda(struct HOGPtr ptr, int val)
{
    ptr->setNumLevels(val);
}

extern "C"
int HOG_getNumLevelsCuda(struct HOGPtr ptr)
{
    return ptr->getNumLevels();
}

extern "C"
void HOG_setHitThresholdCuda(struct HOGPtr ptr, double val)
{
    ptr->setHitThreshold(val);
}

extern "C"
double HOG_getHitThresholdCuda(struct HOGPtr ptr)
{
    return ptr->getHitThreshold();
}

extern "C"
void HOG_setWinStrideCuda(struct HOGPtr ptr, struct SizeWrapper val)
{
    ptr->setWinStride(val);
}

extern "C"
struct SizeWrapper HOG_getWinStrideCuda(struct HOGPtr ptr)
{
    return ptr->getWinStride();
}

extern "C"
void HOG_setScaleFactorCuda(struct HOGPtr ptr, double val)
{
    ptr->setScaleFactor(val);
}

extern "C"
double HOG_getScaleFactorCuda(struct HOGPtr ptr)
{
    return ptr->getScaleFactor();
}

extern "C"
void HOG_setGroupThresholdCuda(struct HOGPtr ptr, int val)
{
    ptr->setGroupThreshold(val);
}

extern "C"
int HOG_getGroupThresholdCuda(struct HOGPtr ptr)
{
    return ptr->getGroupThreshold();
}

extern "C"
void HOG_setDescriptorFormatCuda(struct HOGPtr ptr, int val)
{
    ptr->setDescriptorFormat(val);
}

extern "C"
int HOG_getDescriptorFormatCuda(struct HOGPtr ptr)
{
    return ptr->getDescriptorFormat();
}

extern "C"
size_t HOG_getDescriptorSizeCuda(struct HOGPtr ptr)
{
    return ptr->getDescriptorSize();
}

extern "C"
size_t HOG_getBlockHistogramSizeCuda(struct HOGPtr ptr)
{
    return ptr->getBlockHistogramSize();
}

extern "C"
void HOG_setSVMDetectorCuda(struct HOGPtr ptr, struct TensorWrapper val)
{
    ptr->setSVMDetector(val.toMat());
}

extern "C"
struct TensorWrapper HOG_getDefaultPeopleDetectorCuda(struct HOGPtr ptr)
{
    return TensorWrapper(ptr->getDefaultPeopleDetector());
}

extern "C"
struct TensorPlusPointArray HOG_detectCuda(
        struct cutorchInfo info, struct HOGPtr ptr, struct TensorWrapper img)
{
    std::vector<cv::Point> found_locations;
    std::vector<double> confidences;
    cuda::GpuMat imgMat = img.toGpuMat();
    cuda::GpuMat imgMatByte;
    imgMat.convertTo(imgMatByte, CV_8U, 255.0); // Sorry guys :( #156
    ptr->detect(imgMatByte, found_locations, &confidences);

    TensorPlusPointArray retval;
    new (&retval.points) PointArray(found_locations);
    new (&retval.tensor) TensorWrapper(cv::Mat(confidences));
    return retval;
}

extern "C"
struct TensorPlusRectArray HOG_detectMultiScaleCuda(
        struct cutorchInfo info, struct HOGPtr ptr, struct TensorWrapper img)
{
    std::vector<cv::Rect> found_locations;
    std::vector<double> confidences;
    cuda::GpuMat imgMat = img.toGpuMat();
    cuda::GpuMat imgMatByte;
    imgMat.convertTo(imgMatByte, CV_8U, 255.0); // Sorry guys :( #156

    ptr->detectMultiScale(imgMatByte, found_locations, &confidences);

    TensorPlusRectArray retval;
    new (&retval.rects) RectArray(found_locations);
    new (&retval.tensor) TensorWrapper(cv::Mat(confidences));
    return retval;
}

extern "C"
struct TensorWrapper HOG_computeCuda(
        struct cutorchInfo info, struct HOGPtr ptr, struct TensorWrapper img,
        struct TensorWrapper descriptors)
{
    GpuMatT descriptorsMat = descriptors.toGpuMatT();
    cuda::GpuMat imgMat = img.toGpuMat();
    cuda::GpuMat imgMatByte;
    imgMat.convertTo(imgMatByte, CV_8U, 255.0); // Sorry guys :( #156

    ptr->compute(imgMatByte, descriptorsMat, prepareStream(info));
    return TensorWrapper(descriptorsMat, info.state);
}

extern "C"
struct CascadeClassifierPtr CascadeClassifier_ctor_filenameCuda(const char *filename)
{
    return rescueObjectFromPtr(cuda::CascadeClassifier::create(filename));
}

extern "C"
struct CascadeClassifierPtr CascadeClassifier_ctor_fileCuda(struct FileStoragePtr file)
{
    return rescueObjectFromPtr(cuda::CascadeClassifier::create(*file));
}

extern "C"
void CascadeClassifier_setMaxObjectSizeCuda(struct CascadeClassifierPtr ptr, struct SizeWrapper val)
{
    ptr->setMaxObjectSize(val);
}

extern "C"
struct SizeWrapper CascadeClassifier_getMaxObjectSizeCuda(struct CascadeClassifierPtr ptr)
{
    return ptr->getMaxObjectSize();
}

extern "C"
void CascadeClassifier_setMinObjectSizeCuda(struct CascadeClassifierPtr ptr, struct SizeWrapper val)
{
    ptr->setMinObjectSize(val);
}

extern "C"
struct SizeWrapper CascadeClassifier_getMinObjectSizeCuda(struct CascadeClassifierPtr ptr)
{
    return ptr->getMinObjectSize();
}

extern "C"
void CascadeClassifier_setScaleFactorCuda(struct CascadeClassifierPtr ptr, double val)
{
    ptr->setScaleFactor(val);
}

extern "C"
double CascadeClassifier_getScaleFactorCuda(struct CascadeClassifierPtr ptr)
{
    return ptr->getScaleFactor();
}

extern "C"
void CascadeClassifier_setMinNeighborsCuda(struct CascadeClassifierPtr ptr, int val)
{
    ptr->setMinNeighbors(val);
}

extern "C"
int CascadeClassifier_getMinNeighborsCuda(struct CascadeClassifierPtr ptr)
{
    return ptr->getMinNeighbors();
}

extern "C"
void CascadeClassifier_setFindLargestObjectCuda(struct CascadeClassifierPtr ptr, bool val)
{
    ptr->setFindLargestObject(val);
}

extern "C"
bool CascadeClassifier_getFindLargestObjectCuda(struct CascadeClassifierPtr ptr)
{
    return ptr->getFindLargestObject();
}

extern "C"
void CascadeClassifier_setMaxNumObjectsCuda(struct CascadeClassifierPtr ptr, int val)
{
    ptr->setMaxNumObjects(val);
}

extern "C"
int CascadeClassifier_getMaxNumObjectsCuda(struct CascadeClassifierPtr ptr)
{
    return ptr->getMaxNumObjects();
}

extern "C"
struct SizeWrapper CascadeClassifier_getClassifierSizeCuda(struct CascadeClassifierPtr ptr)
{
    return ptr->getClassifierSize();
}

extern "C"
struct TensorWrapper CascadeClassifier_detectMultiScaleCuda(
        struct cutorchInfo info, struct CascadeClassifierPtr ptr,
        struct TensorWrapper image, struct TensorWrapper objects)
{
    GpuMatT objectsMat = objects.toGpuMatT();
    cuda::GpuMat imageMat = image.toGpuMat();
    cuda::GpuMat imageByte;
    imageMat.convertTo(imageByte, CV_8U, 255.0); // Sorry guys :(
    ptr->detectMultiScale(imageByte, objectsMat, prepareStream(info));
    return TensorWrapper(objectsMat, info.state);
}

extern "C"
struct RectArray CascadeClassifier_convertCuda(
        struct CascadeClassifierPtr ptr, struct TensorWrapper gpu_objects)
{
    auto mat = gpu_objects.toGpuMat(CV_32S);
    std::vector<cv::Rect> objects;
    ptr->convert(mat, objects);
    return RectArray(objects);
}
<commit_msg>Error in HOG_detectMultiScaleCuda, #156<commit_after>#include <cudaobjdetect.hpp>

extern "C"
struct HOGPtr HOG_ctorCuda(
        struct SizeWrapper win_size, struct SizeWrapper block_size,
        struct SizeWrapper block_stride, struct SizeWrapper cell_size, int nbins)
{
    return rescueObjectFromPtr(cuda::HOG::create(
            win_size, block_size, block_stride, cell_size, nbins));
}

extern "C"
void HOG_setWinSigmaCuda(struct HOGPtr ptr, double val)
{
    ptr->setWinSigma(val);
}

extern "C"
double HOG_getWinSigmaCuda(struct HOGPtr ptr)
{
    return ptr->getWinSigma();
}

extern "C"
void HOG_setL2HysThresholdCuda(struct HOGPtr ptr, double val)
{
    ptr->setL2HysThreshold(val);
}

extern "C"
double HOG_getL2HysThresholdCuda(struct HOGPtr ptr)
{
    return ptr->getL2HysThreshold();
}

extern "C"
void HOG_setGammaCorrectionCuda(struct HOGPtr ptr, bool val)
{
    ptr->setGammaCorrection(val);
}

extern "C"
bool HOG_getGammaCorrectionCuda(struct HOGPtr ptr)
{
    return ptr->getGammaCorrection();
}

extern "C"
void HOG_setNumLevelsCuda(struct HOGPtr ptr, int val)
{
    ptr->setNumLevels(val);
}

extern "C"
int HOG_getNumLevelsCuda(struct HOGPtr ptr)
{
    return ptr->getNumLevels();
}

extern "C"
void HOG_setHitThresholdCuda(struct HOGPtr ptr, double val)
{
    ptr->setHitThreshold(val);
}

extern "C"
double HOG_getHitThresholdCuda(struct HOGPtr ptr)
{
    return ptr->getHitThreshold();
}

extern "C"
void HOG_setWinStrideCuda(struct HOGPtr ptr, struct SizeWrapper val)
{
    ptr->setWinStride(val);
}

extern "C"
struct SizeWrapper HOG_getWinStrideCuda(struct HOGPtr ptr)
{
    return ptr->getWinStride();
}

extern "C"
void HOG_setScaleFactorCuda(struct HOGPtr ptr, double val)
{
    ptr->setScaleFactor(val);
}

extern "C"
double HOG_getScaleFactorCuda(struct HOGPtr ptr)
{
    return ptr->getScaleFactor();
}

extern "C"
void HOG_setGroupThresholdCuda(struct HOGPtr ptr, int val)
{
    ptr->setGroupThreshold(val);
}

extern "C"
int HOG_getGroupThresholdCuda(struct HOGPtr ptr)
{
    return ptr->getGroupThreshold();
}

extern "C"
void HOG_setDescriptorFormatCuda(struct HOGPtr ptr, int val)
{
    ptr->setDescriptorFormat(val);
}

extern "C"
int HOG_getDescriptorFormatCuda(struct HOGPtr ptr)
{
    return ptr->getDescriptorFormat();
}

extern "C"
size_t HOG_getDescriptorSizeCuda(struct HOGPtr ptr)
{
    return ptr->getDescriptorSize();
}

extern "C"
size_t HOG_getBlockHistogramSizeCuda(struct HOGPtr ptr)
{
    return ptr->getBlockHistogramSize();
}

extern "C"
void HOG_setSVMDetectorCuda(struct HOGPtr ptr, struct TensorWrapper val)
{
    ptr->setSVMDetector(val.toMat());
}

extern "C"
struct TensorWrapper HOG_getDefaultPeopleDetectorCuda(struct HOGPtr ptr)
{
    return TensorWrapper(ptr->getDefaultPeopleDetector());
}

extern "C"
struct TensorPlusPointArray HOG_detectCuda(
        struct cutorchInfo info, struct HOGPtr ptr, struct TensorWrapper img)
{
    std::vector<cv::Point> found_locations;
    std::vector<double> confidences;
    cuda::GpuMat imgMat = img.toGpuMat();
    cuda::GpuMat imgMatByte;
    imgMat.convertTo(imgMatByte, CV_8U, 255.0); // Sorry guys :( #156
    ptr->detect(imgMatByte, found_locations, &confidences);

    TensorPlusPointArray retval;
    new (&retval.points) PointArray(found_locations);
    new (&retval.tensor) TensorWrapper(cv::Mat(confidences));
    return retval;
}

extern "C"
struct TensorPlusRectArray HOG_detectMultiScaleCuda(
        struct cutorchInfo info, struct HOGPtr ptr, struct TensorWrapper img)
{
    std::vector<cv::Rect> found_locations;
    std::vector<double> confidences;
    cuda::GpuMat imgMat = img.toGpuMat();
    cuda::GpuMat imgMatByte;
    imgMat.convertTo(imgMatByte, CV_8U, 255.0); // Sorry guys :( #156

    if (ptr->getGroupThreshold() == 0) {
        ptr->detectMultiScale(imgMatByte, found_locations, &confidences);
    } else {
        ptr->detectMultiScale(imgMatByte, found_locations, nullptr);
    }

    TensorPlusRectArray retval;
    new (&retval.rects) RectArray(found_locations);
    new (&retval.tensor) TensorWrapper(cv::Mat(confidences));
    return retval;
}

extern "C"
struct TensorWrapper HOG_computeCuda(
        struct cutorchInfo info, struct HOGPtr ptr, struct TensorWrapper img,
        struct TensorWrapper descriptors)
{
    GpuMatT descriptorsMat = descriptors.toGpuMatT();
    cuda::GpuMat imgMat = img.toGpuMat();
    cuda::GpuMat imgMatByte;
    imgMat.convertTo(imgMatByte, CV_8U, 255.0); // Sorry guys :( #156

    ptr->compute(imgMatByte, descriptorsMat, prepareStream(info));
    return TensorWrapper(descriptorsMat, info.state);
}

extern "C"
struct CascadeClassifierPtr CascadeClassifier_ctor_filenameCuda(const char *filename)
{
    return rescueObjectFromPtr(cuda::CascadeClassifier::create(filename));
}

extern "C"
struct CascadeClassifierPtr CascadeClassifier_ctor_fileCuda(struct FileStoragePtr file)
{
    return rescueObjectFromPtr(cuda::CascadeClassifier::create(*file));
}

extern "C"
void CascadeClassifier_setMaxObjectSizeCuda(struct CascadeClassifierPtr ptr, struct SizeWrapper val)
{
    ptr->setMaxObjectSize(val);
}

extern "C"
struct SizeWrapper CascadeClassifier_getMaxObjectSizeCuda(struct CascadeClassifierPtr ptr)
{
    return ptr->getMaxObjectSize();
}

extern "C"
void CascadeClassifier_setMinObjectSizeCuda(struct CascadeClassifierPtr ptr, struct SizeWrapper val)
{
    ptr->setMinObjectSize(val);
}

extern "C"
struct SizeWrapper CascadeClassifier_getMinObjectSizeCuda(struct CascadeClassifierPtr ptr)
{
    return ptr->getMinObjectSize();
}

extern "C"
void CascadeClassifier_setScaleFactorCuda(struct CascadeClassifierPtr ptr, double val)
{
    ptr->setScaleFactor(val);
}

extern "C"
double CascadeClassifier_getScaleFactorCuda(struct CascadeClassifierPtr ptr)
{
    return ptr->getScaleFactor();
}

extern "C"
void CascadeClassifier_setMinNeighborsCuda(struct CascadeClassifierPtr ptr, int val)
{
    ptr->setMinNeighbors(val);
}

extern "C"
int CascadeClassifier_getMinNeighborsCuda(struct CascadeClassifierPtr ptr)
{
    return ptr->getMinNeighbors();
}

extern "C"
void CascadeClassifier_setFindLargestObjectCuda(struct CascadeClassifierPtr ptr, bool val)
{
    ptr->setFindLargestObject(val);
}

extern "C"
bool CascadeClassifier_getFindLargestObjectCuda(struct CascadeClassifierPtr ptr)
{
    return ptr->getFindLargestObject();
}

extern "C"
void CascadeClassifier_setMaxNumObjectsCuda(struct CascadeClassifierPtr ptr, int val)
{
    ptr->setMaxNumObjects(val);
}

extern "C"
int CascadeClassifier_getMaxNumObjectsCuda(struct CascadeClassifierPtr ptr)
{
    return ptr->getMaxNumObjects();
}

extern "C"
struct SizeWrapper CascadeClassifier_getClassifierSizeCuda(struct CascadeClassifierPtr ptr)
{
    return ptr->getClassifierSize();
}

extern "C"
struct TensorWrapper CascadeClassifier_detectMultiScaleCuda(
        struct cutorchInfo info, struct CascadeClassifierPtr ptr,
        struct TensorWrapper image, struct TensorWrapper objects)
{
    GpuMatT objectsMat = objects.toGpuMatT();
    cuda::GpuMat imageMat = image.toGpuMat();
    cuda::GpuMat imageByte;
    imageMat.convertTo(imageByte, CV_8U, 255.0); // Sorry guys :(
    ptr->detectMultiScale(imageByte, objectsMat, prepareStream(info));
    return TensorWrapper(objectsMat, info.state);
}

extern "C"
struct RectArray CascadeClassifier_convertCuda(
        struct CascadeClassifierPtr ptr, struct TensorWrapper gpu_objects)
{
    auto mat = gpu_objects.toGpuMat(CV_32S);
    std::vector<cv::Rect> objects;
    ptr->convert(mat, objects);
    return RectArray(objects);
}
<|endoftext|>


<commit_before>/* cclive
 * Copyright (C) 2010-2011  Toni Gundogdu <legatvs@gmail.com>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <ccinternal>

#include <iomanip>
#include <ctime>

#include <boost/foreach.hpp>
#include <boost/random.hpp>

#include <curl/curl.h>

#ifndef foreach
#define foreach BOOST_FOREACH
#endif

#include <ccapplication>
#include <ccquvi>
#include <ccutil>
#include <cclog>

namespace cc
{

static boost::mt19937 _rng;

static void rand_decor()
{
  boost::uniform_int<> r(2,5);
  boost::variate_generator<boost::mt19937&, boost::uniform_int<> > v(_rng,r);

  const int n = v();
  for (int i=0; i<n; ++i) cc::log << ".";
}

static void handle_fetch(const quvi_word type, void*)
{
  rand_decor();
  if (type == QUVISTATUSTYPE_DONE)
    cc::log << " ";
}

static void print_done()
{
  cc::log << "done.\n";
}

static void handle_verify(const quvi_word type)
{
  rand_decor();
  if (type == QUVISTATUSTYPE_DONE)
    print_done();
}

static void handle_resolve(const quvi_word type)
{
  rand_decor();
  if (type == QUVISTATUSTYPE_DONE)
    cc::log << " ";
}

static int status_callback(long param, void *ptr)
{
  const quvi_word status = quvi_loword(param);
  const quvi_word type   = quvi_hiword(param);

  switch (status)
    {
    case QUVISTATUS_FETCH :
      handle_fetch(type,ptr);
      break;
    case QUVISTATUS_VERIFY:
      handle_verify(type);
      break;
    case QUVISTATUS_RESOLVE:
      handle_resolve(type);
      break;
    }

  cc::log << std::flush;

  return QUVI_OK;
}

template<class Iterator>
static Iterator make_unique(Iterator first, Iterator last)
{
  while (first != last)
    {
      Iterator next(first);
      last  = std::remove(++next, last, *first);
      first = next;
    }
  return last;
}

static void print_retrying(const int retry,
                           const int max_retries,
                           const int retry_wait)
{
  if (retry > 0)
    {
      cc::log
          << "Retrying "
          << retry
          << " of "
          << max_retries
          << " ... "
          << std::flush;

      cc::wait(retry_wait);
    }
}

static void print_checking(const int i, const int n)
{
  if (n > 1)  cc::log << "(" << i << " of " << n << ") ";
  cc::log << "Checking ... " << std::flush;
}

static void print_quvi_error(const quvi::error& e)
{
  cc::log << "libquvi: error: " << e.what() << std::endl;
}

static void check_quvi_error(const quvi::error& e)
{
  const long resp_code = e.response_code();

  if (resp_code >= 400 && resp_code <= 500)
    throw e;

  else
    {
      switch (e.quvi_code())
        {
        case QUVI_CURL:
          print_quvi_error(e);
          break; // Retry.

        default:
          throw e;
        }
    }
}

static const char depr_msg[] =
  "Warning:\n"
  "   '--format list' is deprecated and will be removed in the later\n"
  "   versions. Use --query-formats instead.";

static const char format_usage[] =
  "Usage:\n"
  "   --format arg                get format arg of media\n"
  "   --format list               print domains with formats\n"
  "   --format list arg           match arg to supported domain names\n"
  "Examples:\n"
  "   --format list youtube       print youtube formats\n"
  "   --format fmt34_360p         get format fmt34_360p of media";

static application::exit_status print_format_help()
{
  std::cout << format_usage << "\n" << depr_msg << std::endl;
  return application::ok;
}

typedef std::map<std::string,std::string> map_ss;

static void print_host(const map_ss::value_type& t)
{
  std::cout
      << t.first
      << ":\n  "
      << t.second
      << "\n"
      << std::endl;
}

namespace po = boost::program_options;

static application::exit_status handle_format_list(
  const po::variables_map& map,
  const quvi::query& query)
{
  map_ss m = query.support();

  // -f list <pattern>

  if (map.count("url"))
    {
      const std::string arg0 =
        map["url"].as< std::vector<std::string> >()[0];

      foreach (map_ss::value_type& t, m)
      {
        if (t.first.find(arg0) != std::string::npos)
          print_host(t);
      }
    }

  // -f list

  else
    {
      foreach (map_ss::value_type& t, m)
      {
        print_host(t);
      }
    }

  std::cout << depr_msg << std::endl;

  return application::ok;
}

static application::exit_status query_formats(
  const quvi::query& query,
  const quvi::options &opts,
  const std::vector<std::string>& input)
{
  const size_t n = input.size();
  size_t i = 0;

  foreach (std::string url, input)
  {
    ++i;

    try
      {
        print_checking(i,n);

        const std::string formats = query.formats(url, opts);

        print_done();

        cc::log
            << std::setw(10)
            << formats
            << " : "
            << url
            << std::endl;
      }
    catch(const quvi::error& e)
      {
        print_quvi_error(e);
      }
  }
  return application::ok;
}

static char *parse_url_scheme(const std::string& s)
{
  char *url = const_cast<char*>(s.c_str());

  char *p = strstr(url, ":/");
  if (!p)
    return NULL;

  char *r = NULL;
  asprintf(&r, "%.*s", (int)(p - url), url);

  return r;
}

#define _free(p) \
  do { if (p) free(p); p=NULL; } while (0)

static int is_url(const std::string& s)
{
  char *p = parse_url_scheme(s);
  if (p)
    {
      _free(p);
      return true;
    }
  return false;
}

#undef _free

static void read_from(std::istream& is, std::vector<std::string>& dst)
{
  std::string s;
  char ch = 0;

  while (is.get(ch))
    s += ch;

  std::istringstream iss(s);
  std::copy(
    std::istream_iterator<std::string >(iss),
    std::istream_iterator<std::string >(),
    std::back_inserter<std::vector<std::string> >(dst)
  );
}

extern char LICENSE[]; // cclive/license.cpp

application::exit_status application::exec(int argc, char **argv)
{
  try
    {
      _opts.exec(argc,argv);
    }
  catch(const std::exception& e)
    {
      std::clog << "error: " << e.what() << std::endl;
      return invalid_option;
    }

  const po::variables_map map = _opts.map();

  // Dump and terminate options.

  if (map.count("help"))
    {
      std::cout << _opts << std::flush;
      return ok;
    }

  if (map.count("version"))
    {
      std::cout
          << "cclive version "
#ifdef GIT_DESCRIBE
          << GIT_DESCRIBE
#else
          << PACKAGE_VERSION
#endif
#ifdef BUILD_DATE
          << " built on " << BUILD_DATE
#endif
          << " for " << CANONICAL_TARGET
          << "\nlibquvi version "
          << quvi_version(QUVI_VERSION_LONG)
          << std::endl;
      return ok;
    }

  if (map.count("license"))
    {
      std::cout << LICENSE << std::endl;
      return ok;
    }

  // --support

  quvi::query query; // Throws quvi::error caught in main.cpp

  if (map.count("support"))
    {
      std::cout << quvi::support_to_s(query.support()) << std::flush;
      return ok;
    }

  // --format [<id> | [<help> | <list> [<pattern]]]

  const std::string format = map["format"].as<std::string>();

  if (format == "help")
    return print_format_help();

  else if (format == "list")
    return handle_format_list(map, query);

  // Parse input.

  std::vector<std::string> input;

  if (map.count("url") == 0)
    read_from(std::cin, input);
  else
    {
      std::vector<std::string> args =
        map["url"].as< std::vector<std::string> >();

      foreach(std::string arg, args)
      {
        if (!is_url(arg))
          {
            std::ifstream f(arg.c_str());
            if (f.is_open())
              read_from(f, input);
            else
              {
                std::clog
                    << "error: "
                    << arg
                    << ": "
                    << cc::perror("unable to open")
                    << std::endl;
              }
          }
        else
          input.push_back(arg);
      }
    }

  if (input.size() == 0)
    {
      std::clog << "error: no input urls" << std::endl;
      return invalid_option;
    }

  // Remove duplicates.

  input.erase(make_unique(input.begin(), input.end()), input.end());

  // Turn on libcurl verbose output.

  if (map.count("verbose-libcurl"))
    curl_easy_setopt(query.curlHandle(), CURLOPT_VERBOSE, 1L);

  // Set up quvi.

  _tweak_curl_opts(query, map);

  quvi::options qopts;

  qopts.statusfunc(status_callback);
  qopts.format(format);
#ifdef _0
  qopts.verify(map.count("no-verify"));
#endif
  qopts.resolve(map.count("no-resolve"));

  // Seed random generator.

  _rng.seed(static_cast<unsigned int>(std::time(0)));

  // Omit flag.

  bool omit = map.count("quiet");

  // Go to background.

#ifdef HAVE_FORK
  const bool background_given = map.count("background");

  if (background_given)
    {

      // (Boost) Throws std::runtime_error if fails.

      cc::go_background(map["log-file"].as<std::string>(), omit);
    }
#endif

  // Omit std output. Note that --background flips this above.

  cc::log.push(cc::omit_sink(omit));
  cc::log.setf(std::ios::fixed);

  // Query formats.

  if (map.count("query-formats"))
    return query_formats(query, qopts, input);

#if defined (HAVE_FORK) && defined (HAVE_GETPID)
  if (background_given)
    {
      cc::log
          << "Running in background (pid: "
          << static_cast<long>(getpid())
          << ")."
          << std::endl;
    }
#endif

  // For each input URL.

  const size_t n = input.size();
  size_t i = 0;

  const int max_retries  = map["max-retries"].as<int>();
  const int retry_wait   = map["retry-wait"].as<int>();

  foreach(std::string url, input)
  {
    ++i;

    try
      {
        int retry = 0;

        while (retry <= max_retries)
          {
            print_retrying(retry, max_retries, retry_wait);

            ++retry;

            print_checking(i, n);

            quvi::media m;

            try
              {
                m = query.parse(url, qopts);
              }
            catch(const quvi::error& e)
              {
                check_quvi_error(e);
              }

            cc::get(query, m, map);

            break; // Stop retrying.
          }
      }

    catch(const quvi::error& e)
      {
        print_quvi_error(e);
      }

    catch(const std::runtime_error& e)
      {
        cc::log << "error: " << e.what() << std::endl;
      }
  }

  return ok;
}

void application::_tweak_curl_opts(const quvi::query& query,
                                   const po::variables_map& map)
{
  CURL *curl = query.curlHandle();

  curl_easy_setopt(curl, CURLOPT_USERAGENT,
                   map["agent"].as<std::string>().c_str());

  if (map.count("verbose-curl"))
    curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);

  if (map.count("proxy"))
    {
      curl_easy_setopt(curl, CURLOPT_PROXY,
                       map["proxy"].as<std::string>().c_str());
    }

  if (map.count("no-proxy"))
    curl_easy_setopt(curl, CURLOPT_PROXY, "");

  if (map.count("throttle"))
    {
      curl_off_t limit = map["throttle"].as<int>()*1024;
      curl_easy_setopt(curl, CURLOPT_MAX_RECV_SPEED_LARGE, limit);
    }

  curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT,
                   map["connect-timeout"].as<int>());
}

} // namespace cc

// vim: set ts=2 sw=2 tw=72 expandtab:
<commit_msg>Use QUVI_CALLBACK instead of QUVI_CURL<commit_after>/* cclive
 * Copyright (C) 2010-2011  Toni Gundogdu <legatvs@gmail.com>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <ccinternal>

#include <iomanip>
#include <ctime>

#include <boost/foreach.hpp>
#include <boost/random.hpp>

#include <curl/curl.h>

#ifndef foreach
#define foreach BOOST_FOREACH
#endif

#include <ccapplication>
#include <ccquvi>
#include <ccutil>
#include <cclog>

namespace cc
{

static boost::mt19937 _rng;

static void rand_decor()
{
  boost::uniform_int<> r(2,5);
  boost::variate_generator<boost::mt19937&, boost::uniform_int<> > v(_rng,r);

  const int n = v();
  for (int i=0; i<n; ++i) cc::log << ".";
}

static void handle_fetch(const quvi_word type, void*)
{
  rand_decor();
  if (type == QUVISTATUSTYPE_DONE)
    cc::log << " ";
}

static void print_done()
{
  cc::log << "done.\n";
}

static void handle_verify(const quvi_word type)
{
  rand_decor();
  if (type == QUVISTATUSTYPE_DONE)
    print_done();
}

static void handle_resolve(const quvi_word type)
{
  rand_decor();
  if (type == QUVISTATUSTYPE_DONE)
    cc::log << " ";
}

static int status_callback(long param, void *ptr)
{
  const quvi_word status = quvi_loword(param);
  const quvi_word type   = quvi_hiword(param);

  switch (status)
    {
    case QUVISTATUS_FETCH :
      handle_fetch(type,ptr);
      break;
    case QUVISTATUS_VERIFY:
      handle_verify(type);
      break;
    case QUVISTATUS_RESOLVE:
      handle_resolve(type);
      break;
    }

  cc::log << std::flush;

  return QUVI_OK;
}

template<class Iterator>
static Iterator make_unique(Iterator first, Iterator last)
{
  while (first != last)
    {
      Iterator next(first);
      last  = std::remove(++next, last, *first);
      first = next;
    }
  return last;
}

static void print_retrying(const int retry,
                           const int max_retries,
                           const int retry_wait)
{
  if (retry > 0)
    {
      cc::log
          << "Retrying "
          << retry
          << " of "
          << max_retries
          << " ... "
          << std::flush;

      cc::wait(retry_wait);
    }
}

static void print_checking(const int i, const int n)
{
  if (n > 1)  cc::log << "(" << i << " of " << n << ") ";
  cc::log << "Checking ... " << std::flush;
}

static void print_quvi_error(const quvi::error& e)
{
  cc::log << "libquvi: error: " << e.what() << std::endl;
}

static void check_quvi_error(const quvi::error& e)
{
  const long resp_code = e.response_code();

  if (resp_code >= 400 && resp_code <= 500)
    throw e;

  else
    {
      switch (e.quvi_code())
        {
        case QUVI_CALLBACK:
          print_quvi_error(e);
          break; // Retry.

        default:
          throw e;
        }
    }
}

static const char depr_msg[] =
  "Warning:\n"
  "   '--format list' is deprecated and will be removed in the later\n"
  "   versions. Use --query-formats instead.";

static const char format_usage[] =
  "Usage:\n"
  "   --format arg                get format arg of media\n"
  "   --format list               print domains with formats\n"
  "   --format list arg           match arg to supported domain names\n"
  "Examples:\n"
  "   --format list youtube       print youtube formats\n"
  "   --format fmt34_360p         get format fmt34_360p of media";

static application::exit_status print_format_help()
{
  std::cout << format_usage << "\n" << depr_msg << std::endl;
  return application::ok;
}

typedef std::map<std::string,std::string> map_ss;

static void print_host(const map_ss::value_type& t)
{
  std::cout
      << t.first
      << ":\n  "
      << t.second
      << "\n"
      << std::endl;
}

namespace po = boost::program_options;

static application::exit_status handle_format_list(
  const po::variables_map& map,
  const quvi::query& query)
{
  map_ss m = query.support();

  // -f list <pattern>

  if (map.count("url"))
    {
      const std::string arg0 =
        map["url"].as< std::vector<std::string> >()[0];

      foreach (map_ss::value_type& t, m)
      {
        if (t.first.find(arg0) != std::string::npos)
          print_host(t);
      }
    }

  // -f list

  else
    {
      foreach (map_ss::value_type& t, m)
      {
        print_host(t);
      }
    }

  std::cout << depr_msg << std::endl;

  return application::ok;
}

static application::exit_status query_formats(
  const quvi::query& query,
  const quvi::options &opts,
  const std::vector<std::string>& input)
{
  const size_t n = input.size();
  size_t i = 0;

  foreach (std::string url, input)
  {
    ++i;

    try
      {
        print_checking(i,n);

        const std::string formats = query.formats(url, opts);

        print_done();

        cc::log
            << std::setw(10)
            << formats
            << " : "
            << url
            << std::endl;
      }
    catch(const quvi::error& e)
      {
        print_quvi_error(e);
      }
  }
  return application::ok;
}

static char *parse_url_scheme(const std::string& s)
{
  char *url = const_cast<char*>(s.c_str());

  char *p = strstr(url, ":/");
  if (!p)
    return NULL;

  char *r = NULL;
  asprintf(&r, "%.*s", (int)(p - url), url);

  return r;
}

#define _free(p) \
  do { if (p) free(p); p=NULL; } while (0)

static int is_url(const std::string& s)
{
  char *p = parse_url_scheme(s);
  if (p)
    {
      _free(p);
      return true;
    }
  return false;
}

#undef _free

static void read_from(std::istream& is, std::vector<std::string>& dst)
{
  std::string s;
  char ch = 0;

  while (is.get(ch))
    s += ch;

  std::istringstream iss(s);
  std::copy(
    std::istream_iterator<std::string >(iss),
    std::istream_iterator<std::string >(),
    std::back_inserter<std::vector<std::string> >(dst)
  );
}

extern char LICENSE[]; // cclive/license.cpp

application::exit_status application::exec(int argc, char **argv)
{
  try
    {
      _opts.exec(argc,argv);
    }
  catch(const std::exception& e)
    {
      std::clog << "error: " << e.what() << std::endl;
      return invalid_option;
    }

  const po::variables_map map = _opts.map();

  // Dump and terminate options.

  if (map.count("help"))
    {
      std::cout << _opts << std::flush;
      return ok;
    }

  if (map.count("version"))
    {
      std::cout
          << "cclive version "
#ifdef GIT_DESCRIBE
          << GIT_DESCRIBE
#else
          << PACKAGE_VERSION
#endif
#ifdef BUILD_DATE
          << " built on " << BUILD_DATE
#endif
          << " for " << CANONICAL_TARGET
          << "\nlibquvi version "
          << quvi_version(QUVI_VERSION_LONG)
          << std::endl;
      return ok;
    }

  if (map.count("license"))
    {
      std::cout << LICENSE << std::endl;
      return ok;
    }

  // --support

  quvi::query query; // Throws quvi::error caught in main.cpp

  if (map.count("support"))
    {
      std::cout << quvi::support_to_s(query.support()) << std::flush;
      return ok;
    }

  // --format [<id> | [<help> | <list> [<pattern]]]

  const std::string format = map["format"].as<std::string>();

  if (format == "help")
    return print_format_help();

  else if (format == "list")
    return handle_format_list(map, query);

  // Parse input.

  std::vector<std::string> input;

  if (map.count("url") == 0)
    read_from(std::cin, input);
  else
    {
      std::vector<std::string> args =
        map["url"].as< std::vector<std::string> >();

      foreach(std::string arg, args)
      {
        if (!is_url(arg))
          {
            std::ifstream f(arg.c_str());
            if (f.is_open())
              read_from(f, input);
            else
              {
                std::clog
                    << "error: "
                    << arg
                    << ": "
                    << cc::perror("unable to open")
                    << std::endl;
              }
          }
        else
          input.push_back(arg);
      }
    }

  if (input.size() == 0)
    {
      std::clog << "error: no input urls" << std::endl;
      return invalid_option;
    }

  // Remove duplicates.

  input.erase(make_unique(input.begin(), input.end()), input.end());

  // Turn on libcurl verbose output.

  if (map.count("verbose-libcurl"))
    curl_easy_setopt(query.curlHandle(), CURLOPT_VERBOSE, 1L);

  // Set up quvi.

  _tweak_curl_opts(query, map);

  quvi::options qopts;

  qopts.statusfunc(status_callback);
  qopts.format(format);
#ifdef _0
  qopts.verify(map.count("no-verify"));
#endif
  qopts.resolve(map.count("no-resolve"));

  // Seed random generator.

  _rng.seed(static_cast<unsigned int>(std::time(0)));

  // Omit flag.

  bool omit = map.count("quiet");

  // Go to background.

#ifdef HAVE_FORK
  const bool background_given = map.count("background");

  if (background_given)
    {

      // (Boost) Throws std::runtime_error if fails.

      cc::go_background(map["log-file"].as<std::string>(), omit);
    }
#endif

  // Omit std output. Note that --background flips this above.

  cc::log.push(cc::omit_sink(omit));
  cc::log.setf(std::ios::fixed);

  // Query formats.

  if (map.count("query-formats"))
    return query_formats(query, qopts, input);

#if defined (HAVE_FORK) && defined (HAVE_GETPID)
  if (background_given)
    {
      cc::log
          << "Running in background (pid: "
          << static_cast<long>(getpid())
          << ")."
          << std::endl;
    }
#endif

  // For each input URL.

  const size_t n = input.size();
  size_t i = 0;

  const int max_retries  = map["max-retries"].as<int>();
  const int retry_wait   = map["retry-wait"].as<int>();

  foreach(std::string url, input)
  {
    ++i;

    try
      {
        int retry = 0;

        while (retry <= max_retries)
          {
            print_retrying(retry, max_retries, retry_wait);

            ++retry;

            print_checking(i, n);

            quvi::media m;

            try
              {
                m = query.parse(url, qopts);
              }
            catch(const quvi::error& e)
              {
                check_quvi_error(e);
              }

            cc::get(query, m, map);

            break; // Stop retrying.
          }
      }

    catch(const quvi::error& e)
      {
        print_quvi_error(e);
      }

    catch(const std::runtime_error& e)
      {
        cc::log << "error: " << e.what() << std::endl;
      }
  }

  return ok;
}

void application::_tweak_curl_opts(const quvi::query& query,
                                   const po::variables_map& map)
{
  CURL *curl = query.curlHandle();

  curl_easy_setopt(curl, CURLOPT_USERAGENT,
                   map["agent"].as<std::string>().c_str());

  if (map.count("verbose-curl"))
    curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);

  if (map.count("proxy"))
    {
      curl_easy_setopt(curl, CURLOPT_PROXY,
                       map["proxy"].as<std::string>().c_str());
    }

  if (map.count("no-proxy"))
    curl_easy_setopt(curl, CURLOPT_PROXY, "");

  if (map.count("throttle"))
    {
      curl_off_t limit = map["throttle"].as<int>()*1024;
      curl_easy_setopt(curl, CURLOPT_MAX_RECV_SPEED_LARGE, limit);
    }

  curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT,
                   map["connect-timeout"].as<int>());
}

} // namespace cc

// vim: set ts=2 sw=2 tw=72 expandtab:
<|endoftext|>