<commit_before>//
// Copyright (c) 2014-2015, THUNDERBEAST GAMES LLC All rights reserved
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
//

#include "../Precompiled.h"

#include "../Core/Profiler.h"
#include "../Container/HashMap.h"
#include "../IO/BufferQueue.h"
#include "../IO/Log.h"
#include "../Web/WebRequest.h"

#ifdef EMSCRIPTEN

#include "../DebugNew.h"
// Add code to use an XMLHttpRequest or ActiveX XMLHttpRequest here.

#else

#include "../Web/WebInternalConfig.h"
#include <asio.hpp>
#include <functional>
#include <curl/curl.h>

#include "../DebugNew.h"

namespace Atomic
{

struct WebRequestInternalState
{
    /// The WebRequest external state.
    WebRequest& es;
    /// The WebRequest external state to force it to stay around.
    SharedPtr<WebRequest> es_hold;
    /// The work queue.
    asio::io_service* service;
    /// URL.
    String url;
    /// Verb.
    String verb;
    /// Response headers.
    HashMap<StringHash, Pair<String, String>> responseHeaders;
    /// Upload stream.
    SharedPtr<Object> upload;
    /// Download stream.
    SharedPtr<Object> download;
    /// Request Headers.
    curl_slist* headers = NULL;
    /// Connection state.
    WebRequestState state;
    /// cURL multi handle.
    CURLM* curlm;
    /// cURL easy handle.
    CURL* curl;
    /// A flag to know if the request has contents (has data to upload).
    curl_off_t requestContentSize;
    /// A flag to know if the operation has been aborted.
    bool isAborted;
    /// A flag to know if the easy handle has been added to the Web class's multi handle.
    bool isAddedToMulti;
    /// Error string. Empty if no error.
    char error[CURL_ERROR_SIZE];

    WebRequestInternalState(WebRequest &es_) :
        es(es_)
    {
        LOGDEBUG("Create WebRequestInternalState");
    }

    ~WebRequestInternalState()
    {
        LOGDEBUG("Destroy WebRequestInternalState");
    }

    static int onProgress(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
    {
        WebRequestInternalState *is_(reinterpret_cast<WebRequestInternalState*>(clientp));
        if (is_->isAborted)
        {
            // This should probably be CURL_XFERINFO_ABORT, but that doesn't
            // exist. It probably would be the same numeric value, if it did.
            // The docs say that it just has to be a nonzero to abort.
            return CURL_READFUNC_ABORT;
        }

        VariantMap eventData;
        eventData.Insert(MakePair(StringHash("down_total"), Variant((double)dltotal)));
        eventData.Insert(MakePair(StringHash("down_loaded"), Variant((double)dlnow)));
        eventData.Insert(MakePair(StringHash("up_total"), Variant((double)ultotal)));
        eventData.Insert(MakePair(StringHash("up_loaded"), Variant((double)ulnow)));
        is_->es.SendEvent("progress", eventData);
        return 0;
    }

    static size_t onHeader(char *ptr, size_t size, size_t nmemb, void *userdata)
    {
        WebRequestInternalState *is_(reinterpret_cast<WebRequestInternalState*>(userdata));
        if (is_->isAborted)
        {
            is_->state = HTTP_CLOSED;
            // This should probably be CURL_HEADERFUNC_ABORT, but that doesn't
            // exist. It probably would be the same numeric value, if it did.
            // The docs say that it just has to be a number of bytes that is
            // not "size * nmemb" to abort.
            return CURL_READFUNC_ABORT;
        }

        // Find the size in bytes.
        size_t real_size(size * nmemb);

        // Check for some known values.
        if (real_size == 2 && ptr[0] == '\r' && ptr[1] == '\n')
        {
            return real_size;
        }
        if (real_size > 5 && !strncmp(ptr, "HTTP/", 5))
        {
            return real_size;
        }

        // Get the header key and value, and add them to the map.
        unsigned int key_end = 0;
        unsigned int value_begin = 2;
        while (value_begin < real_size)
        {
            if (ptr[key_end] == ':' && ptr[key_end + 1] == ' ')
            {
                break;
            }
            ++key_end;
            ++value_begin;
        }
        if (value_begin == real_size)
        {
            String key(ptr, (unsigned int)real_size);
            is_->responseHeaders.InsertNew(key.ToUpper(), MakePair(key, String()));
        }
        else
        {
            String key(ptr, (unsigned int)key_end);
            is_->responseHeaders.InsertNew(key.ToUpper(), MakePair(key, String(ptr + value_begin, (unsigned int)real_size - value_begin - 2)));
        }

        return real_size;
    }

    static size_t onWrite(char *ptr, size_t size, size_t nmemb, void *userdata)
    {
        WebRequestInternalState *is_(reinterpret_cast<WebRequestInternalState*>(userdata));
        is_->state = HTTP_OPEN;
        if (is_->isAborted)
        {
            is_->state = HTTP_CLOSED;
            // This should probably be CURL_WRITEFUNC_ABORT, but that doesn't
            // exist. It probably would be the same numeric value, if it did.
            // The docs say that it just has to be a number of bytes that is
            // not "size * nmemb" to abort.
            return CURL_READFUNC_ABORT;
        }

        // Find the size in bytes.
        size_t real_size(size * nmemb);

        // Write the date into the download buffer queue.
        Serializer* download(dynamic_cast<Serializer*>(is_->download.Get()));
        download->Write(ptr, (unsigned int)real_size);

        // Emit a "download_chunk" event.
        VariantMap eventData;
        eventData.Insert(MakePair(StringHash("download"), Variant(is_->download)));
        eventData.Insert(MakePair(StringHash("size"), Variant((unsigned int)real_size)));
        is_->es.SendEvent("download_chunk", eventData);

        return real_size;
    }

    static size_t onRead(char *buffer, size_t size, size_t nitems, void *instream)
    {
        WebRequestInternalState *is_(reinterpret_cast<WebRequestInternalState*>(instream));
        is_->state = HTTP_OPEN;
        if (is_->isAborted)
        {
            is_->state = HTTP_CLOSED;
            return CURL_READFUNC_ABORT;
        }

        // Find the size in bytes.
        size_t real_size(size * nitems);

        // Read as much as we can from the upload buffer queue.
        Deserializer* upload(dynamic_cast<Deserializer*>(is_->upload.Get()));
        size_t size_queued(upload->GetSize());
        size_t size_left(real_size);
        if ((size_left > 0) && (size_queued > 0))
        {
            size_t read_size(std::min(size_queued, size_left));
            upload->Read(buffer, (unsigned int)read_size);
            size_left -= read_size;
        }

        // If we still have bytes to fill, then emit a "upload_chunk" event.
        if (size_left > 0)
        {
            VariantMap eventData;
            eventData.Insert(MakePair(StringHash("upload"), Variant(is_->upload)));
            eventData.Insert(MakePair(StringHash("size"), Variant((unsigned int)size_left)));
            is_->es.SendEvent("upload_chunk", eventData);
        }

        // Read as much as we can from the upload buffer queue (again).
        size_queued = upload->GetSize();
        size_left = real_size;
        if ((size_left > 0) && (size_queued > 0))
        {
            size_t read_size(std::min(size_queued, size_left));
            upload->Read(buffer, (unsigned int)read_size);
            size_left -= read_size;
        }

        // If we still have bytes to fill, then something went wrong, so we should abort.
        if (size_left > 0)
        {
            is_->isAborted = true;
            return CURL_READFUNC_ABORT;
        }

        return real_size;
    }

    void onEnd(int code)
    {
        VariantMap eventData;
        if (code != CURLE_OK)
        {
            state = HTTP_ERROR;
            eventData.Insert(MakePair(StringHash("error"), Variant(String(error, (unsigned int)strnlen(error, sizeof(error))))));
        }
        else
        {
            state = HTTP_CLOSED;
            eventData.Insert(MakePair(StringHash("download"), Variant(download)));
            eventData.Insert(MakePair(StringHash("upload"), Variant(upload)));
        }
        es.SendEvent("complete", eventData);
    }
};

WebRequest::WebRequest(Context* context, const String& verb, const String& url, double requestContentSize) :
    Object(context),
    is_(new WebRequestInternalState(*this))
{
    is_->url = url.Trimmed();
    is_->verb = verb;
    is_->upload = new BufferQueue(context);
    is_->download = new BufferQueue(context);
    is_->state = HTTP_INITIALIZING;
    is_->curlm = NULL;
    is_->curl = NULL;
    is_->requestContentSize = curl_off_t(std::floor(requestContentSize));
    is_->isAborted = false;
    is_->isAddedToMulti = false;

}

WebRequest::~WebRequest()
{
    LOGDEBUG("Destroy WebRequest");

    curl_slist_free_all(is_->headers);
    if (is_->curlm == NULL)
    {
        return;
    }
    curl_easy_cleanup(is_->curl);
    delete is_;
}

void WebRequest::setup(asio::io_service *service, CURLM *curlm)
{
    LOGDEBUG("Create WebRequest");

    is_->service = service;
    is_->curlm = curlm;
    is_->curl = curl_easy_init();

    LOGDEBUG("HTTP " + is_->verb + " request to URL " + is_->url);

    curl_easy_setopt(is_->curl, CURLOPT_ERRORBUFFER, is_->error);
    is_->error[0] = '\0';

    // This line will eventually go away with a CA bundle in place, or other TLS options.
    curl_easy_setopt(is_->curl, CURLOPT_SSL_VERIFYPEER, FALSE);

    curl_easy_setopt(is_->curl, CURLOPT_URL, is_->url.CString());

    // All callbacks must look at is_->isAborted flag!

    curl_easy_setopt(is_->curl, CURLOPT_HEADERFUNCTION, &WebRequestInternalState::onHeader);
    curl_easy_setopt(is_->curl, CURLOPT_HEADERDATA, is_);

    curl_easy_setopt(is_->curl, CURLOPT_WRITEFUNCTION, &WebRequestInternalState::onWrite);
    curl_easy_setopt(is_->curl, CURLOPT_WRITEDATA, is_);

    curl_easy_setopt(is_->curl, CURLOPT_NOPROGRESS, 0L);
    curl_easy_setopt(is_->curl, CURLOPT_XFERINFOFUNCTION, &WebRequestInternalState::onProgress);
    curl_easy_setopt(is_->curl, CURLOPT_XFERINFODATA, is_);

    curl_easy_setopt(is_->curl, CURLOPT_CUSTOMREQUEST, is_->verb.CString());

    curl_easy_setopt(is_->curl, CURLOPT_PRIVATE, this);

    curl_easy_setopt(is_->curl, CURLOPT_READFUNCTION, &WebRequestInternalState::onRead);
    curl_easy_setopt(is_->curl, CURLOPT_READDATA, is_);

    if (is_->requestContentSize)
    {
        curl_easy_setopt(is_->curl, CURLOPT_UPLOAD, 1L);
        curl_easy_setopt(is_->curl, CURLOPT_INFILESIZE_LARGE, is_->requestContentSize);
    }
}

void WebRequest::internalNotify(WebRequest *wr, int code)
{
    wr->is_->onEnd(code);
    if (wr->is_->isAddedToMulti)
    {
        curl_multi_remove_handle(wr->is_->curlm, wr->is_->curl);
        wr->is_->isAddedToMulti = false;
        wr->is_->es_hold.Reset();
    }
}

void WebRequest::Abort()
{
    is_->isAborted = true;
}

const String& WebRequest::GetURL() const
{
    return is_->url;
}

String WebRequest::GetError() const
{
    return String(is_->error);
}

WebRequestState WebRequest::GetState() const
{
    return is_->state;
}

String WebRequest::GetVerb() const
{
    return is_->verb;
}

bool WebRequest::HasDownloadChunkEvent()
{
    return true; // cURL based implementations always support the "download_chunk" event.
}

void WebRequest::SetRequestHeader(const String& key, const String& value)
{
    // Trim and only add non-empty header strings.
    String header;
    header += key.Trimmed();
    header += ": ";
    header += value;
    if (header.Length())
    {
        is_->headers = curl_slist_append(is_->headers, header.CString());
    }
}

void WebRequest::Send()
{
    if (!is_->isAddedToMulti && !is_->isAborted)
    {
        is_->es_hold = this;
        curl_easy_setopt(is_->curl, CURLOPT_HTTPHEADER, is_->headers);
        curl_multi_add_handle(is_->curlm, is_->curl);
        is_->isAddedToMulti = true;
    }
}

StringVector WebRequest::GetResponseHeaderKeys()
{
    StringVector keys;
    for (auto it(is_->responseHeaders.Begin()),
        itEnd(is_->responseHeaders.End()); it != itEnd; ++it)
    {
        keys.Push(it->second_.first_);
    }
    return keys;
}

String WebRequest::GetResponseHeader(const String& header)
{
    auto it(is_->responseHeaders.Find(header.ToUpper()));
    if (it == is_->responseHeaders.End())
    {
        return "";
    }
    return it->second_.second_;
}

String WebRequest::GetAllResponseHeaders()
{
    String allHeaders;
    for (auto it(is_->responseHeaders.Begin()),
        itEnd(is_->responseHeaders.End()); it != itEnd; ++it)
    {
        allHeaders += it->second_.first_;
        allHeaders += ": ";
        allHeaders += it->second_.second_;
        allHeaders += "\r\n";
    }
    return allHeaders;
}

}

#endif
<commit_msg>Not using FALSE, which may not be defined.<commit_after>//
// Copyright (c) 2014-2015, THUNDERBEAST GAMES LLC All rights reserved
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
//

#include "../Precompiled.h"

#include "../Core/Profiler.h"
#include "../Container/HashMap.h"
#include "../IO/BufferQueue.h"
#include "../IO/Log.h"
#include "../Web/WebRequest.h"

#ifdef EMSCRIPTEN

#include "../DebugNew.h"
// Add code to use an XMLHttpRequest or ActiveX XMLHttpRequest here.

#else

#include "../Web/WebInternalConfig.h"
#include <asio.hpp>
#include <functional>
#include <curl/curl.h>

#include "../DebugNew.h"

namespace Atomic
{

struct WebRequestInternalState
{
    /// The WebRequest external state.
    WebRequest& es;
    /// The WebRequest external state to force it to stay around.
    SharedPtr<WebRequest> es_hold;
    /// The work queue.
    asio::io_service* service;
    /// URL.
    String url;
    /// Verb.
    String verb;
    /// Response headers.
    HashMap<StringHash, Pair<String, String>> responseHeaders;
    /// Upload stream.
    SharedPtr<Object> upload;
    /// Download stream.
    SharedPtr<Object> download;
    /// Request Headers.
    curl_slist* headers = NULL;
    /// Connection state.
    WebRequestState state;
    /// cURL multi handle.
    CURLM* curlm;
    /// cURL easy handle.
    CURL* curl;
    /// A flag to know if the request has contents (has data to upload).
    curl_off_t requestContentSize;
    /// A flag to know if the operation has been aborted.
    bool isAborted;
    /// A flag to know if the easy handle has been added to the Web class's multi handle.
    bool isAddedToMulti;
    /// Error string. Empty if no error.
    char error[CURL_ERROR_SIZE];

    WebRequestInternalState(WebRequest &es_) :
        es(es_)
    {
        LOGDEBUG("Create WebRequestInternalState");
    }

    ~WebRequestInternalState()
    {
        LOGDEBUG("Destroy WebRequestInternalState");
    }

    static int onProgress(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
    {
        WebRequestInternalState *is_(reinterpret_cast<WebRequestInternalState*>(clientp));
        if (is_->isAborted)
        {
            // This should probably be CURL_XFERINFO_ABORT, but that doesn't
            // exist. It probably would be the same numeric value, if it did.
            // The docs say that it just has to be a nonzero to abort.
            return CURL_READFUNC_ABORT;
        }

        VariantMap eventData;
        eventData.Insert(MakePair(StringHash("down_total"), Variant((double)dltotal)));
        eventData.Insert(MakePair(StringHash("down_loaded"), Variant((double)dlnow)));
        eventData.Insert(MakePair(StringHash("up_total"), Variant((double)ultotal)));
        eventData.Insert(MakePair(StringHash("up_loaded"), Variant((double)ulnow)));
        is_->es.SendEvent("progress", eventData);
        return 0;
    }

    static size_t onHeader(char *ptr, size_t size, size_t nmemb, void *userdata)
    {
        WebRequestInternalState *is_(reinterpret_cast<WebRequestInternalState*>(userdata));
        if (is_->isAborted)
        {
            is_->state = HTTP_CLOSED;
            // This should probably be CURL_HEADERFUNC_ABORT, but that doesn't
            // exist. It probably would be the same numeric value, if it did.
            // The docs say that it just has to be a number of bytes that is
            // not "size * nmemb" to abort.
            return CURL_READFUNC_ABORT;
        }

        // Find the size in bytes.
        size_t real_size(size * nmemb);

        // Check for some known values.
        if (real_size == 2 && ptr[0] == '\r' && ptr[1] == '\n')
        {
            return real_size;
        }
        if (real_size > 5 && !strncmp(ptr, "HTTP/", 5))
        {
            return real_size;
        }

        // Get the header key and value, and add them to the map.
        unsigned int key_end = 0;
        unsigned int value_begin = 2;
        while (value_begin < real_size)
        {
            if (ptr[key_end] == ':' && ptr[key_end + 1] == ' ')
            {
                break;
            }
            ++key_end;
            ++value_begin;
        }
        if (value_begin == real_size)
        {
            String key(ptr, (unsigned int)real_size);
            is_->responseHeaders.InsertNew(key.ToUpper(), MakePair(key, String()));
        }
        else
        {
            String key(ptr, (unsigned int)key_end);
            is_->responseHeaders.InsertNew(key.ToUpper(), MakePair(key, String(ptr + value_begin, (unsigned int)real_size - value_begin - 2)));
        }

        return real_size;
    }

    static size_t onWrite(char *ptr, size_t size, size_t nmemb, void *userdata)
    {
        WebRequestInternalState *is_(reinterpret_cast<WebRequestInternalState*>(userdata));
        is_->state = HTTP_OPEN;
        if (is_->isAborted)
        {
            is_->state = HTTP_CLOSED;
            // This should probably be CURL_WRITEFUNC_ABORT, but that doesn't
            // exist. It probably would be the same numeric value, if it did.
            // The docs say that it just has to be a number of bytes that is
            // not "size * nmemb" to abort.
            return CURL_READFUNC_ABORT;
        }

        // Find the size in bytes.
        size_t real_size(size * nmemb);

        // Write the date into the download buffer queue.
        Serializer* download(dynamic_cast<Serializer*>(is_->download.Get()));
        download->Write(ptr, (unsigned int)real_size);

        // Emit a "download_chunk" event.
        VariantMap eventData;
        eventData.Insert(MakePair(StringHash("download"), Variant(is_->download)));
        eventData.Insert(MakePair(StringHash("size"), Variant((unsigned int)real_size)));
        is_->es.SendEvent("download_chunk", eventData);

        return real_size;
    }

    static size_t onRead(char *buffer, size_t size, size_t nitems, void *instream)
    {
        WebRequestInternalState *is_(reinterpret_cast<WebRequestInternalState*>(instream));
        is_->state = HTTP_OPEN;
        if (is_->isAborted)
        {
            is_->state = HTTP_CLOSED;
            return CURL_READFUNC_ABORT;
        }

        // Find the size in bytes.
        size_t real_size(size * nitems);

        // Read as much as we can from the upload buffer queue.
        Deserializer* upload(dynamic_cast<Deserializer*>(is_->upload.Get()));
        size_t size_queued(upload->GetSize());
        size_t size_left(real_size);
        if ((size_left > 0) && (size_queued > 0))
        {
            size_t read_size(std::min(size_queued, size_left));
            upload->Read(buffer, (unsigned int)read_size);
            size_left -= read_size;
        }

        // If we still have bytes to fill, then emit a "upload_chunk" event.
        if (size_left > 0)
        {
            VariantMap eventData;
            eventData.Insert(MakePair(StringHash("upload"), Variant(is_->upload)));
            eventData.Insert(MakePair(StringHash("size"), Variant((unsigned int)size_left)));
            is_->es.SendEvent("upload_chunk", eventData);
        }

        // Read as much as we can from the upload buffer queue (again).
        size_queued = upload->GetSize();
        size_left = real_size;
        if ((size_left > 0) && (size_queued > 0))
        {
            size_t read_size(std::min(size_queued, size_left));
            upload->Read(buffer, (unsigned int)read_size);
            size_left -= read_size;
        }

        // If we still have bytes to fill, then something went wrong, so we should abort.
        if (size_left > 0)
        {
            is_->isAborted = true;
            return CURL_READFUNC_ABORT;
        }

        return real_size;
    }

    void onEnd(int code)
    {
        VariantMap eventData;
        if (code != CURLE_OK)
        {
            state = HTTP_ERROR;
            eventData.Insert(MakePair(StringHash("error"), Variant(String(error, (unsigned int)strnlen(error, sizeof(error))))));
        }
        else
        {
            state = HTTP_CLOSED;
            eventData.Insert(MakePair(StringHash("download"), Variant(download)));
            eventData.Insert(MakePair(StringHash("upload"), Variant(upload)));
        }
        es.SendEvent("complete", eventData);
    }
};

WebRequest::WebRequest(Context* context, const String& verb, const String& url, double requestContentSize) :
    Object(context),
    is_(new WebRequestInternalState(*this))
{
    is_->url = url.Trimmed();
    is_->verb = verb;
    is_->upload = new BufferQueue(context);
    is_->download = new BufferQueue(context);
    is_->state = HTTP_INITIALIZING;
    is_->curlm = NULL;
    is_->curl = NULL;
    is_->requestContentSize = curl_off_t(std::floor(requestContentSize));
    is_->isAborted = false;
    is_->isAddedToMulti = false;

}

WebRequest::~WebRequest()
{
    LOGDEBUG("Destroy WebRequest");

    curl_slist_free_all(is_->headers);
    if (is_->curlm == NULL)
    {
        return;
    }
    curl_easy_cleanup(is_->curl);
    delete is_;
}

void WebRequest::setup(asio::io_service *service, CURLM *curlm)
{
    LOGDEBUG("Create WebRequest");

    is_->service = service;
    is_->curlm = curlm;
    is_->curl = curl_easy_init();

    LOGDEBUG("HTTP " + is_->verb + " request to URL " + is_->url);

    curl_easy_setopt(is_->curl, CURLOPT_ERRORBUFFER, is_->error);
    is_->error[0] = '\0';

    // This line will eventually go away with a CA bundle in place, or other TLS options.
    curl_easy_setopt(is_->curl, CURLOPT_SSL_VERIFYPEER, 0);

    curl_easy_setopt(is_->curl, CURLOPT_URL, is_->url.CString());

    // All callbacks must look at is_->isAborted flag!

    curl_easy_setopt(is_->curl, CURLOPT_HEADERFUNCTION, &WebRequestInternalState::onHeader);
    curl_easy_setopt(is_->curl, CURLOPT_HEADERDATA, is_);

    curl_easy_setopt(is_->curl, CURLOPT_WRITEFUNCTION, &WebRequestInternalState::onWrite);
    curl_easy_setopt(is_->curl, CURLOPT_WRITEDATA, is_);

    curl_easy_setopt(is_->curl, CURLOPT_NOPROGRESS, 0L);
    curl_easy_setopt(is_->curl, CURLOPT_XFERINFOFUNCTION, &WebRequestInternalState::onProgress);
    curl_easy_setopt(is_->curl, CURLOPT_XFERINFODATA, is_);

    curl_easy_setopt(is_->curl, CURLOPT_CUSTOMREQUEST, is_->verb.CString());

    curl_easy_setopt(is_->curl, CURLOPT_PRIVATE, this);

    curl_easy_setopt(is_->curl, CURLOPT_READFUNCTION, &WebRequestInternalState::onRead);
    curl_easy_setopt(is_->curl, CURLOPT_READDATA, is_);

    if (is_->requestContentSize)
    {
        curl_easy_setopt(is_->curl, CURLOPT_UPLOAD, 1L);
        curl_easy_setopt(is_->curl, CURLOPT_INFILESIZE_LARGE, is_->requestContentSize);
    }
}

void WebRequest::internalNotify(WebRequest *wr, int code)
{
    wr->is_->onEnd(code);
    if (wr->is_->isAddedToMulti)
    {
        curl_multi_remove_handle(wr->is_->curlm, wr->is_->curl);
        wr->is_->isAddedToMulti = false;
        wr->is_->es_hold.Reset();
    }
}

void WebRequest::Abort()
{
    is_->isAborted = true;
}

const String& WebRequest::GetURL() const
{
    return is_->url;
}

String WebRequest::GetError() const
{
    return String(is_->error);
}

WebRequestState WebRequest::GetState() const
{
    return is_->state;
}

String WebRequest::GetVerb() const
{
    return is_->verb;
}

bool WebRequest::HasDownloadChunkEvent()
{
    return true; // cURL based implementations always support the "download_chunk" event.
}

void WebRequest::SetRequestHeader(const String& key, const String& value)
{
    // Trim and only add non-empty header strings.
    String header;
    header += key.Trimmed();
    header += ": ";
    header += value;
    if (header.Length())
    {
        is_->headers = curl_slist_append(is_->headers, header.CString());
    }
}

void WebRequest::Send()
{
    if (!is_->isAddedToMulti && !is_->isAborted)
    {
        is_->es_hold = this;
        curl_easy_setopt(is_->curl, CURLOPT_HTTPHEADER, is_->headers);
        curl_multi_add_handle(is_->curlm, is_->curl);
        is_->isAddedToMulti = true;
    }
}

StringVector WebRequest::GetResponseHeaderKeys()
{
    StringVector keys;
    for (auto it(is_->responseHeaders.Begin()),
        itEnd(is_->responseHeaders.End()); it != itEnd; ++it)
    {
        keys.Push(it->second_.first_);
    }
    return keys;
}

String WebRequest::GetResponseHeader(const String& header)
{
    auto it(is_->responseHeaders.Find(header.ToUpper()));
    if (it == is_->responseHeaders.End())
    {
        return "";
    }
    return it->second_.second_;
}

String WebRequest::GetAllResponseHeaders()
{
    String allHeaders;
    for (auto it(is_->responseHeaders.Begin()),
        itEnd(is_->responseHeaders.End()); it != itEnd; ++it)
    {
        allHeaders += it->second_.first_;
        allHeaders += ": ";
        allHeaders += it->second_.second_;
        allHeaders += "\r\n";
    }
    return allHeaders;
}

}

#endif
<|endoftext|>


<commit_before>//
// Copyright (c) 2008-2019 the Urho3D project.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
//

#include "../Precompiled.h"

#include "../IO/File.h"
#include "../IO/Log.h"
#include "../IO/PackageFile.h"
#include "../IO/FileSystem.h"

namespace Urho3D
{

PackageFile::PackageFile(Context* context) :
    Object(context),
    totalSize_(0),
    totalDataSize_(0),
    checksum_(0),
    compressed_(false)
{
}

PackageFile::PackageFile(Context* context, const ea::string& fileName, unsigned startOffset) :
    Object(context),
    totalSize_(0),
    totalDataSize_(0),
    checksum_(0),
    compressed_(false)
{
    Open(fileName, startOffset);
}

PackageFile::~PackageFile() = default;

bool PackageFile::Open(const ea::string& fileName, unsigned startOffset)
{
    SharedPtr<File> file(new File(context_, fileName));
    if (!file->IsOpen())
        return false;

    // Check ID, then read the directory
    file->Seek(startOffset);
    ea::string id = file->ReadFileID();
    if (id != "UPAK" && id != "ULZ4")
    {
        // If start offset has not been explicitly specified, also try to read package size from the end of file
        // to know how much we must rewind to find the package start
        if (!startOffset)
        {
            unsigned fileSize = file->GetSize();
            file->Seek((unsigned)(fileSize - sizeof(unsigned)));
            unsigned newStartOffset = fileSize - file->ReadUInt();
            if (newStartOffset < fileSize)
            {
                startOffset = newStartOffset;
                file->Seek(startOffset);
                id = file->ReadFileID();
            }
        }

        if (id != "UPAK" && id != "ULZ4")
        {
            URHO3D_LOGERROR(fileName + " is not a valid package file");
            return false;
        }
    }

    fileName_ = fileName;
    nameHash_ = fileName_;
    totalSize_ = file->GetSize();
    compressed_ = id == "ULZ4";

    unsigned numFiles = file->ReadUInt();
    checksum_ = file->ReadUInt();

    for (unsigned i = 0; i < numFiles; ++i)
    {
        ea::string entryName = file->ReadString();
        PackageEntry newEntry{};
        newEntry.offset_ = file->ReadUInt() + startOffset;
        totalDataSize_ += (newEntry.size_ = file->ReadUInt());
        newEntry.checksum_ = file->ReadUInt();
        if (!compressed_ && newEntry.offset_ + newEntry.size_ > totalSize_)
        {
            URHO3D_LOGERROR("File entry " + entryName + " outside package file");
            return false;
        }
        else
            entries_[entryName] = newEntry;
    }

    return true;
}

bool PackageFile::Exists(const ea::string& fileName) const
{
    bool found = entries_.find(fileName) != entries_.end();

#ifdef _WIN32
    // On Windows perform a fallback case-insensitive search
    if (!found)
    {
        for (auto i = entries_.begin(); i != entries_.end(); ++i)
        {
            if (!i->first.comparei(fileName))
            {
                found = true;
                break;
            }
        }
    }
#endif

    return found;
}

const PackageEntry* PackageFile::GetEntry(const ea::string& fileName) const
{
    auto i = entries_.find(fileName);
    if (i != entries_.end())
        return &i->second;

#ifdef _WIN32
    // On Windows perform a fallback case-insensitive search
    else
    {
        for (auto j = entries_.begin(); j != entries_.end(); ++j)
        {
            if (!j->first.comparei(fileName))
                return &j->second;
        }
    }
#endif

    return nullptr;
}

void PackageFile::Scan(ea::vector<ea::string>& result, const ea::string& pathName, const ea::string& filter, bool recursive) const
{
    result.clear();

    ea::string sanitizedPath = GetSanitizedPath(pathName);
    ea::string filterExtension = filter.substr(filter.find_last_of('.'));
    if (filterExtension.contains('*'))
        filterExtension.clear();

    bool caseSensitive = true;
#ifdef _WIN32
    // On Windows ignore case in string comparisons
    caseSensitive = false;
#endif

    const StringVector& entryNames = GetEntryNames();
    for (auto i = entryNames.begin(); i != entryNames.end(); ++i)
    {
        ea::string entryName = GetSanitizedPath(*i);
        if ((filterExtension.empty() || entryName.ends_with(filterExtension, caseSensitive)) &&
            entryName.starts_with(sanitizedPath, caseSensitive))
        {
            ea::string fileName = entryName.substr(sanitizedPath.length());
            if (fileName.starts_with("\\") || fileName.starts_with("/"))
                fileName = fileName.substr(1, fileName.length() - 1);
            if (!recursive && (fileName.contains("\\") || fileName.contains("/")))
                continue;

            result.push_back(fileName);
        }
    }
}

}
<commit_msg>IO: Implement support for new rbfx pak format.<commit_after>//
// Copyright (c) 2008-2019 the Urho3D project.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
//

#include "../Precompiled.h"

#include "../IO/File.h"
#include "../IO/Log.h"
#include "../IO/PackageFile.h"
#include "../IO/FileSystem.h"

namespace Urho3D
{

PackageFile::PackageFile(Context* context) :
    Object(context),
    totalSize_(0),
    totalDataSize_(0),
    checksum_(0),
    compressed_(false)
{
}

PackageFile::PackageFile(Context* context, const ea::string& fileName, unsigned startOffset) :
    Object(context),
    totalSize_(0),
    totalDataSize_(0),
    checksum_(0),
    compressed_(false)
{
    Open(fileName, startOffset);
}

PackageFile::~PackageFile() = default;

bool PackageFile::Open(const ea::string& fileName, unsigned startOffset)
{
    SharedPtr<File> file(new File(context_, fileName));
    if (!file->IsOpen())
        return false;

    // Check ID, then read the directory
    file->Seek(startOffset);
    ea::string id = file->ReadFileID();
    if (id != "UPAK" && id != "ULZ4" && id != "RPAK" && id != "RLZ4")
    {
        // If start offset has not been explicitly specified, also try to read package size from the end of file
        // to know how much we must rewind to find the package start
        if (!startOffset)
        {
            unsigned fileSize = file->GetSize();
            file->Seek((unsigned)(fileSize - sizeof(unsigned)));
            unsigned newStartOffset = fileSize - file->ReadUInt();
            if (newStartOffset < fileSize)
            {
                startOffset = newStartOffset;
                file->Seek(startOffset);
                id = file->ReadFileID();
            }
        }

        if (id != "UPAK" && id != "ULZ4" && id != "RPAK" && id != "RLZ4")
        {
            URHO3D_LOGERROR(fileName + " is not a valid package file");
            return false;
        }
    }

    fileName_ = fileName;
    nameHash_ = fileName_;
    totalSize_ = file->GetSize();
    compressed_ = id == "ULZ4" || id == "RLZ4";
    unsigned numFiles = file->ReadUInt();
    checksum_ = file->ReadUInt();

    if (id == "RPAK" || id == "RLZ4")
    {
        // New PAK file format includes two extra PAK header fields:
        // * Version. At this time this field is unused and is always 0. It will be used in the future if PAK format needs to be extended.
        // * File list offset. New format writes file list in the end of the file. This allows PAK creation without knowing entire file list
        //   beforehand.
        unsigned version = file->ReadUInt();                        // Reserved for future use.
        assert(version == 0);
        int64_t fileListOffset = file->ReadInt64();                 // New format has file list at the end of the file.
        file->Seek(fileListOffset);                                 // TODO: Serializer/Deserializer do not support files bigger than 4 GB
    }

    for (unsigned i = 0; i < numFiles; ++i)
    {
        ea::string entryName = file->ReadString();
        PackageEntry newEntry{};
        newEntry.offset_ = file->ReadUInt() + startOffset;
        totalDataSize_ += (newEntry.size_ = file->ReadUInt());
        newEntry.checksum_ = file->ReadUInt();
        if (!compressed_ && newEntry.offset_ + newEntry.size_ > totalSize_)
        {
            URHO3D_LOGERROR("File entry " + entryName + " outside package file");
            return false;
        }
        else
            entries_[entryName] = newEntry;
    }

    return true;
}

bool PackageFile::Exists(const ea::string& fileName) const
{
    bool found = entries_.find(fileName) != entries_.end();

#ifdef _WIN32
    // On Windows perform a fallback case-insensitive search
    if (!found)
    {
        for (auto i = entries_.begin(); i != entries_.end(); ++i)
        {
            if (!i->first.comparei(fileName))
            {
                found = true;
                break;
            }
        }
    }
#endif

    return found;
}

const PackageEntry* PackageFile::GetEntry(const ea::string& fileName) const
{
    auto i = entries_.find(fileName);
    if (i != entries_.end())
        return &i->second;

#ifdef _WIN32
    // On Windows perform a fallback case-insensitive search
    else
    {
        for (auto j = entries_.begin(); j != entries_.end(); ++j)
        {
            if (!j->first.comparei(fileName))
                return &j->second;
        }
    }
#endif

    return nullptr;
}

void PackageFile::Scan(ea::vector<ea::string>& result, const ea::string& pathName, const ea::string& filter, bool recursive) const
{
    result.clear();

    ea::string sanitizedPath = GetSanitizedPath(pathName);
    ea::string filterExtension = filter.substr(filter.find_last_of('.'));
    if (filterExtension.contains('*'))
        filterExtension.clear();

    bool caseSensitive = true;
#ifdef _WIN32
    // On Windows ignore case in string comparisons
    caseSensitive = false;
#endif

    const StringVector& entryNames = GetEntryNames();
    for (auto i = entryNames.begin(); i != entryNames.end(); ++i)
    {
        ea::string entryName = GetSanitizedPath(*i);
        if ((filterExtension.empty() || entryName.ends_with(filterExtension, caseSensitive)) &&
            entryName.starts_with(sanitizedPath, caseSensitive))
        {
            ea::string fileName = entryName.substr(sanitizedPath.length());
            if (fileName.starts_with("\\") || fileName.starts_with("/"))
                fileName = fileName.substr(1, fileName.length() - 1);
            if (!recursive && (fileName.contains("\\") || fileName.contains("/")))
                continue;

            result.push_back(fileName);
        }
    }
}

}
<|endoftext|>


<commit_before>#include <string>
#include <fstream>
#include <errno.h>
#include <math.h>
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#ifdef DARWIN
# include <sys/param.h>
# include <sys/sysctl.h>
#endif
#include <sys/time.h>
#include <sys/stat.h>
#include <limits.h>
#include <sstream>
#include <stdlib.h>
#include <libgen.h>
#ifdef LINUX
# include <sched.h>
# ifdef HAS_LIBNUMA
#  include <numaif.h>
# endif
#endif

#include "tools.h"
#include "failure.h"
#include "log.h"

using std::string;
using std::vector;

/* purpose: formats ISO 8601 timestamp into given buffer (simplified)
 * paramtr: seconds (IN): time stamp
 *          buffer (OUT): where to put the results
 *          size (IN): capacity of buffer
 * returns: pointer to start of buffer for convenience.
 */
char * isodate( time_t seconds, char* buffer, size_t size ) {
    struct tm zulu = *gmtime(&seconds);
    struct tm local = *localtime(&seconds);
    zulu.tm_isdst = local.tm_isdst;
    {
        time_t distance = (seconds - mktime(&zulu)) / 60;
        int hours = distance / 60;
        int minutes = distance < 0 ? -distance % 60 : distance % 60;
        size_t len = strftime( buffer, size, "%Y-%m-%dT%H:%M:%S", &local );
        snprintf( buffer+len, size-len, "%+03d:%02d", hours, minutes );
    }
    return buffer;
}

/* purpose: formats ISO 8601 timestamp into given buffer (simplified)
 * paramtr: seconds_wf (IN): time stamp with fractional seconds (millis)
 *          buffer (OUT): where to put the results
 *          size (IN): capacity of buffer
 * returns: pointer to start of buffer for convenience.
 */
char * iso2date( double seconds_wf, char* buffer, size_t size ) {
    char millis[8];
    double integral, fractional = modf(seconds_wf,&integral);
    time_t seconds = (time_t) integral;
    struct tm zulu = *gmtime(&seconds);
    struct tm local = *localtime(&seconds);
    zulu.tm_isdst = local.tm_isdst;
    snprintf( millis, sizeof(millis), "%.3f", fractional );
    {
        time_t distance = (seconds - mktime(&zulu)) / 60;
        int hours = distance / 60;
        int minutes = distance < 0 ? -distance % 60 : distance % 60;
        size_t len = strftime( buffer, size, "%Y-%m-%dT%H:%M:%S", &local );
        snprintf( buffer+len, size-len, "%s%+03d:%02d", millis+1, hours, minutes );
    }
    return buffer;
}

/* Get the local host name */
void get_host_name(std::string &hostname) {
    char name[HOST_NAME_MAX];
    if (gethostname(name, HOST_NAME_MAX) < 0) {
        myfailures("Unable to get host name");
    }
    hostname = name;
}

/* Get the current time as the number of seconds since the epoch */
double current_time() {
    struct timeval now;
    gettimeofday(&now, NULL);
    double ts = now.tv_sec + (now.tv_usec/1000000.0);
    return ts;
}

/* Get the total amount of physical memory in bytes */
unsigned long get_host_memory() {
    unsigned long memory;
#ifdef DARWIN
    size_t size = sizeof(memory);
    if (sysctlbyname("hw.memsize", &memory, &size, NULL, 0) < 0) {
        myfailures("Unable to get host physical memory size");
    }
#else
    long pages = sysconf(_SC_PHYS_PAGES);
    if (pages < 0) {
        myfailures("Unable to get number of host physical memory pages");
    }
    long pagesize = sysconf(_SC_PAGE_SIZE);
    if (pagesize < 0) {
        myfailures("Unable to get physical page size");
    }
    memory = pages * pagesize;
#endif
    if (memory == 0) {
        myfailure("Invalid memory size: %lu bytes", memory);
    }
    return memory;
}

struct cpuinfo get_host_cpuinfo() {
    struct cpuinfo c;
    c.threads = 0;
    c.cores = 0;
    c.sockets = 0;
#ifdef __MACH__
    unsigned int temp;
    size_t size = sizeof(temp);
    if (sysctlbyname("hw.logicalcpu", &temp, &size, NULL, 0) < 0) {
        myfailures("Unable to get number of CPUs (logical CPUs)");
    }
    c.threads = temp;
    if (sysctlbyname("hw.physicalcpu", &temp, &size, NULL, 0) < 0) {
        myfailures("Unable to get number of cores (physical CPUs)");
    }
    c.cores = temp;
    if (sysctlbyname("hw.packages", &temp, &size, NULL, 0) < 0) {
        myfailures("Unable to get number of CPU sockets");
    }
    c.threads = temp;
#else
    std::ifstream infile;
    infile.open("/proc/cpuinfo");
    if (!infile.good()) {
        myfailures("Error opening /proc/cpuinfo");
    }

    int last_physical_id = -1;
    bool new_socket = false;
    string rec;
    while (getline(infile, rec)) {
        if (rec.find("processor\t:", 0, 11) == 0) {
            // Each time we encounter a processor field, we increment the
            // number of cpus/threads
            c.threads += 1;
        } else if (rec.find("physical id\t:", 0, 13) == 0) {
            // Each time we encounter a new physical id, we increment the
            // number of sockets
            int new_physical_id;
            if (sscanf(rec.c_str(), "physical id\t: %d", &new_physical_id) != 1) {
                myfailures("Error reading 'physical id' field from /proc/cpuinfo");
            }
            if (new_physical_id > last_physical_id) {
                c.sockets += 1;
                last_physical_id = new_physical_id;
                new_socket = true;
            }
        } else if (rec.find("cpu cores\t:", 0, 11) == 0) {
            // Each time we encounter a new socket, we count the number of
            // cores it has
            if (new_socket) {
                cpu_t cores;
                if (sscanf(rec.c_str(), "cpu cores\t: %" SCNcpu_t, &cores) != 1) {
                    myfailures("Error reading 'cpu cores' field from /proc/cpuinfo");
                }
                c.cores += cores;
                new_socket = false;
            }
        }
    }

    if (infile.bad() || !infile.eof()) {
        myfailures("Error reading /proc/cpuinfo");
    }

    infile.close();
#endif
    if (c.threads == 0 || c.cores == 0 || c.sockets == 0 ||
            c.cores > c.threads || c.sockets > c.cores ||
            c.threads % c.cores > 0 || c.cores % c.sockets > 0) {
        myfailure("Invalid cpuinfo: %" PRIcpu_t " %" PRIcpu_t " %" PRIcpu_t, c.threads, c.cores, c.sockets);
    }
    return c;
}

int mkdirs(const char *path) {
    if (path == NULL || strlen(path) == 0) {
        return 0;
    }

    // No need to create cwd
    if (path[0] == '.' && path[1] == '\0') {
        return 0;
    }

    if (strlen(path) > PATH_MAX) {
        errno = ENAMETOOLONG;
        return -1;
    }

    char mypath[PATH_MAX];
    char *p = mypath;

    if (path[0] == '.' && path[1] == '.') {
        if (getcwd(mypath, PATH_MAX) == NULL) {
            return -1;
        }

        char *parent = strrchr(mypath, '/');
        if (parent) *parent = '\0';

        int off = strlen(mypath);
        strcpy(mypath+off, path+2);

        // In this case we don't need to go back to the root
        p = mypath + off;
    } else if (path[0] == '.' && path[1] == '/') {
        strcpy(mypath, path+2);
    } else {
        strcpy(mypath, path);
    }

    while (*p == '/') p++;

    int created = 0;
    struct stat st;
    while ((p = strchr(p, '/'))) {
        *p = '\0';

        if (stat(mypath, &st)) {
            if (errno == ENOENT) {
                if (mkdir(mypath, 0777)) {
                    return -1;
                }
                created++;
            } else {
                return -1;
            }
        } else {
            // If it exists, make sure it is a dir
            if (S_ISDIR(st.st_mode) == 0) {
                errno = ENOTDIR;
                return -1;
            }
        }

        *p = '/';
        while (*p == '/') p++;
    }

    // Last element of path
    if (stat(mypath, &st)) {
        if (errno == ENOENT) {
            if (mkdir(mypath, 0777)) {
                return -1;
            }
            created++;
        } else {
            return -1;
        }
    } else {
        if (S_ISDIR(st.st_mode) == 0) {
            errno = ENOTDIR;
            return -1;
        }
    }

    return created;
}

bool is_executable(const string &file) {
    // Invalid path
    if (file.size() == 0) {
        return false;
    }

    struct stat st;
    if (stat(file.c_str(), &st) == 0) {
        if (S_ISREG(st.st_mode)) {
            if ((st.st_uid == geteuid() && (S_IXUSR & st.st_mode) == S_IXUSR) ||
                (st.st_gid == getegid() && (S_IXGRP & st.st_mode) == S_IXGRP) ||
                ((S_IXOTH & st.st_mode) == S_IXOTH)) {
                // It is a regular file and we can execute it
                return true;
            }
        }
    }

    // In all other cases, the file is not executable
    return false;
}

int read_file(const string &file, char *buf, size_t size) {
    // Invalid path
    if (file.size() == 0) {
        errno = ENOENT;
        return -1;
    }

    FILE *f = fopen(file.c_str(), "r");
    if (f == NULL) {
        return -1;
    }

    size_t read = fread(buf, 1, size, f);

    if (fclose(f)) {
        return -1;
    }

    return read;
}

string pathfind(const string &file) {
    if (file.size() == 0) {
        return file;
    }

    // files that have a / should be returned as-is
    if (file.find('/') != string::npos) {
        return file;
    }

    // normally we wouldn't allow this
    if (is_executable(file)) {
        return file;
    }

    string path;
    char *env = getenv("PATH");
    if (env == NULL) {
#ifdef _PATH_DEFPATH
        path = _PATH_DEFPATH;
#else
        return file;
#endif
    } else {
        /* yes, there is a PATH variable */
        path = env;
    }

    string element;
    std::istringstream split(path);
    while(std::getline(split, element, ':')) {
        if (element.size() == 0) {
            continue;
        }
        string myfile = element + "/" + file;
        if (is_executable(myfile)) {
            return myfile;
        }
    }

    return file;
}

/* Return the directory part of a path */
string dirname(const string &path) {
    char *temp = strdup(path.c_str());
    string result = ::dirname(temp);
    free(temp);
    return result;
}

/* Return the last part of a path */
string filename(const string &path) {
    char *temp = strdup(path.c_str());
    string result = ::basename(temp);
    free(temp);
    return result;
}

/* Set the cpu affinity to values in bindings */
int set_cpu_affinity(vector<cpu_t> &bindings) {
#ifdef LINUX
    struct cpuinfo c = get_host_cpuinfo();
    cpu_set_t *cpuset = CPU_ALLOC(c.threads);
    if (cpuset == NULL) {
        return -1;
    }
    size_t cpusetsize = CPU_ALLOC_SIZE(c.threads);
    CPU_ZERO_S(cpusetsize, cpuset);

    for (vector<cpu_t>::iterator i = bindings.begin(); i != bindings.end(); i++) {
        cpu_t j = *i;
        if (j >= c.threads) {
            CPU_FREE(cpuset);
            errno = ERANGE;
            return -1;
        }
        CPU_SET_S(j, cpusetsize, cpuset);
    }

    int rc = sched_setaffinity(0, cpusetsize, cpuset);
    CPU_FREE(cpuset);
    if (rc < 0) {
        return -1;
    }
#endif
    return 0;
}

int clear_cpu_affinity() {
#ifdef LINUX
    struct cpuinfo c = get_host_cpuinfo();
    cpu_set_t *cpuset = CPU_ALLOC(c.threads);
    if (cpuset == NULL) {
        return -1;
    }
    size_t cpusetsize = CPU_ALLOC_SIZE(c.threads);
    CPU_ZERO_S(cpusetsize, cpuset);

    for (unsigned i=0; i<c.threads; i++) {
        CPU_SET_S(i, cpusetsize, cpuset);
    }

    int rc = sched_setaffinity(0, cpusetsize, cpuset);
    CPU_FREE(cpuset);
    if (rc < 0) {
        return -1;
    }
#endif
    return 0;
}

int clear_memory_affinity() {
#ifdef HAS_LIBNUMA
    int rc = set_mempolicy(MPOL_DEFAULT, NULL, 0);
    if (rc < 0) {
        return -1;
    }
#endif
    return 0;
}
<commit_msg>PM-1403: POWER9 cpuinfo fix<commit_after>#include <string>
#include <fstream>
#include <errno.h>
#include <math.h>
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#ifdef DARWIN
# include <sys/param.h>
# include <sys/sysctl.h>
#endif
#include <sys/time.h>
#include <sys/stat.h>
#include <limits.h>
#include <sstream>
#include <stdlib.h>
#include <libgen.h>
#ifdef LINUX
# include <sched.h>
# ifdef HAS_LIBNUMA
#  include <numaif.h>
# endif
#endif

#include "tools.h"
#include "failure.h"
#include "log.h"

using std::string;
using std::vector;

/* purpose: formats ISO 8601 timestamp into given buffer (simplified)
 * paramtr: seconds (IN): time stamp
 *          buffer (OUT): where to put the results
 *          size (IN): capacity of buffer
 * returns: pointer to start of buffer for convenience.
 */
char * isodate( time_t seconds, char* buffer, size_t size ) {
    struct tm zulu = *gmtime(&seconds);
    struct tm local = *localtime(&seconds);
    zulu.tm_isdst = local.tm_isdst;
    {
        time_t distance = (seconds - mktime(&zulu)) / 60;
        int hours = distance / 60;
        int minutes = distance < 0 ? -distance % 60 : distance % 60;
        size_t len = strftime( buffer, size, "%Y-%m-%dT%H:%M:%S", &local );
        snprintf( buffer+len, size-len, "%+03d:%02d", hours, minutes );
    }
    return buffer;
}

/* purpose: formats ISO 8601 timestamp into given buffer (simplified)
 * paramtr: seconds_wf (IN): time stamp with fractional seconds (millis)
 *          buffer (OUT): where to put the results
 *          size (IN): capacity of buffer
 * returns: pointer to start of buffer for convenience.
 */
char * iso2date( double seconds_wf, char* buffer, size_t size ) {
    char millis[8];
    double integral, fractional = modf(seconds_wf,&integral);
    time_t seconds = (time_t) integral;
    struct tm zulu = *gmtime(&seconds);
    struct tm local = *localtime(&seconds);
    zulu.tm_isdst = local.tm_isdst;
    snprintf( millis, sizeof(millis), "%.3f", fractional );
    {
        time_t distance = (seconds - mktime(&zulu)) / 60;
        int hours = distance / 60;
        int minutes = distance < 0 ? -distance % 60 : distance % 60;
        size_t len = strftime( buffer, size, "%Y-%m-%dT%H:%M:%S", &local );
        snprintf( buffer+len, size-len, "%s%+03d:%02d", millis+1, hours, minutes );
    }
    return buffer;
}

/* Get the local host name */
void get_host_name(std::string &hostname) {
    char name[HOST_NAME_MAX];
    if (gethostname(name, HOST_NAME_MAX) < 0) {
        myfailures("Unable to get host name");
    }
    hostname = name;
}

/* Get the current time as the number of seconds since the epoch */
double current_time() {
    struct timeval now;
    gettimeofday(&now, NULL);
    double ts = now.tv_sec + (now.tv_usec/1000000.0);
    return ts;
}

/* Get the total amount of physical memory in bytes */
unsigned long get_host_memory() {
    unsigned long memory;
#ifdef DARWIN
    size_t size = sizeof(memory);
    if (sysctlbyname("hw.memsize", &memory, &size, NULL, 0) < 0) {
        myfailures("Unable to get host physical memory size");
    }
#else
    long pages = sysconf(_SC_PHYS_PAGES);
    if (pages < 0) {
        myfailures("Unable to get number of host physical memory pages");
    }
    long pagesize = sysconf(_SC_PAGE_SIZE);
    if (pagesize < 0) {
        myfailures("Unable to get physical page size");
    }
    memory = pages * pagesize;
#endif
    if (memory == 0) {
        myfailure("Invalid memory size: %lu bytes", memory);
    }
    return memory;
}

struct cpuinfo get_host_cpuinfo() {
    struct cpuinfo c;
    c.threads = 0;
    c.cores = 0;
    c.sockets = 0;
#ifdef __MACH__
    unsigned int temp;
    size_t size = sizeof(temp);
    if (sysctlbyname("hw.logicalcpu", &temp, &size, NULL, 0) < 0) {
        myfailures("Unable to get number of CPUs (logical CPUs)");
    }
    c.threads = temp;
    if (sysctlbyname("hw.physicalcpu", &temp, &size, NULL, 0) < 0) {
        myfailures("Unable to get number of cores (physical CPUs)");
    }
    c.cores = temp;
    if (sysctlbyname("hw.packages", &temp, &size, NULL, 0) < 0) {
        myfailures("Unable to get number of CPU sockets");
    }
    c.threads = temp;
#else
    std::ifstream infile;
    infile.open("/proc/cpuinfo");
    if (!infile.good()) {
        myfailures("Error opening /proc/cpuinfo");
    }

    int last_physical_id = -1;
    bool new_socket = false;
    string rec;
    while (getline(infile, rec)) {
        if (rec.find("processor\t:", 0, 11) == 0) {
            // Each time we encounter a processor field, we increment the
            // number of cpus/threads
            c.threads += 1;
        } else if (rec.find("physical id\t:", 0, 13) == 0) {
            // Each time we encounter a new physical id, we increment the
            // number of sockets
            int new_physical_id;
            if (sscanf(rec.c_str(), "physical id\t: %d", &new_physical_id) != 1) {
                myfailures("Error reading 'physical id' field from /proc/cpuinfo");
            }
            if (new_physical_id > last_physical_id) {
                c.sockets += 1;
                last_physical_id = new_physical_id;
                new_socket = true;
            }
        } else if (rec.find("cpu cores\t:", 0, 11) == 0) {
            // Each time we encounter a new socket, we count the number of
            // cores it has
            if (new_socket) {
                cpu_t cores;
                if (sscanf(rec.c_str(), "cpu cores\t: %" SCNcpu_t, &cores) != 1) {
                    myfailures("Error reading 'cpu cores' field from /proc/cpuinfo");
                }
                c.cores += cores;
                new_socket = false;
            }
        }
    }

    if (infile.bad() || !infile.eof()) {
        myfailures("Error reading /proc/cpuinfo");
    }

    infile.close();

    // On some platforms, we can only detect the number of virtual cores
    // https://jira.isi.edu/browse/PM-1403
    if (c.threads > 0 && c.cores == 0) {
        log_warn("Unable to determine the number of cores from /proc/cpuinfo - using the threads value");
        c.cores = c.threads;
    }
    if (c.threads > 0 && c.sockets == 0) {
        log_warn("Unable to determine the number of sockets from /proc/cpuinfo - setting it to 1");
        c.sockets = 1;
    }

#endif
    if (c.threads == 0 || c.cores == 0 || c.sockets == 0 ||
            c.cores > c.threads || c.sockets > c.cores ||
            c.threads % c.cores > 0 || c.cores % c.sockets > 0) {
        myfailure("Invalid cpuinfo: %" PRIcpu_t " %" PRIcpu_t " %" PRIcpu_t, c.threads, c.cores, c.sockets);
    }
    return c;
}

int mkdirs(const char *path) {
    if (path == NULL || strlen(path) == 0) {
        return 0;
    }

    // No need to create cwd
    if (path[0] == '.' && path[1] == '\0') {
        return 0;
    }

    if (strlen(path) > PATH_MAX) {
        errno = ENAMETOOLONG;
        return -1;
    }

    char mypath[PATH_MAX];
    char *p = mypath;

    if (path[0] == '.' && path[1] == '.') {
        if (getcwd(mypath, PATH_MAX) == NULL) {
            return -1;
        }

        char *parent = strrchr(mypath, '/');
        if (parent) *parent = '\0';

        int off = strlen(mypath);
        strcpy(mypath+off, path+2);

        // In this case we don't need to go back to the root
        p = mypath + off;
    } else if (path[0] == '.' && path[1] == '/') {
        strcpy(mypath, path+2);
    } else {
        strcpy(mypath, path);
    }

    while (*p == '/') p++;

    int created = 0;
    struct stat st;
    while ((p = strchr(p, '/'))) {
        *p = '\0';

        if (stat(mypath, &st)) {
            if (errno == ENOENT) {
                if (mkdir(mypath, 0777)) {
                    return -1;
                }
                created++;
            } else {
                return -1;
            }
        } else {
            // If it exists, make sure it is a dir
            if (S_ISDIR(st.st_mode) == 0) {
                errno = ENOTDIR;
                return -1;
            }
        }

        *p = '/';
        while (*p == '/') p++;
    }

    // Last element of path
    if (stat(mypath, &st)) {
        if (errno == ENOENT) {
            if (mkdir(mypath, 0777)) {
                return -1;
            }
            created++;
        } else {
            return -1;
        }
    } else {
        if (S_ISDIR(st.st_mode) == 0) {
            errno = ENOTDIR;
            return -1;
        }
    }

    return created;
}

bool is_executable(const string &file) {
    // Invalid path
    if (file.size() == 0) {
        return false;
    }

    struct stat st;
    if (stat(file.c_str(), &st) == 0) {
        if (S_ISREG(st.st_mode)) {
            if ((st.st_uid == geteuid() && (S_IXUSR & st.st_mode) == S_IXUSR) ||
                (st.st_gid == getegid() && (S_IXGRP & st.st_mode) == S_IXGRP) ||
                ((S_IXOTH & st.st_mode) == S_IXOTH)) {
                // It is a regular file and we can execute it
                return true;
            }
        }
    }

    // In all other cases, the file is not executable
    return false;
}

int read_file(const string &file, char *buf, size_t size) {
    // Invalid path
    if (file.size() == 0) {
        errno = ENOENT;
        return -1;
    }

    FILE *f = fopen(file.c_str(), "r");
    if (f == NULL) {
        return -1;
    }

    size_t read = fread(buf, 1, size, f);

    if (fclose(f)) {
        return -1;
    }

    return read;
}

string pathfind(const string &file) {
    if (file.size() == 0) {
        return file;
    }

    // files that have a / should be returned as-is
    if (file.find('/') != string::npos) {
        return file;
    }

    // normally we wouldn't allow this
    if (is_executable(file)) {
        return file;
    }

    string path;
    char *env = getenv("PATH");
    if (env == NULL) {
#ifdef _PATH_DEFPATH
        path = _PATH_DEFPATH;
#else
        return file;
#endif
    } else {
        /* yes, there is a PATH variable */
        path = env;
    }

    string element;
    std::istringstream split(path);
    while(std::getline(split, element, ':')) {
        if (element.size() == 0) {
            continue;
        }
        string myfile = element + "/" + file;
        if (is_executable(myfile)) {
            return myfile;
        }
    }

    return file;
}

/* Return the directory part of a path */
string dirname(const string &path) {
    char *temp = strdup(path.c_str());
    string result = ::dirname(temp);
    free(temp);
    return result;
}

/* Return the last part of a path */
string filename(const string &path) {
    char *temp = strdup(path.c_str());
    string result = ::basename(temp);
    free(temp);
    return result;
}

/* Set the cpu affinity to values in bindings */
int set_cpu_affinity(vector<cpu_t> &bindings) {
#ifdef LINUX
    struct cpuinfo c = get_host_cpuinfo();
    cpu_set_t *cpuset = CPU_ALLOC(c.threads);
    if (cpuset == NULL) {
        return -1;
    }
    size_t cpusetsize = CPU_ALLOC_SIZE(c.threads);
    CPU_ZERO_S(cpusetsize, cpuset);

    for (vector<cpu_t>::iterator i = bindings.begin(); i != bindings.end(); i++) {
        cpu_t j = *i;
        if (j >= c.threads) {
            CPU_FREE(cpuset);
            errno = ERANGE;
            return -1;
        }
        CPU_SET_S(j, cpusetsize, cpuset);
    }

    int rc = sched_setaffinity(0, cpusetsize, cpuset);
    CPU_FREE(cpuset);
    if (rc < 0) {
        return -1;
    }
#endif
    return 0;
}

int clear_cpu_affinity() {
#ifdef LINUX
    struct cpuinfo c = get_host_cpuinfo();
    cpu_set_t *cpuset = CPU_ALLOC(c.threads);
    if (cpuset == NULL) {
        return -1;
    }
    size_t cpusetsize = CPU_ALLOC_SIZE(c.threads);
    CPU_ZERO_S(cpusetsize, cpuset);

    for (unsigned i=0; i<c.threads; i++) {
        CPU_SET_S(i, cpusetsize, cpuset);
    }

    int rc = sched_setaffinity(0, cpusetsize, cpuset);
    CPU_FREE(cpuset);
    if (rc < 0) {
        return -1;
    }
#endif
    return 0;
}

int clear_memory_affinity() {
#ifdef HAS_LIBNUMA
    int rc = set_mempolicy(MPOL_DEFAULT, NULL, 0);
    if (rc < 0) {
        return -1;
    }
#endif
    return 0;
}
<|endoftext|>


<commit_before>#include <chrono>
#include <cstdint>
#include <exception>
#include <fstream>
#include <iomanip>
#include <iostream>
#include <stdexcept>
#include <string>
#include <vector>

#include <gflags/gflags.h>
#include <glog/logging.h>

#include <tudocomp/Compressor.hpp>
#include <tudocomp/io.hpp>
#include <tudocomp/io/IOUtil.hpp>
#include <tudocomp/util/Json.hpp>
#include <tudocomp_driver/Registry.hpp>

/*namespace validate {
  static bool algorithm(const char* flagname, int32 value) {
    if (value > 0 && value < 32768)   // value is ok
      return true;
    printf("Invalid value for --%s: %d\n", flagname, (int)value);
    return false;
  }
  static const bool algorithm_dummy = RegisterFlagValidator(&FLAGS_algorithm, &algorithm);
}*/

DEFINE_string(algorithm, "", "Algorithm to use for (de)compression. See --list for a complete list of them.");
DEFINE_bool(decompress, false, "Decompress input instead of compressing it.");
DEFINE_string(output, "", "Choose output filename instead the the default generated one of <input>.tdc or stdout.");
DEFINE_bool(stats, false, "Print statistics to stdout.");
DEFINE_bool(force, false, "Overwrite output even if it exists.");
DEFINE_bool(list, false, "List all compression algorithms supported by this tool.");
DEFINE_bool(raw, false, "Do not emit an header into the output file when compressing.");
DEFINE_bool(usestdin, false, "Read from stdin instead of trying to open a file.");
DEFINE_bool(usestdout, false, "Output to stdout instead of writing to a file");

namespace tdc_driver {

using namespace tdc;
using namespace tdc_algorithms;

const std::string COMPRESSED_FILE_ENDING = "tdc";

static void exit(std::string msg) {
    // TODO: Replace with more specific logic-error-exception
    throw std::runtime_error(msg);
}

static void validate_flag_combination() {
    auto err = [](std::string s) {
        exit(s);
    };

    if (!FLAGS_list) {
        if (!FLAGS_decompress && (std::string(FLAGS_algorithm) == "")) {
            err("Need to give an algorithm for compression");
        }
        if (FLAGS_decompress && FLAGS_raw && (std::string(FLAGS_algorithm) == "")) {
            err("Need to give an algorithm for raw decompression");
        }
    }
}

static bool fexists(std::string filename)
{
  std::ifstream ifile(filename);
  return bool(ifile);
}

static bool check_for_file_already_exist(std::string& ofile,
                                         bool allow_overwrite) {
    // Don't accidentially overwrite files
    if (!allow_overwrite && fexists(ofile)) {
        std::cerr << "Outputfile already exists\n";
        return false;
    }
    return true;
}

} // namespace tdc_driver

#include <iomanip>

int main(int argc, char** argv)
{
    using namespace tdc_driver;
    using namespace tdc_algorithms;

    google::InitGoogleLogging(argv[0]);
    google::SetUsageMessage("This tool compresses and decompresses files");
    if(argc == 1 || strcmp(argv[1],"-h") == 0 || strcmp(argv[1],"--help") == 0 || strcmp(argv[1],"-help") == 0) {
//        google::ShowUsageWithFlagsRestrict(argv[0], __FILE__); //shortcut
        std::vector<google::CommandLineFlagInfo> info;
        google::GetAllFlags(&info);

        std::cout << argv[0] << " [options] {file to compress/decompress}" << std::endl;
        std::cout << "You need to provide at least an algorithm and a source (a file) to compress/decompress." << std::endl << std::endl;
        std::cout
            << std::setw(20) << std::setiosflags(std::ios::left) << "Parameter"
            << std::setw(10) << "Type"
            << std::setw(20) << "Default"
            << "Description" << std::endl;
        std::cout << std::endl;
         for(auto it = info.cbegin(); it != info.cend(); ++it) {
             if(it->filename != __FILE__) continue;
                 std::cout
                    << std::setw(20) << std::setiosflags(std::ios::left)<< (std::string("--")+ it->name)
                    << std::setw(10) << it->type
                    << std::setw(20) << (std::string("(") + it->default_value + ")")
                    << it->description << std::endl;
         }
        return 0;
    }

    int first_cmd_arg = google::ParseCommandLineFlags(&argc, &argv, true);

    try {
        validate_flag_combination();

        // Set up algorithms
        Registry& registry = REGISTRY;

        if (FLAGS_list) {
            std::cout << "This build supports the following algorithms:\n";
            std::cout << std::endl;
            std::cout << registry.generate_doc_string();

            return 0;
        }

        bool print_stats = FLAGS_stats;
        bool do_compress = !FLAGS_decompress;
        bool do_raw = FLAGS_raw;

        /////////////////////////////////////////////////////////////////////////
        // Select algorithm

        class Selection {
            std::string m_id_string;
            std::unique_ptr<Compressor> m_compressor;
            bool m_needs_sentinel_terminator;
            std::shared_ptr<EnvRoot> m_algorithm_env;
        public:
            Selection():
                m_id_string(),
                m_compressor(),
                m_needs_sentinel_terminator(false),
                m_algorithm_env() {}
            Selection(std::string&& id_string,
                      std::unique_ptr<Compressor>&& compressor,
                      bool needs_sentinel_terminator,
                      std::shared_ptr<EnvRoot>&& algorithm_env):
                m_id_string(std::move(id_string)),
                m_compressor(std::move(compressor)),
                m_needs_sentinel_terminator(needs_sentinel_terminator),
                m_algorithm_env(std::move(algorithm_env)) {}
            const std::string& id_string() const {
                return m_id_string;
            }
            Compressor& compressor() {
                return *m_compressor;
            }
            bool needs_sentinel_terminator() const {
                return m_needs_sentinel_terminator;
            }
            const std::shared_ptr<EnvRoot>& algorithm_env() const {
                return m_algorithm_env;
            }
        };
        Selection selection;

        if (!FLAGS_algorithm.empty()) {
            auto id_string = FLAGS_algorithm;

            auto av = registry.parse_algorithm_id(id_string);
            auto needs_sentinel_terminator = av.needs_sentinel_terminator();
            auto compressor = registry.select_algorithm_or_exit(av);
            auto algorithm_env = compressor->env().root();

            selection = Selection {
                std::move(id_string),
                std::move(compressor),
                needs_sentinel_terminator,
                std::move(algorithm_env),
            };
        }

        /////////////////////////////////////////////////////////////////////////
        // Select where the input comes from

        std::string file;

        if ((!FLAGS_usestdin) && (first_cmd_arg < argc)) {
            file = argv[first_cmd_arg];
            if (!fexists(file)) {
                std::cerr << "input file " << file << " does not exist\n";
                return 1;
            }
        } else if (!FLAGS_usestdin) {
            std::cerr << "No input file given\n";
            return 1;
        }

        bool use_stdin = FLAGS_usestdin;

        /////////////////////////////////////////////////////////////////////////
        // Select where the output goes to

        std::string ofile;
        bool use_stdout = FLAGS_usestdout;
        bool use_explict_output = std::string(FLAGS_output) != "" ;

        if (use_explict_output) {
            // Output to manually specifed file
            ofile = FLAGS_output;
        } else if (!use_stdin && do_compress) {
            // Output to a automatically determined file
            ofile = file + "." + COMPRESSED_FILE_ENDING;
        } else if (!use_stdin && !do_compress && !use_stdout) {
            std::cerr << "Need to specify a output filename\n";
            return 1;
        }

        /////////////////////////////////////////////////////////////////////////
        // open streams

        using clk = std::chrono::high_resolution_clock;

        clk::time_point start_time = clk::now();
        clk::time_point setup_time;
        clk::time_point comp_time;
        clk::time_point end_time;

        {
            std::vector<uint8_t> stream_buffer;
            Input inp;

            if (use_stdin) {
                // Input from stdin
                inp = Input(std::cin);
            } else {
                // Input from specified file
                inp = Input::from_path(file);
            }

            Output out;
            if (use_stdout) {
                // Output to stdout
                out = Output::from_stream(std::cout);
            } else {
                // Output to specified file
                bool force = FLAGS_force;
                if (!check_for_file_already_exist(ofile, force)) {
                    return 1;
                } else {
                    out = Output::from_path(ofile, true);
                }
            }

            /////////////////////////////////////////////////////////////////////////
            // call into actual library

            if (do_compress) {
                if (!do_raw) {
                    CHECK(selection.id_string().find('%') == std::string::npos);

                    auto o_stream = out.as_stream();
                    o_stream << selection.id_string() << '%';
                }

                if (selection.needs_sentinel_terminator()) {
                    inp.escape_and_terminate();
                }

                selection.algorithm_env()->restart_stats("Compress");
                setup_time = clk::now();
                selection.compressor().compress(inp, out);
                comp_time = clk::now();
            } else {
                // 3 cases
                // --decompress                   : read and use header
                // --decompress --algorithm       : read but ignore header
                // --decompress --raw --algorithm : no header

                std::string algorithm_header;

                if (!do_raw) {
                    // read header
                    auto i_stream = inp.as_stream();

                    char c;
                    size_t sanity_size_check = 0;
                    bool err = false;
                    while (i_stream.get(c)) {
                        err = false;
                        if (sanity_size_check > 1023) {
                            err = true;
                            break;
                        } else if (c == '%') {
                            break;
                        } else {
                            algorithm_header.push_back(c);
                        }
                        sanity_size_check++;
                        err = true;
                    }
                    if (err) {
                        exit("Input did not have an algorithm header!");
                    }
                }

                if (!do_raw && !selection.id_string().empty()) {
                    DLOG(INFO) << "Ignoring header " << algorithm_header
                        << " and using manually given " << selection.id_string();
                } else if (!do_raw) {
                    DLOG(INFO) << "Using header id string " << algorithm_header;

                    auto id_string = std::move(algorithm_header);
                    auto av = registry.parse_algorithm_id(id_string);
                    auto compressor = registry.select_algorithm_or_exit(av);
                    auto needs_sentinel_terminator = av.needs_sentinel_terminator();
                    auto algorithm_env = compressor->env().root();

                    selection = Selection {
                        std::move(id_string),
                        std::move(compressor),
                        needs_sentinel_terminator,
                        std::move(algorithm_env),
                    };
                } else {
                    DLOG(INFO) << "Using manually given " << selection.id_string();
                }

                if (selection.needs_sentinel_terminator()) {
                    out.unescape_and_trim();
                }

                selection.algorithm_env()->restart_stats("Decompress");
                setup_time = clk::now();
                selection.compressor().decompress(inp, out);
                comp_time = clk::now();
            }
        }

        auto algo_stats = selection.algorithm_env()->finish_stats();
        end_time = clk::now();

        if (print_stats) {
            auto setup_duration = setup_time - start_time;
            auto comp_duration = comp_time - setup_time;
            auto end_duration = end_time - comp_time;

            size_t in_size  = use_stdin  ? 0 : io::read_file_size(file);
            size_t out_size = use_stdout ? 0 : io::read_file_size(ofile);

            json::Object meta;
            meta.set("startTime",
                std::chrono::duration_cast<std::chrono::seconds>(
                    start_time.time_since_epoch()).count());

            meta.set("config", selection.id_string());
            meta.set("input", use_stdin ? "<stdin>" : file);
            meta.set("inputSize", in_size);
            meta.set("output", use_stdout ? "<stdin>" : file);
            meta.set("outputSize", out_size);
            meta.set("rate", (use_stdin || use_stdout) ? 0.0 :
                double(out_size) / double(in_size));

            json::Object stats;
            stats.set("meta", meta);
            stats.set("data", algo_stats.to_json());

            stats.str(std::cout);
            std::cout << std::endl;
        }
    } catch (std::exception& e) {
        std::cout << "Error: " << e.what() << '\n';
        return 1;
    }

    return 0;
}
<commit_msg>Fix cnp mistake in stat JSON building. Fixes #18379<commit_after>#include <chrono>
#include <cstdint>
#include <exception>
#include <fstream>
#include <iomanip>
#include <iostream>
#include <stdexcept>
#include <string>
#include <vector>

#include <gflags/gflags.h>
#include <glog/logging.h>

#include <tudocomp/Compressor.hpp>
#include <tudocomp/io.hpp>
#include <tudocomp/io/IOUtil.hpp>
#include <tudocomp/util/Json.hpp>
#include <tudocomp_driver/Registry.hpp>

/*namespace validate {
  static bool algorithm(const char* flagname, int32 value) {
    if (value > 0 && value < 32768)   // value is ok
      return true;
    printf("Invalid value for --%s: %d\n", flagname, (int)value);
    return false;
  }
  static const bool algorithm_dummy = RegisterFlagValidator(&FLAGS_algorithm, &algorithm);
}*/

DEFINE_string(algorithm, "", "Algorithm to use for (de)compression. See --list for a complete list of them.");
DEFINE_bool(decompress, false, "Decompress input instead of compressing it.");
DEFINE_string(output, "", "Choose output filename instead the the default generated one of <input>.tdc or stdout.");
DEFINE_bool(stats, false, "Print statistics to stdout.");
DEFINE_bool(force, false, "Overwrite output even if it exists.");
DEFINE_bool(list, false, "List all compression algorithms supported by this tool.");
DEFINE_bool(raw, false, "Do not emit an header into the output file when compressing.");
DEFINE_bool(usestdin, false, "Read from stdin instead of trying to open a file.");
DEFINE_bool(usestdout, false, "Output to stdout instead of writing to a file");

namespace tdc_driver {

using namespace tdc;
using namespace tdc_algorithms;

const std::string COMPRESSED_FILE_ENDING = "tdc";

static void exit(std::string msg) {
    // TODO: Replace with more specific logic-error-exception
    throw std::runtime_error(msg);
}

static void validate_flag_combination() {
    auto err = [](std::string s) {
        exit(s);
    };

    if (!FLAGS_list) {
        if (!FLAGS_decompress && (std::string(FLAGS_algorithm) == "")) {
            err("Need to give an algorithm for compression");
        }
        if (FLAGS_decompress && FLAGS_raw && (std::string(FLAGS_algorithm) == "")) {
            err("Need to give an algorithm for raw decompression");
        }
    }
}

static bool fexists(std::string filename)
{
  std::ifstream ifile(filename);
  return bool(ifile);
}

static bool check_for_file_already_exist(std::string& ofile,
                                         bool allow_overwrite) {
    // Don't accidentially overwrite files
    if (!allow_overwrite && fexists(ofile)) {
        std::cerr << "Outputfile already exists\n";
        return false;
    }
    return true;
}

} // namespace tdc_driver

#include <iomanip>

int main(int argc, char** argv)
{
    using namespace tdc_driver;
    using namespace tdc_algorithms;

    google::InitGoogleLogging(argv[0]);
    google::SetUsageMessage("This tool compresses and decompresses files");
    if(argc == 1 || strcmp(argv[1],"-h") == 0 || strcmp(argv[1],"--help") == 0 || strcmp(argv[1],"-help") == 0) {
//        google::ShowUsageWithFlagsRestrict(argv[0], __FILE__); //shortcut
        std::vector<google::CommandLineFlagInfo> info;
        google::GetAllFlags(&info);

        std::cout << argv[0] << " [options] {file to compress/decompress}" << std::endl;
        std::cout << "You need to provide at least an algorithm and a source (a file) to compress/decompress." << std::endl << std::endl;
        std::cout
            << std::setw(20) << std::setiosflags(std::ios::left) << "Parameter"
            << std::setw(10) << "Type"
            << std::setw(20) << "Default"
            << "Description" << std::endl;
        std::cout << std::endl;
         for(auto it = info.cbegin(); it != info.cend(); ++it) {
             if(it->filename != __FILE__) continue;
                 std::cout
                    << std::setw(20) << std::setiosflags(std::ios::left)<< (std::string("--")+ it->name)
                    << std::setw(10) << it->type
                    << std::setw(20) << (std::string("(") + it->default_value + ")")
                    << it->description << std::endl;
         }
        return 0;
    }

    int first_cmd_arg = google::ParseCommandLineFlags(&argc, &argv, true);

    try {
        validate_flag_combination();

        // Set up algorithms
        Registry& registry = REGISTRY;

        if (FLAGS_list) {
            std::cout << "This build supports the following algorithms:\n";
            std::cout << std::endl;
            std::cout << registry.generate_doc_string();

            return 0;
        }

        bool print_stats = FLAGS_stats;
        bool do_compress = !FLAGS_decompress;
        bool do_raw = FLAGS_raw;

        /////////////////////////////////////////////////////////////////////////
        // Select algorithm

        class Selection {
            std::string m_id_string;
            std::unique_ptr<Compressor> m_compressor;
            bool m_needs_sentinel_terminator;
            std::shared_ptr<EnvRoot> m_algorithm_env;
        public:
            Selection():
                m_id_string(),
                m_compressor(),
                m_needs_sentinel_terminator(false),
                m_algorithm_env() {}
            Selection(std::string&& id_string,
                      std::unique_ptr<Compressor>&& compressor,
                      bool needs_sentinel_terminator,
                      std::shared_ptr<EnvRoot>&& algorithm_env):
                m_id_string(std::move(id_string)),
                m_compressor(std::move(compressor)),
                m_needs_sentinel_terminator(needs_sentinel_terminator),
                m_algorithm_env(std::move(algorithm_env)) {}
            const std::string& id_string() const {
                return m_id_string;
            }
            Compressor& compressor() {
                return *m_compressor;
            }
            bool needs_sentinel_terminator() const {
                return m_needs_sentinel_terminator;
            }
            const std::shared_ptr<EnvRoot>& algorithm_env() const {
                return m_algorithm_env;
            }
        };
        Selection selection;

        if (!FLAGS_algorithm.empty()) {
            auto id_string = FLAGS_algorithm;

            auto av = registry.parse_algorithm_id(id_string);
            auto needs_sentinel_terminator = av.needs_sentinel_terminator();
            auto compressor = registry.select_algorithm_or_exit(av);
            auto algorithm_env = compressor->env().root();

            selection = Selection {
                std::move(id_string),
                std::move(compressor),
                needs_sentinel_terminator,
                std::move(algorithm_env),
            };
        }

        /////////////////////////////////////////////////////////////////////////
        // Select where the input comes from

        std::string file;

        if ((!FLAGS_usestdin) && (first_cmd_arg < argc)) {
            file = argv[first_cmd_arg];
            if (!fexists(file)) {
                std::cerr << "input file " << file << " does not exist\n";
                return 1;
            }
        } else if (!FLAGS_usestdin) {
            std::cerr << "No input file given\n";
            return 1;
        }

        bool use_stdin = FLAGS_usestdin;

        /////////////////////////////////////////////////////////////////////////
        // Select where the output goes to

        std::string ofile;
        bool use_stdout = FLAGS_usestdout;
        bool use_explict_output = std::string(FLAGS_output) != "" ;

        if (use_explict_output) {
            // Output to manually specifed file
            ofile = FLAGS_output;
        } else if (!use_stdin && do_compress) {
            // Output to a automatically determined file
            ofile = file + "." + COMPRESSED_FILE_ENDING;
        } else if (!use_stdin && !do_compress && !use_stdout) {
            std::cerr << "Need to specify a output filename\n";
            return 1;
        }

        /////////////////////////////////////////////////////////////////////////
        // open streams

        using clk = std::chrono::high_resolution_clock;

        clk::time_point start_time = clk::now();
        clk::time_point setup_time;
        clk::time_point comp_time;
        clk::time_point end_time;

        {
            std::vector<uint8_t> stream_buffer;
            Input inp;

            if (use_stdin) {
                // Input from stdin
                inp = Input(std::cin);
            } else {
                // Input from specified file
                inp = Input::from_path(file);
            }

            Output out;
            if (use_stdout) {
                // Output to stdout
                out = Output::from_stream(std::cout);
            } else {
                // Output to specified file
                bool force = FLAGS_force;
                if (!check_for_file_already_exist(ofile, force)) {
                    return 1;
                } else {
                    out = Output::from_path(ofile, true);
                }
            }

            /////////////////////////////////////////////////////////////////////////
            // call into actual library

            if (do_compress) {
                if (!do_raw) {
                    CHECK(selection.id_string().find('%') == std::string::npos);

                    auto o_stream = out.as_stream();
                    o_stream << selection.id_string() << '%';
                }

                if (selection.needs_sentinel_terminator()) {
                    inp.escape_and_terminate();
                }

                selection.algorithm_env()->restart_stats("Compress");
                setup_time = clk::now();
                selection.compressor().compress(inp, out);
                comp_time = clk::now();
            } else {
                // 3 cases
                // --decompress                   : read and use header
                // --decompress --algorithm       : read but ignore header
                // --decompress --raw --algorithm : no header

                std::string algorithm_header;

                if (!do_raw) {
                    // read header
                    auto i_stream = inp.as_stream();

                    char c;
                    size_t sanity_size_check = 0;
                    bool err = false;
                    while (i_stream.get(c)) {
                        err = false;
                        if (sanity_size_check > 1023) {
                            err = true;
                            break;
                        } else if (c == '%') {
                            break;
                        } else {
                            algorithm_header.push_back(c);
                        }
                        sanity_size_check++;
                        err = true;
                    }
                    if (err) {
                        exit("Input did not have an algorithm header!");
                    }
                }

                if (!do_raw && !selection.id_string().empty()) {
                    DLOG(INFO) << "Ignoring header " << algorithm_header
                        << " and using manually given " << selection.id_string();
                } else if (!do_raw) {
                    DLOG(INFO) << "Using header id string " << algorithm_header;

                    auto id_string = std::move(algorithm_header);
                    auto av = registry.parse_algorithm_id(id_string);
                    auto compressor = registry.select_algorithm_or_exit(av);
                    auto needs_sentinel_terminator = av.needs_sentinel_terminator();
                    auto algorithm_env = compressor->env().root();

                    selection = Selection {
                        std::move(id_string),
                        std::move(compressor),
                        needs_sentinel_terminator,
                        std::move(algorithm_env),
                    };
                } else {
                    DLOG(INFO) << "Using manually given " << selection.id_string();
                }

                if (selection.needs_sentinel_terminator()) {
                    out.unescape_and_trim();
                }

                selection.algorithm_env()->restart_stats("Decompress");
                setup_time = clk::now();
                selection.compressor().decompress(inp, out);
                comp_time = clk::now();
            }
        }

        auto algo_stats = selection.algorithm_env()->finish_stats();
        end_time = clk::now();

        if (print_stats) {
            auto setup_duration = setup_time - start_time;
            auto comp_duration = comp_time - setup_time;
            auto end_duration = end_time - comp_time;

            size_t in_size  = use_stdin  ? 0 : io::read_file_size(file);
            size_t out_size = use_stdout ? 0 : io::read_file_size(ofile);

            json::Object meta;
            meta.set("startTime",
                std::chrono::duration_cast<std::chrono::seconds>(
                    start_time.time_since_epoch()).count());

            meta.set("config", selection.id_string());
            meta.set("input", use_stdin ? "<stdin>" : file);
            meta.set("inputSize", in_size);
            meta.set("output", use_stdout ? "<stdin>" : ofile);
            meta.set("outputSize", out_size);
            meta.set("rate", (use_stdin || use_stdout) ? 0.0 :
                double(out_size) / double(in_size));

            json::Object stats;
            stats.set("meta", meta);
            stats.set("data", algo_stats.to_json());

            stats.str(std::cout);
            std::cout << std::endl;
        }
    } catch (std::exception& e) {
        std::cout << "Error: " << e.what() << '\n';
        return 1;
    }

    return 0;
}
<|endoftext|>