<commit_before>// Copyright (c) 2009 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include <cstring>

#include "cff.h"
#include "ots.h"

// name - Naming Table
// http://www.microsoft.com/opentype/otspec/name.htm

namespace ots {

bool ots_name_parse(OpenTypeFile *, const uint8_t *, size_t) {
  return true;
}

bool ots_name_should_serialise(OpenTypeFile *) {
  return true;
}

bool ots_name_serialise(OTSStream *out, OpenTypeFile *file) {
  // NAME is a required table, but we don't want anything to do with it. Thus,
  // we don't bother parsing it and we just serialise an empty name table.

  const char* kStrings[] = {
      "Derived font data",  // 0: copyright
      "OTS derived font",  // 1: the name the user sees
      "Unspecified",  // 2: face weight
      "UniqueID",  // 3: unique id
      "OTS derivied font",  // 4: human readable name
      "1.000",  // 5: version
      "False",  // 6: postscript name
      NULL,  // 7: trademark data
      "OTS",  // 8: foundary
      "OTS",  // 9: designer
  };
  static const size_t kStringsLen = sizeof(kStrings) / sizeof(kStrings[0]);

  // The spec says that "In CFF OpenType fonts, these two name strings, when
  // translated to ASCII, must also be identical to the font name as stored in
  // the CFF's Name INDEX." And actually, Mac OS X's font parser requires that.
  if (file->cff && !file->cff->name.empty()) {
    kStrings[6] = file->cff->name.c_str();
  }

  unsigned num_strings = 0;
  for (unsigned i = 0; i < kStringsLen; ++i) {
    if (kStrings[i]) num_strings++;
  }

  if (!out->WriteU16(0) ||  // version
      // Magic numbers:
      //   6:  This entry (U16 * 3 = 6 bytes)
      //   2:  Mac Roman & Windows Roman = 2 types
      //   12: Each string entry (U16 * 6 = 12 bytes)
      !out->WriteU16(num_strings * 2) ||  // count
      !out->WriteU16(6 + num_strings * 2 * 12)) {  // string data offset
    return OTS_FAILURE();
  }

  unsigned current_offset = 0;
  for (unsigned i = 0; i < kStringsLen; ++i) {
    if (!kStrings[i]) continue;

    // string length in UTF-8 (ASCII).
    size_t len = std::strlen(kStrings[i]);

    if (!out->WriteU16(1) ||  // Mac
        !out->WriteU16(0) ||  // Roman
        !out->WriteU16(0) ||  // English
        !out->WriteU16(i) ||
        !out->WriteU16(len) ||
        !out->WriteU16(current_offset)) {
      return OTS_FAILURE();
    }

    current_offset += len;
  }

  for (unsigned i = 0; i < kStringsLen; ++i) {
    if (!kStrings[i]) continue;

    // string length in UTF-16.
    size_t len = std::strlen(kStrings[i]) * 2;

    if (!out->WriteU16(3) ||  // Windows
        !out->WriteU16(1) ||  // Unicode BMP (UCS-2)
        !out->WriteU16(0x0409) ||  // US English
        !out->WriteU16(i) ||
        !out->WriteU16(len) ||
        !out->WriteU16(current_offset)) {
      return OTS_FAILURE();
    }

    current_offset += len;
  }

  // Write strings in Mac Roman compatible with ASCII.
  // Because all the entries are ASCII, we can just copy.
  for (unsigned i = 0; i < kStringsLen; ++i) {
    if (!kStrings[i]) continue;

    const size_t len = std::strlen(kStrings[i]);
    if (!out->Write(kStrings[i], len)) {
      return OTS_FAILURE();
    }
  }

  // Write strings in UCS-2. Because all the entries are ASCII,
  // we can just expand each byte to U16.
  for (unsigned i = 0; i < kStringsLen; ++i) {
    if (!kStrings[i]) continue;

    const size_t len = std::strlen(kStrings[i]);
    for (size_t j = 0; j < len; ++j) {
      uint16_t v = kStrings[i][j];
      if (!out->WriteU16(v)) {
        return OTS_FAILURE();
      }
    }
  }

  return true;
}

void ots_name_free(OpenTypeFile *) {
}

}  // namespace
<commit_msg>Typo fix.<commit_after>// Copyright (c) 2009 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include <cstring>

#include "cff.h"
#include "ots.h"

// name - Naming Table
// http://www.microsoft.com/opentype/otspec/name.htm

namespace ots {

bool ots_name_parse(OpenTypeFile *, const uint8_t *, size_t) {
  return true;
}

bool ots_name_should_serialise(OpenTypeFile *) {
  return true;
}

bool ots_name_serialise(OTSStream *out, OpenTypeFile *file) {
  // NAME is a required table, but we don't want anything to do with it. Thus,
  // we don't bother parsing it and we just serialise an empty name table.

  const char* kStrings[] = {
      "Derived font data",  // 0: copyright
      "OTS derived font",  // 1: the name the user sees
      "Unspecified",  // 2: face weight
      "UniqueID",  // 3: unique id
      "OTS derived font",  // 4: human readable name
      "1.000",  // 5: version
      "False",  // 6: postscript name
      NULL,  // 7: trademark data
      "OTS",  // 8: foundary
      "OTS",  // 9: designer
  };
  static const size_t kStringsLen = sizeof(kStrings) / sizeof(kStrings[0]);

  // The spec says that "In CFF OpenType fonts, these two name strings, when
  // translated to ASCII, must also be identical to the font name as stored in
  // the CFF's Name INDEX." And actually, Mac OS X's font parser requires that.
  if (file->cff && !file->cff->name.empty()) {
    kStrings[6] = file->cff->name.c_str();
  }

  unsigned num_strings = 0;
  for (unsigned i = 0; i < kStringsLen; ++i) {
    if (kStrings[i]) num_strings++;
  }

  if (!out->WriteU16(0) ||  // version
      // Magic numbers:
      //   6:  This entry (U16 * 3 = 6 bytes)
      //   2:  Mac Roman & Windows Roman = 2 types
      //   12: Each string entry (U16 * 6 = 12 bytes)
      !out->WriteU16(num_strings * 2) ||  // count
      !out->WriteU16(6 + num_strings * 2 * 12)) {  // string data offset
    return OTS_FAILURE();
  }

  unsigned current_offset = 0;
  for (unsigned i = 0; i < kStringsLen; ++i) {
    if (!kStrings[i]) continue;

    // string length in UTF-8 (ASCII).
    size_t len = std::strlen(kStrings[i]);

    if (!out->WriteU16(1) ||  // Mac
        !out->WriteU16(0) ||  // Roman
        !out->WriteU16(0) ||  // English
        !out->WriteU16(i) ||
        !out->WriteU16(len) ||
        !out->WriteU16(current_offset)) {
      return OTS_FAILURE();
    }

    current_offset += len;
  }

  for (unsigned i = 0; i < kStringsLen; ++i) {
    if (!kStrings[i]) continue;

    // string length in UTF-16.
    size_t len = std::strlen(kStrings[i]) * 2;

    if (!out->WriteU16(3) ||  // Windows
        !out->WriteU16(1) ||  // Unicode BMP (UCS-2)
        !out->WriteU16(0x0409) ||  // US English
        !out->WriteU16(i) ||
        !out->WriteU16(len) ||
        !out->WriteU16(current_offset)) {
      return OTS_FAILURE();
    }

    current_offset += len;
  }

  // Write strings in Mac Roman compatible with ASCII.
  // Because all the entries are ASCII, we can just copy.
  for (unsigned i = 0; i < kStringsLen; ++i) {
    if (!kStrings[i]) continue;

    const size_t len = std::strlen(kStrings[i]);
    if (!out->Write(kStrings[i], len)) {
      return OTS_FAILURE();
    }
  }

  // Write strings in UCS-2. Because all the entries are ASCII,
  // we can just expand each byte to U16.
  for (unsigned i = 0; i < kStringsLen; ++i) {
    if (!kStrings[i]) continue;

    const size_t len = std::strlen(kStrings[i]);
    for (size_t j = 0; j < len; ++j) {
      uint16_t v = kStrings[i][j];
      if (!out->WriteU16(v)) {
        return OTS_FAILURE();
      }
    }
  }

  return true;
}

void ots_name_free(OpenTypeFile *) {
}

}  // namespace
<|endoftext|>


<commit_before>// Copyright (c) 2009-2012 The Bitcoin developers
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include <boost/assign/list_of.hpp> // for 'map_list_of()'
#include <boost/foreach.hpp>

#include "checkpoints.h"

#include "main.h"
#include "uint256.h"

namespace Checkpoints
{
    typedef std::map<int, uint256> MapCheckpoints;

    // How many times we expect transactions after the last checkpoint to
    // be slower. This number is a compromise, as it can't be accurate for
    // every system. When reindexing from a fast disk with a slow CPU, it
    // can be up to 20, while when downloading from a slow network with a
    // fast multicore CPU, it won't be much higher than 1.
    static const double fSigcheckVerificationFactor = 5.0;

    struct CCheckpointData {
        const MapCheckpoints *mapCheckpoints;
        int64 nTimeLastCheckpoint;
        int64 nTransactionsLastCheckpoint;
        double fTransactionsPerDay;
    };

    // What makes a good checkpoint block?
    // + Is surrounded by blocks with reasonable timestamps
    //   (no blocks before with a timestamp after, none after with
    //    timestamp before)
    // + Contains no strange transactions
    static MapCheckpoints mapCheckpoints =
        boost::assign::map_list_of
        (  0, uint256("0x835638aa4511b7482c70a4b91de5e04b8dfee38204779207bdd307eb5728ffd2"))
        (  1, uint256("0x4191a0dbc253afd4409d85e6f40b9e5a2c4257bba60362567d333de481b72dd8"))
        (  137, uint256("0xa202b975be7bd20e0bb566d04ad3e8323bb95ddad37795972c0155706a2f60b1"))
        (  1337, uint256("0x5cdcb8f076baa31e00ab92ce393ef8ef035373c9779fbde291cdd59668c342ca"))
        (  13337, uint256("0xe9e83c681d7247c13d2c0c2b493d8e404a058b2ac0506dc369cd62f91e0dcb02"))
        ;
    static const CCheckpointData data = {
        &mapCheckpoints,
        1381125340, // * UNIX timestamp of last checkpoint block
        0,    // * total number of transactions between genesis and last checkpoint
                    //   (the tx=... number in the SetBestChain debug.log lines)
        500.0     // * estimated number of transactions per day after checkpoint
    };

    static MapCheckpoints mapCheckpointsTestnet =
        boost::assign::map_list_of
        (   0, uint256("6458d9895dc9dc9cb15248dbd0ce0145830b0f677471aea1d77bad9deb2008f4"))
        ;
    static const CCheckpointData dataTestnet = {
        &mapCheckpointsTestnet,
        1380502719,
        0,
        100
    };

    const CCheckpointData &Checkpoints() {
        if (fTestNet)
            return dataTestnet;
        else
            return data;
    }

    bool CheckBlock(int nHeight, const uint256& hash)
    {
        if (fTestNet) return true; // Testnet has no checkpoints
        if (!GetBoolArg("-checkpoints", true))
            return true;

        const MapCheckpoints& checkpoints = *Checkpoints().mapCheckpoints;

        MapCheckpoints::const_iterator i = checkpoints.find(nHeight);
        if (i == checkpoints.end()) return true;
        return hash == i->second;
    }

    // Guess how far we are in the verification process at the given block index
    double GuessVerificationProgress(CBlockIndex *pindex) {
        if (pindex==NULL)
            return 0.0;

        int64 nNow = time(NULL);

        double fWorkBefore = 0.0; // Amount of work done before pindex
        double fWorkAfter = 0.0;  // Amount of work left after pindex (estimated)
        // Work is defined as: 1.0 per transaction before the last checkoint, and
        // fSigcheckVerificationFactor per transaction after.

        const CCheckpointData &data = Checkpoints();

        if (pindex->nChainTx <= data.nTransactionsLastCheckpoint) {
            double nCheapBefore = pindex->nChainTx;
            double nCheapAfter = data.nTransactionsLastCheckpoint - pindex->nChainTx;
            double nExpensiveAfter = (nNow - data.nTimeLastCheckpoint)/86400.0*data.fTransactionsPerDay;
            fWorkBefore = nCheapBefore;
            fWorkAfter = nCheapAfter + nExpensiveAfter*fSigcheckVerificationFactor;
        } else {
            double nCheapBefore = data.nTransactionsLastCheckpoint;
            double nExpensiveBefore = pindex->nChainTx - data.nTransactionsLastCheckpoint;
            double nExpensiveAfter = (nNow - pindex->nTime)/86400.0*data.fTransactionsPerDay;
            fWorkBefore = nCheapBefore + nExpensiveBefore*fSigcheckVerificationFactor;
            fWorkAfter = nExpensiveAfter*fSigcheckVerificationFactor;
        }

        return fWorkBefore / (fWorkBefore + fWorkAfter);
    }

    int GetTotalBlocksEstimate()
    {
        if (fTestNet) return 0; // Testnet has no checkpoints
        if (!GetBoolArg("-checkpoints", true))
            return 0;

        const MapCheckpoints& checkpoints = *Checkpoints().mapCheckpoints;

        return checkpoints.rbegin()->first;
    }

    CBlockIndex* GetLastCheckpoint(const std::map<uint256, CBlockIndex*>& mapBlockIndex)
    {
        if (fTestNet) return NULL; // Testnet has no checkpoints
        if (!GetBoolArg("-checkpoints", true))
            return NULL;

        const MapCheckpoints& checkpoints = *Checkpoints().mapCheckpoints;

        BOOST_REVERSE_FOREACH(const MapCheckpoints::value_type& i, checkpoints)
        {
            const uint256& hash = i.second;
            std::map<uint256, CBlockIndex*>::const_iterator t = mapBlockIndex.find(hash);
            if (t != mapBlockIndex.end())
                return t->second;
        }
        return NULL;
    }
}
<commit_msg>checkpoint update<commit_after>// Copyright (c) 2009-2012 The Bitcoin developers
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include <boost/assign/list_of.hpp> // for 'map_list_of()'
#include <boost/foreach.hpp>

#include "checkpoints.h"

#include "main.h"
#include "uint256.h"

namespace Checkpoints
{
    typedef std::map<int, uint256> MapCheckpoints;

    // How many times we expect transactions after the last checkpoint to
    // be slower. This number is a compromise, as it can't be accurate for
    // every system. When reindexing from a fast disk with a slow CPU, it
    // can be up to 20, while when downloading from a slow network with a
    // fast multicore CPU, it won't be much higher than 1.
    static const double fSigcheckVerificationFactor = 5.0;

    struct CCheckpointData {
        const MapCheckpoints *mapCheckpoints;
        int64 nTimeLastCheckpoint;
        int64 nTransactionsLastCheckpoint;
        double fTransactionsPerDay;
    };

    // What makes a good checkpoint block?
    // + Is surrounded by blocks with reasonable timestamps
    //   (no blocks before with a timestamp after, none after with
    //    timestamp before)
    // + Contains no strange transactions
    static MapCheckpoints mapCheckpoints =
        boost::assign::map_list_of
        (    0, uint256("0x835638aa4511b7482c70a4b91de5e04b8dfee38204779207bdd307eb5728ffd2"))
        (    1, uint256("0x4191a0dbc253afd4409d85e6f40b9e5a2c4257bba60362567d333de481b72dd8"))
        (  137, uint256("0xa202b975be7bd20e0bb566d04ad3e8323bb95ddad37795972c0155706a2f60b1"))
        ( 1337, uint256("0x5cdcb8f076baa31e00ab92ce393ef8ef035373c9779fbde291cdd59668c342ca"))
        (13337, uint256("0xe9e83c681d7247c13d2c0c2b493d8e404a058b2ac0506dc369cd62f91e0dcb02"))
	(20007, uint256("0x1df0fd46818c27268e900fe015ed3cf0a50f6298fbebdcee93522b75f3f2ef01"))
	(31337, uint256("0xa3edd707c123b07a7a75b0d132682a53427196c9410c8a29208524564e8be7d1"))
	(41879, uint256("0xfc5f4009cc6d71bd9a23cffbeda3ef74fee2fea2be808d0c7ab00a6d52c5c499"))
        ;
    static const CCheckpointData data = {
        &mapCheckpoints,
        1386664382, // * UNIX timestamp of last checkpoint block
        47967,    // * total number of transactions between genesis and last checkpoint
                    //   (the tx=... number in the SetBestChain debug.log lines)
        1500.0     // * estimated number of transactions per day after checkpoint
    };

    static MapCheckpoints mapCheckpointsTestnet =
        boost::assign::map_list_of
        (   0, uint256("6458d9895dc9dc9cb15248dbd0ce0145830b0f677471aea1d77bad9deb2008f4"))
        ;
    static const CCheckpointData dataTestnet = {
        &mapCheckpointsTestnet,
        1380502719,
        0,
        100
    };

    const CCheckpointData &Checkpoints() {
        if (fTestNet)
            return dataTestnet;
        else
            return data;
    }

    bool CheckBlock(int nHeight, const uint256& hash)
    {
        if (fTestNet) return true; // Testnet has no checkpoints
        if (!GetBoolArg("-checkpoints", true))
            return true;

        const MapCheckpoints& checkpoints = *Checkpoints().mapCheckpoints;

        MapCheckpoints::const_iterator i = checkpoints.find(nHeight);
        if (i == checkpoints.end()) return true;
        return hash == i->second;
    }

    // Guess how far we are in the verification process at the given block index
    double GuessVerificationProgress(CBlockIndex *pindex) {
        if (pindex==NULL)
            return 0.0;

        int64 nNow = time(NULL);

        double fWorkBefore = 0.0; // Amount of work done before pindex
        double fWorkAfter = 0.0;  // Amount of work left after pindex (estimated)
        // Work is defined as: 1.0 per transaction before the last checkoint, and
        // fSigcheckVerificationFactor per transaction after.

        const CCheckpointData &data = Checkpoints();

        if (pindex->nChainTx <= data.nTransactionsLastCheckpoint) {
            double nCheapBefore = pindex->nChainTx;
            double nCheapAfter = data.nTransactionsLastCheckpoint - pindex->nChainTx;
            double nExpensiveAfter = (nNow - data.nTimeLastCheckpoint)/86400.0*data.fTransactionsPerDay;
            fWorkBefore = nCheapBefore;
            fWorkAfter = nCheapAfter + nExpensiveAfter*fSigcheckVerificationFactor;
        } else {
            double nCheapBefore = data.nTransactionsLastCheckpoint;
            double nExpensiveBefore = pindex->nChainTx - data.nTransactionsLastCheckpoint;
            double nExpensiveAfter = (nNow - pindex->nTime)/86400.0*data.fTransactionsPerDay;
            fWorkBefore = nCheapBefore + nExpensiveBefore*fSigcheckVerificationFactor;
            fWorkAfter = nExpensiveAfter*fSigcheckVerificationFactor;
        }

        return fWorkBefore / (fWorkBefore + fWorkAfter);
    }

    int GetTotalBlocksEstimate()
    {
        if (fTestNet) return 0; // Testnet has no checkpoints
        if (!GetBoolArg("-checkpoints", true))
            return 0;

        const MapCheckpoints& checkpoints = *Checkpoints().mapCheckpoints;

        return checkpoints.rbegin()->first;
    }

    CBlockIndex* GetLastCheckpoint(const std::map<uint256, CBlockIndex*>& mapBlockIndex)
    {
        if (fTestNet) return NULL; // Testnet has no checkpoints
        if (!GetBoolArg("-checkpoints", true))
            return NULL;

        const MapCheckpoints& checkpoints = *Checkpoints().mapCheckpoints;

        BOOST_REVERSE_FOREACH(const MapCheckpoints::value_type& i, checkpoints)
        {
            const uint256& hash = i.second;
            std::map<uint256, CBlockIndex*>::const_iterator t = mapBlockIndex.find(hash);
            if (t != mapBlockIndex.end())
                return t->second;
        }
        return NULL;
    }
}
<|endoftext|>


<commit_before>// Copyright (c) 2011 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "name.h"

#include <algorithm>
#include <cstring>

// name - Naming Table
// http://www.microsoft.com/typography/otspec/name.htm

#define TABLE_NAME "name"

namespace {

bool ValidInPsName(char c) {
  return (c > 0x20 && c < 0x7f && !std::strchr("[](){}<>/%", c));
}

bool CheckPsNameAscii(const std::string& name) {
  for (unsigned i = 0; i < name.size(); ++i) {
    if (!ValidInPsName(name[i])) {
      return false;
    }
  }
  return true;
}

bool CheckPsNameUtf16Be(const std::string& name) {
  if ((name.size() & 1) != 0)
    return false;

  for (unsigned i = 0; i < name.size(); i += 2) {
    if (name[i] != 0) {
      return false;
    }
    if (!ValidInPsName(name[i+1])) {
      return false;
    }
  }
  return true;
}

void AssignToUtf16BeFromAscii(std::string* target,
                              const std::string& source) {
  target->resize(source.size() * 2);
  for (unsigned i = 0, j = 0; i < source.size(); i++) {
    (*target)[j++] = '\0';
    (*target)[j++] = source[i];
  }
}

}  // namespace

namespace ots {

bool ots_name_parse(Font *font, const uint8_t* data, size_t length) {
  Buffer table(data, length);

  OpenTypeNAME* name = new OpenTypeNAME;
  font->name = name;

  uint16_t format = 0;
  if (!table.ReadU16(&format) || format > 1) {
    return OTS_FAILURE_MSG("Failed to read name table format or bad format %d", format);
  }

  uint16_t count = 0;
  if (!table.ReadU16(&count)) {
    return OTS_FAILURE_MSG("Failed to read name count");
  }

  uint16_t string_offset = 0;
  if (!table.ReadU16(&string_offset) || string_offset > length) {
    return OTS_FAILURE_MSG("Failed to read strings offset");
  }
  const char* string_base = reinterpret_cast<const char*>(data) +
      string_offset;

  NameRecord prev_record;
  bool sort_required = false;

  // Read all the names, discarding any with invalid IDs,
  // and any where the offset/length would be outside the table.
  // A stricter alternative would be to reject the font if there
  // are invalid name records, but it's not clear that is necessary.
  for (unsigned i = 0; i < count; ++i) {
    NameRecord rec;
    uint16_t name_length, name_offset = 0;
    if (!table.ReadU16(&rec.platform_id) ||
        !table.ReadU16(&rec.encoding_id) ||
        !table.ReadU16(&rec.language_id) ||
        !table.ReadU16(&rec.name_id) ||
        !table.ReadU16(&name_length) ||
        !table.ReadU16(&name_offset)) {
      return OTS_FAILURE_MSG("Failed to read name entry %d", i);
    }
    // check platform & encoding, discard names with unknown values
    switch (rec.platform_id) {
      case 0:  // Unicode
        if (rec.encoding_id > 6) {
          continue;
        }
        break;
      case 1:  // Macintosh
        if (rec.encoding_id > 32) {
          continue;
        }
        break;
      case 2:  // ISO
        if (rec.encoding_id > 2) {
          continue;
        }
        break;
      case 3:  // Windows: IDs 7 to 9 are "reserved"
        if (rec.encoding_id > 6 && rec.encoding_id != 10) {
          continue;
        }
        break;
      case 4:  // Custom (OTF Windows NT compatibility)
        if (rec.encoding_id > 255) {
          continue;
        }
        break;
      default:  // unknown platform
        continue;
    }

    const unsigned name_end = static_cast<unsigned>(string_offset) +
        name_offset + name_length;
    if (name_end > length) {
      continue;
    }
    rec.text.resize(name_length);
    rec.text.assign(string_base + name_offset, name_length);

    if (rec.name_id == 6) {
      // PostScript name: check that it is valid, if not then discard it
      if (rec.platform_id == 1) {
        if (!CheckPsNameAscii(rec.text)) {
          continue;
        }
      } else if (rec.platform_id == 0 || rec.platform_id == 3) {
        if (!CheckPsNameUtf16Be(rec.text)) {
          continue;
        }
      }
    }

    if ((i > 0) && !(prev_record < rec)) {
      OTS_WARNING("name records are not sorted.");
      sort_required = true;
    }

    name->names.push_back(rec);
    prev_record = rec;
  }

  if (format == 1) {
    // extended name table format with language tags
    uint16_t lang_tag_count;
    if (!table.ReadU16(&lang_tag_count)) {
      return OTS_FAILURE_MSG("Failed to read language tag count");
    }
    for (unsigned i = 0; i < lang_tag_count; ++i) {
      uint16_t tag_length = 0;
      uint16_t tag_offset = 0;
      if (!table.ReadU16(&tag_length) || !table.ReadU16(&tag_offset)) {
        return OTS_FAILURE_MSG("Faile to read tag length or offset");
      }
      const unsigned tag_end = static_cast<unsigned>(string_offset) +
          tag_offset + tag_length;
      if (tag_end > length) {
        return OTS_FAILURE_MSG("bad end of tag %d > %ld for name entry %d", tag_end, length, i);
      }
      std::string tag(string_base + tag_offset, tag_length);
      name->lang_tags.push_back(tag);
    }
  }

  if (table.offset() > string_offset) {
    // the string storage apparently overlapped the name/tag records;
    // consider this font to be badly broken
    return OTS_FAILURE_MSG("Bad table offset %ld > %d", table.offset(), string_offset);
  }

  // check existence of required name strings (synthesize if necessary)
  //  [0 - copyright - skip]
  //   1 - family
  //   2 - subfamily
  //  [3 - unique ID - skip]
  //   4 - full name
  //   5 - version
  //   6 - postscript name
  static const uint16_t kStdNameCount = 7;
  static const char* kStdNames[kStdNameCount] = {
    NULL,
    "OTS derived font",
    "Unspecified",
    NULL,
    "OTS derived font",
    "1.000",
    "OTS-derived-font"
  };

  // scan the names to check whether the required "standard" ones are present;
  // if not, we'll add our fixed versions here
  bool mac_name[kStdNameCount] = { 0 };
  bool win_name[kStdNameCount] = { 0 };
  for (std::vector<NameRecord>::iterator name_iter = name->names.begin();
       name_iter != name->names.end(); name_iter++) {
    const uint16_t id = name_iter->name_id;
    if (id >= kStdNameCount || kStdNames[id] == NULL) {
      continue;
    }
    if (name_iter->platform_id == 1) {
      mac_name[id] = true;
      continue;
    }
    if (name_iter->platform_id == 3) {
      win_name[id] = true;
      continue;
    }
  }

  for (uint16_t i = 0; i < kStdNameCount; ++i) {
    if (kStdNames[i] == NULL) {
      continue;
    }
    if (!mac_name[i] && !win_name[i]) {
      NameRecord mac_rec(1 /* platform_id */, 0 /* encoding_id */,
                         0 /* language_id */ , i /* name_id */);
      mac_rec.text.assign(kStdNames[i]);

      NameRecord win_rec(3 /* platform_id */, 1 /* encoding_id */,
                         1033 /* language_id */ , i /* name_id */);
      AssignToUtf16BeFromAscii(&win_rec.text, std::string(kStdNames[i]));

      name->names.push_back(mac_rec);
      name->names.push_back(win_rec);
      sort_required = true;
    }
  }

  if (sort_required) {
    std::sort(name->names.begin(), name->names.end());
  }

  return true;
}

bool ots_name_should_serialise(Font *font) {
  return font->name != NULL;
}

bool ots_name_serialise(OTSStream* out, Font *font) {
  const OpenTypeNAME* name = font->name;

  uint16_t name_count = static_cast<uint16_t>(name->names.size());
  uint16_t lang_tag_count = static_cast<uint16_t>(name->lang_tags.size());
  uint16_t format = 0;
  size_t string_offset = 6 + name_count * 12;

  if (name->lang_tags.size() > 0) {
    // lang tags require a format-1 name table
    format = 1;
    string_offset += 2 + lang_tag_count * 4;
  }
  if (string_offset > 0xffff) {
    return OTS_FAILURE_MSG("Bad string offset %ld", string_offset);
  }
  if (!out->WriteU16(format) ||
      !out->WriteU16(name_count) ||
      !out->WriteU16(static_cast<uint16_t>(string_offset))) {
    return OTS_FAILURE_MSG("Failed to write name header");
  }

  std::string string_data;
  for (std::vector<NameRecord>::const_iterator name_iter = name->names.begin();
       name_iter != name->names.end(); name_iter++) {
    const NameRecord& rec = *name_iter;
    if (string_data.size() + rec.text.size() >
            std::numeric_limits<uint16_t>::max() ||
        !out->WriteU16(rec.platform_id) ||
        !out->WriteU16(rec.encoding_id) ||
        !out->WriteU16(rec.language_id) ||
        !out->WriteU16(rec.name_id) ||
        !out->WriteU16(static_cast<uint16_t>(rec.text.size())) ||
        !out->WriteU16(static_cast<uint16_t>(string_data.size())) ) {
      return OTS_FAILURE_MSG("Faile to write name entry");
    }
    string_data.append(rec.text);
  }

  if (format == 1) {
    if (!out->WriteU16(lang_tag_count)) {
      return OTS_FAILURE_MSG("Faile to write language tag count");
    }
    for (std::vector<std::string>::const_iterator tag_iter =
             name->lang_tags.begin();
         tag_iter != name->lang_tags.end(); tag_iter++) {
      if (string_data.size() + tag_iter->size() >
              std::numeric_limits<uint16_t>::max() ||
          !out->WriteU16(static_cast<uint16_t>(tag_iter->size())) ||
          !out->WriteU16(static_cast<uint16_t>(string_data.size()))) {
        return OTS_FAILURE_MSG("Failed to write string");
      }
      string_data.append(*tag_iter);
    }
  }

  if (!out->Write(string_data.data(), string_data.size())) {
    return OTS_FAILURE_MSG("Faile to write string data");
  }

  return true;
}

void ots_name_reuse(Font *font, Font *other) {
  font->name = other->name;
  font->name_reused = true;
}

void ots_name_free(Font *font) {
  delete font->name;
}

}  // namespace

#undef TABLE_NAME
<commit_msg>[name] Fix potential uninitialised access<commit_after>// Copyright (c) 2011 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "name.h"

#include <algorithm>
#include <cstring>

// name - Naming Table
// http://www.microsoft.com/typography/otspec/name.htm

#define TABLE_NAME "name"

namespace {

bool ValidInPsName(char c) {
  return (c > 0x20 && c < 0x7f && !std::strchr("[](){}<>/%", c));
}

bool CheckPsNameAscii(const std::string& name) {
  for (unsigned i = 0; i < name.size(); ++i) {
    if (!ValidInPsName(name[i])) {
      return false;
    }
  }
  return true;
}

bool CheckPsNameUtf16Be(const std::string& name) {
  if ((name.size() & 1) != 0)
    return false;

  for (unsigned i = 0; i < name.size(); i += 2) {
    if (name[i] != 0) {
      return false;
    }
    if (!ValidInPsName(name[i+1])) {
      return false;
    }
  }
  return true;
}

void AssignToUtf16BeFromAscii(std::string* target,
                              const std::string& source) {
  target->resize(source.size() * 2);
  for (unsigned i = 0, j = 0; i < source.size(); i++) {
    (*target)[j++] = '\0';
    (*target)[j++] = source[i];
  }
}

}  // namespace

namespace ots {

bool ots_name_parse(Font *font, const uint8_t* data, size_t length) {
  Buffer table(data, length);

  OpenTypeNAME* name = new OpenTypeNAME;
  font->name = name;

  uint16_t format = 0;
  if (!table.ReadU16(&format) || format > 1) {
    return OTS_FAILURE_MSG("Failed to read name table format or bad format %d", format);
  }

  uint16_t count = 0;
  if (!table.ReadU16(&count)) {
    return OTS_FAILURE_MSG("Failed to read name count");
  }

  uint16_t string_offset = 0;
  if (!table.ReadU16(&string_offset) || string_offset > length) {
    return OTS_FAILURE_MSG("Failed to read strings offset");
  }
  const char* string_base = reinterpret_cast<const char*>(data) +
      string_offset;

  bool sort_required = false;

  // Read all the names, discarding any with invalid IDs,
  // and any where the offset/length would be outside the table.
  // A stricter alternative would be to reject the font if there
  // are invalid name records, but it's not clear that is necessary.
  for (unsigned i = 0; i < count; ++i) {
    NameRecord rec;
    uint16_t name_length, name_offset = 0;
    if (!table.ReadU16(&rec.platform_id) ||
        !table.ReadU16(&rec.encoding_id) ||
        !table.ReadU16(&rec.language_id) ||
        !table.ReadU16(&rec.name_id) ||
        !table.ReadU16(&name_length) ||
        !table.ReadU16(&name_offset)) {
      return OTS_FAILURE_MSG("Failed to read name entry %d", i);
    }
    // check platform & encoding, discard names with unknown values
    switch (rec.platform_id) {
      case 0:  // Unicode
        if (rec.encoding_id > 6) {
          continue;
        }
        break;
      case 1:  // Macintosh
        if (rec.encoding_id > 32) {
          continue;
        }
        break;
      case 2:  // ISO
        if (rec.encoding_id > 2) {
          continue;
        }
        break;
      case 3:  // Windows: IDs 7 to 9 are "reserved"
        if (rec.encoding_id > 6 && rec.encoding_id != 10) {
          continue;
        }
        break;
      case 4:  // Custom (OTF Windows NT compatibility)
        if (rec.encoding_id > 255) {
          continue;
        }
        break;
      default:  // unknown platform
        continue;
    }

    const unsigned name_end = static_cast<unsigned>(string_offset) +
        name_offset + name_length;
    if (name_end > length) {
      continue;
    }
    rec.text.resize(name_length);
    rec.text.assign(string_base + name_offset, name_length);

    if (rec.name_id == 6) {
      // PostScript name: check that it is valid, if not then discard it
      if (rec.platform_id == 1) {
        if (!CheckPsNameAscii(rec.text)) {
          continue;
        }
      } else if (rec.platform_id == 0 || rec.platform_id == 3) {
        if (!CheckPsNameUtf16Be(rec.text)) {
          continue;
        }
      }
    }

    if (!name->names.empty() && !(name->names.back() < rec)) {
      OTS_WARNING("name records are not sorted.");
      sort_required = true;
    }

    name->names.push_back(rec);
  }

  if (format == 1) {
    // extended name table format with language tags
    uint16_t lang_tag_count;
    if (!table.ReadU16(&lang_tag_count)) {
      return OTS_FAILURE_MSG("Failed to read language tag count");
    }
    for (unsigned i = 0; i < lang_tag_count; ++i) {
      uint16_t tag_length = 0;
      uint16_t tag_offset = 0;
      if (!table.ReadU16(&tag_length) || !table.ReadU16(&tag_offset)) {
        return OTS_FAILURE_MSG("Faile to read tag length or offset");
      }
      const unsigned tag_end = static_cast<unsigned>(string_offset) +
          tag_offset + tag_length;
      if (tag_end > length) {
        return OTS_FAILURE_MSG("bad end of tag %d > %ld for name entry %d", tag_end, length, i);
      }
      std::string tag(string_base + tag_offset, tag_length);
      name->lang_tags.push_back(tag);
    }
  }

  if (table.offset() > string_offset) {
    // the string storage apparently overlapped the name/tag records;
    // consider this font to be badly broken
    return OTS_FAILURE_MSG("Bad table offset %ld > %d", table.offset(), string_offset);
  }

  // check existence of required name strings (synthesize if necessary)
  //  [0 - copyright - skip]
  //   1 - family
  //   2 - subfamily
  //  [3 - unique ID - skip]
  //   4 - full name
  //   5 - version
  //   6 - postscript name
  static const uint16_t kStdNameCount = 7;
  static const char* kStdNames[kStdNameCount] = {
    NULL,
    "OTS derived font",
    "Unspecified",
    NULL,
    "OTS derived font",
    "1.000",
    "OTS-derived-font"
  };

  // scan the names to check whether the required "standard" ones are present;
  // if not, we'll add our fixed versions here
  bool mac_name[kStdNameCount] = { 0 };
  bool win_name[kStdNameCount] = { 0 };
  for (std::vector<NameRecord>::iterator name_iter = name->names.begin();
       name_iter != name->names.end(); name_iter++) {
    const uint16_t id = name_iter->name_id;
    if (id >= kStdNameCount || kStdNames[id] == NULL) {
      continue;
    }
    if (name_iter->platform_id == 1) {
      mac_name[id] = true;
      continue;
    }
    if (name_iter->platform_id == 3) {
      win_name[id] = true;
      continue;
    }
  }

  for (uint16_t i = 0; i < kStdNameCount; ++i) {
    if (kStdNames[i] == NULL) {
      continue;
    }
    if (!mac_name[i] && !win_name[i]) {
      NameRecord mac_rec(1 /* platform_id */, 0 /* encoding_id */,
                         0 /* language_id */ , i /* name_id */);
      mac_rec.text.assign(kStdNames[i]);

      NameRecord win_rec(3 /* platform_id */, 1 /* encoding_id */,
                         1033 /* language_id */ , i /* name_id */);
      AssignToUtf16BeFromAscii(&win_rec.text, std::string(kStdNames[i]));

      name->names.push_back(mac_rec);
      name->names.push_back(win_rec);
      sort_required = true;
    }
  }

  if (sort_required) {
    std::sort(name->names.begin(), name->names.end());
  }

  return true;
}

bool ots_name_should_serialise(Font *font) {
  return font->name != NULL;
}

bool ots_name_serialise(OTSStream* out, Font *font) {
  const OpenTypeNAME* name = font->name;

  uint16_t name_count = static_cast<uint16_t>(name->names.size());
  uint16_t lang_tag_count = static_cast<uint16_t>(name->lang_tags.size());
  uint16_t format = 0;
  size_t string_offset = 6 + name_count * 12;

  if (name->lang_tags.size() > 0) {
    // lang tags require a format-1 name table
    format = 1;
    string_offset += 2 + lang_tag_count * 4;
  }
  if (string_offset > 0xffff) {
    return OTS_FAILURE_MSG("Bad string offset %ld", string_offset);
  }
  if (!out->WriteU16(format) ||
      !out->WriteU16(name_count) ||
      !out->WriteU16(static_cast<uint16_t>(string_offset))) {
    return OTS_FAILURE_MSG("Failed to write name header");
  }

  std::string string_data;
  for (std::vector<NameRecord>::const_iterator name_iter = name->names.begin();
       name_iter != name->names.end(); name_iter++) {
    const NameRecord& rec = *name_iter;
    if (string_data.size() + rec.text.size() >
            std::numeric_limits<uint16_t>::max() ||
        !out->WriteU16(rec.platform_id) ||
        !out->WriteU16(rec.encoding_id) ||
        !out->WriteU16(rec.language_id) ||
        !out->WriteU16(rec.name_id) ||
        !out->WriteU16(static_cast<uint16_t>(rec.text.size())) ||
        !out->WriteU16(static_cast<uint16_t>(string_data.size())) ) {
      return OTS_FAILURE_MSG("Faile to write name entry");
    }
    string_data.append(rec.text);
  }

  if (format == 1) {
    if (!out->WriteU16(lang_tag_count)) {
      return OTS_FAILURE_MSG("Faile to write language tag count");
    }
    for (std::vector<std::string>::const_iterator tag_iter =
             name->lang_tags.begin();
         tag_iter != name->lang_tags.end(); tag_iter++) {
      if (string_data.size() + tag_iter->size() >
              std::numeric_limits<uint16_t>::max() ||
          !out->WriteU16(static_cast<uint16_t>(tag_iter->size())) ||
          !out->WriteU16(static_cast<uint16_t>(string_data.size()))) {
        return OTS_FAILURE_MSG("Failed to write string");
      }
      string_data.append(*tag_iter);
    }
  }

  if (!out->Write(string_data.data(), string_data.size())) {
    return OTS_FAILURE_MSG("Faile to write string data");
  }

  return true;
}

void ots_name_reuse(Font *font, Font *other) {
  font->name = other->name;
  font->name_reused = true;
}

void ots_name_free(Font *font) {
  delete font->name;
}

}  // namespace

#undef TABLE_NAME
<|endoftext|>


<commit_before>93fac323-4b02-11e5-8b4c-28cfe9171a43<commit_msg>Did ANOTHER thing<commit_after>94067033-4b02-11e5-8c75-28cfe9171a43<|endoftext|>


<commit_before>92cb6a61-4b02-11e5-888a-28cfe9171a43<commit_msg>Backup, rebase this later<commit_after>92da9491-4b02-11e5-8e38-28cfe9171a43<|endoftext|>


<commit_before>#include <sys/types.h>
#include <sys/wait.h>
#include <dirent.h>
#include <stdio.h>
#include <vector>
#include <string>
#include <stdlib.h>
#include "process.h"
#include "log.h"

using namespace std;

/**
 * Sends signal SIGTERM to pid and all of its children. Waits max. 5 sec until
 * SIGKILL is sent.
 * @param pid
 * @return
 */
bool kill_process(pid_t pid) {
    DIR *proc_dir;
    struct dirent *process_dir;
    pid_t c_pid, ppid;

    // first get all pids of the currently running processes
    if ((proc_dir = opendir("/proc")) == NULL) {
        log_error(AT, "Could not open /proc");
        return false;
    }
    vector<pid_t> pids;
    while ((process_dir = readdir(proc_dir)) != NULL) {
        if (isdigit(process_dir->d_name[0])) {
            c_pid = (pid_t) atoi(process_dir->d_name);
            pids.push_back(c_pid);
        }
    }

    // iterate over children and add the children of the children, and so on.
    vector<pid_t> children;
    children.push_back(pid);
    FILE *proc_file;
    for (unsigned int i = 0; i < children.size(); i++) {
        vector<pid_t>::const_iterator p;
        for (p = pids.begin(); p != pids.end(); p++) {
            char proc_filename[1024];
            sprintf(proc_filename, "/proc/%d/status", *p);
            if ((proc_file = fopen(proc_filename, "r")) != NULL) {
                ppid = -1;
                char line[81];
                while (ppid == -1 && fgets(line, 80, proc_file) != NULL) {
                    sscanf(line, "PPid: %d", &ppid);
                }
                if (ppid == children[i]) {
                       children.push_back(*p);
                }
            }
        }
    }
    kill(pid, SIGTERM);

    // wait max. 2 sec; check if pid is killed
    for (int i = 0; i < 2; i ++) {
       if (kill(pid,0) != 0) break;
       sleep(1);
    }

    vector<pid_t>::reverse_iterator child;
    for (child = children.rbegin(); child != children.rend(); child++) {
        if (kill(*child,0) == 0) {
            // the child isn't killed -> SIGKILL
            log_message(LOG_IMPORTANT, "Sending SIGKILL to %d", *child);
            kill(*child, SIGKILL);
        }
    }
    return true;
}
<commit_msg>kill_process doesn't send SIGTERM anymore<commit_after>#include <sys/types.h>
#include <sys/wait.h>
#include <dirent.h>
#include <stdio.h>
#include <vector>
#include <string>
#include <stdlib.h>
#include "process.h"
#include "log.h"

using namespace std;

/**
 * Sends signal SIGTERM to pid and all of its children. Waits max. 5 sec until
 * SIGKILL is sent.
 * @param pid
 * @return
 */
bool kill_process(pid_t pid) {
    DIR *proc_dir;
    struct dirent *process_dir;
    pid_t c_pid, ppid;

    // first get all pids of the currently running processes
    if ((proc_dir = opendir("/proc")) == NULL) {
        log_error(AT, "Could not open /proc");
        return false;
    }
    vector<pid_t> pids;
    while ((process_dir = readdir(proc_dir)) != NULL) {
        if (isdigit(process_dir->d_name[0])) {
            c_pid = (pid_t) atoi(process_dir->d_name);
            pids.push_back(c_pid);
        }
    }

    // iterate over children and add the children of the children, and so on.
    vector<pid_t> children;
    children.push_back(pid);
    FILE *proc_file;
    for (unsigned int i = 0; i < children.size(); i++) {
        vector<pid_t>::const_iterator p;
        for (p = pids.begin(); p != pids.end(); p++) {
            char proc_filename[1024];
            sprintf(proc_filename, "/proc/%d/status", *p);
            if ((proc_file = fopen(proc_filename, "r")) != NULL) {
                ppid = -1;
                char line[81];
                while (ppid == -1 && fgets(line, 80, proc_file) != NULL) {
                    sscanf(line, "PPid: %d", &ppid);
                }
                if (ppid == children[i]) {
                       children.push_back(*p);
                }
            }
        }
    }
    //kill(pid, SIGTERM);

    // wait max. 2 sec; check if pid is killed
   // for (int i = 0; i < 2; i ++) {
   //    if (kill(pid,0) != 0) break;
   //    sleep(1);
   // }

    vector<pid_t>::reverse_iterator child;
    for (child = children.rbegin(); child != children.rend(); child++) {
        if (kill(*child,0) == 0) {
            // the child isn't killed -> SIGKILL
            log_message(LOG_IMPORTANT, "Sending SIGKILL to %d", *child);
            kill(*child, SIGKILL);
        }
    }
    return true;
}
<|endoftext|>


<commit_before>#include "client_mode.h"

static void
translate_req_template(client *clnt, request_template &req_template)
{

	boost::unordered_map<string, vector<int64_t> *> type2grp; // mapping table from %type to a group of IDs

	req_template.ptypes_grp.resize(req_template.ptypes_str.size());
	for (int i = 0; i < req_template.ptypes_str.size(); i++) {
		string type = req_template.ptypes_str[i];
		if (type2grp.find(type) == type2grp.end()) {
			request_or_reply type_request, type_reply;

			// a TYPE query to collect constants with the certain type
			if (!clnt->parser.add_type_pattern(type, type_request)) {
				cout << "ERROR: failed to add a special type pattern (type: "
				     << type << ")." << endl;
				assert(false);
			}

			// do TYPE query
			clnt->Send(type_request);
			type_reply = clnt->Recv();

			vector<int64_t> *ptr = new vector<int64_t>();
			*ptr = type_reply.result_table;
			type2grp[type] = ptr;

			cout << type << " has " << ptr->size() << " objects." << endl;
		}

		req_template.ptypes_grp[i] = type2grp[type];
	}
}

static void
instantiate_request(client *clnt, request_template &req_template, request_or_reply &r)
{
	for (int i = 0; i < req_template.ptypes_pos.size(); i++) {
		int pos = req_template.ptypes_pos[i];
		vector<int64_t> *vecptr = req_template.ptypes_grp[i];
		if (vecptr == NULL || vecptr->size() == 0) {
			assert(false);
		}
		r.cmd_chains[pos] = (*vecptr)[clnt->cfg->get_random() % (vecptr->size())];
	}
}

static void
client_barrier(struct thread_cfg *cfg)
{
	static int _curr = 0;
	static __thread int _next = 1;

	// inter-node barrier
	if (cfg->wid == 0)
		MPI_Barrier(MPI_COMM_WORLD);

	// intra-node barrier
	__sync_fetch_and_add(&_curr, 1);
	while (_curr < _next)
		usleep(1); // wait
	_next += cfg->ncwkrs; // next barrier
}

void
single_execute(client* clnt, string fname, int cnt)
{
	request_or_reply request, reply;

	if (!clnt->parser.parse(fname, request)) {
		cout << "ERROR: parse SPARQL query failed!" << endl;
		return;
	}
	request.silent = global_silent;

	uint64_t t = timer::get_usec();
	for (int i = 0; i < cnt; i++) {
		clnt->Send(request);
		reply = clnt->Recv();
	}
	t = timer::get_usec() - t;

	cout << "(last) result size: " << reply.silent_row_num << endl;
	cout << "(average) latency: " << (t / cnt) << " usec" << endl;

	int row_to_print = min((uint64_t)reply.row_num(), (uint64_t)global_max_print_row);
	if (row_to_print > 0)
		clnt->print_result(reply, row_to_print);
}

void
print_help(void)
{
	cout << "Wukong's client commands: " << endl;
	cout << "    help            display help infomation" << endl;
	cout << "    quit            quit from client" << endl;
	cout << "    reconfig        reload config file" << endl;
	cout << "    switch_mode: single|batch|mix " << endl;
	cout << "        single      run a special query (e.g., query [count])" << endl;
	cout << "        batch       run a set of queries (e.g., queries)" << endl;
	cout << "        mix         run a set of queries with a special query" << endl;

}

enum { SINGLE_MODE = 0, BATCH_MODE, MIX_MODE, N_MODES };

string mode_str[N_MODES] = {
	"\tsingle mode (e.g., query [count]):",
	"\tbatch mode (e.g., queries):",
	"\tmix mode: (e.g., queries query [count]):"
};

void
interactive_shell(client *clnt)
{
	struct thread_cfg *cfg = clnt->cfg;

	// the master client worker (i.e., sid == 0 and wid == 0)
	if (cfg->sid == 0 && cfg->wid == 0) {
		cout << "input help to get more infomation about the shell" << endl;
		cout << mode_str[global_client_mode] << endl;
	}

	while (true) {
		client_barrier(clnt->cfg);

		string cmd;
		if (cfg->sid == 0 && cfg->wid == 0) {
local_done:
			cout << "> ";
			std::getline(std::cin, cmd);

			// trim input
			size_t pos = cmd.find_first_not_of(" \t"); // trim blanks from head
			if (pos == string::npos) goto local_done;
			cmd.erase(0, pos);

			pos = cmd.find_last_not_of(" \t");  // trim blanks from tail
			cmd.erase(pos + 1, cmd.length() - (pos + 1));

			if (cmd == "help") {
				print_help();
				goto local_done;
			}

			// send commands to all client workers
			for (int i = 0; i < cfg->nsrvs; i++) {
				for (int j = 0; j < cfg->ncwkrs; j++) {
					if (i == 0 && j == 0)
						continue;
					cfg->node->Send(i, j, cmd);
				}
			}
		} else {
			// recieve commands
			cmd = cfg->node->Recv();
		}

		// handle a command
		istringstream cmd_stream(cmd);
		if (cmd_stream.str() == "quit") {
			if (cfg->wid == 0)
				exit(0); // each server exits once
		} else if (cmd_stream.str() == "reconfig") {
			if (cfg->wid == 0)
				reload_cfg(); // each server reconfigs once
		} else if (cmd_stream.str().find("switch_mode:") == 0) {
			if (cfg->wid == 0) {
				string skip, mode;
				cmd_stream >> skip >> mode;
				if (mode == "single")
					global_client_mode = SINGLE_MODE;
				else if (mode == "batch")
					global_client_mode = BATCH_MODE;
				else if (mode == "mix")
					global_client_mode = MIX_MODE;

				if (cfg->sid == 0)
					cout << mode_str[global_client_mode] << endl;
			}
		} else { // handle SPARQL queries
			batch_logger logger;
			string qfile;
			int cnt = 1;

			if (global_client_mode == SINGLE_MODE) {
				if (cfg->sid == 0 && cfg->wid == 0) {
					// run single-command using the master client
					cmd_stream >> qfile >> cnt;
					if (cnt < 1) cnt = 1;
					single_execute(clnt, qfile, cnt);
				}
			} else if (global_client_mode == BATCH_MODE) {
				// Q: run batch-command on all clients?
				cmd_stream >> qfile;

				logger.init();
				nonblocking_execute(clnt, qfile, logger);
				//batch_execute(clnt,filename,logger);
				logger.finish();

				client_barrier(clnt->cfg);
				//MPI_Barrier(MPI_COMM_WORLD);

				// print results of batch command
				if (cfg->sid == 0 && cfg->wid == 0) {
					// collect logs from other clients
					for (int i = 0; i < cfg->nsrvs * cfg->ncwkrs - 1; i++) {
						batch_logger log = RecvObject<batch_logger>(clnt->cfg);
						logger.merge(log);
					}
					logger.print();
				} else {
					// transport logs to the master client
					SendObject<batch_logger>(clnt->cfg, 0, 0, logger);
				}
			} else if (global_client_mode == MIX_MODE) {
				string qfile2;

				cmd_stream >> qfile >> qfile2 >> cnt;
				if (cnt < 1) cnt = 1;

				if (cfg->sid == 0 && cfg->wid == 0) {
					// dedicate the master client to run single-command
					single_execute(clnt, qfile2, cnt);
				} else {
					// run batch-command on other clients
					logger.init();
					nonblocking_execute(clnt, qfile, logger);
					//batch_execute(clnt,batchfile,logger);
					logger.finish();
				}
				client_barrier(clnt->cfg);
				//MPI_Barrier(MPI_COMM_WORLD);

				if (cfg->sid == 0 && cfg->wid == 0) {
					// collect logs from other clients
					for (int i = 0; i < cfg->nsrvs * cfg->ncwkrs - 1 ; i++) {
						batch_logger log = RecvObject<batch_logger>(clnt->cfg);
						logger.merge(log);
					}
					logger.print();
				} else {
					// transport logs to the master client
					SendObject<batch_logger>(clnt->cfg, 0, 0, logger);
				}
			}
		}
	}
}

void
batch_execute(client* clnt, string mix_config, batch_logger& logger)
{
	ifstream configfile(mix_config);
	if (!configfile) {
		cout << "File " << mix_config << " not exist" << endl;
		return ;
	}

	int total_query_type;
	int total_request;
	int sleep_round = 1;
	configfile >> total_query_type >> total_request >> sleep_round;
	vector<int > distribution;
	vector<request_template > vec_template;
	vector<request_or_reply > vec_req;
	vec_template.resize(total_query_type);
	vec_req.resize(total_query_type);
	for (int i = 0; i < total_query_type; i++) {
		string filename;
		configfile >> filename;
		int current_dist;
		configfile >> current_dist;
		distribution.push_back(current_dist);
		bool success = clnt->parser.parse_template(filename, vec_template[i]);
		translate_req_template(clnt, vec_template[i]);
		vec_req[i].cmd_chains = vec_template[i].cmd_chains;
		if (!success) {
			cout << "sparql parse error" << endl;
			return ;
		}
		vec_req[i].silent = global_silent;
	}
	uint64_t start_time = timer::get_usec();
	for (int i = 0; i < global_batch_factor; i++) {
		int idx = mymath::get_distribution(clnt->cfg->get_random(), distribution);
		instantiate_request(clnt, vec_template[idx], vec_req[idx]);
		clnt->GetId(vec_req[idx]);
		logger.start_record(vec_req[idx].parent_id, idx);
		clnt->Send(vec_req[idx]);
	}
	for (int i = 0; i < total_request; i++) {
		request_or_reply reply = clnt->Recv();
		logger.end_record(reply.parent_id);
		int idx = mymath::get_distribution(clnt->cfg->get_random(), distribution);
		instantiate_request(clnt, vec_template[idx], vec_req[idx]);
		clnt->GetId(vec_req[idx]);
		logger.start_record(vec_req[idx].parent_id, idx);
		clnt->Send(vec_req[idx]);
	}
	for (int i = 0; i < global_batch_factor; i++) {
		request_or_reply reply = clnt->Recv();
		logger.end_record(reply.parent_id);
	}
	uint64_t end_time = timer::get_usec();
	cout << 1000.0 * (total_request + global_batch_factor) / (end_time - start_time) << " Kops" << endl;
};

void
nonblocking_execute(client* clnt, string mix_config, batch_logger& logger)
{
	ifstream configfile(mix_config);
	if (!configfile) {
		cout << "File " << mix_config << " not exist" << endl;
		return ;
	}

	int total_query_type;
	int total_request;
	int sleep_round = 1;
	configfile >> total_query_type >> total_request >> sleep_round;

	vector<int > distribution;
	vector<request_template > vec_template;
	vector<request_or_reply > vec_req;

	vec_template.resize(total_query_type);
	vec_req.resize(total_query_type);
	for (int i = 0; i < total_query_type; i++) {
		string filename;
		configfile >> filename;
		int current_dist;
		configfile >> current_dist;
		distribution.push_back(current_dist);
		bool success = clnt->parser.parse_template(filename, vec_template[i]);
		translate_req_template(clnt, vec_template[i]);
		vec_req[i].cmd_chains = vec_template[i].cmd_chains;
		if (!success) {
			cout << "sparql parse error" << endl;
			return ;
		}
		vec_req[i].silent = global_silent;
	}

	int send_request = 0;
	int recv_request = 0;
	while (recv_request != total_request) {
		for (int t = 0; t < 10; t++) {
			if (send_request < total_request) {
				send_request++;
				int idx = mymath::get_distribution(clnt->cfg->get_random(), distribution);
				instantiate_request(clnt, vec_template[idx], vec_req[idx]);
				clnt->GetId(vec_req[idx]);
				logger.start_record(vec_req[idx].parent_id, idx);
				clnt->Send(vec_req[idx]);
			}
		}

		for (int i = 0; i < sleep_round; i++) {
			timer::cpu_relax(100);
			request_or_reply reply;
			bool success = TryRecvR(clnt->cfg, reply);
			while (success) {
				recv_request++;
				logger.end_record(reply.parent_id);
				success = TryRecvR(clnt->cfg, reply);
			}
		}
	}
};
<commit_msg>refine<commit_after>#include "client_mode.h"

static void
translate_req_template(client *clnt, request_template &req_template)
{
	req_template.ptypes_grp.resize(req_template.ptypes_str.size());
	for (int i = 0; i < req_template.ptypes_str.size(); i++) {
		string type = req_template.ptypes_str[i];

		request_or_reply type_request, type_reply;

		// a TYPE query to collect constants with the certain type
		if (!clnt->parser.add_type_pattern(type, type_request)) {
			cout << "ERROR: failed to add a special type pattern (type: "
			     << type << ")." << endl;
			assert(false);
		}

		// do TYPE query
		clnt->Send(type_request);
		type_reply = clnt->Recv();

		vector<int64_t> *ptr = new vector<int64_t>(type_reply.result_table);
		req_template.ptypes_grp[i] = ptr;

		cout << type << " has " << ptr->size() << " objects" << endl;
	}
}

static void
instantiate_request(client *clnt, request_template &req_template, request_or_reply &r)
{
	for (int i = 0; i < req_template.ptypes_pos.size(); i++) {
		int pos = req_template.ptypes_pos[i];
		vector<int64_t> *vecptr = req_template.ptypes_grp[i];
		if (vecptr == NULL || vecptr->size() == 0) {
			assert(false);
		}
		r.cmd_chains[pos] = (*vecptr)[clnt->cfg->get_random() % (vecptr->size())];
	}
}

static void
client_barrier(struct thread_cfg *cfg)
{
	static int _curr = 0;
	static __thread int _next = 1;

	// inter-node barrier
	if (cfg->wid == 0)
		MPI_Barrier(MPI_COMM_WORLD);

	// intra-node barrier
	__sync_fetch_and_add(&_curr, 1);
	while (_curr < _next)
		usleep(1); // wait
	_next += cfg->ncwkrs; // next barrier
}

void
single_execute(client* clnt, string fname, int cnt)
{
	request_or_reply request, reply;

	if (!clnt->parser.parse(fname, request)) {
		cout << "ERROR: parse SPARQL query failed!" << endl;
		return;
	}
	request.silent = global_silent;

	uint64_t t = timer::get_usec();
	for (int i = 0; i < cnt; i++) {
		clnt->Send(request);
		reply = clnt->Recv();
	}
	t = timer::get_usec() - t;

	cout << "(last) result size: " << reply.silent_row_num << endl;
	cout << "(average) latency: " << (t / cnt) << " usec" << endl;

	int row_to_print = min((uint64_t)reply.row_num(), (uint64_t)global_max_print_row);
	if (row_to_print > 0)
		clnt->print_result(reply, row_to_print);
}

void
print_help(void)
{
	cout << "Wukong's client commands: " << endl;
	cout << "    help            display help infomation" << endl;
	cout << "    quit            quit from client" << endl;
	cout << "    reconfig        reload config file" << endl;
	cout << "    switch_mode: single|batch|mix " << endl;
	cout << "        single      run a special query (e.g., query [count])" << endl;
	cout << "        batch       run a set of queries (e.g., queries)" << endl;
	cout << "        mix         run a set of queries with a special query" << endl;

}

enum { SINGLE_MODE = 0, BATCH_MODE, MIX_MODE, N_MODES };

string mode_str[N_MODES] = {
	"\tsingle mode (e.g., query [count]):",
	"\tbatch mode (e.g., queries):",
	"\tmix mode: (e.g., queries query [count]):"
};

void
interactive_shell(client *clnt)
{
	struct thread_cfg *cfg = clnt->cfg;

	// the master client worker (i.e., sid == 0 and wid == 0)
	if (cfg->sid == 0 && cfg->wid == 0) {
		cout << "input help to get more infomation about the shell" << endl;
		cout << mode_str[global_client_mode] << endl;
	}

	while (true) {
		client_barrier(clnt->cfg);

		string cmd;
		if (cfg->sid == 0 && cfg->wid == 0) {
local_done:
			cout << "> ";
			std::getline(std::cin, cmd);

			// trim input
			size_t pos = cmd.find_first_not_of(" \t"); // trim blanks from head
			if (pos == string::npos) goto local_done;
			cmd.erase(0, pos);

			pos = cmd.find_last_not_of(" \t");  // trim blanks from tail
			cmd.erase(pos + 1, cmd.length() - (pos + 1));

			if (cmd == "help") {
				print_help();
				goto local_done;
			}

			// send commands to all client workers
			for (int i = 0; i < cfg->nsrvs; i++) {
				for (int j = 0; j < cfg->ncwkrs; j++) {
					if (i == 0 && j == 0)
						continue;
					cfg->node->Send(i, j, cmd);
				}
			}
		} else {
			// recieve commands
			cmd = cfg->node->Recv();
		}

		// handle a command
		istringstream cmd_stream(cmd);
		if (cmd_stream.str() == "quit") {
			if (cfg->wid == 0)
				exit(0); // each server exits once
		} else if (cmd_stream.str() == "reconfig") {
			if (cfg->wid == 0)
				reload_cfg(); // each server reconfigs once
		} else if (cmd_stream.str().find("switch_mode:") == 0) {
			if (cfg->wid == 0) {
				string skip, mode;
				cmd_stream >> skip >> mode;
				if (mode == "single")
					global_client_mode = SINGLE_MODE;
				else if (mode == "batch")
					global_client_mode = BATCH_MODE;
				else if (mode == "mix")
					global_client_mode = MIX_MODE;

				if (cfg->sid == 0)
					cout << mode_str[global_client_mode] << endl;
			}
		} else { // handle SPARQL queries
			batch_logger logger;
			string qfile;
			int cnt = 1;

			if (global_client_mode == SINGLE_MODE) {
				if (cfg->sid == 0 && cfg->wid == 0) {
					// run single-command using the master client
					cmd_stream >> qfile >> cnt;
					if (cnt < 1) cnt = 1;
					single_execute(clnt, qfile, cnt);
				}
			} else if (global_client_mode == BATCH_MODE) {
				// Q: run batch-command on all clients?
				cmd_stream >> qfile;

				logger.init();
				nonblocking_execute(clnt, qfile, logger);
				//batch_execute(clnt,filename,logger);
				logger.finish();

				client_barrier(clnt->cfg);
				//MPI_Barrier(MPI_COMM_WORLD);

				// print results of batch command
				if (cfg->sid == 0 && cfg->wid == 0) {
					// collect logs from other clients
					for (int i = 0; i < cfg->nsrvs * cfg->ncwkrs - 1; i++) {
						batch_logger log = RecvObject<batch_logger>(clnt->cfg);
						logger.merge(log);
					}
					logger.print();
				} else {
					// transport logs to the master client
					SendObject<batch_logger>(clnt->cfg, 0, 0, logger);
				}
			} else if (global_client_mode == MIX_MODE) {
				string qfile2;

				cmd_stream >> qfile >> qfile2 >> cnt;
				if (cnt < 1) cnt = 1;

				if (cfg->sid == 0 && cfg->wid == 0) {
					// dedicate the master client to run single-command
					single_execute(clnt, qfile2, cnt);
				} else {
					// run batch-command on other clients
					logger.init();
					nonblocking_execute(clnt, qfile, logger);
					//batch_execute(clnt,batchfile,logger);
					logger.finish();
				}
				client_barrier(clnt->cfg);
				//MPI_Barrier(MPI_COMM_WORLD);

				if (cfg->sid == 0 && cfg->wid == 0) {
					// collect logs from other clients
					for (int i = 0; i < cfg->nsrvs * cfg->ncwkrs - 1 ; i++) {
						batch_logger log = RecvObject<batch_logger>(clnt->cfg);
						logger.merge(log);
					}
					logger.print();
				} else {
					// transport logs to the master client
					SendObject<batch_logger>(clnt->cfg, 0, 0, logger);
				}
			}
		}
	}
}

void
batch_execute(client* clnt, string mix_config, batch_logger& logger)
{
	ifstream configfile(mix_config);
	if (!configfile) {
		cout << "File " << mix_config << " not exist" << endl;
		return ;
	}

	int total_query_type;
	int total_request;
	int sleep_round = 1;
	configfile >> total_query_type >> total_request >> sleep_round;
	vector<int > distribution;
	vector<request_template > vec_template;
	vector<request_or_reply > vec_req;
	vec_template.resize(total_query_type);
	vec_req.resize(total_query_type);
	for (int i = 0; i < total_query_type; i++) {
		string filename;
		configfile >> filename;
		int current_dist;
		configfile >> current_dist;
		distribution.push_back(current_dist);
		bool success = clnt->parser.parse_template(filename, vec_template[i]);
		translate_req_template(clnt, vec_template[i]);
		vec_req[i].cmd_chains = vec_template[i].cmd_chains;
		if (!success) {
			cout << "sparql parse error" << endl;
			return ;
		}
		vec_req[i].silent = global_silent;
	}
	uint64_t start_time = timer::get_usec();
	for (int i = 0; i < global_batch_factor; i++) {
		int idx = mymath::get_distribution(clnt->cfg->get_random(), distribution);
		instantiate_request(clnt, vec_template[idx], vec_req[idx]);
		clnt->GetId(vec_req[idx]);
		logger.start_record(vec_req[idx].parent_id, idx);
		clnt->Send(vec_req[idx]);
	}
	for (int i = 0; i < total_request; i++) {
		request_or_reply reply = clnt->Recv();
		logger.end_record(reply.parent_id);
		int idx = mymath::get_distribution(clnt->cfg->get_random(), distribution);
		instantiate_request(clnt, vec_template[idx], vec_req[idx]);
		clnt->GetId(vec_req[idx]);
		logger.start_record(vec_req[idx].parent_id, idx);
		clnt->Send(vec_req[idx]);
	}
	for (int i = 0; i < global_batch_factor; i++) {
		request_or_reply reply = clnt->Recv();
		logger.end_record(reply.parent_id);
	}
	uint64_t end_time = timer::get_usec();
	cout << 1000.0 * (total_request + global_batch_factor) / (end_time - start_time) << " Kops" << endl;
};

void
nonblocking_execute(client* clnt, string mix_config, batch_logger& logger)
{
	ifstream configfile(mix_config);
	if (!configfile) {
		cout << "File " << mix_config << " not exist" << endl;
		return ;
	}

	int total_query_type;
	int total_request;
	int sleep_round = 1;
	configfile >> total_query_type >> total_request >> sleep_round;

	vector<int > distribution;
	vector<request_template > vec_template;
	vector<request_or_reply > vec_req;

	vec_template.resize(total_query_type);
	vec_req.resize(total_query_type);
	for (int i = 0; i < total_query_type; i++) {
		string filename;
		configfile >> filename;
		int current_dist;
		configfile >> current_dist;
		distribution.push_back(current_dist);
		bool success = clnt->parser.parse_template(filename, vec_template[i]);
		translate_req_template(clnt, vec_template[i]);
		vec_req[i].cmd_chains = vec_template[i].cmd_chains;
		if (!success) {
			cout << "sparql parse error" << endl;
			return ;
		}
		vec_req[i].silent = global_silent;
	}

	int send_request = 0;
	int recv_request = 0;
	while (recv_request != total_request) {
		for (int t = 0; t < 10; t++) {
			if (send_request < total_request) {
				send_request++;
				int idx = mymath::get_distribution(clnt->cfg->get_random(), distribution);
				instantiate_request(clnt, vec_template[idx], vec_req[idx]);
				clnt->GetId(vec_req[idx]);
				logger.start_record(vec_req[idx].parent_id, idx);
				clnt->Send(vec_req[idx]);
			}
		}

		for (int i = 0; i < sleep_round; i++) {
			timer::cpu_relax(100);
			request_or_reply reply;
			bool success = TryRecvR(clnt->cfg, reply);
			while (success) {
				recv_request++;
				logger.end_record(reply.parent_id);
				success = TryRecvR(clnt->cfg, reply);
			}
		}
	}
};
<|endoftext|>


<commit_before>#include <Magick++.h>
#include <json.hpp>
#include <boost/filesystem.hpp>
#include <boost/crc.hpp>
#include <vector>
#include <algorithm>
#include <math.h>
#include <fstream>
#include <iostream>
#include <thread>
#include <iterator>
#include <set>
#include <string>

typedef std::vector<boost::filesystem::directory_entry> path_listing;
typedef std::vector<boost::filesystem::path> extension_list;

struct image_meta{
  std::string checksum;
  std::string exif_date_original;
};

bool operator<(const image_meta &a,const image_meta &b)
{
  return a.checksum < b.checksum;
}

void to_json(nlohmann::json& j, const image_meta& im) {
   j = nlohmann::json{{"id", im.checksum}, {"date", im.exif_date_original}};
}

void from_json(const nlohmann::json& j, image_meta& im) {

        im.checksum = j.at("id").get<std::string>();
        im.exif_date_original= j.at("date").get<std::string>();
}

typedef std::set<image_meta> image_set;

bool calculate_checksum(boost::filesystem::path filep, std::string &checksum)
{
  std::ifstream file(filep.string(),std::ios::in | std::ios::binary);
  std::vector<uint8_t> raw((std::istreambuf_iterator<char>(file)),std::istreambuf_iterator<char>());

  boost::crc_32_type result;
  result.process_bytes(&raw[0],raw.size());

  std::stringstream s;
  s << std::hex << result.checksum();
  s >> checksum;

  return true;
}

bool list_directory(const boost::filesystem::path root_directory,path_listing &files, path_listing &subdirectories, extension_list extensions)
{

  bool is_directory = true;

  if(boost::filesystem::exists(root_directory) && boost::filesystem::is_directory(root_directory))   {
    for(auto &entry: boost::filesystem::directory_iterator(root_directory))
      {
        if(boost::filesystem::is_directory(entry))
        {
          subdirectories.push_back(entry);
        }

        if(boost::filesystem::is_regular_file(entry) && std::find(extensions.begin(),extensions.end(),entry.path().extension()) != extensions.end())
        {
          files.push_back(entry);
        }
      }

  }else
  {
    is_directory = false;
  }

  return is_directory;

}

bool resize_single_picture(boost::filesystem::directory_entry entry, boost::filesystem::path output_directory, int new_width, std::string checksum, image_set &pictures_added)
{
   if(!boost::filesystem::is_directory(output_directory))
  {
    return false;
  }

   std::string filepath = entry.path().string();

   Magick::Image image;

   image.read(filepath);
   auto size = image.size();

   double w = size.width();
   double h = size.height();

   double ratio = h/w;
   int new_height = floor(new_width * ratio);

   std::string file_with_extension = checksum + entry.path().extension().string();
   auto output_filepath = output_directory / std::to_string(new_width) / file_with_extension;
   std::string output_name(output_filepath.string());

   std::cout << "Resizing..." << output_name << std::endl;

   //image.filterType(Magick::FilterType::LanczosFilter); // already default + not supported in earlier lib versions
   image.resize(Magick::Geometry(new_width, new_height));

   image.write(output_name);

   image_meta imgi;
   imgi.checksum = checksum;
   imgi.exif_date_original = image.attribute("EXIF:DateTimeOriginal");

   pictures_added.insert(imgi);
   return true;
}

bool create_direectories()
{
  return true;
}

bool resize_pictures_in_directory(path_listing files, boost::filesystem::path output_directory, int new_width, image_set &pictures_added)
{
  std::string checksum = "";

  for(auto &entry : files)
  {
    calculate_checksum(entry.path(), checksum);
    resize_single_picture(entry, output_directory, new_width, checksum, pictures_added);
  }

  return true;
}

bool parallel_resize(path_listing files, boost::filesystem::path output_directory, int new_width, image_set &pictures_added)
{

  size_t cores = std::thread::hardware_concurrency()/2;

  int batch_size = files.size()/cores;
  int start = 0;
  int stop = batch_size;

  std::vector<std::thread> thread_pool;

  for(int p=0;p<cores;p++)
  {
    path_listing partitioned_files(files.begin()+start, files.begin()+stop);
    start += batch_size;
    stop += batch_size;

    thread_pool.push_back(std::thread(resize_pictures_in_directory,partitioned_files,output_directory,new_width, std::ref(pictures_added)));
  }

  for(auto &t : thread_pool)
  {
    t.join();
  }
  return true;

}

bool load_json(boost::filesystem::path path,nlohmann::json &json)
{
  path = path / "digest.json";
  std::ifstream j(path.string());
  if(j)
  {
    j >> json;
    return true;
  }

  return false;
}

bool save_json(boost::filesystem::path path, nlohmann::json json)
{
  path = path / "digest.json";
  std::ofstream j(path.string());
  if(j)
  {
  std::cout << "SAVING" << path.string() << std::endl;
    j << json.dump(4);
    return true;
  }

    return false;
}

int main(int argc, char** argv)
{

nlohmann::json json;

bool existing_json = load_json(boost::filesystem::path(argv[2]),json);
path_listing files, subdirectories;

extension_list allowed_extensions({".jpg",".jpeg",".JPG",".png",".PNG"});

image_set pictures_added;
std::set<int> resolutions;

try
{
image_set pa = json[argv[3]]["pictures"];
std::set<int> res = json[argv[3]]["widths"];

pictures_added = pa;
resolutions = res;
}catch(std::exception &e)
{
}

list_directory(boost::filesystem::path(argv[1]),files, subdirectories, allowed_extensions);
if(argc > 4)
{
  for(int p=4; p< argc;p++)
  {
    boost::filesystem::path sub = boost::filesystem::path(argv[2]) / std::to_string(std::stoi(argv[p]));

    boost::filesystem::create_directory(sub);
    resolutions.insert(std::stoi(argv[p]));
    resize_pictures_in_directory(files,boost::filesystem::path(argv[2]),std::stoi(argv[p]), pictures_added);
   // parallel_resize(files,boost::filesystem::path(argv[2]),std::stoi(argv[p]), pictures_added);;
  }
}

json[argv[3]]["pictures"] = nlohmann::json(pictures_added);
json[argv[3]]["widths"] = resolutions;

save_json(boost::filesystem::path(argv[2]), json);

return 0;
}
<commit_msg>🌀 recursive directory scan<commit_after>#include <Magick++.h>
#include <json.hpp>
#include <boost/filesystem.hpp>
#include <boost/crc.hpp>
#include <vector>
#include <algorithm>
#include <math.h>
#include <fstream>
#include <iostream>
#include <thread>
#include <iterator>
#include <set>
#include <string>

typedef std::vector<boost::filesystem::directory_entry> path_listing;
typedef std::vector<boost::filesystem::path> extension_list;

const size_t max_directory_depth = 10;

struct image_meta{
  std::string checksum;
  std::string exif_date_original;
};

bool operator<(const image_meta &a,const image_meta &b)
{
  return a.checksum < b.checksum;
}

void to_json(nlohmann::json& j, const image_meta& im) {
   j = nlohmann::json{{"id", im.checksum}, {"date", im.exif_date_original}};
}

void from_json(const nlohmann::json& j, image_meta& im) {

        im.checksum = j.at("id").get<std::string>();
        im.exif_date_original= j.at("date").get<std::string>();
}

typedef std::set<image_meta> image_set;

bool calculate_checksum(boost::filesystem::path filep, std::string &checksum)
{
  std::ifstream file(filep.string(),std::ios::in | std::ios::binary);
  std::vector<uint8_t> raw((std::istreambuf_iterator<char>(file)),std::istreambuf_iterator<char>());

  boost::crc_32_type result;
  result.process_bytes(&raw[0],raw.size());

  std::stringstream s;
  s << std::hex << result.checksum();
  s >> checksum;

  return true;
}

bool list_directory(const boost::filesystem::path root_directory,path_listing &files, extension_list extensions, size_t recursions = 0)
{
  recursions++;
  bool is_directory = true;

  if(boost::filesystem::exists(root_directory) && boost::filesystem::is_directory(root_directory))   {
    for(auto &entry: boost::filesystem::directory_iterator(root_directory))
      {
        if(boost::filesystem::is_directory(entry))
        {
          if(recursions <= max_directory_depth)
          {
            list_directory(entry.path(),files,extensions, recursions);
          }
        }

        if(boost::filesystem::is_regular_file(entry) && std::find(extensions.begin(),extensions.end(),entry.path().extension()) != extensions.end())
        {
          files.push_back(entry);
        }
      }

  }else
  {
    is_directory = false;
  }

  return is_directory;

}

bool resize_single_picture(boost::filesystem::directory_entry entry, boost::filesystem::path output_directory, int new_width, std::string checksum, image_set &pictures_added)
{
   if(!boost::filesystem::is_directory(output_directory))
  {
    return false;
  }

   std::string filepath = entry.path().string();

   Magick::Image image;

   image.read(filepath);
   auto size = image.size();

   double w = size.width();
   double h = size.height();

   double ratio = h/w;
   int new_height = floor(new_width * ratio);

   std::string file_with_extension = checksum + entry.path().extension().string();
   auto output_filepath = output_directory / std::to_string(new_width) / file_with_extension;
   std::string output_name(output_filepath.string());

   std::cout << "Resizing..." << output_name << std::endl;

   //image.filterType(Magick::FilterType::LanczosFilter); // already default + not supported in earlier lib versions
   image.resize(Magick::Geometry(new_width, new_height));

   image.write(output_name);

   image_meta imgi;
   imgi.checksum = checksum;
   imgi.exif_date_original = image.attribute("EXIF:DateTimeOriginal");

   pictures_added.insert(imgi);
   return true;
}

bool create_direectories()
{
  return true;
}

bool resize_pictures_in_directory(path_listing files, boost::filesystem::path output_directory, int new_width, image_set &pictures_added)
{
  std::string checksum = "";

  for(auto &entry : files)
  {
    calculate_checksum(entry.path(), checksum);
    resize_single_picture(entry, output_directory, new_width, checksum, pictures_added);
  }

  return true;
}

bool parallel_resize(path_listing files, boost::filesystem::path output_directory, int new_width, image_set &pictures_added)
{

  size_t cores = std::thread::hardware_concurrency()/2;

  int batch_size = files.size()/cores;
  int start = 0;
  int stop = batch_size;

  std::vector<std::thread> thread_pool;

  for(int p=0;p<cores;p++)
  {
    path_listing partitioned_files(files.begin()+start, files.begin()+stop);
    start += batch_size;
    stop += batch_size;

    thread_pool.push_back(std::thread(resize_pictures_in_directory,partitioned_files,output_directory,new_width, std::ref(pictures_added)));
  }

  for(auto &t : thread_pool)
  {
    t.join();
  }
  return true;

}

bool load_json(boost::filesystem::path path,nlohmann::json &json)
{
  path = path / "digest.json";
  std::ifstream j(path.string());
  if(j)
  {
    j >> json;
    return true;
  }

  return false;
}

bool save_json(boost::filesystem::path path, nlohmann::json json)
{
  path = path / "digest.json";
  std::ofstream j(path.string());
  if(j)
  {
  std::cout << "SAVING" << path.string() << std::endl;
    j << json.dump(4);
    return true;
  }

    return false;
}

int main(int argc, char** argv)
{

nlohmann::json json;

bool existing_json = load_json(boost::filesystem::path(argv[2]),json);
path_listing files;

extension_list allowed_extensions({".jpg",".jpeg",".JPG",".png",".PNG"});

image_set pictures_added;
std::set<int> resolutions;

try
{
image_set pa = json[argv[3]]["pictures"];
std::set<int> res = json[argv[3]]["widths"];

pictures_added = pa;
resolutions = res;
}catch(std::exception &e)
{
}

list_directory(boost::filesystem::path(argv[1]),files, allowed_extensions);
if(argc > 4)
{
  for(int p=4; p< argc;p++)
  {
    boost::filesystem::path sub = boost::filesystem::path(argv[2]) / std::to_string(std::stoi(argv[p]));

    boost::filesystem::create_directory(sub);
    resolutions.insert(std::stoi(argv[p]));
    resize_pictures_in_directory(files,boost::filesystem::path(argv[2]),std::stoi(argv[p]), pictures_added);
   // parallel_resize(files,boost::filesystem::path(argv[2]),std::stoi(argv[p]), pictures_added);;
  }
}

json[argv[3]]["pictures"] = nlohmann::json(pictures_added);
json[argv[3]]["widths"] = resolutions;

save_json(boost::filesystem::path(argv[2]), json);

return 0;
}
<|endoftext|>


<commit_before>/*
 * KDots
 * Copyright (c) 2011-2012 Minh Ngo <nlminhtl@gmail.com>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
#include <KApplication>
#include <KAboutData>
#include <KLocale>
#include <KCmdLineArgs>
#include <brushcombo.hpp>
#include "mainwindow.hpp"
#include "gameconfig.hpp"

int main (int argc, char **argv)
{
	KAboutData aboutData ("kdots", 0,
		ki18n ("kdots"), "0.5b",
		ki18n ("A prototype of the game of dots."),
		KAboutData::License_BSD,
		ki18n ("Copyright (c) 2011-2012 Minh Ngo"));

	aboutData.setHomepage ("https://github.com/Ignotus/kdots");
	aboutData.setBugAddress ("nlminhtl@gmail.com");
	aboutData.addAuthor (ki18n ("Minh Ngo"),
			ki18n ("Architecture. Project Management"),
			"nlminhtl@gmail.com",
			"http://ignotusp.fedoraproject.org");

	aboutData.addCredit (ki18n ("Bohdan Rybak"),
			ki18n ("Game engine tester"),
			"brainerazer@gmail.com");

	KCmdLineArgs::init (argc, argv, &aboutData);

	KApplication app;

	KDots::GameConfig::registerMeta ();

	KDots::MainWindow *w = new KDots::MainWindow ();
	w->show ();

	return app.exec ();
}
<commit_msg>Adding a contributor<commit_after>/*
 * KDots
 * Copyright (c) 2011-2012 Minh Ngo <nlminhtl@gmail.com>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
#include <KApplication>
#include <KAboutData>
#include <KLocale>
#include <KCmdLineArgs>
#include <brushcombo.hpp>
#include "mainwindow.hpp"
#include "gameconfig.hpp"

void initAboutData(KAboutData& aboutData)
{
	aboutData.setHomepage ("https://github.com/Ignotus/kdots");
	aboutData.setBugAddress ("nlminhtl@gmail.com");
	aboutData.addAuthor (ki18n ("Minh Ngo"),
			ki18n ("Architecture. Project Management"),
			"nlminhtl@gmail.com",
			"http://ignotusp.fedoraproject.org");

	aboutData.addCredit (ki18n ("Bohdan Rybak"),
			ki18n ("Game engine tester"),
			"brainerazer@gmail.com");

	aboutData.addCredit (ki18n ("Artem Sereda"),
			ki18n ("Arch Linux package maintainer"),
			"overmind88@gmail.com");
}

int main (int argc, char **argv)
{
	KAboutData aboutData ("kdots", 0,
		ki18n ("kdots"), "0.5b",
		ki18n ("A prototype of the game of dots."),
		KAboutData::License_BSD,
		ki18n ("Copyright (c) 2011-2012 Minh Ngo"));

	initAboutData (aboutData);

	KCmdLineArgs::init (argc, argv, &aboutData);

	KApplication app;

	KDots::GameConfig::registerMeta ();

	KDots::MainWindow *w = new KDots::MainWindow ();
	w->show ();

	return app.exec ();
}
<|endoftext|>