<commit_before>/*
    vcflib C++ library for parsing and manipulating VCF files

    Copyright © 2010-2022 Erik Garrison
    Copyright © 2020-2022 Pjotr Prins

    This software is published under the MIT License. See the LICENSE file.
*/

#include "Variant.h"
#include "convert.h"
#include "join.h"
#include "split.h"
#include <omp.h>
#include <set>
#include <algorithm>
#include <getopt.h>

using namespace std;
using namespace vcflib;

#define ALLELE_NULL -1

double convertStrDbl(const string& s) {
    double r;
    convert(s, r);
    return r;
}

void printSummary(char** argv) {
    std::string text = R"(
usage: vcfwave [options] [file]

Realign reference and alternate alleles with WFA, parsing out the primitive alleles
into multiple VCF records. New records have IDs that reference the source record ID.
Genotypes are handled. Deletions generate haploid/missing genotypes at overlapping sites.

options:
    -p, --wf-params PARAMS  use the given BiWFA params (default: 0,19,39,3,81,1)
                            format=match,mismatch,gap1-open,gap1-ext,gap2-open,gap2-ext
    -f, --tag-parsed FLAG   Annotate decomposed records with the source record position
                            (default: ORIGIN).
    -L, --max-length LEN    Do not manipulate records in which either the ALT or
                            REF is longer than LEN (default: unlimited).
    -K, --inv-kmer K        Length of k-mer to use for inversion detection sketching (default: 17).
    -I, --inv-min LEN       Minimum allele length to consider for inverted alignment (default: 64).
    -k, --keep-info         Maintain site and allele-level annotations when decomposing.
                            Note that in many cases, such as multisample VCFs, these won't
                            be valid post-decomposition.  For biallelic loci in single-sample
                            VCFs, they should be usable with caution.
    -t, --threads N         use this many threads for variant decomposition
    -n, --nextgen           next gen mode.
    -d, --debug             debug mode.

Type: transformation
)";

    cerr << text;

    exit(0);
}

int main(int argc, char** argv) {

    bool includePreviousBaseForIndels = true;
    bool useMNPs = true;
    string parseFlag = "ORIGIN";
    string algorithm = "WF";
    string paramString = "0,19,39,3,81,1";
    int maxLength = 0;
    bool keepInfo = false;
    bool keepGeno = false;
    bool useWaveFront = true;
    bool nextGen  = false;
    bool debug    = false;

    int thread_count = 1;
    int inv_sketch_kmer = 17;
    int min_inv_len = 64;

    VariantCallFile variantFile;

    int c;
    while (true) {
        static struct option long_options[] =
            {
                /* These options set a flag. */
                //{"verbose", no_argument,       &verbose_flag, 1},
                {"help", no_argument, 0, 'h'},
                {"wf-params", required_argument, 0, 'p'},
                {"max-length", required_argument, 0, 'L'},
                {"inv-kmer", required_argument, 0, 'K'},
                {"inv-min", required_argument, 0, 'I'},
                {"tag-parsed", required_argument, 0, 'f'},
                {"keep-info", no_argument, 0, 'k'},
                {"keep-geno", no_argument, 0, 'g'},
                {"threads", required_argument, 0, 't'},
                {"nextgen", no_argument, 0, 'n'},
                {"debug", no_argument, 0, 'd'},
                {0, 0, 0, 0}
            };
        /* getopt_long stores the option index here. */
        int option_index = 0;

        c = getopt_long (argc, argv, "ndhkt:L:p:t:K:I:f:",
                         long_options, &option_index);

        if (c == -1)
            break;

        switch (c) {

	    case 'k':
            keepInfo = true;
            break;

	    case 'g':
            keepGeno = true;
            break;

        case 'p':
            paramString = optarg;
            break;

        case 't':
            thread_count = atoi(optarg);
            break;

        case 'n':
            nextGen = true;
            break;

        case 'd':
            debug = true;
            break;

        case 'h':
            printSummary(argv);
            break;

	    case 'f':
            parseFlag = optarg;
            break;

        case 'L':
            maxLength = atoi(optarg);
            break;

        case 'K':
            inv_sketch_kmer = atoi(optarg);
            break;

        case 'I':
            min_inv_len = atoi(optarg);
            break;

        case '?':
            printSummary(argv);
            exit(1);
            break;

        default:
            abort ();
        }
    }

    omp_set_num_threads(thread_count);

    if (optind < argc) {
        string filename = argv[optind];
        variantFile.open(filename);
    } else {
        variantFile.open(std::cin);
    }

    if (!variantFile.is_open()) {
        return 1;
    }

    // parse the alignment parameters
    vector<string> p_str = split(paramString, ',');
    vector<int> p;
    for (auto& s : p_str) { p.push_back(atoi(s.c_str())); }

    auto wfa_params = wavefront_aligner_attr_default;
    wfa_params.memory_mode = wavefront_memory_ultralow; // note this is overridden in Variant.cpp
    wfa_params.distance_metric = gap_affine_2p;
    wfa_params.affine2p_penalties.match = p[0];
    wfa_params.affine2p_penalties.mismatch = p[1];
    wfa_params.affine2p_penalties.gap_opening1 = p[2];
    wfa_params.affine2p_penalties.gap_extension1 = p[3];
    wfa_params.affine2p_penalties.gap_opening2 = p[4];
    wfa_params.affine2p_penalties.gap_extension2 = p[5];
    wfa_params.alignment_scope = compute_alignment;

    variantFile.addHeaderLine("##INFO=<ID=TYPE,Number=A,Type=String,Description=\"The type of allele, either snp, mnp, ins, del, or complex.\">");
    variantFile.addHeaderLine("##INFO=<ID=LEN,Number=A,Type=Integer,Description=\"allele length\">");
    variantFile.addHeaderLine("##INFO=<ID="+parseFlag+",Number=1,Type=String,Description=\"Decomposed from a complex record using vcflib vcfwave and alignment with WFA2-lib.\">");
    variantFile.addHeaderLine("##INFO=<ID=INV,Number=A,Type=String,Description=\"Count of haplotypes which are aligned in the inverted orientation using vcflib vcfwave.\">");
    cout << variantFile.header << endl;

    Variant var(variantFile);
    while (variantFile.getNextVariant(var)) {
        // we can't decompose *1* bp events, these are already in simplest-form whether SNPs or indels
        // we also don't handle anything larger than maxLength bp
        int max_allele_length = 0;
        for (auto allele: var.alt) {
          if (debug) cerr << allele << ":" << allele.length() << "," << max_allele_length << endl;
          if (allele.length() >= max_allele_length) {
             max_allele_length = allele.length();
             // cerr << max_allele_length << endl;
          }
        }

        if ((maxLength && max_allele_length > maxLength) || max_allele_length == 1 ||
            (var.alt.size() == 1 &&
             (var.ref.size() == 1 || (maxLength && var.ref.size() > maxLength)))) {
            // nothing to do
            cout << var << endl;
            continue;
        }

        map<string, pair<vector<VariantAllele>, bool> > varAlleles =
           var.parsedAlternates(includePreviousBaseForIndels, useMNPs,
                                false, // bool useEntropy = false,
                                "",    // string flankingRefLeft = "",
                                "",    // string flankingRefRight = "",
                                &wfa_params,
                                inv_sketch_kmer,
                                min_inv_len,
                                debug);  // bool debug=false

        if (nextGen) {

            struct trackinfo {
                size_t pos0 = 0;
                string ref0, alt0, ref1, algn;
                size_t pos1 = 0;
                size_t altidx;
                int relpos;
                int AC,AF,AN;
                bool is_rev = false;
                string type;
                string origin;
            };
            typedef map<string, trackinfo> TrackInfo;
            TrackInfo unique;

            for (const auto [alt0, wfvalue] : varAlleles) {
                // auto alt0 = k.first;
                // auto wfvalue = k.second;
                bool is_rev = wfvalue.second;
                for (auto wfmatch: wfvalue.first) {
                    auto ref = wfmatch.ref;
                    auto aligned = wfmatch.alt;
                    auto wfpos = wfmatch.position;
                    int alt_index,AC,AN = -1;
                    double AF = 0.0;
                    string wftag = alt0+":"+to_string(wfpos)+":"+ref+"/"+aligned;
                    if (var.ref == aligned) {
                        cout << "EQ: ";
                    }
                    else {
                        auto index = [](vector<string> v, string allele) {
                            auto it = find(v.begin(), v.end(), allele);
                            return (it == v.end() ? throw std::runtime_error("Unexpected value error for allele "+allele) : it - v.begin() );
                        };
                        alt_index = index(var.alt,alt0); // throws error if missing
                        AC = stoi(var.info["AC"].at(alt_index));
                        AF = stod(var.info["AF"].at(alt_index));
                        AN = stoi(var.info["AN"].at(0));
                    }
                    auto relpos = wfpos - var.position;
                    auto u = &unique[wftag];
                    u->pos0 = var.position;
                    u->ref0 = var.ref;
                    u->alt0 = alt0;
                    u->ref1 = ref;
                    u->algn = aligned;
                    u->pos1 = wfpos;
                    u->altidx = alt_index;
                    u->relpos = relpos;
                    u->AC = AC;
                    u->AF = AF;
                    u->AN = AN;
                    u->is_rev = is_rev;
                }
            }
            // Adjust TYPE field to set snp/mnp/ins/del
            auto variants = unique;
            unique.clear();
            for (auto [key,v] : variants) {
                auto ref_len = v.ref1.length();
                auto aln_len = v.algn.length();
                string type;
                if (aln_len < ref_len)
                    type = "del";
                else if (aln_len > ref_len)
                    type = "ins";
                else if (aln_len == ref_len)
                    if (ref_len == 1)
                        type = "snp";
                    else
                        type = "mnp";
                v.type = type;
                v.origin = var.sequenceName+to_string(var.position);
                variants[key] = v;
                cout.precision(2);
                cout << key << " HEY " << ref_len << "," << aln_len << " AC=" << v.AC << fixed << " AF=" << v.AF << " " << variants[key].type << " " << variants.size() << endl;
            }
            cerr << "WIP " << endl;
        }
        else {
            var.legacy_reduceAlleles(
                varAlleles,
                variantFile,
                var,
                parseFlag,
                keepInfo,
                keepGeno,
                debug);
        }
    }

    return 0;
}
<commit_msg>vfwave postprocessing: transpiling to C++ splitting genotypes<commit_after>/*
    vcflib C++ library for parsing and manipulating VCF files

    Copyright © 2010-2022 Erik Garrison
    Copyright © 2020-2022 Pjotr Prins

    This software is published under the MIT License. See the LICENSE file.
*/

#include "Variant.h"
#include "convert.h"
#include "join.h"
#include "split.h"
#include <omp.h>
#include <set>
#include <algorithm>
#include <getopt.h>

using namespace std;
using namespace vcflib;

#define ALLELE_NULL -1

double convertStrDbl(const string& s) {
    double r;
    convert(s, r);
    return r;
}

void printSummary(char** argv) {
    std::string text = R"(
usage: vcfwave [options] [file]

Realign reference and alternate alleles with WFA, parsing out the primitive alleles
into multiple VCF records. New records have IDs that reference the source record ID.
Genotypes are handled. Deletions generate haploid/missing genotypes at overlapping sites.

options:
    -p, --wf-params PARAMS  use the given BiWFA params (default: 0,19,39,3,81,1)
                            format=match,mismatch,gap1-open,gap1-ext,gap2-open,gap2-ext
    -f, --tag-parsed FLAG   Annotate decomposed records with the source record position
                            (default: ORIGIN).
    -L, --max-length LEN    Do not manipulate records in which either the ALT or
                            REF is longer than LEN (default: unlimited).
    -K, --inv-kmer K        Length of k-mer to use for inversion detection sketching (default: 17).
    -I, --inv-min LEN       Minimum allele length to consider for inverted alignment (default: 64).
    -k, --keep-info         Maintain site and allele-level annotations when decomposing.
                            Note that in many cases, such as multisample VCFs, these won't
                            be valid post-decomposition.  For biallelic loci in single-sample
                            VCFs, they should be usable with caution.
    -t, --threads N         use this many threads for variant decomposition
    -n, --nextgen           next gen mode.
    -d, --debug             debug mode.

Type: transformation
)";

    cerr << text;

    exit(0);
}

int main(int argc, char** argv) {

    bool includePreviousBaseForIndels = true;
    bool useMNPs = true;
    string parseFlag = "ORIGIN";
    string algorithm = "WF";
    string paramString = "0,19,39,3,81,1";
    int maxLength = 0;
    bool keepInfo = false;
    bool keepGeno = false;
    bool useWaveFront = true;
    bool nextGen  = false;
    bool debug    = false;

    int thread_count = 1;
    int inv_sketch_kmer = 17;
    int min_inv_len = 64;

    VariantCallFile variantFile;

    int c;
    while (true) {
        static struct option long_options[] =
            {
                /* These options set a flag. */
                //{"verbose", no_argument,       &verbose_flag, 1},
                {"help", no_argument, 0, 'h'},
                {"wf-params", required_argument, 0, 'p'},
                {"max-length", required_argument, 0, 'L'},
                {"inv-kmer", required_argument, 0, 'K'},
                {"inv-min", required_argument, 0, 'I'},
                {"tag-parsed", required_argument, 0, 'f'},
                {"keep-info", no_argument, 0, 'k'},
                {"keep-geno", no_argument, 0, 'g'},
                {"threads", required_argument, 0, 't'},
                {"nextgen", no_argument, 0, 'n'},
                {"debug", no_argument, 0, 'd'},
                {0, 0, 0, 0}
            };
        /* getopt_long stores the option index here. */
        int option_index = 0;

        c = getopt_long (argc, argv, "ndhkt:L:p:t:K:I:f:",
                         long_options, &option_index);

        if (c == -1)
            break;

        switch (c) {

	    case 'k':
            keepInfo = true;
            break;

	    case 'g':
            keepGeno = true;
            break;

        case 'p':
            paramString = optarg;
            break;

        case 't':
            thread_count = atoi(optarg);
            break;

        case 'n':
            nextGen = true;
            break;

        case 'd':
            debug = true;
            break;

        case 'h':
            printSummary(argv);
            break;

	    case 'f':
            parseFlag = optarg;
            break;

        case 'L':
            maxLength = atoi(optarg);
            break;

        case 'K':
            inv_sketch_kmer = atoi(optarg);
            break;

        case 'I':
            min_inv_len = atoi(optarg);
            break;

        case '?':
            printSummary(argv);
            exit(1);
            break;

        default:
            abort ();
        }
    }

    omp_set_num_threads(thread_count);

    if (optind < argc) {
        string filename = argv[optind];
        variantFile.open(filename);
    } else {
        variantFile.open(std::cin);
    }

    if (!variantFile.is_open()) {
        return 1;
    }

    // parse the alignment parameters
    vector<string> p_str = split(paramString, ',');
    vector<int> p;
    for (auto& s : p_str) { p.push_back(atoi(s.c_str())); }

    auto wfa_params = wavefront_aligner_attr_default;
    wfa_params.memory_mode = wavefront_memory_ultralow; // note this is overridden in Variant.cpp
    wfa_params.distance_metric = gap_affine_2p;
    wfa_params.affine2p_penalties.match = p[0];
    wfa_params.affine2p_penalties.mismatch = p[1];
    wfa_params.affine2p_penalties.gap_opening1 = p[2];
    wfa_params.affine2p_penalties.gap_extension1 = p[3];
    wfa_params.affine2p_penalties.gap_opening2 = p[4];
    wfa_params.affine2p_penalties.gap_extension2 = p[5];
    wfa_params.alignment_scope = compute_alignment;

    variantFile.addHeaderLine("##INFO=<ID=TYPE,Number=A,Type=String,Description=\"The type of allele, either snp, mnp, ins, del, or complex.\">");
    variantFile.addHeaderLine("##INFO=<ID=LEN,Number=A,Type=Integer,Description=\"allele length\">");
    variantFile.addHeaderLine("##INFO=<ID="+parseFlag+",Number=1,Type=String,Description=\"Decomposed from a complex record using vcflib vcfwave and alignment with WFA2-lib.\">");
    variantFile.addHeaderLine("##INFO=<ID=INV,Number=A,Type=String,Description=\"Count of haplotypes which are aligned in the inverted orientation using vcflib vcfwave.\">");
    cout << variantFile.header << endl;

    Variant var(variantFile);
    while (variantFile.getNextVariant(var)) {
        // we can't decompose *1* bp events, these are already in simplest-form whether SNPs or indels
        // we also don't handle anything larger than maxLength bp
        int max_allele_length = 0;
        for (auto allele: var.alt) {
          if (debug) cerr << allele << ":" << allele.length() << "," << max_allele_length << endl;
          if (allele.length() >= max_allele_length) {
             max_allele_length = allele.length();
             // cerr << max_allele_length << endl;
          }
        }

        if ((maxLength && max_allele_length > maxLength) || max_allele_length == 1 ||
            (var.alt.size() == 1 &&
             (var.ref.size() == 1 || (maxLength && var.ref.size() > maxLength)))) {
            // nothing to do
            cout << var << endl;
            continue;
        }

        map<string, pair<vector<VariantAllele>, bool> > varAlleles =
           var.parsedAlternates(includePreviousBaseForIndels, useMNPs,
                                false, // bool useEntropy = false,
                                "",    // string flankingRefLeft = "",
                                "",    // string flankingRefRight = "",
                                &wfa_params,
                                inv_sketch_kmer,
                                min_inv_len,
                                debug);  // bool debug=false

        if (nextGen) {

            typedef vector<int> Genotypes;
            typedef map<string, Genotypes> SampleGenotypes;
            struct trackinfo {
                size_t pos0 = 0;
                string ref0, alt0, ref1, algn;
                size_t pos1 = 0;
                size_t altidx;
                int relpos;
                int AC,AF,AN;
                bool is_rev = false;
                string type;
                string origin;
                SampleGenotypes genotypes;
            };
            typedef map<string, trackinfo> TrackInfo;
            TrackInfo unique;

            for (const auto [alt0, wfvalue] : varAlleles) {
                // auto alt0 = k.first;
                // auto wfvalue = k.second;
                bool is_rev = wfvalue.second;
                for (auto wfmatch: wfvalue.first) {
                    auto ref = wfmatch.ref;
                    auto aligned = wfmatch.alt;
                    auto wfpos = wfmatch.position;
                    int alt_index,AC,AN = -1;
                    double AF = 0.0;
                    string wftag = alt0+":"+to_string(wfpos)+":"+ref+"/"+aligned;
                    if (var.ref == aligned) {
                        cout << "EQ: ";
                    }
                    else {
                        auto index = [](vector<string> v, string allele) {
                            auto it = find(v.begin(), v.end(), allele);
                            return (it == v.end() ? throw std::runtime_error("Unexpected value error for allele "+allele) : it - v.begin() );
                        };
                        alt_index = index(var.alt,alt0); // throws error if missing
                        AC = stoi(var.info["AC"].at(alt_index));
                        AF = stod(var.info["AF"].at(alt_index));
                        AN = stoi(var.info["AN"].at(0));
                    }
                    auto relpos = wfpos - var.position;
                    auto u = &unique[wftag];
                    u->pos0 = var.position;
                    u->ref0 = var.ref;
                    u->alt0 = alt0;
                    u->ref1 = ref;
                    u->algn = aligned;
                    u->pos1 = wfpos;
                    u->altidx = alt_index;
                    u->relpos = relpos;
                    u->AC = AC;
                    u->AF = AF;
                    u->AN = AN;
                    u->is_rev = is_rev;
                }
            }
            // Collect genotypes in gts
            auto samples = var.samples;
            for (auto sname: var.sampleNames) {
                auto genotype1 = samples[sname]["GT"].front();
                auto genotypeStrs = split(genotype1, "|/");
                cout << "gts " << genotypeStrs[0] << "," << genotypeStrs[1] << endl;
                // auto gts =
            }
            // Merge records in a new dict named variants and adjust AC, AF and AN
            TrackInfo variants;
            for (auto [key,v] : unique) {
                auto ref = v.ref1;
                auto aligned = v.algn;
                if (ref != aligned) {
                    auto ntag = to_string(v.pos1) + ":" + ref + "/" + aligned;
                    if (variants.count(ntag)>0) {
                        variants[ntag].AC += v.AC;
                        // Check AN number is equal so we can compute AF by addition
                        assert(variants[ntag].AN == v.AN);
                        variants[ntag].AF += v.AF;
                        // Merge genotypes if they come from different alleles
                        if (v.altidx != variants[ntag].altidx) {
                            // later
                        }
                    }
                    else {
                        variants[ntag] = v;
                    }
                    /*
                    self.assertEqual(variants[ntag]['AN'],v['AN'])
                    variants[ntag]['AF'] += v['AF']
                    # Merge genotypes if they come from different alleles
                    if v['altidx'] != variants[ntag]['altidx']:
                        for i,samplesi in enumerate(variants[ntag]['samples']):
                            result = samplesi.copy()
                            g2 = v['samples'][i]
                            for j,samplej in enumerate(g2):
                                if g2[j] and g2[j]>0:
                                    result[j] = g2[j]
                            # print(i,samplesi,v['samples'][i],result)
                else:
                    variants[ntag] = v
                    */
                }
            }
            unique.clear();
            // Adjust TYPE field to set snp/mnp/ins/del
            for (auto [key,v] : variants) {
                auto ref_len = v.ref1.length();
                auto aln_len = v.algn.length();
                string type;
                if (aln_len < ref_len)
                    type = "del";
                else if (aln_len > ref_len)
                    type = "ins";
                else if (aln_len == ref_len)
                    if (ref_len == 1)
                        type = "snp";
                    else
                        type = "mnp";
                v.type = type;
                v.origin = var.sequenceName+to_string(var.position);
                variants[key] = v;
                cout.precision(2);
                cout << key << " HEY " << ref_len << "," << aln_len << " AC=" << v.AC << fixed << " AF=" << v.AF << " " << variants[key].type << " " << variants.size() << endl;
            }
            cerr << "WIP " << endl;
        }
        else {
            var.legacy_reduceAlleles(
                varAlleles,
                variantFile,
                var,
                parseFlag,
                keepInfo,
                keepGeno,
                debug);
        }
    }

    return 0;
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "walltime.h"

#include <sys/time.h>
#include <time.h>
#include <cstdio>
#include <cstdint>
#include <cstring>
#include <atomic>
#include <limits>
#include <type_traits>

#include "check.h"
#include "cycleclock.h"
#include "sysinfo.h"

namespace benchmark {
namespace walltime {
namespace {

inline bool SplitTimezone(WallTime value, bool local, struct tm* t,
                             double* subsecond) {
  memset(t, 0, sizeof(*t));
  if ((value < 0) || (value > std::numeric_limits<time_t>::max())) {
    *subsecond = 0.0;
    return false;
  }
  const time_t whole_time = static_cast<time_t>(value);
  *subsecond = value - whole_time;
  if (local)
    localtime_r(&whole_time, t);
  else
    gmtime_r(&whole_time, t);
  return true;
}

} // end anonymous namespace

namespace internal {

class WallTimeImp
{
public:
  WallTime Now();

  static WallTimeImp & GetWallTimeImp() {
    static WallTimeImp imp;
#if __cplusplus >= 201103L
    static_assert(std::is_trivially_destructible<WallTimeImp>::value,
                  "WallTimeImp must be trivially destructible to prevent "
                  "issues with static destruction");
#endif
    return imp;
  }

private:
  WallTimeImp();
  // Helper routines to load/store a float from an AtomicWord. Required because
  // g++ < 4.7 doesn't support std::atomic<float> correctly. I cannot wait to
  // get rid of this horror show.
  void SetDrift(float f) {
    int32_t w;
    memcpy(&w, &f, sizeof(f));
    std::atomic_store(&drift_adjust_, w);
  }

  float GetDrift() const {
    float f;
    int32_t w = std::atomic_load(&drift_adjust_);
    memcpy(&f, &w, sizeof(f));
    return f;
  }

  WallTime Slow() {
    struct timeval tv;
    gettimeofday(&tv, NULL);
    return tv.tv_sec + tv.tv_usec * 1e-6;
  }

private:
  static_assert(sizeof(float) <= sizeof(int32_t),
               "type sizes don't allow the drift_adjust hack");

  static constexpr double kMaxErrorInterval = 100e-6;

  WallTime base_walltime_;
  int64_t base_cycletime_;
  int64_t cycles_per_second_;
  double seconds_per_cycle_;
  uint32_t last_adjust_time_;
  std::atomic<int32_t> drift_adjust_;
  int64_t max_interval_cycles_;

  BENCHMARK_DISALLOW_COPY_AND_ASSIGN(WallTimeImp);
};

WallTime WallTimeImp::Now() {
  WallTime now = 0.0;
  WallTime result = 0.0;
  int64_t ct = 0;
  uint32_t top_bits = 0;
  do {
    ct = cycleclock::Now();
    int64_t cycle_delta = ct - base_cycletime_;
    result = base_walltime_ + cycle_delta * seconds_per_cycle_;

    top_bits = static_cast<uint32_t>(uint64_t(ct) >> 32);
    // Recompute drift no more often than every 2^32 cycles.
    // I.e., @2GHz, ~ every two seconds
    if (top_bits == last_adjust_time_) {  // don't need to recompute drift
      return result + GetDrift();
    }

    now = Slow();
  } while (cycleclock::Now() - ct > max_interval_cycles_);
  // We are now sure that "now" and "result" were produced within
  // kMaxErrorInterval of one another.

  SetDrift(now - result);
  last_adjust_time_ = top_bits;
  return now;
}

WallTimeImp::WallTimeImp()
    : base_walltime_(0.0), base_cycletime_(0),
      cycles_per_second_(0), seconds_per_cycle_(0.0),
      last_adjust_time_(0), drift_adjust_(0),
      max_interval_cycles_(0) {
  cycles_per_second_ = static_cast<int64_t>(CyclesPerSecond());
  CHECK(cycles_per_second_ != 0);
  seconds_per_cycle_ = 1.0 / cycles_per_second_;
  max_interval_cycles_ =
      static_cast<int64_t>(cycles_per_second_ * kMaxErrorInterval);
  do {
    base_cycletime_ = cycleclock::Now();
    base_walltime_ = Slow();
  } while (cycleclock::Now() - base_cycletime_ > max_interval_cycles_);
  // We are now sure that "base_walltime" and "base_cycletime" were produced
  // within kMaxErrorInterval of one another.

  SetDrift(0.0);
  last_adjust_time_ = static_cast<uint32_t>(uint64_t(base_cycletime_) >> 32);
}

} // end namespace internal

WallTime Now()
{
    static internal::WallTimeImp& imp = internal::WallTimeImp::GetWallTimeImp();
    return imp.Now();
}

std::string Print(WallTime time, const char* format, bool local,
                  int* remainder_us) {
  char storage[32];
  struct tm split;
  double subsecond;
  if (!SplitTimezone(time, local, &split, &subsecond)) {
    snprintf(storage, sizeof(storage), "Invalid time: %f", time);
  } else {
    if (remainder_us != NULL) {
      *remainder_us = static_cast<int>((subsecond * 1000000) + 0.5);
      if (*remainder_us > 999999) *remainder_us = 999999;
      if (*remainder_us < 0) *remainder_us = 0;
    }
    strftime(storage, sizeof(storage), format, &split);
  }
  return std::string(storage);
}

}  // end namespace walltime
}  // end namespace benchmark
<commit_msg>address comments<commit_after>// Copyright 2015 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "walltime.h"

#include <sys/time.h>

#include <cstdio>
#include <cstdint>
#include <cstring>
#include <ctime>

#include <atomic>
#include <limits>
#include <type_traits>

#include "check.h"
#include "cycleclock.h"
#include "sysinfo.h"

namespace benchmark {
namespace walltime {
namespace {

bool SplitTimezone(WallTime value, bool local, struct tm* t,
                   double* subsecond) {
  memset(t, 0, sizeof(*t));
  if ((value < 0) || (value > std::numeric_limits<time_t>::max())) {
    *subsecond = 0.0;
    return false;
  }
  const time_t whole_time = static_cast<time_t>(value);
  *subsecond = value - whole_time;
  if (local)
    localtime_r(&whole_time, t);
  else
    gmtime_r(&whole_time, t);
  return true;
}

} // end anonymous namespace

namespace {

class WallTimeImp
{
public:
  WallTime Now();

  static WallTimeImp& GetWallTimeImp() {
    static WallTimeImp imp;
#if __cplusplus >= 201103L
    static_assert(std::is_trivially_destructible<WallTimeImp>::value,
                  "WallTimeImp must be trivially destructible to prevent "
                  "issues with static destruction");
#endif
    return imp;
  }

private:
  WallTimeImp();
  // Helper routines to load/store a float from an AtomicWord. Required because
  // g++ < 4.7 doesn't support std::atomic<float> correctly. I cannot wait to
  // get rid of this horror show.
  void SetDrift(float f) {
    int32_t w;
    memcpy(&w, &f, sizeof(f));
    std::atomic_store(&drift_adjust_, w);
  }

  float GetDrift() const {
    float f;
    int32_t w = std::atomic_load(&drift_adjust_);
    memcpy(&f, &w, sizeof(f));
    return f;
  }

  WallTime Slow() const {
    struct timeval tv;
    gettimeofday(&tv, NULL);
    return tv.tv_sec + tv.tv_usec * 1e-6;
  }

private:
  static_assert(sizeof(float) <= sizeof(int32_t),
               "type sizes don't allow the drift_adjust hack");

  static constexpr double kMaxErrorInterval = 100e-6;

  WallTime base_walltime_;
  int64_t base_cycletime_;
  int64_t cycles_per_second_;
  double seconds_per_cycle_;
  uint32_t last_adjust_time_;
  std::atomic<int32_t> drift_adjust_;
  int64_t max_interval_cycles_;

  BENCHMARK_DISALLOW_COPY_AND_ASSIGN(WallTimeImp);
};

WallTime WallTimeImp::Now() {
  WallTime now = 0.0;
  WallTime result = 0.0;
  int64_t ct = 0;
  uint32_t top_bits = 0;
  do {
    ct = cycleclock::Now();
    int64_t cycle_delta = ct - base_cycletime_;
    result = base_walltime_ + cycle_delta * seconds_per_cycle_;

    top_bits = static_cast<uint32_t>(uint64_t(ct) >> 32);
    // Recompute drift no more often than every 2^32 cycles.
    // I.e., @2GHz, ~ every two seconds
    if (top_bits == last_adjust_time_) {  // don't need to recompute drift
      return result + GetDrift();
    }

    now = Slow();
  } while (cycleclock::Now() - ct > max_interval_cycles_);
  // We are now sure that "now" and "result" were produced within
  // kMaxErrorInterval of one another.

  SetDrift(now - result);
  last_adjust_time_ = top_bits;
  return now;
}

WallTimeImp::WallTimeImp()
    : base_walltime_(0.0), base_cycletime_(0),
      cycles_per_second_(0), seconds_per_cycle_(0.0),
      last_adjust_time_(0), drift_adjust_(0),
      max_interval_cycles_(0) {
  cycles_per_second_ = static_cast<int64_t>(CyclesPerSecond());
  CHECK(cycles_per_second_ != 0);
  seconds_per_cycle_ = 1.0 / cycles_per_second_;
  max_interval_cycles_ =
      static_cast<int64_t>(cycles_per_second_ * kMaxErrorInterval);
  do {
    base_cycletime_ = cycleclock::Now();
    base_walltime_ = Slow();
  } while (cycleclock::Now() - base_cycletime_ > max_interval_cycles_);
  // We are now sure that "base_walltime" and "base_cycletime" were produced
  // within kMaxErrorInterval of one another.

  SetDrift(0.0);
  last_adjust_time_ = static_cast<uint32_t>(uint64_t(base_cycletime_) >> 32);
}

} // end anonymous namespace

WallTime Now()
{
    static WallTimeImp& imp = WallTimeImp::GetWallTimeImp();
    return imp.Now();
}

std::string Print(WallTime time, const char* format, bool local,
                  int* remainder_us) {
  char storage[32];
  struct tm split;
  double subsecond;
  if (!SplitTimezone(time, local, &split, &subsecond)) {
    snprintf(storage, sizeof(storage), "Invalid time: %f", time);
  } else {
    if (remainder_us != NULL) {
      *remainder_us = static_cast<int>((subsecond * 1000000) + 0.5);
      if (*remainder_us > 999999) *remainder_us = 999999;
      if (*remainder_us < 0) *remainder_us = 0;
    }
    strftime(storage, sizeof(storage), format, &split);
  }
  return std::string(storage);
}

}  // end namespace walltime
}  // end namespace benchmark
<|endoftext|>


<commit_before>
//
// wasm intepreter for asm2wasm output, in a js environment. receives asm.js,
// generates a runnable module suitable as a polyfill.
//
// this polyfills an emscripten --separate-asm *.asm.js file, as a drop-in
// replacement. it writes the wasm module to Module.asm, and sets Module.buffer.
//

#include <emscripten.h>

#include "asm2wasm.h"
#include "wasm-interpreter.h"

using namespace cashew;
using namespace wasm;

// global singletons
Asm2WasmBuilder* asm2wasm = nullptr;
ModuleInstance* instance = nullptr;
Module* module = nullptr;

// receives asm.js code, parses into wasm and returns an instance handle.
// this creates a module, an external interface, a builder, and a module instance,
// all of which are then the responsibility of the caller to free.
// note: this modifies the input.
extern "C" void EMSCRIPTEN_KEEPALIVE load_asm(char *input) {
  assert(instance == nullptr); // singleton

  // emcc --separate-asm modules look like
  //
  //    Module["asm"] = (function(global, env, buffer) {
  //      ..
  //    });
  //
  // we need to clean that up.
  size_t num = strlen(input);
  assert(*input == 'M');
  while (*input != 'f') {
    input++;
    num--;
  }
  char *end = input + num - 1;
  while (*end != '}') {
    *end = 0;
    end--;
  }

  int debug = 0;

  if (debug) std::cerr << "parsing...\n";
  cashew::Parser<Ref, DotZeroValueBuilder> builder;
  Ref asmjs = builder.parseToplevel(input);

  module = new Module();

  if (debug) std::cerr << "wasming...\n";
  asm2wasm = new Asm2WasmBuilder(*module);
  asm2wasm->processAsm(asmjs);

  if (debug) std::cerr << "optimizing...\n";
  asm2wasm->optimize();

  //std::cerr << *module << '\n';

  if (debug) std::cerr << "generating exports...\n";
  EM_ASM({
    Module['asmExports'] = {};
  });
  for (auto& curr : module->exports) {
    EM_ASM_({
      var name = Pointer_stringify($0);
      Module['asmExports'][name] = function() {
        Module['tempArguments'] = Array.prototype.slice.call(arguments);
        return Module['_call_from_js']($0);
      };
    }, curr.name.str);
  }

  if (debug) std::cerr << "creating instance...\n";

  struct JSExternalInterface : ModuleInstance::ExternalInterface {
    Literal callImport(Import *import, ModuleInstance::LiteralList& arguments) override {
      EM_ASM({
        Module['tempArguments'] = [];
      });
      for (auto& argument : arguments) {
        if (argument.type == i32) {
          EM_ASM_({ Module['tempArguments'].push($0) }, argument.geti32());
        } else if (argument.type == f64) {
          EM_ASM_({ Module['tempArguments'].push($0) }, argument.getf64());
        } else {
          abort();
        }
      }
      return Literal(EM_ASM_DOUBLE({
        var mod = Pointer_stringify($0);
        var base = Pointer_stringify($1);
        var tempArguments = Module['tempArguments'];
        Module['tempArguments'] = null;
        var lookup = Module['lookupImport'](mod, base);
        return lookup.apply(null, tempArguments);
      }, import->module.str, import->base.str));
    }

    Literal load(Load* load, Literal ptr) override {
      size_t addr = ptr.geti32();
      assert(load->align == load->bytes);
      if (!load->float_) {
        if (load->bytes == 1) {
          if (load->signed_) {
            return Literal(EM_ASM_INT({ return Module['info'].parent['HEAP8'][$0] }, addr));
          } else {
            return Literal(EM_ASM_INT({ return Module['info'].parent['HEAPU8'][$0] }, addr));
          }
        } else if (load->bytes == 2) {
          if (load->signed_) {
            return Literal(EM_ASM_INT({ return Module['info'].parent['HEAP16'][$0] }, addr));
          } else {
            return Literal(EM_ASM_INT({ return Module['info'].parent['HEAPU16'][$0] }, addr));
          }
        } else if (load->bytes == 4) {
          if (load->signed_) {
            return Literal(EM_ASM_INT({ return Module['info'].parent['HEAP32'][$0] }, addr));
          } else {
            return Literal(EM_ASM_INT({ return Module['info'].parent['HEAPU32'][$0] }, addr));
          }
        }
        abort();
      } else {
        if (load->bytes == 4) {
          return Literal(EM_ASM_DOUBLE({ return Module['info'].parent['HEAPF32'][$0] }, addr)); // XXX expands into double
        } else if (load->bytes == 8) {
          return Literal(EM_ASM_DOUBLE({ return Module['info'].parent['HEAPF64'][$0] }, addr));
        }
        abort();
      }
    }

    void store(Store* store, Literal ptr, Literal value) override {
      size_t addr = ptr.geti32();
      assert(store->align == store->bytes);
      if (!store->float_) {
        if (store->bytes == 1) {
          EM_ASM_INT({ Module['info'].parent['HEAP8'][$0] = $1 }, addr, value.geti32());
        } else if (store->bytes == 2) {
          EM_ASM_INT({ Module['info'].parent['HEAP16'][$0] = $1 }, addr, value.geti32());
        } else if (store->bytes == 4) {
          EM_ASM_INT({ Module['info'].parent['HEAP32'][$0] = $1 }, addr, value.geti32());
        } else {
          abort();
        }
      } else {
        if (store->bytes == 4) {
          EM_ASM_DOUBLE({ Module['info'].parent['HEAPF32'][$0] = $1 }, addr, value.getf64());
        } else if (store->bytes == 8) {
          EM_ASM_DOUBLE({ Module['info'].parent['HEAPF64'][$0] = $1 }, addr, value.getf64());
        } else {
          abort();
        }
      }
    }
  };

  instance = new ModuleInstance(*module, new JSExternalInterface());
}

// Ready the provided imported globals, copying them to their mapped locations.
extern "C" void EMSCRIPTEN_KEEPALIVE load_mapped_globals() {
  for (auto& pair : asm2wasm->mappedGlobals) {
    auto name = pair.first;
    auto& global = pair.second;
    if (!global.import) continue; // non-imports are initialized to zero in the typed array anyhow, so nothing to do here
    double value = EM_ASM_DOUBLE({ return Module['lookupImport'](Pointer_stringify($0), Pointer_stringify($1)) }, global.module.str, global.base.str);
    unsigned address = global.address;
    switch (global.type) {
      case i32: EM_ASM_({ Module['info'].parent['HEAP32'][$0] = $1 }, address, value); break;
      case f32: EM_ASM_({ Module['info'].parent['HEAPF32'][$0] = $1 }, address, value); break;
      case f64: EM_ASM_({ Module['info'].parent['HEAPF64'][$0] = $1 }, address, value); break;
      default: abort();
    }
  }
}

// Does a call from js into an export of the module.
extern "C" double EMSCRIPTEN_KEEPALIVE call_from_js(const char *target) {
  IString name(target);
  assert(instance->functions.find(name) != instance->functions.end());
  Function *function = instance->functions[name];
  size_t seen = EM_ASM_INT_V({ return Module['tempArguments'].length });
  size_t actual = function->params.size();
  ModuleInstance::LiteralList arguments;
  for (size_t i = 0; i < actual; i++) {
    WasmType type = function->params[i].type;
    // add the parameter, with a zero value if JS did not provide it.
    if (type == i32) {
      arguments.push_back(Literal(i < seen ? EM_ASM_INT({ return Module['tempArguments'][$0] }, i) : (int32_t)0));
    } else if (type == f32) {
      arguments.push_back(Literal(i < seen ? (float)EM_ASM_DOUBLE({ return Module['tempArguments'][$0] }, i) : (float)0.0));
    } else if (type == f64) {
      arguments.push_back(Literal(i < seen ? EM_ASM_DOUBLE({ return Module['tempArguments'][$0] }, i) : (double)0.0));
    } else {
      abort();
    }
  }

  Literal ret = instance->callFunction(name, arguments);
  if (ret.type == none) return 0;
  if (ret.type == i32) return ret.i32;
  if (ret.type == f32) return ret.f32;
  if (ret.type == f64) return ret.f64;
  abort();
}

<commit_msg>fix import call return type handling<commit_after>
//
// wasm intepreter for asm2wasm output, in a js environment. receives asm.js,
// generates a runnable module suitable as a polyfill.
//
// this polyfills an emscripten --separate-asm *.asm.js file, as a drop-in
// replacement. it writes the wasm module to Module.asm, and sets Module.buffer.
//

#include <emscripten.h>

#include "asm2wasm.h"
#include "wasm-interpreter.h"

using namespace cashew;
using namespace wasm;

// global singletons
Asm2WasmBuilder* asm2wasm = nullptr;
ModuleInstance* instance = nullptr;
Module* module = nullptr;

// receives asm.js code, parses into wasm and returns an instance handle.
// this creates a module, an external interface, a builder, and a module instance,
// all of which are then the responsibility of the caller to free.
// note: this modifies the input.
extern "C" void EMSCRIPTEN_KEEPALIVE load_asm(char *input) {
  assert(instance == nullptr); // singleton

  // emcc --separate-asm modules look like
  //
  //    Module["asm"] = (function(global, env, buffer) {
  //      ..
  //    });
  //
  // we need to clean that up.
  size_t num = strlen(input);
  assert(*input == 'M');
  while (*input != 'f') {
    input++;
    num--;
  }
  char *end = input + num - 1;
  while (*end != '}') {
    *end = 0;
    end--;
  }

  int debug = 0;

  if (debug) std::cerr << "parsing...\n";
  cashew::Parser<Ref, DotZeroValueBuilder> builder;
  Ref asmjs = builder.parseToplevel(input);

  module = new Module();

  if (debug) std::cerr << "wasming...\n";
  asm2wasm = new Asm2WasmBuilder(*module);
  asm2wasm->processAsm(asmjs);

  if (debug) std::cerr << "optimizing...\n";
  asm2wasm->optimize();

  //std::cerr << *module << '\n';

  if (debug) std::cerr << "generating exports...\n";
  EM_ASM({
    Module['asmExports'] = {};
  });
  for (auto& curr : module->exports) {
    EM_ASM_({
      var name = Pointer_stringify($0);
      Module['asmExports'][name] = function() {
        Module['tempArguments'] = Array.prototype.slice.call(arguments);
        return Module['_call_from_js']($0);
      };
    }, curr.name.str);
  }

  if (debug) std::cerr << "creating instance...\n";

  struct JSExternalInterface : ModuleInstance::ExternalInterface {
    Literal callImport(Import *import, ModuleInstance::LiteralList& arguments) override {
      EM_ASM({
        Module['tempArguments'] = [];
      });
      for (auto& argument : arguments) {
        if (argument.type == i32) {
          EM_ASM_({ Module['tempArguments'].push($0) }, argument.geti32());
        } else if (argument.type == f64) {
          EM_ASM_({ Module['tempArguments'].push($0) }, argument.getf64());
        } else {
          abort();
        }
      }
      double ret = EM_ASM_DOUBLE({
        var mod = Pointer_stringify($0);
        var base = Pointer_stringify($1);
        var tempArguments = Module['tempArguments'];
        Module['tempArguments'] = null;
        var lookup = Module['lookupImport'](mod, base);
        return lookup.apply(null, tempArguments);
      }, import->module.str, import->base.str);
      switch (import->type.result) {
        case none: return Literal(0);
        case i32: return Literal((int32_t)ret);
        case f32: return Literal((float)ret);
        case f64: return Literal((double)ret);
        default: abort();
      }
    }

    Literal load(Load* load, Literal ptr) override {
      size_t addr = ptr.geti32();
      assert(load->align == load->bytes);
      if (!load->float_) {
        if (load->bytes == 1) {
          if (load->signed_) {
            return Literal(EM_ASM_INT({ return Module['info'].parent['HEAP8'][$0] }, addr));
          } else {
            return Literal(EM_ASM_INT({ return Module['info'].parent['HEAPU8'][$0] }, addr));
          }
        } else if (load->bytes == 2) {
          if (load->signed_) {
            return Literal(EM_ASM_INT({ return Module['info'].parent['HEAP16'][$0] }, addr));
          } else {
            return Literal(EM_ASM_INT({ return Module['info'].parent['HEAPU16'][$0] }, addr));
          }
        } else if (load->bytes == 4) {
          if (load->signed_) {
            return Literal(EM_ASM_INT({ return Module['info'].parent['HEAP32'][$0] }, addr));
          } else {
            return Literal(EM_ASM_INT({ return Module['info'].parent['HEAPU32'][$0] }, addr));
          }
        }
        abort();
      } else {
        if (load->bytes == 4) {
          return Literal(EM_ASM_DOUBLE({ return Module['info'].parent['HEAPF32'][$0] }, addr)); // XXX expands into double
        } else if (load->bytes == 8) {
          return Literal(EM_ASM_DOUBLE({ return Module['info'].parent['HEAPF64'][$0] }, addr));
        }
        abort();
      }
    }

    void store(Store* store, Literal ptr, Literal value) override {
      size_t addr = ptr.geti32();
      assert(store->align == store->bytes);
      if (!store->float_) {
        if (store->bytes == 1) {
          EM_ASM_INT({ Module['info'].parent['HEAP8'][$0] = $1 }, addr, value.geti32());
        } else if (store->bytes == 2) {
          EM_ASM_INT({ Module['info'].parent['HEAP16'][$0] = $1 }, addr, value.geti32());
        } else if (store->bytes == 4) {
          EM_ASM_INT({ Module['info'].parent['HEAP32'][$0] = $1 }, addr, value.geti32());
        } else {
          abort();
        }
      } else {
        if (store->bytes == 4) {
          EM_ASM_DOUBLE({ Module['info'].parent['HEAPF32'][$0] = $1 }, addr, value.getf64());
        } else if (store->bytes == 8) {
          EM_ASM_DOUBLE({ Module['info'].parent['HEAPF64'][$0] = $1 }, addr, value.getf64());
        } else {
          abort();
        }
      }
    }
  };

  instance = new ModuleInstance(*module, new JSExternalInterface());
}

// Ready the provided imported globals, copying them to their mapped locations.
extern "C" void EMSCRIPTEN_KEEPALIVE load_mapped_globals() {
  for (auto& pair : asm2wasm->mappedGlobals) {
    auto name = pair.first;
    auto& global = pair.second;
    if (!global.import) continue; // non-imports are initialized to zero in the typed array anyhow, so nothing to do here
    double value = EM_ASM_DOUBLE({ return Module['lookupImport'](Pointer_stringify($0), Pointer_stringify($1)) }, global.module.str, global.base.str);
    unsigned address = global.address;
    switch (global.type) {
      case i32: EM_ASM_({ Module['info'].parent['HEAP32'][$0] = $1 }, address, value); break;
      case f32: EM_ASM_({ Module['info'].parent['HEAPF32'][$0] = $1 }, address, value); break;
      case f64: EM_ASM_({ Module['info'].parent['HEAPF64'][$0] = $1 }, address, value); break;
      default: abort();
    }
  }
}

// Does a call from js into an export of the module.
extern "C" double EMSCRIPTEN_KEEPALIVE call_from_js(const char *target) {
  IString name(target);
  assert(instance->functions.find(name) != instance->functions.end());
  Function *function = instance->functions[name];
  size_t seen = EM_ASM_INT_V({ return Module['tempArguments'].length });
  size_t actual = function->params.size();
  ModuleInstance::LiteralList arguments;
  for (size_t i = 0; i < actual; i++) {
    WasmType type = function->params[i].type;
    // add the parameter, with a zero value if JS did not provide it.
    if (type == i32) {
      arguments.push_back(Literal(i < seen ? EM_ASM_INT({ return Module['tempArguments'][$0] }, i) : (int32_t)0));
    } else if (type == f32) {
      arguments.push_back(Literal(i < seen ? (float)EM_ASM_DOUBLE({ return Module['tempArguments'][$0] }, i) : (float)0.0));
    } else if (type == f64) {
      arguments.push_back(Literal(i < seen ? EM_ASM_DOUBLE({ return Module['tempArguments'][$0] }, i) : (double)0.0));
    } else {
      abort();
    }
  }

  Literal ret = instance->callFunction(name, arguments);
  if (ret.type == none) return 0;
  if (ret.type == i32) return ret.i32;
  if (ret.type == f32) return ret.f32;
  if (ret.type == f64) return ret.f64;
  abort();
}

<|endoftext|>


<commit_before>// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab
/*
 * Ceph - scalable distributed file system
 *
 * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
 *
 * This is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License version 2.1, as published by the Free Software
 * Foundation.  See file COPYING.
 *
 */

#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>

#include <sys/stat.h>
#include <iostream>
#include <string>
using namespace std;

#include "config.h"

#include "mon/MonMap.h"
#include "mds/MDS.h"
#include "osd/OSDMap.h"

#include "msg/SimpleMessenger.h"

#include "common/Timer.h"
#include "common/common_init.h"

#include "mon/MonClient.h"

#include "osdc/Objecter.h"

#include "messages/MOSDGetMap.h"
#include "messages/MClientMount.h"
#include "messages/MClientMountAck.h"

class RadosClient : public Dispatcher
{
  MonMap monmap;
  OSDMap osdmap;
  Messenger *messenger;
  MonClient *mc;
  SimpleMessenger rank;

  bool _dispatch(Message *m);
  bool dispatch_impl(Message *m);

  Objecter *objecter;

  Mutex lock;
  Cond cond;

 class C_WriteAck : public Context {
    object_t oid;
    loff_t start;
    size_t *length;
    Cond *pcond;
  public:
    tid_t tid;
    C_WriteAck(object_t o, loff_t s, size_t *l, Cond *cond) : oid(o), start(s), length(l), pcond(cond) {}
    void finish(int r) {
      if (pcond) {
        *length = r;
        pcond->Signal();
      }
    }
  };
  class C_WriteCommit : public Context {
    object_t oid;
    loff_t start;
    size_t *length;
    Cond *pcond;
  public:
    tid_t tid;
    C_WriteCommit(object_t o, loff_t s, size_t *l, Cond *cond) : oid(o), start(s), length(l), pcond(cond) {}
    void finish(int r) {
      if (pcond) {
        *length = r;
        pcond->Signal();
      }
    }
  };
  class C_ExecCommit : public Context {
    object_t oid;
    loff_t start;
    size_t *length;
    Cond *pcond;
  public:
    tid_t tid;
    C_ExecCommit(object_t o, loff_t s, size_t *l, Cond *cond) : oid(o), start(s), length(l), pcond(cond) {}
    void finish(int r) {
      if (pcond) {
        *length = r;
        pcond->Signal();
      }
    }
  };
  class C_ReadCommit : public Context {
    object_t oid;
    loff_t start;
    size_t *length;
    bufferlist *bl;
    Cond *pcond;
  public:
    tid_t tid;
    C_ReadCommit(object_t o, loff_t s, size_t *l, bufferlist *b, Cond *cond) : oid(o), start(s), length(l), bl(b),
        pcond(cond) {}
    void finish(int r) {
      *length = r;
      if (pcond)
        pcond->Signal();
    }
  };

public:
  RadosClient() : messenger(NULL), mc(NULL), lock("c3") {}
  ~RadosClient();
  bool init();

  int lookup_pool(const char *name) {
    return osdmap.lookup_pg_pool_name(name);
  }

  int write(int pool, object_t& oid, const char *buf, off_t off, size_t len);
  int read(int pool, object_t& oid, char *buf, off_t off, size_t len);

  int exec(int pool, object_t& oid, const char *cls, const char *method, const char *inbuf, size_t in_len, char *buf, size_t out_len);
};

bool RadosClient::init()
{
  mc = new MonClient(&monmap, NULL);

  // get monmap
  if (!mc->get_monmap())
    return false;

  rank.bind();
  cout << "starting c3." << g_conf.id
       << " at " << rank.get_rank_addr()
       << " fsid " << monmap.get_fsid()
       << std::endl;

  messenger = rank.register_entity(entity_name_t::CLIENT(-1));
  assert_warn(messenger);
  if (!messenger)
    return false;

  mc->set_messenger(messenger);

  rank.set_policy(entity_name_t::TYPE_MON, SimpleMessenger::Policy::lossy_fail_after(1.0));
  rank.set_policy(entity_name_t::TYPE_MDS, SimpleMessenger::Policy::lossless());
  rank.set_policy(entity_name_t::TYPE_OSD, SimpleMessenger::Policy::lossless());
  rank.set_policy(entity_name_t::TYPE_CLIENT, SimpleMessenger::Policy::lossless());  // mds does its own timeout/markdown

  rank.start(1);

  mc->link_dispatcher(this);

  objecter = new Objecter(messenger, &monmap, &osdmap, lock);
  if (!objecter)
    return false;

  mc->mount(g_conf.client_mount_timeout);

  lock.Lock();

  objecter->set_client_incarnation(0);
  objecter->init();

  while (osdmap.get_epoch() == 0) {
    dout(0) << "waiting for osdmap" << dendl;
    cond.Wait(lock);
  }
  lock.Unlock();

  dout(0) << "init done" << dendl;

  return true;
}

RadosClient::~RadosClient()
{
  if (mc)
    delete mc;
  if (messenger)
    messenger->shutdown();
}

bool RadosClient::dispatch_impl(Message *m)
{
  bool ret;

  // verify protocol version
  if (m->get_orig_source().is_mds() &&
      m->get_header().mds_protocol != CEPH_MDS_PROTOCOL) {
    dout(0) << "mds protocol v " << (int)m->get_header().mds_protocol << " != my " << CEPH_MDS_PROTOCOL
	    << " from " << m->get_orig_source_inst() << " " << *m << dendl;
    delete m;
    return true;
  }

  if (m->get_header().mdsc_protocol != CEPH_MDSC_PROTOCOL) {
    dout(0) << "mdsc protocol v " << (int)m->get_header().mdsc_protocol << " != my " << CEPH_MDSC_PROTOCOL
	    << " from " << m->get_orig_source_inst() << " " << *m << dendl;
    delete m;
    return true;
  }
  if (m->get_orig_source().is_mon() &&
      m->get_header().monc_protocol != CEPH_MONC_PROTOCOL) {
    dout(0) << "monc protocol v " << (int)m->get_header().monc_protocol << " != my " << CEPH_MONC_PROTOCOL
	    << " from " << m->get_orig_source_inst() << " " << *m << dendl;
    delete m;
    return true;
  }
  if (m->get_orig_source().is_osd() &&
      m->get_header().osdc_protocol != CEPH_OSDC_PROTOCOL) {
    dout(0) << "osdc protocol v " << (int)m->get_header().osdc_protocol << " != my " << CEPH_OSDC_PROTOCOL
	    << " from " << m->get_orig_source_inst() << " " << *m << dendl;
    delete m;
    return true;
  }

  lock.Lock();
  ret = _dispatch(m);
  lock.Unlock();

  return ret;
}

bool RadosClient::_dispatch(Message *m)
{
  switch (m->get_type()) {
  // OSD
  case CEPH_MSG_OSD_OPREPLY:
    objecter->handle_osd_op_reply((class MOSDOpReply*)m);
    break;
  case CEPH_MSG_OSD_MAP:
    objecter->handle_osd_map((MOSDMap*)m);
    cond.Signal();
    break;
  case CEPH_MSG_MDS_MAP:
    break;

  default:
    return false;
  }

  return true;
}

int RadosClient::write(int pool, object_t& oid, const char *buf, off_t off, size_t len)
{
  SnapContext snapc;
  bufferlist bl;
  utime_t ut = g_clock.now();

  Mutex lock("RadosClient::write");
  Cond write_wait;
  bl.append(&buf[off], len);

  C_WriteAck *onack = new C_WriteAck(oid, off, &len, &write_wait);
  C_WriteCommit *oncommit = new C_WriteCommit(oid, off, &len, NULL);

  ceph_object_layout layout = objecter->osdmap->make_object_layout(oid, pool);

  dout(0) << "going to write" << dendl;

  lock.Lock();

  objecter->write(oid, layout,
	      off, len, snapc, bl, ut, 0,
              onack, oncommit);

  dout(0) << "after write call" << dendl;

  write_wait.Wait(lock);

  lock.Unlock();

  return len;
}

int RadosClient::exec(int pool, object_t& oid, const char *cls, const char *method, const char *inbuf, size_t in_len, char *buf, size_t out_len)
{
  SnapContext snapc;
  utime_t ut = g_clock.now();

  Mutex lock("RadosClient::exec");
  Cond exec_wait;

  C_ExecCommit *oncommit = new C_ExecCommit(oid, 0, &out_len, &exec_wait);

  ceph_object_layout layout = objecter->osdmap->make_object_layout(oid, pool);

  dout(0) << "going to exec" << dendl;

  lock.Lock();

  ObjectRead rd;
  bufferlist inbl, outbl;

  inbl.append(inbuf, in_len);
  rd.rdcall(cls, method, inbl);
  objecter->read(oid, layout, rd, CEPH_NOSNAP, &outbl, 0, oncommit);

  dout(0) << "after rdcall got " << outbl.length() << " bytes" << dendl;

  exec_wait.Wait(lock);

  lock.Unlock();

  if (out_len)
    memcpy(buf, outbl.c_str(), out_len);

  return out_len;
}

int RadosClient::read(int pool, object_t& oid, char *buf, off_t off, size_t len)
{
  SnapContext snapc;
  bufferlist bl;
  Mutex lock("RadosClient::read");
  Cond read_wait;

  C_ReadCommit *oncommit = new C_ReadCommit(oid, off, &len, &bl, &read_wait);

  ceph_object_layout layout = objecter->osdmap->make_object_layout(oid, pool);

  dout(0) << "going to read" << dendl;

  lock.Lock();

  objecter->read(oid, layout,
	      off, len, CEPH_NOSNAP, &bl, 0,
              oncommit);

  dout(0) << "after read call" << dendl;
  read_wait.Wait(lock);

  lock.Unlock();

  if (len)
    memcpy(buf, bl.c_str(), len);

  return len;
}

// ---------------------------------------------

static void __rados_init(int argc, const char *argv[])
{
  vector<const char*> args;

  if (argc && argv) {
    argv_to_vec(argc, argv, args);
    env_to_vec(args);
  }
  common_init(args, "librados", false);

  if (g_conf.clock_tare) g_clock.tare();
}

static Mutex rados_init_mutex("rados_init");
static int rados_initialized = 0;

static RadosClient *radosp;

#include "include/librados.h"

extern "C" int rados_initialize(int argc, const char **argv)
{
  int ret = 0;
  rados_init_mutex.Lock();

  if (!rados_initialized) {
    __rados_init(argc, argv);
    radosp = new RadosClient;

    if (!radosp) {
      dout(0) <<  "radosp is NULL" << dendl;
      ret = ENOMEM;
      goto out;
    }
    radosp->init();
  }
  ++rados_initialized;

out:
  rados_init_mutex.Unlock();
  return ret;
}

extern "C" void rados_deinitialize()
{
  rados_init_mutex.Lock();
  --rados_initialized;

  if (!rados_initialized)
    delete radosp;

  radosp = NULL;

  rados_init_mutex.Unlock();
}

extern "C" int rados_open_pool(const char *name, rados_pool_t *pool)
{
  int poolid = radosp->lookup_pool(name);
  if (poolid >= 0) {
    *pool = poolid;
    return 0;
  }
  return poolid;
}

extern "C" int rados_close_pool(rados_pool_t pool)
{
  return 0;
}

extern "C" int rados_write(rados_pool_t pool, ceph_object *o, const char *buf, off_t off, size_t len)
{
  object_t oid(*o);
  return radosp->write(pool, oid, buf, off, len);
}

extern "C" int rados_read(rados_pool_t pool, ceph_object *o, char *buf, off_t off, size_t len)
{
  object_t oid(*o);
  return radosp->read(pool, oid, buf, off, len);
}

extern "C" int rados_exec(rados_pool_t pool, ceph_object *o, const char *cls, const char *method,
                         const char *inbuf, size_t in_len, char *buf, size_t out_len)
{
  object_t oid(*o);
  return radosp->exec(pool, oid, cls, method, inbuf, in_len, buf, out_len);
}

<commit_msg>librados: use generic C_SafeCond and wait safely for acks<commit_after>// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab
/*
 * Ceph - scalable distributed file system
 *
 * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
 *
 * This is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License version 2.1, as published by the Free Software
 * Foundation.  See file COPYING.
 *
 */

#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>

#include <sys/stat.h>
#include <iostream>
#include <string>
using namespace std;

#include "config.h"

#include "mon/MonMap.h"
#include "mds/MDS.h"
#include "osd/OSDMap.h"

#include "msg/SimpleMessenger.h"

#include "common/Timer.h"
#include "common/common_init.h"

#include "mon/MonClient.h"

#include "osdc/Objecter.h"

#include "messages/MOSDGetMap.h"
#include "messages/MClientMount.h"
#include "messages/MClientMountAck.h"

class RadosClient : public Dispatcher
{
  MonMap monmap;
  OSDMap osdmap;
  Messenger *messenger;
  MonClient *mc;
  SimpleMessenger rank;

  bool _dispatch(Message *m);
  bool dispatch_impl(Message *m);

  Objecter *objecter;

  Mutex lock;
  Cond cond;

public:
  RadosClient() : messenger(NULL), mc(NULL), lock("c3") {}
  ~RadosClient();
  bool init();

  int lookup_pool(const char *name) {
    return osdmap.lookup_pg_pool_name(name);
  }

  int write(int pool, object_t& oid, const char *buf, off_t off, size_t len);
  int read(int pool, object_t& oid, char *buf, off_t off, size_t len);

  int exec(int pool, object_t& oid, const char *cls, const char *method, const char *inbuf, size_t in_len, char *buf, size_t out_len);
};

bool RadosClient::init()
{
  mc = new MonClient(&monmap, NULL);

  // get monmap
  if (!mc->get_monmap())
    return false;

  rank.bind();
  cout << "starting c3." << g_conf.id
       << " at " << rank.get_rank_addr()
       << " fsid " << monmap.get_fsid()
       << std::endl;

  messenger = rank.register_entity(entity_name_t::CLIENT(-1));
  assert_warn(messenger);
  if (!messenger)
    return false;

  mc->set_messenger(messenger);

  rank.set_policy(entity_name_t::TYPE_MON, SimpleMessenger::Policy::lossy_fail_after(1.0));
  rank.set_policy(entity_name_t::TYPE_MDS, SimpleMessenger::Policy::lossless());
  rank.set_policy(entity_name_t::TYPE_OSD, SimpleMessenger::Policy::lossless());
  rank.set_policy(entity_name_t::TYPE_CLIENT, SimpleMessenger::Policy::lossless());  // mds does its own timeout/markdown

  rank.start(1);

  mc->link_dispatcher(this);

  objecter = new Objecter(messenger, &monmap, &osdmap, lock);
  if (!objecter)
    return false;

  mc->mount(g_conf.client_mount_timeout);

  lock.Lock();

  objecter->set_client_incarnation(0);
  objecter->init();

  while (osdmap.get_epoch() == 0) {
    dout(0) << "waiting for osdmap" << dendl;
    cond.Wait(lock);
  }
  lock.Unlock();

  dout(0) << "init done" << dendl;

  return true;
}

RadosClient::~RadosClient()
{
  if (mc)
    delete mc;
  if (messenger)
    messenger->shutdown();
}

bool RadosClient::dispatch_impl(Message *m)
{
  bool ret;

  // verify protocol version
  if (m->get_orig_source().is_mds() &&
      m->get_header().mds_protocol != CEPH_MDS_PROTOCOL) {
    dout(0) << "mds protocol v " << (int)m->get_header().mds_protocol << " != my " << CEPH_MDS_PROTOCOL
	    << " from " << m->get_orig_source_inst() << " " << *m << dendl;
    delete m;
    return true;
  }

  if (m->get_header().mdsc_protocol != CEPH_MDSC_PROTOCOL) {
    dout(0) << "mdsc protocol v " << (int)m->get_header().mdsc_protocol << " != my " << CEPH_MDSC_PROTOCOL
	    << " from " << m->get_orig_source_inst() << " " << *m << dendl;
    delete m;
    return true;
  }
  if (m->get_orig_source().is_mon() &&
      m->get_header().monc_protocol != CEPH_MONC_PROTOCOL) {
    dout(0) << "monc protocol v " << (int)m->get_header().monc_protocol << " != my " << CEPH_MONC_PROTOCOL
	    << " from " << m->get_orig_source_inst() << " " << *m << dendl;
    delete m;
    return true;
  }
  if (m->get_orig_source().is_osd() &&
      m->get_header().osdc_protocol != CEPH_OSDC_PROTOCOL) {
    dout(0) << "osdc protocol v " << (int)m->get_header().osdc_protocol << " != my " << CEPH_OSDC_PROTOCOL
	    << " from " << m->get_orig_source_inst() << " " << *m << dendl;
    delete m;
    return true;
  }

  lock.Lock();
  ret = _dispatch(m);
  lock.Unlock();

  return ret;
}

bool RadosClient::_dispatch(Message *m)
{
  switch (m->get_type()) {
  // OSD
  case CEPH_MSG_OSD_OPREPLY:
    objecter->handle_osd_op_reply((class MOSDOpReply*)m);
    break;
  case CEPH_MSG_OSD_MAP:
    objecter->handle_osd_map((MOSDMap*)m);
    cond.Signal();
    break;
  case CEPH_MSG_MDS_MAP:
    break;

  default:
    return false;
  }

  return true;
}

int RadosClient::write(int pool, object_t& oid, const char *buf, off_t off, size_t len)
{
  SnapContext snapc;
  bufferlist bl;
  utime_t ut = g_clock.now();

  Mutex lock("RadosClient::write");
  Cond cond;
  bool done;
  int r;
  Context *onack = new C_SafeCond(&lock, &cond, &done, &r);

  bl.append(&buf[off], len);

  ceph_object_layout layout = objecter->osdmap->make_object_layout(oid, pool);

  dout(0) << "going to write" << dendl;

  lock.Lock();
  objecter->write(oid, layout,
		  off, len, snapc, bl, ut, 0,
		  onack, NULL);
  while (!done)
    cond.Wait(lock);
  lock.Unlock();

  return len;
}

int RadosClient::exec(int pool, object_t& oid, const char *cls, const char *method, const char *inbuf, size_t in_len, char *buf, size_t out_len)
{
  SnapContext snapc;
  utime_t ut = g_clock.now();

  Mutex lock("RadosClient::rdcall");
  Cond cond;
  bool done;
  int r;
  Context *onack = new C_SafeCond(&lock, &cond, &done, &r);

  ceph_object_layout layout = objecter->osdmap->make_object_layout(oid, pool);

  lock.Lock();

  ObjectRead rd;
  bufferlist inbl, outbl;

  inbl.append(inbuf, in_len);
  rd.rdcall(cls, method, inbl);
  objecter->read(oid, layout, rd, CEPH_NOSNAP, &outbl, 0, onack);

  dout(0) << "after rdcall got " << outbl.length() << " bytes" << dendl;

  while (!done)
    cond.Wait(lock);

  lock.Unlock();

  if (out_len)
    memcpy(buf, outbl.c_str(), out_len);

  return out_len;
}

int RadosClient::read(int pool, object_t& oid, char *buf, off_t off, size_t len)
{
  SnapContext snapc;
  bufferlist bl;

  Mutex lock("RadosClient::rdcall");
  Cond cond;
  bool done;
  int r;
  Context *onack = new C_SafeCond(&lock, &cond, &done, &r);

  ceph_object_layout layout = objecter->osdmap->make_object_layout(oid, pool);

  dout(0) << "going to read" << dendl;

  lock.Lock();
  objecter->read(oid, layout,
	      off, len, CEPH_NOSNAP, &bl, 0,
              onack);
  while (!done)
    cond.Wait(lock);

  lock.Unlock();

  if (len)
    memcpy(buf, bl.c_str(), len);

  return len;
}

// ---------------------------------------------

static void __rados_init(int argc, const char *argv[])
{
  vector<const char*> args;

  if (argc && argv) {
    argv_to_vec(argc, argv, args);
    env_to_vec(args);
  }
  common_init(args, "librados", false);

  if (g_conf.clock_tare) g_clock.tare();
}

static Mutex rados_init_mutex("rados_init");
static int rados_initialized = 0;

static RadosClient *radosp;

#include "include/librados.h"

extern "C" int rados_initialize(int argc, const char **argv)
{
  int ret = 0;
  rados_init_mutex.Lock();

  if (!rados_initialized) {
    __rados_init(argc, argv);
    radosp = new RadosClient;

    if (!radosp) {
      dout(0) <<  "radosp is NULL" << dendl;
      ret = ENOMEM;
      goto out;
    }
    radosp->init();
  }
  ++rados_initialized;

out:
  rados_init_mutex.Unlock();
  return ret;
}

extern "C" void rados_deinitialize()
{
  rados_init_mutex.Lock();
  --rados_initialized;

  if (!rados_initialized)
    delete radosp;

  radosp = NULL;

  rados_init_mutex.Unlock();
}

extern "C" int rados_open_pool(const char *name, rados_pool_t *pool)
{
  int poolid = radosp->lookup_pool(name);
  if (poolid >= 0) {
    *pool = poolid;
    return 0;
  }
  return poolid;
}

extern "C" int rados_close_pool(rados_pool_t pool)
{
  return 0;
}

extern "C" int rados_write(rados_pool_t pool, ceph_object *o, const char *buf, off_t off, size_t len)
{
  object_t oid(*o);
  return radosp->write(pool, oid, buf, off, len);
}

extern "C" int rados_read(rados_pool_t pool, ceph_object *o, char *buf, off_t off, size_t len)
{
  object_t oid(*o);
  return radosp->read(pool, oid, buf, off, len);
}

extern "C" int rados_exec(rados_pool_t pool, ceph_object *o, const char *cls, const char *method,
                         const char *inbuf, size_t in_len, char *buf, size_t out_len)
{
  object_t oid(*o);
  return radosp->exec(pool, oid, cls, method, inbuf, in_len, buf, out_len);
}

<|endoftext|>


<commit_before>#include <logger.h>
#include <context.h>

#include <ll.h>

#include <lisp/ll.h>
#include <lisp/parser.h>

Logger* lisp_logger;
Logger* lisp_parser_logger;
Logger* lisp_atom_logger;

void init_lisp(Context* context)
{
    lisp_logger = new Logger("lisp");
    lisp_parser_logger = new Logger(lisp_logger, "parser");
    lisp_atom_logger = new Logger(lisp_parser_logger, "atom");

    // system functions
    add_symbol("eval", context->add_ll_fn(ll_eval));
    add_symbol("deval", context->add_ll_fn(ll_deval));
    add_symbol("nop", context->add_ll_fn(ll_nop));

    add_symbol("if", context->add_ll_fn(ll_cond));
    add_symbol("eq", context->add_ll_fn(ll_eq));

    add_symbol("resw", context->add_ll_fn(ll_resw));
    add_symbol("map", context->add_ll_fn(ll_map));
    add_symbol("exit", context->add_ll_fn(ll_exit));
    add_symbol("printi", context->add_ll_fn(ll_printi));
    add_symbol("printb", context->add_ll_fn(ll_printb));
    add_symbol("addi", context->add_ll_fn(ll_addi));

    // lisp macro
    add_symbol("quote", context->add_ll_fn(ll_quote));
    add_symbol("asm", context->add_ll_fn(ll_asm));
    add_symbol("declare", context->add_ll_fn(ll_declare));
    add_symbol("genfn", context->add_ll_fn(ll_gen_fn));
}

vword_t ll_gen_fn(Context* context, vword_t p)
{
    // get the two inputs
    vword_t p1 = context->read_word(p);
    p += VWORD_SIZE;
    vword_t p2 = context->read_word(p);
    p += VWORD_SIZE;

    SNode* a1 = new SNode(context, p1); // function
    SNode* a2 = new SNode(context, p2); // needed length

    if(a2->type != INTEGER)
    {
        printf("fn wrapper needs expected length\n");
        return p;
    }

    // collect applied arguments (ast) until the required length is reached
    size_t l = a2->integer;

    List<vword_t> list = List<vword_t>();

    vword_t pt = p; // working pointer (temporal pushs)
    vword_t ps = p;
    while(pt+l > ps)
    {
        // get snode
        vword_t p3 = context->read_word(p);
        p += VWORD_SIZE;
        SNode* sn = new SNode(context, p3);

        // parse it on stack
        pt -= lisp_parse_size(sn);
        size_t n = lisp_parse(context, pt, sn);

        // self-eval lists
        if(sn->type == LIST)
        {
            n += pt;
            pt = ll_eval(context, pt + VWORD_SIZE);
            n -= pt;
        }

        list.pushBack(pt); // pointer to start
        list.pushBack(n);  // length
    }

    // copy evaled arguments (reverse order)
    vbyte_t* buf = (vbyte_t*) malloc(l);

    vbyte_t* dest = buf;
    while(! list.isEmpty())
    {
        vword_t start = list.popFront();
        vword_t n = list.popFront();
        dest += context->read(start, n, dest);
    }

    p -= l;
    context->write(p, l, buf);

    free(buf);

    p -= lisp_parse_size(a1);
    lisp_parse(context, p, a1);

//    context->dump(p, 8);

    return ll_eval(context, p);
}

vword_t ll_declare(Context* context, vword_t p)
{
    static Logger* logger = new Logger(lisp_logger, "declare");

    static vword_t def_top = 0xA0000;

    vword_t pn = context->read_word(p);
    p += VWORD_SIZE;
    vword_t vn = context->read_word(p);
    p += VWORD_SIZE;

    SNode* name = new SNode(context, pn);
    SNode* value = new SNode(context, vn);

    if(name->type == SYMBOL)
    {
        if(resolve_symbol(name->string) == (vword_t)0)
        {
            def_top -= lisp_parse_size(value);
            size_t len = lisp_parse(context, def_top, value);

            // execute lists
            if(value->type == LIST)
                def_top = ll_eval(context, def_top);

            logger->log(linfo, "declared \'%s\': 0x%x, %d bytes", name->string, def_top, len);

            add_symbol(name->string, def_top);
//            context->dump(def_top, len/VWORD_SIZE);
        }
        else
            logger->log(lerror, "symbol \'%s\' already in use", name->string);
    }
    else
        logger->log(lerror, "first argument must be a symbol!");

    delete name;
    delete value;

    return p;
}

vword_t ll_asm(Context* context, vword_t p)
{
    vword_t n = context->read_word(p);

    SNode* ast = new SNode(context, n);

    size_t l = asm_parse(context, 0x2000, ast);

    context->write_word(p, 0x2000);
    p = ll_eval(context, p);

    return p;
}

vword_t ll_quote(Context* context, vword_t p)
{
    vword_t p1 = context->read_word(p);
    p += VWORD_SIZE;

    SNode* ast = new SNode(context, p1);

    p -= lisp_parse_size(ast);
    lisp_parse(context, p, ast);

    return p;
}

<commit_msg>fix length in genfn<commit_after>#include <logger.h>
#include <context.h>

#include <ll.h>

#include <lisp/ll.h>
#include <lisp/parser.h>

Logger* lisp_logger;
Logger* lisp_parser_logger;
Logger* lisp_atom_logger;

void init_lisp(Context* context)
{
    lisp_logger = new Logger("lisp");
    lisp_parser_logger = new Logger(lisp_logger, "parser");
    lisp_atom_logger = new Logger(lisp_parser_logger, "atom");

    // system functions
    add_symbol("eval", context->add_ll_fn(ll_eval));
    add_symbol("deval", context->add_ll_fn(ll_deval));
    add_symbol("nop", context->add_ll_fn(ll_nop));

    add_symbol("if", context->add_ll_fn(ll_cond));
    add_symbol("eq", context->add_ll_fn(ll_eq));

    add_symbol("resw", context->add_ll_fn(ll_resw));
    add_symbol("map", context->add_ll_fn(ll_map));
    add_symbol("exit", context->add_ll_fn(ll_exit));
    add_symbol("printi", context->add_ll_fn(ll_printi));
    add_symbol("printb", context->add_ll_fn(ll_printb));
    add_symbol("addi", context->add_ll_fn(ll_addi));

    // lisp macro
    add_symbol("quote", context->add_ll_fn(ll_quote));
    add_symbol("asm", context->add_ll_fn(ll_asm));
    add_symbol("declare", context->add_ll_fn(ll_declare));
    add_symbol("genfn", context->add_ll_fn(ll_gen_fn));
}

vword_t ll_gen_fn(Context* context, vword_t p)
{
    // get the two inputs
    vword_t p1 = context->read_word(p);
    p += VWORD_SIZE;
    vword_t p2 = context->read_word(p);
    p += VWORD_SIZE;

    SNode* a1 = new SNode(context, p1); // function
    SNode* a2 = new SNode(context, p2); // needed length

    if(a2->type != INTEGER)
    {
        printf("fn wrapper needs expected length\n");
        return p;
    }

    // collect applied arguments (ast) until the required length is reached
    size_t l = a2->integer;

    List<vword_t> list = List<vword_t>();

    vword_t pt = p; // working pointer (temporal pushs)
    size_t i = 0;
    while(i < l)
    {
        // get snode
        vword_t p3 = context->read_word(p);
        p += VWORD_SIZE;
        SNode* sn = new SNode(context, p3);

        // parse it on stack
        pt -= lisp_parse_size(sn);
        size_t n = lisp_parse(context, pt, sn);

        // self-eval lists
        if(sn->type == LIST)
        {
            pt = ll_eval(context, pt + VWORD_SIZE);
            n = VWORD_SIZE; // FIXME
        }

        list.pushBack(pt); // pointer to start
        list.pushBack(n);  // length

        i += n;
    }

//	context->dump(pt, l/VWORD_SIZE);

    // copy evaled arguments (reverse order)
    vbyte_t* buf = (vbyte_t*) malloc(l);

    vbyte_t* dest = buf;
    while(! list.isEmpty())
    {
        vword_t start = list.popFront();
        vword_t n = list.popFront();

//		printf("genfn: 0x%x, %d\n", start, n);
        dest += context->read(start, n, dest);
    }

    p -= l;
    context->write(p, l, buf);

    free(buf);

    p -= lisp_parse_size(a1);
    lisp_parse(context, p, a1);

//    context->dump(p, 8);

    return ll_eval(context, p);
}

vword_t ll_declare(Context* context, vword_t p)
{
    static Logger* logger = new Logger(lisp_logger, "declare");

    static vword_t def_top = 0xA0000;

    vword_t pn = context->read_word(p);
    p += VWORD_SIZE;
    vword_t vn = context->read_word(p);
    p += VWORD_SIZE;

    SNode* name = new SNode(context, pn);
    SNode* value = new SNode(context, vn);

    if(name->type == SYMBOL)
    {
        if(resolve_symbol(name->string) == (vword_t)0)
        {
            def_top -= lisp_parse_size(value);
            size_t len = lisp_parse(context, def_top, value);

            // execute lists
            if(value->type == LIST)
                def_top = ll_eval(context, def_top);

            logger->log(linfo, "declared \'%s\': 0x%x, %d bytes", name->string, def_top, len);

            add_symbol(name->string, def_top);
//            context->dump(def_top, len/VWORD_SIZE);
        }
        else
            logger->log(lerror, "symbol \'%s\' already in use", name->string);
    }
    else
        logger->log(lerror, "first argument must be a symbol!");

    delete name;
    delete value;

    return p;
}

vword_t ll_asm(Context* context, vword_t p)
{
    vword_t n = context->read_word(p);

    SNode* ast = new SNode(context, n);

    size_t l = asm_parse(context, 0x2000, ast);

    context->write_word(p, 0x2000);
    p = ll_eval(context, p);

    return p;
}

vword_t ll_quote(Context* context, vword_t p)
{
    vword_t p1 = context->read_word(p);
    p += VWORD_SIZE;

    SNode* ast = new SNode(context, p1);

    p -= lisp_parse_size(ast);
    lisp_parse(context, p, ast);

    return p;
}

<|endoftext|>


<commit_before>
// Bindings
#include "CPyCppyy.h"
#include "PyROOTPythonize.h"
#include "CPPInstance.h"
#include "ProxyWrappers.h"
#include "Converters.h"
#include "Utility.h"

// ROOT
#include "TClass.h"
#include "TTree.h"
#include "TBranch.h"
#include "TBranchElement.h"
#include "TBranchObject.h"
#include "TLeaf.h"
#include "TLeafElement.h"
#include "TLeafObject.h"
#include "TStreamerElement.h"
#include "TStreamerInfo.h"

using namespace CPyCppyy;

static TClass *GetClass(const CPPInstance *pyobj)
{
   return TClass::GetClass(Cppyy::GetFinalName(pyobj->ObjectIsA()).c_str());
}

// Allow access to branches/leaves as if they were data members
PyObject *GetAttr(const CPPInstance *self, PyObject *pyname)
{
   const char *name_possibly_alias = CPyCppyy_PyUnicode_AsString(pyname);
   if (!name_possibly_alias)
      return 0;

   // get hold of actual tree
   TTree *tree = (TTree *)GetClass(self)->DynamicCast(TTree::Class(), self->GetObject());

   if (!tree) {
      PyErr_SetString(PyExc_ReferenceError, "attempt to access a null-pointer");
      return 0;
   }

   // deal with possible aliasing
   const char *name = tree->GetAlias(name_possibly_alias);
   if (!name)
      name = name_possibly_alias;

   // search for branch first (typical for objects)
   TBranch *branch = tree->GetBranch(name);
   if (!branch) {
      // for benefit of naming of sub-branches, the actual name may have a trailing '.'
      branch = tree->GetBranch((std::string(name) + '.').c_str());
   }

   if (branch) {
      // found a branched object, wrap its address for the object it represents

      // for partial return of a split object
      if (branch->InheritsFrom(TBranchElement::Class())) {
         TBranchElement *be = (TBranchElement *)branch;
         if (be->GetCurrentClass() && (be->GetCurrentClass() != be->GetTargetClass()) && (0 <= be->GetID())) {
            Long_t offset = ((TStreamerElement *)be->GetInfo()->GetElements()->At(be->GetID()))->GetOffset();
            return BindCppObjectNoCast(be->GetObject() + offset, Cppyy::GetScope(be->GetCurrentClass()->GetName()));
         }
      }

      // for return of a full object
      if (branch->IsA() == TBranchElement::Class() || branch->IsA() == TBranchObject::Class()) {
         TClass *klass = TClass::GetClass(branch->GetClassName());
         if (klass && branch->GetAddress())
            return BindCppObjectNoCast(*(void **)branch->GetAddress(), Cppyy::GetScope(branch->GetClassName()));

         // try leaf, otherwise indicate failure by returning a typed null-object
         TObjArray *leaves = branch->GetListOfLeaves();
         if (klass && !tree->GetLeaf(name) && !(leaves->GetSize() && (leaves->First() == leaves->Last())))
            return BindCppObjectNoCast(NULL, Cppyy::GetScope(branch->GetClassName()));
      }
   }

   // if not, try leaf
   TLeaf *leaf = tree->GetLeaf(name);
   if (branch && !leaf) {
      leaf = branch->GetLeaf(name);
      if (!leaf) {
         TObjArray *leaves = branch->GetListOfLeaves();
         if (leaves->GetSize() && (leaves->First() == leaves->Last())) {
            // i.e., if unambiguously only this one
            leaf = (TLeaf *)leaves->At(0);
         }
      }
   }

   if (leaf) {
      // found a leaf, extract value and wrap
      if (1 < leaf->GetLenStatic() || leaf->GetLeafCount()) {
         // array types
         std::string typeName = leaf->GetTypeName();
         Converter *pcnv = CreateConverter(typeName + '*', leaf->GetNdata());

         void *address = 0;
         if (leaf->GetBranch())
            address = (void *)leaf->GetBranch()->GetAddress();
         if (!address)
            address = (void *)leaf->GetValuePointer();

         PyObject *value = pcnv->FromMemory(&address);
         delete pcnv;

         return value;
      } else if (leaf->GetValuePointer()) {
         // value types
         Converter *pcnv = CreateConverter(leaf->GetTypeName());
         PyObject *value = 0;
         if (leaf->IsA() == TLeafElement::Class() || leaf->IsA() == TLeafObject::Class())
            value = pcnv->FromMemory((void *)*(void **)leaf->GetValuePointer());
         else
            value = pcnv->FromMemory((void *)leaf->GetValuePointer());
         delete pcnv;

         return value;
      }
   }

   // confused
   PyErr_Format(PyExc_AttributeError, "\'%s\' object has no attribute \'%s\'", tree->IsA()->GetName(), name);
   return 0;
}

// Public function
PyObject *PyROOT::PythonizeTTree(PyObject *, PyObject *args)
{
   PyObject *pyclass = PyTuple_GetItem(args, 0);
   Utility::AddToClass(pyclass, "__getattr__", (PyCFunction)GetAttr, METH_O);
   Py_RETURN_NONE;
}
<commit_msg>Refactor code in implementation of TTree __getattr__<commit_after>
// Bindings
#include "CPyCppyy.h"
#include "PyROOTPythonize.h"
#include "CPPInstance.h"
#include "ProxyWrappers.h"
#include "Converters.h"
#include "Utility.h"

// ROOT
#include "TClass.h"
#include "TTree.h"
#include "TBranch.h"
#include "TBranchElement.h"
#include "TBranchObject.h"
#include "TLeaf.h"
#include "TLeafElement.h"
#include "TLeafObject.h"
#include "TStreamerElement.h"
#include "TStreamerInfo.h"

using namespace CPyCppyy;

static TClass *GetClass(const CPPInstance *pyobj)
{
   return TClass::GetClass(Cppyy::GetFinalName(pyobj->ObjectIsA()).c_str());
}

static TBranch *SearchForBranch(TTree *tree, const char *name)
{
   TBranch *branch = tree->GetBranch(name);
   if (!branch) {
      // for benefit of naming of sub-branches, the actual name may have a trailing '.'
      branch = tree->GetBranch((std::string(name) + '.').c_str());
   }
   return branch;
}

static TLeaf *SearchForLeaf(TTree *tree, const char *name, TBranch *branch)
{
   TLeaf *leaf = tree->GetLeaf(name);
   if (branch && !leaf) {
      leaf = branch->GetLeaf(name);
      if (!leaf) {
         TObjArray *leaves = branch->GetListOfLeaves();
         if (leaves->GetSize() && (leaves->First() == leaves->Last())) {
            // i.e., if unambiguously only this one
            leaf = (TLeaf *)leaves->At(0);
         }
      }
   }
   return leaf;
}

static PyObject *BindBranchToProxy(TTree *tree, const char *name, TBranch *branch)
{
   // for partial return of a split object
   if (branch->InheritsFrom(TBranchElement::Class())) {
      TBranchElement *be = (TBranchElement *)branch;
      if (be->GetCurrentClass() && (be->GetCurrentClass() != be->GetTargetClass()) && (0 <= be->GetID())) {
         Long_t offset = ((TStreamerElement *)be->GetInfo()->GetElements()->At(be->GetID()))->GetOffset();
         return BindCppObjectNoCast(be->GetObject() + offset, Cppyy::GetScope(be->GetCurrentClass()->GetName()));
      }
   }

   // for return of a full object
   if (branch->IsA() == TBranchElement::Class() || branch->IsA() == TBranchObject::Class()) {
      TClass *klass = TClass::GetClass(branch->GetClassName());
      if (klass && branch->GetAddress())
         return BindCppObjectNoCast(*(void **)branch->GetAddress(), Cppyy::GetScope(branch->GetClassName()));

      // try leaf, otherwise indicate failure by returning a typed null-object
      TObjArray *leaves = branch->GetListOfLeaves();
      if (klass && !tree->GetLeaf(name) && !(leaves->GetSize() && (leaves->First() == leaves->Last())))
         return BindCppObjectNoCast(nullptr, Cppyy::GetScope(branch->GetClassName()));
   }

   return nullptr;
}

static PyObject *WrapLeaf(TLeaf *leaf)
{
   if (1 < leaf->GetLenStatic() || leaf->GetLeafCount()) {
      // array types
      std::string typeName = leaf->GetTypeName();
      Converter *pcnv = CreateConverter(typeName + '*', leaf->GetNdata());

      void *address = 0;
      if (leaf->GetBranch())
         address = (void *)leaf->GetBranch()->GetAddress();
      if (!address)
         address = (void *)leaf->GetValuePointer();

      PyObject *value = pcnv->FromMemory(&address);
      delete pcnv;

      return value;
   } else if (leaf->GetValuePointer()) {
      // value types
      Converter *pcnv = CreateConverter(leaf->GetTypeName());
      PyObject *value = 0;
      if (leaf->IsA() == TLeafElement::Class() || leaf->IsA() == TLeafObject::Class())
         value = pcnv->FromMemory((void *)*(void **)leaf->GetValuePointer());
      else
         value = pcnv->FromMemory((void *)leaf->GetValuePointer());
      delete pcnv;

      return value;
   }

   return nullptr;
}

// Allow access to branches/leaves as if they were data members
PyObject *GetAttr(const CPPInstance *self, PyObject *pyname)
{
   const char *name_possibly_alias = CPyCppyy_PyUnicode_AsString(pyname);
   if (!name_possibly_alias)
      return 0;

   // get hold of actual tree
   TTree *tree = (TTree *)GetClass(self)->DynamicCast(TTree::Class(), self->GetObject());

   if (!tree) {
      PyErr_SetString(PyExc_ReferenceError, "attempt to access a null-pointer");
      return 0;
   }

   // deal with possible aliasing
   const char *name = tree->GetAlias(name_possibly_alias);
   if (!name)
      name = name_possibly_alias;

   // search for branch first (typical for objects)
   TBranch *branch = SearchForBranch(tree, name);

   if (branch) {
      // found a branched object, wrap its address for the object it represents
      auto proxy = BindBranchToProxy(tree, name, branch);
      if (proxy != nullptr)
         return proxy;
   }

   // if not, try leaf
   TLeaf *leaf = SearchForLeaf(tree, name, branch);

   if (leaf) {
      // found a leaf, extract value and wrap with a Python object according to its type
      auto wrapper = WrapLeaf(leaf);
      if (wrapper != nullptr)
         return wrapper;
   }

   // confused
   PyErr_Format(PyExc_AttributeError, "\'%s\' object has no attribute \'%s\'", tree->IsA()->GetName(), name);
   return 0;
}

// Public function
PyObject *PyROOT::PythonizeTTree(PyObject *, PyObject *args)
{
   PyObject *pyclass = PyTuple_GetItem(args, 0);
   Utility::AddToClass(pyclass, "__getattr__", (PyCFunction)GetAttr, METH_O);
   Py_RETURN_NONE;
}
<|endoftext|>