<commit_before>#include "opencv2/core/utility.hpp"
#include "opencv2/imgproc.hpp"
#include "opencv2/imgcodecs.hpp"
#include "opencv2/highgui.hpp"

#include <iostream>

using namespace cv;
using namespace std;

static void help()
{
    cout <<   "no help"      << endl;
}

const auto RED = cv::Scalar(0,0,255);
const auto PINK = cv::Scalar(230,130,255);
const auto BLUE = cv::Scalar(255,0,0);
const auto LIGHTBLUE = cv::Scalar(255,255,160);
const auto GREEN = cv::Scalar(0,255,0);

class GCApplication
{
public:
    enum{ NOT_SET = 0, IN_PROCESS = 1, SET = 2 };
    static const int radius = 2;
    static const int thickness = -1;

    void reset();
    void setImageAndWinName( const Mat& _image, const string& _winName );
    void showImage() const;
    void setImagePath(const string& _s) {imagePath = &_s;};
    string* getImagePath() {return &imagePath;};
private:
    void preprocessImage( const Mat& inImage, Mat& outImage );

    const string* winName;
    const string* imagePath;
    const Mat* image;
};

GCApplication gcapp;

const char* keys =
{
    "{help h||}{@image|../../testdata/A/A05_38.bmp|input image file}"
};

int main( int argc, const char** argv )
{
    CommandLineParser parser(argc, argv, keys);
    if (parser.has("help"))
    {
        help();
        return 0;
    }
    gcapp.setImagePath( parser.get<string>(0) );

    // Load the source image. HighGUI use.
    Mat image = imread( gcapp.getImagePath(), CV_LOAD_IMAGE_GRAYSCALE );
    if(image.empty())
    {
        std::cerr << "Cannot read image file: " << gcapp.getImagePath() << std::endl;
        return -1;
    }

    // TODO: check size and reduce it only if needed
    cv::resize(image, image, cv::Size(), 0.28, 0.28);
    help();

    /// Create the GUI
    const string winName = "image";
    namedWindow( winName, WINDOW_AUTOSIZE );

    gcapp.setImageAndWinName( image, winName );
    gcapp.showImage();

    int iterCount_ = 0;

    for(;;)
    {
        int c = waitKey(0);
        switch( (char) c )
        {
        case '\x1b':
            cout << "Exiting ..." << endl;
            goto exit_main;
        case 'r':
            cout << endl;
            gcapp.reset();
            gcapp.showImage();
            break;
        case 'n':
            cout << "<" << iterCount_ << "... ";
            cout << "rect must be determined>" << endl;
            break;
        }
    }

exit_main:
    destroyWindow( winName );
    return 0;
}

void GCApplication::reset()
{
    /* if( !mask.empty() ) */
    /*     mask.setTo(Scalar::all(GC_BGD)); */
    /* bgdPxls.clear(); fgdPxls.clear(); */
    /* prBgdPxls.clear();  prFgdPxls.clear(); */

    std::cout << " nothing here" << std::endl;
}

void GCApplication::setImageAndWinName( const Mat& _image, const string& _winName  )
{
    if( _image.empty() || _winName.empty() )
        return;
    image = &_image;
    winName = &_winName;
}

void GCApplication::preprocessImage( const Mat& inImage, Mat& outImage )
// TODO: add image->copy(out)
{

}

void GCApplication::showImage()
{
    if( image->empty() || winName->empty() )
        return;

    // Treat the image
    Mat res;
    image->copyTo( res );

    /*if( !isInitialized )
        image->copyTo( res );
    else
    {
        getBinMask( mask, binMask );
        image->copyTo( res, binMask );
    }*/

    /// Set a contour and its rotated bounding box
    vector<vector<Point> > contours;
    vector<Vec4i> hierarchy;
    /* if( isInitialized ){ */
    /*     findContours( binMask, contours, hierarchy, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE ); */
    /* } */
    /* if( isInitialized ) */
    /* { */
    /*     std::cout << "num of contours: " << contours.size() << "->"; */
    /*     // eliminate small contours */
    /*     contours.erase( */
    /*             std::remove_if( */
    /*                 contours.begin(), contours.end(), */
    /*                 [](const vector<Point>& c) */
    /*                 { const int min_size = 200; return c.size() < min_size; } */
    /*                 // or a functor/plain function/Boost.Lambda expression */
    /*                 ), contours.end() */
    /*             ); */
    /*     std::cout << contours.size() << " ..."; */
    /* } */

    /// Find the rotated rectangles for each contour
    /* vector<RotatedRect> minRect( contours.size() ); */
    /* for( size_t i = 0; i < contours.size(); i++ ) */
    /*     minRect[i] = minAreaRect( Mat(contours[i]) ); */

    /* for( size_t i = 0; i< contours.size(); i++ ) */
    /*     { */
    /*     Scalar color( rand()&255, rand()&255, rand()&255 ); */
    /*     // contour */
    /*     drawContours( res, contours, (int)i, color, 1, 8, vector<Vec4i>(), 0, Point() ); */
    /*     // rotated rectangle */
    /*     Point2f rect_points[4]; minRect[i].points( rect_points ); */
    /*     for( int j = 0; j < 4; j++ ) */
    /*         line( res, rect_points[j], rect_points[(j+1)%4], color, 1, 8 ); */
    /*     } */

    imshow( *winName, res );
}
<commit_msg>added remove background<commit_after>#include "opencv2/core/utility.hpp"
#include "opencv2/imgproc.hpp"
#include "opencv2/imgcodecs.hpp"
#include "opencv2/highgui.hpp"

#include <iostream>

using namespace cv;
using namespace std;

static void help()
{
    cout <<   "r - reset\n"
         << "b - remove background\n"
         << "t - threshold (not working)\n"
         << "f - find contours (not working)\n"
         << endl;
}

const auto RED = cv::Scalar(0,0,255);
const auto PINK = cv::Scalar(230,130,255);
const auto BLUE = cv::Scalar(255,0,0);
const auto LIGHTBLUE = cv::Scalar(255,255,160);
const auto GREEN = cv::Scalar(0,255,0);
const auto IMAGE_SCALE = .25;

class GCApplication
{
public:
    enum{ NOT_SET = 0, IN_PROCESS = 1, SET = 2 };
    static const int radius = 2;
    static const int thickness = -1;

    void reset();
    void setImageAndWinName( const Mat& _image, const string& _winName );
    void showImage();
private:
    int preprocessImage( const Mat& _inImage, Mat& _outImage );

    const string* winName;
    const Mat* image;
    Mat* outImage;
};

GCApplication gcapp;

const char* keys =
{
    "{help h||}{@image|../../testdata/A/A05_38.bmp|input image file}"
};

int main( int argc, const char** argv )
{
    CommandLineParser parser(argc, argv, keys);
    if (parser.has("help"))
    {
        help();
        return 0;
    }
    string inputImage = parser.get<string>(0);

    // Load the source image. HighGUI use.
    Mat image = imread( inputImage, CV_LOAD_IMAGE_GRAYSCALE );
    if(image.empty())
    {
        std::cerr << "Cannot read image file: " << inputImage << std::endl;
        return -1;
    }

    // TODO: check size and reduce it only if needed
    cv::resize(image, image, cv::Size(), IMAGE_SCALE, IMAGE_SCALE);
    help();

    /// Create the GUI
    const string winName = "image";
    namedWindow( winName, WINDOW_AUTOSIZE );

    gcapp.setImageAndWinName( image, winName );
    gcapp.showImage();

    int iterCount_ = 0;

    for(;;)
    {
        int c = waitKey(0);
        switch( (char) c )
        {
        case '\x1b':
            cout << "Exiting ..." << endl;
            goto exit_main;
        case 'r':
            cout << endl;
            gcapp.reset();
            gcapp.showImage();
            break;
        case 'b':
            help();
            break;
        case 't':
            help();
            break;
        case 'f':
            help();
            break;
        }
    }

exit_main:
    destroyWindow( winName );
    return 0;
}

void GCApplication::reset()
{
    /* if( !mask.empty() ) */
    /*     mask.setTo(Scalar::all(GC_BGD)); */
    /* bgdPxls.clear(); fgdPxls.clear(); */
    /* prBgdPxls.clear();  prFgdPxls.clear(); */

    /* image->copyTo( outImage ); */
}

void GCApplication::setImageAndWinName( const Mat& _image, const string& _winName  )
{
    if( _image.empty() || _winName.empty() )
        return;
    image = &_image;
    winName = &_winName;
}

int GCApplication::preprocessImage( const Mat& _inImage, Mat& _outImage )
// TODO: add image->copy(out)
{
    const string bkgPath = "../../testdata/A/background.bmp";
    Mat background = imread( bkgPath, CV_LOAD_IMAGE_GRAYSCALE );
    if(background.empty())
    {
        std::cerr << "Cannot read image file: " << bkgPath << std::endl;
        return -1;
    }

    cv::resize(background, background, cv::Size(), IMAGE_SCALE, IMAGE_SCALE);
    /* _inImage.copyTo( _outImage ); */
    /* _outImage -= background; */
    cv::absdiff(background, _outImage, _outImage);
    return 0;
 }

void GCApplication::showImage()
{
    if( image->empty() || winName->empty() )
        return;

    // Treat the image
    Mat res;
    image->copyTo( res );
    gcapp.preprocessImage(*image, res);

    /*if( !isInitialized )
        image->copyTo( res );
    else
    {
        getBinMask( mask, binMask );
        image->copyTo( res, binMask );
    }*/

    /// Set a contour and its rotated bounding box
    vector<vector<Point> > contours;
    vector<Vec4i> hierarchy;
    /* if( isInitialized ){ */
    /*     findContours( binMask, contours, hierarchy, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE ); */
    /* } */
    /* if( isInitialized ) */
    /* { */
    /*     std::cout << "num of contours: " << contours.size() << "->"; */
    /*     // eliminate small contours */
    /*     contours.erase( */
    /*             std::remove_if( */
    /*                 contours.begin(), contours.end(), */
    /*                 [](const vector<Point>& c) */
    /*                 { const int min_size = 200; return c.size() < min_size; } */
    /*                 // or a functor/plain function/Boost.Lambda expression */
    /*                 ), contours.end() */
    /*             ); */
    /*     std::cout << contours.size() << " ..."; */
    /* } */

    /// Find the rotated rectangles for each contour
    /* vector<RotatedRect> minRect( contours.size() ); */
    /* for( size_t i = 0; i < contours.size(); i++ ) */
    /*     minRect[i] = minAreaRect( Mat(contours[i]) ); */

    /* for( size_t i = 0; i< contours.size(); i++ ) */
    /*     { */
    /*     Scalar color( rand()&255, rand()&255, rand()&255 ); */
    /*     // contour */
    /*     drawContours( res, contours, (int)i, color, 1, 8, vector<Vec4i>(), 0, Point() ); */
    /*     // rotated rectangle */
    /*     Point2f rect_points[4]; minRect[i].points( rect_points ); */
    /*     for( int j = 0; j < 4; j++ ) */
    /*         line( res, rect_points[j], rect_points[(j+1)%4], color, 1, 8 ); */
    /*     } */

    imshow( *winName, res );
}

<|endoftext|>


<commit_before><commit_msg>Refactored `cuda::device::current::scoped_override_t`:<commit_after><|endoftext|>


<commit_before><commit_msg>Default Generator Includes<commit_after><|endoftext|>


<commit_before>#include <deque>
#include <iostream>
#include <string>
#include <tclap/CmdLine.h>

#include "Pore_Model.hpp"
#include "Builtin_Model.hpp"
#include "State_Transitions.hpp"
#include "Event.hpp"
#include "Fast5_Summary.hpp"
#include "Viterbi.hpp"
#include "Forward_Backward.hpp"
#include "logger.hpp"
#include "zstr.hpp"
#include "fast5.hpp"
#include "pfor.hpp"
#include "fs_support.hpp"

using namespace std;

namespace opts
{
    using namespace TCLAP;
    string description = "Call bases in ONT reads";
    CmdLine cmd_parser(description);
    MultiArg< string > log_level("", "log", "Log level.", false, "string", cmd_parser);
    ValueArg< unsigned > num_threads("t", "threads", "Number of parallel threads.", false, 1, "int", cmd_parser);
    MultiArg< string > model_fn("m", "model", "Pore model.", false, "file", cmd_parser);
    ValueArg< string > model_fofn("", "model-fofn", "File of pore models.", false, "", "file", cmd_parser);
    ValueArg< string > trans_fn("s", "trans", "Initial state transitions.", false, "", "file", cmd_parser);
    ValueArg< string > output_fn("o", "output", "Output.", false, "", "file", cmd_parser);
    ValueArg< float > pr_stay("", "pr-stay", "Transition probability of staying in the same state.", false, .1, "float", cmd_parser);
    ValueArg< float > pr_skip("", "pr-skip", "Transition probability of skipping at least 1 state.", false, .1, "float", cmd_parser);
    ValueArg< float > pr_cutoff("", "pr-cutoff", "Minimum value for transition probabilities; smaller values are set to zero.", false, .001, "float", cmd_parser);
    ValueArg< unsigned > min_read_len("", "min-len", "Minimum read length.", false, 1000, "int", cmd_parser);
    UnlabeledMultiArg< string > input_fn("inputs", "Input files/directories", true, "path", cmd_parser);
} // namespace opts

void init_models(Pore_Model_Dict_Type& models)
{
    auto parse_model_name = [] (const string& s) {
        if (s.size() < 3
            or (s[0] != '0' and s[0] != '1' and s[0] != '2')
            or s[1] != ':')
        {
            cerr << "could not parse model name: \"" << s << "\"; format should be \"[0|1|2]:<file>\"" << endl;
            exit(EXIT_FAILURE);
        }
        unsigned st = s[0] - '0';
        return make_pair(st, s.substr(2));
    };

    map< unsigned, list< string > > model_list;
    if (not opts::model_fn.get().empty())
    {
        for (const auto& s : opts::model_fn.get())
        {
            auto p = parse_model_name(s);
            model_list[p.first].push_back(p.second);
        }
    }
    if (not opts::model_fofn.get().empty())
    {
        zstr::ifstream ifs(opts::model_fofn);
        string s;
        while (getline(ifs, s))
        {
            auto p = parse_model_name(s);
            model_list[p.first].push_back(p.second);
        }
    }
    if (model_list[2].empty() and (model_list[0].empty() != model_list[1].empty()))
    {
        cerr << "models were specified only for strand " << (int)model_list[0].empty()
             << "! give models for both strands, or for neither." << endl;
        exit(EXIT_FAILURE);
    }
    if (not (model_list[0].empty() and model_list[1].empty() and model_list[2].empty()))
    {
        for (unsigned st = 0; st < 3; ++st)
        {
            for (const auto& e : model_list[st])
            {
                Pore_Model_Type pm;
                string pm_name = e;
                zstr::ifstream(e) >> pm;
                pm.strand() = st;
                models[pm_name] = move(pm);
                LOG("main", info) << "loaded module [" << pm_name << "] for strand [" << st << "]" << endl;
            }
        }
    }
    else
    {
        // use built-in models
        for (unsigned i = 0; i < Builtin_Model::num; ++i)
        {
            Pore_Model_Type pm;
            string pm_name = Builtin_Model::names[i];
            pm.load_from_vector(Builtin_Model::init_lists[i]);
            pm.strand() = Builtin_Model::strands[i];
            models[Builtin_Model::names[i]] = move(pm);
            LOG("main", info)
                << "loaded builtin module [" << Builtin_Model::names[i] << "] for strand ["
                << Builtin_Model::strands[i] << "] statistics [mean=" << pm.mean() << ", stdv="
                << pm.stdv() << "]" << endl;
        }
    }
} // init_models

void init_transitions(State_Transitions_Type& transitions)
{
    if (not opts::trans_fn.get().empty())
    {
        zstr::ifstream(opts::trans_fn) >> transitions;
        LOG("main", info) << "loaded state transitions from [" << opts::trans_fn.get() << "]" << endl;
    }
    else
    {
        transitions.compute_transitions(opts::pr_skip, opts::pr_stay, opts::pr_cutoff);
        LOG("main", info) << "initialized state transitions with parameters p_skip=[" << opts::pr_skip
                          << "], pr_stay=[" << opts::pr_stay << "], pr_cutoff=[" << opts::pr_cutoff << "]" << endl;
    }
} // init_transitions

// Parse command line arguments. For each of them:
// - if it is a directory, find all fast5 files in it, ignore non-fast5 files.
// - if it is a file, check that it is indeed a fast5 file.
void init_files(list< string >& files)
{
    for (const auto& f : opts::input_fn)
    {
        if (is_directory(f))
        {
            auto l = list_directory(f);
            for (const auto& g : l)
            {
                string f2 = f + (f[f.size() - 1] != '/'? "/" : "") + g;
                if (fast5::File::is_valid_file(f2))
                {
                    files.push_back(f2);
                    LOG("main", info) << "adding input file [" << f2 << "]" << endl;
                }
                else
                {
                    LOG("main", info) << "ignoring file [" << f2 << "]" << endl;
                }
            }
        }
        else // not a directory
        {
            if (fast5::File::is_valid_file(f))
            {
                files.push_back(f);
                LOG("main", info) << "adding input file [" << f << "]" << endl;
            }
            else
            {
                cerr << "error opening fast5 file [" << f << "]" << endl;
                exit(EXIT_FAILURE);
            }
        }
    }
} // init_files

void init_reads(const Pore_Model_Dict_Type& models,
                const list< string >& files,
                deque< Fast5_Summary_Type >& reads)
{
    for (const auto& f : files)
    {
        Fast5_Summary_Type s(f, models);
        LOG("main", info) << "summary: " << s << endl;
        if (s.have_ed_events)
        {
            reads.emplace_back(move(s));
        }
    }
} // init_reads

void train_reads(const Pore_Model_Dict_Type& models,
                 State_Transitions_Type& transitions,
                 deque< Fast5_Summary_Type >& reads)
{
    return;
    //
    // this is only a stub
    //
    for (unsigned round = 0; round < 5; ++round)
    {
        LOG("main", info) << "starting training round [" << round << "]" << endl;
        unsigned crt_idx = 0;
        pfor::pfor< unsigned >(
            opts::num_threads,
            10,
            // get_item
            [&] (unsigned& i) {
                if (crt_idx >= reads.size()) return false;
                i = crt_idx++;
                return true;
            },
            // process item
            [&] (unsigned& i) {
                Fast5_Summary_Type& read_summary = reads[i];
                read_summary.load_events();

                for (unsigned st = 0; st < 2; ++st)
                {
                    // if not enough events, ignore strand
                    if (read_summary.events[st].size() < opts::min_read_len) continue;
                    // create list of models to try
                    list< string > model_sublist;
                    if (models.count(read_summary.preferred_model[st]))
                    {
                        // if we have a preferred model, use that
                        model_sublist.push_back(read_summary.preferred_model[st]);
                    }
                    else
                    {
                        // no preferred model, try all that apply to this strand
                        for (const auto& p : models)
                        {
                            if (p.second.strand() == st or p.second.strand() == 2)
                            {
                                model_sublist.push_back(p.first);
                            }
                        }
                    }
                    assert(not model_sublist.empty());
                    // initialize parameters if not existing already
                    for (const auto& m_name : model_sublist)
                    {
                        if (not read_summary.params[st].count(m_name))
                        {
                            read_summary.params[st][m_name] = Pore_Model_Parameters_Type();
                        }
                    }
                    // run fwbw
                    map< string, Forward_Backward_Type > results;
                    for (const auto& m_name : model_sublist)
                    {
                        // scale model using current parameters
                        Pore_Model_Type pm(models.at(m_name));
                        Pore_Model_Parameters_Type pm_params = read_summary.params[st].at(m_name);
                        pm.scale(pm_params);
                        // main work: fill matrix
                        results[m_name].fill(pm, transitions, read_summary.events[st]);
                    }
                    //
                    // use fwbw results to update parameters
                    // ...
                    //
                }
                read_summary.drop_events();
            },
            // progress_report
            [&] (unsigned items, unsigned seconds) {
                clog << "Processed " << setw(6) << right << items << " reads in "
                     << setw(6) << right << seconds << " seconds\r";
            }); // pfor

        //
        // update state_transitions
        // ...
        //

    } // for round
} // train_reads

void basecall_reads(const Pore_Model_Dict_Type& models,
                    const State_Transitions_Type& transitions,
                    deque< Fast5_Summary_Type >& reads)
{
    strict_fstream::ofstream ofs;
    ostream* os_p = nullptr;
    if (not opts::output_fn.get().empty())
    {
        ofs.open(opts::output_fn);
        os_p = &ofs;
    }
    else
    {
        os_p = &cout;
    }

    unsigned crt_idx = 0;
    pfor::pfor< unsigned, std::ostringstream >(
        opts::num_threads,
        10,
        // get_item
        [&] (unsigned& i) {
            if (crt_idx >= reads.size()) return false;
            i = crt_idx++;
            return true;
        },
        // process_item
        [&] (unsigned& i, std::ostringstream& oss) {
            Fast5_Summary_Type& read_summary = reads[i];
            read_summary.load_events();
            for (unsigned st = 0; st < 2; ++st)
            {
                // if not enough events, ignore strand
                if (read_summary.events[st].size() < opts::min_read_len) continue;
                // create list of models to try
                list< string > model_sublist;
                if (models.count(read_summary.preferred_model[st]))
                {
                    // if we have a preferred model, use that
                    model_sublist.push_back(read_summary.preferred_model[st]);
                }
                else
                {
                    // no preferred model, try all that apply to this strand
                    for (const auto& p : models)
                    {
                        if (p.second.strand() == st or p.second.strand() == 2)
                        {
                            model_sublist.push_back(p.first);
                        }
                    }
                }
                assert(not model_sublist.empty());
                // initialize parameters if not existing already
                for (const auto& m_name : model_sublist)
                {
                    if (not read_summary.params[st].count(m_name))
                    {
                        read_summary.params[st][m_name] = Pore_Model_Parameters_Type();
                    }
                }
                // check main scaling parameters
                auto r = get_mean_stdv< float >(
                    read_summary.events[st],
                    [] (const Event_Type& ev) { return ev.mean; });
                // deque of results
                deque< tuple< float, string, string > > results;
                for (const auto& m_name : model_sublist)
                {
                    Pore_Model_Type pm(models.at(m_name));
                    Pore_Model_Parameters_Type pm_params = read_summary.params[st].at(m_name);
                    pm.scale(pm_params);
                    if (std::abs(r.first - pm.mean()) > 5.0)
                    {
                        LOG("main", warning)
                            << "basecalling read [" << read_summary.read_id << "] strand [" << st
                            << "] using model [" << m_name << "] and parameters "
                            << read_summary.params[st].at(m_name)
                            << " but model_mean=[" << pm.mean() << "] and events_mean=[" << r.first
                            << "] are far apart!" << endl;
                    }
                    Viterbi_Type vit;
                    vit.fill(pm, transitions, read_summary.events.at(st));
                    results.emplace_back(make_tuple(vit.path_probability(), m_name, vit.base_seq()));
                }
                std::sort(results.begin(), results.end());
                string& best_m_name = get<1>(results.back());
                string& base_seq = get<2>(results.back());
                LOG("main", info) << "basecalled read [" << read_summary.read_id << "] strand [" << st
                                  << "] using model [" << best_m_name << "] and parameters "
                                  << read_summary.params[st].at(best_m_name) << endl;
                oss << ">" << read_summary.read_id << ":" << st << endl
                    << base_seq << endl;
            } // for st
            read_summary.drop_events();
        },
        // output_chunk
        [&] (std::ostringstream& oss) {
            *os_p << oss.str();
        },
        // progress_report
        [&] (unsigned items, unsigned seconds) {
            clog << "Processed " << setw(6) << right << items << " reads in "
                 << setw(6) << right << seconds << " seconds\r";
        }); // pfor
} // basecall_reads

void real_main()
{
    Pore_Model_Dict_Type models;
    State_Transitions_Type transitions;
    deque< Fast5_Summary_Type > reads;
    list< string > files;
    // initialize structs
    init_models(models);
    init_transitions(transitions);
    init_files(files);
    init_reads(models, files, reads);
    // do some training
    train_reads(models, transitions, reads);
    // basecall reads
    basecall_reads(models, transitions, reads);
}

int main(int argc, char * argv[])
{
    opts::cmd_parser.parse(argc, argv);
    Logger::set_default_level(Logger::level::info);
    Logger::set_levels_from_options(opts::log_level);
    Fast5_Summary_Type::min_read_len() = opts::min_read_len;
    real_main();
}
<commit_msg>fasta line width<commit_after>#include <deque>
#include <iostream>
#include <string>
#include <tclap/CmdLine.h>

#include "Pore_Model.hpp"
#include "Builtin_Model.hpp"
#include "State_Transitions.hpp"
#include "Event.hpp"
#include "Fast5_Summary.hpp"
#include "Viterbi.hpp"
#include "Forward_Backward.hpp"
#include "logger.hpp"
#include "zstr.hpp"
#include "fast5.hpp"
#include "pfor.hpp"
#include "fs_support.hpp"

using namespace std;

namespace opts
{
    using namespace TCLAP;
    string description = "Call bases in ONT reads";
    CmdLine cmd_parser(description);
    MultiArg< string > log_level("", "log", "Log level.", false, "string", cmd_parser);
    ValueArg< unsigned > num_threads("t", "threads", "Number of parallel threads.", false, 1, "int", cmd_parser);
    MultiArg< string > model_fn("m", "model", "Pore model.", false, "file", cmd_parser);
    ValueArg< string > model_fofn("", "model-fofn", "File of pore models.", false, "", "file", cmd_parser);
    ValueArg< string > trans_fn("s", "trans", "Initial state transitions.", false, "", "file", cmd_parser);
    ValueArg< string > output_fn("o", "output", "Output.", false, "", "file", cmd_parser);
    ValueArg< float > pr_stay("", "pr-stay", "Transition probability of staying in the same state.", false, .1, "float", cmd_parser);
    ValueArg< float > pr_skip("", "pr-skip", "Transition probability of skipping at least 1 state.", false, .1, "float", cmd_parser);
    ValueArg< float > pr_cutoff("", "pr-cutoff", "Minimum value for transition probabilities; smaller values are set to zero.", false, .001, "float", cmd_parser);
    ValueArg< unsigned > min_read_len("", "min-len", "Minimum read length.", false, 1000, "int", cmd_parser);
    ValueArg< unsigned > fasta_line_width("", "fasta-line-width", "Maximum fasta line width.", false, 80, "int", cmd_parser);
    UnlabeledMultiArg< string > input_fn("inputs", "Input files/directories", true, "path", cmd_parser);
} // namespace opts

void init_models(Pore_Model_Dict_Type& models)
{
    auto parse_model_name = [] (const string& s) {
        if (s.size() < 3
            or (s[0] != '0' and s[0] != '1' and s[0] != '2')
            or s[1] != ':')
        {
            cerr << "could not parse model name: \"" << s << "\"; format should be \"[0|1|2]:<file>\"" << endl;
            exit(EXIT_FAILURE);
        }
        unsigned st = s[0] - '0';
        return make_pair(st, s.substr(2));
    };

    map< unsigned, list< string > > model_list;
    if (not opts::model_fn.get().empty())
    {
        for (const auto& s : opts::model_fn.get())
        {
            auto p = parse_model_name(s);
            model_list[p.first].push_back(p.second);
        }
    }
    if (not opts::model_fofn.get().empty())
    {
        zstr::ifstream ifs(opts::model_fofn);
        string s;
        while (getline(ifs, s))
        {
            auto p = parse_model_name(s);
            model_list[p.first].push_back(p.second);
        }
    }
    if (model_list[2].empty() and (model_list[0].empty() != model_list[1].empty()))
    {
        cerr << "models were specified only for strand " << (int)model_list[0].empty()
             << "! give models for both strands, or for neither." << endl;
        exit(EXIT_FAILURE);
    }
    if (not (model_list[0].empty() and model_list[1].empty() and model_list[2].empty()))
    {
        for (unsigned st = 0; st < 3; ++st)
        {
            for (const auto& e : model_list[st])
            {
                Pore_Model_Type pm;
                string pm_name = e;
                zstr::ifstream(e) >> pm;
                pm.strand() = st;
                models[pm_name] = move(pm);
                LOG("main", info) << "loaded module [" << pm_name << "] for strand [" << st << "]" << endl;
            }
        }
    }
    else
    {
        // use built-in models
        for (unsigned i = 0; i < Builtin_Model::num; ++i)
        {
            Pore_Model_Type pm;
            string pm_name = Builtin_Model::names[i];
            pm.load_from_vector(Builtin_Model::init_lists[i]);
            pm.strand() = Builtin_Model::strands[i];
            models[Builtin_Model::names[i]] = move(pm);
            LOG("main", info)
                << "loaded builtin module [" << Builtin_Model::names[i] << "] for strand ["
                << Builtin_Model::strands[i] << "] statistics [mean=" << pm.mean() << ", stdv="
                << pm.stdv() << "]" << endl;
        }
    }
} // init_models

void init_transitions(State_Transitions_Type& transitions)
{
    if (not opts::trans_fn.get().empty())
    {
        zstr::ifstream(opts::trans_fn) >> transitions;
        LOG("main", info) << "loaded state transitions from [" << opts::trans_fn.get() << "]" << endl;
    }
    else
    {
        transitions.compute_transitions(opts::pr_skip, opts::pr_stay, opts::pr_cutoff);
        LOG("main", info) << "initialized state transitions with parameters p_skip=[" << opts::pr_skip
                          << "], pr_stay=[" << opts::pr_stay << "], pr_cutoff=[" << opts::pr_cutoff << "]" << endl;
    }
} // init_transitions

// Parse command line arguments. For each of them:
// - if it is a directory, find all fast5 files in it, ignore non-fast5 files.
// - if it is a file, check that it is indeed a fast5 file.
void init_files(list< string >& files)
{
    for (const auto& f : opts::input_fn)
    {
        if (is_directory(f))
        {
            auto l = list_directory(f);
            for (const auto& g : l)
            {
                string f2 = f + (f[f.size() - 1] != '/'? "/" : "") + g;
                if (fast5::File::is_valid_file(f2))
                {
                    files.push_back(f2);
                    LOG("main", info) << "adding input file [" << f2 << "]" << endl;
                }
                else
                {
                    LOG("main", info) << "ignoring file [" << f2 << "]" << endl;
                }
            }
        }
        else // not a directory
        {
            if (fast5::File::is_valid_file(f))
            {
                files.push_back(f);
                LOG("main", info) << "adding input file [" << f << "]" << endl;
            }
            else
            {
                cerr << "error opening fast5 file [" << f << "]" << endl;
                exit(EXIT_FAILURE);
            }
        }
    }
} // init_files

void init_reads(const Pore_Model_Dict_Type& models,
                const list< string >& files,
                deque< Fast5_Summary_Type >& reads)
{
    for (const auto& f : files)
    {
        Fast5_Summary_Type s(f, models);
        LOG("main", info) << "summary: " << s << endl;
        if (s.have_ed_events)
        {
            reads.emplace_back(move(s));
        }
    }
} // init_reads

void train_reads(const Pore_Model_Dict_Type& models,
                 State_Transitions_Type& transitions,
                 deque< Fast5_Summary_Type >& reads)
{
    return;
    //
    // this is only a stub
    //
    for (unsigned round = 0; round < 5; ++round)
    {
        LOG("main", info) << "starting training round [" << round << "]" << endl;
        unsigned crt_idx = 0;
        pfor::pfor< unsigned >(
            opts::num_threads,
            10,
            // get_item
            [&] (unsigned& i) {
                if (crt_idx >= reads.size()) return false;
                i = crt_idx++;
                return true;
            },
            // process item
            [&] (unsigned& i) {
                Fast5_Summary_Type& read_summary = reads[i];
                read_summary.load_events();

                for (unsigned st = 0; st < 2; ++st)
                {
                    // if not enough events, ignore strand
                    if (read_summary.events[st].size() < opts::min_read_len) continue;
                    // create list of models to try
                    list< string > model_sublist;
                    if (models.count(read_summary.preferred_model[st]))
                    {
                        // if we have a preferred model, use that
                        model_sublist.push_back(read_summary.preferred_model[st]);
                    }
                    else
                    {
                        // no preferred model, try all that apply to this strand
                        for (const auto& p : models)
                        {
                            if (p.second.strand() == st or p.second.strand() == 2)
                            {
                                model_sublist.push_back(p.first);
                            }
                        }
                    }
                    assert(not model_sublist.empty());
                    // initialize parameters if not existing already
                    for (const auto& m_name : model_sublist)
                    {
                        if (not read_summary.params[st].count(m_name))
                        {
                            read_summary.params[st][m_name] = Pore_Model_Parameters_Type();
                        }
                    }
                    // run fwbw
                    map< string, Forward_Backward_Type > results;
                    for (const auto& m_name : model_sublist)
                    {
                        // scale model using current parameters
                        Pore_Model_Type pm(models.at(m_name));
                        Pore_Model_Parameters_Type pm_params = read_summary.params[st].at(m_name);
                        pm.scale(pm_params);
                        // main work: fill matrix
                        results[m_name].fill(pm, transitions, read_summary.events[st]);
                    }
                    //
                    // use fwbw results to update parameters
                    // ...
                    //
                }
                read_summary.drop_events();
            },
            // progress_report
            [&] (unsigned items, unsigned seconds) {
                clog << "Processed " << setw(6) << right << items << " reads in "
                     << setw(6) << right << seconds << " seconds\r";
            }); // pfor

        //
        // update state_transitions
        // ...
        //

    } // for round
} // train_reads

void write_fasta(ostream& os, const string& name, const string& seq)
{
    os << ">" << name << endl;
    for (unsigned pos = 0; pos < seq.size(); pos += opts::fasta_line_width)
    {
        os << seq.substr(pos, opts::fasta_line_width) << endl;
    }
} // write_fasta

void basecall_reads(const Pore_Model_Dict_Type& models,
                    const State_Transitions_Type& transitions,
                    deque< Fast5_Summary_Type >& reads)
{
    strict_fstream::ofstream ofs;
    ostream* os_p = nullptr;
    if (not opts::output_fn.get().empty())
    {
        ofs.open(opts::output_fn);
        os_p = &ofs;
    }
    else
    {
        os_p = &cout;
    }

    unsigned crt_idx = 0;
    pfor::pfor< unsigned, std::ostringstream >(
        opts::num_threads,
        10,
        // get_item
        [&] (unsigned& i) {
            if (crt_idx >= reads.size()) return false;
            i = crt_idx++;
            return true;
        },
        // process_item
        [&] (unsigned& i, std::ostringstream& oss) {
            Fast5_Summary_Type& read_summary = reads[i];
            read_summary.load_events();
            for (unsigned st = 0; st < 2; ++st)
            {
                // if not enough events, ignore strand
                if (read_summary.events[st].size() < opts::min_read_len) continue;
                // create list of models to try
                list< string > model_sublist;
                if (models.count(read_summary.preferred_model[st]))
                {
                    // if we have a preferred model, use that
                    model_sublist.push_back(read_summary.preferred_model[st]);
                }
                else
                {
                    // no preferred model, try all that apply to this strand
                    for (const auto& p : models)
                    {
                        if (p.second.strand() == st or p.second.strand() == 2)
                        {
                            model_sublist.push_back(p.first);
                        }
                    }
                }
                assert(not model_sublist.empty());
                // initialize parameters if not existing already
                for (const auto& m_name : model_sublist)
                {
                    if (not read_summary.params[st].count(m_name))
                    {
                        read_summary.params[st][m_name] = Pore_Model_Parameters_Type();
                    }
                }
                // check main scaling parameters
                auto r = get_mean_stdv< float >(
                    read_summary.events[st],
                    [] (const Event_Type& ev) { return ev.mean; });
                // deque of results
                deque< tuple< float, string, string > > results;
                for (const auto& m_name : model_sublist)
                {
                    Pore_Model_Type pm(models.at(m_name));
                    Pore_Model_Parameters_Type pm_params = read_summary.params[st].at(m_name);
                    pm.scale(pm_params);
                    if (std::abs(r.first - pm.mean()) > 5.0)
                    {
                        LOG("main", warning)
                            << "basecalling read [" << read_summary.read_id << "] strand [" << st
                            << "] using model [" << m_name << "] and parameters "
                            << read_summary.params[st].at(m_name)
                            << " but model_mean=[" << pm.mean() << "] and events_mean=[" << r.first
                            << "] are far apart!" << endl;
                    }
                    Viterbi_Type vit;
                    vit.fill(pm, transitions, read_summary.events.at(st));
                    results.emplace_back(make_tuple(vit.path_probability(), m_name, vit.base_seq()));
                }
                std::sort(results.begin(), results.end());
                string& best_m_name = get<1>(results.back());
                string& base_seq = get<2>(results.back());
                LOG("main", info)
                    << "best_model read [" << read_summary.read_id
                    << "] strand [" << st
                    << "] model [" << best_m_name
                    << "] parameters " << read_summary.params[st].at(best_m_name) << endl;
                ostringstream tmp;
                tmp << read_summary.read_id << ":" << st;
                write_fasta(oss, tmp.str(), base_seq);
            } // for st
            read_summary.drop_events();
        },
        // output_chunk
        [&] (std::ostringstream& oss) {
            *os_p << oss.str();
        },
        // progress_report
        [&] (unsigned items, unsigned seconds) {
            clog << "Processed " << setw(6) << right << items << " reads in "
                 << setw(6) << right << seconds << " seconds\r";
        }); // pfor
} // basecall_reads

void real_main()
{
    Pore_Model_Dict_Type models;
    State_Transitions_Type transitions;
    deque< Fast5_Summary_Type > reads;
    list< string > files;
    // initialize structs
    init_models(models);
    init_transitions(transitions);
    init_files(files);
    init_reads(models, files, reads);
    // do some training
    train_reads(models, transitions, reads);
    // basecall reads
    basecall_reads(models, transitions, reads);
}

int main(int argc, char * argv[])
{
    opts::cmd_parser.parse(argc, argv);
    Logger::set_default_level(Logger::level::info);
    Logger::set_levels_from_options(opts::log_level);
    Fast5_Summary_Type::min_read_len() = opts::min_read_len;
    real_main();
}
<|endoftext|>


<commit_before>// Copyright (C) 2012-2014 Leap Motion, Inc. All rights reserved.
#include "stdafx.h"
#include "BasicThread.h"
#include "Autowired.h"
#include "BasicThreadStateBlock.h"
#include "ContextEnumerator.h"
#include "fast_pointer_cast.h"
#include "move_only.h"

// Explicit instantiation of supported time point types:
template<> bool BasicThread::WaitUntil(std::chrono::steady_clock::time_point);
template<> bool BasicThread::WaitUntil(std::chrono::system_clock::time_point);

BasicThread::BasicThread(const char* pName):
  ContextMember(pName),
  m_state(std::make_shared<BasicThreadStateBlock>()),
  m_stop(false),
  m_running(false),
  m_completed(false),
  m_priority(ThreadPriority::Default)
{}

std::mutex& BasicThread::GetLock(void) {
  return m_state->m_lock;
}

void BasicThread::DoRun(std::shared_ptr<Object>&& refTracker) {
  assert(m_running);

  // Make our own session current before we do anything else:
  CurrentContextPusher pusher(GetContext());

  // Set the thread name no matter what:
  if(GetName())
    SetCurrentThreadName();

  // Now we wait for the thread to be good to go:
  try {
    Run();
  }
  catch(dispatch_aborted_exception&) {
    // Okay, this is fine, a dispatcher is terminating--this is a normal way to
    // end a dispatcher loop.
  }
  catch(...) {
    try {
      // Ask that the enclosing context filter this exception, if possible:
      GetContext()->FilterException();
    }
    catch(...) {
      // Generic exception, unhandled, we can't do anything about this
    }

    // Signal shutdown on the enclosing context--cannot wait, if we wait we WILL deadlock
    GetContext()->SignalShutdown(false);
  }

  // Run loop is over, time to clean up
  DoRunLoopCleanup(pusher.Pop(), std::move(refTracker));
}

void BasicThread::DoRunLoopCleanup(std::shared_ptr<CoreContext>&& ctxt, std::shared_ptr<Object>&& refTracker) {
  // Take a copy of our state condition shared pointer while we still hold a reference to
  // ourselves.  This is the only member out of our collection of members that we actually
  // need to hold a reference to.
  auto state = m_state;

  // Perform a manual notification of teardown listeners
  NotifyTeardownListeners();

  // Notify everyone that we're completed:
  std::lock_guard<std::mutex> lk(state->m_lock);
  m_stop = true;
  m_completed = true;
  m_running = false;

  // No longer running, we MUST release the thread pointer to ensure proper teardown order
  state->m_thisThread.detach();

  // Release our hold on the context.  After this point, we have to be VERY CAREFUL that we
  // don't try to refer to any of our own member variables, because our own object may have
  // already gone out of scope.  [this] is potentially dangling.
  ctxt.reset();

  // Clear our reference tracker, which will notify anyone who is asleep and also maybe
  // will destroy the entire underlying context.
  refTracker.reset();

  // Notify other threads that we are done.  At this point, any held references that might
  // still exist are held by entities other than ourselves.
  state->m_stateCondition.notify_all();
}

void BasicThread::WaitForStateUpdate(const std::function<bool()>& fn) {
  std::unique_lock<std::mutex> lk(m_state->m_lock);
  m_state->m_stateCondition.wait(
    lk,
    [&fn, this] {
      return fn() || ShouldStop();
    }
  );
  if(ShouldStop())
    throw dispatch_aborted_exception();
}

void BasicThread::PerformStatusUpdate(const std::function<void()>& fn) {
  std::unique_lock<std::mutex> lk(m_state->m_lock);
  fn();
  m_state->m_stateCondition.notify_all();
}

bool BasicThread::ShouldStop(void) const {
  auto context = ContextMember::GetContext();
  return m_stop || !context || context->IsShutdown();
}

bool BasicThread::IsRunning(void) const {
  std::lock_guard<std::mutex> lk(m_state->m_lock);
  return m_running;
}

bool BasicThread::ThreadSleep(std::chrono::nanoseconds timeout) {
  std::unique_lock<std::mutex> lk(m_state->m_lock);
  return m_state->m_stateCondition.wait_for(lk, timeout, [this] { return ShouldStop(); });
}

bool BasicThread::Start(std::shared_ptr<Object> outstanding) {
  std::shared_ptr<CoreContext> context = m_context.lock();
  if(!context)
    return false;

  {
    std::lock_guard<std::mutex> lk(m_state->m_lock);
    if(m_running)
      // Already running, short-circuit
      return true;

    if(m_completed)
      // Already completed (perhaps cancelled), short-circuit
      return false;

    // Currently running:
    m_running = true;
    m_state->m_stateCondition.notify_all();
  }

  // Kick off a thread and return here
  MoveOnly<std::shared_ptr<Object>> out(std::move(outstanding));
  m_state->m_thisThread = std::thread(
    [this, out] {
      this->DoRun(std::move(out.value));
    }
  );
  return true;
}

void BasicThread::Wait(void) {
  std::unique_lock<std::mutex> lk(m_state->m_lock);
  m_state->m_stateCondition.wait(
    lk,
    [this] {return this->m_completed; }
  );
}

bool BasicThread::WaitFor(std::chrono::nanoseconds duration) {
  std::unique_lock<std::mutex> lk(m_state->m_lock);
  return m_state->m_stateCondition.wait_for(
    lk,
    duration,
    [this] {return this->m_completed; }
  );
}

template<class TimeType>
bool BasicThread::WaitUntil(TimeType timepoint) {
  std::unique_lock<std::mutex> lk(m_state->m_lock);
  return m_state->m_stateCondition.wait_until(
    lk,
    timepoint,
    [this] {return this->m_completed; }
  );
}

void BasicThread::Stop(bool graceful) {
  {
    std::lock_guard<std::mutex> lk(m_state->m_lock);

    // Trivial return check:
    if(m_stop)
      return;

    // If we're not running, mark ourselves complete
    if(!m_running)
      m_completed = true;

    // Now we send the appropriate trigger:
    m_stop = true;
    m_state->m_stateCondition.notify_all();
  }

  // Event notification takes place outside of the context of a lock
  OnStop();
}

void BasicThread::ForceCoreThreadReidentify(void) {
  for(const auto& ctxt : ContextEnumerator(AutoGlobalContext())) {
    auto threadListCpy = ctxt->CopyBasicThreadList();
    for(auto q = threadListCpy.begin(); q != threadListCpy.end(); q++)
      (**q).SetCurrentThreadName();
  }
}

void ForceCoreThreadReidentify(void) {
  BasicThread::ForceCoreThreadReidentify();
}
<commit_msg>Deprecating any use of MoveOnly<commit_after>// Copyright (C) 2012-2014 Leap Motion, Inc. All rights reserved.
#include "stdafx.h"
#include "BasicThread.h"
#include "Autowired.h"
#include "BasicThreadStateBlock.h"
#include "ContextEnumerator.h"
#include "fast_pointer_cast.h"
#include "move_only.h"

// Explicit instantiation of supported time point types:
template<> bool BasicThread::WaitUntil(std::chrono::steady_clock::time_point);
template<> bool BasicThread::WaitUntil(std::chrono::system_clock::time_point);

BasicThread::BasicThread(const char* pName):
  ContextMember(pName),
  m_state(std::make_shared<BasicThreadStateBlock>()),
  m_stop(false),
  m_running(false),
  m_completed(false),
  m_priority(ThreadPriority::Default)
{}

std::mutex& BasicThread::GetLock(void) {
  return m_state->m_lock;
}

void BasicThread::DoRun(std::shared_ptr<Object>&& refTracker) {
  assert(m_running);

  // Make our own session current before we do anything else:
  CurrentContextPusher pusher(GetContext());

  // Set the thread name no matter what:
  if(GetName())
    SetCurrentThreadName();

  // Now we wait for the thread to be good to go:
  try {
    Run();
  }
  catch(dispatch_aborted_exception&) {
    // Okay, this is fine, a dispatcher is terminating--this is a normal way to
    // end a dispatcher loop.
  }
  catch(...) {
    try {
      // Ask that the enclosing context filter this exception, if possible:
      GetContext()->FilterException();
    }
    catch(...) {
      // Generic exception, unhandled, we can't do anything about this
    }

    // Signal shutdown on the enclosing context--cannot wait, if we wait we WILL deadlock
    GetContext()->SignalShutdown(false);
  }

  // Run loop is over, time to clean up
  DoRunLoopCleanup(pusher.Pop(), std::move(refTracker));
}

void BasicThread::DoRunLoopCleanup(std::shared_ptr<CoreContext>&& ctxt, std::shared_ptr<Object>&& refTracker) {
  // Take a copy of our state condition shared pointer while we still hold a reference to
  // ourselves.  This is the only member out of our collection of members that we actually
  // need to hold a reference to.
  auto state = m_state;

  // Perform a manual notification of teardown listeners
  NotifyTeardownListeners();

  // Notify everyone that we're completed:
  std::lock_guard<std::mutex> lk(state->m_lock);
  m_stop = true;
  m_completed = true;
  m_running = false;

  // No longer running, we MUST release the thread pointer to ensure proper teardown order
  state->m_thisThread.detach();

  // Release our hold on the context.  After this point, we have to be VERY CAREFUL that we
  // don't try to refer to any of our own member variables, because our own object may have
  // already gone out of scope.  [this] is potentially dangling.
  ctxt.reset();

  // Clear our reference tracker, which will notify anyone who is asleep and also maybe
  // will destroy the entire underlying context.
  refTracker.reset();

  // Notify other threads that we are done.  At this point, any held references that might
  // still exist are held by entities other than ourselves.
  state->m_stateCondition.notify_all();
}

void BasicThread::WaitForStateUpdate(const std::function<bool()>& fn) {
  std::unique_lock<std::mutex> lk(m_state->m_lock);
  m_state->m_stateCondition.wait(
    lk,
    [&fn, this] {
      return fn() || ShouldStop();
    }
  );
  if(ShouldStop())
    throw dispatch_aborted_exception();
}

void BasicThread::PerformStatusUpdate(const std::function<void()>& fn) {
  std::unique_lock<std::mutex> lk(m_state->m_lock);
  fn();
  m_state->m_stateCondition.notify_all();
}

bool BasicThread::ShouldStop(void) const {
  auto context = ContextMember::GetContext();
  return m_stop || !context || context->IsShutdown();
}

bool BasicThread::IsRunning(void) const {
  std::lock_guard<std::mutex> lk(m_state->m_lock);
  return m_running;
}

bool BasicThread::ThreadSleep(std::chrono::nanoseconds timeout) {
  std::unique_lock<std::mutex> lk(m_state->m_lock);
  return m_state->m_stateCondition.wait_for(lk, timeout, [this] { return ShouldStop(); });
}

bool BasicThread::Start(std::shared_ptr<Object> outstanding) {
  std::shared_ptr<CoreContext> context = m_context.lock();
  if(!context)
    return false;

  {
    std::lock_guard<std::mutex> lk(m_state->m_lock);
    if(m_running)
      // Already running, short-circuit
      return true;

    if(m_completed)
      // Already completed (perhaps cancelled), short-circuit
      return false;

    // Currently running:
    m_running = true;
    m_state->m_stateCondition.notify_all();
  }

  // Kick off a thread and return here
  m_state->m_thisThread = std::thread(
    [this, outstanding] () mutable {
      this->DoRun(std::move(outstanding));
    }
  );
  return true;
}

void BasicThread::Wait(void) {
  std::unique_lock<std::mutex> lk(m_state->m_lock);
  m_state->m_stateCondition.wait(
    lk,
    [this] {return this->m_completed; }
  );
}

bool BasicThread::WaitFor(std::chrono::nanoseconds duration) {
  std::unique_lock<std::mutex> lk(m_state->m_lock);
  return m_state->m_stateCondition.wait_for(
    lk,
    duration,
    [this] {return this->m_completed; }
  );
}

template<class TimeType>
bool BasicThread::WaitUntil(TimeType timepoint) {
  std::unique_lock<std::mutex> lk(m_state->m_lock);
  return m_state->m_stateCondition.wait_until(
    lk,
    timepoint,
    [this] {return this->m_completed; }
  );
}

void BasicThread::Stop(bool graceful) {
  {
    std::lock_guard<std::mutex> lk(m_state->m_lock);

    // Trivial return check:
    if(m_stop)
      return;

    // If we're not running, mark ourselves complete
    if(!m_running)
      m_completed = true;

    // Now we send the appropriate trigger:
    m_stop = true;
    m_state->m_stateCondition.notify_all();
  }

  // Event notification takes place outside of the context of a lock
  OnStop();
}

void BasicThread::ForceCoreThreadReidentify(void) {
  for(const auto& ctxt : ContextEnumerator(AutoGlobalContext())) {
    auto threadListCpy = ctxt->CopyBasicThreadList();
    for(auto q = threadListCpy.begin(); q != threadListCpy.end(); q++)
      (**q).SetCurrentThreadName();
  }
}

void ForceCoreThreadReidentify(void) {
  BasicThread::ForceCoreThreadReidentify();
}
<|endoftext|>


<commit_before><commit_msg>Adds a consumer for listening to DCC accessory setting packets. Exposes last set state via identify consumer commands using a 2040-length address scheme.<commit_after><|endoftext|>


<commit_before><commit_msg>filestore: don't start commit if nothing new is _applied_<commit_after><|endoftext|>


<commit_before>/*
 * Web Application Socket protocol, input data channel library.
 *
 * author: Max Kellermann <mk@cm4all.com>
 */

#ifndef BENG_PROXY_WAS_INPUT_HXX
#define BENG_PROXY_WAS_INPUT_HXX

#include "glibfwd.hxx"

#include <stdint.h>

struct pool;
class Istream;
class WasInput;

class WasInputHandler {
public:
    /**
     * Istream::Close() has been called.
     *
     * @param received the number of bytes received so far (includes
     * data that hasn't been delivered to the #IstreamHandler yet)
     */
    virtual void WasInputClose(uint64_t received) = 0;

    /**
     * All data was received from the pipe to the input buffer; we
     * don't need the pipe anymore for this request.
     *
     * Optional method.
     */
    virtual void WasInputRelease() = 0;

    virtual void WasInputEof() = 0;

    virtual void WasInputError() = 0;
};

WasInput *
was_input_new(struct pool *pool, int fd,
              WasInputHandler &handler);

/**
 * @param error the error reported to the istream handler
 */
void
was_input_free(WasInput *input, GError *error);

static inline void
was_input_free_p(WasInput **input_p, GError *error)
{
    WasInput *input = *input_p;
    *input_p = nullptr;
    was_input_free(input, error);
}

/**
 * Like was_input_free(), but assumes that was_input_enable() has not
 * been called yet (no istream handler).
 */
void
was_input_free_unused(WasInput *input);

static inline void
was_input_free_unused_p(WasInput **input_p)
{
    WasInput *input = *input_p;
    *input_p = nullptr;
    was_input_free_unused(input);
}

Istream &
was_input_enable(WasInput &input);

/**
 * Set the new content length of this entity.
 *
 * @return false if the value is invalid (callback "abort" has been
 * invoked in this case)
 */
bool
was_input_set_length(WasInput *input, uint64_t length);

/**
 * Signals premature end of this stream.
 *
 * @param length the total number of bytes the peer has written to the
 * pipe
 * @return true if recovery was successful, false if the object has
 * been closed
 */
bool
was_input_premature(WasInput *input, uint64_t length);

void
was_input_enable_timeout(WasInput *input);

/**
 * Can the piope be released?
 *
 * @see WasInput::release()
 */
bool
was_input_can_release(const WasInput &input);

#endif
<commit_msg>was/input: more WasInputHandler documentation<commit_after>/*
 * Web Application Socket protocol, input data channel library.
 *
 * author: Max Kellermann <mk@cm4all.com>
 */

#ifndef BENG_PROXY_WAS_INPUT_HXX
#define BENG_PROXY_WAS_INPUT_HXX

#include "glibfwd.hxx"

#include <stdint.h>

struct pool;
class Istream;
class WasInput;

class WasInputHandler {
public:
    /**
     * Istream::Close() has been called.
     *
     * The #Istream will be destroyed right after returning from this
     * method; the method should abandon all pointers to it, and not
     * call it.
     *
     * @param received the number of bytes received so far (includes
     * data that hasn't been delivered to the #IstreamHandler yet)
     */
    virtual void WasInputClose(uint64_t received) = 0;

    /**
     * All data was received from the pipe to the input buffer; we
     * don't need the pipe anymore for this request.
     */
    virtual void WasInputRelease() = 0;

    /**
     * Called right before reporting end-of-file to the #IstreamHandler.
     *
     * The #Istream will be destroyed right after returning from this
     * method; the method should abandon all pointers to it, and not
     * call it.
     */
    virtual void WasInputEof() = 0;

    /**
     * There was an I/O error on the pipe.  Called right before
     * reporting the error to the #IstreamHandler.
     *
     * The #Istream will be destroyed right after returning from this
     * method; the method should abandon all pointers to it, and not
     * call it.
     */
    virtual void WasInputError() = 0;
};

WasInput *
was_input_new(struct pool *pool, int fd,
              WasInputHandler &handler);

/**
 * @param error the error reported to the istream handler
 */
void
was_input_free(WasInput *input, GError *error);

static inline void
was_input_free_p(WasInput **input_p, GError *error)
{
    WasInput *input = *input_p;
    *input_p = nullptr;
    was_input_free(input, error);
}

/**
 * Like was_input_free(), but assumes that was_input_enable() has not
 * been called yet (no istream handler).
 */
void
was_input_free_unused(WasInput *input);

static inline void
was_input_free_unused_p(WasInput **input_p)
{
    WasInput *input = *input_p;
    *input_p = nullptr;
    was_input_free_unused(input);
}

Istream &
was_input_enable(WasInput &input);

/**
 * Set the new content length of this entity.
 *
 * @return false if the value is invalid (callback "abort" has been
 * invoked in this case)
 */
bool
was_input_set_length(WasInput *input, uint64_t length);

/**
 * Signals premature end of this stream.
 *
 * @param length the total number of bytes the peer has written to the
 * pipe
 * @return true if recovery was successful, false if the object has
 * been closed
 */
bool
was_input_premature(WasInput *input, uint64_t length);

void
was_input_enable_timeout(WasInput *input);

/**
 * Can the piope be released?
 *
 * @see WasInput::release()
 */
bool
was_input_can_release(const WasInput &input);

#endif
<|endoftext|>


<commit_before>/*
 * author: Max Kellermann <mk@cm4all.com>
 */

#include "Ping.hxx"
#include "pool.hxx"
#include "system/Error.hxx"
#include "net/SocketAddress.hxx"
#include "event/SocketEvent.hxx"
#include "event/Duration.hxx"
#include "util/Cancellable.hxx"

#include <sys/socket.h>
#include <errno.h>
#include <string.h>
#include <netinet/in.h>
#include <netinet/ip_icmp.h>
#include <unistd.h>

class PingClient final : Cancellable {
    struct pool &pool;

    const int fd;

    const uint16_t ident;

    SocketEvent event;

    PingClientHandler &handler;

public:
    PingClient(EventLoop &event_loop, struct pool &_pool,
               int _fd, uint16_t _ident,
               PingClientHandler &_handler,
               CancellablePointer &cancel_ptr)
        :pool(_pool), fd(_fd), ident(_ident),
         event(event_loop, fd, SocketEvent::READ,
               BIND_THIS_METHOD(EventCallback)),
         handler(_handler) {
        cancel_ptr = *this;
    }

    void ScheduleRead() {
        event.Add(EventDuration<10>::value);
    }

private:
    void EventCallback(unsigned events);

    void Read();

    /* virtual methods from class Cancellable */
    void Cancel() override;
};

static u_short
in_cksum(const u_short *addr, register int len, u_short csum)
{
	int nleft = len;
	const u_short *w = addr;
	u_short answer;
	int sum = csum;

	/*
	 *  Our algorithm is simple, using a 32 bit accumulator (sum),
	 *  we add sequential 16 bit words to it, and at the end, fold
	 *  back all the carry bits from the top 16 bits into the lower
	 *  16 bits.
	 */
	while (nleft > 1)  {
		sum += *w++;
		nleft -= 2;
	}

	/* mop up an odd byte, if necessary */
	if (nleft == 1)
		sum += htons(*(const u_char *)w << 8);

	/*
	 * add back carry outs from top 16 bits to low 16 bits
	 */
	sum = (sum >> 16) + (sum & 0xffff);	/* add hi 16 to low 16 */
	sum += (sum >> 16);			/* add carry */
	answer = ~sum;				/* truncate to 16 bits */
	return (answer);
}

static void *
deconst_address(const struct sockaddr *address)
{
    return const_cast<struct sockaddr *>(address);
}

static bool
parse_reply(struct msghdr *msg, size_t cc, uint16_t ident)
{
    const void *buf = (const void *)msg->msg_iov->iov_base;
    const struct icmphdr *icp = (const struct icmphdr *)buf;
    if (cc < sizeof(*icp))
        return false;

    return icp->type == ICMP_ECHOREPLY && icp->un.echo.id == ident;
}

inline void
PingClient::Read()
{
    char buffer[1024];
    struct iovec iov = {
        .iov_base = buffer,
        .iov_len = sizeof(buffer),
    };

    struct msghdr msg;
    memset(&msg, 0, sizeof(msg));

    char addrbuf[128];
    msg.msg_name = addrbuf;
    msg.msg_namelen = sizeof(addrbuf);
    msg.msg_iov = &iov;
    msg.msg_iovlen = 1;
    char ans_data[4096];
    msg.msg_control = ans_data;
    msg.msg_controllen = sizeof(ans_data);

    int cc = recvmsg(fd, &msg, MSG_DONTWAIT);
    if (cc >= 0) {
        if (parse_reply(&msg, cc, ident)) {
            close(fd);
            handler.PingResponse();
            pool_unref(&pool);
        } else
            ScheduleRead();
    } else if (errno == EAGAIN || errno == EINTR) {
        ScheduleRead();
    } else {
        const int e = errno;
        close(fd);
        handler.PingError(std::make_exception_ptr(MakeErrno(e)));
        pool_unref(&pool);
    }
}

/*
 * libevent callback
 *
 */

inline void
PingClient::EventCallback(unsigned events)
{
    assert(fd >= 0);

    if (events & SocketEvent::READ) {
        Read();
    } else {
        close(fd);
        handler.PingTimeout();
        pool_unref(&pool);
    }
}

/*
 * async operation
 *
 */

void
PingClient::Cancel()
{
    event.Delete();
    close(fd);
    pool_unref(&pool);
}

/*
 * constructor
 *
 */

bool
ping_available(void)
{
    int fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_ICMP);
    if (fd < 0)
        return false;
    close(fd);
    return true;
}

void
ping(EventLoop &event_loop, struct pool &pool, SocketAddress address,
     PingClientHandler &handler,
     CancellablePointer &cancel_ptr)
{
    int fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_ICMP);
    if (fd < 0) {
        handler.PingError(std::make_exception_ptr(MakeErrno("Failed to create ping socket")));
        return;
    }

    struct sockaddr_in sin;
    sin.sin_family = AF_INET;
    sin.sin_addr.s_addr = 0;
    socklen_t sin_length = sizeof(sin);
    if (bind(fd, (const struct sockaddr *)&sin, sin_length) < 0 ||
        getsockname(fd, (struct sockaddr *)&sin, &sin_length) < 0) {
        handler.PingError(std::make_exception_ptr(MakeErrno()));
        return;
    }

    uint16_t ident = sin.sin_port;

    struct {
        struct icmphdr header;
        char data[8];
    } packet;

    packet.header.type = ICMP_ECHO;
    packet.header.code = 0;
    packet.header.checksum = 0;
    packet.header.un.echo.sequence = htons(1);
    packet.header.un.echo.id = ident;
    memset(packet.data, 0, sizeof(packet.data));
    packet.header.checksum = in_cksum((u_short *)&packet, sizeof(packet), 0);

    struct iovec iov = {
        .iov_base = &packet,
        .iov_len = sizeof(packet),
    };

    struct msghdr m = {
        .msg_name = deconst_address(address.GetAddress()),
        .msg_namelen = socklen_t(address.GetSize()),
        .msg_iov = &iov,
        .msg_iovlen = 1,
        .msg_control = nullptr,
        .msg_controllen = 0,
        .msg_flags = 0,
    };

    ssize_t nbytes = sendmsg(fd, &m, 0);
    if (nbytes < 0) {
        const int e = errno;
        close(fd);
        handler.PingError(std::make_exception_ptr(MakeErrno(e)));
        return;
    }

    pool_ref(&pool);
    auto p = NewFromPool<PingClient>(pool, event_loop, pool, fd, ident,
                                     handler, cancel_ptr);
    p->ScheduleRead();
}
<commit_msg>net/Ping: call destructor<commit_after>/*
 * author: Max Kellermann <mk@cm4all.com>
 */

#include "Ping.hxx"
#include "pool.hxx"
#include "system/Error.hxx"
#include "net/SocketAddress.hxx"
#include "event/SocketEvent.hxx"
#include "event/Duration.hxx"
#include "util/Cancellable.hxx"

#include <sys/socket.h>
#include <errno.h>
#include <string.h>
#include <netinet/in.h>
#include <netinet/ip_icmp.h>
#include <unistd.h>

class PingClient final : Cancellable {
    struct pool &pool;

    const int fd;

    const uint16_t ident;

    SocketEvent event;

    PingClientHandler &handler;

public:
    PingClient(EventLoop &event_loop, struct pool &_pool,
               int _fd, uint16_t _ident,
               PingClientHandler &_handler,
               CancellablePointer &cancel_ptr)
        :pool(_pool), fd(_fd), ident(_ident),
         event(event_loop, fd, SocketEvent::READ,
               BIND_THIS_METHOD(EventCallback)),
         handler(_handler) {
        cancel_ptr = *this;
    }

    void Destroy() {
        DeleteUnrefPool(pool, this);
    }

    void ScheduleRead() {
        event.Add(EventDuration<10>::value);
    }

private:
    void EventCallback(unsigned events);

    void Read();

    /* virtual methods from class Cancellable */
    void Cancel() override;
};

static u_short
in_cksum(const u_short *addr, register int len, u_short csum)
{
	int nleft = len;
	const u_short *w = addr;
	u_short answer;
	int sum = csum;

	/*
	 *  Our algorithm is simple, using a 32 bit accumulator (sum),
	 *  we add sequential 16 bit words to it, and at the end, fold
	 *  back all the carry bits from the top 16 bits into the lower
	 *  16 bits.
	 */
	while (nleft > 1)  {
		sum += *w++;
		nleft -= 2;
	}

	/* mop up an odd byte, if necessary */
	if (nleft == 1)
		sum += htons(*(const u_char *)w << 8);

	/*
	 * add back carry outs from top 16 bits to low 16 bits
	 */
	sum = (sum >> 16) + (sum & 0xffff);	/* add hi 16 to low 16 */
	sum += (sum >> 16);			/* add carry */
	answer = ~sum;				/* truncate to 16 bits */
	return (answer);
}

static void *
deconst_address(const struct sockaddr *address)
{
    return const_cast<struct sockaddr *>(address);
}

static bool
parse_reply(struct msghdr *msg, size_t cc, uint16_t ident)
{
    const void *buf = (const void *)msg->msg_iov->iov_base;
    const struct icmphdr *icp = (const struct icmphdr *)buf;
    if (cc < sizeof(*icp))
        return false;

    return icp->type == ICMP_ECHOREPLY && icp->un.echo.id == ident;
}

inline void
PingClient::Read()
{
    char buffer[1024];
    struct iovec iov = {
        .iov_base = buffer,
        .iov_len = sizeof(buffer),
    };

    struct msghdr msg;
    memset(&msg, 0, sizeof(msg));

    char addrbuf[128];
    msg.msg_name = addrbuf;
    msg.msg_namelen = sizeof(addrbuf);
    msg.msg_iov = &iov;
    msg.msg_iovlen = 1;
    char ans_data[4096];
    msg.msg_control = ans_data;
    msg.msg_controllen = sizeof(ans_data);

    int cc = recvmsg(fd, &msg, MSG_DONTWAIT);
    if (cc >= 0) {
        if (parse_reply(&msg, cc, ident)) {
            close(fd);
            handler.PingResponse();
            Destroy();
        } else
            ScheduleRead();
    } else if (errno == EAGAIN || errno == EINTR) {
        ScheduleRead();
    } else {
        const int e = errno;
        close(fd);
        handler.PingError(std::make_exception_ptr(MakeErrno(e)));
        Destroy();
    }
}

/*
 * libevent callback
 *
 */

inline void
PingClient::EventCallback(unsigned events)
{
    assert(fd >= 0);

    if (events & SocketEvent::READ) {
        Read();
    } else {
        close(fd);
        handler.PingTimeout();
        Destroy();
    }
}

/*
 * async operation
 *
 */

void
PingClient::Cancel()
{
    event.Delete();
    close(fd);
    Destroy();
}

/*
 * constructor
 *
 */

bool
ping_available(void)
{
    int fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_ICMP);
    if (fd < 0)
        return false;
    close(fd);
    return true;
}

void
ping(EventLoop &event_loop, struct pool &pool, SocketAddress address,
     PingClientHandler &handler,
     CancellablePointer &cancel_ptr)
{
    int fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_ICMP);
    if (fd < 0) {
        handler.PingError(std::make_exception_ptr(MakeErrno("Failed to create ping socket")));
        return;
    }

    struct sockaddr_in sin;
    sin.sin_family = AF_INET;
    sin.sin_addr.s_addr = 0;
    socklen_t sin_length = sizeof(sin);
    if (bind(fd, (const struct sockaddr *)&sin, sin_length) < 0 ||
        getsockname(fd, (struct sockaddr *)&sin, &sin_length) < 0) {
        handler.PingError(std::make_exception_ptr(MakeErrno()));
        return;
    }

    uint16_t ident = sin.sin_port;

    struct {
        struct icmphdr header;
        char data[8];
    } packet;

    packet.header.type = ICMP_ECHO;
    packet.header.code = 0;
    packet.header.checksum = 0;
    packet.header.un.echo.sequence = htons(1);
    packet.header.un.echo.id = ident;
    memset(packet.data, 0, sizeof(packet.data));
    packet.header.checksum = in_cksum((u_short *)&packet, sizeof(packet), 0);

    struct iovec iov = {
        .iov_base = &packet,
        .iov_len = sizeof(packet),
    };

    struct msghdr m = {
        .msg_name = deconst_address(address.GetAddress()),
        .msg_namelen = socklen_t(address.GetSize()),
        .msg_iov = &iov,
        .msg_iovlen = 1,
        .msg_control = nullptr,
        .msg_controllen = 0,
        .msg_flags = 0,
    };

    ssize_t nbytes = sendmsg(fd, &m, 0);
    if (nbytes < 0) {
        const int e = errno;
        close(fd);
        handler.PingError(std::make_exception_ptr(MakeErrno(e)));
        return;
    }

    pool_ref(&pool);
    auto p = NewFromPool<PingClient>(pool, event_loop, pool, fd, ident,
                                     handler, cancel_ptr);
    p->ScheduleRead();
}
<|endoftext|>


<commit_before><commit_msg>prepending po to blockname<commit_after><|endoftext|>


<commit_before><commit_msg>Minor tweaks.<commit_after><|endoftext|>


<commit_before><commit_msg>camera: Handle starting/stopping the camera correctly<commit_after><|endoftext|>


<commit_before><commit_msg>INTEGRATION: CWS impress73 (1.101.30); FILE MERGED 2005/11/03 11:23:56 af 1.101.30.2: RESYNC: (1.101-1.102); FILE MERGED 2005/10/21 13:25:55 cl 1.101.30.1: #i56371# format text if unformated when completing online spelling<commit_after><|endoftext|>


<commit_before><commit_msg>INTEGRATION: CWS sb59 (1.2.488); FILE MERGED 2006/08/28 12:58:09 sb 1.2.488.1: #i67487# Made code warning-free (wntmsci10).<commit_after><|endoftext|>


<commit_before><commit_msg>Slight touch-up of German translation.<commit_after><|endoftext|>


<commit_before>#include <iostream>
#include <ostream>
#include <sstream>
#include <sys/time.h>
#include <zmq.hpp>
#include "chlog.hpp"

using namespace std;

static string
__attribute__ ((format(printf,1,2)))
stringprintf(const char* format, ...) {
    va_list lst;
    char temps[256];
    va_start(lst, format);
    // truncates if length > size of 'temps'
    if (vsnprintf(temps, sizeof(temps), format, lst) < 0)
        throw runtime_error("stringprintf failed: " + string(strerror(errno)));
    va_end(lst);
    return string(temps);
}

#if 0
class chime_log_stream : public std::ostringstream {

public:
    chime_log_stream() {
        cout << "chime_log_stream() constructor" << endl;
    }
    virtual ~chime_log_stream() {
        cout << "~chime_log_stream()" << endl;
        cout << "string: " << str() << endl;
    }

    /*
    virtual basic_ostream& flush() {
        cout << "chime_log_stream::flush()" << endl;
        return *this;
    }

    template<class T>
    chime_log_stream& operator<<(T val) {
        cout << "chime_log_stream:: operator<< " << val << endl;
        return *this;
    }
     */
};
#endif

// inspired by http://stackoverflow.com/questions/2212776/overload-handling-of-stdendl
class chime_log_stream : public std::ostream {
    class MyStreamBuf: public std::stringbuf {
        //std::ostream& output;
    public:
        MyStreamBuf()
        //std::ostream& str)
        // : output(str)
        {}

        virtual int sync() {
            cout << "[blah]" << str();
            str("");
            cout.flush();
            cout << "SYNC" << endl;
            return 0;
        }
    };
    MyStreamBuf buffer;
public:
    chime_log_stream() //std::ostream& str)
        : std::ostream(&buffer),
          buffer()
    {}
};

chime_log_stream chstream;

ostream& chime_log(log_level lev, const char* file, int line, const char* function) {
    struct timeval tv;

    if (gettimeofday(&tv, NULL)) {
        cout << "Error calling gettimeofday(): " << strerror(errno) << endl;
        //return cout;
    }

    struct tm cal;
    if (!gmtime_r(&tv.tv_sec, &cal)) {
        cout << "Error calling gmtime_r(): " << strerror(errno) << endl;
        //return cout;
    }

    /*
     string datestring = stringprintf("%04i-%02i-%02i-%02i:%02i:%02i.%03i",
     cal.tm_year + 1900, cal.tm_mon + 1, cal.tm_mday,
     cal.tm_hour, cal.tm_min, cal.tm_sec,
     int(tv.tv_usec / 1000));
     return cout << "[" << file << ":" << line << " [" << function << "] "
     << datestring << "] ";
     */

    string datestring = stringprintf("%04i-%02i-%02i-%02i:%02i:%02i.%03i",
                                     cal.tm_year + 1900, cal.tm_mon + 1, cal.tm_mday,
                                     cal.tm_hour, cal.tm_min, cal.tm_sec,
                                     int(tv.tv_usec / 1000));

    //chime_log_stream stream;
    //stringbuf stream;
    //ostringstream* stream = new chime_log_stream(); //ostringstream();
    chstream << "[" << file << ":" << line << " [" << function << "] "
           << datestring << "] ";
    return chstream;
}

int main() {
    //chlog();
    chlog("Hello world");
    chlog("Hello " << 1 << ", " << 2 << ", 3");
    chdebug("Debug" << 42+43);
}

<commit_msg>basic PUB/SUB distributed logger<commit_after>#include <thread>
#include <functional>
#include <iostream>
#include <ostream>
#include <sstream>
#include <sys/time.h>
#include <unistd.h>
#include <zmq.hpp>
#include "chlog.hpp"

using namespace std;

static string
__attribute__ ((format(printf,1,2)))
stringprintf(const char* format, ...) {
    va_list lst;
    char temps[256];
    va_start(lst, format);
    // truncates if length > size of 'temps'
    if (vsnprintf(temps, sizeof(temps), format, lst) < 0)
        throw runtime_error("stringprintf failed: " + string(strerror(errno)));
    va_end(lst);
    return string(temps);
}

class chime_log_socket {
public:
     chime_log_socket(zmq::context_t &ctx) :
         //_ctx(ctx),
         _socket(ctx, ZMQ_PUB)
    {}

    void add_server(std::string port) {
        cout << "chime_log connecting to " << port << endl;
        _socket.connect(port);
    }

    void send(std::string msg) {
        cout << "chime_log sending message: " << msg << endl;
        size_t n = _socket.send(static_cast<const void*>(msg.data()), msg.size());
        cout << "sent " << n << endl;
    }

protected:
    //zmq::context_t &_ctx;

    zmq::socket_t _socket;

};

static chime_log_socket* logsock = NULL;

void chime_log_init(zmq::context_t &ctx) {
    if (logsock)
        delete logsock;
    logsock = new chime_log_socket(ctx);
}

void chime_log_add_server(string port) {
    if (!logsock)
        return;
    logsock->add_server(port);
}

static void chime_send_log(const string &s) {
    if (logsock) {
        logsock->send(s);
    } else {
        cout << s;
        cout.flush();
        cout << "SEND" << endl;
    }
}

// inspired by http://stackoverflow.com/questions/2212776/overload-handling-of-stdendl
class chime_log_stream : public std::ostream {
public:
    chime_log_stream()
        : std::ostream(&buffer),
          buffer()
    {}
protected:
    // A little string buffer subclass that intercepts the sync() call.
    class chstringbuf : public std::stringbuf {
    public:
        chstringbuf() {}

        virtual int sync() {
            chime_send_log(str());
            str("");
            return 0;
        }
    };
    chstringbuf buffer;
};

static chime_log_stream chstream;

ostream& chime_log(log_level lev, const char* file, int line, const char* function) {
    struct timeval tv;

    if (gettimeofday(&tv, NULL)) {
        cout << "Error calling gettimeofday(): " << strerror(errno) << endl;
        return chstream;
    }

    struct tm cal;
    if (!gmtime_r(&tv.tv_sec, &cal)) {
        cout << "Error calling gmtime_r(): " << strerror(errno) << endl;
        return chstream;
    }

    string datestring = stringprintf("%04i-%02i-%02i-%02i:%02i:%02i.%03i",
                                     cal.tm_year + 1900, cal.tm_mon + 1, cal.tm_mday,
                                     cal.tm_hour, cal.tm_min, cal.tm_sec,
                                     int(tv.tv_usec / 1000));

    chstream << "[" << file << ":" << line << " [" << function << "] "
           << datestring << "] ";
    return chstream;
}

static string msg_string(zmq::message_t &msg) {
    return string(static_cast<const char*>(msg.data()), msg.size());
}

std::mutex cout_mutex;

static void server_main(zmq::context_t* ctx, string port) {
    zmq::socket_t sock(*ctx, ZMQ_SUB);
    sock.setsockopt(ZMQ_SUBSCRIBE, "", 0);
    sock.bind(port);
    cout << "server_main: bound " << port << endl;
    for (;;) {
        zmq::message_t msg;
        cout << "server_main: waiting to receive message." << endl;
        sock.recv(&msg);

        {
            std::lock_guard<std::mutex> lock(cout_mutex);
            cout << "Server received message: " << msg_string(msg) << endl;
        }
    }
}

int main() {
    zmq::context_t ctx;

    string port = "tcp://127.0.0.1:6666";
    thread serverthread(std::bind(server_main, &ctx, port));

    string port2 = "tcp://127.0.0.1:6667";
    thread serverthread2(std::bind(server_main, &ctx, port2));

    chime_log_init(ctx);
    chime_log_add_server(port);
    chime_log_add_server(port2);

    usleep(1000000);

    chlog("Hello world");
    chlog("Hello " << 1 << ", " << 2 << ", 3");
    chdebug("Debug" << 42+43);

    usleep(1000000);
}

<|endoftext|>


<commit_before><commit_msg>删去了并没有什么用的C++代码<commit_after><|endoftext|>


<commit_before><commit_msg>Added generation of jump to global code<commit_after><|endoftext|>