<commit_before>/** \file    add_synonyms.cc
 *  \brief   Generic version for augmenting title data with synonyms found
             in the authority data
 *  \author  Johannes Riedl
 */

/*
    Copyright (C) 2016, Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

/*  We offer a list of tags and subfields where the primary data resides along
    with a list of tags and subfields where the synonym data is found and
    a list of unused fields in the title data where the synonyms can be stored
*/

#include <iostream>
#include <map>
#include <vector>
#include <cstdlib>
#include "Compiler.h"
#include "FileUtil.h"
#include "MarcReader.h"
#include "MarcRecord.h"
#include "MarcWriter.h"
#include "MediaTypeUtil.h"
#include "RegexMatcher.h"
#include "StringUtil.h"
#include "Subfields.h"
#include "util.h"

static unsigned modified_count(0);
static unsigned record_count(0);

void Usage() {
    std::cerr << "Usage: " << ::progname << " master_marc_input norm_data_marc_input marc_output\n";
    std::exit(EXIT_FAILURE);
}

std::string GetTag(const std::string &tag_and_subfields_spec) {
    return tag_and_subfields_spec.substr(0, 3);
}

std::string GetSubfieldCodes(const std::string &tag_and_subfields_spec) {
    return tag_and_subfields_spec.substr(3);
}

bool FilterPasses(const MarcRecord &record, const std::map<std::string, std::pair<std::string, std::string>> &filter_specs, const std::string &field_spec) {
      auto filter_spec(filter_specs.find(field_spec));
      if (filter_spec == filter_specs.cend())
          return true;

      auto rule(filter_spec->second);
      // We have field_spec in key and rule to match in value
      std::string subfield_codes(GetSubfieldCodes(rule.first));
      if (subfield_codes.length() != 1)
         Error("Invalid subfield specification "  + subfield_codes + " for filter");

      std::string subfield_value;
      if ((subfield_value = record.extractFirstSubfield(GetTag(rule.first), subfield_codes.c_str()[0])).empty())
          return false;

      return subfield_value == rule.second;
}

void ExtractSynonyms(MarcReader * const authority_reader,
                     const std::set<std::string> &primary_tags_and_subfield_codes,
                     const std::set<std::string> &synonym_tags_and_subfield_codes,
                     std::vector<std::map<std::string, std::string>> * const synonym_maps,
                     const std::map<std::string, std::pair<std::string, std::string>> &filter_spec)
{
    while (const MarcRecord record = authority_reader->read()) {
        std::set<std::string>::const_iterator primary;
        std::set<std::string>::const_iterator synonym;
        unsigned int i(0);
        for (primary = primary_tags_and_subfield_codes.begin(), synonym = synonym_tags_and_subfield_codes.begin();
            primary != primary_tags_and_subfield_codes.end();
            ++primary, ++synonym, ++i)
        {
            // Fill maps with synonyms
            std::vector<std::string> primary_values;
            std::vector<std::string> synonym_values;

            if (FilterPasses(record, filter_spec, *primary) and
                record.extractSubfields(GetTag(*primary), GetSubfieldCodes(*primary), &primary_values) and
                record.extractSubfields(GetTag(*synonym), GetSubfieldCodes(*synonym), &synonym_values))
                    (*synonym_maps)[i].emplace(StringUtil::Join(primary_values, ','),
                                               StringUtil::Join(synonym_values, ','));
        }
    }
}

inline std::string GetMapValueOrEmptyString(const std::map<std::string, std::string> &map,
                                            const std::string &searchterm)
{
    auto value(map.find(searchterm));
    return (value != map.cend()) ? value->second : "";
}

void ProcessRecord(MarcRecord * const record, const std::vector<std::map<std::string, std::string>> &synonym_maps,
                   const std::set<std::string> &primary_tags_and_subfield_codes,
                   const std::set<std::string> &output_tags_and_subfield_codes)
{
    std::set<std::string>::const_iterator primary;
    std::set<std::string>::const_iterator output;
    unsigned int i(0);

    if (primary_tags_and_subfield_codes.size() == output_tags_and_subfield_codes.size()) {
        for (primary = primary_tags_and_subfield_codes.begin(), output = output_tags_and_subfield_codes.begin();
            primary != primary_tags_and_subfield_codes.end();
            ++primary, ++output, ++i)
        {
            std::vector<std::string> primary_values;
            std::set<std::string> synonym_values;
            std::vector<size_t> field_indices;
            if (record->getFieldIndices(GetTag(*primary), &field_indices) != MarcRecord::FIELD_NOT_FOUND) {
                for (auto field_index : field_indices) {
                     primary_values.clear();
                     if (record->getSubfields(field_index).extractSubfields(GetSubfieldCodes(*primary), &primary_values)) {
                        std::string searchterm = StringUtil::Join(primary_values, ',');
                        // First case: Look up synonyms only in one category
                        if (i < synonym_maps.size()) {
                            const auto &synonym_map(synonym_maps[i]);
                            const auto synonym(GetMapValueOrEmptyString(synonym_map, searchterm));
                            if (not synonym.empty())
                                synonym_values.insert(synonym);
                        }

                        // Second case: Look up synonyms in all categories
                        else {
                            for (auto &synonym_map : synonym_maps) {
                                const auto &synonym(GetMapValueOrEmptyString(synonym_map, searchterm));
                                if (not synonym.empty())
                                   synonym_values.insert(synonym);
                            }
                        }
                    }
                 }

                if (synonym_values.empty())
                     continue;

                // Insert synonyms
                // Abort if field is already populated
                std::string tag(GetTag(*output));
                if (record->getFieldIndex(tag) != MarcRecord::FIELD_NOT_FOUND)
                    Error("Field with tag " + tag + " is not empty for PPN " + record->getControlNumber() + '\n');
                std::string subfield_spec = GetSubfieldCodes(*output);
                if (unlikely(subfield_spec.size() != 1))
                    Error("We currently only support a single subfield and thus specifying " + subfield_spec
                          + " as output subfield is not valid\n");

                std::string synonyms;
                unsigned current_length(0);
                unsigned indicator2(0);

                for (auto synonym_it(synonym_values.cbegin()); synonym_it != synonym_values.cend(); /*Intentionally empty*/) {

                    if (indicator2 > 9)
                        Error ("Currently cannot handle synonyms with total length greater " + std::to_string(9 * MarcRecord::MAX_FIELD_LENGTH) + '\n');

                    if (current_length + (*synonym_it).length() < MarcRecord::MAX_FIELD_LENGTH) {
                         bool synonyms_empty(synonyms.empty());
                         synonyms += (synonyms_empty ? *synonym_it : " , " + *synonym_it);
                         current_length += (*synonym_it).length() + (synonyms_empty ? 0 : 3);
                         ++synonym_it;
                    }
                    else {
                        if (not(record->insertSubfield(tag, subfield_spec.at(0), synonyms, '0', indicator2 + '0')))
                            Error("Could not insert field " + tag + " for PPN " + record->getControlNumber() + '\n');
                        synonyms = "";
                        current_length = 0;
                        ++indicator2;
                    }
                }
                // Write rest of data
                if (not synonyms.empty()) {
                    if (not(record->insertSubfield(tag, subfield_spec.at(0), synonyms, '0', indicator2 + '0')))
                            Error("Could not insert field " + tag + " for PPN " + record->getControlNumber() + '\n');
                }
                ++modified_count;
            }
        }
    }
}

void InsertSynonyms(MarcReader * const marc_reader, MarcWriter * const marc_writer,
                    const std::set<std::string> &primary_tags_and_subfield_codes,
                    const std::set<std::string> &output_tags_and_subfield_codes,
                    std::vector<std::map<std::string, std::string>> &synonym_maps)
{
    while (MarcRecord record = marc_reader->read()) {
        ProcessRecord(&record, synonym_maps, primary_tags_and_subfield_codes, output_tags_and_subfield_codes);
        marc_writer->write(record);
        ++record_count;
    }

    std::cerr << "Modified " << modified_count << " of " << record_count << " record(s).\n";
}

bool ParseSpec(std::string spec_str, std::set<std::string> * const field_specs, std::map<std::string, std::pair<std::string, std::string>> * filter_specs = nullptr) {
    std::set<std::string> raw_field_specs;

    if (unlikely(StringUtil::Split(spec_str, ':', &raw_field_specs) == 0)) {
        Error("Need at least one field");
        return false;
    }

    if (filter_specs == nullptr) {
        *field_specs = raw_field_specs;
	return true;
    }

    // Iterate over all Field-specs and extract possible filters
    static RegexMatcher * const matcher(RegexMatcher::RegexMatcherFactory("(\\d{1,3}[a-z]+)\\[(\\d{1,3}[a-z])=(.*)\\]"));

    for (auto field_spec : raw_field_specs) {
        if (matcher->matched(field_spec)) {
            filter_specs->emplace((*matcher)[1], std::make_pair((*matcher)[2], (*matcher)[3]));
            auto bracket = field_spec.find("[");
            field_spec = (bracket != std::string::npos) ? field_spec.erase(bracket, field_spec.length()) : field_spec;
        }
        field_specs->insert(field_spec);
    }
    return true;
}

int main(int argc, char **argv) {
    ::progname = argv[0];

    if (argc != 4)
        Usage();

    const std::string marc_input_filename(argv[1]);
    const std::string authority_data_marc_input_filename(argv[2]);
    const std::string marc_output_filename(argv[3]);
    if (unlikely(marc_input_filename == marc_output_filename))
        Error("Title data input file name equals output file name!");
    if (unlikely(authority_data_marc_input_filename == marc_output_filename))
        Error("Authority data input file name equals output file name!");

    std::unique_ptr<MarcReader> marc_reader(MarcReader::Factory(marc_input_filename, MarcReader::BINARY));
    std::unique_ptr<MarcReader> authority_reader(MarcReader::Factory(authority_data_marc_input_filename,
                                                                     MarcReader::BINARY));
    std::unique_ptr<MarcWriter> marc_writer(MarcWriter::Factory(marc_output_filename, MarcWriter::BINARY));

    try {
        // Determine possible mappings
        // Values in square brackets specify a positive criterion for values to be taken into account
        const std::string AUTHORITY_DATA_PRIMARY_SPEC("100abcd[079v=piz]:110abcd:111abcd:130abcd:150abcd:151abcd");
        const std::string AUTHORITY_DATA_SYNONYM_SPEC("400abcd:410abcd:411abcd:430abcd:450abcd:451abcd");
        const std::string TITLE_DATA_PRIMARY_SPEC("600abcd:610abcd:611abcd:630abcd:650abcd:651abcd:689abcd");
        const std::string TITLE_DATA_UNUSED_FIELDS_FOR_SYNONYMS("180a:181a:182a:183a:184a:185a:186a");

        // Determine fields to handle
        std::set<std::string> primary_tags_and_subfield_codes;
        std::set<std::string> synonym_tags_and_subfield_codes;
        std::set<std::string> input_tags_and_subfield_codes;
        std::set<std::string> output_tags_and_subfield_codes;

        std::map<std::string, std::pair<std::string, std::string>> filter_specs;

        if (not unlikely(ParseSpec(AUTHORITY_DATA_PRIMARY_SPEC, &primary_tags_and_subfield_codes, &filter_specs)))
            Error("Could not properly parse " + AUTHORITY_DATA_PRIMARY_SPEC);

        if (unlikely(StringUtil::Split(AUTHORITY_DATA_SYNONYM_SPEC, ":", &synonym_tags_and_subfield_codes) == 0))
            Error("Need at least one synonym field");

        if (unlikely(StringUtil::Split(TITLE_DATA_PRIMARY_SPEC, ":", &input_tags_and_subfield_codes) == 0))
            Error("Need at least one input field");

        if (unlikely(StringUtil::Split(TITLE_DATA_UNUSED_FIELDS_FOR_SYNONYMS, ":", &output_tags_and_subfield_codes)
                     == 0))
            Error("Need at least one output field");

        unsigned num_of_authority_entries(primary_tags_and_subfield_codes.size());

        if (synonym_tags_and_subfield_codes.size() != num_of_authority_entries)
            Error("Number of authority primary specs must match number of synonym specs");
        if (input_tags_and_subfield_codes.size() != output_tags_and_subfield_codes.size())
            Error("Number of fields title entry specs must match number of output specs");

        std::vector<std::map<std::string, std::string>> synonym_maps(num_of_authority_entries,
                                                                     std::map<std::string, std::string>());

        // Extract the synonyms from authority data
        ExtractSynonyms(authority_reader.get(), primary_tags_and_subfield_codes, synonym_tags_and_subfield_codes,
                        &synonym_maps, filter_specs);

        // Iterate over the title data
        InsertSynonyms(marc_reader.get(), marc_writer.get(), input_tags_and_subfield_codes,
                       output_tags_and_subfield_codes, synonym_maps);

    } catch (const std::exception &x) {
        Error("caught exception: " + std::string(x.what()));
    }
}
<commit_msg>Minor changes<commit_after>/** \file    add_synonyms.cc
 *  \brief   Generic version for augmenting title data with synonyms found
             in the authority data
 *  \author  Johannes Riedl
 */

/*
    Copyright (C) 2016, Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

/*  We offer a list of tags and subfields where the primary data resides along
    with a list of tags and subfields where the synonym data is found and
    a list of unused fields in the title data where the synonyms can be stored
*/

#include <iostream>
#include <map>
#include <vector>
#include <cstdlib>
#include "Compiler.h"
#include "FileUtil.h"
#include "MarcReader.h"
#include "MarcRecord.h"
#include "MarcWriter.h"
#include "MediaTypeUtil.h"
#include "RegexMatcher.h"
#include "StringUtil.h"
#include "Subfields.h"
#include "util.h"

static unsigned modified_count(0);
static unsigned record_count(0);

void Usage() {
    std::cerr << "Usage: " << ::progname << " master_marc_input norm_data_marc_input marc_output\n";
    std::exit(EXIT_FAILURE);
}

std::string GetTag(const std::string &tag_and_subfields_spec) {
    return tag_and_subfields_spec.substr(0, 3);
}

std::string GetSubfieldCodes(const std::string &tag_and_subfields_spec) {
    return tag_and_subfields_spec.substr(3);
}

bool FilterPasses(const MarcRecord &record, const std::map<std::string, std::pair<std::string, std::string>> &filter_specs, const std::string &field_spec) {
      auto filter_spec(filter_specs.find(field_spec));
      if (filter_spec == filter_specs.cend())
          return true;

      auto rule(filter_spec->second);
      // We have field_spec in key and rule to match in value
      std::string subfield_codes(GetSubfieldCodes(rule.first));
      if (subfield_codes.length() != 1)
         Error("Invalid subfield specification "  + subfield_codes + " for filter");

      std::string subfield_value;
      if ((subfield_value = record.extractFirstSubfield(GetTag(rule.first), subfield_codes.c_str()[0])).empty())
          return false;

      return subfield_value == rule.second;
}

void ExtractSynonyms(MarcReader * const authority_reader,
                     const std::set<std::string> &primary_tags_and_subfield_codes,
                     const std::set<std::string> &synonym_tags_and_subfield_codes,
                     std::vector<std::map<std::string, std::string>> * const synonym_maps,
                     const std::map<std::string, std::pair<std::string, std::string>> &filter_spec)
{
    while (const MarcRecord record = authority_reader->read()) {
        std::set<std::string>::const_iterator primary;
        std::set<std::string>::const_iterator synonym;
        unsigned int i(0);
        for (primary = primary_tags_and_subfield_codes.begin(), synonym = synonym_tags_and_subfield_codes.begin();
            primary != primary_tags_and_subfield_codes.end();
            ++primary, ++synonym, ++i)
        {
            // Fill maps with synonyms
            std::vector<std::string> primary_values;
            std::vector<std::string> synonym_values;

            if (FilterPasses(record, filter_spec, *primary) and
                record.extractSubfields(GetTag(*primary), GetSubfieldCodes(*primary), &primary_values) and
                record.extractSubfields(GetTag(*synonym), GetSubfieldCodes(*synonym), &synonym_values))
                    (*synonym_maps)[i].emplace(StringUtil::Join(primary_values, ','),
                                               StringUtil::Join(synonym_values, ','));
        }
    }
}

inline std::string GetMapValueOrEmptyString(const std::map<std::string, std::string> &map,
                                            const std::string &searchterm)
{
    auto value(map.find(searchterm));
    return (value != map.cend()) ? value->second : "";
}

void ProcessRecord(MarcRecord * const record, const std::vector<std::map<std::string, std::string>> &synonym_maps,
                   const std::set<std::string> &primary_tags_and_subfield_codes,
                   const std::set<std::string> &output_tags_and_subfield_codes)
{
    std::set<std::string>::const_iterator primary;
    std::set<std::string>::const_iterator output;
    unsigned int i(0);

    if (primary_tags_and_subfield_codes.size() == output_tags_and_subfield_codes.size()) {
        for (primary = primary_tags_and_subfield_codes.begin(), output = output_tags_and_subfield_codes.begin();
            primary != primary_tags_and_subfield_codes.end();
            ++primary, ++output, ++i)
        {
            std::vector<std::string> primary_values;
            std::set<std::string> synonym_values;
            std::vector<size_t> field_indices;
            if (record->getFieldIndices(GetTag(*primary), &field_indices) != MarcRecord::FIELD_NOT_FOUND) {
                for (auto field_index : field_indices) {
                     primary_values.clear();
                     if (record->getSubfields(field_index).extractSubfields(GetSubfieldCodes(*primary), &primary_values)) {
                        std::string searchterm = StringUtil::Join(primary_values, ',');
                        // First case: Look up synonyms only in one category
                        if (i < synonym_maps.size()) {
                            const auto &synonym_map(synonym_maps[i]);
                            const auto synonym(GetMapValueOrEmptyString(synonym_map, searchterm));
                            if (not synonym.empty())
                                synonym_values.insert(synonym);
                        }

                        // Second case: Look up synonyms in all categories
                        else {
                            for (auto &synonym_map : synonym_maps) {
                                const auto &synonym(GetMapValueOrEmptyString(synonym_map, searchterm));
                                if (not synonym.empty())
                                   synonym_values.insert(synonym);
                            }
                        }
                    }
                 }

                if (synonym_values.empty())
                     continue;

                // Insert synonyms
                // Abort if field is already populated
                std::string tag(GetTag(*output));
                if (record->getFieldIndex(tag) != MarcRecord::FIELD_NOT_FOUND)
                    Error("Field with tag " + tag + " is not empty for PPN " + record->getControlNumber() + '\n');
                std::string subfield_spec = GetSubfieldCodes(*output);
                if (unlikely(subfield_spec.size() != 1))
                    Error("We currently only support a single subfield and thus specifying " + subfield_spec
                          + " as output subfield is not valid\n");

                std::string synonyms;
                unsigned current_length(0);
                unsigned indicator2(0);

                for (auto synonym_it(synonym_values.cbegin()); synonym_it != synonym_values.cend(); /*Intentionally empty*/) {

                    if (indicator2 > 9)
                        Error ("Currently cannot handle synonyms with total length greater than " + std::to_string(9 * MarcRecord::MAX_FIELD_LENGTH) + '\n');

                    if (current_length + (*synonym_it).length() < MarcRecord::MAX_FIELD_LENGTH) {
                         bool synonyms_empty(synonyms.empty());
                         synonyms += (synonyms_empty ? *synonym_it : " , " + *synonym_it);
                         current_length += (*synonym_it).length() + (synonyms_empty ? 0 : 3);
                         ++synonym_it;
                    } else {
                        if (not(record->insertSubfield(tag, subfield_spec.at(0), synonyms, '0', indicator2 + '0')))
                            Error("Could not insert field " + tag + " for PPN " + record->getControlNumber() + '\n');
                        synonyms = "";
                        current_length = 0;
                        ++indicator2;
                    }
                }
                // Write rest of data
                if (not synonyms.empty()) {
                    if (not(record->insertSubfield(tag, subfield_spec.at(0), synonyms, '0', indicator2 + '0')))
                            Error("Could not insert field " + tag + " for PPN " + record->getControlNumber() + '\n');
                }
                ++modified_count;
            }
        }
    }
}

void InsertSynonyms(MarcReader * const marc_reader, MarcWriter * const marc_writer,
                    const std::set<std::string> &primary_tags_and_subfield_codes,
                    const std::set<std::string> &output_tags_and_subfield_codes,
                    std::vector<std::map<std::string, std::string>> &synonym_maps)
{
    while (MarcRecord record = marc_reader->read()) {
        ProcessRecord(&record, synonym_maps, primary_tags_and_subfield_codes, output_tags_and_subfield_codes);
        marc_writer->write(record);
        ++record_count;
    }

    std::cerr << "Modified " << modified_count << " of " << record_count << " record(s).\n";
}

bool ParseSpec(std::string spec_str, std::set<std::string> * const field_specs, std::map<std::string, std::pair<std::string, std::string>> * filter_specs = nullptr) {
    std::set<std::string> raw_field_specs;

    if (unlikely(StringUtil::Split(spec_str, ':', &raw_field_specs) == 0)) {
        Error("Need at least one field");
        return false;
    }

    if (filter_specs == nullptr) {
        *field_specs = raw_field_specs;
	return true;
    }

    // Iterate over all Field-specs and extract possible filters
    static RegexMatcher * const matcher(RegexMatcher::RegexMatcherFactory("(\\d{1,3}[a-z]+)\\[(\\d{1,3}[a-z])=(.*)\\]"));

    for (auto field_spec : raw_field_specs) {
        if (matcher->matched(field_spec)) {
            filter_specs->emplace((*matcher)[1], std::make_pair((*matcher)[2], (*matcher)[3]));
            auto bracket = field_spec.find("[");
            field_spec = (bracket != std::string::npos) ? field_spec.erase(bracket, field_spec.length()) : field_spec;
        }
        field_specs->insert(field_spec);
    }
    return true;
}

int main(int argc, char **argv) {
    ::progname = argv[0];

    if (argc != 4)
        Usage();

    const std::string marc_input_filename(argv[1]);
    const std::string authority_data_marc_input_filename(argv[2]);
    const std::string marc_output_filename(argv[3]);
    if (unlikely(marc_input_filename == marc_output_filename))
        Error("Title data input file name equals output file name!");
    if (unlikely(authority_data_marc_input_filename == marc_output_filename))
        Error("Authority data input file name equals output file name!");

    std::unique_ptr<MarcReader> marc_reader(MarcReader::Factory(marc_input_filename, MarcReader::BINARY));
    std::unique_ptr<MarcReader> authority_reader(MarcReader::Factory(authority_data_marc_input_filename,
                                                                     MarcReader::BINARY));
    std::unique_ptr<MarcWriter> marc_writer(MarcWriter::Factory(marc_output_filename, MarcWriter::BINARY));

    try {
        // Determine possible mappings
        // Values in square brackets specify a positive criterion for values to be taken into account
        const std::string AUTHORITY_DATA_PRIMARY_SPEC("100abcd[079v=piz]:110abcd:111abcd:130abcd:150abcd:151abcd");
        const std::string AUTHORITY_DATA_SYNONYM_SPEC("400abcd:410abcd:411abcd:430abcd:450abcd:451abcd");
        const std::string TITLE_DATA_PRIMARY_SPEC("600abcd:610abcd:611abcd:630abcd:650abcd:651abcd:689abcd");
        const std::string TITLE_DATA_UNUSED_FIELDS_FOR_SYNONYMS("180a:181a:182a:183a:184a:185a:186a");

        // Determine fields to handle
        std::set<std::string> primary_tags_and_subfield_codes;
        std::set<std::string> synonym_tags_and_subfield_codes;
        std::set<std::string> input_tags_and_subfield_codes;
        std::set<std::string> output_tags_and_subfield_codes;

        std::map<std::string, std::pair<std::string, std::string>> filter_specs;

        if (not unlikely(ParseSpec(AUTHORITY_DATA_PRIMARY_SPEC, &primary_tags_and_subfield_codes, &filter_specs)))
            Error("Could not properly parse " + AUTHORITY_DATA_PRIMARY_SPEC);

        if (unlikely(StringUtil::Split(AUTHORITY_DATA_SYNONYM_SPEC, ":", &synonym_tags_and_subfield_codes) == 0))
            Error("Need at least one synonym field");

        if (unlikely(StringUtil::Split(TITLE_DATA_PRIMARY_SPEC, ":", &input_tags_and_subfield_codes) == 0))
            Error("Need at least one input field");

        if (unlikely(StringUtil::Split(TITLE_DATA_UNUSED_FIELDS_FOR_SYNONYMS, ":", &output_tags_and_subfield_codes)
                     == 0))
            Error("Need at least one output field");

        unsigned num_of_authority_entries(primary_tags_and_subfield_codes.size());

        if (synonym_tags_and_subfield_codes.size() != num_of_authority_entries)
            Error("Number of authority primary specs must match number of synonym specs");
        if (input_tags_and_subfield_codes.size() != output_tags_and_subfield_codes.size())
            Error("Number of fields title entry specs must match number of output specs");

        std::vector<std::map<std::string, std::string>> synonym_maps(num_of_authority_entries,
                                                                     std::map<std::string, std::string>());

        // Extract the synonyms from authority data
        ExtractSynonyms(authority_reader.get(), primary_tags_and_subfield_codes, synonym_tags_and_subfield_codes,
                        &synonym_maps, filter_specs);

        // Iterate over the title data
        InsertSynonyms(marc_reader.get(), marc_writer.get(), input_tags_and_subfield_codes,
                       output_tags_and_subfield_codes, synonym_maps);

    } catch (const std::exception &x) {
        Error("caught exception: " + std::string(x.what()));
    }
}
<|endoftext|>


<commit_before>/** \file    add_synonyms.cc
 *  \brief   Generic version for augmenting title data with synonyms found
             in the authority data
 *  \author  Johannes Riedl
 */

/*
    Copyright (C) 2016, Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

/*  We offer a list of tags and subfields where the primary data resides along
    with a list of tags and subfields where the synonym data is found and
    a list of unused fields in the title data where the synonyms can be stored
*/

#include <iostream>
#include <map>
#include <vector>
#include <cstdlib>
#include "Compiler.h"
#include "FileUtil.h"
#include "MarcReader.h"
#include "MarcRecord.h"
#include "MarcWriter.h"
#include "MediaTypeUtil.h"
#include "RegexMatcher.h"
#include "StringUtil.h"
#include "Subfields.h"
#include "util.h"

static unsigned modified_count(0);
static unsigned record_count(0);

void Usage() {
    std::cerr << "Usage: " << ::progname << " master_marc_input norm_data_marc_input marc_output\n";
    std::exit(EXIT_FAILURE);
}

std::string GetTag(const std::string &tag_and_subfields_spec) {
    return tag_and_subfields_spec.substr(0, 3);
}

std::string GetSubfieldCodes(const std::string &tag_and_subfields_spec) {
    return tag_and_subfields_spec.substr(3);
}

bool FilterPasses(const MarcRecord &record, const std::map<std::string, std::pair<std::string, std::string>> &filter_specs, const std::string &field_spec) {
      auto filter_spec(filter_specs.find(field_spec));
      if (filter_spec == filter_specs.cend())
          return true;

      auto rule(filter_spec->second);
      // We have field_spec in key and rule to match in value
      std::string subfield_codes(GetSubfieldCodes(rule.first));
      if (subfield_codes.length() != 1)
         Error("Invalid subfield specification "  + subfield_codes + " for filter");

      std::string subfield_value;
      if ((subfield_value = record.extractFirstSubfield(GetTag(rule.first), subfield_codes.c_str()[0])).empty())
          return false;

      return subfield_value == rule.second;
}

void ExtractSynonyms(MarcReader * const authority_reader,
                     const std::set<std::string> &primary_tags_and_subfield_codes,
                     const std::set<std::string> &synonym_tags_and_subfield_codes,
                     std::vector<std::map<std::string, std::string>> * const synonym_maps,
                     const std::map<std::string, std::pair<std::string, std::string>> &filter_spec)
{
    while (const MarcRecord record = authority_reader->read()) {
        std::set<std::string>::const_iterator primary;
        std::set<std::string>::const_iterator synonym;
        unsigned int i(0);
        for (primary = primary_tags_and_subfield_codes.begin(), synonym = synonym_tags_and_subfield_codes.begin();
            primary != primary_tags_and_subfield_codes.end();
            ++primary, ++synonym, ++i)
        {
            // Fill maps with synonyms
            std::vector<std::string> primary_values;
            std::vector<std::string> synonym_values;

            if (FilterPasses(record, filter_spec, *primary) and
                record.extractSubfields(GetTag(*primary), GetSubfieldCodes(*primary), &primary_values) and
                record.extractSubfields(GetTag(*synonym), GetSubfieldCodes(*synonym), &synonym_values))
                    (*synonym_maps)[i].emplace(StringUtil::Join(primary_values, ','),
                                               StringUtil::Join(synonym_values, ','));
        }
    }
}

inline std::string GetMapValueOrEmptyString(const std::map<std::string, std::string> &map,
                                            const std::string &searchterm)
{
    auto value(map.find(searchterm));
    return (value != map.cend()) ? value->second : "";
}

void ProcessRecord(MarcRecord * const record, const std::vector<std::map<std::string, std::string>> &synonym_maps,
                   const std::set<std::string> &primary_tags_and_subfield_codes,
                   const std::set<std::string> &output_tags_and_subfield_codes)
{
    std::set<std::string>::const_iterator primary;
    std::set<std::string>::const_iterator output;
    unsigned int i(0);

    if (primary_tags_and_subfield_codes.size() == output_tags_and_subfield_codes.size()) {
        for (primary = primary_tags_and_subfield_codes.begin(), output = output_tags_and_subfield_codes.begin();
            primary != primary_tags_and_subfield_codes.end();
            ++primary, ++output, ++i)
        {
            std::vector<std::string> primary_values;
            std::set<std::string> synonym_values;
            std::vector<size_t> field_indices;
            if (record->getFieldIndices(GetTag(*primary), &field_indices) != MarcRecord::FIELD_NOT_FOUND) {
                for (auto field_index : field_indices) {
                     primary_values.clear();
                     if (record->getSubfields(field_index).extractSubfields(GetSubfieldCodes(*primary), &primary_values)) {
                        std::string searchterm = StringUtil::Join(primary_values, ',');
                        // First case: Look up synonyms only in one category
                        if (i < synonym_maps.size()) {
                            const auto &synonym_map(synonym_maps[i]);
                            const auto synonym(GetMapValueOrEmptyString(synonym_map, searchterm));
                            if (not synonym.empty())
                                synonym_values.insert(synonym);
                        }

                        // Second case: Look up synonyms in all categories
                        else {
                            for (auto &synonym_map : synonym_maps) {
                                const auto &synonym(GetMapValueOrEmptyString(synonym_map, searchterm));
                                if (not synonym.empty())
                                   synonym_values.insert(synonym);
                            }
                        }
                    }
                 }

                if (synonym_values.empty())
                     continue;

                // Insert synonyms
                // Abort if field is already populated
                std::string tag(GetTag(*output));
                if (record->getFieldIndex(tag) != MarcRecord::FIELD_NOT_FOUND)
                    Error("Field with tag " + tag + " is not empty for PPN " + record->getControlNumber() + '\n');
                std::string subfield_spec = GetSubfieldCodes(*output);
                if (unlikely(subfield_spec.size() != 1))
                    Error("We currently only support a single subfield and thus specifying " + subfield_spec
                          + " as output subfield is not valid\n");

                std::string synonyms;
                unsigned current_length(0);
                unsigned indicator2(0);

                for (auto synonym_it(synonym_values.cbegin()); synonym_it != synonym_values.cend(); /*Intentionally empty*/) {

                    if (indicator2 > 9)
                        Error ("Currently cannot handle synonyms with total length greater than " + std::to_string(9 * MarcRecord::MAX_FIELD_LENGTH) + '\n');

                    if (current_length + (*synonym_it).length() < MarcRecord::MAX_FIELD_LENGTH) {
                         bool synonyms_empty(synonyms.empty());
                         synonyms += (synonyms_empty ? *synonym_it : " , " + *synonym_it);
                         current_length += (*synonym_it).length() + (synonyms_empty ? 0 : 3);
                         ++synonym_it;
                    } else {
                        if (not(record->insertSubfield(tag, subfield_spec.at(0), synonyms, '0', indicator2 + '0')))
                            Error("Could not insert field " + tag + " for PPN " + record->getControlNumber() + '\n');
                        synonyms = "";
                        current_length = 0;
                        ++indicator2;
                    }
                }
                // Write rest of data
                if (not synonyms.empty()) {
                    if (not(record->insertSubfield(tag, subfield_spec.at(0), synonyms, '0', indicator2 + '0')))
                            Error("Could not insert field " + tag + " for PPN " + record->getControlNumber() + '\n');
                }
                ++modified_count;
            }
        }
    }
}

void InsertSynonyms(MarcReader * const marc_reader, MarcWriter * const marc_writer,
                    const std::set<std::string> &primary_tags_and_subfield_codes,
                    const std::set<std::string> &output_tags_and_subfield_codes,
                    std::vector<std::map<std::string, std::string>> &synonym_maps)
{
    while (MarcRecord record = marc_reader->read()) {
        ProcessRecord(&record, synonym_maps, primary_tags_and_subfield_codes, output_tags_and_subfield_codes);
        marc_writer->write(record);
        ++record_count;
    }

    std::cerr << "Modified " << modified_count << " of " << record_count << " record(s).\n";
}

bool ParseSpec(std::string spec_str, std::set<std::string> * const field_specs, std::map<std::string, std::pair<std::string, std::string>> * filter_specs = nullptr) {
    std::set<std::string> raw_field_specs;

    if (unlikely(StringUtil::Split(spec_str, ':', &raw_field_specs) == 0)) {
        Error("Need at least one field");
        return false;
    }

    if (filter_specs == nullptr) {
        *field_specs = raw_field_specs;
	return true;
    }

    // Iterate over all Field-specs and extract possible filters
    static RegexMatcher * const matcher(RegexMatcher::RegexMatcherFactory("(\\d{1,3}[a-z]+)\\[(\\d{1,3}[a-z])=(.*)\\]"));

    for (auto field_spec : raw_field_specs) {
        if (matcher->matched(field_spec)) {
            filter_specs->emplace((*matcher)[1], std::make_pair((*matcher)[2], (*matcher)[3]));
            auto bracket = field_spec.find("[");
            field_spec = (bracket != std::string::npos) ? field_spec.erase(bracket, field_spec.length()) : field_spec;
        }
        field_specs->insert(field_spec);
    }
    return true;
}

int main(int argc, char **argv) {
    ::progname = argv[0];

    if (argc != 4)
        Usage();

    const std::string marc_input_filename(argv[1]);
    const std::string authority_data_marc_input_filename(argv[2]);
    const std::string marc_output_filename(argv[3]);
    if (unlikely(marc_input_filename == marc_output_filename))
        Error("Title data input file name equals output file name!");
    if (unlikely(authority_data_marc_input_filename == marc_output_filename))
        Error("Authority data input file name equals output file name!");

    std::unique_ptr<MarcReader> marc_reader(MarcReader::Factory(marc_input_filename, MarcReader::BINARY));
    std::unique_ptr<MarcReader> authority_reader(MarcReader::Factory(authority_data_marc_input_filename,
                                                                     MarcReader::BINARY));
    std::unique_ptr<MarcWriter> marc_writer(MarcWriter::Factory(marc_output_filename, MarcWriter::BINARY));

    try {
        // Determine possible mappings
        // Values in square brackets specify a positive criterion for values to be taken into account
        const std::string AUTHORITY_DATA_PRIMARY_SPEC("100abcd[079v=piz]:110abcd:111abcd:130abcd:150abcd:151abcd");
        const std::string AUTHORITY_DATA_SYNONYM_SPEC("400abcd:410abcd:411abcd:430abcd:450abcd:451abcd");
        const std::string TITLE_DATA_PRIMARY_SPEC("600abcd:610abcd:611abcd:630abcd:650abcd:651abcd:689abcd");
        const std::string TITLE_DATA_UNUSED_FIELDS_FOR_SYNONYMS("180a:181a:182a:183a:184a:185a:186a");

        // Determine fields to handle
        std::set<std::string> primary_tags_and_subfield_codes;
        std::set<std::string> synonym_tags_and_subfield_codes;
        std::set<std::string> input_tags_and_subfield_codes;
        std::set<std::string> output_tags_and_subfield_codes;

        std::map<std::string, std::pair<std::string, std::string>> filter_specs;

        if (not unlikely(ParseSpec(AUTHORITY_DATA_PRIMARY_SPEC, &primary_tags_and_subfield_codes, &filter_specs)))
            Error("Could not properly parse " + AUTHORITY_DATA_PRIMARY_SPEC);

        if (unlikely(StringUtil::Split(AUTHORITY_DATA_SYNONYM_SPEC, ":", &synonym_tags_and_subfield_codes) == 0))
            Error("Need at least one synonym field");

        if (unlikely(StringUtil::Split(TITLE_DATA_PRIMARY_SPEC, ":", &input_tags_and_subfield_codes) == 0))
            Error("Need at least one input field");

        if (unlikely(StringUtil::Split(TITLE_DATA_UNUSED_FIELDS_FOR_SYNONYMS, ":", &output_tags_and_subfield_codes)
                     == 0))
            Error("Need at least one output field");

        unsigned num_of_authority_entries(primary_tags_and_subfield_codes.size());

        if (synonym_tags_and_subfield_codes.size() != num_of_authority_entries)
            Error("Number of authority primary specs must match number of synonym specs");
        if (input_tags_and_subfield_codes.size() != output_tags_and_subfield_codes.size())
            Error("Number of fields title entry specs must match number of output specs");

        std::vector<std::map<std::string, std::string>> synonym_maps(num_of_authority_entries,
                                                                     std::map<std::string, std::string>());

        // Extract the synonyms from authority data
        ExtractSynonyms(authority_reader.get(), primary_tags_and_subfield_codes, synonym_tags_and_subfield_codes,
                        &synonym_maps, filter_specs);

        // Iterate over the title data
        InsertSynonyms(marc_reader.get(), marc_writer.get(), input_tags_and_subfield_codes,
                       output_tags_and_subfield_codes, synonym_maps);

    } catch (const std::exception &x) {
        Error("caught exception: " + std::string(x.what()));
    }
}
<commit_msg>Minor adjustments<commit_after>/** \file    add_synonyms.cc
 *  \brief   Generic version for augmenting title data with synonyms found
             in the authority data
 *  \author  Johannes Riedl
 */

/*
    Copyright (C) 2016, Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

/*  We offer a list of tags and subfields where the primary data resides along
    with a list of tags and subfields where the synonym data is found and
    a list of unused fields in the title data where the synonyms can be stored
*/

#include <iostream>
#include <map>
#include <vector>
#include <cstdlib>
#include "Compiler.h"
#include "FileUtil.h"
#include "MarcReader.h"
#include "MarcRecord.h"
#include "MarcWriter.h"
#include "MediaTypeUtil.h"
#include "RegexMatcher.h"
#include "StringUtil.h"
#include "Subfields.h"
#include "util.h"

static unsigned modified_count(0);
static unsigned record_count(0);

void Usage() {
    std::cerr << "Usage: " << ::progname << " master_marc_input norm_data_marc_input marc_output\n";
    std::exit(EXIT_FAILURE);
}

std::string GetTag(const std::string &tag_and_subfields_spec) {
    return tag_and_subfields_spec.substr(0, 3);
}

std::string GetSubfieldCodes(const std::string &tag_and_subfields_spec) {
    return tag_and_subfields_spec.substr(3);
}

bool FilterPasses(const MarcRecord &record, const std::map<std::string, std::pair<std::string, std::string>> &filter_specs, const std::string &field_spec) {
      auto filter_spec(filter_specs.find(field_spec));
      if (filter_spec == filter_specs.cend())
          return true;

      auto rule(filter_spec->second);
      // We have field_spec in key and rule to match in value
      std::string subfield_codes(GetSubfieldCodes(rule.first));
      if (subfield_codes.length() != 1)
         Error("Invalid subfield specification "  + subfield_codes + " for filter");

      std::string subfield_value;
      if ((subfield_value = record.extractFirstSubfield(GetTag(rule.first), subfield_codes.c_str()[0])).empty())
          return false;

      return subfield_value == rule.second;
}

void ExtractSynonyms(MarcReader * const authority_reader,
                     const std::set<std::string> &primary_tags_and_subfield_codes,
                     const std::set<std::string> &synonym_tags_and_subfield_codes,
                     std::vector<std::map<std::string, std::string>> * const synonym_maps,
                     const std::map<std::string, std::pair<std::string, std::string>> &filter_spec)
{
    while (const MarcRecord record = authority_reader->read()) {
        std::set<std::string>::const_iterator primary;
        std::set<std::string>::const_iterator synonym;
        unsigned int i(0);
        for (primary = primary_tags_and_subfield_codes.begin(), synonym = synonym_tags_and_subfield_codes.begin();
            primary != primary_tags_and_subfield_codes.end();
            ++primary, ++synonym, ++i)
        {
            // Fill maps with synonyms
            std::vector<std::string> primary_values;
            std::vector<std::string> synonym_values;

            if (FilterPasses(record, filter_spec, *primary) and
                record.extractSubfields(GetTag(*primary), GetSubfieldCodes(*primary), &primary_values) and
                record.extractSubfields(GetTag(*synonym), GetSubfieldCodes(*synonym), &synonym_values))
                    (*synonym_maps)[i].emplace(StringUtil::Join(primary_values, ','),
                                               StringUtil::Join(synonym_values, ','));
        }
    }
}

inline std::string GetMapValueOrEmptyString(const std::map<std::string, std::string> &map,
                                            const std::string &searchterm)
{
    auto value(map.find(searchterm));
    return (value != map.cend()) ? value->second : "";
}

void ProcessRecord(MarcRecord * const record, const std::vector<std::map<std::string, std::string>> &synonym_maps,
                   const std::set<std::string> &primary_tags_and_subfield_codes,
                   const std::set<std::string> &output_tags_and_subfield_codes)
{
    std::set<std::string>::const_iterator primary;
    std::set<std::string>::const_iterator output;
    unsigned int i(0);

    if (primary_tags_and_subfield_codes.size() == output_tags_and_subfield_codes.size()) {
        for (primary = primary_tags_and_subfield_codes.begin(), output = output_tags_and_subfield_codes.begin();
            primary != primary_tags_and_subfield_codes.end();
            ++primary, ++output, ++i)
        {
            std::vector<std::string> primary_values;
            std::set<std::string> synonym_values;
            std::vector<size_t> field_indices;
            if (record->getFieldIndices(GetTag(*primary), &field_indices) != MarcRecord::FIELD_NOT_FOUND) {
                for (auto field_index : field_indices) {
                     primary_values.clear();
                     if (record->getSubfields(field_index).extractSubfields(GetSubfieldCodes(*primary), &primary_values)) {
                        std::string searchterm = StringUtil::Join(primary_values, ',');
                        // First case: Look up synonyms only in one category
                        if (i < synonym_maps.size()) {
                            const auto &synonym_map(synonym_maps[i]);
                            const auto synonym(GetMapValueOrEmptyString(synonym_map, searchterm));
                            if (not synonym.empty())
                                synonym_values.insert(synonym);
                        }

                        // Second case: Look up synonyms in all categories
                        else {
                            for (auto &synonym_map : synonym_maps) {
                                const auto &synonym(GetMapValueOrEmptyString(synonym_map, searchterm));
                                if (not synonym.empty())
                                   synonym_values.insert(synonym);
                            }
                        }
                    }
                 }

                if (synonym_values.empty())
                     continue;

                // Insert synonyms
                // Abort if field is already populated
                std::string tag(GetTag(*output));
                if (record->getFieldIndex(tag) != MarcRecord::FIELD_NOT_FOUND)
                    Error("Field with tag " + tag + " is not empty for PPN " + record->getControlNumber() + '\n');
                std::string subfield_spec(GetSubfieldCodes(*output));
                if (unlikely(subfield_spec.size() != 1))
                    Error("We currently only support a single subfield and thus specifying " + subfield_spec
                          + " as output subfield is not valid\n");

                std::string synonyms;
                unsigned current_length(0);
                unsigned indicator2(0);

                for (auto synonym_it(synonym_values.cbegin()); synonym_it != synonym_values.cend(); /*Intentionally empty*/) {
                    if (indicator2 > 9)
                        Error ("Currently cannot handle synonyms with total length greater than " + std::to_string(9 * MarcRecord::MAX_FIELD_LENGTH) + '\n');

                    if (current_length + synonym_it->length() < MarcRecord::MAX_FIELD_LENGTH) {
                         bool synonyms_empty(synonyms.empty());
                         synonyms += (synonyms_empty ? *synonym_it : " , " + *synonym_it);
                         current_length += synonym_it->length() + (synonyms_empty ? 0 : 3);
                         ++synonym_it;
                    } else {
                        if (not(record->insertSubfield(tag, subfield_spec[0], synonyms, '0', indicator2 + '0')))
                            Error("Could not insert field " + tag + " for PPN " + record->getControlNumber() + '\n');
                        synonyms.clear();
                        current_length = 0;
                        ++indicator2;
                    }
                }
                // Write rest of data
                if (not synonyms.empty()) {
                    if (not(record->insertSubfield(tag, subfield_spec[0], synonyms, '0', indicator2 + '0')))
                            Error("Could not insert field " + tag + " for PPN " + record->getControlNumber() + '\n');
                }
                ++modified_count;
            }
        }
    }
}

void InsertSynonyms(MarcReader * const marc_reader, MarcWriter * const marc_writer,
                    const std::set<std::string> &primary_tags_and_subfield_codes,
                    const std::set<std::string> &output_tags_and_subfield_codes,
                    std::vector<std::map<std::string, std::string>> &synonym_maps)
{
    while (MarcRecord record = marc_reader->read()) {
        ProcessRecord(&record, synonym_maps, primary_tags_and_subfield_codes, output_tags_and_subfield_codes);
        marc_writer->write(record);
        ++record_count;
    }

    std::cerr << "Modified " << modified_count << " of " << record_count << " record(s).\n";
}

bool ParseSpec(std::string spec_str, std::set<std::string> * const field_specs, std::map<std::string, std::pair<std::string, std::string>> * filter_specs = nullptr) {
    std::set<std::string> raw_field_specs;

    if (unlikely(StringUtil::Split(spec_str, ':', &raw_field_specs) == 0)) {
        Error("Need at least one field");
        return false;
    }

    if (filter_specs == nullptr) {
        *field_specs = raw_field_specs;
	return true;
    }

    // Iterate over all Field-specs and extract possible filters
    static RegexMatcher * const matcher(RegexMatcher::RegexMatcherFactory("(\\d{1,3}[a-z]+)\\[(\\d{1,3}[a-z])=(.*)\\]"));

    for (auto field_spec : raw_field_specs) {
        if (matcher->matched(field_spec)) {
            filter_specs->emplace((*matcher)[1], std::make_pair((*matcher)[2], (*matcher)[3]));
            auto bracket = field_spec.find("[");
            field_spec = (bracket != std::string::npos) ? field_spec.erase(bracket, field_spec.length()) : field_spec;
        }
        field_specs->insert(field_spec);
    }
    return true;
}

int main(int argc, char **argv) {
    ::progname = argv[0];

    if (argc != 4)
        Usage();

    const std::string marc_input_filename(argv[1]);
    const std::string authority_data_marc_input_filename(argv[2]);
    const std::string marc_output_filename(argv[3]);
    if (unlikely(marc_input_filename == marc_output_filename))
        Error("Title data input file name equals output file name!");
    if (unlikely(authority_data_marc_input_filename == marc_output_filename))
        Error("Authority data input file name equals output file name!");

    std::unique_ptr<MarcReader> marc_reader(MarcReader::Factory(marc_input_filename, MarcReader::BINARY));
    std::unique_ptr<MarcReader> authority_reader(MarcReader::Factory(authority_data_marc_input_filename,
                                                                     MarcReader::BINARY));
    std::unique_ptr<MarcWriter> marc_writer(MarcWriter::Factory(marc_output_filename, MarcWriter::BINARY));

    try {
        // Determine possible mappings
        // Values in square brackets specify a positive criterion for values to be taken into account
        const std::string AUTHORITY_DATA_PRIMARY_SPEC("100abcd[079v=piz]:110abcd:111abcd:130abcd:150abcd:151abcd");
        const std::string AUTHORITY_DATA_SYNONYM_SPEC("400abcd:410abcd:411abcd:430abcd:450abcd:451abcd");
        const std::string TITLE_DATA_PRIMARY_SPEC("600abcd:610abcd:611abcd:630abcd:650abcd:651abcd:689abcd");
        const std::string TITLE_DATA_UNUSED_FIELDS_FOR_SYNONYMS("180a:181a:182a:183a:184a:185a:186a");

        // Determine fields to handle
        std::set<std::string> primary_tags_and_subfield_codes;
        std::set<std::string> synonym_tags_and_subfield_codes;
        std::set<std::string> input_tags_and_subfield_codes;
        std::set<std::string> output_tags_and_subfield_codes;

        std::map<std::string, std::pair<std::string, std::string>> filter_specs;

        if (not unlikely(ParseSpec(AUTHORITY_DATA_PRIMARY_SPEC, &primary_tags_and_subfield_codes, &filter_specs)))
            Error("Could not properly parse " + AUTHORITY_DATA_PRIMARY_SPEC);

        if (unlikely(StringUtil::Split(AUTHORITY_DATA_SYNONYM_SPEC, ":", &synonym_tags_and_subfield_codes) == 0))
            Error("Need at least one synonym field");

        if (unlikely(StringUtil::Split(TITLE_DATA_PRIMARY_SPEC, ":", &input_tags_and_subfield_codes) == 0))
            Error("Need at least one input field");

        if (unlikely(StringUtil::Split(TITLE_DATA_UNUSED_FIELDS_FOR_SYNONYMS, ":", &output_tags_and_subfield_codes)
                     == 0))
            Error("Need at least one output field");

        unsigned num_of_authority_entries(primary_tags_and_subfield_codes.size());

        if (synonym_tags_and_subfield_codes.size() != num_of_authority_entries)
            Error("Number of authority primary specs must match number of synonym specs");
        if (input_tags_and_subfield_codes.size() != output_tags_and_subfield_codes.size())
            Error("Number of fields title entry specs must match number of output specs");

        std::vector<std::map<std::string, std::string>> synonym_maps(num_of_authority_entries,
                                                                     std::map<std::string, std::string>());

        // Extract the synonyms from authority data
        ExtractSynonyms(authority_reader.get(), primary_tags_and_subfield_codes, synonym_tags_and_subfield_codes,
                        &synonym_maps, filter_specs);

        // Iterate over the title data
        InsertSynonyms(marc_reader.get(), marc_writer.get(), input_tags_and_subfield_codes,
                       output_tags_and_subfield_codes, synonym_maps);

    } catch (const std::exception &x) {
        Error("caught exception: " + std::string(x.what()));
    }
}
<|endoftext|>


<commit_before>#include "includeextractor.h"

#include <extensionsystem/pluginmanager.h>

#include <cplusplus/LookupContext.h>

#include <utils/qtcassert.h>

using namespace CPlusPlus;

static bool hasNonForwardDeclaration (const QList<LookupItem> &matches) {
  auto ok = false;
  for (const auto &i: matches) {
    ok |= (i.declaration () && !i.declaration ()->isForwardClassDeclaration ());
  }
  return ok;
}

IncludeExtractor::IncludeExtractor (Document::Ptr document,
                                    const Snapshot &snapshot) :
  ASTVisitor (document ? document->translationUnit () : nullptr),
  document_ (document),
  snapshot_ (snapshot),
  bindings_ (new CreateBindings (document, snapshot)) {
  QTC_ASSERT (document, return );

  //  bindings_->setExpandTemplates (true);

  if (!translationUnit () || !translationUnit ()->ast ()) {
    return;
  }

  accept (translationUnit ()->ast ());
}

bool IncludeExtractor::visit (NamedTypeSpecifierAST *ast) {
  QTC_ASSERT (ast, return false);
  if (!ast->name || !ast->name->name) {
    return true;
  }
  const auto name = Overview () (ast->name->name);
  qCritical () << "NamedTypeSpecifierAST" << name;
  const auto scope = scopeAtToken (ast->firstToken ());
  QTC_ASSERT (scope, return true);
  addExpression (name, scope);
  return true;
}

bool IncludeExtractor::visit (DeclaratorIdAST *ast) {
  QTC_ASSERT (ast, return false);
  if (!ast->name || !ast->name->name) {
    return true;
  }
  const auto name = Overview () (ast->name->name);
  qCritical () << "DeclaratorIdAST" << name;
  const auto scope = scopeAtToken (ast->firstToken ());
  QTC_ASSERT (scope, return true);
  addExpression (name, scope);
  return true;
}

bool IncludeExtractor::visit (IdExpressionAST *ast) {
  QTC_ASSERT (ast, return false);
  if (!ast->name || !ast->name->name) {
    return true;
  }
  const auto name = Overview () (ast->name->name);
  qCritical () << "IdExpressionAST" << name;
  const auto scope = scopeAtToken (ast->firstToken ());
  QTC_ASSERT (scope, return true);
  addExpression (name, scope);
  return true;
}

bool IncludeExtractor::visit (CallAST *ast) {
  QTC_ASSERT (ast, return false);
  qCritical () << "CallAST";
  const auto scope = scopeAtToken (ast->firstToken ());
  QTC_ASSERT (scope, return true);
  QTC_ASSERT (ast->base_expression, return true);
  addExpression (ast->base_expression, scope);

  auto expression = ast->expression_list;
  while (expression) {
    qCritical () << "CallAST expression_list";
    addExpression (expression->value, scope);
    expression = expression->next;
  }

  return true;
}

bool IncludeExtractor::visit (TemplateIdAST *ast) {
  QTC_ASSERT (ast, return false);
  if (!ast->name) {
    return true;
  }
  const auto name = Overview () (ast->name);
  qCritical () << "TemplateIdAST" << name;
  const auto scope = scopeAtToken (ast->firstToken ());
  QTC_ASSERT (scope, return true);
  addExpression (name, scope);
  return true;
}

bool IncludeExtractor::visit (UsingDirectiveAST *ast) {
  QTC_ASSERT (ast, return false);
  if (!ast->name || !ast->name->name) {
    return true;
  }
  const auto name = Overview () (ast->name->name);
  qCritical () << "UsingDirectiveAST" << name;
  const auto scope = scopeAtToken (ast->firstToken ());
  QTC_ASSERT (scope, return true);
  addExpression (name, scope);
  return true;
}

bool IncludeExtractor::visit (MemberAccessAST *ast) {
  QTC_ASSERT (ast, return false);
  if (!ast->member_name || !ast->member_name->name) {
    return true;
  }
  const auto name = Overview () (ast->member_name->name);
  qCritical () << "MemberAccessAST" << name;
  const auto scope = scopeAtToken (ast->firstToken ());
  QTC_ASSERT (scope, return true);
  addExpression (name, scope);
  return true;
}

Scope *IncludeExtractor::scopeAtToken (unsigned token) const {
  QTC_ASSERT (translationUnit (), return nullptr);
  unsigned line = 0;
  unsigned column = 0;
  translationUnit ()->getTokenStartPosition (token, &line, &column);
  return document_->scopeAt (line);
}

void IncludeExtractor::addExpression (const QString &name, CPlusPlus::Scope *scope) {
  if (name.isEmpty ()) {
    return;
  }
  QTC_ASSERT (scope, return );

  const auto pair = qMakePair (name, scope);
  if (checkedTypes_.contains (pair)) {
    return;
  }
  checkedTypes_.insert (pair);
  qCritical () << "add string expression" << name;

  TypeOfExpression toe;
  toe.init (document_, snapshot_, bindings_);
  const auto matches = toe (name.toUtf8 (), scope);

  addDeclarations (matches);

  if (toe.ast ()) {
    accept (toe.ast ());
  }
}

void IncludeExtractor::addExpression (ExpressionAST *ast, Scope *scope) {
  QTC_ASSERT (scope, return );
  QTC_ASSERT (ast, return );
  QString callName;
  if (auto e = ast->asIdExpression ()) {
    callName = Overview () (e->name->name);
  }
  else if (auto e = ast->asMemberAccess ()) {
    callName = Overview () (e->member_name->name);
  }
  qCritical () << "add ast expression" << callName;

  TypeOfExpression toe;
  toe.init (document_, snapshot_, bindings_);
  const auto matches = toe (ast, document_, scope);

  addDeclarations (matches);

  if (toe.ast ()) {
    accept (toe.ast ());
  }
}

void IncludeExtractor::addDeclarations (const QList<LookupItem> &declarations) {
  const auto hasNonForward = hasNonForwardDeclaration (declarations);

  for (const auto &match: declarations) {
    const auto matchType = Overview ()(match.type ());
    if (hasNonForward
        && match.declaration ()
        && match.declaration ()->isForwardClassDeclaration ()
        && !match.type ().isTypedef ()) {
      continue;
    }
    addDeclaration (match.declaration ());

    QTC_ASSERT (match.scope (), continue);
    addExpression (matchType, match.scope ());
  }
}

bool IncludeExtractor::addDeclaration (Symbol *declaration) {
  if (!declaration
      || document_->fileName () == QString::fromUtf8 (declaration->fileName ())) {
    return false;
  }

  QTC_ASSERT (declaration->fileName (), return false);
  const auto fileName = QString::fromUtf8 (declaration->fileName ());
  includes_.insert (fileName);
  symbols_.insert (declaration);

  qCritical () << ">>>addDeclaration"
               << "at" << fileName
               << declaration
               << typeid (*declaration).name ()
               << Overview () (declaration->type ().type ());

  return true;
}
<commit_msg>Qtc 4.11<commit_after>#include "includeextractor.h"

#include <extensionsystem/pluginmanager.h>

#include <cplusplus/LookupContext.h>

#include <utils/qtcassert.h>

using namespace CPlusPlus;

static bool hasNonForwardDeclaration (const QList<LookupItem> &matches) {
  auto ok = false;
  for (const auto &i: matches) {
    ok |= (i.declaration () && !i.declaration ()->isForwardClassDeclaration ());
  }
  return ok;
}

IncludeExtractor::IncludeExtractor (Document::Ptr document,
                                    const Snapshot &snapshot) :
  ASTVisitor (document ? document->translationUnit () : nullptr),
  document_ (document),
  snapshot_ (snapshot),
  bindings_ (new CreateBindings (document, snapshot)) {
  QTC_ASSERT (document, return );

  //  bindings_->setExpandTemplates (true);

  if (!translationUnit () || !translationUnit ()->ast ()) {
    return;
  }

  accept (translationUnit ()->ast ());
}

bool IncludeExtractor::visit (NamedTypeSpecifierAST *ast) {
  QTC_ASSERT (ast, return false);
  if (!ast->name || !ast->name->name) {
    return true;
  }
  const auto name = Overview () (ast->name->name);
  qCritical () << "NamedTypeSpecifierAST" << name;
  const auto scope = scopeAtToken (ast->firstToken ());
  QTC_ASSERT (scope, return true);
  addExpression (name, scope);
  return true;
}

bool IncludeExtractor::visit (DeclaratorIdAST *ast) {
  QTC_ASSERT (ast, return false);
  if (!ast->name || !ast->name->name) {
    return true;
  }
  const auto name = Overview () (ast->name->name);
  qCritical () << "DeclaratorIdAST" << name;
  const auto scope = scopeAtToken (ast->firstToken ());
  QTC_ASSERT (scope, return true);
  addExpression (name, scope);
  return true;
}

bool IncludeExtractor::visit (IdExpressionAST *ast) {
  QTC_ASSERT (ast, return false);
  if (!ast->name || !ast->name->name) {
    return true;
  }
  const auto name = Overview () (ast->name->name);
  qCritical () << "IdExpressionAST" << name;
  const auto scope = scopeAtToken (ast->firstToken ());
  QTC_ASSERT (scope, return true);
  addExpression (name, scope);
  return true;
}

bool IncludeExtractor::visit (CallAST *ast) {
  QTC_ASSERT (ast, return false);
  qCritical () << "CallAST";
  const auto scope = scopeAtToken (ast->firstToken ());
  QTC_ASSERT (scope, return true);
  QTC_ASSERT (ast->base_expression, return true);
  addExpression (ast->base_expression, scope);

  auto expression = ast->expression_list;
  while (expression) {
    qCritical () << "CallAST expression_list";
    addExpression (expression->value, scope);
    expression = expression->next;
  }

  return true;
}

bool IncludeExtractor::visit (TemplateIdAST *ast) {
  QTC_ASSERT (ast, return false);
  if (!ast->name) {
    return true;
  }
  const auto name = Overview () (ast->name);
  qCritical () << "TemplateIdAST" << name;
  const auto scope = scopeAtToken (ast->firstToken ());
  QTC_ASSERT (scope, return true);
  addExpression (name, scope);
  return true;
}

bool IncludeExtractor::visit (UsingDirectiveAST *ast) {
  QTC_ASSERT (ast, return false);
  if (!ast->name || !ast->name->name) {
    return true;
  }
  const auto name = Overview () (ast->name->name);
  qCritical () << "UsingDirectiveAST" << name;
  const auto scope = scopeAtToken (ast->firstToken ());
  QTC_ASSERT (scope, return true);
  addExpression (name, scope);
  return true;
}

bool IncludeExtractor::visit (MemberAccessAST *ast) {
  QTC_ASSERT (ast, return false);
  if (!ast->member_name || !ast->member_name->name) {
    return true;
  }
  const auto name = Overview () (ast->member_name->name);
  qCritical () << "MemberAccessAST" << name;
  const auto scope = scopeAtToken (ast->firstToken ());
  QTC_ASSERT (scope, return true);
  addExpression (name, scope);
  return true;
}

Scope *IncludeExtractor::scopeAtToken (unsigned token) const {
  QTC_ASSERT (translationUnit (), return nullptr);
  int line = 0;
  int column = 0;
  translationUnit ()->getTokenStartPosition (token, &line, &column);
  return document_->scopeAt (line);
}

void IncludeExtractor::addExpression (const QString &name, CPlusPlus::Scope *scope) {
  if (name.isEmpty ()) {
    return;
  }
  QTC_ASSERT (scope, return );

  const auto pair = qMakePair (name, scope);
  if (checkedTypes_.contains (pair)) {
    return;
  }
  checkedTypes_.insert (pair);
  qCritical () << "add string expression" << name;

  TypeOfExpression toe;
  toe.init (document_, snapshot_, bindings_);
  const auto matches = toe (name.toUtf8 (), scope);

  addDeclarations (matches);

  if (toe.ast ()) {
    accept (toe.ast ());
  }
}

void IncludeExtractor::addExpression (ExpressionAST *ast, Scope *scope) {
  QTC_ASSERT (scope, return );
  QTC_ASSERT (ast, return );
  QString callName;
  if (auto e = ast->asIdExpression ()) {
    callName = Overview () (e->name->name);
  }
  else if (auto e = ast->asMemberAccess ()) {
    callName = Overview () (e->member_name->name);
  }
  qCritical () << "add ast expression" << callName;

  TypeOfExpression toe;
  toe.init (document_, snapshot_, bindings_);
  const auto matches = toe (ast, document_, scope);

  addDeclarations (matches);

  if (toe.ast ()) {
    accept (toe.ast ());
  }
}

void IncludeExtractor::addDeclarations (const QList<LookupItem> &declarations) {
  const auto hasNonForward = hasNonForwardDeclaration (declarations);

  for (const auto &match: declarations) {
    const auto matchType = Overview ()(match.type ());
    if (hasNonForward
        && match.declaration ()
        && match.declaration ()->isForwardClassDeclaration ()
        && !match.type ().isTypedef ()) {
      continue;
    }
    addDeclaration (match.declaration ());

    QTC_ASSERT (match.scope (), continue);
    addExpression (matchType, match.scope ());
  }
}

bool IncludeExtractor::addDeclaration (Symbol *declaration) {
  if (!declaration
      || document_->fileName () == QString::fromUtf8 (declaration->fileName ())) {
    return false;
  }

  QTC_ASSERT (declaration->fileName (), return false);
  const auto fileName = QString::fromUtf8 (declaration->fileName ());
  includes_.insert (fileName);
  symbols_.insert (declaration);

  qCritical () << ">>>addDeclaration"
               << "at" << fileName
               << declaration
               << typeid (*declaration).name ()
               << Overview () (declaration->type ().type ());

  return true;
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2003 The Regents of The University of Michigan
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met: redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer;
 * redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution;
 * neither the name of the copyright holders nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include <string>

#include "cpu/base_cpu.hh"
#include "cpu/exec_context.hh"

#ifdef FULL_SYSTEM
#include "sim/system.hh"
#else
#include "sim/process.hh"
#endif

using namespace std;

// constructor
#ifdef FULL_SYSTEM
ExecContext::ExecContext(BaseCPU *_cpu, int _thread_num, System *_sys,
                         AlphaItb *_itb, AlphaDtb *_dtb,
                         FunctionalMemory *_mem)
    : _status(ExecContext::Unallocated),
      kernelStats(this, _cpu), cpu(_cpu), thread_num(_thread_num),
      cpu_id(-1), mem(_mem), itb(_itb), dtb(_dtb), system(_sys),
      memCtrl(_sys->memCtrl), physmem(_sys->physmem),
      swCtx(NULL), func_exe_inst(0), storeCondFailures(0)
{
    memset(&regs, 0, sizeof(RegFile));
}
#else
ExecContext::ExecContext(BaseCPU *_cpu, int _thread_num,
                         Process *_process, int _asid)
    : _status(ExecContext::Unallocated),
      cpu(_cpu), thread_num(_thread_num), cpu_id(-1),
      process(_process), mem(process->getMemory()), asid(_asid),
      func_exe_inst(0), storeCondFailures(0)
{
}

ExecContext::ExecContext(BaseCPU *_cpu, int _thread_num,
                         FunctionalMemory *_mem, int _asid)
    : cpu(_cpu), thread_num(_thread_num), process(0), mem(_mem), asid(_asid),
      func_exe_inst(0), storeCondFailures(0)
{
}
#endif

void
ExecContext::takeOverFrom(ExecContext *oldContext)
{
    // some things should already be set up
    assert(mem == oldContext->mem);
#ifdef FULL_SYSTEM
    assert(system == oldContext->system);
#else
    assert(process == oldContext->process);
#endif

    // copy over functional state
    _status = oldContext->_status;
#ifdef FULL_SYSTEM
    kernelStats = oldContext->kernelStats;
#endif
    regs = oldContext->regs;
    cpu_id = oldContext->cpu_id;
    func_exe_inst = oldContext->func_exe_inst;

    storeCondFailures = 0;

    oldContext->_status = ExecContext::Unallocated;
}

void
ExecContext::serialize(ostream &os)
{
    SERIALIZE_ENUM(_status);
    regs.serialize(os);
    // thread_num and cpu_id are deterministic from the config
    SERIALIZE_SCALAR(func_exe_inst);

    bool ctx = false;
    if (swCtx) {
        ctx = true;
        SERIALIZE_SCALAR(ctx);
        SERIALIZE_SCALAR(swCtx->calls);
        std::stack<fnCall *> *stack = &(swCtx->callStack);
        fnCall *top;
        int size = stack->size();
        SERIALIZE_SCALAR(size);

        for (int j=0; j<size; ++j) {
            top = stack->top();
            paramOut(os, csprintf("stackpos[%d]",j), top->name);
            delete top;
            stack->pop();
        }
    } else {
        SERIALIZE_SCALAR(ctx);
    }
    if (system->bin) {
        Statistics::MainBin *cur = Statistics::MainBin::curBin();
        string bin_name = cur->name();
        SERIALIZE_SCALAR(bin_name);
    }
}

void
ExecContext::unserialize(Checkpoint *cp, const std::string &section)
{
    UNSERIALIZE_ENUM(_status);
    regs.unserialize(cp, section);
    // thread_num and cpu_id are deterministic from the config
    UNSERIALIZE_SCALAR(func_exe_inst);

    bool ctx;
    UNSERIALIZE_SCALAR(ctx);
    if (ctx) {
        swCtx = new SWContext;
        UNSERIALIZE_SCALAR(swCtx->calls);
        int size;
        UNSERIALIZE_SCALAR(size);

        vector<fnCall *> calls;
        fnCall *call;
        for (int i=0; i<size; ++i) {
            call = new fnCall;
            paramIn(cp, section, csprintf("stackpos[%d]",i), call->name);
            call->myBin = system->getBin(call->name);
            calls.push_back(call);
        }

        for (int i=size-1; i>=0; --i) {
            swCtx->callStack.push(calls[i]);
        }

    }

    if (system->bin) {
        string bin_name;
        UNSERIALIZE_SCALAR(bin_name);
        system->getBin(bin_name)->activate();
    }
}

void
ExecContext::activate(int delay)
{
    if (status() == Active)
        return;

    _status = Active;
    cpu->activateContext(thread_num, delay);
}

void
ExecContext::suspend()
{
    if (status() == Suspended)
        return;

#ifdef FULL_SYSTEM
    // Don't change the status from active if there are pending interrupts
    if (cpu->check_interrupts()) {
        assert(status() == Active);
        return;
    }
#endif

    _status = Suspended;
    cpu->suspendContext(thread_num);
}

void
ExecContext::deallocate()
{
    if (status() == Unallocated)
        return;

    _status = Unallocated;
    cpu->deallocateContext(thread_num);
}

void
ExecContext::halt()
{
    if (status() == Halted)
        return;

    _status = Halted;
    cpu->haltContext(thread_num);
}

void
ExecContext::regStats(const string &name)
{
#ifdef FULL_SYSTEM
    kernelStats.regStats(name + ".kern");
#endif
}
<commit_msg>fix ALPHA<commit_after>/*
 * Copyright (c) 2003 The Regents of The University of Michigan
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met: redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer;
 * redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution;
 * neither the name of the copyright holders nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include <string>

#include "cpu/base_cpu.hh"
#include "cpu/exec_context.hh"

#ifdef FULL_SYSTEM
#include "sim/system.hh"
#else
#include "sim/process.hh"
#endif

using namespace std;

// constructor
#ifdef FULL_SYSTEM
ExecContext::ExecContext(BaseCPU *_cpu, int _thread_num, System *_sys,
                         AlphaItb *_itb, AlphaDtb *_dtb,
                         FunctionalMemory *_mem)
    : _status(ExecContext::Unallocated),
      kernelStats(this, _cpu), cpu(_cpu), thread_num(_thread_num),
      cpu_id(-1), mem(_mem), itb(_itb), dtb(_dtb), system(_sys),
      memCtrl(_sys->memCtrl), physmem(_sys->physmem),
      swCtx(NULL), func_exe_inst(0), storeCondFailures(0)
{
    memset(&regs, 0, sizeof(RegFile));
}
#else
ExecContext::ExecContext(BaseCPU *_cpu, int _thread_num,
                         Process *_process, int _asid)
    : _status(ExecContext::Unallocated),
      cpu(_cpu), thread_num(_thread_num), cpu_id(-1),
      process(_process), mem(process->getMemory()), asid(_asid),
      func_exe_inst(0), storeCondFailures(0)
{
}

ExecContext::ExecContext(BaseCPU *_cpu, int _thread_num,
                         FunctionalMemory *_mem, int _asid)
    : cpu(_cpu), thread_num(_thread_num), process(0), mem(_mem), asid(_asid),
      func_exe_inst(0), storeCondFailures(0)
{
}
#endif

void
ExecContext::takeOverFrom(ExecContext *oldContext)
{
    // some things should already be set up
    assert(mem == oldContext->mem);
#ifdef FULL_SYSTEM
    assert(system == oldContext->system);
#else
    assert(process == oldContext->process);
#endif

    // copy over functional state
    _status = oldContext->_status;
#ifdef FULL_SYSTEM
    kernelStats = oldContext->kernelStats;
#endif
    regs = oldContext->regs;
    cpu_id = oldContext->cpu_id;
    func_exe_inst = oldContext->func_exe_inst;

    storeCondFailures = 0;

    oldContext->_status = ExecContext::Unallocated;
}

void
ExecContext::serialize(ostream &os)
{
    SERIALIZE_ENUM(_status);
    regs.serialize(os);
    // thread_num and cpu_id are deterministic from the config
    SERIALIZE_SCALAR(func_exe_inst);

#ifdef FULL_SYSTEM
    bool ctx = false;
    if (swCtx) {
        ctx = true;
        SERIALIZE_SCALAR(ctx);
        SERIALIZE_SCALAR(swCtx->calls);
        std::stack<fnCall *> *stack = &(swCtx->callStack);
        fnCall *top;
        int size = stack->size();
        SERIALIZE_SCALAR(size);

        for (int j=0; j<size; ++j) {
            top = stack->top();
            paramOut(os, csprintf("stackpos[%d]",j), top->name);
            delete top;
            stack->pop();
        }
    } else {
        SERIALIZE_SCALAR(ctx);
    }
    if (system->bin) {
        Statistics::MainBin *cur = Statistics::MainBin::curBin();
        string bin_name = cur->name();
        SERIALIZE_SCALAR(bin_name);
    }
#endif //FULL_SYSTEM
}

void
ExecContext::unserialize(Checkpoint *cp, const std::string &section)
{
    UNSERIALIZE_ENUM(_status);
    regs.unserialize(cp, section);
    // thread_num and cpu_id are deterministic from the config
    UNSERIALIZE_SCALAR(func_exe_inst);

#ifdef FULL_SYSTEM
    bool ctx;
    UNSERIALIZE_SCALAR(ctx);
    if (ctx) {
        swCtx = new SWContext;
        UNSERIALIZE_SCALAR(swCtx->calls);
        int size;
        UNSERIALIZE_SCALAR(size);

        vector<fnCall *> calls;
        fnCall *call;
        for (int i=0; i<size; ++i) {
            call = new fnCall;
            paramIn(cp, section, csprintf("stackpos[%d]",i), call->name);
            call->myBin = system->getBin(call->name);
            calls.push_back(call);
        }

        for (int i=size-1; i>=0; --i) {
            swCtx->callStack.push(calls[i]);
        }

    }

    if (system->bin) {
        string bin_name;
        UNSERIALIZE_SCALAR(bin_name);
        system->getBin(bin_name)->activate();
    }
#endif //FULL_SYSTEM
}

void
ExecContext::activate(int delay)
{
    if (status() == Active)
        return;

    _status = Active;
    cpu->activateContext(thread_num, delay);
}

void
ExecContext::suspend()
{
    if (status() == Suspended)
        return;

#ifdef FULL_SYSTEM
    // Don't change the status from active if there are pending interrupts
    if (cpu->check_interrupts()) {
        assert(status() == Active);
        return;
    }
#endif

    _status = Suspended;
    cpu->suspendContext(thread_num);
}

void
ExecContext::deallocate()
{
    if (status() == Unallocated)
        return;

    _status = Unallocated;
    cpu->deallocateContext(thread_num);
}

void
ExecContext::halt()
{
    if (status() == Halted)
        return;

    _status = Halted;
    cpu->haltContext(thread_num);
}

void
ExecContext::regStats(const string &name)
{
#ifdef FULL_SYSTEM
    kernelStats.regStats(name + ".kern");
#endif
}
<|endoftext|>