<commit_before>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Utility tool for generating an action table, word & affix lexicons,
// and a fully fleshed-out master spec.
// Input arguments:
// - Path to a commons store.
// - File pattern of training documents.
// - Name of output directory.
//
// The action table, all the lexicons, and the master spec are dumped in the
// output directory.
//
// Sample usage:
//   bazel-bin/nlp/parser/trainer/generate-action-table
//       --documents='/tmp/documents.*'
//       --commons=/tmp/common_store.encoded
//       --output_dir='/tmp/out'

#include <string>
#include <vector>

#include "base/flags.h"
#include "base/init.h"
#include "base/logging.h"
#include "base/macros.h"
#include "dragnn/protos/spec.pb.h"
#include "file/file.h"
#include "frame/object.h"
#include "frame/serialization.h"
#include "frame/store.h"
#include "nlp/parser/trainer/action-table-generator.h"
#include "nlp/parser/trainer/feature.h"
#include "nlp/parser/trainer/shared-resources.h"
#include "string/strcat.h"
#include "syntaxnet/affix.h"
#include "syntaxnet/task_context.h"
#include "syntaxnet/task_spec.pb.h"
#include "syntaxnet/term_frequency_map.h"
#include "syntaxnet/utils.h"
#include "tensorflow/core/lib/strings/str_util.h"

using sling::File;
using sling::FileDecoder;
using sling::Object;
using sling::Store;
using sling::nlp::ActionTable;
using sling::nlp::ActionTableGenerator;
using sling::nlp::SemparFeatureExtractor;
using sling::nlp::SharedResources;
using sling::nlp::Document;

using syntaxnet::dragnn::ComponentSpec;
using syntaxnet::dragnn::MasterSpec;
using syntaxnet::dragnn::RegisteredModuleSpec;

DEFINE_string(documents, "", "File pattern of training documents.");
DEFINE_string(commons, "", "Path to common store.");
DEFINE_string(output_dir, "/tmp/sempar_out", "Output directory.");

// Various options for generating the action table, lexicons, spec.
constexpr int kActionTableCoveragePercentile = 99;
constexpr bool kActionTableFromPerSentence = true;
constexpr int kLexiconMaxPrefixLength = 3;
constexpr int kLexiconMaxSuffixLength = 3;

// Workspace for various artifacts that are used/created by this tool.
struct Artifacts {
  SharedResources resources;

  std::vector<string> train_files;  // all training documents
  string commons_filename;          // full path to commons
  string action_table_filename;     // full path of generated action table
  MasterSpec spec;                  // generated master spec
  string spec_file;                 // path to the master spec

  // Lexicon name -> Full path to the generated lexicon.
  std::unordered_map<string, string> lexicon_paths;

  Store *global() { return resources.global; }
  const ActionTable &table() { return resources.table; }
};

// Returns the full output path for 'basename'.
string FullName(const string &basename) {
  string s = FLAGS_output_dir;
  CHECK(!s.empty());
  if (s.back() == '/') s.pop_back();
  return sling::StrCat(s, "/", basename);
}

void OutputActionTable(Artifacts *artifacts) {
  ActionTableGenerator generator(artifacts->global());
  generator.set_coverage_percentile(kActionTableCoveragePercentile);
  generator.set_per_sentence(kActionTableFromPerSentence);

  LOG(INFO) << "Processing " << artifacts->train_files.size() << " documents..";
  int count = 0;
  for (const string &file : artifacts->train_files) {
    Store local(artifacts->global());
    FileDecoder decoder(&local, file);
    Object top = decoder.Decode();
    if (top.invalid()) continue;

    count++;
    Document document(top.AsFrame());
    generator.Add(document);
    if (count % 100 == 1) LOG(INFO) << count << " documents processed.";
  }
  LOG(INFO) << "Processed " << count << " documents.";

  string table_file = FullName("table");
  string summary_file = FullName("table.summary");
  string unknown_file = FullName("table.unknown_symbols");
  generator.Save(table_file, summary_file, unknown_file);
  artifacts->action_table_filename = table_file;

  LOG(INFO) << "Wrote action table to " << table_file
            << ", " << summary_file << ", " << unknown_file;
  artifacts->resources.LoadActionTable(table_file);
}

void WriteAffixTable(const syntaxnet::AffixTable &affixes,
                     const string &output_file) {
  syntaxnet::ProtoRecordWriter writer(output_file);
  affixes.Write(&writer);
}

/*
void OutputLexicons(Artifacts *artifacts) {
  using syntaxnet::TermFrequencyMap;
  using syntaxnet::AffixTable;

  // Term frequency maps to be populated by the corpus.
  TermFrequencyMap words;
  TermFrequencyMap lcwords;

  // Affix tables to be populated by the corpus.
  AffixTable prefixes(AffixTable::PREFIX, kLexiconMaxPrefixLength);
  AffixTable suffixes(AffixTable::SUFFIX, kLexiconMaxSuffixLength);

  // Make a pass over the corpus.
  int64 num_tokens = 0;
  int64 num_documents = 0;
  for (const string &file : artifacts->train_files) {
    Store local(artifacts->global());
    FileDecoder decoder(&local, file);
    Object top = decoder.Decode();
    if (top.invalid()) continue;

    num_documents++;
    Document document(top.AsFrame());

    // Gather token information.
    for (int t = 0; t < document.num_tokens(); ++t) {
      // Get token and lowercased word.
      const auto &token = document.token(t);
      string word = token.text();
      syntaxnet::utils::NormalizeDigits(&word);
      string lcword = tensorflow::str_util::Lowercase(word);

      // Make sure the token does not contain a newline.
      CHECK(lcword.find('\n') == string::npos);

      // Increment frequencies (only for terms that exist).
      if (!word.empty() && !HasSpaces(word)) words.Increment(word);
      if (!lcword.empty() && !HasSpaces(lcword)) lcwords.Increment(lcword);

      // Add prefixes/suffixes for the current word.
      prefixes.AddAffixesForWord(word.c_str(), word.size());
      suffixes.AddAffixesForWord(word.c_str(), word.size());

      // Update the number of processed tokens.
      ++num_tokens;
    }
  }
  LOG(INFO) << "Term maps collected over " << num_tokens << " tokens from "
            << num_documents << " documents";

  // Write mappings to disk.
  words.Save(FullName("word-map"));
  lcwords.Save(FullName("lcword-map"));

  // Write affixes to disk.
  WriteAffixTable(prefixes, FullName("prefix-table"));
  WriteAffixTable(suffixes, FullName("suffix-table"));

  artifacts->lexicon_paths["word-map"] = FullName("word-map");
  artifacts->lexicon_paths["lcword-map"] = FullName("lcword-map");
  artifacts->lexicon_paths["prefix-table"] = FullName("prefix-table");
  artifacts->lexicon_paths["suffix-table"] = FullName("suffix-table");
  LOG(INFO) << "Wrote term maps and affix tables.";
} */

syntaxnet::dragnn::ComponentSpec *AddComponent(
    const string &name,
    const string &backend,
    const string &network_unit,
    const string &transition_system,
    Artifacts *artifacts) {
  auto *c = artifacts->spec.add_component();
  c->set_name(name);
  c->mutable_backend()->set_registered_name(backend);
  c->mutable_network_unit()->set_registered_name(network_unit);
  c->mutable_transition_system()->set_registered_name(transition_system);
  c->mutable_component_builder()->set_registered_name(
      "DynamicComponentBuilder");
  return c;
}

void SetParam(RegisteredModuleSpec *spec,
              const string &key,
              const string &value) {
  (*spec->mutable_parameters())[key] = value;
}

void AddFixedFeature(ComponentSpec *component,
                     const string &name,
                     const string &fml,
                     int embedding_dim) {
  auto *f = component->add_fixed_feature();
  f->set_name(name);
  f->set_fml(fml);
  f->set_embedding_dim(embedding_dim);
  f->set_predicate_map("hashed");
}

void AddLinkedFeature(ComponentSpec *component,
                      const string &name,
                      const string &fml_pattern,
                      int fml_arg_max,
                      int embedding_dim,
                      const string &source,
                      const string &translator) {
  // Replace "XX" with i, for every i in [0, fml_arg_max).
  // e.g. constant(XX) would yield constant(0), constant(1) etc.
  string fml;
  size_t i = fml_pattern.find("XX");
  if (i == string::npos) {
    fml = fml_pattern;
  } else {
    for (int j = 0; j < fml_arg_max; ++j) {
      if (j > 0) sling::StrAppend(&fml, " ");
      sling::StrAppend(
          &fml, fml_pattern.substr(0, i), j, fml_pattern.substr(i + 2));
    }
  }
  auto *f = component->add_linked_feature();
  f->set_name(name);
  f->set_fml(fml);
  f->set_embedding_dim(embedding_dim);
  f->set_source_component(source);
  f->set_source_translator(translator);
  f->set_source_layer("layer_0");
  std::vector<string> parts = syntaxnet::utils::Split(fml, ' ');
  f->set_size(parts.size());
}

void AddLinkedFeature(ComponentSpec *component,
                      const string &name,
                      const string &fml_pattern,
                      int fml_arg_max,
                      int embedding_dim,
                      const string &source) {
  AddLinkedFeature(component, name, fml_pattern, fml_arg_max,
                   embedding_dim, source, "identity");
}

void AddResource(ComponentSpec *spec,
                 const string &name,
                 const string &file_pattern,
                 const string &format,
                 const string &record) {
  auto *r = spec->add_resource();
  r->set_name(name);
  auto *part = r->add_part();
  part->set_file_pattern(file_pattern);
  part->set_file_format(format);
  part->set_record_format(record);
}

// Finds and fills the domain size of all fixed features.
// This is done by calling ParserEmbeddingFeatureExtractor::Init(),
// which in turn requires us to create the requisite task context.
void TrainFeatures(Artifacts *artifacts, ComponentSpec *spec) {
  using sling::StrAppend;
  using sling::StrCat;

  SemparFeatureExtractor fixed_feature_extractor;
  for (const auto &fixed_channel : spec->fixed_feature()) {
    fixed_feature_extractor.AddChannel(fixed_channel);
  }

  // Note: We are NOT copying spec->transition_system().parameters() over to
  // the features. Therefore any parameters for the features should be
  // specified in the FML itself.

  fixed_feature_extractor.Train(artifacts->train_files,
                                FLAGS_output_dir,
                                true /* fill vocabulary sizes */,
                                &artifacts->resources,
                                spec);

  SemparFeatureExtractor linked_feature_extractor;
  for (const auto &linked_channel : spec->linked_feature()) {
    linked_feature_extractor.AddChannel(linked_channel);
  }

  linked_feature_extractor.Train(
      artifacts->train_files,
      FLAGS_output_dir,
      false /* linked features don't need vocab sizes */,
      &artifacts->resources,
      spec);
}

void OutputMasterSpec(Artifacts *artifacts) {
  // Left to right LSTM.
  auto *lr_lstm = AddComponent(
      "lr_lstm", "SemparComponent", "LSTMNetwork", "shift-only", artifacts);
  SetParam(lr_lstm->mutable_transition_system(), "left_to_right", "true");
  SetParam(lr_lstm->mutable_network_unit(), "hidden_layer_sizes", "256");
  lr_lstm->set_num_actions(1);
  AddFixedFeature(lr_lstm, "words", "word", 32);
  AddFixedFeature(lr_lstm, "suffix", "suffix(length=2)", 16);
  AddFixedFeature(
      lr_lstm, "shape",
      "digit hyphen punctuation quote capitalization", 8);

  // Right to left LSTM.
  auto *rl_lstm = artifacts->spec.add_component();
  *rl_lstm = *lr_lstm;
  rl_lstm->set_name("rl_lstm");
  SetParam(rl_lstm->mutable_transition_system(), "left_to_right", "false");

  // Feed forward unit.
  auto *ff = AddComponent(
      "ff", "SemparComponent", "FeedForwardNetwork", "sempar", artifacts);
  ff->set_num_actions(artifacts->table().NumActions());
  //AddFixedFeature(ff, "roles", "roles", 16);
  AddLinkedFeature(
      ff, "frame-creation-steps", "frame-creation(XX)", 5, 64, "ff");
  AddLinkedFeature(
      ff, "frame-focus-steps", "frame-focus(XX)", 5, 64, "ff");
  AddLinkedFeature(
      ff, "frame-end-lr", "frame-end(XX)", 5, 32, "lr_lstm");
  AddLinkedFeature(
      ff, "frame-end-rl", "frame-end(XX)", 5, 32, "rl_lstm",
      "reverse_token");
  AddLinkedFeature(ff, "history", "constant(XX)", 4, 64, "ff");
  AddLinkedFeature(ff, "lr", "current-token", -1, 32, "lr_lstm");
  AddLinkedFeature(
      ff, "rl", "current-token", -1, 32, "rl_lstm", "reverse-token");

  // Add any resources required by the feed forward unit's features.
  AddResource(ff, "commons", artifacts->commons_filename, "store", "encoded");
  AddResource(
      ff, "action-table", artifacts->action_table_filename, "store", "encoded");

  // Fill vocabulary sizes and feature sizes. Recall that this will also
  // add any lexicons as resources (e.g. needed by input.word, input.suffix).
  TrainFeatures(artifacts, lr_lstm);
  TrainFeatures(artifacts, rl_lstm);
  TrainFeatures(artifacts, ff);

  // Dump the master spec.
  string spec_file = FullName("master_spec");
  CHECK_OK(File::WriteContents(spec_file, artifacts->spec.DebugString()));
  artifacts->spec_file = spec_file;
  LOG(INFO) << "Wrote master spec to " << spec_file;
}

int main(int argc, char **argv) {
  sling::InitProgram(&argc, &argv);

  CHECK(!FLAGS_documents.empty()) << "No documents specified.";
  CHECK(!FLAGS_commons.empty()) << "No commons specified.";
  CHECK(!FLAGS_output_dir.empty()) << "No output_dir specified.";

  if (!File::Exists(FLAGS_output_dir)) {
    CHECK_OK(File::Mkdir(FLAGS_output_dir));
  }

  Artifacts artifacts;
  artifacts.commons_filename = FLAGS_commons;
  artifacts.resources.LoadGlobalStore(FLAGS_commons);

  // Get a list of all training files.
  CHECK_OK(File::Match(FLAGS_documents, &artifacts.train_files));

  // Dump action table.
  OutputActionTable(&artifacts);

  // Make feature lexicons.
  //OutputLexicons(&artifacts);

  // Make master spec.
  OutputMasterSpec(&artifacts);

  return 0;
}
<commit_msg>Actually add generate-master-spec.cc (left out in previous commit).<commit_after>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Utility tool for generating a fully populated master spec.
// In particular, it creates the action table, all resources needed by the
// features, computes the feature domain sizes and uses all this to output
// a full MasterSpec.
//
// Input arguments:
// - Path to a commons store.
// - File pattern of training documents.
// - Name of output directory.
//
// Sample usage:
//   bazel-bin/nlp/parser/trainer/generate-master-spec
//       --documents='/tmp/documents.*'
//       --commons=/tmp/common_store.encoded
//       --output_dir='/tmp/out'

#include <string>
#include <vector>

#include "base/flags.h"
#include "base/init.h"
#include "base/logging.h"
#include "base/macros.h"
#include "dragnn/protos/spec.pb.h"
#include "file/file.h"
#include "frame/object.h"
#include "frame/serialization.h"
#include "frame/store.h"
#include "nlp/parser/trainer/action-table-generator.h"
#include "nlp/parser/trainer/feature.h"
#include "nlp/parser/trainer/shared-resources.h"
#include "string/strcat.h"
#include "syntaxnet/affix.h"
#include "syntaxnet/task_context.h"
#include "syntaxnet/task_spec.pb.h"
#include "syntaxnet/term_frequency_map.h"
#include "syntaxnet/utils.h"
#include "tensorflow/core/lib/strings/str_util.h"

using sling::File;
using sling::FileDecoder;
using sling::Object;
using sling::Store;
using sling::nlp::ActionTable;
using sling::nlp::ActionTableGenerator;
using sling::nlp::SemparFeatureExtractor;
using sling::nlp::SharedResources;
using sling::nlp::Document;

using syntaxnet::dragnn::ComponentSpec;
using syntaxnet::dragnn::MasterSpec;
using syntaxnet::dragnn::RegisteredModuleSpec;

DEFINE_string(documents, "", "File pattern of training documents.");
DEFINE_string(commons, "", "Path to common store.");
DEFINE_string(output_dir, "/tmp/sempar_out", "Output directory.");

// Various options for generating the action table, lexicons, spec.
constexpr int kActionTableCoveragePercentile = 99;
constexpr bool kActionTableFromPerSentence = true;
constexpr int kLexiconMaxPrefixLength = 3;
constexpr int kLexiconMaxSuffixLength = 3;

// Workspace for various artifacts that are used/created by this tool.
struct Artifacts {
  SharedResources resources;

  std::vector<string> train_files;  // all training documents
  string commons_filename;          // full path to commons
  string action_table_filename;     // full path of generated action table
  MasterSpec spec;                  // generated master spec
  string spec_file;                 // path to the master spec

  // Lexicon name -> Full path to the generated lexicon.
  std::unordered_map<string, string> lexicon_paths;

  Store *global() { return resources.global; }
  const ActionTable &table() { return resources.table; }
};

// Returns the full output path for 'basename'.
string FullName(const string &basename) {
  string s = FLAGS_output_dir;
  CHECK(!s.empty());
  if (s.back() == '/') s.pop_back();
  return sling::StrCat(s, "/", basename);
}

void OutputActionTable(Artifacts *artifacts) {
  ActionTableGenerator generator(artifacts->global());
  generator.set_coverage_percentile(kActionTableCoveragePercentile);
  generator.set_per_sentence(kActionTableFromPerSentence);

  LOG(INFO) << "Processing " << artifacts->train_files.size() << " documents..";
  int count = 0;
  for (const string &file : artifacts->train_files) {
    Store local(artifacts->global());
    FileDecoder decoder(&local, file);
    Object top = decoder.Decode();
    if (top.invalid()) continue;

    count++;
    Document document(top.AsFrame());
    generator.Add(document);
    if (count % 100 == 1) LOG(INFO) << count << " documents processed.";
  }
  LOG(INFO) << "Processed " << count << " documents.";

  string table_file = FullName("table");
  string summary_file = FullName("table.summary");
  string unknown_file = FullName("table.unknown_symbols");
  generator.Save(table_file, summary_file, unknown_file);
  artifacts->action_table_filename = table_file;

  LOG(INFO) << "Wrote action table to " << table_file
            << ", " << summary_file << ", " << unknown_file;
  artifacts->resources.LoadActionTable(table_file);
}

void WriteAffixTable(const syntaxnet::AffixTable &affixes,
                     const string &output_file) {
  syntaxnet::ProtoRecordWriter writer(output_file);
  affixes.Write(&writer);
}

/*
void OutputLexicons(Artifacts *artifacts) {
  using syntaxnet::TermFrequencyMap;
  using syntaxnet::AffixTable;

  // Term frequency maps to be populated by the corpus.
  TermFrequencyMap words;
  TermFrequencyMap lcwords;

  // Affix tables to be populated by the corpus.
  AffixTable prefixes(AffixTable::PREFIX, kLexiconMaxPrefixLength);
  AffixTable suffixes(AffixTable::SUFFIX, kLexiconMaxSuffixLength);

  // Make a pass over the corpus.
  int64 num_tokens = 0;
  int64 num_documents = 0;
  for (const string &file : artifacts->train_files) {
    Store local(artifacts->global());
    FileDecoder decoder(&local, file);
    Object top = decoder.Decode();
    if (top.invalid()) continue;

    num_documents++;
    Document document(top.AsFrame());

    // Gather token information.
    for (int t = 0; t < document.num_tokens(); ++t) {
      // Get token and lowercased word.
      const auto &token = document.token(t);
      string word = token.text();
      syntaxnet::utils::NormalizeDigits(&word);
      string lcword = tensorflow::str_util::Lowercase(word);

      // Make sure the token does not contain a newline.
      CHECK(lcword.find('\n') == string::npos);

      // Increment frequencies (only for terms that exist).
      if (!word.empty() && !HasSpaces(word)) words.Increment(word);
      if (!lcword.empty() && !HasSpaces(lcword)) lcwords.Increment(lcword);

      // Add prefixes/suffixes for the current word.
      prefixes.AddAffixesForWord(word.c_str(), word.size());
      suffixes.AddAffixesForWord(word.c_str(), word.size());

      // Update the number of processed tokens.
      ++num_tokens;
    }
  }
  LOG(INFO) << "Term maps collected over " << num_tokens << " tokens from "
            << num_documents << " documents";

  // Write mappings to disk.
  words.Save(FullName("word-map"));
  lcwords.Save(FullName("lcword-map"));

  // Write affixes to disk.
  WriteAffixTable(prefixes, FullName("prefix-table"));
  WriteAffixTable(suffixes, FullName("suffix-table"));

  artifacts->lexicon_paths["word-map"] = FullName("word-map");
  artifacts->lexicon_paths["lcword-map"] = FullName("lcword-map");
  artifacts->lexicon_paths["prefix-table"] = FullName("prefix-table");
  artifacts->lexicon_paths["suffix-table"] = FullName("suffix-table");
  LOG(INFO) << "Wrote term maps and affix tables.";
} */

syntaxnet::dragnn::ComponentSpec *AddComponent(
    const string &name,
    const string &backend,
    const string &network_unit,
    const string &transition_system,
    Artifacts *artifacts) {
  auto *c = artifacts->spec.add_component();
  c->set_name(name);
  c->mutable_backend()->set_registered_name(backend);
  c->mutable_network_unit()->set_registered_name(network_unit);
  c->mutable_transition_system()->set_registered_name(transition_system);
  c->mutable_component_builder()->set_registered_name(
      "DynamicComponentBuilder");
  return c;
}

void SetParam(RegisteredModuleSpec *spec,
              const string &key,
              const string &value) {
  (*spec->mutable_parameters())[key] = value;
}

void AddFixedFeature(ComponentSpec *component,
                     const string &name,
                     const string &fml,
                     int embedding_dim) {
  auto *f = component->add_fixed_feature();
  f->set_name(name);
  f->set_fml(fml);
  f->set_embedding_dim(embedding_dim);
  f->set_predicate_map("hashed");
}

void AddLinkedFeature(ComponentSpec *component,
                      const string &name,
                      const string &fml_pattern,
                      int fml_arg_max,
                      int embedding_dim,
                      const string &source,
                      const string &translator) {
  // Replace "XX" with i, for every i in [0, fml_arg_max).
  // e.g. constant(XX) would yield constant(0), constant(1) etc.
  string fml;
  size_t i = fml_pattern.find("XX");
  if (i == string::npos) {
    fml = fml_pattern;
  } else {
    for (int j = 0; j < fml_arg_max; ++j) {
      if (j > 0) sling::StrAppend(&fml, " ");
      sling::StrAppend(
          &fml, fml_pattern.substr(0, i), j, fml_pattern.substr(i + 2));
    }
  }
  auto *f = component->add_linked_feature();
  f->set_name(name);
  f->set_fml(fml);
  f->set_embedding_dim(embedding_dim);
  f->set_source_component(source);
  f->set_source_translator(translator);
  f->set_source_layer("layer_0");
  std::vector<string> parts = syntaxnet::utils::Split(fml, ' ');
  f->set_size(parts.size());
}

void AddLinkedFeature(ComponentSpec *component,
                      const string &name,
                      const string &fml_pattern,
                      int fml_arg_max,
                      int embedding_dim,
                      const string &source) {
  AddLinkedFeature(component, name, fml_pattern, fml_arg_max,
                   embedding_dim, source, "identity");
}

void AddResource(ComponentSpec *spec,
                 const string &name,
                 const string &file_pattern,
                 const string &format,
                 const string &record) {
  auto *r = spec->add_resource();
  r->set_name(name);
  auto *part = r->add_part();
  part->set_file_pattern(file_pattern);
  part->set_file_format(format);
  part->set_record_format(record);
}

// Finds and fills the domain size of all fixed features.
// This is done by calling ParserEmbeddingFeatureExtractor::Init(),
// which in turn requires us to create the requisite task context.
void TrainFeatures(Artifacts *artifacts, ComponentSpec *spec) {
  using sling::StrAppend;
  using sling::StrCat;

  SemparFeatureExtractor fixed_feature_extractor;
  for (const auto &fixed_channel : spec->fixed_feature()) {
    fixed_feature_extractor.AddChannel(fixed_channel);
  }

  // Note: We are NOT copying spec->transition_system().parameters() over to
  // the features. Therefore any parameters for the features should be
  // specified in the FML itself.

  fixed_feature_extractor.Train(artifacts->train_files,
                                FLAGS_output_dir,
                                true /* fill vocabulary sizes */,
                                &artifacts->resources,
                                spec);

  SemparFeatureExtractor linked_feature_extractor;
  for (const auto &linked_channel : spec->linked_feature()) {
    linked_feature_extractor.AddChannel(linked_channel);
  }

  linked_feature_extractor.Train(
      artifacts->train_files,
      FLAGS_output_dir,
      false /* linked features don't need vocab sizes */,
      &artifacts->resources,
      spec);
}

void OutputMasterSpec(Artifacts *artifacts) {
  // Left to right LSTM.
  auto *lr_lstm = AddComponent(
      "lr_lstm", "SemparComponent", "LSTMNetwork", "shift-only", artifacts);
  SetParam(lr_lstm->mutable_transition_system(), "left_to_right", "true");
  SetParam(lr_lstm->mutable_network_unit(), "hidden_layer_sizes", "256");
  lr_lstm->set_num_actions(1);
  AddFixedFeature(lr_lstm, "words", "word", 32);
  AddFixedFeature(lr_lstm, "suffix", "suffix(length=2)", 16);
  AddFixedFeature(
      lr_lstm, "shape",
      "digit hyphen punctuation quote capitalization", 8);

  // Right to left LSTM.
  auto *rl_lstm = artifacts->spec.add_component();
  *rl_lstm = *lr_lstm;
  rl_lstm->set_name("rl_lstm");
  SetParam(rl_lstm->mutable_transition_system(), "left_to_right", "false");

  // Feed forward unit.
  auto *ff = AddComponent(
      "ff", "SemparComponent", "FeedForwardNetwork", "sempar", artifacts);
  ff->set_num_actions(artifacts->table().NumActions());
  //AddFixedFeature(ff, "roles", "roles", 16);
  AddLinkedFeature(
      ff, "frame-creation-steps", "frame-creation(XX)", 5, 64, "ff");
  AddLinkedFeature(
      ff, "frame-focus-steps", "frame-focus(XX)", 5, 64, "ff");
  AddLinkedFeature(
      ff, "frame-end-lr", "frame-end(XX)", 5, 32, "lr_lstm");
  AddLinkedFeature(
      ff, "frame-end-rl", "frame-end(XX)", 5, 32, "rl_lstm",
      "reverse_token");
  AddLinkedFeature(ff, "history", "constant(XX)", 4, 64, "ff");
  AddLinkedFeature(ff, "lr", "current-token", -1, 32, "lr_lstm");
  AddLinkedFeature(
      ff, "rl", "current-token", -1, 32, "rl_lstm", "reverse-token");

  // Add any resources required by the feed forward unit's features.
  AddResource(ff, "commons", artifacts->commons_filename, "store", "encoded");
  AddResource(
      ff, "action-table", artifacts->action_table_filename, "store", "encoded");

  // Fill vocabulary sizes and feature sizes. Recall that this will also
  // add any lexicons as resources (e.g. needed by input.word, input.suffix).
  TrainFeatures(artifacts, lr_lstm);
  TrainFeatures(artifacts, rl_lstm);
  TrainFeatures(artifacts, ff);

  // Dump the master spec.
  string spec_file = FullName("master_spec");
  CHECK_OK(File::WriteContents(spec_file, artifacts->spec.DebugString()));
  artifacts->spec_file = spec_file;
  LOG(INFO) << "Wrote master spec to " << spec_file;
}

int main(int argc, char **argv) {
  sling::InitProgram(&argc, &argv);

  CHECK(!FLAGS_documents.empty()) << "No documents specified.";
  CHECK(!FLAGS_commons.empty()) << "No commons specified.";
  CHECK(!FLAGS_output_dir.empty()) << "No output_dir specified.";

  if (!File::Exists(FLAGS_output_dir)) {
    CHECK_OK(File::Mkdir(FLAGS_output_dir));
  }

  Artifacts artifacts;
  artifacts.commons_filename = FLAGS_commons;
  artifacts.resources.LoadGlobalStore(FLAGS_commons);

  // Get a list of all training files.
  CHECK_OK(File::Match(FLAGS_documents, &artifacts.train_files));

  // Dump action table.
  OutputActionTable(&artifacts);

  // Make feature lexicons.
  //OutputLexicons(&artifacts);

  // Make master spec.
  OutputMasterSpec(&artifacts);

  return 0;
}
<|endoftext|>


<commit_before>// Copyright (C) 2007-2015  CEA/DEN, EDF R&D, OPEN CASCADE
//
// Copyright (C) 2003-2007  OPEN CASCADE, EADS/CCR, LIP6, CEA/DEN,
// CEDRAT, EDF R&D, LEG, PRINCIPIA R&D, BUREAU VERITAS
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
//
// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
//

#include "SalomeApp_VisualState.h"

#include "SalomeApp_Module.h"
#include "SalomeApp_Study.h"
#include "SalomeApp_Application.h"

//#include <SUIT_ResourceMgr.h>
#include <SUIT_ViewManager.h>
#include <SUIT_ViewWindow.h>
#include <QtxWorkstack.h>

#include <STD_TabDesktop.h>

#include <QList>
#include <QApplication>
#include <QMultiHash>

#include <SALOMEDSClient_ClientFactory.hxx>//?
#include <SALOMEDSClient_IParameters.hxx>//?

#include <vector>//?
#include <string>//?
#include <cstdio>

/*!
  Constructor.
*/
SalomeApp_VisualState::SalomeApp_VisualState( SalomeApp_Application* app )
  : QObject(),
    myApp( app )
{
}

/*!
  Destructor.
*/
SalomeApp_VisualState::~SalomeApp_VisualState()
{
}

/*!
  Sets names of all view windows in given list.  This is used
  in order to apply the same naming algorithm when saving and restoring
  view windows.  Names of view windows must be the same before saving
  workstack (splitters) information, and before its restoring!
  Naming rule: ViewerType_IndexOfViewerOfThisType_IndexOfViewInThisViewer
               VTKViewer_0_0
               OCCViewer_0_0  OCCViewer_0_1  OCCViewer_0_2
               VTKViewer_1_0
*/
void nameViewWindows( const ViewManagerList& lst )
{
  QMap<QString, int> viewersCounter;
  for ( QList<SUIT_ViewManager*>::const_iterator it = lst.begin(); it != lst.end(); ++it )
  {
    SUIT_ViewManager* aVM = *it;
    if ( !aVM )
      continue;

    int view_count = aVM->getViewsCount();
    QString vType = aVM->getType();
    if ( !view_count )
      continue; //No views is opened in the viewer

    if ( !viewersCounter.contains( vType ) )
      viewersCounter.insert( vType, 0 );

    int& viewerID = viewersCounter[vType];

    QVector<SUIT_ViewWindow*> views = aVM->getViews();
    for ( int i = 0; i < view_count; i++ )
    {
      QString vName = QString( "%1_%2_%3" ).arg( vType ).arg( viewerID ).arg( i );
      views[i]->setObjectName( vName );
    }
    viewerID++;
  }
}

/*!
  Stores the visual parameters of the viewers
*/
int SalomeApp_VisualState::storeState()
{
  SalomeApp_Study* study = dynamic_cast<SalomeApp_Study*>( myApp->activeStudy() );
  if ( !study )
    return -1;

  // unlock study if it is locked
  bool aLocked = study->studyDS()->GetProperties()->IsLocked();
  if (aLocked) study->studyDS()->GetProperties()->SetLocked(false);

  int savePoint = 1;
  std::vector<int> savePoints = study->getSavePoints();
  //Calculate a new savePoint number = the last save point number + 1
  if ( savePoints.size() > 0)
    savePoint = savePoints[savePoints.size()-1] + 1;

  _PTR(AttributeParameter) ap = study->studyDS()->GetCommonParameters( study->getVisualComponentName().toLatin1().constData(),
								       savePoint );
  _PTR(IParameters) ip = ClientFactory::getIParameters( ap );

  ViewManagerList lst;
  myApp->viewManagers( lst );

  // setting unique names for view windows in order to save this view inside
  // workstack's structure (see below).  On restore the views with the same names will
  // be placed to the same place inside the workstack's splitters.
  nameViewWindows( lst );

  // store active window's name
  SUIT_ViewWindow* win = myApp->desktop()->activeWindow();
  if ( win )
    ip->setProperty("AP_ACTIVE_VIEW", win->objectName().toStdString() );

  int viewerID = 0;
  SUIT_ViewManager* vm = 0;
  QListIterator<SUIT_ViewManager*> it( lst );
  while ( it.hasNext() ) {
    vm = it.next();
    if ( !vm ) continue;

    int view_count = vm->getViewsCount();
    if ( !view_count )
      continue; //No views is opened in the viewer

    std::string viewerEntry = QString( "%1_%2" ).arg( vm->getType() ).arg( ++viewerID ).toStdString();
    ip->append("AP_VIEWERS_LIST", viewerEntry);

    QVector<SUIT_ViewWindow*> views = vm->getViews();
    for(int i = 0; i<view_count; i++) {
      ip->append( viewerEntry, views[i]->windowTitle().toStdString() );
      ip->append( viewerEntry, views[i]->getVisualParameters().toStdString() );
    }
  }

  //Save information about split areas
  if ( myApp->desktop()->inherits( "STD_TabDesktop" ) )
  {
    QString workstackInfo;
    QtxWorkstack* workstack = ((STD_TabDesktop*)myApp->desktop())->workstack();
    QByteArray geomState = workstack->saveState( 0 );
    ip->setProperty( "AP_WORKSTACK_INFO", QString( geomState.toHex() ).toStdString() );
  }

  //Save a name of the active module
  if ( CAM_Module* activeModule = myApp->activeModule() )
    ip->setProperty( "AP_ACTIVE_MODULE", activeModule->moduleName().toStdString() );

  //Store visual parameters of the modules
  QList<CAM_Module*> mlist;
  myApp->modules( mlist );
  QListIterator<CAM_Module*> itM( mlist );
  CAM_Module* module = 0;
  while ( itM.hasNext() ) {
    module = itM.next();
    if ( !module ) continue;

    if ( SalomeApp_Module* sModule = dynamic_cast<SalomeApp_Module*>( module ) ) {
      ip->append( "AP_MODULES_LIST", sModule->moduleName().toStdString() );
      sModule->storeVisualParameters( savePoint );
    }
  }

  // set default name of new savePoint
  study->setNameOfSavePoint( savePoint, QObject::tr( "SAVE_POINT_DEF_NAME" ) + QString::number( savePoint ) );

  if (aLocked) study->studyDS()->GetProperties()->SetLocked(true);

  return savePoint;
}

/*!
  Restores the visual parameters of the viewers
*/
void SalomeApp_VisualState::restoreState(int savePoint)
{
  SalomeApp_Study* study = dynamic_cast<SalomeApp_Study*>( myApp->activeStudy() );
  if ( !study )
    return;

  _PTR(AttributeParameter) ap = study->studyDS()->GetCommonParameters( study->getVisualComponentName().toLatin1().constData(),
								       savePoint );
  _PTR(IParameters) ip = ClientFactory::getIParameters(ap);

  qApp->installEventFilter( this );

  //Remove all already existent veiwers and their views
  //myApp->clearViewManagers();
  myApp->clearKnownViewManagers();

  //Restore the viewers and view windows
  int nbViewers = ip->nbValues( "AP_VIEWERS_LIST" );
  SUIT_ViewWindow* viewWin = 0;

  // parameters of view windows are stored in a map for restoring after restoring of the workstack
  QMap<SUIT_ViewWindow*, QString> viewersParameters;

  for ( int i = 0; i < nbViewers; i++ )
  {
    std::string viewerEntry = ip->getValue( "AP_VIEWERS_LIST", i );
    std::vector<std::string> veiewerParams = ip->parseValue(viewerEntry,'_');
    std::string type = veiewerParams[0];
    std::string viewerID = veiewerParams[1];
    SUIT_ViewManager* vm = myApp->newViewManager( type.c_str() );
    if ( !vm )
      continue; //Unknown viewer

    int nbViews = (ip->nbValues(viewerEntry))/2;

    //Create nbViews-1 view (-1 because 1 view is created by createViewManager)
    for ( int i = 1; i< nbViews; i++ )
    {
      SUIT_ViewWindow* aView = vm->createViewWindow();
      aView->show();
    }

    int viewCount = vm->getViewsCount();
    if ( viewCount != nbViews )
    {
      printf( "\nRestore visual state: Unknow error, Can't create a view!\n" );
      continue;
    }

    //Resize the views, set their captions and apply visual parameters.
    QVector<SUIT_ViewWindow*> views = vm->getViews();
    for ( int i = 0, j = 0; i<viewCount; i++, j+=2 )
    {
      viewWin = views[i];
      if ( !viewWin )
        continue;

      // wait untill the window is really shown.  This step fixes MANY bugs..
      //      while ( !vm->isVisible() )
      //        qApp->processEvents();

      viewWin->setWindowTitle( ip->getValue( viewerEntry, j ).c_str() );

      //      printf ( "VP for viewWin \"%s\": %s\n", viewerEntry.c_str(), ip->getValue(viewerEntry, j+1).c_str() );
      viewersParameters[ viewWin ] = ip->getValue( viewerEntry, j + 1 ).c_str();
      //viewWin->setVisualParameters(ip->getValue(viewerEntry, j+1).c_str());
    }
  }

  qApp->processEvents( QEventLoop::ExcludeUserInputEvents, 5000 );

  // restore modules' visual parameters
  std::vector<std::string> v = ip->getValues( "AP_MODULES_LIST" );
  for ( int i = 0; i < v.size(); i++ )
  {
    myApp->activateModule( v[i].c_str() );
    if ( SalomeApp_Module* module = dynamic_cast<SalomeApp_Module*>( myApp->activeModule() ) )
      module->restoreVisualParameters( savePoint );
  }

  // new view windows may have been created in  module->restoreVisualParameters() [GAUSS]
  // so here we store their visual parameters for later restoring..
  QList<SUIT_ViewManager*> lst;
  myApp->viewManagers( lst );
  for ( QList<SUIT_ViewManager*>::iterator itVM = lst.begin(); itVM != lst.end(); ++itVM )
  {
    SUIT_ViewManager* aVM = *itVM;
    if ( !aVM )
      continue;

    int view_count = aVM->getViewsCount();
    QVector<SUIT_ViewWindow*> views = aVM->getViews();
    for ( int i = 0; i < view_count; i++ )
    {
      if ( !viewersParameters.contains( views[i] ) )
      {
        viewersParameters[ views[i] ] = views[i]->getVisualParameters();
        //      printf ( "store VP for viewWin \"%s\": %s\n", views[i]->name(), views[i]->getVisualParameters().toLatin1().constData() );
      }
    }
  }

  // activate module that was active on save
  QString activeModuleName( ip->getProperty("AP_ACTIVE_MODULE" ).c_str() );
  if ( !activeModuleName.isEmpty() )
    myApp->activateModule( activeModuleName );

  // setting unique names for view windows in order to restore positions of view windows inside
  // workstack's structure (see below).  During save the same naming algorithm was used,
  // so the same views will get the same names.
  nameViewWindows( lst );

  qApp->processEvents( QEventLoop::ExcludeUserInputEvents, 5000 );

  // restore workstack parameters.  should be done after module's restoreVisualParameters(), because
  // some modules can create their own viewers (like VISU creates GaussViewers)
  if ( myApp->desktop()->inherits( "STD_TabDesktop" ) )
  {
    QtxWorkstack* workstack = ((STD_TabDesktop*)myApp->desktop())->workstack();
    workstack->restoreState( QByteArray::fromHex( QByteArray( ip->getProperty( "AP_WORKSTACK_INFO" ).c_str() ) ), 0 );
  }

  // restore visual parameters of view windows.  it must be done AFTER restoring workstack.
  // also set active view
  std::string activeViewName = ip->getProperty("AP_ACTIVE_VIEW");
  QMap<SUIT_ViewWindow*, QString>::Iterator mapIt;
  for ( mapIt = viewersParameters.begin(); mapIt != viewersParameters.end(); ++mapIt ) {
    mapIt.key()->setVisualParameters( mapIt.value() );
    if ( activeViewName == mapIt.key()->objectName().toStdString() )
      mapIt.key()->setFocus();
  }

  qApp->removeEventFilter( this );

  //  for ( it.toFirst(); it.current(); ++it ) {
  //    int view_count = it.current()->getViewsCount();
  //    QPtrVector<SUIT_ViewWindow> views = it.current()->getViews();
  //    for ( int i = 0; i < view_count; i++ )
  //      views[i]->setVisualParameters( viewersParameters[ views[i]->name() ] );
  //  }

  // set focus to previously saved active view window
  //  std::string activeViewName = ip->getProperty("AP_ACTIVE_VIEW");
  //  for ( it.toFirst(); it.current(); ++it ) {
  //    int view_count = it.current()->getViewsCount();
  //    QPtrVector<SUIT_ViewWindow> views = it.current()->getViews();
  //    for ( int i = 0; i < view_count; i++ )  {
  //      if ( activeViewName == views[i]->name() )
  //    views[i]->setFocus();
  //    }
  //  }
}

/*!
  Custom event filter
*/
bool SalomeApp_VisualState::eventFilter( QObject* o, QEvent* e )
{
  // eat keyboard and mouse events
  QEvent::Type aType = e->type();
  if ( aType == QEvent::MouseButtonDblClick ||
       aType == QEvent::MouseButtonPress ||
       aType == QEvent::MouseButtonRelease ||
       aType == QEvent::MouseMove ||
       aType == QEvent::KeyPress ||
       aType == QEvent::KeyRelease )
    return true;

  return QObject::eventFilter( o, e );
}
<commit_msg>allow auto loading of light modules when opening study file<commit_after>// Copyright (C) 2007-2015  CEA/DEN, EDF R&D, OPEN CASCADE
//
// Copyright (C) 2003-2007  OPEN CASCADE, EADS/CCR, LIP6, CEA/DEN,
// CEDRAT, EDF R&D, LEG, PRINCIPIA R&D, BUREAU VERITAS
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
//
// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
//

#include "SalomeApp_VisualState.h"

#include "SalomeApp_Module.h"
#include "SalomeApp_Study.h"
#include "SalomeApp_Application.h"

//#include <SUIT_ResourceMgr.h>
#include <SUIT_ViewManager.h>
#include <SUIT_ViewWindow.h>
#include <QtxWorkstack.h>

#include <STD_TabDesktop.h>

#include <QList>
#include <QApplication>
#include <QMultiHash>

#include <SALOMEDSClient_ClientFactory.hxx>//?
#include <SALOMEDSClient_IParameters.hxx>//?

#include <vector>//?
#include <string>//?
#include <cstdio>

/*!
  Constructor.
*/
SalomeApp_VisualState::SalomeApp_VisualState( SalomeApp_Application* app )
  : QObject(),
    myApp( app )
{
}

/*!
  Destructor.
*/
SalomeApp_VisualState::~SalomeApp_VisualState()
{
}

/*!
  Sets names of all view windows in given list.  This is used
  in order to apply the same naming algorithm when saving and restoring
  view windows.  Names of view windows must be the same before saving
  workstack (splitters) information, and before its restoring!
  Naming rule: ViewerType_IndexOfViewerOfThisType_IndexOfViewInThisViewer
               VTKViewer_0_0
               OCCViewer_0_0  OCCViewer_0_1  OCCViewer_0_2
               VTKViewer_1_0
*/
void nameViewWindows( const ViewManagerList& lst )
{
  QMap<QString, int> viewersCounter;
  for ( QList<SUIT_ViewManager*>::const_iterator it = lst.begin(); it != lst.end(); ++it )
  {
    SUIT_ViewManager* aVM = *it;
    if ( !aVM )
      continue;

    int view_count = aVM->getViewsCount();
    QString vType = aVM->getType();
    if ( !view_count )
      continue; //No views is opened in the viewer

    if ( !viewersCounter.contains( vType ) )
      viewersCounter.insert( vType, 0 );

    int& viewerID = viewersCounter[vType];

    QVector<SUIT_ViewWindow*> views = aVM->getViews();
    for ( int i = 0; i < view_count; i++ )
    {
      QString vName = QString( "%1_%2_%3" ).arg( vType ).arg( viewerID ).arg( i );
      views[i]->setObjectName( vName );
    }
    viewerID++;
  }
}

/*!
  Stores the visual parameters of the viewers
*/
int SalomeApp_VisualState::storeState()
{
  SalomeApp_Study* study = dynamic_cast<SalomeApp_Study*>( myApp->activeStudy() );
  if ( !study )
    return -1;

  // unlock study if it is locked
  bool aLocked = study->studyDS()->GetProperties()->IsLocked();
  if (aLocked) study->studyDS()->GetProperties()->SetLocked(false);

  int savePoint = 1;
  std::vector<int> savePoints = study->getSavePoints();
  //Calculate a new savePoint number = the last save point number + 1
  if ( savePoints.size() > 0)
    savePoint = savePoints[savePoints.size()-1] + 1;

  _PTR(AttributeParameter) ap = study->studyDS()->GetCommonParameters( study->getVisualComponentName().toLatin1().constData(),
								       savePoint );
  _PTR(IParameters) ip = ClientFactory::getIParameters( ap );

  ViewManagerList lst;
  myApp->viewManagers( lst );

  // setting unique names for view windows in order to save this view inside
  // workstack's structure (see below).  On restore the views with the same names will
  // be placed to the same place inside the workstack's splitters.
  nameViewWindows( lst );

  // store active window's name
  SUIT_ViewWindow* win = myApp->desktop()->activeWindow();
  if ( win )
    ip->setProperty("AP_ACTIVE_VIEW", win->objectName().toStdString() );

  int viewerID = 0;
  SUIT_ViewManager* vm = 0;
  QListIterator<SUIT_ViewManager*> it( lst );
  while ( it.hasNext() ) {
    vm = it.next();
    if ( !vm ) continue;

    int view_count = vm->getViewsCount();
    if ( !view_count )
      continue; //No views is opened in the viewer

    std::string viewerEntry = QString( "%1_%2" ).arg( vm->getType() ).arg( ++viewerID ).toStdString();
    ip->append("AP_VIEWERS_LIST", viewerEntry);

    QVector<SUIT_ViewWindow*> views = vm->getViews();
    for(int i = 0; i<view_count; i++) {
      ip->append( viewerEntry, views[i]->windowTitle().toStdString() );
      ip->append( viewerEntry, views[i]->getVisualParameters().toStdString() );
    }
  }

  //Save information about split areas
  if ( myApp->desktop()->inherits( "STD_TabDesktop" ) )
  {
    QString workstackInfo;
    QtxWorkstack* workstack = ((STD_TabDesktop*)myApp->desktop())->workstack();
    QByteArray geomState = workstack->saveState( 0 );
    ip->setProperty( "AP_WORKSTACK_INFO", QString( geomState.toHex() ).toStdString() );
  }

  //Save a name of the active module
  if ( CAM_Module* activeModule = myApp->activeModule() )
    ip->setProperty( "AP_ACTIVE_MODULE", activeModule->moduleName().toStdString() );

  //Store visual parameters of the modules
  QList<CAM_Module*> mlist;
  myApp->modules( mlist );
  QListIterator<CAM_Module*> itM( mlist );
  CAM_Module* module = 0;
  while ( itM.hasNext() ) {
    module = itM.next();
    //DEBTRACE("---");
    if ( !module ) continue;
    //DEBTRACE("--- " << module->name().toStdString());

    if ( LightApp_Module* lModule = dynamic_cast<LightApp_Module*>( module ) ) {
      //DEBTRACE("AP_MODULES_LIST append " << lModule->moduleName().toStdString());
      ip->append( "AP_MODULES_LIST", lModule->moduleName().toStdString() );
      if ( SalomeApp_Module* sModule = dynamic_cast<SalomeApp_Module*>( module ) )
        sModule->storeVisualParameters( savePoint );
    }
  }

  // set default name of new savePoint
  study->setNameOfSavePoint( savePoint, QObject::tr( "SAVE_POINT_DEF_NAME" ) + QString::number( savePoint ) );

  if (aLocked) study->studyDS()->GetProperties()->SetLocked(true);

  return savePoint;
}

/*!
  Restores the visual parameters of the viewers
*/
void SalomeApp_VisualState::restoreState(int savePoint)
{
  SalomeApp_Study* study = dynamic_cast<SalomeApp_Study*>( myApp->activeStudy() );
  if ( !study )
    return;

  _PTR(AttributeParameter) ap = study->studyDS()->GetCommonParameters( study->getVisualComponentName().toLatin1().constData(),
								       savePoint );
  _PTR(IParameters) ip = ClientFactory::getIParameters(ap);

  qApp->installEventFilter( this );

  //Remove all already existent veiwers and their views
  //myApp->clearViewManagers();
  myApp->clearKnownViewManagers();

  //Restore the viewers and view windows
  int nbViewers = ip->nbValues( "AP_VIEWERS_LIST" );
  SUIT_ViewWindow* viewWin = 0;

  // parameters of view windows are stored in a map for restoring after restoring of the workstack
  QMap<SUIT_ViewWindow*, QString> viewersParameters;

  for ( int i = 0; i < nbViewers; i++ )
  {
    std::string viewerEntry = ip->getValue( "AP_VIEWERS_LIST", i );
    std::vector<std::string> veiewerParams = ip->parseValue(viewerEntry,'_');
    std::string type = veiewerParams[0];
    std::string viewerID = veiewerParams[1];
    SUIT_ViewManager* vm = myApp->newViewManager( type.c_str() );
    if ( !vm )
      continue; //Unknown viewer

    int nbViews = (ip->nbValues(viewerEntry))/2;

    //Create nbViews-1 view (-1 because 1 view is created by createViewManager)
    for ( int i = 1; i< nbViews; i++ )
    {
      SUIT_ViewWindow* aView = vm->createViewWindow();
      aView->show();
    }

    int viewCount = vm->getViewsCount();
    if ( viewCount != nbViews )
    {
      printf( "\nRestore visual state: Unknow error, Can't create a view!\n" );
      continue;
    }

    //Resize the views, set their captions and apply visual parameters.
    QVector<SUIT_ViewWindow*> views = vm->getViews();
    for ( int i = 0, j = 0; i<viewCount; i++, j+=2 )
    {
      viewWin = views[i];
      if ( !viewWin )
        continue;

      // wait untill the window is really shown.  This step fixes MANY bugs..
      //      while ( !vm->isVisible() )
      //        qApp->processEvents();

      viewWin->setWindowTitle( ip->getValue( viewerEntry, j ).c_str() );

      //      printf ( "VP for viewWin \"%s\": %s\n", viewerEntry.c_str(), ip->getValue(viewerEntry, j+1).c_str() );
      viewersParameters[ viewWin ] = ip->getValue( viewerEntry, j + 1 ).c_str();
      //viewWin->setVisualParameters(ip->getValue(viewerEntry, j+1).c_str());
    }
  }

  qApp->processEvents( QEventLoop::ExcludeUserInputEvents, 5000 );

  // restore modules' visual parameters
  std::vector<std::string> v = ip->getValues( "AP_MODULES_LIST" );
  for ( int i = 0; i < v.size(); i++ )
  {
    myApp->activateModule( v[i].c_str() );
    if ( SalomeApp_Module* module = dynamic_cast<SalomeApp_Module*>( myApp->activeModule() ) )
      module->restoreVisualParameters( savePoint );
  }

  // new view windows may have been created in  module->restoreVisualParameters() [GAUSS]
  // so here we store their visual parameters for later restoring..
  QList<SUIT_ViewManager*> lst;
  myApp->viewManagers( lst );
  for ( QList<SUIT_ViewManager*>::iterator itVM = lst.begin(); itVM != lst.end(); ++itVM )
  {
    SUIT_ViewManager* aVM = *itVM;
    if ( !aVM )
      continue;

    int view_count = aVM->getViewsCount();
    QVector<SUIT_ViewWindow*> views = aVM->getViews();
    for ( int i = 0; i < view_count; i++ )
    {
      if ( !viewersParameters.contains( views[i] ) )
      {
        viewersParameters[ views[i] ] = views[i]->getVisualParameters();
        //      printf ( "store VP for viewWin \"%s\": %s\n", views[i]->name(), views[i]->getVisualParameters().toLatin1().constData() );
      }
    }
  }

  // activate module that was active on save
  QString activeModuleName( ip->getProperty("AP_ACTIVE_MODULE" ).c_str() );
  if ( !activeModuleName.isEmpty() )
    myApp->activateModule( activeModuleName );

  // setting unique names for view windows in order to restore positions of view windows inside
  // workstack's structure (see below).  During save the same naming algorithm was used,
  // so the same views will get the same names.
  nameViewWindows( lst );

  qApp->processEvents( QEventLoop::ExcludeUserInputEvents, 5000 );

  // restore workstack parameters.  should be done after module's restoreVisualParameters(), because
  // some modules can create their own viewers (like VISU creates GaussViewers)
  if ( myApp->desktop()->inherits( "STD_TabDesktop" ) )
  {
    QtxWorkstack* workstack = ((STD_TabDesktop*)myApp->desktop())->workstack();
    workstack->restoreState( QByteArray::fromHex( QByteArray( ip->getProperty( "AP_WORKSTACK_INFO" ).c_str() ) ), 0 );
  }

  // restore visual parameters of view windows.  it must be done AFTER restoring workstack.
  // also set active view
  std::string activeViewName = ip->getProperty("AP_ACTIVE_VIEW");
  QMap<SUIT_ViewWindow*, QString>::Iterator mapIt;
  for ( mapIt = viewersParameters.begin(); mapIt != viewersParameters.end(); ++mapIt ) {
    mapIt.key()->setVisualParameters( mapIt.value() );
    if ( activeViewName == mapIt.key()->objectName().toStdString() )
      mapIt.key()->setFocus();
  }

  qApp->removeEventFilter( this );

  //  for ( it.toFirst(); it.current(); ++it ) {
  //    int view_count = it.current()->getViewsCount();
  //    QPtrVector<SUIT_ViewWindow> views = it.current()->getViews();
  //    for ( int i = 0; i < view_count; i++ )
  //      views[i]->setVisualParameters( viewersParameters[ views[i]->name() ] );
  //  }

  // set focus to previously saved active view window
  //  std::string activeViewName = ip->getProperty("AP_ACTIVE_VIEW");
  //  for ( it.toFirst(); it.current(); ++it ) {
  //    int view_count = it.current()->getViewsCount();
  //    QPtrVector<SUIT_ViewWindow> views = it.current()->getViews();
  //    for ( int i = 0; i < view_count; i++ )  {
  //      if ( activeViewName == views[i]->name() )
  //    views[i]->setFocus();
  //    }
  //  }
}

/*!
  Custom event filter
*/
bool SalomeApp_VisualState::eventFilter( QObject* o, QEvent* e )
{
  // eat keyboard and mouse events
  QEvent::Type aType = e->type();
  if ( aType == QEvent::MouseButtonDblClick ||
       aType == QEvent::MouseButtonPress ||
       aType == QEvent::MouseButtonRelease ||
       aType == QEvent::MouseMove ||
       aType == QEvent::KeyPress ||
       aType == QEvent::KeyRelease )
    return true;

  return QObject::eventFilter( o, e );
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2015, 2016, Intel Corporation
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in
 *       the documentation and/or other materials provided with the
 *       distribution.
 *
 *     * Neither the name of Intel Corporation nor the names of its
 *       contributors may be used to endorse or promote products derived
 *       from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY LOG OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "gtest/gtest.h"
#include "mpi.h"

#ifndef NAME_MAX
#define NAME_MAX 1024
#endif

int main(int argc, char **argv)
{
    int err = 0;
    int rank = 0;
    int comm_size = 0;

    MPI_Init(&argc, &argv);
    testing::InitGoogleTest(&argc, argv);
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &comm_size);
    char per_rank_log_name[NAME_MAX];
    char per_rank_err_name[NAME_MAX];
    snprintf(per_rank_log_name, NAME_MAX, ".geopm_mpi_test.%.3d.log", rank);
    snprintf(per_rank_err_name, NAME_MAX, ".geopm_mpi_test.%.3d.err", rank);
    int stdout_fileno_dup;
    int stderr_fileno_dup;

    stdout_fileno_dup = dup(STDOUT_FILENO);
    stderr_fileno_dup = dup(STDERR_FILENO);

    freopen(per_rank_log_name, "w", stdout);
    freopen(per_rank_err_name, "w", stderr);
    try {
        err = RUN_ALL_TESTS();
    }
    catch (std::exception ex) {
        err = err ? err : 1;
        std::cerr << "Error: <geopm_mpi_test> [" << rank << "] " << ex.what() << std::endl;
    }

    fflush(stdout);
    dup2(stdout_fileno_dup, STDOUT_FILENO);
    dup2(stderr_fileno_dup, STDERR_FILENO);

    MPI_Barrier(MPI_COMM_WORLD);

    if (!rank) {
        FILE *fid_in = NULL;
        int nread;
        char buffer[NAME_MAX];
        for (int i = 0; i < comm_size; ++i) {
            snprintf(per_rank_log_name, NAME_MAX, ".geopm_mpi_test.%.3d.log", i);
            fid_in = fopen(per_rank_log_name, "r");
            fprintf(stdout, "**********       Log: <geopm_mpi_test> [%.3d]      **********\n", i);
            nread = -1;
            while (fid_in && nread) {
                nread = fread(buffer, 1, NAME_MAX, fid_in);
                fwrite(buffer, 1, nread, stdout);
            }
            fprintf(stdout,     "************************************************************\n");
            fclose(fid_in);
            unlink(per_rank_log_name);

            snprintf(per_rank_err_name, NAME_MAX, ".geopm_mpi_test.%.3d.err", i);
            fid_in = fopen(per_rank_err_name, "r");
            nread = -1;

            bool is_first_print = true;
            bool is_err_empty = true;
            while (fid_in && nread) {
                nread = fread(buffer, 1, NAME_MAX, fid_in);
                if (nread && is_first_print) {
                    fprintf(stdout, "**********      Error: <geopm_mpi_test> [%.3d]     **********\n", i);
                    is_err_empty = false;
                }
                fwrite(buffer, 1, nread, stdout);
                is_first_print = false;
            }
            if (!is_err_empty) {
                fprintf(stdout,     "************************************************************\n");
            }
            fclose(fid_in);
            unlink(per_rank_err_name);
        }
        fflush(stdout);
    }

    int all_err;
    MPI_Allreduce(&err, &all_err, 1, MPI_INT, MPI_LOR, MPI_COMM_WORLD);
    if (all_err) {
        all_err = -255;
    }
    MPI_Finalize();

    if (!err) {
        _exit(err);
    }
    else {
        _exit(all_err);
    }
}
<commit_msg>Allow rank zero to exit normally through gtest, while all other ranks call _exit().<commit_after>/*
 * Copyright (c) 2015, 2016, Intel Corporation
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in
 *       the documentation and/or other materials provided with the
 *       distribution.
 *
 *     * Neither the name of Intel Corporation nor the names of its
 *       contributors may be used to endorse or promote products derived
 *       from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY LOG OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "gtest/gtest.h"
#include "mpi.h"

#ifndef NAME_MAX
#define NAME_MAX 1024
#endif

int main(int argc, char **argv)
{
    int err = 0;
    int rank = 0;
    int comm_size = 0;

    MPI_Init(&argc, &argv);
    testing::InitGoogleTest(&argc, argv);
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &comm_size);
    char per_rank_log_name[NAME_MAX];
    char per_rank_err_name[NAME_MAX];
    snprintf(per_rank_log_name, NAME_MAX, ".geopm_mpi_test.%.3d.log", rank);
    snprintf(per_rank_err_name, NAME_MAX, ".geopm_mpi_test.%.3d.err", rank);
    int stdout_fileno_dup;
    int stderr_fileno_dup;

    stdout_fileno_dup = dup(STDOUT_FILENO);
    stderr_fileno_dup = dup(STDERR_FILENO);

    freopen(per_rank_log_name, "w", stdout);
    freopen(per_rank_err_name, "w", stderr);
    try {
        err = RUN_ALL_TESTS();
    }
    catch (std::exception ex) {
        err = err ? err : 1;
        std::cerr << "Error: <geopm_mpi_test> [" << rank << "] " << ex.what() << std::endl;
    }

    fflush(stdout);
    dup2(stdout_fileno_dup, STDOUT_FILENO);
    dup2(stderr_fileno_dup, STDERR_FILENO);

    MPI_Barrier(MPI_COMM_WORLD);

    if (!rank) {
        FILE *fid_in = NULL;
        int nread;
        char buffer[NAME_MAX];
        for (int i = 0; i < comm_size; ++i) {
            snprintf(per_rank_log_name, NAME_MAX, ".geopm_mpi_test.%.3d.log", i);
            fid_in = fopen(per_rank_log_name, "r");
            fprintf(stdout, "**********       Log: <geopm_mpi_test> [%.3d]      **********\n", i);
            nread = -1;
            while (fid_in && nread) {
                nread = fread(buffer, 1, NAME_MAX, fid_in);
                fwrite(buffer, 1, nread, stdout);
            }
            fprintf(stdout,     "************************************************************\n");
            fclose(fid_in);
            unlink(per_rank_log_name);

            snprintf(per_rank_err_name, NAME_MAX, ".geopm_mpi_test.%.3d.err", i);
            fid_in = fopen(per_rank_err_name, "r");
            nread = -1;

            bool is_first_print = true;
            bool is_err_empty = true;
            while (fid_in && nread) {
                nread = fread(buffer, 1, NAME_MAX, fid_in);
                if (nread && is_first_print) {
                    fprintf(stdout, "**********      Error: <geopm_mpi_test> [%.3d]     **********\n", i);
                    is_err_empty = false;
                }
                fwrite(buffer, 1, nread, stdout);
                is_first_print = false;
            }
            if (!is_err_empty) {
                fprintf(stdout,     "************************************************************\n");
            }
            fclose(fid_in);
            unlink(per_rank_err_name);
        }
        fflush(stdout);
    }

    int all_err;
    MPI_Allreduce(&err, &all_err, 1, MPI_INT, MPI_LOR, MPI_COMM_WORLD);
    if (all_err) {
        all_err = -255;
    }
    MPI_Finalize();

    if (!err) {
        err = all_err;
    }
    if (!rank) {
        _exit(err);
    }
    return err;
}
<|endoftext|>


<commit_before>#include <user_config.h>
#include <SmingCore/SmingCore.h>
#include "mavlink/ardupilotmega/mavlink.h"

// If you want, you can define WiFi settings globally in Eclipse Environment Variables
#ifndef WIFI_SSID
	#define WIFI_SSID "Ares" // Put you SSID and Password here
	#define WIFI_PWD "w1f1p455"
#endif

#define MAX_CLIENTS 5

IPAddress client_list[MAX_CLIENTS];
static uint8_t num_clients=0;

// Will be called when WiFi hardware and software initialization was finished
// And system initialization was completed
void ready()
{
	debugf("READY!");

	// If AP is enabled:
	debugf("AP. ip: %s mac: %s", WifiAccessPoint.getIP().toString().c_str(), WifiAccessPoint.getMAC().c_str());
}

void udp_receive_callback(UdpConnection& connection, char *data, int size,
        IPAddress remoteIP, uint16_t remotePort) {

    //test if this IP is already in client list
    if (num_clients > 0) {
        for (uint8_t i=0; i < num_clients; i++) {
            if (client_list[i] == remoteIP)
                goto existing_client;
        }
    }
    client_list[num_clients] = remoteIP;
    num_clients++;

existing_client:
    static mavlink_message_t udp_in_msg;
    static mavlink_status_t  udp_in_status;
    static uint8_t buffer[512];

    //try to decode incoming message
    for (int i = i; i < size; i++) {
        if (mavlink_parse_char(MAVLINK_COMM_0, (uint8_t)data[i], &udp_in_msg, &udp_in_status)) {
            debugf("UDP > : %d\n", udp_in_msg.msgid);
            //message decoded, send via serial
            uint16_t len = mavlink_msg_to_send_buffer(buffer, &udp_in_msg);
            if (len > 0) {
                //really inefficient but usable
                for (uint16_t j=0; j < len; j++)
                    Serial.write(buffer[j]);
            }
        }
    }

    return;
}

// UDP server
const uint16_t listen_port = 14550;
UdpConnection udp(udp_receive_callback);

//executed frequently, reads available data from UART
void timer_interrupt() {
    static mavlink_message_t msg;
    static mavlink_status_t status;
    static uint8_t buffer[512];

    while (Serial.available() > 0) {
        if (mavlink_parse_char(MAVLINK_COMM_0, (uint8_t)Serial.read(), &msg, &status)) {
            //mavlink packet received from UART
            debugf("UART > : %d\n", msg.msgid);

            uint16_t len = mavlink_msg_to_send_buffer(buffer, &msg);
            if (msg.msgid == 0) {
                //heartbeat messages are broadcast
                udp.sendTo(IPAddress(192, 168, 13, 255), 14550, (const char*)buffer, len);
            } else {
                //forward other packets to connected clients
                for (uint8_t i=0; i < num_clients; i++)
                    udp.sendTo(client_list[i], 14550, (const char*)buffer, len);
            }

        }
    }
}

Timer uart_recv_timer;

void init()
{
	Serial.begin(921600);
	Serial.systemDebugOutput(true); // Allow debug print to serial
	Serial.println("Sming. Let's do smart things!");
	// Set system ready callback method
	System.onReady(ready);

	// Soft access point
	WifiAccessPoint.enable(true);
	WifiAccessPoint.config("ESP8266-MAVLink-bridge", "", AUTH_OPEN);
    WifiStation.enable(false);

	// Optional: Change IP addresses (and disable DHCP)
	WifiAccessPoint.setIP(IPAddress(192, 168, 13, 1));
    uart_recv_timer.initializeMs(10, timer_interrupt).start();

    udp.listen(listen_port);
}
<commit_msg>Adds first working version. Requires ESP IOT-SDK 1.5.0.<commit_after>#include <user_config.h>
#include <SmingCore/SmingCore.h>
#include "mavlink/ardupilotmega/mavlink.h"

// If you want, you can define WiFi settings globally in Eclipse Environment Variables
#ifndef WIFI_SSID
	#define WIFI_SSID "Ares" // Put you SSID and Password here
	#define WIFI_PWD "w1f1p455"
#endif

#define MAX_CLIENTS 5

IPAddress client_list[MAX_CLIENTS];
static uint8_t num_clients=0;

// Will be called when WiFi hardware and software initialization was finished
// And system initialization was completed
void ready()
{
	debugf("READY!");

	// If AP is enabled:
	debugf("AP. ip: %s mac: %s", WifiAccessPoint.getIP().toString().c_str(), WifiAccessPoint.getMAC().c_str());
}

void udp_receive_callback(UdpConnection& connection, char *data, int size,
        IPAddress remoteIP, uint16_t remotePort) {

    //test if this IP is already in client list
    if (num_clients > 0) {
        for (uint8_t i=0; i < num_clients; i++) {
            if (client_list[i] == remoteIP)
                goto existing_client;
        }
    }
    client_list[num_clients] = remoteIP;
    num_clients++;

existing_client:
    static mavlink_message_t udp_in_msg;
    static mavlink_status_t  udp_in_status;
    static uint8_t buffer[512];
    //try to decode incoming message
    for (int i = 0; i < size; i++) {
        if (mavlink_parse_char(MAVLINK_COMM_0, (uint8_t)data[i], &udp_in_msg, &udp_in_status)) {
            debugf("UDP > : %d\n", udp_in_msg.msgid);
            //message decoded, send via serial
            uint16_t len = mavlink_msg_to_send_buffer(buffer, &udp_in_msg);
            if (len > 0) {
                //really inefficient but should do the trick
                for (uint16_t j=0; j < len; j++)
                    Serial.write(buffer[j]);
            }
        }
    }

    return;
}

// UDP server
const uint16_t listen_port = 14550;
UdpConnection udp(udp_receive_callback);

//executed frequently, reads available data from UART
void timer_interrupt() {
    static mavlink_message_t msg;
    static mavlink_status_t status;
    static uint8_t buffer[512];

    while (Serial.available() > 0) {
        if (mavlink_parse_char(MAVLINK_COMM_0, (uint8_t)Serial.read(), &msg, &status)) {
            //mavlink packet received from UART
            debugf("UART > : %d\n", msg.msgid, msg.len);

            uint16_t len = mavlink_msg_to_send_buffer(buffer, &msg);
            if (msg.msgid == 0) {
                //heartbeat messages are also broadcast
                udp.sendTo(IPAddress(192, 168, 13, 255), 14550, (const char*)buffer, len);
            }

            //forward other packets to connected clients
            for (uint8_t i=0; i < num_clients; i++)
                udp.sendTo(client_list[i], 14550, (const char*)buffer, len);

        }
    }
}

Timer uart_recv_timer;

void init()
{
	Serial.begin(921600);
	Serial.systemDebugOutput(false); // Allow debug print to serial
	// Set system ready callback method
	System.onReady(ready);

	// Soft access point
	WifiAccessPoint.enable(true);
	WifiAccessPoint.config("ESP8266-MAVLink-bridge", "", AUTH_OPEN);
    WifiStation.enable(false);

	// Optional: Change IP addresses (and disable DHCP)
	WifiAccessPoint.setIP(IPAddress(192, 168, 13, 1));
    uart_recv_timer.initializeMs(10, timer_interrupt).start();

    udp.listen(listen_port);
}
<|endoftext|>


<commit_before>#include "clunk_ex.h"
#include <errno.h>
#include <stdio.h>
#include <string.h>

void clunk::Exception::add_message(const char *file, int line) {
	char buf[1024];
	snprintf(buf, sizeof(buf), "[%s:%d] ", file, line);
	message += buf;
}

void clunk::Exception::add_message(const std::string &msg) {
	message += msg;
	message += ' ';
}

void clunk::IOException::add_custom_message() {
	char buf[1024];
	memset(buf, 0, sizeof(buf));

#ifdef _WINDOWS
	strncpy(buf, _strerror(NULL), sizeof(buf));
#else
	strncpy(buf, strerror(errno), sizeof(buf));
//	if (strerror_r(errno, buf, sizeof(buf)-1) != 0) perror("strerror");
#endif
	add_message(buf);
}

<commit_msg>defined snprinf for windows<commit_after>#include "clunk_ex.h"
#include <errno.h>
#include <stdio.h>
#include <string.h>

#if defined _WINDOWS
#    ifndef snprintf
#        define snprintf _snprintf
#    endif
#endif

void clunk::Exception::add_message(const char *file, int line) {
	char buf[1024];
	snprintf(buf, sizeof(buf), "[%s:%d] ", file, line);
	message += buf;
}

void clunk::Exception::add_message(const std::string &msg) {
	message += msg;
	message += ' ';
}

void clunk::IOException::add_custom_message() {
	char buf[1024];
	memset(buf, 0, sizeof(buf));

#ifdef _WINDOWS
	strncpy(buf, _strerror(NULL), sizeof(buf));
#else
	strncpy(buf, strerror(errno), sizeof(buf));
//	if (strerror_r(errno, buf, sizeof(buf)-1) != 0) perror("strerror");
#endif
	add_message(buf);
}

<|endoftext|>


<commit_before>/*
 *
 * This file is distributed under the terms in the attached LICENSE file.
 * If you do not find this file, copies can be found by writing to:
 * Intel Research Berkeley, 2150 Shattuck Avenue, Suite 1300,
 * Berkeley, CA, 94704.  Attention:  Intel License Inquiry.
 * Or
 * UC Berkeley EECS Computer Science Division, 387 Soda Hall #1776,
 * Berkeley, CA,  94707. Attention: P2 Group.
 *
 */

#include <iostream>
#include "rdelivery.h"
#include "p2Time.h"
#include "val_uint64.h"
#include "val_uint32.h"
#include "val_double.h"
#include "val_str.h"
#include "val_tuple.h"
#include "netglobals.h"

RDelivery::RDelivery(string n, unsigned m)
  : Element(n, 2, 2),
    _out_cb(0),
    _in_on(true),
    _max_retry(m)
{
}

/**
 * New tuple to send
 */
TuplePtr RDelivery::pull(int port, b_cbv cb)
{
  assert (port == 0);

  TuplePtr tp;
  if (_retry_q.empty() && _in_on &&
      (_in_on = ((tp = input(0)->pull(boost::bind(&RDelivery::input_cb,this))) != NULL))) {
    /* Store the tuple for retry, if failure */
    return tuple(tp);
  }
  else if (!_retry_q.empty()) {
    tp = _retry_q.front();
    _retry_q.pop_front();
    return tp;
  }

  _out_cb = cb;
  return TuplePtr();
}

/**
 * The push method handles input on 2 ports.
 * port 1: Acknowledgement or Failure Tuple
 */
int RDelivery::push(int port, TuplePtr tp, b_cbv cb)
{
  assert(port == 1);

  try {
    for (uint i = 0;
         i < tp->size();
         i++) {
      if ((*tp)[i]->typeCode() == Value::STR &&
          Val_Str::cast((*tp)[i]) == "ACK") {
        // Acknowledge tuple and update measurements.
        ack(tp);
      }
    }
  }
  catch (Value::TypeError e) { }
  return output(1)->push(tp, cb);
}

REMOVABLE_INLINE void RDelivery::handle_failure(ConnectionPtr cp)
{
  cp->_tcb = NULL;

  for (std::deque<RDelivery::Connection::TuplePtr>::iterator iter =
       cp->_outstanding.begin(); iter != cp->_outstanding.end(); ) {
    RDelivery::Connection::TuplePtr ctp = *iter;
    if (ctp->_retry_cnt < _max_retry) {
      ctp->_retry_cnt++;
      _retry_q.push_back(ctp->_tp);
      if (_out_cb) {
        _out_cb();
        _out_cb = 0;
      }
      iter++;
    }
    else {
      std::cerr << "RETRY TOO MANY: DEST = " << cp->_dest->toString()
                << " SEQ = " << ctp->_seq << std::endl;
      if (cp->_cum_seq < ctp->_seq)
        cp->_cum_seq = ctp->_seq;
      assert(iter == cp->_outstanding.begin());
      iter = cp->_outstanding.erase(iter);
    }
  }

  if (cp->_outstanding.size() > 0) {
    cp->_tcb = delayCB((cp->_rtt) / 1000.0,
                       boost::bind(&RDelivery::handle_failure, this, cp), this);
  }
}

RDelivery::ConnectionPtr RDelivery::lookup(ValuePtr dest)
{
  ValueConnectionMap::iterator iter = _index.find(dest);
  if (iter != _index.end()) {
    return iter->second;		// Found it.
  }
  return RDelivery::ConnectionPtr();	// Didn't find it.
}

void RDelivery::map(ValuePtr dest, ConnectionPtr cp)
{
  ValueConnectionMap::iterator iter = _index.find(dest);
  if (iter == _index.end()) {
    _index.insert(std::make_pair(dest,cp));
  }
}

void RDelivery::unmap(ValuePtr dest)
{
  ValueConnectionMap::iterator iter = _index.find(dest);
  if (iter != _index.end()) {
      _index.erase(iter);
  }
}

TuplePtr RDelivery::tuple(TuplePtr tp)
{
  ValuePtr dest = (*tp)[DEST];
  SeqNum   seq  = Val_UInt64::cast((*tp)[SEQ]);
  double   rtt  = Val_Double::cast((*tp)[RTT]);

  RDelivery::ConnectionPtr cp = lookup(dest);
  if (!cp) {
    cp.reset(new Connection(dest, rtt));
    map(dest, cp);
  }

  TuplePtr rtp = Tuple::mk();
  for (unsigned i = 0; i < tp->size(); i++) {
    if (i == CUMSEQ) {
      rtp->append(Val_UInt64::mk(cp->_cum_seq));
    }
    else {
      rtp->append((*tp)[i]);
    }
  }
  rtp->freeze();

  RDelivery::Connection::TuplePtr ctp(new RDelivery::Connection::Tuple(seq, rtp));
  cp->_outstanding.push_back(ctp);
  if (cp->_tcb == NULL) {
    cp->_tcb = delayCB((2 * cp->_rtt) / 1000.0,
                       boost::bind(&RDelivery::handle_failure, this, cp), this);
  }
  return rtp;
}

void RDelivery::ack(TuplePtr tp)
{
  ValuePtr      dest = (*tp)[DEST + 2];
  ConnectionPtr cp   = lookup(dest);
  SeqNum        cseq = Val_UInt64::cast((*tp)[CUMSEQ + 2]);

  if (cp && cp->_cum_seq < cseq) {
    if (cp->_tcb != NULL) {
      timeCBRemove(cp->_tcb);
      cp->_tcb = NULL;
    }
    cp->_cum_seq = cseq;
    while (cp->_outstanding.size() > 0 &&
           cp->_outstanding.front()->_seq <= cp->_cum_seq) {
      cp->_outstanding.pop_front();
    }
    if (cp->_outstanding.size() > 0) {
      cp->_tcb = delayCB((2 * cp->_rtt) / 1000.0,
                         boost::bind(&RDelivery::handle_failure, this, cp), this);
    }
  }
}

void RDelivery::input_cb()
{
  _in_on = true;
  if (_out_cb) {
    _out_cb();
    _out_cb = 0;
  }
}
<commit_msg>*** empty log message ***<commit_after>/*
 *
 * This file is distributed under the terms in the attached LICENSE file.
 * If you do not find this file, copies can be found by writing to:
 * Intel Research Berkeley, 2150 Shattuck Avenue, Suite 1300,
 * Berkeley, CA, 94704.  Attention:  Intel License Inquiry.
 * Or
 * UC Berkeley EECS Computer Science Division, 387 Soda Hall #1776,
 * Berkeley, CA,  94707. Attention: P2 Group.
 *
 */

#include <iostream>
#include "rdelivery.h"
#include "p2Time.h"
#include "val_uint64.h"
#include "val_uint32.h"
#include "val_double.h"
#include "val_str.h"
#include "val_tuple.h"
#include "netglobals.h"

RDelivery::RDelivery(string n, unsigned m)
  : Element(n, 2, 2),
    _out_cb(0),
    _in_on(true),
    _max_retry(m)
{
}

/**
 * New tuple to send
 */
TuplePtr RDelivery::pull(int port, b_cbv cb)
{
  assert (port == 0);
  double flip = rand() / (float)RAND_MAX;

  TuplePtr tp;
  if (_in_on &&
      (_retry_q.empty() ||
       (flip < 0.5 &&
        (_in_on = ((tp = input(0)->pull(boost::bind(&RDelivery::input_cb,this))) != NULL))))) {
    /* Store the tuple for retry, if failure */
    return tuple(tp);
  }
  else if (!_retry_q.empty()) {
    tp = _retry_q.front();
    _retry_q.pop_front();
    return tp;
  }

  _out_cb = cb;
  return TuplePtr();
}

/**
 * The push method handles input on 2 ports.
 * port 1: Acknowledgement or Failure Tuple
 */
int RDelivery::push(int port, TuplePtr tp, b_cbv cb)
{
  assert(port == 1);

  try {
    for (uint i = 0;
         i < tp->size();
         i++) {
      if ((*tp)[i]->typeCode() == Value::STR &&
          Val_Str::cast((*tp)[i]) == "ACK") {
        // Acknowledge tuple and update measurements.
        ack(tp);
      }
    }
  }
  catch (Value::TypeError e) { }
  return output(1)->push(tp, cb);
}

REMOVABLE_INLINE void RDelivery::handle_failure(ConnectionPtr cp)
{
  cp->_tcb = NULL;

  for (std::deque<RDelivery::Connection::TuplePtr>::iterator iter =
       cp->_outstanding.begin(); iter != cp->_outstanding.end(); ) {
    RDelivery::Connection::TuplePtr ctp = *iter;
    if (ctp->_retry_cnt < _max_retry) {
      ctp->_retry_cnt++;
      _retry_q.push_back(ctp->_tp);
      if (_out_cb) {
        _out_cb();
        _out_cb = 0;
      }
      iter++;
    }
    else {
      std::cerr << "RETRY TOO MANY: DEST = " << cp->_dest->toString()
                << " SEQ = " << ctp->_seq << std::endl;
      if (cp->_cum_seq < ctp->_seq)
        cp->_cum_seq = ctp->_seq;
      assert(iter == cp->_outstanding.begin());
      iter = cp->_outstanding.erase(iter);
    }
  }

  if (cp->_outstanding.size() > 0) {
    cp->_tcb = delayCB((cp->_rtt) / 1000.0,
                       boost::bind(&RDelivery::handle_failure, this, cp), this);
  }
}

RDelivery::ConnectionPtr RDelivery::lookup(ValuePtr dest)
{
  ValueConnectionMap::iterator iter = _index.find(dest);
  if (iter != _index.end()) {
    return iter->second;		// Found it.
  }
  return RDelivery::ConnectionPtr();	// Didn't find it.
}

void RDelivery::map(ValuePtr dest, ConnectionPtr cp)
{
  ValueConnectionMap::iterator iter = _index.find(dest);
  if (iter == _index.end()) {
    _index.insert(std::make_pair(dest,cp));
  }
}

void RDelivery::unmap(ValuePtr dest)
{
  ValueConnectionMap::iterator iter = _index.find(dest);
  if (iter != _index.end()) {
      _index.erase(iter);
  }
}

TuplePtr RDelivery::tuple(TuplePtr tp)
{
  ValuePtr dest = (*tp)[DEST];
  SeqNum   seq  = Val_UInt64::cast((*tp)[SEQ]);
  double   rtt  = Val_Double::cast((*tp)[RTT]);

  RDelivery::ConnectionPtr cp = lookup(dest);
  if (!cp) {
    cp.reset(new Connection(dest, rtt));
    map(dest, cp);
  }

  TuplePtr rtp = Tuple::mk();
  for (unsigned i = 0; i < tp->size(); i++) {
    if (i == CUMSEQ) {
      rtp->append(Val_UInt64::mk(cp->_cum_seq));
    }
    else {
      rtp->append((*tp)[i]);
    }
  }
  rtp->freeze();

  RDelivery::Connection::TuplePtr ctp(new RDelivery::Connection::Tuple(seq, rtp));
  cp->_outstanding.push_back(ctp);
  if (cp->_tcb == NULL) {
    cp->_tcb = delayCB((2 * cp->_rtt) / 1000.0,
                       boost::bind(&RDelivery::handle_failure, this, cp), this);
  }
  return rtp;
}

void RDelivery::ack(TuplePtr tp)
{
  ValuePtr      dest = (*tp)[DEST + 2];
  ConnectionPtr cp   = lookup(dest);
  SeqNum        cseq = Val_UInt64::cast((*tp)[CUMSEQ + 2]);

  if (cp && cp->_cum_seq < cseq) {
    if (cp->_tcb != NULL) {
      timeCBRemove(cp->_tcb);
      cp->_tcb = NULL;
    }
    cp->_cum_seq = cseq;
    while (cp->_outstanding.size() > 0 &&
           cp->_outstanding.front()->_seq <= cp->_cum_seq) {
      cp->_outstanding.pop_front();
    }
    if (cp->_outstanding.size() > 0) {
      cp->_tcb = delayCB((2 * cp->_rtt) / 1000.0,
                         boost::bind(&RDelivery::handle_failure, this, cp), this);
    }
  }
}

void RDelivery::input_cb()
{
  _in_on = true;
  if (_out_cb) {
    _out_cb();
    _out_cb = 0;
  }
}
<|endoftext|>


<commit_before>/*
  The ziggurat method for RNOR and REXP
  Combine the code below with the main program in which you want
  normal or exponential variates.   Then use of RNOR in any expression
  will provide a standard normal variate with mean zero, variance 1,
  while use of REXP in any expression will provide an exponential variate
  with density exp(-x),x>0.
  Before using RNOR or REXP in your main, insert a command such as
  zigset(86947731 );
  with your own choice of seed value>0, rather than 86947731.
  (If you do not invoke zigset(...) you will get all zeros for RNOR and REXP.)
  For details of the method, see Marsaglia and Tsang, "The ziggurat method
  for generating random variables", Journ. Statistical Software.

  Original functions converted to a c++ object and static variables removed
  // mavarjok
*/

#ifndef ZIGGURAT_HH
#define ZIGGURAT_HH

#include <cmath>
#include <ctime>
#include <stdint.h>

/** Namespace for Ziggurat N(0,1) random number generator. */
namespace ziggurat {

  class Rnd {

  public:

    Rnd()
    {
      jsr=123456789;
      zigset(clock());
    }

    Rnd(uint32_t s)
    {
      jsr=123456789;
      zigset(s);
    }

    /* Generates a random sample from N(0,1) distribution */
    float rnor()
    {
      hz=shr3();
      iz=hz&127;
      return ((fabs(hz)<kn[iz]) ? hz*wn[iz] : nfix());
    }
    /* Generates a random sample from the exponential distribution */
    float rexp()
    {
      jz=shr3();
      iz=jz&255;
      return ((jz <ke[iz]) ? jz*we[iz] : efix());
    }

  private:

    uint32_t jz,jsr;
    int32_t hz;
    uint32_t iz, kn[128], ke[256];
    float wn[128],fn[128], we[256],fe[256];
    float x, y;
    float r;     /* The start of the right tail */

    /* Generates variates from the residue when rejection in RNOR occurs */
    float nfix(void);

    /* Generates variates from the residue when rejection in REXP occurs */
    float efix(void);

    /* Sets the seed and creates the tables */
    void zigset(uint32_t jsrseed);

    inline uint32_t shr3()
    {
      jz=jsr;
      jsr^=(jsr<<13);
      jsr^=(jsr>>17);
      jsr^=(jsr<<5);
      return jz+jsr;
    }

    inline double uni()
    {
      return .5 + (signed) shr3()*.2328306e-9;
    }

    inline uint32_t iuni()
    {
      return shr3();
    }
  };

  /** Global generator instance for normal use. */
  extern Rnd rnd;

};

#endif /* ZIGGURAT_HH */
<commit_msg>Fixed initialization<commit_after>/*
  The ziggurat method for RNOR and REXP
  Combine the code below with the main program in which you want
  normal or exponential variates.   Then use of RNOR in any expression
  will provide a standard normal variate with mean zero, variance 1,
  while use of REXP in any expression will provide an exponential variate
  with density exp(-x),x>0.
  Before using RNOR or REXP in your main, insert a command such as
  zigset(86947731 );
  with your own choice of seed value>0, rather than 86947731.
  (If you do not invoke zigset(...) you will get all zeros for RNOR and REXP.)
  For details of the method, see Marsaglia and Tsang, "The ziggurat method
  for generating random variables", Journ. Statistical Software.

  Original functions converted to a c++ object and static variables removed
  // mavarjok
*/

#ifndef ZIGGURAT_HH
#define ZIGGURAT_HH

#include <cmath>
#include <ctime>
#include <stdint.h>

/** Namespace for Ziggurat N(0,1) random number generator. */
namespace ziggurat {

  class Rnd {

  public:

    Rnd()
    {
      jsr=123456789;
      zigset(time(NULL));
    }

    Rnd(uint32_t s)
    {
      jsr=123456789;
      zigset(s);
    }

    /* Generates a random sample from N(0,1) distribution */
    float rnor()
    {
      hz=shr3();
      iz=hz&127;
      return ((fabs(hz)<kn[iz]) ? hz*wn[iz] : nfix());
    }
    /* Generates a random sample from the exponential distribution */
    float rexp()
    {
      jz=shr3();
      iz=jz&255;
      return ((jz <ke[iz]) ? jz*we[iz] : efix());
    }

  private:

    uint32_t jz,jsr;
    int32_t hz;
    uint32_t iz, kn[128], ke[256];
    float wn[128],fn[128], we[256],fe[256];
    float x, y;
    float r;     /* The start of the right tail */

    /* Generates variates from the residue when rejection in RNOR occurs */
    float nfix(void);

    /* Generates variates from the residue when rejection in REXP occurs */
    float efix(void);

    /* Sets the seed and creates the tables */
    void zigset(uint32_t jsrseed);

    inline uint32_t shr3()
    {
      jz=jsr;
      jsr^=(jsr<<13);
      jsr^=(jsr>>17);
      jsr^=(jsr<<5);
      return jz+jsr;
    }

    inline double uni()
    {
      return .5 + (signed) shr3()*.2328306e-9;
    }

    inline uint32_t iuni()
    {
      return shr3();
    }
  };

  /** Global generator instance for normal use. */
  extern Rnd rnd;

};

#endif /* ZIGGURAT_HH */
<|endoftext|>