<commit_before>////////////////////////////////////////////////////////////////////////////////
//                                                                            //
// AliFemtoCutMonitorParticleYPtWithWeights - the cut monitor for particles to study    //
// the difference between reconstructed and true momentum                     //
// author: Rafal Maselek rmaselek@cern.ch                                     //
////////////////////////////////////////////////////////////////////////////////
#include "AliFemtoCutMonitorParticleYPtWithWeights.h"
#include "AliFemtoModelHiddenInfo.h"
#include <TH1D.h>
#include <TH2D.h>
#include <TList.h>
#include <TMath.h>

AliFemtoCutMonitorParticleYPtWithWeights::AliFemtoCutMonitorParticleYPtWithWeights(TH2D *filter, int calculateWeights = 1):
  fYPt(0),
  fYPhi(0),
  fPtPhi(0),
  fEtaPhi(0),
  fEtaPt(0),
  fEtaPhiW(0),
  fEtaPtW(0),
  fDCARPt(0),
  fDCAZPt(0),
  fMass(0.13957),
  filterHist(filter),
  fCalcWeights(calculateWeights)
{
  // Default constructor
  fYPt = new TH2D("YPt", "Rapidity vs Pt",              100, 0.0, 6.0, 100, 0.0, 3.0);
  fYPhi = new TH2D("YPhi", "Rapidity vs Phi",           100, 0.0, 6.0, 100, -TMath::Pi(), TMath::Pi());
  fPtPhi = new TH2D("PtPhi", "Pt vs Phi",               100,  0.0, 3.0, 100, -TMath::Pi(), TMath::Pi());
  fEtaPhi = new TH2D("EtaPhi", "Pseudorapidity vs Phi", 100, 0.0, 6.0, 100, -TMath::Pi(), TMath::Pi());
  fEtaPt = new TH2D("EtaPt", "Pseudorapidity vs Pt",    100, 0.0, 6.0, 100, 0.0, 3.0);
  // fEtaPhiW = new TH2D("EtaPhiW", "Pseudorapidity vs Phi chi2/N weighted", 140, -1.4, 1.4, 100, -TMath::Pi(), TMath::Pi());
  // fEtaPtW = new TH2D("EtaPtW", "Pseudorapidity vs Pt chi2/N weighted",    140, -1.4, 1.4, 100, 0.0, 5.0);
  fDCARPt = new TH2D("DCARPt", "DCA in XY vs. Pt", 400, -3.0, 3.0, 100,0.0,3.0);
  fDCAZPt = new TH2D("DCAZPt", "DCA in Z vs. Pt", 400, -3.0, 3.0, 100,0.0,3.0);

}

AliFemtoCutMonitorParticleYPtWithWeights::AliFemtoCutMonitorParticleYPtWithWeights(const char *aName, float aMass, TH2D *filter, int calculateWeights=1):
  fYPt(0),
  fYPhi(0),
  fPtPhi(0),
  fEtaPhi(0),
  fEtaPt(0),
  fEtaPhiW(0),
  fEtaPtW(0),
  fDCARPt(0),
  fDCAZPt(0),
  fMass(aMass),
  filterHist(filter),
  fCalcWeights(calculateWeights)
  {
  // Normal constructor
  char name[200];
  snprintf(name, 200, "YPt%s", aName);
  fYPt = new TH2D(name, "Rapdity vs Pt", 100, 0.0, 6.0, 100, 0.0, 3.0);
  snprintf(name, 200, "YPhi%s", aName);
  fYPhi = new TH2D(name, "Rapidity vs Phi",           100, 0.0, 6.0, 100, -TMath::Pi(), TMath::Pi());
  snprintf(name, 200, "PtPhi%s", aName);
  fPtPhi = new TH2D(name, "Pt vs Phi",               100,  0.0, 3.0, 100, -TMath::Pi(), TMath::Pi());
  snprintf(name, 200, "EtaPhi%s", aName);
  fEtaPhi = new TH2D(name, "Pseudorapidity vs Phi", 100, 0.0, 6.0, 100, -TMath::Pi(), TMath::Pi());
  snprintf(name, 200, "EtaPt%s", aName);
  fEtaPt = new TH2D(name, "Pseudorapidity vs Pt",    100, 0.0, 6.0, 100, 0.0, 3.0);
  // snprintf(name, 200, "EtaPhiW%s", aName);
  // fEtaPhiW = new TH2D(name, "Pseudorapidity vs Phi chi2/N weighted", 140, -1.4, 1.4, 100, -TMath::Pi(), TMath::Pi());
  // snprintf(name, 200, "EtaPtW%s", aName);
  // fEtaPtW = new TH2D(name, "Pseudorapidity vs Pt chi2/N weighted",    140, -1.4, 1.4, 100, 0.0, 2.0);
  snprintf(name, 200, "DCARPt%s", aName);
  fDCARPt = new TH2D(name, "DCA in XY vs. Pt", 400, -3.0, 3.0, 100,0.0,3.0);
  snprintf(name, 200, "DCAZPt%s", aName);
  fDCAZPt = new TH2D(name, "DCA in Z vs. Pt", 400, -3.0, 3.0, 100,0.0,3.0);

}

AliFemtoCutMonitorParticleYPtWithWeights::AliFemtoCutMonitorParticleYPtWithWeights(const AliFemtoCutMonitorParticleYPtWithWeights &aCut):
  fYPt(0),
  fYPhi(0),
  fPtPhi(0),
  fEtaPhi(0),
  fEtaPt(0),
  fEtaPhiW(0),
  fEtaPtW(0),
  fDCARPt(0),
  fDCAZPt(0),
  fMass(0.13957),
  filterHist(0)
{
  // copy constructor
  fYPt = new TH2D(*aCut.fYPt);
  fYPhi = new TH2D(*aCut.fYPhi);
  fPtPhi = new TH2D(*aCut.fPtPhi);
  fEtaPhi = new TH2D(*aCut.fEtaPhi);
  fEtaPt = new TH2D(*aCut.fEtaPt);
  // fEtaPhiW = new TH2D(*aCut.fEtaPhiW);
  // fEtaPtW = new TH2D(*aCut.fEtaPtW);
  fDCARPt = new TH2D(*aCut.fDCARPt);
  fDCAZPt = new TH2D(*aCut.fDCAZPt);
  filterHist = new TH2D(*aCut.filterHist);
  fMass = aCut.fMass;
  fCalcWeights = aCut.fCalcWeights;
}

AliFemtoCutMonitorParticleYPtWithWeights::~AliFemtoCutMonitorParticleYPtWithWeights()
{
  // Destructor
  delete fYPt;
  delete fYPhi;
  delete fPtPhi;
  delete fEtaPhi;
  delete fEtaPt;
  // delete fEtaPhiW;
  // delete fEtaPtW;
  delete fDCARPt;
  delete fDCAZPt;
  if(filterHist) delete filterHist;
}

AliFemtoCutMonitorParticleYPtWithWeights& AliFemtoCutMonitorParticleYPtWithWeights::operator=(const AliFemtoCutMonitorParticleYPtWithWeights& aCut)
{
  // assignment operator
  if (this == &aCut)
    return *this;

  if (fYPt) delete fYPt;
  fYPt = new TH2D(*aCut.fYPt);
  if (fYPhi) delete fYPhi;
  fYPhi = new TH2D(*aCut.fYPhi);
  if (fPtPhi) delete fPtPhi;
  fPtPhi = new TH2D(*aCut.fPtPhi);
  if (fEtaPhi) delete fEtaPhi;
  fEtaPhi = new TH2D(*aCut.fEtaPhi);
  if (fEtaPt) delete fEtaPt;
  fEtaPt = new TH2D(*aCut.fEtaPt);
  // if (fEtaPhiW) delete fEtaPhiW;
  // fEtaPhiW = new TH2D(*aCut.fEtaPhiW);
  // if (fEtaPtW) delete fEtaPtW;
  // fEtaPtW = new TH2D(*aCut.fEtaPtW);
  if (fDCARPt) delete fDCARPt;
  fDCARPt = new TH2D(*aCut.fDCARPt);
  if (fDCAZPt) delete fDCAZPt;
  fDCAZPt = new TH2D(*aCut.fDCAZPt);

  if(filterHist) delete filterHist;
  filterHist = new TH2D(*aCut.filterHist);

  fCalcWeights = aCut.fCalcWeights;

  return *this;
}

AliFemtoString AliFemtoCutMonitorParticleYPtWithWeights::Report(){
  // Prepare report from the execution
  string stemp = "*** AliFemtoCutMonitorParticleYPtWithWeights report";
  AliFemtoString returnThis = stemp;
  return returnThis;
}

void AliFemtoCutMonitorParticleYPtWithWeights::Fill(const AliFemtoTrack* aTrack)
{
  // Fill in the monitor histograms with the values from the current track
  float tEnergy = ::sqrt(aTrack->P().Mag2()+fMass*fMass);
  if(tEnergy==abs(aTrack->P().z())) tEnergy+=0.001;

  float tRapidity = -1000;
  if((tEnergy+aTrack->P().z())/(tEnergy-aTrack->P().z())>0)
    tRapidity = 0.5*::log((tEnergy+aTrack->P().z())/(tEnergy-aTrack->P().z()));
  else
    tRapidity = -999;

  float tPt = -1000;
  if((aTrack->P().x())*(aTrack->P().x())+(aTrack->P().y())*(aTrack->P().y())>=0)
    tPt = ::sqrt((aTrack->P().x())*(aTrack->P().x())+(aTrack->P().y())*(aTrack->P().y()));
  else
    tPt = -999;

  double weight = filterHist->GetBinContent(filterHist->FindBin(tRapidity, tPt));
  if(fCalcWeights == 0)
    weight=1.0;

  float tEta;
  if(aTrack->P().Theta()==0)
    tEta=0;
  else
    tEta = -TMath::Log(TMath::Tan(aTrack->P().Theta()/2.0));
  float tPhi = aTrack->P().Phi();
  // float chi2w;
  float dcar = aTrack->ImpactD();
  float dcaz = aTrack->ImpactZ();
  // if (aTrack->TPCncls() > 0)
  //   chi2w = aTrack->TPCchi2()/aTrack->TPCncls();
  // else
  //   chi2w = 6.0;

  //  cout << " CMYPt: " << fYPt << " " << fYPt->GetEntries() << " " << tRapidity << " " << tPt << endl;

  fYPt->Fill(tRapidity, tPt, weight);
  fYPhi->Fill(tRapidity, tPhi, weight);
  fPtPhi->Fill(tPt, tPhi, weight);
  fEtaPhi->Fill(tEta, tPhi, weight);
  fEtaPt->Fill(tEta, tPt, weight);
  // fEtaPhiW->Fill(tEta, tPhi, chi2w);
  // fEtaPtW->Fill(tEta, tPt, chi2w);
  fDCARPt->Fill(dcar, tPt, weight);
  fDCAZPt->Fill(dcaz, tPt, weight);
}

void AliFemtoCutMonitorParticleYPtWithWeights::Write()
{
  // Write out the relevant histograms
  fYPt->Write();
  fYPhi->Write();
  fPtPhi->Write();
  fEtaPhi->Write();
  fEtaPt->Write();
  // fEtaPhiW->Write();
  // fEtaPtW->Write();
  fDCARPt->Write();
  fDCAZPt->Write();
}

TList *AliFemtoCutMonitorParticleYPtWithWeights::GetOutputList()
{
  TList *tOutputList = new TList();
  tOutputList->Add(fYPt);
  tOutputList->Add(fYPhi);
  tOutputList->Add(fPtPhi);
  tOutputList->Add(fEtaPhi);
  tOutputList->Add(fEtaPt);
  // tOutputList->Add(fEtaPhiW);
  // tOutputList->Add(fEtaPtW);
  tOutputList->Add(fDCARPt);
  tOutputList->Add(fDCAZPt);

  return tOutputList;
}
<commit_msg>AliFemtoCutMonitorParticleYPtWithWeights - Fix initializer parameter order<commit_after>////////////////////////////////////////////////////////////////////////////////
//                                                                            //
// AliFemtoCutMonitorParticleYPtWithWeights - the cut monitor for particles to study    //
// the difference between reconstructed and true momentum                     //
// author: Rafal Maselek rmaselek@cern.ch                                     //
////////////////////////////////////////////////////////////////////////////////
#include "AliFemtoCutMonitorParticleYPtWithWeights.h"
#include "AliFemtoModelHiddenInfo.h"
#include <TH1D.h>
#include <TH2D.h>
#include <TList.h>
#include <TMath.h>

AliFemtoCutMonitorParticleYPtWithWeights::AliFemtoCutMonitorParticleYPtWithWeights(TH2D *filter, int calculateWeights = 1):
  filterHist(filter),
  fCalcWeights(calculateWeights),
  fYPt(0),
  fYPhi(0),
  fPtPhi(0),
  fEtaPhi(0),
  fEtaPt(0),
  fEtaPhiW(0),
  fEtaPtW(0),
  fDCARPt(0),
  fDCAZPt(0),
  fMass(0.13957)
{
  // Default constructor
  fYPt = new TH2D("YPt", "Rapidity vs Pt",              100, 0.0, 6.0, 100, 0.0, 3.0);
  fYPhi = new TH2D("YPhi", "Rapidity vs Phi",           100, 0.0, 6.0, 100, -TMath::Pi(), TMath::Pi());
  fPtPhi = new TH2D("PtPhi", "Pt vs Phi",               100,  0.0, 3.0, 100, -TMath::Pi(), TMath::Pi());
  fEtaPhi = new TH2D("EtaPhi", "Pseudorapidity vs Phi", 100, 0.0, 6.0, 100, -TMath::Pi(), TMath::Pi());
  fEtaPt = new TH2D("EtaPt", "Pseudorapidity vs Pt",    100, 0.0, 6.0, 100, 0.0, 3.0);
  // fEtaPhiW = new TH2D("EtaPhiW", "Pseudorapidity vs Phi chi2/N weighted", 140, -1.4, 1.4, 100, -TMath::Pi(), TMath::Pi());
  // fEtaPtW = new TH2D("EtaPtW", "Pseudorapidity vs Pt chi2/N weighted",    140, -1.4, 1.4, 100, 0.0, 5.0);
  fDCARPt = new TH2D("DCARPt", "DCA in XY vs. Pt", 400, -3.0, 3.0, 100,0.0,3.0);
  fDCAZPt = new TH2D("DCAZPt", "DCA in Z vs. Pt", 400, -3.0, 3.0, 100,0.0,3.0);

}

AliFemtoCutMonitorParticleYPtWithWeights::AliFemtoCutMonitorParticleYPtWithWeights(const char *aName, float aMass, TH2D *filter, int calculateWeights=1):
  filterHist(filter),
  fCalcWeights(calculateWeights),
  fYPt(0),
  fYPhi(0),
  fPtPhi(0),
  fEtaPhi(0),
  fEtaPt(0),
  fEtaPhiW(0),
  fEtaPtW(0),
  fDCARPt(0),
  fDCAZPt(0),
  fMass(aMass)
{
  // Normal constructor
  char name[200];
  snprintf(name, 200, "YPt%s", aName);
  fYPt = new TH2D(name, "Rapdity vs Pt", 100, 0.0, 6.0, 100, 0.0, 3.0);
  snprintf(name, 200, "YPhi%s", aName);
  fYPhi = new TH2D(name, "Rapidity vs Phi",           100, 0.0, 6.0, 100, -TMath::Pi(), TMath::Pi());
  snprintf(name, 200, "PtPhi%s", aName);
  fPtPhi = new TH2D(name, "Pt vs Phi",               100,  0.0, 3.0, 100, -TMath::Pi(), TMath::Pi());
  snprintf(name, 200, "EtaPhi%s", aName);
  fEtaPhi = new TH2D(name, "Pseudorapidity vs Phi", 100, 0.0, 6.0, 100, -TMath::Pi(), TMath::Pi());
  snprintf(name, 200, "EtaPt%s", aName);
  fEtaPt = new TH2D(name, "Pseudorapidity vs Pt",    100, 0.0, 6.0, 100, 0.0, 3.0);
  // snprintf(name, 200, "EtaPhiW%s", aName);
  // fEtaPhiW = new TH2D(name, "Pseudorapidity vs Phi chi2/N weighted", 140, -1.4, 1.4, 100, -TMath::Pi(), TMath::Pi());
  // snprintf(name, 200, "EtaPtW%s", aName);
  // fEtaPtW = new TH2D(name, "Pseudorapidity vs Pt chi2/N weighted",    140, -1.4, 1.4, 100, 0.0, 2.0);
  snprintf(name, 200, "DCARPt%s", aName);
  fDCARPt = new TH2D(name, "DCA in XY vs. Pt", 400, -3.0, 3.0, 100,0.0,3.0);
  snprintf(name, 200, "DCAZPt%s", aName);
  fDCAZPt = new TH2D(name, "DCA in Z vs. Pt", 400, -3.0, 3.0, 100,0.0,3.0);
}

AliFemtoCutMonitorParticleYPtWithWeights::AliFemtoCutMonitorParticleYPtWithWeights(const AliFemtoCutMonitorParticleYPtWithWeights &aCut):
  filterHist(0),
  fCalcWeights(aCut.fCalcWeights),
  fYPt(0),
  fYPhi(0),
  fPtPhi(0),
  fEtaPhi(0),
  fEtaPt(0),
  fEtaPhiW(0),
  fEtaPtW(0),
  fDCARPt(0),
  fDCAZPt(0),
  fMass(aCut.fMass)
{
  // copy constructor
  fYPt = new TH2D(*aCut.fYPt);
  fYPhi = new TH2D(*aCut.fYPhi);
  fPtPhi = new TH2D(*aCut.fPtPhi);
  fEtaPhi = new TH2D(*aCut.fEtaPhi);
  fEtaPt = new TH2D(*aCut.fEtaPt);
  // fEtaPhiW = new TH2D(*aCut.fEtaPhiW);
  // fEtaPtW = new TH2D(*aCut.fEtaPtW);
  fDCARPt = new TH2D(*aCut.fDCARPt);
  fDCAZPt = new TH2D(*aCut.fDCAZPt);
  filterHist = new TH2D(*aCut.filterHist);
}

AliFemtoCutMonitorParticleYPtWithWeights::~AliFemtoCutMonitorParticleYPtWithWeights()
{
  // Destructor
  delete fYPt;
  delete fYPhi;
  delete fPtPhi;
  delete fEtaPhi;
  delete fEtaPt;
  // delete fEtaPhiW;
  // delete fEtaPtW;
  delete fDCARPt;
  delete fDCAZPt;
  if(filterHist) delete filterHist;
}

AliFemtoCutMonitorParticleYPtWithWeights& AliFemtoCutMonitorParticleYPtWithWeights::operator=(const AliFemtoCutMonitorParticleYPtWithWeights& aCut)
{
  // assignment operator
  if (this == &aCut)
    return *this;

  if (fYPt) delete fYPt;
  fYPt = new TH2D(*aCut.fYPt);
  if (fYPhi) delete fYPhi;
  fYPhi = new TH2D(*aCut.fYPhi);
  if (fPtPhi) delete fPtPhi;
  fPtPhi = new TH2D(*aCut.fPtPhi);
  if (fEtaPhi) delete fEtaPhi;
  fEtaPhi = new TH2D(*aCut.fEtaPhi);
  if (fEtaPt) delete fEtaPt;
  fEtaPt = new TH2D(*aCut.fEtaPt);
  // if (fEtaPhiW) delete fEtaPhiW;
  // fEtaPhiW = new TH2D(*aCut.fEtaPhiW);
  // if (fEtaPtW) delete fEtaPtW;
  // fEtaPtW = new TH2D(*aCut.fEtaPtW);
  if (fDCARPt) delete fDCARPt;
  fDCARPt = new TH2D(*aCut.fDCARPt);
  if (fDCAZPt) delete fDCAZPt;
  fDCAZPt = new TH2D(*aCut.fDCAZPt);

  if(filterHist) delete filterHist;
  filterHist = new TH2D(*aCut.filterHist);

  fCalcWeights = aCut.fCalcWeights;

  return *this;
}

AliFemtoString AliFemtoCutMonitorParticleYPtWithWeights::Report(){
  // Prepare report from the execution
  string stemp = "*** AliFemtoCutMonitorParticleYPtWithWeights report";
  AliFemtoString returnThis = stemp;
  return returnThis;
}

void AliFemtoCutMonitorParticleYPtWithWeights::Fill(const AliFemtoTrack* aTrack)
{
  // Fill in the monitor histograms with the values from the current track
  float tEnergy = ::sqrt(aTrack->P().Mag2()+fMass*fMass);
  if(tEnergy==abs(aTrack->P().z())) tEnergy+=0.001;

  float tRapidity = -1000;
  if((tEnergy+aTrack->P().z())/(tEnergy-aTrack->P().z())>0)
    tRapidity = 0.5*::log((tEnergy+aTrack->P().z())/(tEnergy-aTrack->P().z()));
  else
    tRapidity = -999;

  float tPt = -1000;
  if((aTrack->P().x())*(aTrack->P().x())+(aTrack->P().y())*(aTrack->P().y())>=0)
    tPt = ::sqrt((aTrack->P().x())*(aTrack->P().x())+(aTrack->P().y())*(aTrack->P().y()));
  else
    tPt = -999;

  double weight = filterHist->GetBinContent(filterHist->FindBin(tRapidity, tPt));
  if(fCalcWeights == 0)
    weight=1.0;

  float tEta;
  if(aTrack->P().Theta()==0)
    tEta=0;
  else
    tEta = -TMath::Log(TMath::Tan(aTrack->P().Theta()/2.0));
  float tPhi = aTrack->P().Phi();
  // float chi2w;
  float dcar = aTrack->ImpactD();
  float dcaz = aTrack->ImpactZ();
  // if (aTrack->TPCncls() > 0)
  //   chi2w = aTrack->TPCchi2()/aTrack->TPCncls();
  // else
  //   chi2w = 6.0;

  //  cout << " CMYPt: " << fYPt << " " << fYPt->GetEntries() << " " << tRapidity << " " << tPt << endl;

  fYPt->Fill(tRapidity, tPt, weight);
  fYPhi->Fill(tRapidity, tPhi, weight);
  fPtPhi->Fill(tPt, tPhi, weight);
  fEtaPhi->Fill(tEta, tPhi, weight);
  fEtaPt->Fill(tEta, tPt, weight);
  // fEtaPhiW->Fill(tEta, tPhi, chi2w);
  // fEtaPtW->Fill(tEta, tPt, chi2w);
  fDCARPt->Fill(dcar, tPt, weight);
  fDCAZPt->Fill(dcaz, tPt, weight);
}

void AliFemtoCutMonitorParticleYPtWithWeights::Write()
{
  // Write out the relevant histograms
  fYPt->Write();
  fYPhi->Write();
  fPtPhi->Write();
  fEtaPhi->Write();
  fEtaPt->Write();
  // fEtaPhiW->Write();
  // fEtaPtW->Write();
  fDCARPt->Write();
  fDCAZPt->Write();
}

TList *AliFemtoCutMonitorParticleYPtWithWeights::GetOutputList()
{
  TList *tOutputList = new TList();
  tOutputList->Add(fYPt);
  tOutputList->Add(fYPhi);
  tOutputList->Add(fPtPhi);
  tOutputList->Add(fEtaPhi);
  tOutputList->Add(fEtaPt);
  // tOutputList->Add(fEtaPhiW);
  // tOutputList->Add(fEtaPtW);
  tOutputList->Add(fDCARPt);
  tOutputList->Add(fDCAZPt);

  return tOutputList;
}
<|endoftext|>


<commit_before>
#include "allocore/ui/al_PresetMapper.hpp"

using namespace al;

void PresetMapper::readEntries(std::string path)
{
	Dir presetDir(path);
	while (presetDir.read()) {
		FileInfo entry = presetDir.entry();
		if (entry.type() == FileInfo::DIR) {
			readEntries(entry.name());
		} else if (entry.type() == FileInfo::REG) {
			if (entry.name().size() > 4 && entry.name().substr(entry.name().size() - 4) == ".txt") {
				std::cout << "Found map: " << entry.name();
			}
		}
	}
}

PresetMapper::~PresetMapper() { }

PresetMapper &PresetMapper::registerPresetHandler(PresetHandler &handler) {
	mPresetHandler = &handler;
	if (mFindAutomatically) {
		findPresetMaps();
	}
	return *this;
}

bool PresetMapper::archive(std::string mapName, bool overwrite)
{
	bool ok = true;
	std::string fullPath = mPresetHandler->buildMapPath(mapName) + "_archive";
	if (overwrite) {
		if(File::isDirectory(fullPath)) {
			if (!Dir::removeRecursively(fullPath)) {
				std::cout << "Error removing directory: " << fullPath << " aborting preset map archiving." << std::endl;
				return false;
			}
		} else {
			if (File::remove(fullPath) != 0) {
				std::cout << "Error removing file: " << fullPath << " aborting preset map archiving." << std::endl;
				return false;
			}
		}
		if (!Dir::make(fullPath)) {
			std::cout << "Error creating preset map archive directory " << fullPath << std::endl;
			return false;
		}
	} else {
		int counter = 0;
		while (File::isDirectory(fullPath)) {
			std::string newName = mapName + "_" + std::to_string(counter++);
			fullPath = mPresetHandler->buildMapPath(newName) + "_archive";
			if (counter == 0) { // We've wrapped and run out of names...
				std::cout << "Out of names for preset map archive." << std::endl;
				return false;
			}
		}
		if (!Dir::make(fullPath)) {
			std::cout << "Error creating preset map archive directory " << fullPath << std::endl;
			return false;
		}
	}
	std::map<int, std::string> presetMap = mPresetHandler->readPresetMap(mapName);
	for(auto const &preset : presetMap) {
		std::string presetFilename = mPresetHandler->getCurrentPath() + preset.second + ".preset";
		if (!File::copy(presetFilename, fullPath)) {
			std::cout << "Error copying preset " << presetFilename << " when archiving preset map." << std::endl;
			ok = false;
		}
	}
	if (!File::copy(mPresetHandler->buildMapPath(mapName), fullPath + "default.presetMap")) {
		std::cout << "Error copying map: " << mapName << " when archiving preset map." << std::endl;
		ok = false;
	}

	return ok;
}

bool PresetMapper::load(std::string archiveName)
{
	if (! (archiveName.size() > 8 && archiveName.substr(archiveName.size() - 8) == "_archive") ) {
		archiveName +="_ archive";
	}
	std::string mapPath = mPresetHandler->getRootPath() + archiveName;
	if (File::isDirectory(mapPath)) {
		mPresetHandler->setSubDirectory(archiveName);
		mPresetHandler->setCurrentPresetMap("default");
	} else {
		std::cout << "Error loading archive: " << archiveName << std::endl;
		return false;
	}
	return true;
}

std::vector<std::string> PresetMapper::listAvailableMaps(bool showArchives)
{
	std::vector<std::string> mapList;
	Dir mapperDir(mPresetHandler->getCurrentPath());

	// TODO it makes more sense to sort entries alphabetically
	while(mapperDir.read()) {
		const FileInfo &info = mapperDir.entry();
		if (info.type() == FileInfo::DIR && showArchives) {
			std::string name = info.name();
			if ( (name.size() > 18 && name.substr(name.size() - 18) == ".presetMap_archive")) {
				mapList.push_back(name);
			}
		} else if (info.type() == FileInfo::REG && !showArchives) {
			std::string name = info.name();
			if ( (name.size() > 4 && name.substr(name.size() - 4) == ".txt")
			     || (name.size() > 10 && name.substr(name.size() - 10) == ".presetMap")) {
				mapList.push_back(name);
			}
		}
	}
	return mapList;
}

bool PresetMapper::consumeMessage(osc::Message &m, std::string rootOSCPath)
{
	if(m.addressPattern() == rootOSCPath + "/map" && m.typeTags() == "s"){
		std::string val;
		m >> val;
		std::cout << "OSC: Recall preset map: " << val << std::endl;
		this->restore(val);
		return true;
	}
	return false;
}

bool PresetMapper::restore(std::string mapName, bool overwrite, bool autoCreate)
{
	bool ok = true;
	if (mapName.size() > 18 && mapName.substr(mapName.size() - 18) == ".presetMap_archive") { // restore from archive
		std::cout << "Restoring archive " << mapName << std::endl;
		Dir mapDirectory(mPresetHandler->getCurrentPath() + mapName);
		std::string mapNewName = mapName.substr(0, mapName.size() - sizeof("_archive") + 1);
		while (mapDirectory.read()) {
			const FileInfo &entry = mapDirectory.entry();
			if (entry.type() == FileInfo::REG) {
				if (entry.name().substr(entry.name().size() - 7) == ".preset") {
					if (overwrite && File::exists(mPresetHandler->getCurrentPath() + "/" + entry.name())) {
						File::remove(mPresetHandler->getCurrentPath() + entry.name());
					}
					if (!File::copy(mPresetHandler->getCurrentPath() + mapName + "/" + entry.name(),
					                mPresetHandler->getCurrentPath() + entry.name())) {
						std::cout << "Error restoring preset " << entry.name() << " for " << mapName << std::endl;
						ok = false;
					}
				} else if (entry.name() == "default.presetMap") {

					if (overwrite && File::exists(mPresetHandler->getCurrentPath() + "/" + mapNewName)) {
						File::remove(mPresetHandler->getCurrentPath() + "/" + entry.name());
					}

					if (!File::copy(mPresetHandler->getCurrentPath() + "/" + mapName + "/default.presetMap",
					                mPresetHandler->getCurrentPath() + "/" + mapNewName)) {
						std::cout << "Error restoring preset map " << mapNewName << " for " << mapName << std::endl;
						ok = false;
					}
				} else {
					std::cout << "PresetMapper::restore() invalid file: " << entry.name() << std::endl;
				}
			}
		}
		mPresetHandler->setCurrentPresetMap(mapNewName, autoCreate);
	} else { // set preset map directly
		mPresetHandler->setCurrentPresetMap(mapName, autoCreate);
	}
	return ok;
}

void PresetMapper::findPresetMaps() {
	std::string presetsPath = mPresetHandler->getCurrentPath();

	Dir presetDir(presetsPath);
	while (presetDir.read()) {
		FileInfo entry = presetDir.entry();
		if (entry.name().size() > 4 && entry.name().substr(entry.name().size() - 4) == ".txt") {
			std::cout << "Found map: " << entry.name() << std::endl;
		}
	}
}
<commit_msg>Load preset map instead of restore in OSC server<commit_after>
#include "allocore/ui/al_PresetMapper.hpp"

using namespace al;

void PresetMapper::readEntries(std::string path)
{
	Dir presetDir(path);
	while (presetDir.read()) {
		FileInfo entry = presetDir.entry();
		if (entry.type() == FileInfo::DIR) {
			readEntries(entry.name());
		} else if (entry.type() == FileInfo::REG) {
			if (entry.name().size() > 4 && entry.name().substr(entry.name().size() - 4) == ".txt") {
				std::cout << "Found map: " << entry.name();
			}
		}
	}
}

PresetMapper::~PresetMapper() { }

PresetMapper &PresetMapper::registerPresetHandler(PresetHandler &handler) {
	mPresetHandler = &handler;
	if (mFindAutomatically) {
		findPresetMaps();
	}
	return *this;
}

bool PresetMapper::archive(std::string mapName, bool overwrite)
{
	bool ok = true;
	std::string fullPath = mPresetHandler->buildMapPath(mapName) + "_archive";
	if (overwrite) {
		if(File::isDirectory(fullPath)) {
			if (!Dir::removeRecursively(fullPath)) {
				std::cout << "Error removing directory: " << fullPath << " aborting preset map archiving." << std::endl;
				return false;
			}
		} else {
			if (File::remove(fullPath) != 0) {
				std::cout << "Error removing file: " << fullPath << " aborting preset map archiving." << std::endl;
				return false;
			}
		}
		if (!Dir::make(fullPath)) {
			std::cout << "Error creating preset map archive directory " << fullPath << std::endl;
			return false;
		}
	} else {
		int counter = 0;
		while (File::isDirectory(fullPath)) {
			std::string newName = mapName + "_" + std::to_string(counter++);
			fullPath = mPresetHandler->buildMapPath(newName) + "_archive";
			if (counter == 0) { // We've wrapped and run out of names...
				std::cout << "Out of names for preset map archive." << std::endl;
				return false;
			}
		}
		if (!Dir::make(fullPath)) {
			std::cout << "Error creating preset map archive directory " << fullPath << std::endl;
			return false;
		}
	}
	std::map<int, std::string> presetMap = mPresetHandler->readPresetMap(mapName);
	for(auto const &preset : presetMap) {
		std::string presetFilename = mPresetHandler->getCurrentPath() + preset.second + ".preset";
		if (!File::copy(presetFilename, fullPath)) {
			std::cout << "Error copying preset " << presetFilename << " when archiving preset map." << std::endl;
			ok = false;
		}
	}
	if (!File::copy(mPresetHandler->buildMapPath(mapName), fullPath + "default.presetMap")) {
		std::cout << "Error copying map: " << mapName << " when archiving preset map." << std::endl;
		ok = false;
	}

	return ok;
}

bool PresetMapper::load(std::string archiveName)
{
	if (! (archiveName.size() > 8 && archiveName.substr(archiveName.size() - 8) == "_archive") ) {
		archiveName +="_ archive";
	}
	std::string mapPath = mPresetHandler->getRootPath() + archiveName;
	if (File::isDirectory(mapPath)) {
		mPresetHandler->setSubDirectory(archiveName);
		mPresetHandler->setCurrentPresetMap("default");
	} else {
		std::cout << "Error loading archive: " << archiveName << std::endl;
		return false;
	}
	return true;
}

std::vector<std::string> PresetMapper::listAvailableMaps(bool showArchives)
{
	std::vector<std::string> mapList;
	Dir mapperDir(mPresetHandler->getCurrentPath());

	// TODO it makes more sense to sort entries alphabetically
	while(mapperDir.read()) {
		const FileInfo &info = mapperDir.entry();
		if (info.type() == FileInfo::DIR && showArchives) {
			std::string name = info.name();
			if ( (name.size() > 18 && name.substr(name.size() - 18) == ".presetMap_archive")) {
				mapList.push_back(name);
			}
		} else if (info.type() == FileInfo::REG && !showArchives) {
			std::string name = info.name();
			if ( (name.size() > 4 && name.substr(name.size() - 4) == ".txt")
			     || (name.size() > 10 && name.substr(name.size() - 10) == ".presetMap")) {
				mapList.push_back(name);
			}
		}
	}
	return mapList;
}

bool PresetMapper::consumeMessage(osc::Message &m, std::string rootOSCPath)
{
	if(m.addressPattern() == rootOSCPath + "/map" && m.typeTags() == "s"){
		std::string val;
		m >> val;
		std::cout << "OSC: Recall preset map: " << val << std::endl;
		this->load(val);
		return true;
	}
	return false;
}

bool PresetMapper::restore(std::string mapName, bool overwrite, bool autoCreate)
{
	bool ok = true;
	if (mapName.size() > 18 && mapName.substr(mapName.size() - 18) == ".presetMap_archive") { // restore from archive
		std::cout << "Restoring archive " << mapName << std::endl;
		Dir mapDirectory(mPresetHandler->getCurrentPath() + mapName);
		std::string mapNewName = mapName.substr(0, mapName.size() - sizeof("_archive") + 1);
		while (mapDirectory.read()) {
			const FileInfo &entry = mapDirectory.entry();
			if (entry.type() == FileInfo::REG) {
				if (entry.name().substr(entry.name().size() - 7) == ".preset") {
					if (overwrite && File::exists(mPresetHandler->getCurrentPath() + "/" + entry.name())) {
						File::remove(mPresetHandler->getCurrentPath() + entry.name());
					}
					if (!File::copy(mPresetHandler->getCurrentPath() + mapName + "/" + entry.name(),
					                mPresetHandler->getCurrentPath() + entry.name())) {
						std::cout << "Error restoring preset " << entry.name() << " for " << mapName << std::endl;
						ok = false;
					}
				} else if (entry.name() == "default.presetMap") {

					if (overwrite && File::exists(mPresetHandler->getCurrentPath() + "/" + mapNewName)) {
						File::remove(mPresetHandler->getCurrentPath() + "/" + entry.name());
					}

					if (!File::copy(mPresetHandler->getCurrentPath() + "/" + mapName + "/default.presetMap",
					                mPresetHandler->getCurrentPath() + "/" + mapNewName)) {
						std::cout << "Error restoring preset map " << mapNewName << " for " << mapName << std::endl;
						ok = false;
					}
				} else {
					std::cout << "PresetMapper::restore() invalid file: " << entry.name() << std::endl;
				}
			}
		}
		mPresetHandler->setCurrentPresetMap(mapNewName, autoCreate);
	} else { // set preset map directly
		mPresetHandler->setCurrentPresetMap(mapName, autoCreate);
	}
	return ok;
}

void PresetMapper::findPresetMaps() {
	std::string presetsPath = mPresetHandler->getCurrentPath();

	Dir presetDir(presetsPath);
	while (presetDir.read()) {
		FileInfo entry = presetDir.entry();
		if (entry.name().size() > 4 && entry.name().substr(entry.name().size() - 4) == ".txt") {
			std::cout << "Found map: " << entry.name() << std::endl;
		}
	}
}
<|endoftext|>


<commit_before><commit_msg>imgui.cc: ocd fix of NULL -> nullptr<commit_after><|endoftext|>


<commit_before><commit_msg>Fixed usage message.<commit_after><|endoftext|>


<commit_before><commit_msg>GoldDigger Random Magic 0.7<commit_after><|endoftext|>


<commit_before>// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include "foreachfeature.h"
#include "valuefeature.h"
#include "utils.h"

#include <vespa/searchlib/fef/properties.h>
#include <vespa/vespalib/util/stringfmt.h>
#include <vespa/vespalib/util/stash.h>
#include <boost/algorithm/string/replace.hpp>

#include <vespa/log/log.h>
LOG_SETUP(".features.foreachfeature");

using namespace search::fef;

namespace search::features {

template <typename CO, typename OP>
ForeachExecutor<CO, OP>::ForeachExecutor(const CO & condition, uint32_t numInputs) :
    FeatureExecutor(),
    _condition(condition),
    _operation(),
    _numInputs(numInputs)
{
}

template <typename CO, typename OP>
void
ForeachExecutor<CO, OP>::execute(uint32_t)
{
    _operation.reset();
    for (uint32_t i = 0; i < inputs().size(); ++i) {
        feature_t val = inputs().get_number(i);
        if (_condition.useValue(val)) {
            _operation.onValue(val);
        }
    }
    outputs().set_number(0, _operation.getResult());
}

bool
ForeachBlueprint::decideDimension(const vespalib::string & param)
{
    if (param == "terms") {
        _dimension = TERMS;
    } else if (param == "fields") {
        _dimension = FIELDS;
    } else if (param == "attributes") {
        _dimension = ATTRIBUTES;
    } else {
        LOG(error, "Expected dimension parameter to be 'terms', 'fields', or 'attributes', but was '%s'",
            param.c_str());
        return false;
    }
    return true;
}

bool
ForeachBlueprint::decideCondition(const vespalib::string & condition, const vespalib::string & operation)
{
    if (condition == "true") {
        return decideOperation(TrueCondition(), operation);
    } else if (condition.size() >= 2 && condition[0] == '<') {
        return decideOperation(LessThanCondition(util::strToNum<feature_t>(condition.substr(1))), operation);
    } else if (condition.size() >= 2 && condition[0] == '>') {
        return decideOperation(GreaterThanCondition(util::strToNum<feature_t>(condition.substr(1))), operation);
    } else {
        LOG(error, "Expected condition parameter to be 'true', '<a', or '>a', but was '%s'",
            condition.c_str());
        return false;
    }
}

template <typename CO>
bool
ForeachBlueprint::decideOperation(CO condition, const vespalib::string & operation)
{
    if (operation == "sum") {
        setExecutorCreator<CO, SumOperation>(condition);
    } else if (operation == "product") {
        setExecutorCreator<CO, ProductOperation>(condition);
    } else if (operation == "average") {
        setExecutorCreator<CO, AverageOperation>(condition);
    } else if (operation == "max") {
        setExecutorCreator<CO, MaxOperation>(condition);
    } else if (operation == "min") {
        setExecutorCreator<CO, MinOperation>(condition);
    } else if (operation == "count") {
        setExecutorCreator<CO, CountOperation>(condition);
    } else {
        LOG(error, "Expected operation parameter to be 'sum', 'product', 'average', 'max', 'min', or 'count', but was '%s'",
            operation.c_str());
        return false;
    }
    return true;
}

template <typename CO, typename OP>
void
ForeachBlueprint::setExecutorCreator(CO condition)
{
    class ExecutorCreator : public ExecutorCreatorBase {
    private:
        CO _condition;
    public:
        ExecutorCreator(CO cond) : _condition(cond) {}
        search::fef::FeatureExecutor &create(uint32_t numInputs, vespalib::Stash &stash) const override {
            return stash.create<ForeachExecutor<CO, OP>>(_condition,  numInputs);
        }
    };
    _executorCreator.reset(new ExecutorCreator(condition));
}

ForeachBlueprint::ForeachBlueprint() :
    Blueprint("foreach"),
    _dimension(ILLEGAL),
    _executorCreator(),
    _num_inputs(0)
{
}

ForeachBlueprint::~ForeachBlueprint() = default;

void
ForeachBlueprint::visitDumpFeatures(const IIndexEnvironment &,
                                    IDumpFeatureVisitor &) const
{
}

bool
ForeachBlueprint::setup(const IIndexEnvironment & env,
                        const ParameterList & params)
{
    if (!decideDimension(params[0].getValue())) {
        return false;
    }
    if (!decideCondition(params[3].getValue(), params[4].getValue())) {
        return false;
    }

    const vespalib::string & variable = params[1].getValue();
    const vespalib::string & feature = params[2].getValue();

    if (_dimension == TERMS) {
        uint32_t maxTerms = util::strToNum<uint32_t>(env.getProperties().lookup(getBaseName(), "maxTerms").get("16"));
        for (uint32_t i = 0; i < maxTerms; ++i) {
            defineInput(boost::algorithm::replace_all_copy(feature, variable, vespalib::make_string("%u", i)));
            ++_num_inputs;
        }
    } else {
        for (uint32_t i = 0; i < env.getNumFields(); ++i) {
            const FieldInfo * info = env.getField(i);
            if (info->type() == FieldType::INDEX && _dimension == FIELDS) {
                defineInput(boost::algorithm::replace_all_copy(feature, variable, info->name()));
                ++_num_inputs;
            } else if (info->type() == FieldType::ATTRIBUTE && _dimension == ATTRIBUTES) {
                defineInput(boost::algorithm::replace_all_copy(feature, variable, info->name()));
                ++_num_inputs;
            }
        }
    }

    describeOutput("value", "The result after iterating over the input feature values using the specified operation");

    return true;
}

Blueprint::UP
ForeachBlueprint::createInstance() const
{
    return std::make_unique<ForeachBlueprint>();
}

FeatureExecutor &
ForeachBlueprint::createExecutor(const IQueryEnvironment &, vespalib::Stash &stash) const
{
    if (_executorCreator) {
        return _executorCreator->create(_num_inputs, stash);
    }
    return stash.create<SingleZeroValueExecutor>();
}

}
<commit_msg>use a more specific string replacement algorithm<commit_after>// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include "foreachfeature.h"
#include "valuefeature.h"
#include "utils.h"

#include <vespa/searchlib/fef/properties.h>
#include <vespa/vespalib/util/stringfmt.h>
#include <vespa/vespalib/util/stash.h>
#include <vespa/vespalib/stllike/replace_variable.h>

#include <vespa/log/log.h>
LOG_SETUP(".features.foreachfeature");

using namespace search::fef;

namespace search::features {

template <typename CO, typename OP>
ForeachExecutor<CO, OP>::ForeachExecutor(const CO & condition, uint32_t numInputs) :
    FeatureExecutor(),
    _condition(condition),
    _operation(),
    _numInputs(numInputs)
{
}

template <typename CO, typename OP>
void
ForeachExecutor<CO, OP>::execute(uint32_t)
{
    _operation.reset();
    for (uint32_t i = 0; i < inputs().size(); ++i) {
        feature_t val = inputs().get_number(i);
        if (_condition.useValue(val)) {
            _operation.onValue(val);
        }
    }
    outputs().set_number(0, _operation.getResult());
}

bool
ForeachBlueprint::decideDimension(const vespalib::string & param)
{
    if (param == "terms") {
        _dimension = TERMS;
    } else if (param == "fields") {
        _dimension = FIELDS;
    } else if (param == "attributes") {
        _dimension = ATTRIBUTES;
    } else {
        LOG(error, "Expected dimension parameter to be 'terms', 'fields', or 'attributes', but was '%s'",
            param.c_str());
        return false;
    }
    return true;
}

bool
ForeachBlueprint::decideCondition(const vespalib::string & condition, const vespalib::string & operation)
{
    if (condition == "true") {
        return decideOperation(TrueCondition(), operation);
    } else if (condition.size() >= 2 && condition[0] == '<') {
        return decideOperation(LessThanCondition(util::strToNum<feature_t>(condition.substr(1))), operation);
    } else if (condition.size() >= 2 && condition[0] == '>') {
        return decideOperation(GreaterThanCondition(util::strToNum<feature_t>(condition.substr(1))), operation);
    } else {
        LOG(error, "Expected condition parameter to be 'true', '<a', or '>a', but was '%s'",
            condition.c_str());
        return false;
    }
}

template <typename CO>
bool
ForeachBlueprint::decideOperation(CO condition, const vespalib::string & operation)
{
    if (operation == "sum") {
        setExecutorCreator<CO, SumOperation>(condition);
    } else if (operation == "product") {
        setExecutorCreator<CO, ProductOperation>(condition);
    } else if (operation == "average") {
        setExecutorCreator<CO, AverageOperation>(condition);
    } else if (operation == "max") {
        setExecutorCreator<CO, MaxOperation>(condition);
    } else if (operation == "min") {
        setExecutorCreator<CO, MinOperation>(condition);
    } else if (operation == "count") {
        setExecutorCreator<CO, CountOperation>(condition);
    } else {
        LOG(error, "Expected operation parameter to be 'sum', 'product', 'average', 'max', 'min', or 'count', but was '%s'",
            operation.c_str());
        return false;
    }
    return true;
}

template <typename CO, typename OP>
void
ForeachBlueprint::setExecutorCreator(CO condition)
{
    class ExecutorCreator : public ExecutorCreatorBase {
    private:
        CO _condition;
    public:
        ExecutorCreator(CO cond) : _condition(cond) {}
        search::fef::FeatureExecutor &create(uint32_t numInputs, vespalib::Stash &stash) const override {
            return stash.create<ForeachExecutor<CO, OP>>(_condition,  numInputs);
        }
    };
    _executorCreator.reset(new ExecutorCreator(condition));
}

ForeachBlueprint::ForeachBlueprint() :
    Blueprint("foreach"),
    _dimension(ILLEGAL),
    _executorCreator(),
    _num_inputs(0)
{
}

ForeachBlueprint::~ForeachBlueprint() = default;

void
ForeachBlueprint::visitDumpFeatures(const IIndexEnvironment &,
                                    IDumpFeatureVisitor &) const
{
}

bool
ForeachBlueprint::setup(const IIndexEnvironment & env,
                        const ParameterList & params)
{
    if (!decideDimension(params[0].getValue())) {
        return false;
    }
    if (!decideCondition(params[3].getValue(), params[4].getValue())) {
        return false;
    }

    const vespalib::string & variable = params[1].getValue();
    const vespalib::string & feature = params[2].getValue();

    if (_dimension == TERMS) {
        uint32_t maxTerms = util::strToNum<uint32_t>(env.getProperties().lookup(getBaseName(), "maxTerms").get("16"));
        for (uint32_t i = 0; i < maxTerms; ++i) {
            defineInput(vespalib::replace_variable(feature, variable, vespalib::make_string("%u", i)));
            ++_num_inputs;
        }
    } else {
        for (uint32_t i = 0; i < env.getNumFields(); ++i) {
            const FieldInfo * info = env.getField(i);
            if (info->type() == FieldType::INDEX && _dimension == FIELDS) {
                defineInput(vespalib::replace_variable(feature, variable, info->name()));
                ++_num_inputs;
            } else if (info->type() == FieldType::ATTRIBUTE && _dimension == ATTRIBUTES) {
                defineInput(vespalib::replace_variable(feature, variable, info->name()));
                ++_num_inputs;
            }
        }
    }

    describeOutput("value", "The result after iterating over the input feature values using the specified operation");

    return true;
}

Blueprint::UP
ForeachBlueprint::createInstance() const
{
    return std::make_unique<ForeachBlueprint>();
}

FeatureExecutor &
ForeachBlueprint::createExecutor(const IQueryEnvironment &, vespalib::Stash &stash) const
{
    if (_executorCreator) {
        return _executorCreator->create(_num_inputs, stash);
    }
    return stash.create<SingleZeroValueExecutor>();
}

}
<|endoftext|>


<commit_before>// Copyright 2010-2013 RethinkDB, all rights reserved.
#include "serializer/log/extent_manager.hpp"

#include <queue>

#include "arch/arch.hpp"
#include "logger.hpp"
#include "perfmon/perfmon.hpp"
#include "serializer/log/log_serializer.hpp"

struct extent_info_t {
public:
    enum state_t {
        state_unreserved,
        state_in_use,
        state_free
    };
private:
    state_t state_;

public:
    void set_state(state_t new_state) {
        guarantee(state_ != state_in_use || extent_use_refcount == 0);
        state_ = new_state;
    }

    state_t state() const { return state_; }

    // Valid and non-zero when state_in_use.  There are two ways to own a part of
    // this refcount.  One is if you believe you're currently "using" the extent (if
    // you're the LBA or data_block_manager_t).  The other is (at the time of
    // writing) for every (extent_transaction_t, block_id) for live extent
    // transactions that have set an i_array entry to zero (in the data block
    // manager) but have not yet been commmitted.  The data_block_manager_t and LBA
    // ownership of the refcount also can get passed into the extent_transaction_t
    // object.
    int32_t extent_use_refcount;

    extent_info_t() : state_(state_unreserved),
                      extent_use_refcount(0) { }
};

class extent_zone_t {
    const size_t extent_size;

    unsigned int offset_to_id(int64_t extent) const {
        rassert(divides(extent_size, extent));
        return extent / extent_size;
    }

    /* Combination free-list and extent map. Contains one entry per extent.
    During the state_reserving_extents phase, each extent has state state_unreserved
    or state state_in_use. When we transition to the state_running phase,
    we link all of the state_unreserved entries in each zone together into an
    extent free list, such that each free entry's 'next_in_free_list' field is the
    offset of the next free extent. */

    std::vector<extent_info_t> extents;

    // We want to remove the minimum element from the free_queue first, leaving
    // free extents at the end of the file.
    std::priority_queue<unsigned int,
                        std::vector<unsigned int>,
                        std::greater<unsigned int> > free_queue;

    file_t *const dbfile;

    int held_extents_;

public:
    int held_extents() const {
        return held_extents_;
    }

    extent_zone_t(file_t *_dbfile, size_t _extent_size)
        : extent_size(_extent_size), dbfile(_dbfile), held_extents_(0) {
        // (Avoid a bunch of reallocations by resize calls (avoiding O(n log n)
        // work on average).)
        extents.reserve(dbfile->get_size() / extent_size);
    }

    extent_reference_t reserve_extent(int64_t extent) {
        unsigned int id = offset_to_id(extent);

        if (id >= extents.size()) {
            extents.resize(id + 1);
        }

        rassert(extents[id].state() == extent_info_t::state_unreserved);
        extents[id].set_state(extent_info_t::state_in_use);
        return make_extent_reference(extent);
    }

    void reconstruct_free_list() {
        // RSI: Iterate using extent id.
        for (int64_t extent = 0;
             extent < static_cast<int64_t>(extents.size() * extent_size);
             extent += extent_size) {
            const unsigned int extent_id = offset_to_id(extent);
            if (extents[extent_id].state() == extent_info_t::state_unreserved) {
                extents[extent_id].set_state(extent_info_t::state_free);
                free_queue.push(extent_id);
                ++held_extents_;
            }
        }
    }

    extent_reference_t gen_extent() {
        int64_t extent;

        if (free_queue.empty()) {
            extent = extents.size() * extent_size;
            extents.push_back(extent_info_t());
        } else {
            extent = free_queue.top() * extent_size;
            free_queue.pop();
            --held_extents_;
        }

        extent_info_t *info = &extents[offset_to_id(extent)];
        info->set_state(extent_info_t::state_in_use);

        extent_reference_t extent_ref = make_extent_reference(extent);

        dbfile->set_size_at_least(extent + extent_size);

        return extent_ref;
    }

    extent_reference_t make_extent_reference(const int64_t extent) {
        unsigned int id = offset_to_id(extent);
        guarantee(id < extents.size());
        extent_info_t *info = &extents[id];
        guarantee(info->state() == extent_info_t::state_in_use);
        ++info->extent_use_refcount;
        return extent_reference_t(extent);
    }

    void release_extent(extent_reference_t &&extent_ref) {
        int64_t extent = extent_ref.release();
        extent_info_t *info = &extents[offset_to_id(extent)];
        guarantee(info->state() == extent_info_t::state_in_use);
        guarantee(info->extent_use_refcount > 0);
        --info->extent_use_refcount;
        if (info->extent_use_refcount == 0) {
            info->set_state(extent_info_t::state_free);
            free_queue.push(offset_to_id(extent));
            ++held_extents_;
        }
    }
};

extent_manager_t::extent_manager_t(file_t *file,
                                   const log_serializer_on_disk_static_config_t *static_config,
                                   log_serializer_stats_t *_stats)
    : stats(_stats), extent_size(static_config->extent_size()),
      state(state_reserving_extents) {
    guarantee(divides(DEVICE_BLOCK_SIZE, extent_size));

    zone.init(new extent_zone_t(file, extent_size));
}

extent_manager_t::~extent_manager_t() {
    rassert(state == state_reserving_extents || state == state_shut_down);
}

extent_reference_t extent_manager_t::reserve_extent(int64_t extent) {
    assert_thread();
    rassert(state == state_reserving_extents);
    ++stats->pm_extents_in_use;
    stats->pm_bytes_in_use += extent_size;
    return zone->reserve_extent(extent);
}

void extent_manager_t::prepare_initial_metablock(metablock_mixin_t *mb) {
    mb->padding = 0;
}

void extent_manager_t::start_existing(UNUSED metablock_mixin_t *last_metablock) {
    assert_thread();
    rassert(state == state_reserving_extents);
    current_transaction = NULL;
    zone->reconstruct_free_list();
    state = state_running;

}

void extent_manager_t::prepare_metablock(metablock_mixin_t *metablock) {
    assert_thread();
    rassert(state == state_running);
    metablock->padding = 0;
}

void extent_manager_t::shutdown() {
    assert_thread();
    rassert(state == state_running);
    rassert(!current_transaction);
    state = state_shut_down;
}

void extent_manager_t::begin_transaction(extent_transaction_t *out) {
    assert_thread();
    rassert(!current_transaction);
    current_transaction = out;
    out->init();
}

extent_reference_t extent_manager_t::gen_extent() {
    assert_thread();
    rassert(state == state_running);
    ++stats->pm_extents_in_use;
    stats->pm_bytes_in_use += extent_size;

    return zone->gen_extent();
}

extent_reference_t
extent_manager_t::copy_extent_reference(const extent_reference_t &extent_ref) {
    int64_t offset = extent_ref.offset();
    return zone->make_extent_reference(offset);
}

void extent_manager_t::release_extent_into_transaction(extent_reference_t &&extent_ref, extent_transaction_t *txn) {
    release_extent_preliminaries();
    rassert(current_transaction);
    txn->push_extent(std::move(extent_ref));
}

void extent_manager_t::release_extent(extent_reference_t &&extent_ref) {
    release_extent_preliminaries();
    zone->release_extent(std::move(extent_ref));
}

void extent_manager_t::release_extent_preliminaries() {
    assert_thread();
    rassert(state == state_running);
    --stats->pm_extents_in_use;
    stats->pm_bytes_in_use -= extent_size;
}

void extent_manager_t::end_transaction(extent_transaction_t *t) {
    assert_thread();
    rassert(current_transaction == t);
    current_transaction = NULL;
    t->mark_end();
}

void extent_manager_t::commit_transaction(extent_transaction_t *t) {
    assert_thread();
    std::vector<extent_reference_t> extents = t->reset();
    for (auto it = extents.begin(); it != extents.end(); ++it) {
        zone->release_extent(std::move(*it));
    }
}

int extent_manager_t::held_extents() {
    assert_thread();
    return zone->held_extents();
}
<commit_msg>Made reconstruct_free_list just iterate using extent ids instead of offsets.<commit_after>// Copyright 2010-2013 RethinkDB, all rights reserved.
#include "serializer/log/extent_manager.hpp"

#include <queue>

#include "arch/arch.hpp"
#include "logger.hpp"
#include "perfmon/perfmon.hpp"
#include "serializer/log/log_serializer.hpp"

struct extent_info_t {
public:
    enum state_t {
        state_unreserved,
        state_in_use,
        state_free
    };
private:
    state_t state_;

public:
    void set_state(state_t new_state) {
        guarantee(state_ != state_in_use || extent_use_refcount == 0);
        state_ = new_state;
    }

    state_t state() const { return state_; }

    // Valid and non-zero when state_in_use.  There are two ways to own a part of
    // this refcount.  One is if you believe you're currently "using" the extent (if
    // you're the LBA or data_block_manager_t).  The other is (at the time of
    // writing) for every (extent_transaction_t, block_id) for live extent
    // transactions that have set an i_array entry to zero (in the data block
    // manager) but have not yet been commmitted.  The data_block_manager_t and LBA
    // ownership of the refcount also can get passed into the extent_transaction_t
    // object.
    int32_t extent_use_refcount;

    extent_info_t() : state_(state_unreserved),
                      extent_use_refcount(0) { }
};

class extent_zone_t {
    const size_t extent_size;

    unsigned int offset_to_id(int64_t extent) const {
        rassert(divides(extent_size, extent));
        return extent / extent_size;
    }

    /* Combination free-list and extent map. Contains one entry per extent.
    During the state_reserving_extents phase, each extent has state state_unreserved
    or state state_in_use. When we transition to the state_running phase,
    we link all of the state_unreserved entries in each zone together into an
    extent free list, such that each free entry's 'next_in_free_list' field is the
    offset of the next free extent. */

    std::vector<extent_info_t> extents;

    // We want to remove the minimum element from the free_queue first, leaving
    // free extents at the end of the file.
    std::priority_queue<unsigned int,
                        std::vector<unsigned int>,
                        std::greater<unsigned int> > free_queue;

    file_t *const dbfile;

    int held_extents_;

public:
    int held_extents() const {
        return held_extents_;
    }

    extent_zone_t(file_t *_dbfile, size_t _extent_size)
        : extent_size(_extent_size), dbfile(_dbfile), held_extents_(0) {
        // (Avoid a bunch of reallocations by resize calls (avoiding O(n log n)
        // work on average).)
        extents.reserve(dbfile->get_size() / extent_size);
    }

    extent_reference_t reserve_extent(int64_t extent) {
        unsigned int id = offset_to_id(extent);

        if (id >= extents.size()) {
            extents.resize(id + 1);
        }

        rassert(extents[id].state() == extent_info_t::state_unreserved);
        extents[id].set_state(extent_info_t::state_in_use);
        return make_extent_reference(extent);
    }

    void reconstruct_free_list() {
        // RSI: Iterate using extent id.
        for (unsigned int extent_id = 0; extent_id < extents.size(); ++extent_id) {
            if (extents[extent_id].state() == extent_info_t::state_unreserved) {
                extents[extent_id].set_state(extent_info_t::state_free);
                free_queue.push(extent_id);
                ++held_extents_;
            }
        }
    }

    extent_reference_t gen_extent() {
        int64_t extent;

        if (free_queue.empty()) {
            extent = extents.size() * extent_size;
            extents.push_back(extent_info_t());
        } else {
            extent = free_queue.top() * extent_size;
            free_queue.pop();
            --held_extents_;
        }

        extent_info_t *info = &extents[offset_to_id(extent)];
        info->set_state(extent_info_t::state_in_use);

        extent_reference_t extent_ref = make_extent_reference(extent);

        dbfile->set_size_at_least(extent + extent_size);

        return extent_ref;
    }

    extent_reference_t make_extent_reference(const int64_t extent) {
        unsigned int id = offset_to_id(extent);
        guarantee(id < extents.size());
        extent_info_t *info = &extents[id];
        guarantee(info->state() == extent_info_t::state_in_use);
        ++info->extent_use_refcount;
        return extent_reference_t(extent);
    }

    void release_extent(extent_reference_t &&extent_ref) {
        int64_t extent = extent_ref.release();
        extent_info_t *info = &extents[offset_to_id(extent)];
        guarantee(info->state() == extent_info_t::state_in_use);
        guarantee(info->extent_use_refcount > 0);
        --info->extent_use_refcount;
        if (info->extent_use_refcount == 0) {
            info->set_state(extent_info_t::state_free);
            free_queue.push(offset_to_id(extent));
            ++held_extents_;
        }
    }
};

extent_manager_t::extent_manager_t(file_t *file,
                                   const log_serializer_on_disk_static_config_t *static_config,
                                   log_serializer_stats_t *_stats)
    : stats(_stats), extent_size(static_config->extent_size()),
      state(state_reserving_extents) {
    guarantee(divides(DEVICE_BLOCK_SIZE, extent_size));

    zone.init(new extent_zone_t(file, extent_size));
}

extent_manager_t::~extent_manager_t() {
    rassert(state == state_reserving_extents || state == state_shut_down);
}

extent_reference_t extent_manager_t::reserve_extent(int64_t extent) {
    assert_thread();
    rassert(state == state_reserving_extents);
    ++stats->pm_extents_in_use;
    stats->pm_bytes_in_use += extent_size;
    return zone->reserve_extent(extent);
}

void extent_manager_t::prepare_initial_metablock(metablock_mixin_t *mb) {
    mb->padding = 0;
}

void extent_manager_t::start_existing(UNUSED metablock_mixin_t *last_metablock) {
    assert_thread();
    rassert(state == state_reserving_extents);
    current_transaction = NULL;
    zone->reconstruct_free_list();
    state = state_running;

}

void extent_manager_t::prepare_metablock(metablock_mixin_t *metablock) {
    assert_thread();
    rassert(state == state_running);
    metablock->padding = 0;
}

void extent_manager_t::shutdown() {
    assert_thread();
    rassert(state == state_running);
    rassert(!current_transaction);
    state = state_shut_down;
}

void extent_manager_t::begin_transaction(extent_transaction_t *out) {
    assert_thread();
    rassert(!current_transaction);
    current_transaction = out;
    out->init();
}

extent_reference_t extent_manager_t::gen_extent() {
    assert_thread();
    rassert(state == state_running);
    ++stats->pm_extents_in_use;
    stats->pm_bytes_in_use += extent_size;

    return zone->gen_extent();
}

extent_reference_t
extent_manager_t::copy_extent_reference(const extent_reference_t &extent_ref) {
    int64_t offset = extent_ref.offset();
    return zone->make_extent_reference(offset);
}

void extent_manager_t::release_extent_into_transaction(extent_reference_t &&extent_ref, extent_transaction_t *txn) {
    release_extent_preliminaries();
    rassert(current_transaction);
    txn->push_extent(std::move(extent_ref));
}

void extent_manager_t::release_extent(extent_reference_t &&extent_ref) {
    release_extent_preliminaries();
    zone->release_extent(std::move(extent_ref));
}

void extent_manager_t::release_extent_preliminaries() {
    assert_thread();
    rassert(state == state_running);
    --stats->pm_extents_in_use;
    stats->pm_bytes_in_use -= extent_size;
}

void extent_manager_t::end_transaction(extent_transaction_t *t) {
    assert_thread();
    rassert(current_transaction == t);
    current_transaction = NULL;
    t->mark_end();
}

void extent_manager_t::commit_transaction(extent_transaction_t *t) {
    assert_thread();
    std::vector<extent_reference_t> extents = t->reset();
    for (auto it = extents.begin(); it != extents.end(); ++it) {
        zone->release_extent(std::move(*it));
    }
}

int extent_manager_t::held_extents() {
    assert_thread();
    return zone->held_extents();
}
<|endoftext|>


<commit_before>/*
 *
 * Copyright 2018 Asylo authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

#include <cstdlib>

#include "asylo/platform/primitives/trusted_runtime.h"

// The newlib implementation of malloc and free in mallocr.c depends on symbols
// _malloc_lock and _malloc_unlock, and requires that a thread waiting on a lock
// it already holds will not pause. This file provides a implementation of that
// interface inside the enclave with minimal dependencies on other runtime
// components which expect to call malloc.

#define CACHE_ALIGNED __attribute__((aligned(64)))

// The current owner of the lock, aligned to ensure atomic instructions do not
// straddle a cache line. A value of zero indicates the lock is not held by any
// thread.
static volatile uint64_t lock_owner CACHE_ALIGNED = kInvalidThread;

// The number of times the lock has been recursively acquired, aligned to ensure
// it does not fall in the same cache line as lock_owner to avoid false sharing.
// Notes that the count is only accessed by the thread holding the lock and does
// not need to be updated atomically.
static int lock_count CACHE_ALIGNED = 0;

extern "C" {

void __malloc_lock(struct reent *) {
  const uint64_t self = enc_thread_self();

  while (lock_owner != self) {
    uint64_t prev = 0;

    // Attempt to acquire the mutex by swapping the lock with the current thread
    // id, specifying strong memory ordering to avoid unexpected memory
    // ordering. Try to obtain the lock by atomically testing that it is
    // unlocked and exchanging it with our thread id.
    if (__atomic_compare_exchange_n(&lock_owner,
                                    /*expected=*/&prev,
                                    /*desired=*/self,
                                    /*weak=*/false,
                                    /*success_memorder=*/__ATOMIC_SEQ_CST,
                                    /*failure_memorder=*/__ATOMIC_SEQ_CST) !=
        kInvalidThread) {
      break;
    }

    // If the lock is not free, avoid doing a synchronized read and busy wait
    // until it's release then try to obtain it again.
    while (lock_owner != kInvalidThread) {
      // Issue a busy-wait hint to the CPU if possible.
      enc_pause();
    }
  }

  lock_count++;
}

void __malloc_unlock(struct reent *) {
  // Unlock should only be called by the thread that holds the lock. If this is
  // not the case we abort.
  if (lock_owner != enc_thread_self()) {
    abort();
  }

  lock_count--;
  if (lock_count == 0) {
    // Release the lock with an atomic store. __ATOMIC_RELEASE ensures this
    // write is visible to threads obtaining the lock with __ATOMIC_SEQ_CST.
    __atomic_store_n(&lock_owner, kInvalidThread, __ATOMIC_RELEASE);
  }
}

}  // extern "C"
<commit_msg>Use constants consistently in malloc_lock<commit_after>/*
 *
 * Copyright 2018 Asylo authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

#include <cstdlib>

#include "asylo/platform/primitives/trusted_runtime.h"

// The newlib implementation of malloc and free in mallocr.c depends on symbols
// _malloc_lock and _malloc_unlock, and requires that a thread waiting on a lock
// it already holds will not pause. This file provides a implementation of that
// interface inside the enclave with minimal dependencies on other runtime
// components which expect to call malloc.

#define CACHE_ALIGNED __attribute__((aligned(64)))

// The current owner of the lock, aligned to ensure atomic instructions do not
// straddle a cache line. A value of zero indicates the lock is not held by any
// thread.
static volatile uint64_t lock_owner CACHE_ALIGNED = kInvalidThread;

// The number of times the lock has been recursively acquired, aligned to ensure
// it does not fall in the same cache line as lock_owner to avoid false sharing.
// Notes that the count is only accessed by the thread holding the lock and does
// not need to be updated atomically.
static int lock_count CACHE_ALIGNED = 0;

extern "C" {

void __malloc_lock(struct reent *) {
  const uint64_t self = enc_thread_self();

  while (lock_owner != self) {
    uint64_t unlocked_value = kInvalidThread;

    // Attempt to acquire the mutex by swapping the lock with the current thread
    // id, specifying strong memory ordering to avoid unexpected memory
    // ordering. Try to obtain the lock by atomically testing that it is
    // unlocked and exchanging it with our thread id.
    if (__atomic_compare_exchange_n(&lock_owner,
                                    /*expected=*/&unlocked_value,
                                    /*desired=*/self,
                                    /*weak=*/false,
                                    /*success_memorder=*/__ATOMIC_SEQ_CST,
                                    /*failure_memorder=*/__ATOMIC_SEQ_CST)) {
      break;
    }

    // If the lock is not free, avoid doing a synchronized read and busy wait
    // until it's released then try to obtain it again.
    while (lock_owner != kInvalidThread) {
      // Issue a busy-wait hint to the CPU if possible.
      enc_pause();
    }
  }

  lock_count++;
}

void __malloc_unlock(struct reent *) {
  // Unlock should only be called by the thread that holds the lock. If this is
  // not the case we abort.
  if (lock_owner != enc_thread_self()) {
    abort();
  }

  lock_count--;
  if (lock_count == 0) {
    // Release the lock with an atomic store. __ATOMIC_RELEASE ensures this
    // write is visible to threads obtaining the lock with __ATOMIC_SEQ_CST.
    __atomic_store_n(&lock_owner, kInvalidThread, __ATOMIC_RELEASE);
  }
}

}  // extern "C"
<|endoftext|>


<commit_before><commit_msg>BZOJ 4712<commit_after><|endoftext|>


<commit_before>/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
/*
 * Version: MPL 1.1 / GPLv3+ / LGPLv3+
 *
 * The contents of this file are subject to the Mozilla Public License Version
 * 1.1 (the "License"); you may not use this file except in compliance with
 * the License or as specified alternatively below. You may obtain a copy of
 * the License at http://www.mozilla.org/MPL/
 *
 * Software distributed under the License is distributed on an "AS IS" basis,
 * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
 * for the specific language governing rights and limitations under the
 * License.
 *
 * Major Contributor(s):
 * Copyright (C) 2011 Red Hat, Inc., Stephan Bergmann <sbergman@redhat.com>
 *  (initial developer)
 *
 * All Rights Reserved.
 *
 * For minor contributions see the git repository.
 *
 * Alternatively, the contents of this file may be used under the terms of
 * either the GNU General Public License Version 3 or later (the "GPLv3+"), or
 * the GNU Lesser General Public License Version 3 or later (the "LGPLv3+"),
 * in which case the provisions of the GPLv3+ or the LGPLv3+ are applicable
 * instead of those above.
 */

#include "sal/config.h"

#include <cassert>
#include <cstdarg>
#include <cstddef>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <sstream>

#include <stdio.h> // vsnprintf
#include <string.h> // strdup

#include "osl/thread.hxx"
#include "rtl/string.h"
#include "sal/detail/log.h"
#include "sal/log.hxx"
#include "sal/types.h"

#include "logformat.hxx"

#if defined WNT
#include <process.h>
#define OSL_DETAIL_GETPID _getpid()
#else
#include <unistd.h>
#define OSL_DETAIL_GETPID getpid()
#endif

#ifdef HAVE_SYSLOG_H
#include <syslog.h>
// sal/osl/unx/salinit.cxx::sal_detail_initialize updates this:
bool sal_use_syslog;
#endif

// Avoid the use of other sal code in this file as much as possible, so that
// this code can be called from other sal code without causing endless
// recursion.

namespace {

bool equalStrings(
    char const * string1, std::size_t length1, char const * string2,
    std::size_t length2)
{
    return length1 == length2 && std::memcmp(string1, string2, length1) == 0;
}

char const * toString(sal_detail_LogLevel level) {
    switch (level) {
    default:
        assert(false); // this cannot happen
        // fall through
    case SAL_DETAIL_LOG_LEVEL_INFO:
        return "info";
    case SAL_DETAIL_LOG_LEVEL_WARN:
        return "warn";
    case SAL_DETAIL_LOG_LEVEL_DEBUG:
        return "debug";
    }
}

// getenv is not thread safe, so minimize use of result; except on Android and
// iOS, see 60628799633ffde502cb105b98d3f254f93115aa "Notice if SAL_LOG is
// changed while the process is running":
#if defined ANDROID || defined IOS

char const * getEnvironmentVariable() {
    return std::getenv("SAL_LOG");
}

#else

char const * getEnvironmentVariable_() {
    char const * p1 = std::getenv("SAL_LOG");
    if (p1 == 0) {
        return 0;
    }
    char const * p2 = strdup(p1); // leaked
    if (p2 == 0) {
        std::abort(); // cannot do much here
    }
    return p2;
}

char const * getEnvironmentVariable() {
    static char const * env = getEnvironmentVariable_();
    return env;
}

#endif

#ifdef HAVE_SYSLOG_H
int toSyslogPriority(sal_detail_LogLevel level) {
    switch (level) {
    default:
        assert(false); // this cannot happen
        // fall through
    case SAL_DETAIL_LOG_LEVEL_INFO:
        return LOG_INFO;
    case SAL_DETAIL_LOG_LEVEL_WARN:
        return LOG_WARNING;
    case SAL_DETAIL_LOG_LEVEL_DEBUG:
        return LOG_DEBUG;
    }
}
#endif

bool report(sal_detail_LogLevel level, char const * area) {
    if (level == SAL_DETAIL_LOG_LEVEL_DEBUG)
        return true;
    assert(area != 0);
    char const * env = getEnvironmentVariable();
    if (env == 0) {
        env = "+WARN";
    }
    std::size_t areaLen = std::strlen(area);
    enum Sense { POSITIVE = 0, NEGATIVE = 1 };
    std::size_t senseLen[2] = { 0, 1 };
        // initial senseLen[POSITIVE] < senseLen[NEGATIVE], so that if there are
        // no matching switches at all, the result will be negative (and
        // initializing with 1 is safe as the length of a valid switch, even
        // without the "+"/"-" prefix, will always be > 1)
    for (char const * p = env;;) {
        Sense sense;
        switch (*p++) {
        case '\0':
            return senseLen[POSITIVE] >= senseLen[NEGATIVE];
                // if a specific item is both postiive and negative
                // (senseLen[POSITIVE] == senseLen[NEGATIVE]), default to
                // positive
        case '+':
            sense = POSITIVE;
            break;
        case '-':
            sense = NEGATIVE;
            break;
        default:
            return true; // upon an illegal SAL_LOG value, enable everything
        }
        char const * p1 = p;
        while (*p1 != '.' && *p1 != '+' && *p1 != '-' && *p1 != '\0') {
            ++p1;
        }
        bool match;
        if (equalStrings(p, p1 - p, RTL_CONSTASCII_STRINGPARAM("INFO"))) {
            match = level == SAL_DETAIL_LOG_LEVEL_INFO;
        } else if (equalStrings(p, p1 - p, RTL_CONSTASCII_STRINGPARAM("WARN")))
        {
            match = level == SAL_DETAIL_LOG_LEVEL_WARN;
        } else {
            return true;
                // upon an illegal SAL_LOG value, everything is considered
                // positive
        }
        char const * p2 = p1;
        while (*p2 != '+' && *p2 != '-' && *p2 != '\0') {
            ++p2;
        }
        if (match) {
            if (*p1 == '.') {
                ++p1;
                std::size_t n = p2 - p1;
                if ((n == areaLen && equalStrings(p1, n, area, areaLen))
                    || (n < areaLen && area[n] == '.'
                        && equalStrings(p1, n, area, n)))
                {
                    senseLen[sense] = p2 - p;
                }
            } else {
                senseLen[sense] = p1 - p;
            }
        }
        p = p2;
    }
}

void log(
    sal_detail_LogLevel level, char const * area, char const * where,
    char const * message)
{
    std::ostringstream s;
#ifdef HAVE_SYSLOG_H
    if (!sal_use_syslog)
#endif
        s << toString(level) << ':';
    if (level == SAL_DETAIL_LOG_LEVEL_DEBUG) {
        s << /*no where*/' ' << message << '\n';
    } else {
        s << area << ':' << OSL_DETAIL_GETPID << ':'
          << osl::Thread::getCurrentIdentifier() << ':';
        if (strncmp(where, SRCDIR, sizeof(SRCDIR)-1) == 0)
            s << where+sizeof(SRCDIR);
        else
            s << where;
        s << message << '\n';
    }
#ifdef HAVE_SYSLOG_H
    if (sal_use_syslog)
        syslog(toSyslogPriority(level), "%s", s.str().c_str());
    else
#endif
        std::fputs(s.str().c_str(), stderr);
}

}

void sal_detail_log(
    sal_detail_LogLevel level, char const * area, char const * where,
    char const * message)
{
    if (report(level, area)) {
        log(level, area, where, message);
    }
}

void sal_detail_logFormat(
    sal_detail_LogLevel level, char const * area, char const * where,
    char const * format, ...)
{
    if (report(level, area)) {
        std::va_list args;
        va_start(args, format);
        osl::detail::logFormat(level, area, where, format, args);
        va_end(args);
    }
}

void osl::detail::logFormat(
    sal_detail_LogLevel level, char const * area, char const * where,
    char const * format, std::va_list arguments)
{
    char buf[1024];
    int const len = sizeof buf - RTL_CONSTASCII_LENGTH("...");
    int n = vsnprintf(buf, len, format, arguments);
    if (n < 0) {
        std::strcpy(buf, "???");
    } else if (n >= len) {
        std::strcpy(buf + len - 1, "...");
    }
    log(level, area, where, buf);
}

/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
<commit_msg>Pass log output directly to the Android log mechanism<commit_after>/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
/*
 * Version: MPL 1.1 / GPLv3+ / LGPLv3+
 *
 * The contents of this file are subject to the Mozilla Public License Version
 * 1.1 (the "License"); you may not use this file except in compliance with
 * the License or as specified alternatively below. You may obtain a copy of
 * the License at http://www.mozilla.org/MPL/
 *
 * Software distributed under the License is distributed on an "AS IS" basis,
 * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
 * for the specific language governing rights and limitations under the
 * License.
 *
 * Major Contributor(s):
 * Copyright (C) 2011 Red Hat, Inc., Stephan Bergmann <sbergman@redhat.com>
 *  (initial developer)
 *
 * All Rights Reserved.
 *
 * For minor contributions see the git repository.
 *
 * Alternatively, the contents of this file may be used under the terms of
 * either the GNU General Public License Version 3 or later (the "GPLv3+"), or
 * the GNU Lesser General Public License Version 3 or later (the "LGPLv3+"),
 * in which case the provisions of the GPLv3+ or the LGPLv3+ are applicable
 * instead of those above.
 */

#include "sal/config.h"

#include <cassert>
#include <cstdarg>
#include <cstddef>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <sstream>

#include <stdio.h> // vsnprintf
#include <string.h> // strdup

#include "osl/thread.hxx"
#include "rtl/string.h"
#include "sal/detail/log.h"
#include "sal/log.hxx"
#include "sal/types.h"

#include "logformat.hxx"

#if defined WNT
#include <process.h>
#define OSL_DETAIL_GETPID _getpid()
#else
#include <unistd.h>
#define OSL_DETAIL_GETPID getpid()
#endif

#ifdef HAVE_SYSLOG_H
#include <syslog.h>
// sal/osl/unx/salinit.cxx::sal_detail_initialize updates this:
bool sal_use_syslog;
#endif

#ifdef ANDROID
#include <android/log.h>
#endif

// Avoid the use of other sal code in this file as much as possible, so that
// this code can be called from other sal code without causing endless
// recursion.

namespace {

bool equalStrings(
    char const * string1, std::size_t length1, char const * string2,
    std::size_t length2)
{
    return length1 == length2 && std::memcmp(string1, string2, length1) == 0;
}

char const * toString(sal_detail_LogLevel level) {
    switch (level) {
    default:
        assert(false); // this cannot happen
        // fall through
    case SAL_DETAIL_LOG_LEVEL_INFO:
        return "info";
    case SAL_DETAIL_LOG_LEVEL_WARN:
        return "warn";
    case SAL_DETAIL_LOG_LEVEL_DEBUG:
        return "debug";
    }
}

// getenv is not thread safe, so minimize use of result; except on Android and
// iOS, see 60628799633ffde502cb105b98d3f254f93115aa "Notice if SAL_LOG is
// changed while the process is running":
#if defined ANDROID || defined IOS

char const * getEnvironmentVariable() {
    return std::getenv("SAL_LOG");
}

#else

char const * getEnvironmentVariable_() {
    char const * p1 = std::getenv("SAL_LOG");
    if (p1 == 0) {
        return 0;
    }
    char const * p2 = strdup(p1); // leaked
    if (p2 == 0) {
        std::abort(); // cannot do much here
    }
    return p2;
}

char const * getEnvironmentVariable() {
    static char const * env = getEnvironmentVariable_();
    return env;
}

#endif

#ifdef HAVE_SYSLOG_H
int toSyslogPriority(sal_detail_LogLevel level) {
    switch (level) {
    default:
        assert(false); // this cannot happen
        // fall through
    case SAL_DETAIL_LOG_LEVEL_INFO:
        return LOG_INFO;
    case SAL_DETAIL_LOG_LEVEL_WARN:
        return LOG_WARNING;
    case SAL_DETAIL_LOG_LEVEL_DEBUG:
        return LOG_DEBUG;
    }
}
#endif

bool report(sal_detail_LogLevel level, char const * area) {
    if (level == SAL_DETAIL_LOG_LEVEL_DEBUG)
        return true;
    assert(area != 0);
    char const * env = getEnvironmentVariable();
    if (env == 0) {
        env = "+WARN";
    }
    std::size_t areaLen = std::strlen(area);
    enum Sense { POSITIVE = 0, NEGATIVE = 1 };
    std::size_t senseLen[2] = { 0, 1 };
        // initial senseLen[POSITIVE] < senseLen[NEGATIVE], so that if there are
        // no matching switches at all, the result will be negative (and
        // initializing with 1 is safe as the length of a valid switch, even
        // without the "+"/"-" prefix, will always be > 1)
    for (char const * p = env;;) {
        Sense sense;
        switch (*p++) {
        case '\0':
            return senseLen[POSITIVE] >= senseLen[NEGATIVE];
                // if a specific item is both postiive and negative
                // (senseLen[POSITIVE] == senseLen[NEGATIVE]), default to
                // positive
        case '+':
            sense = POSITIVE;
            break;
        case '-':
            sense = NEGATIVE;
            break;
        default:
            return true; // upon an illegal SAL_LOG value, enable everything
        }
        char const * p1 = p;
        while (*p1 != '.' && *p1 != '+' && *p1 != '-' && *p1 != '\0') {
            ++p1;
        }
        bool match;
        if (equalStrings(p, p1 - p, RTL_CONSTASCII_STRINGPARAM("INFO"))) {
            match = level == SAL_DETAIL_LOG_LEVEL_INFO;
        } else if (equalStrings(p, p1 - p, RTL_CONSTASCII_STRINGPARAM("WARN")))
        {
            match = level == SAL_DETAIL_LOG_LEVEL_WARN;
        } else {
            return true;
                // upon an illegal SAL_LOG value, everything is considered
                // positive
        }
        char const * p2 = p1;
        while (*p2 != '+' && *p2 != '-' && *p2 != '\0') {
            ++p2;
        }
        if (match) {
            if (*p1 == '.') {
                ++p1;
                std::size_t n = p2 - p1;
                if ((n == areaLen && equalStrings(p1, n, area, areaLen))
                    || (n < areaLen && area[n] == '.'
                        && equalStrings(p1, n, area, n)))
                {
                    senseLen[sense] = p2 - p;
                }
            } else {
                senseLen[sense] = p1 - p;
            }
        }
        p = p2;
    }
}

void log(
    sal_detail_LogLevel level, char const * area, char const * where,
    char const * message)
{
    std::ostringstream s;
#ifndef ANDROID
#ifdef HAVE_SYSLOG_H
    if (!sal_use_syslog)
#endif
        s << toString(level) << ':';
#endif
    if (level == SAL_DETAIL_LOG_LEVEL_DEBUG) {
        s << /*no area or where */ ' ' << message << '\n';
    } else {
#ifdef ANDROID
        // The area will be used as the "tag", and log info already contgains the pid on Android
#else
        s << area << ':' << OSL_DETAIL_GETPID << ':';
#endif
        s << osl::Thread::getCurrentIdentifier() << ':';
        if (strncmp(where, SRCDIR, sizeof(SRCDIR)-1) == 0)
            s << where+sizeof(SRCDIR);
        else
            s << where;
        s << message << '\n';
    }
#ifdef ANDROID
    int android_log_level;
    switch (level) {
    case SAL_DETAIL_LOG_LEVEL_INFO:
        android_log_level = ANDROID_LOG_INFO;
        break;
    case SAL_DETAIL_LOG_LEVEL_WARN:
        android_log_level = ANDROID_LOG_WARN;
        break;
    case SAL_DETAIL_LOG_LEVEL_DEBUG:
        android_log_level = ANDROID_LOG_DEBUG;
        break;
    default:
        android_log_level = ANDROID_LOG_INFO;
        break;
    }
    if (area == NULL)
        area = "LibreOffice";
    __android_log_print(android_log_level, area, "%s", s.str().c_str());
#else
#ifdef HAVE_SYSLOG_H
    if (sal_use_syslog)
        syslog(toSyslogPriority(level), "%s", s.str().c_str());
    else
#endif
        std::fputs(s.str().c_str(), stderr);
#endif
}

}

void sal_detail_log(
    sal_detail_LogLevel level, char const * area, char const * where,
    char const * message)
{
    if (report(level, area)) {
        log(level, area, where, message);
    }
}

void sal_detail_logFormat(
    sal_detail_LogLevel level, char const * area, char const * where,
    char const * format, ...)
{
    if (report(level, area)) {
        std::va_list args;
        va_start(args, format);
        osl::detail::logFormat(level, area, where, format, args);
        va_end(args);
    }
}

void osl::detail::logFormat(
    sal_detail_LogLevel level, char const * area, char const * where,
    char const * format, std::va_list arguments)
{
    char buf[1024];
    int const len = sizeof buf - RTL_CONSTASCII_LENGTH("...");
    int n = vsnprintf(buf, len, format, arguments);
    if (n < 0) {
        std::strcpy(buf, "???");
    } else if (n >= len) {
        std::strcpy(buf + len - 1, "...");
    }
    log(level, area, where, buf);
}

/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
<|endoftext|>


<commit_before>#include "Commander.h"
#include <iostream>
#include <string>

using namespace std;

Commander::Commander(CmdContext *v_context)
    : context(v_context)
{
     if (this->context == NULL) throw CmdException("Uninitialized commander context!");
    addCommand("q", new QuitCommand());
}

void Commander::clearCommands()
{
    std::map<std::string, Command*>::iterator itr;
    for (itr = cmds.begin(); itr != cmds.end(); itr++) {
        Command* cmd = itr->second;
        delete cmd;
    }
    this->cmds.clear();
}

void Commander::printHelp()
{
    std::cout << "Avaliable commands: " << cmds.size() << endl;
    std::map<std::string, Command*>::iterator itr;
    for (itr = cmds.begin(); itr != cmds.end(); itr++) {
        Command *cmd = itr->second;
        if (cmd == NULL) continue;
        std::cout << itr->first << " \t- " << cmd->getDescription() << endl;
    }
}

Command* Commander::getCommand(std::string line)
{
    std::string name = line;
    //TODO: split the line...
    if (cmds.find(name) == cmds.end() ) {
       std::cerr << "No such command" << endl;
       return NULL;
    }

    Command *cmd = this->cmds[name];
    return cmd;
}

bool Commander::addCommand(std::string name, Command *cmd, bool overwrite)
{
    if ( cmds.find(name) != cmds.end() ) {
        if (!overwrite) return false; // already exist
        delete this->cmds[name];
        this->cmds[name]  = NULL;
    }
    this->cmds[name] = cmd;
    return true;
}

void Commander::parseCommands()
{
    const std::string PROMPT = "$ ";
    Command *cmd = NULL;

    while (true) {
        if (this->context == NULL) throw CmdException("Uninitialized commander context!");
        if (this->context->isEndProcessing()) break;

        std::cout << PROMPT;
        std::string line;
        std::cin >> line;

        Command *cmd = getCommand(line);

        if (cmd == NULL) {
            this->printHelp();
            continue;
        }
        try {
            CmdParams *params = cmd->fetchParams(line);
            cmd->execute(params, this->context);
        } catch (CustomException e) {
            std::cerr << "ERROR: " << e.what() << endl;
        }
    }
}

<commit_msg>Fix typo<commit_after>#include "Commander.h"
#include <iostream>
#include <string>

using namespace std;

Commander::Commander(CmdContext *v_context)
    : context(v_context)
{
     if (this->context == NULL) throw CmdException("Uninitialized commander context!");
    addCommand("q", new QuitCommand());
}

void Commander::clearCommands()
{
    std::map<std::string, Command*>::iterator itr;
    for (itr = cmds.begin(); itr != cmds.end(); itr++) {
        Command* cmd = itr->second;
        delete cmd;
    }
    this->cmds.clear();
}

void Commander::printHelp()
{
    std::cout << "Available commands: " << cmds.size() << endl;
    std::map<std::string, Command*>::iterator itr;
    for (itr = cmds.begin(); itr != cmds.end(); itr++) {
        Command *cmd = itr->second;
        if (cmd == NULL) continue;
        std::cout << itr->first << " \t- " << cmd->getDescription() << endl;
    }
}

Command* Commander::getCommand(std::string line)
{
    std::string name = line;
    //TODO: split the line...
    if (cmds.find(name) == cmds.end() ) {
       std::cerr << "No such command" << endl;
       return NULL;
    }

    Command *cmd = this->cmds[name];
    return cmd;
}

bool Commander::addCommand(std::string name, Command *cmd, bool overwrite)
{
    if ( cmds.find(name) != cmds.end() ) {
        if (!overwrite) return false; // already exist
        delete this->cmds[name];
        this->cmds[name]  = NULL;
    }
    this->cmds[name] = cmd;
    return true;
}

void Commander::parseCommands()
{
    const std::string PROMPT = "$ ";
    Command *cmd = NULL;

    while (true) {
        if (this->context == NULL) throw CmdException("Uninitialized commander context!");
        if (this->context->isEndProcessing()) break;

        std::cout << PROMPT;
        std::string line;
        std::cin >> line;

        Command *cmd = getCommand(line);

        if (cmd == NULL) {
            this->printHelp();
            continue;
        }
        try {
            CmdParams *params = cmd->fetchParams(line);
            cmd->execute(params, this->context);
        } catch (CustomException e) {
            std::cerr << "ERROR: " << e.what() << endl;
        }
    }
}

<|endoftext|>


<commit_before>// converts scf names into mmap file
//

#include "lvvlib/mmap.h"

// LIBC
#include <fcntl.h>
#include <time.h>

// THRIFT
#include <thrift/protocol/TBinaryProtocol.h>
#include <thrift/protocol/TDenseProtocol.h>
#include <thrift/protocol/TJSONProtocol.h>
#include <thrift/transport/TTransportUtils.h>
#include <thrift/transport/TFDTransport.h>
#include <thrift/transport/TFileTransport.h>
	namespace atp = apache::thrift::protocol;
	namespace att = apache::thrift::transport;

// THRIFT -- FILTERNAMES
#include "filternames_types.h"
#include "filternames_constants.h"
	namespace fn = filternames;

// STD
#include <iostream>
	using std::cerr;
	using std::cout;
	using std::endl;

#include <algorithm>
	using std::copy;
#include <iterator>
	using std::back_inserter;
#include <unordered_map>
	using std::unordered_map;
#include <set>
	using std::set;
#include <string>
	using std::string;
#include <vector>
	using std::vector;
#include <fstream>

// BOOST
#include <boost/program_options.hpp>
	namespace po = boost::program_options;

// LVVLIB (http://github.com/lvv/lvvlib)
//#ifdef LVV
//#include <ro/ro.h>
//#include <scc/simple.h>
//#endif
#include "lvvlib/token.h"

int main(int argc, char **argv) {

	///////////////////////////////////////////////////////////////////////////////  OPTIONS

	// options
	string		filtername_path;

	po::options_description desc("Allowed options");
	desc.add_options()
		("help,h", "help message")
		("filternames,f", po::value<string>(&filtername_path), "filternames file")
	;

	// Parse command line options
	po::variables_map vm;
	po::store(po::parse_command_line(argc, argv, desc), vm);
	po::notify(vm);

	if (vm.count("help")) { cout << desc << "\n"; return 1; }

	////////////////////////////////////////////////////////////// THRIFT READS FILTERNAMES

	int scf_fh = open(filtername_path.c_str(), O_RDONLY);

					if(scf_fh==-1)  {
						cerr << "error: cann't open scf file -- '" << filtername_path << "'\n";
						exit(1);
					}

	boost::shared_ptr<att::TFDTransport>		innerTransportScf(new att::TFDTransport(scf_fh));
	boost::shared_ptr<att::TBufferedTransport>	transportScf(new att::TBufferedTransport(innerTransportScf));
	boost::shared_ptr<atp::TBinaryProtocol>		protocolScf(new atp::TBinaryProtocol(transportScf));
	transportScf->open();

	fn::FilterNames filter_names;
	filter_names.read(protocolScf.get());

	unordered_map<string, set<string>> target_text_map;

	#ifdef LVV
		// a hack to release some memory
		filter_names.target_id_to_names = std::map<std::string, std::vector<std::string>>();
	#endif

	////////////////////////////////////////////////////////////////////////////////// FILTER AND ADD TO NAMES

	vector<char>	names_data;
	vector<char>	name;
	vector<size_t>	names_begin{0};		// names_begin[0] starts at pos 0

	size_t name_min=9999999999;
	size_t name_max=0;
	size_t good_names = 0;

	set<vector<char>> uniq_set;
	long total_tokens = 0;

	// for all names in filter_names
	for(auto& pr : filter_names.name_to_target_ids) {
		auto s  = pr.first.data();
		size_t sz = pr.first.size();

						#ifdef DISPLAY_NAMES
						cout << "\nscf name: \t(" << string(s,s+sz) << ")\n";
						#endif
		// split into tokens
		pos_t  tb, te = s;
		pos_t  e = s + sz;
		name.clear();
		long tokens = 0;
		while ((tb=get_tb(te,e)) != e) {
			te=get_te(tb,e);
			if (!name.empty())  name.push_back(token_separator);
			copy (tb, te, back_inserter(name));
			++tokens;
		}

		// good / bad name
		constexpr long	max_name_length = 1000;
		constexpr long	min_name_length = 3;
		constexpr long	min_name_tokens = 1;

		if (
			    ( min_name_length <= name.size()  &&  name.size() <= max_name_length )
			&&  ( min_name_tokens <= tokens )
			&&  ( uniq_set.find(name) == uniq_set.end() )
		) {
			++good_names;
		} else {
			continue;
		}

		uniq_set.insert(name);
						#ifdef DISPLAY_NAMES
						cout << "added name: \t(" << string(name.begin(), name.end()) << ")\n";
						#endif
		// statistics
		name_min = std::min(name_min, name.size());
		name_max = std::max(name_max, name.size());
		total_tokens += tokens;

		// append name to names_data
		copy (name.begin(), name.end(), back_inserter(names_data));
		names_begin.push_back(names_data.size());
	}

	transportScf->close();

	cerr << "NAMES: "  << filter_names.name_to_target_ids.size()
	     << ";\n\t good:       "         << good_names
	     << ";\n\t min length: "         << name_min
	     << ";\n\t max length: "         << name_max
	     << ";\n\t avg length: "         << double(names_data.size())/(names_begin.size()-1)
	     << ";\n\t avg tokens: "         << double(total_tokens)/(names_begin.size()-1)
	     << ";\n\t total names length: " << names_data.size()
	     << endl;

	///////////////////////////////////////////////////////////////////////////////// WRITE MMAP

	cerr << "writing names memory map file ... ";

	lvv::mmap_write<char>  ("data/names_data.mmap",  &names_data[0],  names_data.size());
	lvv::mmap_write<size_t>("data/names_begin.mmap", &names_begin[0], names_begin.size());

	cerr << "done\n";
}
<commit_msg>-- added comments<commit_after>// converts filternames.scf names into mmap file

// LIBC
#include <fcntl.h>
#include <time.h>

// THRIFT
#include <thrift/protocol/TBinaryProtocol.h>
#include <thrift/protocol/TDenseProtocol.h>
#include <thrift/protocol/TJSONProtocol.h>
#include <thrift/transport/TTransportUtils.h>
#include <thrift/transport/TFDTransport.h>
#include <thrift/transport/TFileTransport.h>
	namespace atp = apache::thrift::protocol;
	namespace att = apache::thrift::transport;

// THRIFT -- FILTERNAMES
#include "filternames_types.h"
#include "filternames_constants.h"
	namespace fn = filternames;

// STD
#include <iostream>
	using std::cerr;
	using std::cout;
	using std::endl;

#include <algorithm>
	using std::copy;
#include <iterator>
	using std::back_inserter;
#include <unordered_map>
	using std::unordered_map;
#include <set>
	using std::set;
#include <string>
	using std::string;
#include <vector>
	using std::vector;
#include <fstream>

// BOOST
#include <boost/program_options.hpp>
	namespace po = boost::program_options;

// LVVLIB (http://github.com/lvv/lvvlib)
//#ifdef LVV
//#include <ro/ro.h>
//#include <scc/simple.h>
//#endif
#include "lvvlib/token.h"
#include "lvvlib/mmap.h"

int main(int argc, char **argv) {

	///////////////////////////////////////////////////////////////////////////////  OPTIONS

	string		filtername_path;

	po::options_description desc("Allowed options");
	desc.add_options()
		("help,h", "help message")
		("filternames,f", po::value<string>(&filtername_path), "filternames file")
	;

	// Parse command line options
	po::variables_map vm;
	po::store(po::parse_command_line(argc, argv, desc), vm);
	po::notify(vm);

	if (vm.count("help")) { cout << desc << "\n"; return 1; }

	////////////////////////////////////////////////////////////// THRIFT READS FILTERNAMES

	int scf_fh = open(filtername_path.c_str(), O_RDONLY);

					if(scf_fh==-1)  {
						cerr << "error: cann't open scf file -- '" << filtername_path << "'\n";
						exit(1);
					}

	boost::shared_ptr<att::TFDTransport>		innerTransportScf(new att::TFDTransport(scf_fh));
	boost::shared_ptr<att::TBufferedTransport>	transportScf(new att::TBufferedTransport(innerTransportScf));
	boost::shared_ptr<atp::TBinaryProtocol>		protocolScf(new atp::TBinaryProtocol(transportScf));
	transportScf->open();

	fn::FilterNames filter_names;
	filter_names.read(protocolScf.get());

	unordered_map<string, set<string>> target_text_map;

	#ifdef LVV
		// a hack to release some memory
		filter_names.target_id_to_names = std::map<std::string, std::vector<std::string>>();
	#endif

	////////////////////////////////////////////////////////////////////////////////// FILTER AND ADD TO NAMES

	vector<char>	names_data;		// cancantenated content of names
	vector<char>	name;                   // temp name
	vector<size_t>	names_begin{0};		// array of indexes in names_data which point to begining of a names.
						// a name i is from names_begin[i] to names_begin[i+1]

	size_t name_min=9999999999;
	size_t name_max=0;
	size_t good_names = 0;

	set<vector<char>> uniq_set;            	// used in check for uniqness
	long total_tokens = 0;

	// for all names in filter_names
	for(auto& pr : filter_names.name_to_target_ids) {
		auto s  = pr.first.data();                             		// name
		size_t sz = pr.first.size();                                    // is size

						#ifdef DISPLAY_NAMES
						cout << "\nscf name: \t(" << string(s,s+sz) << ")\n";
						#endif
		// split into tokens
		pos_t  tb, te = s;             					// token begin, end
		pos_t  e = s + sz;
		name.clear();
		long tokens = 0;
		while ((tb=get_tb(te,e)) != e) {
			te=get_te(tb,e);
			if (!name.empty())  name.push_back(token_separator);
			copy (tb, te, back_inserter(name));
			++tokens;
		}

		// filter out bad names
		constexpr long	max_name_length = 1000;
		constexpr long	min_name_length = 3;
		constexpr long	min_name_tokens = 1;

		if (
			    ( min_name_length <= name.size()  &&  name.size() <= max_name_length )
			&&  ( min_name_tokens <= tokens )
			&&  ( uniq_set.find(name) == uniq_set.end() )
		) {
			++good_names;
		} else {
			continue;
		}

		uniq_set.insert(name);
						#ifdef DISPLAY_NAMES
						cout << "added name: \t(" << string(name.begin(), name.end()) << ")\n";
						#endif
		// statistics
		name_min = std::min(name_min, name.size());
		name_max = std::max(name_max, name.size());
		total_tokens += tokens;

		// append name to names_data
		copy (name.begin(), name.end(), back_inserter(names_data));
		names_begin.push_back(names_data.size());
	}

	transportScf->close();

	cerr << "NAMES: "  << filter_names.name_to_target_ids.size()
	     << ";\n\t good:       "         << good_names
	     << ";\n\t min length: "         << name_min
	     << ";\n\t max length: "         << name_max
	     << ";\n\t avg length: "         << double(names_data.size())/(names_begin.size()-1)
	     << ";\n\t avg tokens: "         << double(total_tokens)/(names_begin.size()-1)
	     << ";\n\t total names length: " << names_data.size()
	     << endl;

	///////////////////////////////////////////////////////////////////////////////// WRITE MMAP

	cerr << "writing names memory map file ... ";

	lvv::mmap_write<char>  ("data/names_data.mmap",  &names_data[0],  names_data.size());
	lvv::mmap_write<size_t>("data/names_begin.mmap", &names_begin[0], names_begin.size());

	cerr << "done\n";
}

// vim: ts=8 sw=8
<|endoftext|>